fix(ci): bound queue DB connections to stop E2E 1040 stalls

## Summary

### Why?

Tenant-scoped reconciliation fans out one goroutine per tenant per subscription tick. With `database/sql`'s unlimited open-connection default, the E2E stack opened more sockets than MySQL's ~151-connection ceiling, returned Error 1040, and stalled pipeline progress until Bazel timed out. Gateway integration also published logs under a tenant that was not in `MQ_TENANTS`, so the consumer never saw them.

### What?

- Apply `QUEUE_MYSQL_MAX_OPEN_CONNECTIONS` (default 16) on each service-owned queue `*sql.DB`, and keep that many connections idle so the cap does not churn sockets.
- Point the gateway log-consumer integration at the configured `test-queue` tenant and scope the land-message lookup by tenant.
- Upload Bazel `test.log` files on failure for every CI test job, with `actions: write` on the workflow token so `upload-artifact` can run.

## Test Plan

✅ `./tool/bazel test //service/messagequeue:go_default_test`
✅ `./tool/bazel test //test/integration/submitqueue/gateway:go_default_test --sandbox_writable_path="$HOME/.docker/buildx"`
✅ `./tool/bazel test //test/e2e/submitqueue:go_default_test --sandbox_writable_path="$HOME/.docker/buildx"` (167s, then 161s)
✅ `./tool/bazel test //test/e2e/submitqueue:go_default_test --test_filter='TestGitMergeE2E' --sandbox_writable_path="$HOME/.docker/buildx"`
✅ `make fmt && make gazelle && make tidy`

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: behinddwalls

.github/actions/run-bazel-test/action.yml

@@ -17,3 +17,7 @@ runs:
       env:
         TARGET: ${{ inputs.target }}
       run: ./tool/bazel test "$TARGET" --test_output=streamed
+
+    - name: Upload Bazel failure logs
+      if: ${{ failure() }}
+      uses: ./.github/actions/upload-testlogs

.github/actions/upload-testlogs/action.yml

@@ -0,0 +1,13 @@
+name: Upload Bazel test logs
+description: Upload bazel-testlogs on job failure so truncated CI output is not the only diagnostic.
+
+runs:
+  using: composite
+  steps:
+    - name: Upload Bazel failure logs
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+      with:
+        name: testlogs-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }}
+        path: bazel-testlogs/**/test.log
+        if-no-files-found: warn
+        retention-days: 7

.github/workflows/ci.yml

@@ -19,6 +19,7 @@ on:
 
 permissions:
   contents: read
+  actions: write
 
 # Cancel superseded runs for the same PR to save runner minutes. Never cancel
 # in-progress runs for `push` (main) or `merge_group` — those must complete so
@@ -88,6 +89,10 @@ jobs:
       - name: Run unit tests
         run: make test
 
+      - name: Upload Bazel failure logs
+        if: ${{ failure() }}
+        uses: ./.github/actions/upload-testlogs
+
   # ---------------------------------------------------------------------------
   # INTEGRATION TESTS (e2e, gateway, orchestrator)
   # ---------------------------------------------------------------------------
@@ -106,6 +111,10 @@ jobs:
       - name: Run E2E tests
         run: make e2e-test
 
+      - name: Upload Bazel failure logs
+        if: ${{ failure() }}
+        uses: ./.github/actions/upload-testlogs
+
   gateway-integration-test:
     name: Gateway Integration Test
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
@@ -121,6 +130,10 @@ jobs:
       - name: Run Gateway integration tests
         run: make integration-test-submitqueue-gateway
 
+      - name: Upload Bazel failure logs
+        if: ${{ failure() }}
+        uses: ./.github/actions/upload-testlogs
+
   orchestrator-integration-test:
     name: Orchestrator Integration Test
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
@@ -136,6 +149,10 @@ jobs:
       - name: Run Orchestrator integration tests
         run: make integration-test-submitqueue-orchestrator
 
+      - name: Upload Bazel failure logs
+        if: ${{ failure() }}
+        uses: ./.github/actions/upload-testlogs
+
   # ---------------------------------------------------------------------------
   # EXTENSION TESTS
   # ---------------------------------------------------------------------------
@@ -243,7 +260,8 @@ jobs:
       # malicious actor controlling ci.yml could delete the guard itself, so the
       # real defenses remain (a) secrets scoped to a main-only Environment so a
       # PR-triggered job cannot obtain them, and (b) CODEOWNERS review on
-      # .github/. GITHUB_TOKEN (least-privilege, read-only here) is allowlisted.
+      # .github/. GITHUB_TOKEN is scoped to contents: read and actions: write
+      # (artifact upload on failed test jobs) and is allowlisted.
       - name: Guard — no repository secrets on the untrusted-code path
         run: |
           hits="$(grep -rnE '\$\{\{[^}]*secrets\.' \

service/messagequeue/BUILD.bazel

@@ -2,16 +2,23 @@ load("@rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = ["tenant.go"],
+    srcs = [
+        "database.go",
+        "tenant.go",
+    ],
     importpath = "github.com/uber/submitqueue/service/messagequeue",
     visibility = ["//visibility:public"],
 )
 
 go_test(
     name = "go_default_test",
-    srcs = ["tenant_test.go"],
+    srcs = [
+        "database_test.go",
+        "tenant_test.go",
+    ],
     embed = [":go_default_library"],
     deps = [
+        "@com_github_go_sql_driver_mysql//:go_default_library",
         "@com_github_stretchr_testify//assert:go_default_library",
         "@com_github_stretchr_testify//require:go_default_library",
     ],

service/messagequeue/database.go

@@ -0,0 +1,44 @@
+// Copyright (c) 2026 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package messagequeue
+
+import (
+	"database/sql"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// DefaultQueueMySQLMaxOpenConnections bounds queue database concurrency when
+// the service does not provide an explicit limit.
+const DefaultQueueMySQLMaxOpenConnections = 16
+
+// ConfigureQueueMySQLConnectionPool applies the configured maximum open
+// connection count and keeps that many connections idle. Empty input
+// selects DefaultQueueMySQLMaxOpenConnections.
+func ConfigureQueueMySQLConnectionPool(db *sql.DB, value string) error {
+	maxOpenConnections := DefaultQueueMySQLMaxOpenConnections
+	trimmed := strings.TrimSpace(value)
+	if trimmed != "" {
+		parsed, err := strconv.Atoi(trimmed)
+		if err != nil || parsed <= 0 {
+			return fmt.Errorf("maximum open connections must be a positive integer")
+		}
+		maxOpenConnections = parsed
+	}
+	db.SetMaxOpenConns(maxOpenConnections)
+	db.SetMaxIdleConns(maxOpenConnections)
+	return nil
+}

service/messagequeue/database_test.go

@@ -0,0 +1,56 @@
+// Copyright (c) 2026 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package messagequeue
+
+import (
+	"database/sql"
+	"testing"
+
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConfigureQueueMySQLConnectionPool(t *testing.T) {
+	tests := []struct {
+		name    string
+		value   string
+		want    int
+		wantErr bool
+	}{
+		{name: "empty uses default", want: DefaultQueueMySQLMaxOpenConnections},
+		{name: "explicit value", value: "24", want: 24},
+		{name: "surrounding whitespace", value: " 8 ", want: 8},
+		{name: "zero", value: "0", wantErr: true},
+		{name: "negative", value: "-1", wantErr: true},
+		{name: "non-integer", value: "many", wantErr: true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			db, err := sql.Open("mysql", "")
+			require.NoError(t, err)
+			t.Cleanup(func() { require.NoError(t, db.Close()) })
+
+			err = ConfigureQueueMySQLConnectionPool(db, tt.value)
+			if tt.wantErr {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, db.Stats().MaxOpenConnections)
+		})
+	}
+}

service/runway/server/docker-compose.yml

@@ -50,6 +50,7 @@ services:
       - MERGER=${SQ_RUNWAY_MERGER:-}
       # Queue infrastructure connection
       - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true
+      - QUEUE_MYSQL_MAX_OPEN_CONNECTIONS=${QUEUE_MYSQL_MAX_OPEN_CONNECTIONS:-16}
       # Level for the queue's own logs; info by default so its per-message
       # chatter does not bury the rest of the service at debug.
       - QUEUE_LOG_LEVEL=${QUEUE_LOG_LEVEL:-}

service/runway/server/main.go

@@ -140,6 +140,9 @@ func run() error {
 		return fmt.Errorf("failed to open queue database: %w", err)
 	}
 	defer queueDB.Close()
+	if err := servicemq.ConfigureQueueMySQLConnectionPool(queueDB, os.Getenv("QUEUE_MYSQL_MAX_OPEN_CONNECTIONS")); err != nil {
+		return fmt.Errorf("failed to configure queue database pool: %w", err)
+	}
 
 	tenants, err := servicemq.ParseRequiredTenants(os.Getenv("MQ_TENANTS"))
 	if err != nil {

service/stovepipe/docker-compose.yml

@@ -67,6 +67,7 @@ services:
       - PORT=:8080
       - STORAGE_MYSQL_DSN=root:root@tcp(mysql-app:3306)/submitqueue?parseTime=true
       - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true
+      - QUEUE_MYSQL_MAX_OPEN_CONNECTIONS=${QUEUE_MYSQL_MAX_OPEN_CONNECTIONS:-16}
       # Level for the queue's own logs; info by default so its per-message
       # chatter does not bury the rest of the service at debug.
       - QUEUE_LOG_LEVEL=${QUEUE_LOG_LEVEL:-}

service/stovepipe/server/main.go

@@ -273,6 +273,9 @@ func run() error {
 		return fmt.Errorf("failed to open queue database: %w", err)
 	}
 	defer queueDB.Close()
+	if err := servicemq.ConfigureQueueMySQLConnectionPool(queueDB, os.Getenv("QUEUE_MYSQL_MAX_OPEN_CONNECTIONS")); err != nil {
+		return fmt.Errorf("failed to configure queue database pool: %w", err)
+	}
 
 	tenants, err := servicemq.ParseRequiredTenants(os.Getenv("MQ_TENANTS"))
 	if err != nil {