From 403d96d8b95955fa142a66470cac2ffdbdd145e3 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 8 Oct 2026 09:41:48 -0400 Subject: [PATCH 1/3] Shard the hosted Gradle e2e suite and ungate the yarn/cargo matrices The merge-group CI run took ~46 min, and its critical path was the four Gradle capstone legs: each ran the agent suites plus all 43 hosted real-Gradle tests serially (~31 min of test time per leg). - Split each line's agent+hosted leg into three: the agent suites plus gradle_hosted_[345], gradle_hosted_[b-p], and a catch-all that runs gradle_hosted_ with --skip on exactly those words, so a new test always lands in some leg. test_ci_gradle_prefixes.py's HostedShards checks that every hosted test runs in exactly one leg per line and that the catch-all's skip list matches the other legs. - Drop needs: [test, coverage] from yarn-classic-matrix, yarn-berry-e2e, cargo-old-toolchains and cargo-vex-matrix (which keeps e2e-build). They consume nothing from those jobs and started only after the ~31 min windows test leg. Draft skipping is unchanged: the yarn jobs never run on pull_request, cargo-old-toolchains has its own draft guard, and e2e-build gates cargo-vex-matrix. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ci.yml | 37 +++++++++++---- scripts/tests/test_ci_e2e_tiers.py | 17 +++++-- scripts/tests/test_ci_gradle_prefixes.py | 57 ++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cce660612..298709391 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1218,13 +1218,27 @@ jobs: # way): a missing suite fails the leg, and every suite must run at # least one test on its own. Maven is installed only where a # selected test needs it (gradle_vendor_395's mixed root). - - {os: ubuntu-latest, suite: 'e2e_gradle_discovery_build e2e_gradle_agent_build e2e_redirect_gradle_build', jvm_tool: gradle, gradle: '6.9.4', java: '11', test_filter: '--ignored gradle_agent_ gradle_hosted_'} + # The hosted suite (~43 real-Gradle builds, ~31 min serially) is + # split into three legs per line so it stops being the merge-queue + # critical path: the agent suites plus `gradle_hosted_[345]`, the + # `gradle_hosted_[b-p]` names, and a catch-all that `--skip`s + # exactly those words, so a new test always lands in some leg + # (test_ci_gradle_prefixes.py keeps the skip list in sync). + - {os: ubuntu-latest, suite: 'e2e_gradle_discovery_build e2e_gradle_agent_build e2e_redirect_gradle_build', jvm_tool: gradle, gradle: '6.9.4', java: '11', test_filter: '--ignored gradle_agent_ gradle_hosted_3 gradle_hosted_4 gradle_hosted_5'} + - {os: ubuntu-latest, suite: e2e_redirect_gradle_build, jvm_tool: gradle, gradle: '6.9.4', java: '11', test_filter: '--ignored gradle_hosted_b gradle_hosted_c gradle_hosted_d gradle_hosted_e gradle_hosted_f gradle_hosted_l gradle_hosted_m gradle_hosted_n gradle_hosted_o gradle_hosted_p'} + - {os: ubuntu-latest, suite: e2e_redirect_gradle_build, jvm_tool: gradle, gradle: '6.9.4', java: '11', test_filter: '--ignored gradle_hosted_ --skip gradle_hosted_3 --skip gradle_hosted_4 --skip gradle_hosted_5 --skip gradle_hosted_b --skip gradle_hosted_c --skip gradle_hosted_d --skip gradle_hosted_e --skip gradle_hosted_f --skip gradle_hosted_l --skip gradle_hosted_m --skip gradle_hosted_n --skip gradle_hosted_o --skip gradle_hosted_p'} - {os: ubuntu-latest, suite: 'e2e_vendor_gradle_build e2e_vendor_jvm_build', jvm_tool: gradle, gradle: '6.9.4', java: '11', test_filter: '--ignored gradle_vendor_ gradle_multi_project'} - - {os: ubuntu-latest, suite: 'e2e_gradle_discovery_build e2e_gradle_agent_build e2e_redirect_gradle_build', jvm_tool: gradle, gradle: '7.6.6', java: '17', test_filter: '--ignored gradle_agent_ gradle_hosted_'} + - {os: ubuntu-latest, suite: 'e2e_gradle_discovery_build e2e_gradle_agent_build e2e_redirect_gradle_build', jvm_tool: gradle, gradle: '7.6.6', java: '17', test_filter: '--ignored gradle_agent_ gradle_hosted_3 gradle_hosted_4 gradle_hosted_5'} + - {os: ubuntu-latest, suite: e2e_redirect_gradle_build, jvm_tool: gradle, gradle: '7.6.6', java: '17', test_filter: '--ignored gradle_hosted_b gradle_hosted_c gradle_hosted_d gradle_hosted_e gradle_hosted_f gradle_hosted_l gradle_hosted_m gradle_hosted_n gradle_hosted_o gradle_hosted_p'} + - {os: ubuntu-latest, suite: e2e_redirect_gradle_build, jvm_tool: gradle, gradle: '7.6.6', java: '17', test_filter: '--ignored gradle_hosted_ --skip gradle_hosted_3 --skip gradle_hosted_4 --skip gradle_hosted_5 --skip gradle_hosted_b --skip gradle_hosted_c --skip gradle_hosted_d --skip gradle_hosted_e --skip gradle_hosted_f --skip gradle_hosted_l --skip gradle_hosted_m --skip gradle_hosted_n --skip gradle_hosted_o --skip gradle_hosted_p'} - {os: ubuntu-latest, suite: 'e2e_vendor_gradle_build e2e_vendor_jvm_build', jvm_tool: gradle, gradle: '7.6.6', java: '17', test_filter: '--ignored gradle_vendor_ gradle_multi_project'} - - {os: ubuntu-latest, suite: 'e2e_gradle_discovery_build e2e_gradle_agent_build e2e_redirect_gradle_build', jvm_tool: gradle, gradle: '8.14.3', java: '21', test_filter: '--ignored gradle_agent_ gradle_hosted_'} + - {os: ubuntu-latest, suite: 'e2e_gradle_discovery_build e2e_gradle_agent_build e2e_redirect_gradle_build', jvm_tool: gradle, gradle: '8.14.3', java: '21', test_filter: '--ignored gradle_agent_ gradle_hosted_3 gradle_hosted_4 gradle_hosted_5'} + - {os: ubuntu-latest, suite: e2e_redirect_gradle_build, jvm_tool: gradle, gradle: '8.14.3', java: '21', test_filter: '--ignored gradle_hosted_b gradle_hosted_c gradle_hosted_d gradle_hosted_e gradle_hosted_f gradle_hosted_l gradle_hosted_m gradle_hosted_n gradle_hosted_o gradle_hosted_p'} + - {os: ubuntu-latest, suite: e2e_redirect_gradle_build, jvm_tool: gradle, gradle: '8.14.3', java: '21', test_filter: '--ignored gradle_hosted_ --skip gradle_hosted_3 --skip gradle_hosted_4 --skip gradle_hosted_5 --skip gradle_hosted_b --skip gradle_hosted_c --skip gradle_hosted_d --skip gradle_hosted_e --skip gradle_hosted_f --skip gradle_hosted_l --skip gradle_hosted_m --skip gradle_hosted_n --skip gradle_hosted_o --skip gradle_hosted_p'} - {os: ubuntu-latest, suite: 'e2e_vendor_gradle_build e2e_vendor_jvm_build', jvm_tool: gradle, gradle: '8.14.3', java: '21', test_filter: '--ignored gradle_vendor_ gradle_multi_project'} - - {os: ubuntu-latest, suite: 'e2e_gradle_discovery_build e2e_gradle_agent_build e2e_redirect_gradle_build', jvm_tool: gradle, gradle: '9.8.0', java: '21', test_filter: '--ignored gradle_agent_ gradle_hosted_'} + - {os: ubuntu-latest, suite: 'e2e_gradle_discovery_build e2e_gradle_agent_build e2e_redirect_gradle_build', jvm_tool: gradle, gradle: '9.8.0', java: '21', test_filter: '--ignored gradle_agent_ gradle_hosted_3 gradle_hosted_4 gradle_hosted_5'} + - {os: ubuntu-latest, suite: e2e_redirect_gradle_build, jvm_tool: gradle, gradle: '9.8.0', java: '21', test_filter: '--ignored gradle_hosted_b gradle_hosted_c gradle_hosted_d gradle_hosted_e gradle_hosted_f gradle_hosted_l gradle_hosted_m gradle_hosted_n gradle_hosted_o gradle_hosted_p'} + - {os: ubuntu-latest, suite: e2e_redirect_gradle_build, jvm_tool: gradle, gradle: '9.8.0', java: '21', test_filter: '--ignored gradle_hosted_ --skip gradle_hosted_3 --skip gradle_hosted_4 --skip gradle_hosted_5 --skip gradle_hosted_b --skip gradle_hosted_c --skip gradle_hosted_d --skip gradle_hosted_e --skip gradle_hosted_f --skip gradle_hosted_l --skip gradle_hosted_m --skip gradle_hosted_n --skip gradle_hosted_o --skip gradle_hosted_p'} - {os: ubuntu-latest, suite: 'e2e_vendor_gradle_build e2e_vendor_jvm_build', jvm_tool: gradle, gradle: '9.8.0', java: '21', test_filter: '--ignored gradle_vendor_ gradle_multi_project'} - {os: windows-latest, suite: e2e_vendor_jvm_build, jvm_tool: gradle, gradle: '8.14.3', java: '17', test_filter: '--ignored gradle_multi_project'} # Real-sbt hosted (socket-patch.sbt) + vendored @@ -1904,7 +1918,9 @@ jobs: # ---------------------------------------------------------------------- yarn-classic-matrix: name: yarn-classic ${{ matrix.release }} - needs: [test, coverage] + # No `needs: [test, coverage]`: nothing here consumes their outputs, + # and waiting on the ~30 min windows `test` leg made this the merge + # queue's critical path. runs-on: ubuntu-latest timeout-minutes: 40 strategy: @@ -1940,7 +1956,9 @@ jobs: yarn-berry-e2e: name: yarn-berry ${{ matrix.yarn }} (${{ matrix.os }}) - needs: [test, coverage] + # No `needs: [test, coverage]`: nothing here consumes their outputs, + # and waiting on the ~30 min windows `test` leg made this the merge + # queue's critical path. strategy: fail-fast: false matrix: @@ -2022,7 +2040,8 @@ jobs: # e2e_safety_cargo_build (its headline test honours the knobs). cargo-vex-matrix: name: cargo ${{ matrix.toolchain }} lock-v${{ matrix.lock || 'own' }} (${{ matrix.os }}) - needs: [test, coverage, e2e-build] + # e2e-build only (its binaries); see yarn-classic-matrix on test/coverage. + needs: [e2e-build] runs-on: ${{ matrix.os }} timeout-minutes: 40 # What .cargo/config.toml's [env] gives processes cargo launches; these @@ -2149,7 +2168,9 @@ jobs: # available; this leg pulls both images and requires them. cargo-old-toolchains: name: cargo old toolchains (manifest [patch]) - needs: [test, coverage] + # No `needs: [test, coverage]`: nothing here consumes their outputs, + # and waiting on the ~30 min windows `test` leg made this the merge + # queue's critical path. runs-on: ubuntu-latest timeout-minutes: 30 steps: diff --git a/scripts/tests/test_ci_e2e_tiers.py b/scripts/tests/test_ci_e2e_tiers.py index 9ed68b8c1..c995384c0 100644 --- a/scripts/tests/test_ci_e2e_tiers.py +++ b/scripts/tests/test_ci_e2e_tiers.py @@ -122,8 +122,17 @@ def test_bundle_reads_cargo_json(self): GRADLE_COMPAT = ROOT / ".github" / "workflows" / "gradle-compatibility.yml" GRADLE_LINES = {"6.9.4": "11", "7.6.6": "17", "8.14.3": "21", "9.8.0": "21"} -AGENT_HOSTED = ("e2e_gradle_discovery_build e2e_gradle_agent_build e2e_redirect_gradle_build", - "--ignored gradle_agent_ gradle_hosted_") +# The hosted suite is sharded over three legs per line (test_ci_gradle_prefixes +# HostedShards checks every hosted test runs in exactly one of them). +HOSTED_SHARD_1 = ["gradle_hosted_3", "gradle_hosted_4", "gradle_hosted_5"] +HOSTED_SHARD_2 = ["gradle_hosted_" + c for c in "bcdeflmnop"] +AGENT_HOSTED = ( + ("e2e_gradle_discovery_build e2e_gradle_agent_build e2e_redirect_gradle_build", + " ".join(["--ignored", "gradle_agent_", *HOSTED_SHARD_1])), + ("e2e_redirect_gradle_build", " ".join(["--ignored", *HOSTED_SHARD_2])), + ("e2e_redirect_gradle_build", + " ".join(["--ignored", "gradle_hosted_"] + [w for p in HOSTED_SHARD_1 + HOSTED_SHARD_2 for w in ("--skip", p)])), +) VENDOR = ("e2e_vendor_gradle_build e2e_vendor_jvm_build", "--ignored gradle_vendor_ gradle_multi_project") @@ -160,7 +169,7 @@ def test_pr_rows_are_the_lean_table(self): gradle = [r for r in rows("e2e") if r.get("jvm_tool") == "gradle"] want = [] for line, java in GRADLE_LINES.items(): - for suite, test_filter in (AGENT_HOSTED, VENDOR): + for suite, test_filter in (*AGENT_HOSTED, VENDOR): row = {"os": "ubuntu-latest", "suite": suite, "jvm_tool": "gradle", "gradle": line, "java": java, "test_filter": test_filter} # The allowance lasts only while a suite of the row is unlanded. @@ -169,7 +178,7 @@ def test_pr_rows_are_the_lean_table(self): want.append(row) want.append({"os": "windows-latest", "suite": "e2e_vendor_jvm_build", "jvm_tool": "gradle", "gradle": "8.14.3", "java": "17", "test_filter": "--ignored gradle_multi_project"}) - self.assertEqual(len(gradle), 9) + self.assertEqual(len(gradle), 17) self.assertEqual(sorted(map(str, gradle)), sorted(map(str, want))) self.assertFalse([r for r in rows("e2e-full") if "gradle" in r or "jvm_tool" in r]) diff --git a/scripts/tests/test_ci_gradle_prefixes.py b/scripts/tests/test_ci_gradle_prefixes.py index 2a52cd027..b4dc6e45f 100644 --- a/scripts/tests/test_ci_gradle_prefixes.py +++ b/scripts/tests/test_ci_gradle_prefixes.py @@ -175,6 +175,63 @@ def test_compat_overrides_select_admitted_tests(self): self.assertTrue(row.get("suites"), "a narrowed filter names the suite that owns it") +def libtest_selects(words, name): + """libtest's filter semantics for the argument words a row passes: a + name runs when it contains any positional filter (all names when there + is none) and no `--skip` word.""" + filters, skips, it = [], [], iter(words) + for word in it: + if word == "--skip": + skips.append(next(it)) + elif not word.startswith("--"): + filters.append(word) + return (not filters or any(f in name for f in filters)) and not any(s in name for s in skips) + + +class HostedShards(unittest.TestCase): + """The hosted suite runs as several legs per Gradle line (it is the + merge-queue critical path in one leg). Every hosted test must run in + exactly one leg of each line, and the catch-all leg's `--skip` words + must be exactly the other legs' hosted words.""" + SUITE = "e2e_redirect_gradle_build" + + def rows_by_line(self): + rows = [r for r in rows_mod.job_rows(rows_mod.jobs(CI.read_text(encoding="utf-8")), "e2e") + if r.get("jvm_tool") == "gradle" and self.SUITE in r["suite"].split()] + lines = {} + for row in rows: + lines.setdefault(row["gradle"], []).append(row["test_filter"].split()) + return lines + + def test_every_hosted_test_runs_in_exactly_one_leg_per_line(self): + names = [n for path in bundle.suite_files(self.SUITE) + for n in bundle.ignored_tests(path.read_text(encoding="utf-8"))] + self.assertGreater(len(names), 20) + lines = self.rows_by_line() + self.assertTrue(lines) + for line, filters in lines.items(): + for name in names: + with self.subTest(gradle=line, test=name): + self.assertEqual(sum(libtest_selects(f, name) for f in filters), 1) + + def test_catch_all_skips_exactly_the_other_legs_words(self): + for line, filters in self.rows_by_line().items(): + with self.subTest(gradle=line): + catch_all = [f for f in filters if "--skip" in f] + self.assertEqual(len(catch_all), 1) + skips = {w for a, w in zip(catch_all[0], catch_all[0][1:]) if a == "--skip"} + named = {w for f in filters if f is not catch_all[0] + for w in f if w.startswith("gradle_hosted_")} + self.assertEqual(skips, named) + + def test_libtest_selects_negative(self): + self.assertFalse(libtest_selects(["--ignored", "gradle_hosted_b"], "gradle_hosted_catalog")) + self.assertFalse(libtest_selects(["--ignored", "gradle_hosted_", "--skip", "gradle_hosted_c"], + "gradle_hosted_catalog")) + self.assertTrue(libtest_selects(["--ignored", "gradle_hosted_", "--skip", "gradle_hosted_c"], + "gradle_hosted_tamper_fails")) + + class AllowEmpty(unittest.TestCase): rows = rows_mod.job_rows(rows_mod.jobs(CI.read_text(encoding="utf-8")), "e2e") From 09aded0f7210b630c3acb5d5f7e335a199426c20 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 15:20:50 +0000 Subject: [PATCH 2/3] Skip ungated yarn/cargo matrix jobs on draft PRs Dropping `needs: [test, coverage]` from yarn-classic-matrix, yarn-berry-e2e and cargo-old-toolchains also dropped the draft skip they inherited from those jobs, so draft pushes started compiling and running every leg. Gate them on `github.event.pull_request.draft != true` like the other top-level jobs; push, merge_group and schedule events have no pull_request payload and still run them. Co-Authored-By: Claude --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 298709391..72b3765c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1921,6 +1921,9 @@ jobs: # No `needs: [test, coverage]`: nothing here consumes their outputs, # and waiting on the ~30 min windows `test` leg made this the merge # queue's critical path. + # Dropping that `needs` also dropped the draft skip it inherited, so + # gate on draft here directly (push/merge_group/schedule still run). + if: github.event.pull_request.draft != true runs-on: ubuntu-latest timeout-minutes: 40 strategy: @@ -1959,6 +1962,9 @@ jobs: # No `needs: [test, coverage]`: nothing here consumes their outputs, # and waiting on the ~30 min windows `test` leg made this the merge # queue's critical path. + # Dropping that `needs` also dropped the draft skip it inherited, so + # gate on draft here directly (push/merge_group/schedule still run). + if: github.event.pull_request.draft != true strategy: fail-fast: false matrix: @@ -2171,6 +2177,9 @@ jobs: # No `needs: [test, coverage]`: nothing here consumes their outputs, # and waiting on the ~30 min windows `test` leg made this the merge # queue's critical path. + # Dropping that `needs` also dropped the draft skip it inherited, so + # gate on draft here directly (push/merge_group/schedule still run). + if: github.event.pull_request.draft != true runs-on: ubuntu-latest timeout-minutes: 30 steps: From a1f2d347d2334dc561a063c7f475db204f7ca4c8 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 8 Oct 2026 11:21:49 -0400 Subject: [PATCH 3/3] Shard the test legs, skip test-release in the merge queue, cancel orphaned merge-group runs - test (macOS / Windows): two legs per OS via scripts/ci-test-shard.py. The Windows leg spent ~10 min linking ~240 test binaries and ~15 min running them in one job (~31 min, the next critical path once the Gradle legs are sharded). Shard 1 runs the unit tests, doctests and a third of the integration targets; shard 2 the rest. The shards together run exactly the old cargo test --workspace selection (test_ci_test_shard.py), and a renamed target fails loudly. - test-release: skipped on merge_group. Every PR already ran it on its head, and it still runs on main after each merge; in the queue it re-spent ~30 min (23 compiling) per entry. ci-ok counts skipped as passing. - merge-queue-janitor.yml: on each merge group, cancel queued or running merge-group runs whose gh-readonly-queue ref was deleted (the queue rebuilt or dropped that entry). Only a definite 404 cancels; it never fails the merge group. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ci.yml | 17 ++++- .github/workflows/merge-queue-janitor.yml | 55 ++++++++++++++ scripts/ci-test-shard.py | 87 +++++++++++++++++++++++ scripts/tests/test_ci_test_shard.py | 80 +++++++++++++++++++++ 4 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/merge-queue-janitor.yml create mode 100644 scripts/ci-test-shard.py create mode 100644 scripts/tests/test_ci_test_shard.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72b3765c1..b6fd73854 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -250,6 +250,11 @@ jobs: fail-fast: false matrix: os: [macos-latest, windows-latest] + # Two legs per OS (scripts/ci-test-shard.py): one leg linked ~240 + # test binaries and ran them serially for ~26 min, the merge queue's + # critical path. Shard 1 = unit tests + doctests + a third of the + # integration targets; shard 2 = the rest. + shard: [1, 2] exclude: # macOS legs run on main, the merge queue and nightly, not per PR push. - os: ${{ github.event_name == 'pull_request' && 'macos-latest' || '' }} @@ -400,11 +405,19 @@ jobs: # `zip`, and assert the pinned release. SOCKET_PATCH_GO_E2E_REQUIRED: '1' SOCKET_PATCH_GO_E2E_VERSION: '1.24' + TEST_SHARD: ${{ matrix.shard }} + # This leg's share of `cargo test --workspace --no-fail-fast`; the + # shards together run exactly that selection (test_ci_test_shard.py). run: | - cargo test --workspace --no-fail-fast + python3 scripts/ci-test-shard.py "$TEST_SHARD" 2 test-release: - if: github.event.pull_request.draft != true + # Not in the merge queue: each PR's head already ran this, and so did + # every PR ahead of it, so a merge group would re-spend ~30 min (23 of it + # compiling) on the same release-mode suite while the queue waits. It + # still runs on every PR and on main after each merge; `ci-ok` counts a + # skipped job as passing. + if: github.event.pull_request.draft != true && github.event_name != 'merge_group' runs-on: ubuntu-latest # Every tests/ target is its own optimized link (~240 test binaries). # The manifest-less VEX suites share two multi-module binaries diff --git a/.github/workflows/merge-queue-janitor.yml b/.github/workflows/merge-queue-janitor.yml new file mode 100644 index 000000000..aee0bae53 --- /dev/null +++ b/.github/workflows/merge-queue-janitor.yml @@ -0,0 +1,55 @@ +name: Merge queue janitor + +# Cancels merge-group runs the queue has orphaned. Each queue entry runs CI on +# its own `gh-readonly-queue/main/pr--` ref, so ci.yml's +# concurrency group never lets a newer run cancel an older one. When an entry +# ahead fails or is removed, the queue deletes the refs of every entry behind +# it and rebuilds them on new refs, but the runs on the deleted refs keep going +# (each one is a full ~200-job CI run, macOS legs included). A run whose ref no +# longer exists can never merge, so it is cancelled here. Every rebuild creates +# a new merge group, which triggers this sweep. +# +# Not a required check, and it never fails the merge group: a sweep error is +# only a warning. + +on: + merge_group: + types: [checks_requested] + workflow_dispatch: + +permissions: {} + +concurrency: + group: merge-queue-janitor + cancel-in-progress: false + +jobs: + cancel-orphaned-runs: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: write + contents: read + steps: + - name: Cancel merge-group runs whose queue ref is gone + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -uo pipefail + for status in queued in_progress; do + gh api --paginate "repos/$REPO/actions/runs?event=merge_group&status=$status&per_page=100" \ + -q '.workflow_runs[] | "\(.id) \(.head_branch)"' || echo "::warning::could not list $status runs" + done | sort -u | while read -r id branch; do + case "$branch" in gh-readonly-queue/*) ;; *) continue ;; esac + # Only a definite 404 means the entry is gone; any other lookup + # error (rate limit, 5xx) leaves the run alone. + if err=$(gh api "repos/$REPO/git/ref/heads/$branch" --silent 2>&1); then + continue + fi + case "$err" in *"HTTP 404"*) ;; *) echo "::warning::ref lookup for $branch failed: $err"; continue ;; esac + echo "Cancelling run $id: $branch is no longer in the queue" + gh api -X POST "repos/$REPO/actions/runs/$id/cancel" --silent \ + || echo "::warning::could not cancel run $id" + done + exit 0 diff --git a/scripts/ci-test-shard.py b/scripts/ci-test-shard.py new file mode 100644 index 000000000..6aed76b4d --- /dev/null +++ b/scripts/ci-test-shard.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Run one shard of ci.yml's `test` job: `cargo test --workspace` split over +`COUNT` runners so the macOS / Windows legs stop being the merge queue's +critical path (one leg spent ~10 min linking ~240 test binaries and ~15 min +running them). + +Shard 1 runs the unit tests (`--lib --bins`, ~4 min of the run on Windows) +and the doctests; the integration-test targets (`cargo metadata`, kind +`test`) are dealt out over the shards by name, with shard 1 taking a smaller +share to balance its unit tests. Every target lands in exactly one shard, so +the union of the shards is the old single `cargo test --workspace` run. + +Extra arguments after `SHARD COUNT` go to every `cargo test` invocation. +Each invocation runs with `--no-fail-fast`; the exit status is non-zero if +any of them failed. + + python3 scripts/ci-test-shard.py 1 2 +""" + +import json +import subprocess +import sys + +# Shard 1's share of the integration targets, relative to every other +# shard's 1.0 (it also carries the unit tests and doctests). +FIRST_SHARD_WEIGHT = 0.5 + + +def integration_targets(metadata): + """Sorted, de-duplicated names of the workspace's integration tests.""" + members = set(metadata["workspace_members"]) + return sorted({t["name"] for p in metadata["packages"] if p["id"] in members + for t in p["targets"] if "test" in t["kind"]}) + + +def partition(names, count): + """`count` lists covering `names` exactly once, in order, with the first + list weighted by FIRST_SHARD_WEIGHT.""" + if count < 1: + raise ValueError("count must be >= 1") + weights = [FIRST_SHARD_WEIGHT if count > 1 else 1.0] + [1.0] * (count - 1) + shards = [[] for _ in range(count)] + load = [0.0] * count + for name in names: + # The least-loaded shard relative to its weight; ties go to the lower index. + i = min(range(count), key=lambda k: ((load[k] + 1) / weights[k], k)) + shards[i].append(name) + load[i] += 1 + return shards + + +def invocations(shard, count, names, extra=()): + """The `cargo test` argument lists shard `shard` (1-based) runs.""" + if not 1 <= shard <= count: + raise ValueError(f"shard {shard} not in 1..{count}") + base = ["cargo", "test", "--workspace", "--no-fail-fast", *extra] + mine = partition(names, count)[shard - 1] + runs = [] + if shard == 1: + runs.append(base + ["--lib", "--bins"] + [a for n in mine for a in ("--test", n)]) + runs.append(base + ["--doc"]) + elif mine: + runs.append(base + [a for n in mine for a in ("--test", n)]) + return runs + + +def main(argv): + if len(argv) < 2: + print(__doc__, file=sys.stderr) + return 2 + shard, count = int(argv[0]), int(argv[1]) + metadata = json.loads(subprocess.run( + ["cargo", "metadata", "--no-deps", "--format-version", "1"], + check=True, capture_output=True, text=True).stdout) + names = integration_targets(metadata) + print(f"ci-test-shard: shard {shard}/{count}: " + f"{len(partition(names, count)[shard - 1])} of {len(names)} integration targets", flush=True) + status = 0 + for args in invocations(shard, count, names, argv[2:]): + print("+ " + " ".join(args), flush=True) + if subprocess.run(args).returncode != 0: + status = 1 + return status + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/tests/test_ci_test_shard.py b/scripts/tests/test_ci_test_shard.py new file mode 100644 index 000000000..c35eaa61d --- /dev/null +++ b/scripts/tests/test_ci_test_shard.py @@ -0,0 +1,80 @@ +"""ci-test-shard.py: the `test` job's shards together run exactly the old +single `cargo test --workspace` selection.""" + +import importlib.util +import json +import subprocess +import unittest +from pathlib import Path + +ROOT = Path(__file__).parents[2] +spec = importlib.util.spec_from_file_location("ci_test_shard", ROOT / "scripts" / "ci-test-shard.py") +shard = importlib.util.module_from_spec(spec) +spec.loader.exec_module(shard) + + +def selected(runs): + names = [] + for args in runs: + names += [args[i + 1] for i, a in enumerate(args) if a == "--test"] + return names + + +class Partition(unittest.TestCase): + NAMES = [f"t{i:03}" for i in range(236)] + + def test_every_target_runs_in_exactly_one_shard(self): + for count in (1, 2, 3): + with self.subTest(count=count): + got = [] + for k in range(1, count + 1): + got += selected(shard.invocations(k, count, self.NAMES)) + self.assertEqual(sorted(got), self.NAMES) + + def test_unit_tests_and_doctests_run_once_on_shard_one(self): + for count in (1, 2, 3): + runs = [args for k in range(1, count + 1) for args in shard.invocations(k, count, self.NAMES)] + self.assertEqual(sum("--lib" in a and "--bins" in a for a in runs), 1) + self.assertEqual(sum("--doc" in a for a in runs), 1) + self.assertTrue(all("--doc" not in a or "--test" not in a for a in runs), + "cargo rejects --doc with other target selectors") + + def test_shard_one_takes_the_smaller_share(self): + first, second = shard.partition(self.NAMES, 2) + self.assertLess(len(first), len(second)) + self.assertAlmostEqual(len(first) / len(second), shard.FIRST_SHARD_WEIGHT, delta=0.02) + + def test_every_run_is_workspace_wide_and_keeps_going(self): + for args in shard.invocations(2, 2, self.NAMES, ["--locked"]): + self.assertEqual(args[:4], ["cargo", "test", "--workspace", "--no-fail-fast"]) + self.assertIn("--locked", args) + + def test_negative_bad_shard(self): + with self.assertRaises(ValueError): + shard.invocations(3, 2, self.NAMES) + with self.assertRaises(ValueError): + shard.invocations(0, 2, self.NAMES) + + def test_integration_targets_reads_workspace_test_kinds(self): + metadata = { + "workspace_members": ["a", "b"], + "packages": [ + {"id": "a", "targets": [{"name": "lib_a", "kind": ["lib"]}, {"name": "e2e_x", "kind": ["test"]}]}, + {"id": "b", "targets": [{"name": "e2e_x", "kind": ["test"]}, {"name": "zz", "kind": ["test"]}]}, + {"id": "dep", "targets": [{"name": "not_ours", "kind": ["test"]}]}, + ], + } + self.assertEqual(shard.integration_targets(metadata), ["e2e_x", "zz"]) + + def test_the_checkout_has_integration_targets(self): + try: + out = subprocess.run(["cargo", "metadata", "--no-deps", "--format-version", "1"], + cwd=ROOT, check=True, capture_output=True, text=True).stdout + except (OSError, subprocess.CalledProcessError): + self.skipTest("cargo not available") + names = shard.integration_targets(json.loads(out)) + self.assertGreater(len(names), 100) + + +if __name__ == "__main__": + unittest.main()