diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b671ca6ca1..e55060b5f9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,10 +41,20 @@ on: branches: [main] tags: ['v*'] schedule: - # Nightly full tier at 04:00 UTC. The sweep already runs on every merge; - # this is the daily arm for the slow suites (parity, compile-smoke, - # doc-tests, package smokes, the auto-optimize gap shards). + # Nightly full tier at 04:00 UTC — the daily arm for the slow suites + # (parity, compile-smoke, doc-tests, package smokes, the auto-optimize + # gap shards). Keep this string in lockstep with NIGHTLY_CRON in + # scripts/ci_plan.py: the planner maps THIS cron to `full` and any other + # cron to `sweep`. - cron: '0 4 * * *' + # Two-hourly SWEEP BACKSTOP. The push-triggered sweep coalesces in one + # constant concurrency group (1 running + 1 pending, newest replaces + # pending) — and measured 2026-08-16..18, NO push sweep ever reached a + # runner: merges landed faster than the pending->running transition even + # with an idle queue, so `main-gate` had never once executed. This cron is + # the reliable arm; the push trigger stays for quiet periods. :47 to dodge + # the contended :00 scheduler slot and the six-hourly satellite gates. + - cron: '47 */2 * * *' workflow_dispatch: inputs: tier: @@ -123,6 +133,7 @@ jobs: env: EVENT_NAME: ${{ github.event_name }} REF: ${{ github.ref }} + SCHEDULE: ${{ github.event.schedule }} LABELS: ${{ github.event_name == 'pull_request' && join(github.event.pull_request.labels.*.name, ',') || '' }} TIER_INPUT: ${{ inputs.tier }} UPDATE_GAP_SNAPSHOT: ${{ inputs.update_gap_snapshot }} @@ -132,6 +143,9 @@ jobs: if [ "$EVENT_NAME" = "pull_request" ]; then args+=(--changed-files changed-files.txt) fi + if [ "$EVENT_NAME" = "schedule" ]; then + args+=(--schedule "$SCHEDULE") + fi if [ "$EVENT_NAME" = "workflow_dispatch" ]; then args+=(--tier "${TIER_INPUT:-full}") if [ "$UPDATE_GAP_SNAPSHOT" = "true" ]; then diff --git a/changelog.d/8360-sweep-schedule-backstop.md b/changelog.d/8360-sweep-schedule-backstop.md new file mode 100644 index 0000000000..367ee849b7 --- /dev/null +++ b/changelog.d/8360-sweep-schedule-backstop.md @@ -0,0 +1,15 @@ +### CI: two-hourly cron backstop for the main sweep (`main-gate`) + +Measured 2026-08-16..18: **no push-triggered sweep ever reached a runner.** +The coalescing group keeps one running + one pending run and replaces the +pending with each newer merge — and merges landed faster than the +pending→running transition, even with an idle queue, so `main-gate` had never +once executed (and main-line rust-cache/sccache saves only happened on +nightlies). A `47 */2 * * *` cron now fires the sweep reliably; +`scripts/ci_plan.py` maps the nightly cron to `full` and any other cron to +`sweep` (self-tested both ways). The push arm stays for quiet periods. + +Note: the sweep's full `cargo-test` legitimately fails on #8222 today, so +`main-gate` reports red until that lands — deliberately not registered in +`gate_freshness.json` until then (a freshness alarm on a known-red gate is +noise). diff --git a/docs/src/testing/ci-tiers.md b/docs/src/testing/ci-tiers.md index 4412831e9b..c861cc2e14 100644 --- a/docs/src/testing/ci-tiers.md +++ b/docs/src/testing/ci-tiers.md @@ -13,7 +13,7 @@ python3 scripts/ci_plan.py --self-test # the policy's own invariants | tier | trigger | what it is for | fan-in job | |---|---|---|---| | **pr** | every `pull_request` push | the required gate. Small, fast, must be green on `main`. | `pr-gate` — **the only required status context** | -| **sweep** | every `push` to `main`, coalesced | post-merge truth for `main`: the PR tier unscoped plus the medium-weight jobs that do not fit the PR budget | `main-gate` | +| **sweep** | every `push` to `main` (coalesced) **+ a two-hourly cron backstop** | post-merge truth for `main`: the PR tier unscoped plus the medium-weight jobs that do not fit the PR budget | `main-gate` | | **full** | nightly `schedule`, `v*` tags, `workflow_dispatch`, PRs labelled `run-extended-tests` | everything, incl. parity, compile-smoke, doc-tests, package smokes, the 8-shard auto-optimize gap suite | `full-suite-gate` — what `release-packages.yml` waits for | ## The job × tier matrix @@ -110,7 +110,11 @@ which costs no runner slot. `push` to `main` uses ONE constant concurrency group with `cancel-in-progress: false`: GitHub keeps at most one running + one pending run per group and replaces the pending run with the newest, so a burst of merges is tested at its tip instead of queueing 58 -sweeps. Sweep-only jobs are chained behind `check` so a sweep's fan-out does not take +sweeps. **The push arm alone is not sufficient**: measured 2026-08-16..18, no push +sweep ever reached a runner — merges replaced the pending run faster than the +pending→running transition happened, even with an idle queue. The two-hourly cron +(`47 */2 * * *`; the planner maps any non-nightly cron to the sweep tier) is the +reliable arm; the push trigger stays for quiet periods. Sweep-only jobs are chained behind `check` so a sweep's fan-out does not take every runner slot the moment a merge lands. Attribution of a sweep failure is by window (`previous sweep SHA .. this sweep SHA`), exactly as for the six-hourly gates. diff --git a/scripts/ci_plan.py b/scripts/ci_plan.py index 3679154fbf..5fd6a5d296 100755 --- a/scripts/ci_plan.py +++ b/scripts/ci_plan.py @@ -128,6 +128,11 @@ EXTENDED_LABEL = "run-extended-tests" +# The nightly (full-tier) cron in test.yml `on.schedule`. Any OTHER cron firing +# the workflow is the two-hourly sweep backstop. Keep in lockstep with the +# workflow; the self-test asserts both mappings. +NIGHTLY_CRON = "0 4 * * *" + # --------------------------------------------------------------------------- # PR scope classification. # --------------------------------------------------------------------------- @@ -218,7 +223,13 @@ def classify(changed: list[str]) -> dict[str, bool]: # --------------------------------------------------------------------------- # Tier derivation. # --------------------------------------------------------------------------- -def derive_tier(event: str, ref: str, labels: list[str], tier_input: str | None) -> str: +def derive_tier( + event: str, + ref: str, + labels: list[str], + tier_input: str | None, + schedule: str | None = None, +) -> str: if event == "pull_request": return "full" if EXTENDED_LABEL in labels else "pr" if event == "push": @@ -230,6 +241,14 @@ def derive_tier(event: str, ref: str, labels: list[str], tier_input: str | None) # `on.push.branches`), but be explicit if one ever does. return "sweep" if event == "schedule": + # Two crons fire this workflow. The nightly is the full tier; the + # two-hourly is the SWEEP BACKSTOP: the push-triggered sweep uses one + # constant concurrency group (1 running + 1 pending, newest replaces + # pending), and measured 2026-08-16..18 NO push sweep ever reached a + # runner -- merges landed faster than pending->running could happen, + # even with an idle queue. The cron string is passed via --schedule. + if schedule and schedule != NIGHTLY_CRON: + return "sweep" return "full" if event == "workflow_dispatch": return tier_input or "full" @@ -243,9 +262,10 @@ def plan( changed: list[str] | None = None, tier_input: str | None = None, update_gap_snapshot: bool = False, + schedule: str | None = None, ) -> dict: labels = labels or [] - tier = derive_tier(event, ref, labels, tier_input) + tier = derive_tier(event, ref, labels, tier_input, schedule) if event == "pull_request": scope = classify(changed or []) if scope["unknown"]: @@ -334,6 +354,10 @@ def check(name: str, cond: bool): check("push main is sweep", derive_tier("push", "refs/heads/main", [], None) == "sweep") check("tag is full", derive_tier("push", "refs/tags/v0.5.9999", [], None) == "full") check("schedule is full", derive_tier("schedule", "refs/heads/main", [], None) == "full") + check("nightly cron is full", derive_tier("schedule", "refs/heads/main", [], None, NIGHTLY_CRON) == "full") + check("any other cron is the sweep backstop", derive_tier("schedule", "refs/heads/main", [], None, "47 */2 * * *") == "sweep") + backstop = plan("schedule", "refs/heads/main", schedule="47 */2 * * *") + check("sweep backstop runs windows but not parity", backstop["jobs"]["windows_build"] and not backstop["jobs"]["parity"]) check("dispatch default is full", derive_tier("workflow_dispatch", "refs/heads/x", [], None) == "full") check("dispatch tier honoured", derive_tier("workflow_dispatch", "refs/heads/x", [], "sweep") == "sweep") @@ -431,6 +455,7 @@ def main(argv: list[str]) -> int: ap.add_argument("--labels", default="", help="comma-separated PR label names") ap.add_argument("--changed-files", help="file with one changed path per line (PR only)") ap.add_argument("--tier", choices=TIERS, help="workflow_dispatch tier input") + ap.add_argument("--schedule", help="the firing cron string (github.event.schedule)") ap.add_argument("--update-gap-snapshot", action="store_true") ap.add_argument("--table", action="store_true", help="print the tier table as markdown") ap.add_argument("--self-test", action="store_true") @@ -457,6 +482,7 @@ def main(argv: list[str]) -> int: changed=changed, tier_input=args.tier, update_gap_snapshot=args.update_gap_snapshot, + schedule=args.schedule, ) out = json.dumps(p, separators=(",", ":"), sort_keys=True) print(json.dumps(p, indent=2, sort_keys=True))