diff --git a/.github/scripts/queue-monitor.cjs b/.github/scripts/queue-monitor.cjs index a66745c..6e1ee6d 100644 --- a/.github/scripts/queue-monitor.cjs +++ b/.github/scripts/queue-monitor.cjs @@ -17,10 +17,10 @@ const nonterminalRunStatuses = Object.freeze([ const routingRecoverySummary = ` Confirm the managed runner host is running a release that includes the latest capacity and reconciliation fixes (at least \`v0.1.21\` / current \`main\` tip) -before changing routing. An unrebuilt host on an older controller will keep +before escalating. An unrebuilt host on an older controller will keep queuing work against dead capacity even when \`main\` already carries the fix. -Then follow the [audited CI routing-control procedure](https://github.com/melodic-software/github-iac/blob/main/README.md#local-ci-routing-governance) to make the affected repository's effective \`CI_RUNNER_POLICY\` value \`hosted-only\` and verify the readback. Cancel the affected run, choose **Re-run all jobs** to guarantee that the selector executes again, and confirm that it selects hosted capacity. Do not use a failed-job or single-job rerun for this recovery because partial-rerun dependency behavior does not guarantee a fresh selector decision. A \`workflow_dispatch\` creates a separate run with different event and ref context; it does not recover the original pull-request check. +There is no routing cutoff to reach for. ci-perf Phase 7 deleted the \`select-runner\` selector and the audited CI routing-control procedure, so \`CI_RUNNER_POLICY\` no longer exists and no rerun changes where a job runs: **Re-run all jobs** and a failed-job rerun are equivalent for routing. Queued work waits for fleet capacity to return, so recovery is bringing a host back. Moving a job to hosted capacity is not an incident-time action: it needs a per-job \`exceptions\` entry in the affected repository's \`.github/runner-policy.json\` plus a \`runs-on\` change, and the \`reason\` must already be a member of the governed set, which has no fleet-outage member. A \`workflow_dispatch\` creates a separate run with different event and ref context; it does not recover the original pull-request check. `; // A worst-case queue-wide outage could stall far more than a handful of diff --git a/.github/scripts/queue-monitor.test.cjs b/.github/scripts/queue-monitor.test.cjs index d0df36e..a3a9314 100644 --- a/.github/scripts/queue-monitor.test.cjs +++ b/.github/scripts/queue-monitor.test.cjs @@ -45,16 +45,21 @@ test('splitList accepts comma and newline separated configuration', () => { assert.deepEqual(splitList('medley, standards\nci-runner'), ['medley', 'standards', 'ci-runner']); }); -test('recovery requires a verified hosted-only cutoff and fresh selector evaluation', () => { - assert.match(routingRecoverySummary, /audited CI routing-control procedure/); - assert.match(routingRecoverySummary, /effective `CI_RUNNER_POLICY` value `hosted-only`/); - assert.match(routingRecoverySummary, /Re-run all jobs/); - assert.match(routingRecoverySummary, /guarantee that the selector executes again/); - assert.match(routingRecoverySummary, /partial-rerun dependency behavior/); +test('recovery is bringing a host back, with no routing cutoff to reach for', () => { + assert.match(routingRecoverySummary, /no routing cutoff to reach for/); + assert.match(routingRecoverySummary, /Re-run all jobs.*are equivalent for routing/); + assert.match(routingRecoverySummary, /recovery is bringing a host back/); + assert.match(routingRecoverySummary, /not an incident-time action/); assert.match(routingRecoverySummary, /does not recover the original pull-request check/); assert.match(routingRecoverySummary, /at least `v0\.1\.21`/); assert.match(routingRecoverySummary, /Confirm the managed runner host is running a release/); assert.doesNotMatch(routingRecoverySummary, /retry(?:ing)? (?:the )?workload/i); + // Phase 7 deleted the selector and the routing-control procedure. Naming them + // as retired is fine; LINKING a responder to the dead procedure, or telling + // one to set the deleted variable, is what this guard forbids. + assert.doesNotMatch(routingRecoverySummary, /\]\(https:\/\/github\.com\/melodic-software\/github-iac[^)]*local-ci-routing-governance\)/); + assert.doesNotMatch(routingRecoverySummary, /make the affected repository's effective/); + assert.doesNotMatch(routingRecoverySummary, /`CI_RUNNER_POLICY` value `hosted-only`/); }); test('queries every GitHub nonterminal run status and deduplicates runs', async () => { diff --git a/README.md b/README.md index 783dd3d..1ae64b9 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,13 @@ `ci-runner` is the Windows host controller and disposable Linux worker image for a small, demand-scaled GitHub Actions fleet. Its purpose is to move eligible private-repository CI from paid GitHub-hosted runners onto `melo-desk-001` and -`melo-lap-001` without creating a second CI contract. The centrally governed -routing policy decides whether local capacity is preferred, required, or -bypassed. +`melo-lap-001` without creating a second CI contract. There is no routing +policy: an eligible private job names the governed fleet label +`melodic-ubuntu-24.04-x64` as a literal, and GitHub queues it for that label. -Both hosts passed their acceptance gates and serve the -organization's governed `self-hosted-only` routing default. The former +Both hosts passed their acceptance gates and serve every private job that names +the fleet label, which is every eligible private job that does not hold a +declared hosted exception. The former Compose/restart-in-place implementation is retired and its files are deleted; the only production credential entry point is `ci-runner secret import --file PATH`. @@ -17,8 +18,8 @@ the only production credential entry point is ```mermaid flowchart LR - W["Reusable workflow selector"] -->|"managed route"| S["Host-owned scale sets"] - W -->|"hosted route"| H["GitHub-hosted ubuntu-24.04"] + W["Workflow job"] -->|"names melodic-ubuntu-24.04-x64"| S["Host-owned scale sets"] + W -->|"names an approved hosted label"| H["GitHub-hosted ubuntu-24.04"] S --> C1["Windows controller\nmelo-desk-001"] S --> C2["Windows controller\nmelo-lap-001"] C1 --> D1["Fresh one-job Linux containers"] @@ -45,48 +46,72 @@ with current-user DPAPI and never enter worker containers. The controller talks only to the fixed local Docker Engine endpoint and requires a Linux/amd64 engine; `DOCKER_HOST`, TLS, and API-version environment overrides are ignored. -## Routing and fallback contract - -Eligible workflows call the central selector in -[`melodic-software/ci-workflows`](https://github.com/melodic-software/ci-workflows). -The selector owns routing and fallback; this controller only supplies runners -inside its governed name and label namespaces. Immutable selector revisions are -reviewed and allowlisted by the -[`standards` runner policy](https://github.com/melodic-software/standards/blob/main/components/runner-policy/README.md): - -- `hosted-only` returns the approved GitHub-hosted image; -- `prefer-self-hosted` chooses a safe online managed namespace and otherwise - falls back hosted; and -- `self-hosted-only` permits only its reviewed managed label and never falls - back to paid hosted capacity. - -Adaptive selection treats any matching online ephemeral runner as fleet -liveness. A busy runner can therefore keep the managed route eligible, and -GitHub queues the job until matching capacity is available. Security guards, -invalid inventory, missing adaptive credentials, and API failures retain the -selector's documented fail-open or fail-closed behavior for the chosen policy. - -A rerun is not an automatic hosted fallback. Recovery from unavailable local -capacity first uses the audited -[`github-iac` routing-control procedure](https://github.com/melodic-software/github-iac/blob/main/README.md#local-ci-routing-governance) -to make the affected repository's effective policy `hosted-only` and verify the -readback. Then cancel the affected run and choose **Re-run all jobs** to -guarantee that the selector executes again; confirm its output selects hosted -capacity. Do not use a failed-job or single-job rerun for this recovery because -partial-rerun dependency behavior does not guarantee a fresh selector decision. -A `workflow_dispatch` creates a separate run with different event and ref -context; it does not recover the original pull-request check. The hosted queue -monitor reports the condition; it -never changes policy, cancels, or replays work automatically. GitHub documents -the distinct [full and partial rerun +## Routing contract + +**There is no selector and no fallback.** ci-perf Phase 7 retired the central +`select-runner` reusable workflow in +[`melodic-software/ci-workflows`](https://github.com/melodic-software/ci-workflows) +(ci-workflows#569, merged as `541ee4e90d12d77a90a3ddd72a3af9bc78634ea7`, +released as v0.23.0), and melodic-software/standards#556 (merged as +`771a796628f325c3c418c7b397d09fb7211e2972`) removed the selector grammar from +the runner-policy component. The three policies this section used to document, +`hosted-only`, `prefer-self-hosted` and `self-hosted-only`, no longer exist. The +`CI_RUNNER_POLICY` organization variable that chose between them still exists at +`self-hosted-only` and reads nowhere: the selector that consumed it is deleted, +and its removal from the Pulumi program is decided pending the owner's Phase 7 +step 5 apply, so changing it changes nothing. github-iac `OrgCiRouting.cs` still +declares it, unprotected ahead of that apply. + +An eligible private-repository job names the governed fleet label +`melodic-ubuntu-24.04-x64` as a literal in its own `runs-on`. GitHub queues the +job for that label, and this controller supplies runners inside its governed +name and label namespaces exactly as before. What changed is upstream of the +controller: nothing decides at run time where a job goes. The +[`standards` runner policy](https://github.com/melodic-software/standards/blob/main/components/runner-policy/README.md) +admits that literal under its `managed-literal` routing kind and fails a job +that names anything else without a declared exception. + +A busy runner does not divert work. GitHub queues the job until matching +capacity is available, which was already the behavior under the retired +liveness rule and is now structural rather than policy-dependent. + +**There is no consumer-side recovery from an unavailable fleet.** The audited +`github-iac` routing-control procedure is deleted with the workflow that +implemented it (github-iac#453, merged as +`4c5937e6b6067552d11b87abef04620191dda503`), and nothing replaces it. Affected +jobs queue on the fleet label until a host returns. + +The `exceptions` mechanism does **not** cover this case, and reaching for it +will fail. A job that needs hosted capacity declares an entry under `exceptions` +in its repository's own `.github/runner-policy.json`, keyed +`#`, carrying a `justification` and a `reason` that must +be a member of the closed `hostedExceptionReasons` set: `dependabot`, +`docker-socket`, `job-container`, `privileged-control-plane`, `publication`, +`service-container`, `windows`. **None of those expresses "the fleet is +down"**, and a `reason` outside the set is rejected by the analyzer. Granting a +fleet-outage escape would mean adding a member to that set, which is a governed +change to `melodic-software/standards` and its synced `policy.json`, not +something a consumer repository can do for itself. +(`hosted-exception-required` is the finding the analyzer raises when a required +entry is missing, not a key anyone writes.) +A rerun changes nothing about placement: there is no selector verdict to +recompute, so **Re-run all jobs** and a failed-job rerun are equivalent as far +as routing is concerned. Do not re-run a stale run on a superseded head SHA in a +pull request whose concurrency group key does not vary with the head; that +hazard is unrelated to routing and is recorded in github-iac +`docs/topics/ci-perf/POSTURE.md` under "Stale re-runs". A `workflow_dispatch` +still creates a separate run with different event and ref context and does not +recover the original pull-request check. GitHub documents the distinct +[full and partial rerun operations](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/re-run-workflows-and-jobs) and [`workflow_dispatch` event context](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#workflow_dispatch). -The adaptive/hosted selector's two-minute `ubuntu-slim` control job is -independent of downstream job timeouts. Strict `self-hosted-only` selection -keeps that control job on the reviewed managed label so it does not spend hosted -minutes. A selected build/test job can outlive either selector control path. +The two-minute `ubuntu-slim` selector control job is gone with the selector, so +it no longer sits in front of a build. The decision record is github-iac ADR +0014, which supersedes ADR 0004. It is added by melodic-software/github-iac#466 +and is not on that repository's `main` yet, so cite the pull request until it +merges. Authoritative behavior: @@ -419,10 +444,18 @@ reviewed and are never auto-merged. Deployment uses a versioned install directory plus `current` junction; how many known-good pairs are retained is a floor, stated once in the [freshness policy](docs/releases.md#freshness-policy). -Rollback order is: set routing `hosted-only`, drain without killing work, -restore the prior immutable pair, restore the prior reusable-workflow SHA if -needed, then use **Re-run all jobs** for affected workflows and confirm hosted -selection. +Rollback order is: drain without killing work, restore the prior immutable pair, +restore the prior reusable-workflow SHA if needed, then use **Re-run all jobs** +for affected workflows. The first step used to be flipping routing to +`hosted-only`; there is no routing variable to flip and no consumer-side +substitute, so a fleet the rollback cannot restore means the affected jobs +queue until it is back. Plan the rollback window accordingly: the fleet's +availability is now the availability of every private job that routes to it, +which is all of them except the ones already holding a hosted exception. There +is no quick shortcut either, because widening the reason set is a governed +change to the standards runner policy **and** each affected repository still +needs its own per-job `exceptions` entry and a `runs-on` change before anything +routes differently. ## Troubleshooting diff --git a/docs/queue-monitor.md b/docs/queue-monitor.md index 060d99d..edcdc79 100644 --- a/docs/queue-monitor.md +++ b/docs/queue-monitor.md @@ -97,17 +97,20 @@ instruction: > Confirm the managed runner host is running a release that includes the latest > capacity and reconciliation fixes (at least `v0.1.21` / current `main` tip) -> before changing routing. An unrebuilt host on an older controller will keep +> before escalating. An unrebuilt host on an older controller will keep > queuing work against dead capacity even when `main` already carries the fix. > -> Then follow the audited CI routing-control procedure to make the affected -> repository's effective `CI_RUNNER_POLICY` value `hosted-only` and verify the -> readback. Cancel the affected run, choose **Re-run all jobs** to guarantee -> that the selector executes again, and confirm that it selects hosted capacity. -> Do not use a failed-job or single-job rerun for this recovery because -> partial-rerun dependency behavior does not guarantee a fresh selector -> decision. A `workflow_dispatch` creates a separate run with different event -> and ref context; it does not recover the original pull-request check. +> There is no routing cutoff to reach for. ci-perf Phase 7 deleted the +> `select-runner` selector and the audited CI routing-control procedure, so +> `CI_RUNNER_POLICY` no longer exists and no rerun changes where a job runs: +> **Re-run all jobs** and a failed-job rerun are equivalent for routing. Queued +> work waits for fleet capacity to return, so recovery is bringing a host back. +> Moving a job to hosted capacity is not an incident-time action: it needs a +> per-job `exceptions` entry in the affected repository's +> `.github/runner-policy.json` plus a `runs-on` change, and the `reason` must +> already be a member of the governed set, which has no fleet-outage member. A +> `workflow_dispatch` creates a separate run with different event and ref +> context; it does not recover the original pull-request check. The monitor never changes policy, cancels, reruns, dispatches, or mutates a workload. The central selector applies its policy-driven routing rules on every diff --git a/docs/roadmap.md b/docs/roadmap.md index 329a5a8..021bcb4 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -22,8 +22,13 @@ Docker-daemon workloads remain GitHub-hosted. A future backend may run them in a disposable Linux VM whose Docker daemon, filesystem, and runner identity are destroyed after one job. The host Docker socket must never be mounted or proxied into a worker. The backend must preserve JIT registration, external diagnostics, -resource admission, graceful drain, and the audited hosted-only cutoff with a -full workflow rerun that recomputes selector eligibility. +resource admission, and graceful drain. The audited hosted-only cutoff it used +to have to preserve is gone: ci-perf Phase 7 retired the selector, so there is +no eligibility to recompute on a rerun and no routing variable to cut over +with. A backend that cannot serve a job leaves that job queued on the fleet +label until it can. No `hostedExceptionReasons` member expresses a fleet +outage, so there is no consumer-side escape; a backend whose unavailability +must not block CI needs that case designed in, not waived per repository. This work is motivated by GitHub's warning that self-hosted runners can be persistently compromised by workflow code and Docker's warning that daemon @@ -80,8 +85,10 @@ but it must not depend on the monitored schedule or the local fleet. Cost reporting may automate the same GitHub billing-usage summary used for the rollout baseline. It must keep private billing data out of this public -repository, separate selector spend from workload spend, and normalize by -eligible completed jobs before claiming savings. +repository and normalize by eligible completed jobs before claiming savings. +There is no longer any selector spend to separate from workload spend: the +selector's two-minute `ubuntu-slim` control job is retired with the selector +itself. - [GitHub billing usage API](https://docs.github.com/en/rest/billing/usage) - [GitHub scheduled-event limitations](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#schedule)