diff --git a/.agents/skills/ship-pr/SKILL.md b/.agents/skills/ship-pr/SKILL.md index f92c00620..dbd8247e2 100644 --- a/.agents/skills/ship-pr/SKILL.md +++ b/.agents/skills/ship-pr/SKILL.md @@ -75,7 +75,9 @@ Coverage and native `windows` / `macos` stay GitHub Actions; ## Push and poll Required jobs: `fmt`, `deny`, `clippy`, `ast-grep`, `test`, `windows`, -`macos`, `coverage`, `nixos-module-eval`. Structural scan is +`macos`, `coverage`, `nixos-module-eval`, and `mutants (0/4)` through +`mutants (3/4)`. A mutants shard killed at 30 minutes is a pass; a +finished non-zero `cargo mutants` exit is a fail. Structural scan is `./scripts/ast-grep.sh`. `windows` / `macos` are native store plus `--smoke`, not operator zips. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02b7e94c2..48c560f2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,8 +31,9 @@ env: # check (a skip would hide required deny / ast-grep). # clippy, test, windows, and macos start immediately. clippy stays parallel: # coverage waits on it, and a cold rust-cache is a full workspace compile. -# coverage and the advisory mutants shards wait for fmt / clippy / test. -# Weekly full mutants sweep stays mutants.yml. +# coverage and the mutants shards wait for fmt / clippy / test. +# A shard killed at 30m passes; a finished shard with a non-zero +# cargo-mutants exit fails. Weekly sweep stays mutants.yml. # nixos-module-runtime is a labeled job (nixos-module-runtime.yml), not this file. # Operator binaries are GitHub Releases (release.yml), not this workflow. # @@ -205,8 +206,8 @@ jobs: - name: store + node smoke run: ./scripts/ci-os-smoke.sh - # Crates, Cargo manifest/lock, or these workflow files. Docs-only PRs - # skip the four shard runners. Starts immediately; shards still wait for test. + # Crates, Cargo manifest/lock, or these workflow files. The shard jobs + # still run on a docs-only PR and exit 0 so the required check reports. mutants-diff: if: github.event_name == 'pull_request' runs-on: ubuntu-latest @@ -233,21 +234,21 @@ jobs: echo "rust=false" >> "$GITHUB_OUTPUT" fi - # PR in-diff, four shards, same window as coverage. Advisory: a missed - # mutant does not fail the job. One ~300-mutant diff at -j 2 is ~50m and - # was canceled at 30m. Weekly sweep stays mutants.yml. Owner: TESTING.md. + # PR in-diff, four shards. Required. A shard that finishes with a + # non-zero cargo-mutants exit fails the check. A shard killed at 30m + # (GNU timeout 124) passes, even if the partial log shows MISSED. + # Docs-only diffs exit 0 so the check still reports. Weekly sweep + # stays mutants.yml. Owner: TESTING.md. mutants: name: mutants (${{ matrix.shard }}/4) - needs: [fmt, clippy, test, mutants-diff] + needs: [fmt, clippy, test] if: >- github.event_name == 'pull_request' && needs.fmt.result == 'success' && needs.clippy.result == 'success' && needs.test.result == 'success' - && needs.mutants-diff.result == 'success' - && needs.mutants-diff.outputs.rust == 'true' runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 40 strategy: fail-fast: false matrix: @@ -264,35 +265,73 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 + - name: rust or workflow diff + id: diff + env: + BASE_REF: ${{ github.base_ref }} + run: | + set -euo pipefail + git diff "origin/${BASE_REF}".. --unified=0 -- \ + crates Cargo.toml Cargo.lock \ + .github/workflows/ci.yml .github/workflows/mutants.yml \ + > git.diff + if [[ -s git.diff ]]; then + echo "rust=true" >> "$GITHUB_OUTPUT" + else + echo "rust=false" >> "$GITHUB_OUTPUT" + echo "no rust or workflow diff; shard passes" + fi - uses: dtolnay/rust-toolchain@a5f673d0ba8626c3977bb416a1612774bc82181b # rustc 1.95.0 + if: steps.diff.outputs.rust == 'true' - uses: taiki-e/install-action@9114bf4d891761788c546334fd37538eae1bf8b3 # v2.87.16 + if: steps.diff.outputs.rust == 'true' with: tool: cargo-mutants@27.1.0 - name: in-diff shard id: run - continue-on-error: true - env: - BASE_REF: ${{ github.base_ref }} + if: steps.diff.outputs.rust == 'true' run: | set -euo pipefail - git diff "origin/${BASE_REF}".. --unified=0 > git.diff set +e - cargo mutants --workspace --in-diff git.diff \ - --shard "${{ matrix.shard }}/4" --sharding slice -j 2 \ + timeout --signal=TERM --kill-after=60s 30m \ + cargo mutants --workspace --in-diff git.diff \ + --shard "${{ matrix.shard }}/4" --sharding slice -j 2 \ > mutants-shard.log 2>&1 ec=$? set -e missed=$(grep -c '^MISSED' mutants-shard.log || true) echo "missed=${missed}" >> "$GITHUB_OUTPUT" cat mutants-shard.log + if [[ "$ec" -eq 124 ]]; then + echo "result=timeout" >> "$GITHUB_OUTPUT" + echo "shard hit the 30m limit; counting as success" + exit 0 + fi + echo "result=finished" >> "$GITHUB_OUTPUT" exit "$ec" - name: summary if: always() run: | { echo "### mutants ${{ matrix.shard }}/4" - echo "Advisory (not a merge gate). Outcome: ${{ steps.run.outcome }}. Missed: ${{ steps.run.outputs.missed }}." + echo "Required. Outcome: ${{ steps.run.outcome }}. Result: ${{ steps.run.outputs.result }}. Missed: ${{ steps.run.outputs.missed }}." + echo "A 30m timeout counts as success. A finished non-zero exit fails." } >> "$GITHUB_STEP_SUMMARY" + - name: fail unless success or timeout + if: always() && steps.diff.outputs.rust == 'true' + run: | + set -euo pipefail + result="${{ steps.run.outputs.result }}" + outcome="${{ steps.run.outcome }}" + if [[ "$result" == "timeout" || "$outcome" == "cancelled" ]]; then + echo "timeout counts as success" + exit 0 + fi + if [[ "$outcome" == "success" && "$result" == "finished" ]]; then + exit 0 + fi + echo "mutants shard failed (outcome=${outcome} result=${result})" + exit 1 - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: diff --git a/.github/workflows/mutants.yml b/.github/workflows/mutants.yml index 27b5d2741..fa63d7c9c 100644 --- a/.github/workflows/mutants.yml +++ b/.github/workflows/mutants.yml @@ -1,5 +1,5 @@ # Weekly cargo-mutants. Not a required check. -# PR in-diff shards are ci.yml job `mutants` (parallel with coverage). +# PR in-diff shards are ci.yml job `mutants` (required; a 30m timeout passes). # Must pass --workspace: default-members is rbitcoin-node only. # Owner: TESTING.md (Mutation testing). Snapshot lists: docs/mutants/. diff --git a/AGENTS.md b/AGENTS.md index c08ca012c..9fb76978a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -124,3 +124,6 @@ Playbooks: Owner playbook: [`docs/releases.md`](docs/releases.md). - Core functional harness: [`.agents/skills/core-functional/SKILL.md`](.agents/skills/core-functional/SKILL.md). +- PR `mutants (0/4)` through `mutants (3/4)` are required. A shard that hits + the 30 minute limit passes. A finished shard with a non-zero + `cargo mutants` exit fails. Owner: [`TESTING.md`](TESTING.md). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2913a6543..7f32294b8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -241,7 +241,7 @@ to it. Darwin operator binaries come from the `macos-14` release job, not Nix. Required **jobs** match [`.github/workflows/ci.yml`](./.github/workflows/ci.yml) (`fmt`, `deny`, `clippy`, `ast-grep`, `test`, `windows`, `macos`, -`coverage`, `nixos-module-eval`) so a red run shows which gate failed without +`coverage`, `nixos-module-eval`, `mutants (0/4)` … `mutants (3/4)`) so a red run shows which gate failed without digging into a monolithic job log. Label **`nixos-module-runtime`** to run the NixOS module qemu test (not eval). That job is not required. diff --git a/TESTING.md b/TESTING.md index 2efdcf5b1..758d7d91f 100644 --- a/TESTING.md +++ b/TESTING.md @@ -254,7 +254,7 @@ matches scalar in default tests). Owner: [`docs/quality.md`](./docs/quality.md). | **cargo-crap** | After LCOV, `./scripts/coverage.sh` calls `./scripts/coverage-crap.sh` (skip if `cargo-crap` missing). `--fail-above --threshold 30`; `.cargo-crap.toml` allowlists today's production CRAP>30 functions (remove a name when it scores ≤30). Dry-run: `CRAP_DRY_RUN=1 ./scripts/coverage-crap.sh`. Self-test: `./scripts/coverage-crap.test.sh` | Rides required `coverage`. No `--fail-regression` (llvm-cov coverage % jitters per function) | | **coverage ignore / badge** | `./scripts/coverage.test.sh` (filename ignore, Tier A IBD not skipped, 92% floor, Shields JSON). Publish dry-run: `BADGE_DRY_RUN=1 ./scripts/publish-coverage-badge.sh` | `test` job self-test; `coverage` job writes `coverage/badge.json` and, on green `master`, pushes `badges/coverage.json` | | **Miri** | `./scripts/miri.sh` → `cargo +nightly miri test -p rbitcoin-primitives`. Dry-run: `MIRI_DRY_RUN=1 ./scripts/miri.sh`. Self-test: `./scripts/miri.test.sh` | Nightly `miri.yml` (not required). Never `--workspace` | -| **cargo-mutants** | Must pass `--workspace` (`Cargo.toml` `default-members` is `rbitcoin-node` only; without `--workspace` the list is ~7 mutants). PR: `git diff origin/.. --unified=0 > git.diff` then `cargo mutants --workspace --in-diff git.diff --shard k/4 --sharding slice -j 2` (same flags on every shard). Weekly: `cargo mutants --workspace --shard N/8 -j 2`. Snapshot of missed/timeouts: [`docs/mutants/`](docs/mutants/). | PR: `ci.yml` `mutants`, 4 shards, needs fmt/clippy/test (same window as coverage). `continue-on-error` (not a merge gate). Skipped when the diff does not touch `crates/`, `Cargo.toml`, `Cargo.lock`, `ci.yml`, or `mutants.yml`. Weekly `mutants.yml` Sunday 03:00 UTC, 8 shards, 360min cap (not required). `workflow_dispatch`. First Sunday wall-clock is still a measurement. | +| **cargo-mutants** | Must pass `--workspace` (`Cargo.toml` `default-members` is `rbitcoin-node` only; without `--workspace` the list is ~7 mutants). PR: `git diff origin/.. --unified=0 -- crates Cargo.toml Cargo.lock .github/workflows/ci.yml .github/workflows/mutants.yml > git.diff` then `cargo mutants --workspace --in-diff git.diff --shard k/4 --sharding slice -j 2` (same flags on every shard). Weekly: `cargo mutants --workspace --shard N/8 -j 2`. Snapshot of missed/timeouts: [`docs/mutants/`](docs/mutants/). | PR: `ci.yml` `mutants (0/4)`–`mutants (3/4)`, required, needs fmt/clippy/test. A shard that finishes with a non-zero exit fails. A shard killed at 30 minutes counts as a pass, even if the partial log shows `MISSED`. No rust or workflow diff: the shard exits 0. Weekly `mutants.yml` Sunday 03:00 UTC, 8 shards, 360min cap (not required). `workflow_dispatch`. | Artifact silos above are unchanged: ast-grep / Miri dry-run / crap dry-run do not write `target/`. `mutants.out/` is gitignored. @@ -522,3 +522,9 @@ python3 scripts/core-functional/check_inventory.py ./scripts/overlay-functional/run.sh.test.sh ./scripts/overlay-functional/run.sh --list ``` + +## What a mutant kill looks like + +The PR gate is the four `mutants` shards in [`ci.yml`](.github/workflows/ci.yml), described in the table above. Do not run a different `cargo mutants` command and treat that as the gate. + +A new production behavior needs a test that fails when that behavior is removed or inverted. Assert the observable result (the error variant, the bytes, the height), not the test name. Concurrency, ordering, and cancellation are not what `cargo-mutants` models; those need a direct test. A diff that does not touch `crates/`, the Cargo manifests, or the mutants workflows does not run mutants. diff --git a/docs/releases.md b/docs/releases.md index ea48e8605..63bba5f7e 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -103,7 +103,7 @@ Cargo.toml). Those PRs run Core functional even without a label. | Check | Who | |-------|-----| -| `fmt` `deny` `clippy` `ast-grep` `test` `windows` `macos` `coverage` `nixos-module-eval` | Every PR (`ci.yml`) | +| `fmt` `deny` `clippy` `ast-grep` `test` `windows` `macos` `coverage` `nixos-module-eval` `mutants (0/4)`–`(3/4)` | Every PR (`ci.yml`). A mutants shard killed at 30 minutes passes; a finished non-zero exit fails. | | `core-functional` | Nightly, `workflow_dispatch`, label **`core-functional`**, label **`release`**, **or** ship version | | `overlay-functional` | Nightly (`42 6`), `workflow_dispatch`, label **`overlay-functional`**, label **`release`**, **or** ship version. Not required. Not in `release-extra` yet. | | `warnet-example` | Label **`warnet`** or `workflow_dispatch`. Two-tank Docker lab. Not required. Not a ship gate. | diff --git a/scripts/pr-checks-watch.sh b/scripts/pr-checks-watch.sh index 05ff6b775..93649fac1 100755 --- a/scripts/pr-checks-watch.sh +++ b/scripts/pr-checks-watch.sh @@ -21,6 +21,10 @@ REQUIRED=( macos coverage nixos-module-eval + "mutants (0/4)" + "mutants (1/4)" + "mutants (2/4)" + "mutants (3/4)" ) usage() { diff --git a/scripts/pr-checks-watch.test.sh b/scripts/pr-checks-watch.test.sh index c88bb2bf5..d90c599cf 100755 --- a/scripts/pr-checks-watch.test.sh +++ b/scripts/pr-checks-watch.test.sh @@ -45,7 +45,11 @@ all_green="$( windows pass 1m \ macos pass 50s \ coverage pass 2m \ - nixos-module-eval pass 32s + nixos-module-eval pass 32s \ + 'mutants (0/4)' pass 2m \ + 'mutants (1/4)' pass 2m \ + 'mutants (2/4)' pass 2m \ + 'mutants (3/4)' pass 2m )" rc=0 out="$(CI_PR_CHECKS_TEXT="$all_green" "$RUN" --once 2>&1)" || rc=$? @@ -63,6 +67,10 @@ analyze_pending_windows_green="$( macos pass 50s \ coverage pass 2m \ nixos-module-eval pass 32s \ + 'mutants (0/4)' pass 2m \ + 'mutants (1/4)' pass 2m \ + 'mutants (2/4)' pass 2m \ + 'mutants (3/4)' pass 2m \ 'Analyze (rust)' pending 0 )" rc=0