Use cargo-nextest for Rust tests

.cargo/mutants.toml

@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: Apache-2.0
+
+test_tool = "nextest"
+additional_cargo_test_args = ["--profile=mutants"]
+cap_lints = false

.config/nextest.toml

@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: Apache-2.0
+
+nextest-version = "0.9.133"
+
+[profile.ci]
+fail-fast = false
+failure-output = "immediate-final"
+
+[profile.mutants]
+fail-fast = true
+failure-output = "final"

.github/workflows/ci-rust-python-package.yaml

@@ -7,6 +7,8 @@ on:
       - "Makefile"
       - "Cargo.toml"
       - "Cargo.lock"
+      - ".cargo/**"
+      - ".config/nextest.toml"
       - "deny.toml"
       - "crates/**"
       - "README.md"
@@ -23,6 +25,8 @@ on:
       - "Makefile"
       - "Cargo.toml"
       - "Cargo.lock"
+      - ".cargo/**"
+      - ".config/nextest.toml"
       - "deny.toml"
       - "crates/**"
       - "README.md"
@@ -51,6 +55,9 @@ jobs:
       has_plugins: ${{ steps.detect.outputs.has_plugins }}
       plugin_count: ${{ steps.detect.outputs.plugin_count }}
       cargo_packages: ${{ steps.detect.outputs.cargo_packages }}
+      mutation_cargo_packages: ${{ steps.detect.outputs.mutation_cargo_packages }}
+      mutation_jobs: ${{ steps.detect.outputs.mutation_jobs }}
+      has_mutation_cargo_packages: ${{ steps.detect.outputs.has_mutation_cargo_packages }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
         with:
@@ -65,7 +72,7 @@ jobs:
         run: |
           set -euo pipefail
           if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
-            selection="$(python3 tools/plugin_catalog.py ci-selection . diff "${{ github.event.pull_request.base.sha }}" "${{ github.sha }}")"
+            selection="$(python3 tools/plugin_catalog.py ci-selection . diff "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}")"
           elif [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then
             selection="$(python3 tools/plugin_catalog.py ci-selection . all '' '')"
           elif [[ "${{ github.event.before }}" == "0000000000000000000000000000000000000000" ]]; then
@@ -74,21 +81,28 @@ jobs:
             selection="$(python3 tools/plugin_catalog.py ci-selection . diff "${{ github.event.before }}" "${{ github.sha }}")"
           fi
 
-          selection="$(printf '%s' "${selection}" | python3 -c 'import json, re, sys; payload = json.load(sys.stdin); slug_re = re.compile(r"^[a-z0-9_]+$"); plugins = payload.get("plugins"); cargo_packages = payload.get("cargo_packages"); has_plugins = payload.get("has_plugins"); plugin_count = payload.get("plugin_count"); assert isinstance(plugins, list) and all(isinstance(item, str) and slug_re.fullmatch(item) for item in plugins); assert isinstance(cargo_packages, list) and all(isinstance(item, str) and slug_re.fullmatch(item) for item in cargo_packages); assert isinstance(has_plugins, bool); assert isinstance(plugin_count, int) and plugin_count == len(plugins); print(json.dumps({"plugins": plugins, "has_plugins": has_plugins, "plugin_count": plugin_count, "cargo_packages": cargo_packages}))')"
+          selection="$(printf '%s' "${selection}" | python3 tools/validate_ci_selection.py)"
           plugins="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["plugins"]))')"
           has_plugins="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(str(json.load(sys.stdin)["has_plugins"]).lower())')"
           plugin_count="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.load(sys.stdin)["plugin_count"])')"
           cargo_packages="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["cargo_packages"]))')"
+          mutation_cargo_packages="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["mutation_cargo_packages"]))')"
+          mutation_jobs="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["mutation_jobs"]))')"
+          has_mutation_cargo_packages="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(str(json.load(sys.stdin)["has_mutation_cargo_packages"]).lower())')"
           if [[ "${has_plugins}" == "false" ]]; then
             has_plugins_output="false"
           else
             has_plugins_output="true"
           fi
+          has_mutation_cargo_packages_output="${has_mutation_cargo_packages}"
           {
             echo "plugins=${plugins}"
             echo "plugin_count=${plugin_count}"
             echo "cargo_packages=${cargo_packages}"
             echo "has_plugins=${has_plugins_output}"
+            echo "mutation_cargo_packages=${mutation_cargo_packages}"
+            echo "mutation_jobs=${mutation_jobs}"
+            echo "has_mutation_cargo_packages=${has_mutation_cargo_packages_output}"
           } >> "$GITHUB_OUTPUT"
 
   build-test:
@@ -115,6 +129,23 @@ jobs:
           rustc --version
           cargo --version
 
+      - name: Install cargo-nextest
+        run: |
+          if [[ "${RUNNER_OS}" == "Linux" ]]; then
+            mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin"
+            curl -LsSf https://get.nexte.st/0.9.133/linux | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
+          elif [[ "${RUNNER_OS}" == "macOS" ]]; then
+            mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin"
+            if [[ "$(uname -m)" == "arm64" ]]; then
+              curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
+            else
+              curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
+            fi
+          else
+            cargo install cargo-nextest --version 0.9.133 --locked
+          fi
+          cargo nextest --version
+
       - name: Install uv
         run: python -m pip install uv==0.9.30 maturin==1.12.6
 
@@ -125,11 +156,15 @@ jobs:
       - name: Plugin CI build verification
         if: matrix.os == 'ubuntu-latest'
         working-directory: plugins/rust/python-package/${{ matrix.plugin }}
+        env:
+          NEXTEST_PROFILE: ci
         run: make ci-build
 
       - name: Plugin CI verification
         if: matrix.os != 'ubuntu-latest'
         working-directory: plugins/rust/python-package/${{ matrix.plugin }}
+        env:
+          NEXTEST_PROFILE: ci
         run: make ci
 
   security-policy:
@@ -150,6 +185,73 @@ jobs:
       - name: Run cargo deny
         run: cargo deny check --config deny.toml
 
+  mutation-testing:
+    needs: validate-and-detect
+    if: github.event_name == 'pull_request' && needs.validate-and-detect.outputs.has_mutation_cargo_packages == 'true'
+    strategy:
+      fail-fast: false
+      matrix:
+        mutation_job: ${{ fromJson(needs.validate-and-detect.outputs.mutation_jobs) }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        with:
+          python-version: "3.12"
+
+      - name: Verify Rust toolchain
+        run: |
+          rustc --version
+          cargo --version
+
+      - name: Install Rust mutation testing tooling
+        run: |
+          if [[ "${RUNNER_OS}" == "Linux" ]]; then
+            mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin"
+            curl -LsSf https://get.nexte.st/0.9.133/linux | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
+          elif [[ "${RUNNER_OS}" == "macOS" ]]; then
+            mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin"
+            if [[ "$(uname -m)" == "arm64" ]]; then
+              curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
+            else
+              curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
+            fi
+          else
+            cargo install cargo-nextest --version 0.9.133 --locked
+          fi
+          cargo nextest --version
+          cargo install cargo-mutants --version 27.0.0 --locked
+          cargo mutants --version
+
+      - name: Create mutation diff
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: git diff "${BASE_SHA}..${HEAD_SHA}" -- '*.rs' > cargo-mutants.diff
+
+      - name: Run cargo-mutants with nextest
+        env:
+          CARGO_PACKAGE: ${{ matrix.mutation_job.cargo_package }}
+          IN_DIFF: ${{ matrix.mutation_job.in_diff }}
+          PYO3_PYTHON: python
+          TEST_PACKAGES: ${{ toJson(matrix.mutation_job.test_packages) }}
+        run: |
+          mapfile -t test_packages < <(python3 -c 'import json, os; [print(package) for package in json.loads(os.environ["TEST_PACKAGES"])]')
+          cargo_args=("-p" "${CARGO_PACKAGE}")
+          if [[ "${IN_DIFF}" == "true" ]]; then
+            cargo_args+=("--in-diff" "cargo-mutants.diff")
+          fi
+          for package in "${test_packages[@]}"; do
+            cargo_args+=("--test-package" "${package}")
+          done
+          cargo mutants "${cargo_args[@]}"
+
   coverage:
     needs: validate-and-detect
     if: needs.validate-and-detect.outputs.has_plugins == 'true'
@@ -169,12 +271,30 @@ jobs:
           rustup component add llvm-tools-preview
           cargo install cargo-llvm-cov --version 0.8.4 --locked
 
+      - name: Install cargo-nextest
+        run: |
+          if [[ "${RUNNER_OS}" == "Linux" ]]; then
+            mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin"
+            curl -LsSf https://get.nexte.st/0.9.133/linux | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
+          elif [[ "${RUNNER_OS}" == "macOS" ]]; then
+            mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin"
+            if [[ "$(uname -m)" == "arm64" ]]; then
+              curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
+            else
+              curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
+            fi
+          else
+            cargo install cargo-nextest --version 0.9.133 --locked
+          fi
+          cargo nextest --version
+
       - name: Install Python build tooling
         run: python -m pip install uv==0.9.30 maturin==1.12.6
 
       - name: Generate Rust coverage report
         env:
           CARGO_PACKAGES: ${{ needs.validate-and-detect.outputs.cargo_packages }}
+          NEXTEST_PROFILE: ci
           PLUGINS: ${{ needs.validate-and-detect.outputs.plugins }}
           PYO3_PYTHON: python
         run: |
@@ -186,6 +306,8 @@ jobs:
             cargo_args+=("-p" "${package}")
           done
           cargo llvm-cov clean --workspace
+          mkdir -p coverage
+          cargo llvm-cov nextest --no-report "${cargo_args[@]}" -P "${NEXTEST_PROFILE}"
           eval "$(cargo llvm-cov show-env --sh)"
           export CARGO_TARGET_DIR="${CARGO_LLVM_COV_TARGET_DIR}/llvm-cov-target"
           export CARGO_LLVM_COV_BUILD_DIR="${CARGO_TARGET_DIR}"
@@ -194,7 +316,6 @@ jobs:
           for plugin in "${plugins[@]}"; do
             (cd "plugins/rust/python-package/${plugin}" && make sync && uv run maturin develop)
           done
-          cargo test "${cargo_args[@]}"
           for plugin in "${plugins[@]}"; do
             (cd "plugins/rust/python-package/${plugin}" && make test-integration)
           done

.github/workflows/ci-scaffold-generator.yaml

@@ -27,9 +27,9 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
 
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
         with:
           python-version: "3.12"
 
@@ -52,6 +52,23 @@ jobs:
           rustc --version
           cargo --version
 
+      - name: Install cargo-nextest
+        run: |
+          if [[ "${RUNNER_OS}" == "Linux" ]]; then
+            mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin"
+            curl -LsSf https://get.nexte.st/0.9.133/linux | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
+          elif [[ "${RUNNER_OS}" == "macOS" ]]; then
+            mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin"
+            if [[ "$(uname -m)" == "arm64" ]]; then
+              curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
+            else
+              curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
+            fi
+          else
+            cargo install cargo-nextest --version 0.9.133 --locked
+          fi
+          cargo nextest --version
+
       - name: Generate default plugin (tool_pre_invoke)
         run: |
           python3 tools/scaffold_plugin.py --non-interactive \

Makefile

@@ -1,7 +1,7 @@
-.PHONY: help plugins-list plugins-validate plugin-test plugin-scaffold plugin-scaffold-help
+.PHONY: help plugins-list plugins-validate plugin-test plugin-mutants plugin-mutants-list plugin-scaffold plugin-scaffold-help
 
 help:
-	@printf "plugins-list\nplugins-validate\nplugin-test PLUGIN=<slug>\nplugin-scaffold\nplugin-scaffold-help\n"
+	@printf "plugins-list\nplugins-validate\nplugin-test PLUGIN=<slug>\nplugin-mutants PLUGIN=<slug>\nplugin-mutants-list PLUGIN=<slug>\nplugin-scaffold\nplugin-scaffold-help\n"
 
 plugins-list:
 	@python3 tools/plugin_catalog.py list .
@@ -14,6 +14,14 @@ plugin-test:
 	@test -n "$(PLUGIN)" || (echo "Set PLUGIN=<slug>" && exit 1)
 	@cd plugins/rust/python-package/$(PLUGIN) && make sync && make ci
 
+plugin-mutants:
+	@test -n "$(PLUGIN)" || (echo "Set PLUGIN=<slug>" && exit 1)
+	cargo mutants -p "$(PLUGIN)"
+
+plugin-mutants-list:
+	@test -n "$(PLUGIN)" || (echo "Set PLUGIN=<slug>" && exit 1)
+	cargo mutants --list -p "$(PLUGIN)"
+
 plugin-scaffold:
 	@python3 -m pip install --quiet jinja2 2>/dev/null || pip install --quiet jinja2 2>/dev/null || true
 	@python3 tools/scaffold_plugin.py

TESTING.md

@@ -30,10 +30,13 @@ Each plugin has its own Rust and Python test suite.
 ```bash
 cd plugins/rust/python-package/rate_limiter
 uv sync --dev
+cargo install cargo-nextest --version 0.9.133 --locked
 make install
 make test-all
 ```
 
+Set `NEXTEST_PROFILE=ci` to use the repository CI profile locally. The CI profile is defined in `.config/nextest.toml`; it disables fail-fast so all Rust test failures are reported in one run.
+
 Equivalent repo-level helper:
 
 ```bash
@@ -51,6 +54,7 @@ To run the same coverage check locally for all managed Rust plugins:
 ```bash
 rustup component add llvm-tools-preview
 cargo install cargo-llvm-cov --version 0.8.4 --locked
+cargo install cargo-nextest --version 0.9.133 --locked
 mkdir -p coverage
 CARGO_PACKAGES="$(python3 tools/plugin_catalog.py ci-selection-field . all '' '' cargo_packages)"
 PLUGINS="$(python3 tools/plugin_catalog.py ci-selection-field . all '' '' plugins)"
@@ -61,6 +65,7 @@ for package in "${cargo_packages[@]}"; do
   cargo_args+=("-p" "${package}")
 done
 cargo llvm-cov clean --workspace
+cargo llvm-cov nextest --no-report "${cargo_args[@]}" -P ci
 eval "$(cargo llvm-cov show-env --sh)"
 export CARGO_TARGET_DIR="${CARGO_LLVM_COV_TARGET_DIR}/llvm-cov-target"
 export CARGO_LLVM_COV_BUILD_DIR="${CARGO_TARGET_DIR}"
@@ -69,14 +74,30 @@ mkdir -p "${CARGO_TARGET_DIR}"
 for plugin in "${plugins[@]}"; do
   (cd "plugins/rust/python-package/${plugin}" && make sync && uv run maturin develop)
 done
-cargo test "${cargo_args[@]}"
 for plugin in "${plugins[@]}"; do
   (cd "plugins/rust/python-package/${plugin}" && make test-integration)
 done
 env -u CARGO_TARGET_DIR -u CARGO_LLVM_COV_BUILD_DIR -u CARGO_LLVM_COV_TARGET_DIR -u LLVM_PROFILE_FILE cargo llvm-cov report "${cargo_args[@]}" --cobertura --output-path coverage/cobertura.xml
 python3 tools/plugin_catalog.py coverage-check . coverage/cobertura.xml 90.00 "${PLUGINS}"
 ```
 
+Rust unit tests use `cargo nextest run`. Coverage uses `cargo llvm-cov nextest --no-report` for the Rust test phase, then runs pytest before generating the final report so PyO3 paths stay covered. CI uses the `ci` nextest profile, which disables fail-fast and prints failure output immediately and again at the end. Nextest does not run Rust doctests; this repo currently has no Rust doctest code blocks, so there is no separate doctest step.
+
+Criterion benchmarks are verified in CI with `cargo nextest run --benches -E 'kind(bench)' --no-run`, which compiles benchmark test targets without rerunning normal unit tests or collecting noisy performance measurements on shared CI runners.
+
+## 4. Mutation Testing
+
+Mutation testing runs in PR CI on Ubuntu for Rust code touched by the pull request diff. It is also available locally through cargo-mutants and runs Rust tests with nextest.
+
+```bash
+cargo install cargo-nextest --version 0.9.133 --locked
+cargo install cargo-mutants --version 27.0.0 --locked
+make plugin-mutants-list PLUGIN=retry_with_backoff
+make plugin-mutants PLUGIN=retry_with_backoff
+```
+
+`.cargo/mutants.toml` sets `test_tool = "nextest"`, selects the `mutants` nextest profile, and keeps `cap_lints = false` so Rust warnings are not downgraded during mutant builds. The `mutants` profile keeps fail-fast enabled because cargo-mutants only needs one failing test to mark a mutant as caught. CI installs `cargo-mutants` with `cargo install cargo-mutants --version 27.0.0 --locked` and runs `cargo mutants "${cargo_args[@]}"`, using `--in-diff cargo-mutants.diff` for Rust source changes and full-package mutation for mutation-tooling config changes.
+
 ## CI Behavior
 
 Repo contract tests run in their own CI workflow. The Rust plugin CI workflow uses the same plugin catalog to select affected plugin build, integration, and coverage jobs.