diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml new file mode 100644 index 0000000..9a5d9ad --- /dev/null +++ b/.cargo/mutants.toml @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 + +test_tool = "nextest" +additional_cargo_test_args = ["--profile=mutants"] +cap_lints = false diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 0000000..ac330e2 --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 + +nextest-version = "0.9.133" + +[profile.ci] +fail-fast = false +failure-output = "immediate-final" + +[profile.mutants] +fail-fast = true +failure-output = "final" diff --git a/.github/workflows/ci-rust-python-package.yaml b/.github/workflows/ci-rust-python-package.yaml index 94c3c4e..a4f9090 100644 --- a/.github/workflows/ci-rust-python-package.yaml +++ b/.github/workflows/ci-rust-python-package.yaml @@ -7,6 +7,8 @@ on: - "Makefile" - "Cargo.toml" - "Cargo.lock" + - ".cargo/**" + - ".config/nextest.toml" - "deny.toml" - "crates/**" - "README.md" @@ -23,6 +25,8 @@ on: - "Makefile" - "Cargo.toml" - "Cargo.lock" + - ".cargo/**" + - ".config/nextest.toml" - "deny.toml" - "crates/**" - "README.md" @@ -51,6 +55,9 @@ jobs: has_plugins: ${{ steps.detect.outputs.has_plugins }} plugin_count: ${{ steps.detect.outputs.plugin_count }} cargo_packages: ${{ steps.detect.outputs.cargo_packages }} + mutation_cargo_packages: ${{ steps.detect.outputs.mutation_cargo_packages }} + mutation_jobs: ${{ steps.detect.outputs.mutation_jobs }} + has_mutation_cargo_packages: ${{ steps.detect.outputs.has_mutation_cargo_packages }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd with: @@ -65,7 +72,7 @@ jobs: run: | set -euo pipefail if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then - selection="$(python3 tools/plugin_catalog.py ci-selection . diff "${{ github.event.pull_request.base.sha }}" "${{ github.sha }}")" + selection="$(python3 tools/plugin_catalog.py ci-selection . diff "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}")" elif [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then selection="$(python3 tools/plugin_catalog.py ci-selection . all '' '')" elif [[ "${{ github.event.before }}" == "0000000000000000000000000000000000000000" ]]; then @@ -74,21 +81,28 @@ jobs: selection="$(python3 tools/plugin_catalog.py ci-selection . diff "${{ github.event.before }}" "${{ github.sha }}")" fi - selection="$(printf '%s' "${selection}" | python3 -c 'import json, re, sys; payload = json.load(sys.stdin); slug_re = re.compile(r"^[a-z0-9_]+$"); plugins = payload.get("plugins"); cargo_packages = payload.get("cargo_packages"); has_plugins = payload.get("has_plugins"); plugin_count = payload.get("plugin_count"); assert isinstance(plugins, list) and all(isinstance(item, str) and slug_re.fullmatch(item) for item in plugins); assert isinstance(cargo_packages, list) and all(isinstance(item, str) and slug_re.fullmatch(item) for item in cargo_packages); assert isinstance(has_plugins, bool); assert isinstance(plugin_count, int) and plugin_count == len(plugins); print(json.dumps({"plugins": plugins, "has_plugins": has_plugins, "plugin_count": plugin_count, "cargo_packages": cargo_packages}))')" + selection="$(printf '%s' "${selection}" | python3 tools/validate_ci_selection.py)" plugins="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["plugins"]))')" has_plugins="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(str(json.load(sys.stdin)["has_plugins"]).lower())')" plugin_count="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.load(sys.stdin)["plugin_count"])')" cargo_packages="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["cargo_packages"]))')" + mutation_cargo_packages="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["mutation_cargo_packages"]))')" + mutation_jobs="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["mutation_jobs"]))')" + has_mutation_cargo_packages="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(str(json.load(sys.stdin)["has_mutation_cargo_packages"]).lower())')" if [[ "${has_plugins}" == "false" ]]; then has_plugins_output="false" else has_plugins_output="true" fi + has_mutation_cargo_packages_output="${has_mutation_cargo_packages}" { echo "plugins=${plugins}" echo "plugin_count=${plugin_count}" echo "cargo_packages=${cargo_packages}" echo "has_plugins=${has_plugins_output}" + echo "mutation_cargo_packages=${mutation_cargo_packages}" + echo "mutation_jobs=${mutation_jobs}" + echo "has_mutation_cargo_packages=${has_mutation_cargo_packages_output}" } >> "$GITHUB_OUTPUT" build-test: @@ -115,6 +129,23 @@ jobs: rustc --version cargo --version + - name: Install cargo-nextest + run: | + if [[ "${RUNNER_OS}" == "Linux" ]]; then + mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin" + curl -LsSf https://get.nexte.st/0.9.133/linux | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin" + elif [[ "${RUNNER_OS}" == "macOS" ]]; then + mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin" + if [[ "$(uname -m)" == "arm64" ]]; then + curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin" + else + curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin" + fi + else + cargo install cargo-nextest --version 0.9.133 --locked + fi + cargo nextest --version + - name: Install uv run: python -m pip install uv==0.9.30 maturin==1.12.6 @@ -125,11 +156,15 @@ jobs: - name: Plugin CI build verification if: matrix.os == 'ubuntu-latest' working-directory: plugins/rust/python-package/${{ matrix.plugin }} + env: + NEXTEST_PROFILE: ci run: make ci-build - name: Plugin CI verification if: matrix.os != 'ubuntu-latest' working-directory: plugins/rust/python-package/${{ matrix.plugin }} + env: + NEXTEST_PROFILE: ci run: make ci security-policy: @@ -150,6 +185,73 @@ jobs: - name: Run cargo deny run: cargo deny check --config deny.toml + mutation-testing: + needs: validate-and-detect + if: github.event_name == 'pull_request' && needs.validate-and-detect.outputs.has_mutation_cargo_packages == 'true' + strategy: + fail-fast: false + matrix: + mutation_job: ${{ fromJson(needs.validate-and-detect.outputs.mutation_jobs) }} + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + fetch-depth: 0 + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 + with: + python-version: "3.12" + + - name: Verify Rust toolchain + run: | + rustc --version + cargo --version + + - name: Install Rust mutation testing tooling + run: | + if [[ "${RUNNER_OS}" == "Linux" ]]; then + mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin" + curl -LsSf https://get.nexte.st/0.9.133/linux | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin" + elif [[ "${RUNNER_OS}" == "macOS" ]]; then + mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin" + if [[ "$(uname -m)" == "arm64" ]]; then + curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin" + else + curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin" + fi + else + cargo install cargo-nextest --version 0.9.133 --locked + fi + cargo nextest --version + cargo install cargo-mutants --version 27.0.0 --locked + cargo mutants --version + + - name: Create mutation diff + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: git diff "${BASE_SHA}..${HEAD_SHA}" -- '*.rs' > cargo-mutants.diff + + - name: Run cargo-mutants with nextest + env: + CARGO_PACKAGE: ${{ matrix.mutation_job.cargo_package }} + IN_DIFF: ${{ matrix.mutation_job.in_diff }} + PYO3_PYTHON: python + TEST_PACKAGES: ${{ toJson(matrix.mutation_job.test_packages) }} + run: | + mapfile -t test_packages < <(python3 -c 'import json, os; [print(package) for package in json.loads(os.environ["TEST_PACKAGES"])]') + cargo_args=("-p" "${CARGO_PACKAGE}") + if [[ "${IN_DIFF}" == "true" ]]; then + cargo_args+=("--in-diff" "cargo-mutants.diff") + fi + for package in "${test_packages[@]}"; do + cargo_args+=("--test-package" "${package}") + done + cargo mutants "${cargo_args[@]}" + coverage: needs: validate-and-detect if: needs.validate-and-detect.outputs.has_plugins == 'true' @@ -169,12 +271,30 @@ jobs: rustup component add llvm-tools-preview cargo install cargo-llvm-cov --version 0.8.4 --locked + - name: Install cargo-nextest + run: | + if [[ "${RUNNER_OS}" == "Linux" ]]; then + mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin" + curl -LsSf https://get.nexte.st/0.9.133/linux | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin" + elif [[ "${RUNNER_OS}" == "macOS" ]]; then + mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin" + if [[ "$(uname -m)" == "arm64" ]]; then + curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin" + else + curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin" + fi + else + cargo install cargo-nextest --version 0.9.133 --locked + fi + cargo nextest --version + - name: Install Python build tooling run: python -m pip install uv==0.9.30 maturin==1.12.6 - name: Generate Rust coverage report env: CARGO_PACKAGES: ${{ needs.validate-and-detect.outputs.cargo_packages }} + NEXTEST_PROFILE: ci PLUGINS: ${{ needs.validate-and-detect.outputs.plugins }} PYO3_PYTHON: python run: | @@ -186,6 +306,8 @@ jobs: cargo_args+=("-p" "${package}") done cargo llvm-cov clean --workspace + mkdir -p coverage + cargo llvm-cov nextest --no-report "${cargo_args[@]}" -P "${NEXTEST_PROFILE}" eval "$(cargo llvm-cov show-env --sh)" export CARGO_TARGET_DIR="${CARGO_LLVM_COV_TARGET_DIR}/llvm-cov-target" export CARGO_LLVM_COV_BUILD_DIR="${CARGO_TARGET_DIR}" @@ -194,7 +316,6 @@ jobs: for plugin in "${plugins[@]}"; do (cd "plugins/rust/python-package/${plugin}" && make sync && uv run maturin develop) done - cargo test "${cargo_args[@]}" for plugin in "${plugins[@]}"; do (cd "plugins/rust/python-package/${plugin}" && make test-integration) done diff --git a/.github/workflows/ci-scaffold-generator.yaml b/.github/workflows/ci-scaffold-generator.yaml index aa74335..f74ecfe 100644 --- a/.github/workflows/ci-scaffold-generator.yaml +++ b/.github/workflows/ci-scaffold-generator.yaml @@ -27,9 +27,9 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 with: python-version: "3.12" @@ -52,6 +52,23 @@ jobs: rustc --version cargo --version + - name: Install cargo-nextest + run: | + if [[ "${RUNNER_OS}" == "Linux" ]]; then + mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin" + curl -LsSf https://get.nexte.st/0.9.133/linux | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin" + elif [[ "${RUNNER_OS}" == "macOS" ]]; then + mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin" + if [[ "$(uname -m)" == "arm64" ]]; then + curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin" + else + curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin" + fi + else + cargo install cargo-nextest --version 0.9.133 --locked + fi + cargo nextest --version + - name: Generate default plugin (tool_pre_invoke) run: | python3 tools/scaffold_plugin.py --non-interactive \ diff --git a/Makefile b/Makefile index 7dfaa3c..8e8dd9c 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ -.PHONY: help plugins-list plugins-validate plugin-test plugin-scaffold plugin-scaffold-help +.PHONY: help plugins-list plugins-validate plugin-test plugin-mutants plugin-mutants-list plugin-scaffold plugin-scaffold-help help: - @printf "plugins-list\nplugins-validate\nplugin-test PLUGIN=\nplugin-scaffold\nplugin-scaffold-help\n" + @printf "plugins-list\nplugins-validate\nplugin-test PLUGIN=\nplugin-mutants PLUGIN=\nplugin-mutants-list PLUGIN=\nplugin-scaffold\nplugin-scaffold-help\n" plugins-list: @python3 tools/plugin_catalog.py list . @@ -14,6 +14,14 @@ plugin-test: @test -n "$(PLUGIN)" || (echo "Set PLUGIN=" && exit 1) @cd plugins/rust/python-package/$(PLUGIN) && make sync && make ci +plugin-mutants: + @test -n "$(PLUGIN)" || (echo "Set PLUGIN=" && exit 1) + cargo mutants -p "$(PLUGIN)" + +plugin-mutants-list: + @test -n "$(PLUGIN)" || (echo "Set PLUGIN=" && exit 1) + cargo mutants --list -p "$(PLUGIN)" + plugin-scaffold: @python3 -m pip install --quiet jinja2 2>/dev/null || pip install --quiet jinja2 2>/dev/null || true @python3 tools/scaffold_plugin.py diff --git a/TESTING.md b/TESTING.md index 404178f..6d99f98 100644 --- a/TESTING.md +++ b/TESTING.md @@ -30,10 +30,13 @@ Each plugin has its own Rust and Python test suite. ```bash cd plugins/rust/python-package/rate_limiter uv sync --dev +cargo install cargo-nextest --version 0.9.133 --locked make install make test-all ``` +Set `NEXTEST_PROFILE=ci` to use the repository CI profile locally. The CI profile is defined in `.config/nextest.toml`; it disables fail-fast so all Rust test failures are reported in one run. + Equivalent repo-level helper: ```bash @@ -51,6 +54,7 @@ To run the same coverage check locally for all managed Rust plugins: ```bash rustup component add llvm-tools-preview cargo install cargo-llvm-cov --version 0.8.4 --locked +cargo install cargo-nextest --version 0.9.133 --locked mkdir -p coverage CARGO_PACKAGES="$(python3 tools/plugin_catalog.py ci-selection-field . all '' '' cargo_packages)" PLUGINS="$(python3 tools/plugin_catalog.py ci-selection-field . all '' '' plugins)" @@ -61,6 +65,7 @@ for package in "${cargo_packages[@]}"; do cargo_args+=("-p" "${package}") done cargo llvm-cov clean --workspace +cargo llvm-cov nextest --no-report "${cargo_args[@]}" -P ci eval "$(cargo llvm-cov show-env --sh)" export CARGO_TARGET_DIR="${CARGO_LLVM_COV_TARGET_DIR}/llvm-cov-target" export CARGO_LLVM_COV_BUILD_DIR="${CARGO_TARGET_DIR}" @@ -69,7 +74,6 @@ mkdir -p "${CARGO_TARGET_DIR}" for plugin in "${plugins[@]}"; do (cd "plugins/rust/python-package/${plugin}" && make sync && uv run maturin develop) done -cargo test "${cargo_args[@]}" for plugin in "${plugins[@]}"; do (cd "plugins/rust/python-package/${plugin}" && make test-integration) done @@ -77,6 +81,23 @@ env -u CARGO_TARGET_DIR -u CARGO_LLVM_COV_BUILD_DIR -u CARGO_LLVM_COV_TARGET_DIR python3 tools/plugin_catalog.py coverage-check . coverage/cobertura.xml 90.00 "${PLUGINS}" ``` +Rust unit tests use `cargo nextest run`. Coverage uses `cargo llvm-cov nextest --no-report` for the Rust test phase, then runs pytest before generating the final report so PyO3 paths stay covered. CI uses the `ci` nextest profile, which disables fail-fast and prints failure output immediately and again at the end. Nextest does not run Rust doctests; this repo currently has no Rust doctest code blocks, so there is no separate doctest step. + +Criterion benchmarks are verified in CI with `cargo nextest run --benches -E 'kind(bench)' --no-run`, which compiles benchmark test targets without rerunning normal unit tests or collecting noisy performance measurements on shared CI runners. + +## 4. Mutation Testing + +Mutation testing runs in PR CI on Ubuntu for Rust code touched by the pull request diff. It is also available locally through cargo-mutants and runs Rust tests with nextest. + +```bash +cargo install cargo-nextest --version 0.9.133 --locked +cargo install cargo-mutants --version 27.0.0 --locked +make plugin-mutants-list PLUGIN=retry_with_backoff +make plugin-mutants PLUGIN=retry_with_backoff +``` + +`.cargo/mutants.toml` sets `test_tool = "nextest"`, selects the `mutants` nextest profile, and keeps `cap_lints = false` so Rust warnings are not downgraded during mutant builds. The `mutants` profile keeps fail-fast enabled because cargo-mutants only needs one failing test to mark a mutant as caught. CI installs `cargo-mutants` with `cargo install cargo-mutants --version 27.0.0 --locked` and runs `cargo mutants "${cargo_args[@]}"`, using `--in-diff cargo-mutants.diff` for Rust source changes and full-package mutation for mutation-tooling config changes. + ## CI Behavior Repo contract tests run in their own CI workflow. The Rust plugin CI workflow uses the same plugin catalog to select affected plugin build, integration, and coverage jobs. diff --git a/plugins/rust/python-package/encoded_exfil_detection/Makefile b/plugins/rust/python-package/encoded_exfil_detection/Makefile index e89ebf6..3abf6b5 100644 --- a/plugins/rust/python-package/encoded_exfil_detection/Makefile +++ b/plugins/rust/python-package/encoded_exfil_detection/Makefile @@ -5,6 +5,8 @@ help: PACKAGE_NAME := cpex-encoded-exfil-detection WHEEL_PREFIX := cpex_encoded_exfil_detection CARGO := cargo +CARGO_PACKAGE := encoded_exfil_detection +NEXTEST_PROFILE ?= default STUB_FILES := cpex_encoded_exfil_detection/__init__.pyi cpex_encoded_exfil_detection/encoded_exfil_detection_rust/__init__.pyi WHEEL_DIR := ../../../../target/wheels @@ -38,7 +40,7 @@ sync: test-unit: @echo "$(GREEN)Running encoded_exfil_detection Rust tests...$(NC)" - $(CARGO) test + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) test: test-unit test-integration @@ -85,7 +87,7 @@ uninstall: @uv pip uninstall -y $(PACKAGE_NAME) 2>/dev/null || true # help: bench - Run Criterion benchmarks -# help: bench-no-run - Compile Criterion benchmarks without running them +# help: bench-no-run - Compile Criterion benchmark targets with nextest .PHONY: bench bench-no-run bench: @@ -93,8 +95,8 @@ bench: $(CARGO) bench bench-no-run: - @echo "$(GREEN)Compiling benchmarks without running them...$(NC)" - $(CARGO) bench --no-run + @echo "$(GREEN)Compiling Criterion benchmark targets with nextest...$(NC)" + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --benches -E 'kind(bench)' --no-run .PHONY: clean clean-all diff --git a/plugins/rust/python-package/pii_filter/Makefile b/plugins/rust/python-package/pii_filter/Makefile index 670f78e..a0b416b 100644 --- a/plugins/rust/python-package/pii_filter/Makefile +++ b/plugins/rust/python-package/pii_filter/Makefile @@ -5,6 +5,8 @@ help: PACKAGE_NAME := cpex-pii-filter WHEEL_PREFIX := cpex_pii_filter CARGO := cargo +CARGO_PACKAGE := pii_filter +NEXTEST_PROFILE ?= default STUB_FILES := cpex_pii_filter/__init__.pyi cpex_pii_filter/pii_filter_rust/__init__.pyi WHEEL_DIR := ../../../../target/wheels @@ -39,13 +41,13 @@ sync: test-unit: @echo "$(GREEN)Running pii_filter Rust tests...$(NC)" - $(CARGO) test + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) test: test-unit test-integration test-verbose: @echo "$(GREEN)Running pii_filter Rust tests (verbose)...$(NC)" - $(CARGO) test -- --nocapture + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --no-capture test-python: $(MAKE) test-integration @@ -90,7 +92,7 @@ uninstall: @uv pip uninstall -y $(PACKAGE_NAME) 2>/dev/null || true # help: bench - Run Criterion benchmarks -# help: bench-no-run - Compile Criterion benchmarks without running them +# help: bench-no-run - Compile Criterion benchmark targets with nextest # help: bench-compare - Compare against saved baseline .PHONY: bench bench-no-run bench-baseline bench-compare @@ -99,8 +101,8 @@ bench: $(CARGO) bench bench-no-run: - @echo "$(GREEN)Compiling benchmarks without running them...$(NC)" - $(CARGO) bench --no-run + @echo "$(GREEN)Compiling Criterion benchmark targets with nextest...$(NC)" + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --benches -E 'kind(bench)' --no-run bench-baseline: $(CARGO) bench --bench pii_filter -- --save-baseline main diff --git a/plugins/rust/python-package/rate_limiter/Makefile b/plugins/rust/python-package/rate_limiter/Makefile index 8cb48b5..2e2e963 100644 --- a/plugins/rust/python-package/rate_limiter/Makefile +++ b/plugins/rust/python-package/rate_limiter/Makefile @@ -13,6 +13,8 @@ help: PACKAGE_NAME := cpex-rate-limiter WHEEL_PREFIX := cpex_rate_limiter CARGO := cargo +CARGO_PACKAGE := rate_limiter +NEXTEST_PROFILE ?= default STUB_FILES := cpex_rate_limiter/__init__.pyi cpex_rate_limiter/rate_limiter_rust/__init__.pyi WHEEL_DIR := ../../../../target/wheels @@ -53,13 +55,13 @@ sync: test-unit: @echo "$(GREEN)Running rate_limiter Rust tests...$(NC)" - $(CARGO) test + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) test: test-unit test-integration test-verbose: @echo "$(GREEN)Running rate_limiter Rust tests (verbose)...$(NC)" - $(CARGO) test -- --nocapture + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --no-capture test-python: $(MAKE) test-integration @@ -110,7 +112,7 @@ uninstall: # 📊 BENCHMARKS # ============================================================================= # help: bench - Run Criterion benchmarks -# help: bench-no-run - Compile Criterion benchmarks without running them +# help: bench-no-run - Compile Criterion benchmark targets with nextest # help: bench-compare - Compare against saved baseline .PHONY: bench bench-no-run bench-baseline bench-compare @@ -119,8 +121,8 @@ bench: $(CARGO) bench bench-no-run: - @echo "$(GREEN)Compiling benchmarks without running them...$(NC)" - $(CARGO) bench --no-run + @echo "$(GREEN)Compiling Criterion benchmark targets with nextest...$(NC)" + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --benches -E 'kind(bench)' --no-run bench-baseline: $(CARGO) bench --bench rate_limiter -- --save-baseline main diff --git a/plugins/rust/python-package/rate_limiter/src/plugin.rs b/plugins/rust/python-package/rate_limiter/src/plugin.rs index 87573cb..c585bcd 100644 --- a/plugins/rust/python-package/rate_limiter/src/plugin.rs +++ b/plugins/rust/python-package/rate_limiter/src/plugin.rs @@ -523,6 +523,10 @@ mod tests { #[test] fn await_async_tuple_parses_successful_result() -> PyResult<()> { + // Ensure the embedded interpreter is initialized for this test process. + // `cargo-nextest` runs tests in separate processes, so this cannot rely + // on another test having already touched Python. + Python::initialize(); Python::attach(|py| -> PyResult<()> { let sys = py.import("sys")?; let asyncio = py.import("asyncio")?; diff --git a/plugins/rust/python-package/retry_with_backoff/Makefile b/plugins/rust/python-package/retry_with_backoff/Makefile index 520f2de..0157169 100644 --- a/plugins/rust/python-package/retry_with_backoff/Makefile +++ b/plugins/rust/python-package/retry_with_backoff/Makefile @@ -6,6 +6,7 @@ PACKAGE_NAME := cpex-retry-with-backoff WHEEL_PREFIX := cpex_retry_with_backoff CARGO := cargo CARGO_PACKAGE := retry_with_backoff +NEXTEST_PROFILE ?= default PLUGIN_SLUG := retry_with_backoff REPO_ROOT := ../../../.. STUB_FILES := cpex_retry_with_backoff/__init__.pyi cpex_retry_with_backoff/retry_with_backoff_rust/__init__.pyi @@ -45,13 +46,13 @@ sync: test-unit: @echo "$(GREEN)Running retry_with_backoff Rust tests...$(NC)" - $(CARGO) test + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) test: test-unit test-integration test-verbose: @echo "$(GREEN)Running retry_with_backoff Rust tests (verbose)...$(NC)" - $(CARGO) test -- --nocapture + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --no-capture test-python: $(MAKE) test-integration @@ -121,17 +122,17 @@ ci-build: check-all verify-stubs build install-wheel coverage: @echo "$(GREEN)Generating Rust coverage...$(NC)" - mkdir -p $(REPO_ROOT)/coverage rustup component add llvm-tools-preview cargo llvm-cov --version >/dev/null 2>&1 || cargo install cargo-llvm-cov --version $(CARGO_LLVM_COV_VERSION) --locked cargo llvm-cov clean --workspace + mkdir -p $(REPO_ROOT)/coverage + $(CARGO) llvm-cov nextest --no-report -p $(CARGO_PACKAGE) -P $(NEXTEST_PROFILE) eval "$$(cargo llvm-cov show-env --sh)" && \ export CARGO_TARGET_DIR="$${CARGO_LLVM_COV_TARGET_DIR}/llvm-cov-target" && \ export CARGO_LLVM_COV_BUILD_DIR="$${CARGO_TARGET_DIR}" && \ export LLVM_PROFILE_FILE="$${CARGO_TARGET_DIR}/cpex-plugins-%p-%10m.profraw" && \ mkdir -p "$${CARGO_TARGET_DIR}" && \ uv run maturin develop && \ - $(CARGO) test -p $(CARGO_PACKAGE) && \ $(MAKE) test-integration && \ env -u CARGO_TARGET_DIR -u CARGO_LLVM_COV_BUILD_DIR -u CARGO_LLVM_COV_TARGET_DIR -u LLVM_PROFILE_FILE cargo llvm-cov report -p $(CARGO_PACKAGE) --cobertura --output-path $(REPO_ROOT)/$(COVERAGE_REPORT) python3 $(REPO_ROOT)/tools/plugin_catalog.py coverage-check $(REPO_ROOT) $(COVERAGE_REPORT) $(COVERAGE_MIN) '["$(PLUGIN_SLUG)"]' diff --git a/plugins/rust/python-package/secrets_detection/Makefile b/plugins/rust/python-package/secrets_detection/Makefile index f9bf602..90947a4 100644 --- a/plugins/rust/python-package/secrets_detection/Makefile +++ b/plugins/rust/python-package/secrets_detection/Makefile @@ -5,6 +5,8 @@ help: PACKAGE_NAME := cpex-secrets-detection WHEEL_PREFIX := cpex_secrets_detection CARGO := cargo +CARGO_PACKAGE := secrets_detection +NEXTEST_PROFILE ?= default STUB_FILES := cpex_secrets_detection/__init__.pyi cpex_secrets_detection/secrets_detection_rust/__init__.pyi WHEEL_DIR := ../../../../target/wheels @@ -39,13 +41,13 @@ sync: test-unit: @echo "$(GREEN)Running secrets_detection Rust tests...$(NC)" - $(CARGO) test + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) test: test-unit test-integration test-verbose: @echo "$(GREEN)Running secrets_detection Rust tests (verbose)...$(NC)" - $(CARGO) test -- --nocapture + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --no-capture test-integration: @echo "$(GREEN)Running integration tests...$(NC)" @@ -87,7 +89,7 @@ uninstall: @uv pip uninstall -y $(PACKAGE_NAME) 2>/dev/null || true # help: bench - Run Criterion benchmarks -# help: bench-no-run - Compile Criterion benchmarks without running them +# help: bench-no-run - Compile Criterion benchmark targets with nextest .PHONY: bench bench-no-run bench: @@ -95,8 +97,8 @@ bench: $(CARGO) bench bench-no-run: - @echo "$(GREEN)Compiling benchmarks without running them...$(NC)" - $(CARGO) bench --no-run + @echo "$(GREEN)Compiling Criterion benchmark targets with nextest...$(NC)" + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --benches -E 'kind(bench)' --no-run .PHONY: clean clean-all diff --git a/plugins/rust/python-package/url_reputation/Makefile b/plugins/rust/python-package/url_reputation/Makefile index 4bbc19e..6f884b5 100644 --- a/plugins/rust/python-package/url_reputation/Makefile +++ b/plugins/rust/python-package/url_reputation/Makefile @@ -5,6 +5,8 @@ help: PACKAGE_NAME := cpex-url-reputation WHEEL_PREFIX := cpex_url_reputation CARGO := cargo +CARGO_PACKAGE := url_reputation +NEXTEST_PROFILE ?= default STUB_FILES := cpex_url_reputation/__init__.pyi cpex_url_reputation/url_reputation_rust/__init__.pyi WHEEL_DIR := ../../../../target/wheels @@ -39,13 +41,13 @@ sync: test-unit: @echo "$(GREEN)Running url_reputation Rust tests...$(NC)" - $(CARGO) test + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) test: test-unit test-integration test-verbose: @echo "$(GREEN)Running url_reputation Rust tests (verbose)...$(NC)" - $(CARGO) test -- --nocapture + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --no-capture test-python: $(MAKE) test-integration @@ -89,12 +91,12 @@ uninstall: @echo "$(YELLOW)Uninstalling $(PACKAGE_NAME)...$(NC)" @uv pip uninstall -y $(PACKAGE_NAME) 2>/dev/null || true -# help: bench-no-run - Compile benchmarks without running them +# help: bench-no-run - Compile benchmark targets with nextest .PHONY: bench-no-run bench-no-run: - @echo "$(GREEN)Compiling benchmarks without running them...$(NC)" - $(CARGO) bench --no-run + @echo "$(GREEN)Compiling Criterion benchmark targets with nextest...$(NC)" + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --benches -E 'kind(bench)' --no-run .PHONY: clean clean-all diff --git a/plugins/rust/python-package/url_reputation/README.md b/plugins/rust/python-package/url_reputation/README.md index 1840890..23eabab 100644 --- a/plugins/rust/python-package/url_reputation/README.md +++ b/plugins/rust/python-package/url_reputation/README.md @@ -163,7 +163,7 @@ flowchart LR **Run tests:** ```bash -cargo test --lib # Run all unit tests +cargo nextest run -p url_reputation # Run Rust unit tests cargo llvm-cov --lib --html # Generate coverage report ``` diff --git a/tests/test_plugin_catalog.py b/tests/test_plugin_catalog.py index 7d995de..8070af5 100644 --- a/tests/test_plugin_catalog.py +++ b/tests/test_plugin_catalog.py @@ -1,6 +1,7 @@ import json import os import ast +import shutil import subprocess import tempfile import textwrap @@ -1533,6 +1534,9 @@ def test_ci_selection_returns_has_plugins_contract(self) -> None: "has_plugins": True, "plugin_count": 2, "cargo_packages": ["pii_filter", "rate_limiter"], + "mutation_cargo_packages": [], + "has_mutation_cargo_packages": False, + "mutation_jobs": [], }, ) @@ -1574,6 +1578,9 @@ def test_ci_selection_treats_catalog_test_change_as_not_shared(self) -> None: "has_plugins": False, "plugin_count": 0, "cargo_packages": [], + "mutation_cargo_packages": [], + "has_mutation_cargo_packages": False, + "mutation_jobs": [], }, ) @@ -1615,9 +1622,93 @@ def test_ci_selection_treats_shared_tool_changes_as_all_plugins(self) -> None: "has_plugins": True, "plugin_count": 2, "cargo_packages": ["pii_filter", "rate_limiter"], + "mutation_cargo_packages": [], + "has_mutation_cargo_packages": False, + "mutation_jobs": [], }, ) + def test_ci_selection_treats_tooling_config_changes_as_all_plugins(self) -> None: + for config_path in (".cargo/mutants.toml", ".config/nextest.toml"): + with self.subTest(config_path=config_path): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + git = lambda *args: subprocess.run( # noqa: E731 + ["git", *args], + cwd=root, + text=True, + capture_output=True, + check=True, + ) + git("init") + git("config", "user.name", "Test User") + git("config", "user.email", "test@example.com") + (root / "Cargo.toml").write_text( + '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' + ) + self._create_plugin(root, "rate_limiter") + self._create_plugin(root, "pii_filter") + git("add", ".") + git("commit", "--no-verify", "-m", "seed layout") + base_sha = git("rev-parse", "HEAD").stdout.strip() + + path = root / config_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("# shared tooling config change\n") + git("add", ".") + git("commit", "--no-verify", "-m", "shared tooling config input") + + result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") + self.assertEqual(result.returncode, 0, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual( + payload, + { + "plugins": ["pii_filter", "rate_limiter"], + "has_plugins": True, + "plugin_count": 2, + "cargo_packages": ["pii_filter", "rate_limiter"], + "mutation_cargo_packages": [], + "has_mutation_cargo_packages": False, + "mutation_jobs": [], + }, + ) + + def test_ci_selection_skips_mutation_for_unrelated_tooling_config(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + git = lambda *args: subprocess.run( # noqa: E731 + ["git", *args], + cwd=root, + text=True, + capture_output=True, + check=True, + ) + git("init") + git("config", "user.name", "Test User") + git("config", "user.email", "test@example.com") + (root / "Cargo.toml").write_text( + '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' + ) + self._create_plugin(root, "rate_limiter") + self._create_plugin(root, "pii_filter") + git("add", ".") + git("commit", "--no-verify", "-m", "seed layout") + base_sha = git("rev-parse", "HEAD").stdout.strip() + + path = root / ".config" / "editor.toml" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("# unrelated tooling config change\n") + git("add", ".") + git("commit", "--no-verify", "-m", "unrelated tooling config input") + + result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") + self.assertEqual(result.returncode, 0, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual(payload["plugins"], ["pii_filter", "rate_limiter"]) + self.assertEqual(payload["mutation_cargo_packages"], []) + self.assertEqual(payload["mutation_jobs"], []) + def test_ci_selection_treats_cargo_lock_change_as_all_plugins(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -1655,6 +1746,9 @@ def test_ci_selection_treats_cargo_lock_change_as_all_plugins(self) -> None: "has_plugins": True, "plugin_count": 2, "cargo_packages": ["pii_filter", "rate_limiter"], + "mutation_cargo_packages": [], + "has_mutation_cargo_packages": False, + "mutation_jobs": [], }, ) @@ -1695,6 +1789,9 @@ def test_ci_selection_treats_deny_config_change_as_all_plugins(self) -> None: "has_plugins": True, "plugin_count": 2, "cargo_packages": ["pii_filter", "rate_limiter"], + "mutation_cargo_packages": [], + "has_mutation_cargo_packages": False, + "mutation_jobs": [], }, ) @@ -1737,6 +1834,9 @@ def test_changed_returns_plugin_for_plugin_integration_test_change(self) -> None "has_plugins": True, "plugin_count": 1, "cargo_packages": ["pii_filter"], + "mutation_cargo_packages": [], + "has_mutation_cargo_packages": False, + "mutation_jobs": [], }, ) @@ -1779,6 +1879,9 @@ def test_ci_selection_treats_shared_plugin_tests_change_as_all_plugins(self) -> "has_plugins": True, "plugin_count": 2, "cargo_packages": ["pii_filter", "rate_limiter"], + "mutation_cargo_packages": [], + "has_mutation_cargo_packages": False, + "mutation_jobs": [], }, ) @@ -1800,6 +1903,10 @@ def test_ci_selection_treats_shared_crate_changes_as_all_plugins(self) -> None: ) self._create_plugin(root, "rate_limiter") self._create_plugin(root, "pii_filter") + (root / "plugins" / "rust" / "python-package" / "rate_limiter" / "Cargo.toml").write_text( + '[package]\nname = "rate_limiter"\nversion = "0.0.1"\nrepository = "https://github.com/IBM/cpex-plugins"\n\n' + "[dependencies]\ncpex_framework_bridge = { workspace = true }\n" + ) shared_crate = root / "crates" / "framework_bridge" / "src" shared_crate.mkdir(parents=True) (shared_crate / "lib.rs").write_text("// shared crate change\n") @@ -1821,9 +1928,214 @@ def test_ci_selection_treats_shared_crate_changes_as_all_plugins(self) -> None: "has_plugins": True, "plugin_count": 2, "cargo_packages": ["pii_filter", "rate_limiter"], + "mutation_cargo_packages": ["cpex_framework_bridge"], + "has_mutation_cargo_packages": True, + "mutation_jobs": [ + { + "cargo_package": "cpex_framework_bridge", + "in_diff": True, + "test_packages": ["rate_limiter"], + } + ], }, ) + def test_ci_selection_ignores_tooling_config_for_shared_crate_mutation_jobs(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + git = lambda *args: subprocess.run( # noqa: E731 + ["git", *args], + cwd=root, + text=True, + capture_output=True, + check=True, + ) + git("init") + git("config", "user.name", "Test User") + git("config", "user.email", "test@example.com") + (root / "Cargo.toml").write_text( + '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' + ) + self._create_plugin(root, "rate_limiter") + self._create_plugin(root, "pii_filter") + (root / "plugins" / "rust" / "python-package" / "rate_limiter" / "Cargo.toml").write_text( + '[package]\nname = "rate_limiter"\nversion = "0.0.1"\nrepository = "https://github.com/IBM/cpex-plugins"\n\n' + "[dependencies]\ncpex_framework_bridge = { workspace = true }\n" + ) + shared_crate = root / "crates" / "framework_bridge" / "src" + shared_crate.mkdir(parents=True) + (shared_crate / "lib.rs").write_text("// shared crate change\n") + config_path = root / ".cargo" / "mutants.toml" + config_path.parent.mkdir(parents=True) + config_path.write_text("# seed\n") + git("add", ".") + git("commit", "--no-verify", "-m", "seed layout") + base_sha = git("rev-parse", "HEAD").stdout.strip() + + (shared_crate / "lib.rs").write_text("// shared crate change\n// update\n") + config_path.write_text("# updated\n") + git("add", ".") + git("commit", "--no-verify", "-m", "mixed mutation inputs") + + result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") + self.assertEqual(result.returncode, 0, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual( + payload["mutation_jobs"], + [ + { + "cargo_package": "cpex_framework_bridge", + "in_diff": True, + "test_packages": ["rate_limiter"], + }, + ], + ) + + def test_ci_selection_ignores_tooling_config_for_plugin_rust_mutation_jobs(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + git = lambda *args: subprocess.run( # noqa: E731 + ["git", *args], + cwd=root, + text=True, + capture_output=True, + check=True, + ) + git("init") + git("config", "user.name", "Test User") + git("config", "user.email", "test@example.com") + (root / "Cargo.toml").write_text( + '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' + ) + rate_limiter = self._create_plugin(root, "rate_limiter") + self._create_plugin(root, "pii_filter") + src_dir = rate_limiter / "src" + src_dir.mkdir() + (src_dir / "lib.rs").write_text("pub fn rate_limit() {}\n") + config_path = root / ".cargo" / "mutants.toml" + config_path.parent.mkdir(parents=True) + config_path.write_text("# seed\n") + git("add", ".") + git("commit", "--no-verify", "-m", "seed layout") + base_sha = git("rev-parse", "HEAD").stdout.strip() + + (src_dir / "lib.rs").write_text("pub fn rate_limit() -> bool { true }\n") + config_path.write_text("# updated\n") + git("add", ".") + git("commit", "--no-verify", "-m", "mixed plugin and config inputs") + + result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") + self.assertEqual(result.returncode, 0, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual( + payload["mutation_jobs"], + [ + {"cargo_package": "rate_limiter", "in_diff": True, "test_packages": []}, + ], + ) + + def test_framework_bridge_mutation_dependents_match_real_manifests(self) -> None: + plugin_root = REPO_ROOT / "plugins" / "rust" / "python-package" + dependents = [] + for manifest_path in sorted(plugin_root.glob("*/Cargo.toml")): + with manifest_path.open("rb") as handle: + manifest = tomllib.load(handle) + if "cpex_framework_bridge" in manifest.get("dependencies", {}): + dependents.append(manifest_path.parent.name) + + self.assertEqual( + dependents, + [ + "encoded_exfil_detection", + "pii_filter", + "rate_limiter", + "secrets_detection", + ], + ) + + def test_framework_bridge_mutation_job_uses_real_dependents(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + subprocess.run( + ["git", "init"], + cwd=root, + text=True, + capture_output=True, + check=True, + ) + subprocess.run( + ["git", "config", "user.name", "Test User"], + cwd=root, + text=True, + capture_output=True, + check=True, + ) + subprocess.run( + ["git", "config", "user.email", "test@example.com"], + cwd=root, + text=True, + capture_output=True, + check=True, + ) + shutil.copytree(REPO_ROOT / "plugins", root / "plugins") + shutil.copytree(REPO_ROOT / "crates", root / "crates") + (root / "Cargo.toml").write_text((REPO_ROOT / "Cargo.toml").read_text()) + subprocess.run( + ["git", "add", "."], + cwd=root, + text=True, + capture_output=True, + check=True, + ) + subprocess.run( + ["git", "commit", "--no-verify", "-m", "seed real manifests"], + cwd=root, + text=True, + capture_output=True, + check=True, + ) + base_sha = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=root, + text=True, + capture_output=True, + check=True, + ).stdout.strip() + + bridge_lib = root / "crates" / "framework_bridge" / "src" / "lib.rs" + bridge_lib.write_text(bridge_lib.read_text() + "\n// mutation route test\n") + subprocess.run( + ["git", "add", "."], + cwd=root, + text=True, + capture_output=True, + check=True, + ) + subprocess.run( + ["git", "commit", "--no-verify", "-m", "shared crate change"], + cwd=root, + text=True, + capture_output=True, + check=True, + ) + + result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") + self.assertEqual(result.returncode, 0, result.stderr) + payload = json.loads(result.stdout) + self.assertIn( + { + "cargo_package": "cpex_framework_bridge", + "in_diff": True, + "test_packages": [ + "encoded_exfil_detection", + "pii_filter", + "rate_limiter", + "secrets_detection", + ], + }, + payload["mutation_jobs"], + ) + def test_ci_selection_reports_cargo_packages_for_single_plugin_diff(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -1862,23 +2174,72 @@ def test_ci_selection_reports_cargo_packages_for_single_plugin_diff(self) -> Non "has_plugins": True, "plugin_count": 1, "cargo_packages": ["rate_limiter"], + "mutation_cargo_packages": [], + "has_mutation_cargo_packages": False, + "mutation_jobs": [], + }, + ) + + def test_ci_selection_reports_mutation_package_for_single_rust_diff(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + git = lambda *args: subprocess.run( # noqa: E731 + ["git", *args], + cwd=root, + text=True, + capture_output=True, + check=True, + ) + git("init") + git("config", "user.name", "Test User") + git("config", "user.email", "test@example.com") + (root / "Cargo.toml").write_text( + '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' + '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' + ) + plugin_dir = self._create_plugin(root, "rate_limiter") + self._create_plugin(root, "pii_filter") + src_dir = plugin_dir / "src" + src_dir.mkdir() + (src_dir / "lib.rs").write_text("pub fn rate_limit() {}\n") + git("add", ".") + git("commit", "--no-verify", "-m", "seed layout") + base_sha = git("rev-parse", "HEAD").stdout.strip() + + (src_dir / "lib.rs").write_text("pub fn rate_limit() -> bool { true }\n") + git("add", ".") + git("commit", "--no-verify", "-m", "single rust change") + + result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") + self.assertEqual(result.returncode, 0, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual( + payload, + { + "plugins": ["rate_limiter"], + "has_plugins": True, + "plugin_count": 1, + "cargo_packages": ["rate_limiter"], + "mutation_cargo_packages": ["rate_limiter"], + "has_mutation_cargo_packages": True, + "mutation_jobs": [ + {"cargo_package": "rate_limiter", "in_diff": True, "test_packages": []} + ], }, ) def test_ci_selection_field_prints_json_and_bool_scalars(self) -> None: + expected_plugins = [ + "encoded_exfil_detection", + "pii_filter", + "rate_limiter", + "retry_with_backoff", + "secrets_detection", + "url_reputation", + ] result = run_catalog("ci-selection-field", str(REPO_ROOT), "all", "", "", "plugins") self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual( - json.loads(result.stdout), - [ - "encoded_exfil_detection", - "pii_filter", - "rate_limiter", - "retry_with_backoff", - "secrets_detection", - "url_reputation", - ], - ) + self.assertEqual(json.loads(result.stdout), expected_plugins) result = run_catalog("ci-selection-field", str(REPO_ROOT), "all", "", "", "has_plugins") self.assertEqual(result.returncode, 0, result.stderr) @@ -1890,18 +2251,26 @@ def test_ci_selection_field_prints_json_and_bool_scalars(self) -> None: result = run_catalog("ci-selection-field", str(REPO_ROOT), "all", "", "", "cargo_packages") self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout), expected_plugins) + + result = run_catalog("ci-selection-field", str(REPO_ROOT), "all", "", "", "mutation_cargo_packages") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout), expected_plugins) + + result = run_catalog("ci-selection-field", str(REPO_ROOT), "all", "", "", "mutation_jobs") + self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual( json.loads(result.stdout), [ - "encoded_exfil_detection", - "pii_filter", - "rate_limiter", - "retry_with_backoff", - "secrets_detection", - "url_reputation", + {"cargo_package": plugin, "in_diff": False, "test_packages": []} + for plugin in expected_plugins ], ) + result = run_catalog("ci-selection-field", str(REPO_ROOT), "all", "", "", "has_mutation_cargo_packages") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "true") + def test_ci_selection_field_supports_diff_mode(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -1960,6 +2329,8 @@ def test_ci_workflow_shared_paths_match_catalog_contract(self) -> None: "Makefile", "Cargo.toml", "Cargo.lock", + ".cargo/**", + ".config/nextest.toml", "deny.toml", "crates/**", "README.md", @@ -2198,6 +2569,7 @@ def test_ci_workflow_includes_parity_jobs_for_rust_plugin_checks(self) -> None: REPO_ROOT / ".github" / "workflows" / "ci-rust-python-package.yaml" ).read_text() security_section = self._extract_workflow_job_section(workflow, "security-policy") + mutants_section = self._extract_workflow_job_section(workflow, "mutation-testing") coverage_section = self._extract_workflow_job_section(workflow, "coverage") documentation_section = self._extract_workflow_job_section( workflow, "documentation" @@ -2208,6 +2580,24 @@ def test_ci_workflow_includes_parity_jobs_for_rust_plugin_checks(self) -> None: deny_run = self._extract_workflow_step_run( workflow, "security-policy", step_name="Run cargo deny" ) + mutants_install_run = self._extract_workflow_step_run( + workflow, "mutation-testing", step_name="Install Rust mutation testing tooling" + ) + mutants_diff_run = self._extract_workflow_step_run( + workflow, "mutation-testing", step_name="Create mutation diff" + ) + mutants_run = self._extract_workflow_step_run( + workflow, "mutation-testing", step_name="Run cargo-mutants with nextest" + ) + build_test_nextest_install = self._extract_workflow_step_section( + workflow, "build-test", step_name="Install cargo-nextest" + ) + build_test_ci_build = self._extract_workflow_step_section( + workflow, "build-test", step_name="Plugin CI build verification" + ) + build_test_ci = self._extract_workflow_step_section( + workflow, "build-test", step_name="Plugin CI verification" + ) coverage_run = self._extract_workflow_step_run( workflow, "coverage", step_name="Generate Rust coverage report" ) @@ -2221,14 +2611,23 @@ def test_ci_workflow_includes_parity_jobs_for_rust_plugin_checks(self) -> None: self.assertIn("workflow_dispatch:", workflow) self.assertIn('elif [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then', detect_run) self.assertIn("ci-selection . all '' ''", detect_run) + self.assertIn( + "ci-selection . diff \"${{ github.event.pull_request.base.sha }}\" \"${{ github.event.pull_request.head.sha }}\"", + detect_run, + ) self.assertIn("plugin_count: ${{ steps.detect.outputs.plugin_count }}", workflow) self.assertNotIn("single_cargo_package", workflow) self.assertIn("cargo_packages: ${{ steps.detect.outputs.cargo_packages }}", workflow) + self.assertIn("mutation_cargo_packages: ${{ steps.detect.outputs.mutation_cargo_packages }}", workflow) + self.assertIn("mutation_jobs: ${{ steps.detect.outputs.mutation_jobs }}", workflow) + self.assertIn("has_mutation_cargo_packages: ${{ steps.detect.outputs.has_mutation_cargo_packages }}", workflow) self.assertIn("security-policy:", workflow) + self.assertIn("mutation-testing:", workflow) self.assertIn("coverage:", workflow) self.assertIn("documentation:", workflow) self.assertNotIn("benchmark-build-verification:", workflow) self.assertIn("if: needs.validate-and-detect.outputs.has_plugins == 'true'", security_section) + self.assertIn("if: github.event_name == 'pull_request' && needs.validate-and-detect.outputs.has_mutation_cargo_packages == 'true'", mutants_section) self.assertIn("if: needs.validate-and-detect.outputs.has_plugins == 'true'", coverage_section) self.assertIn("if: needs.validate-and-detect.outputs.has_plugins == 'true'", documentation_section) self.assertNotIn("cargo-audit", security_section) @@ -2239,7 +2638,32 @@ def test_ci_workflow_includes_parity_jobs_for_rust_plugin_checks(self) -> None: self.assertNotIn("--workspace", security_section) self.assertNotIn("--manifest-path", security_section) self.assertNotIn("matrix:", security_section) + self.assertIn("NEXTEST_PROFILE: ci", build_test_ci_build) + self.assertIn("NEXTEST_PROFILE: ci", build_test_ci) self.assertIn("cargo install cargo-llvm-cov --version 0.8.4 --locked", coverage_section) + self.assertIn("cargo install cargo-nextest --version 0.9.133 --locked", build_test_nextest_install) + self.assertIn("cargo nextest --version", build_test_nextest_install) + self.assertIn("https://get.nexte.st/0.9.133/mac", build_test_nextest_install) + self.assertNotIn("mac-arm", workflow) + self.assertIn("cargo install cargo-nextest --version 0.9.133 --locked", coverage_section) + self.assertIn("cargo nextest --version", coverage_section) + self.assertIn("cargo install cargo-nextest --version 0.9.133 --locked", mutants_install_run) + self.assertIn("cargo nextest --version", mutants_install_run) + self.assertIn("cargo install cargo-mutants --version 27.0.0 --locked", mutants_install_run) + self.assertIn("cargo mutants --version", mutants_install_run) + self.assertIn("fetch-depth: 0", mutants_section) + self.assertEqual('git diff "${BASE_SHA}..${HEAD_SHA}" -- \'*.rs\' > cargo-mutants.diff\n', mutants_diff_run) + self.assertIn("BASE_SHA: ${{ github.event.pull_request.base.sha }}", mutants_section) + self.assertIn("HEAD_SHA: ${{ github.event.pull_request.head.sha }}", mutants_section) + self.assertIn("mutation_job: ${{ fromJson(needs.validate-and-detect.outputs.mutation_jobs) }}", mutants_section) + self.assertIn("CARGO_PACKAGE: ${{ matrix.mutation_job.cargo_package }}", mutants_section) + self.assertIn("IN_DIFF: ${{ matrix.mutation_job.in_diff }}", mutants_section) + self.assertIn("TEST_PACKAGES: ${{ toJson(matrix.mutation_job.test_packages) }}", mutants_section) + self.assertIn('cargo_args=("-p" "${CARGO_PACKAGE}")', mutants_run) + self.assertIn('cargo_args+=("--in-diff" "cargo-mutants.diff")', mutants_run) + self.assertIn('cargo_args+=("--test-package" "${package}")', mutants_run) + self.assertIn('cargo mutants "${cargo_args[@]}"', mutants_run) + self.assertIn("PYO3_PYTHON: python", mutants_section) self.assertIn("python -m pip install uv==0.9.30 maturin==1.12.6", coverage_section) self.assertIn("CARGO_PACKAGES: ${{ needs.validate-and-detect.outputs.cargo_packages }}", coverage_section) self.assertIn("PLUGINS: ${{ needs.validate-and-detect.outputs.plugins }}", coverage_section) @@ -2264,7 +2688,10 @@ def test_ci_workflow_includes_parity_jobs_for_rust_plugin_checks(self) -> None: 'mkdir -p "${CARGO_TARGET_DIR}"', coverage_run, ) - self.assertIn('cargo test "${cargo_args[@]}"', coverage_run) + self.assertIn( + 'cargo llvm-cov nextest --no-report "${cargo_args[@]}" -P "${NEXTEST_PROFILE}"', + coverage_run, + ) self.assertIn('make sync && uv run maturin develop', coverage_run) self.assertIn('make test-integration', coverage_run) self.assertIn( @@ -2325,20 +2752,59 @@ def test_ci_workflow_dispatch_detect_step_selects_all_plugins(self) -> None: ] self.assertEqual(json.loads(outputs["plugins"]), expected_plugins) self.assertEqual(json.loads(outputs["cargo_packages"]), expected_plugins) + self.assertEqual(json.loads(outputs["mutation_cargo_packages"]), expected_plugins) + self.assertEqual( + json.loads(outputs["mutation_jobs"]), + [ + {"cargo_package": plugin, "in_diff": False, "test_packages": []} + for plugin in expected_plugins + ], + ) + self.assertEqual(outputs["has_mutation_cargo_packages"], "true") + + def test_scaffold_workflow_installs_nextest_for_generated_plugin_ci(self) -> None: + workflow = ( + REPO_ROOT / ".github" / "workflows" / "ci-scaffold-generator.yaml" + ).read_text() + nextest_install = self._extract_workflow_step_section( + workflow, "test", step_name="Install cargo-nextest" + ) + self.assertIn("cargo install cargo-nextest --version 0.9.133 --locked", nextest_install) + self.assertIn("https://get.nexte.st/0.9.133/mac", nextest_install) + self.assertNotIn("mac-arm", workflow) + self.assertIn("cargo nextest --version", nextest_install) def test_testing_docs_include_local_rust_coverage_command(self) -> None: testing_doc = (REPO_ROOT / "TESTING.md").read_text() + mutants_config = (REPO_ROOT / ".cargo" / "mutants.toml").read_text() + nextest_config = (REPO_ROOT / ".config" / "nextest.toml").read_text() self.assertIn("cargo install cargo-llvm-cov --version 0.8.4 --locked", testing_doc) + self.assertIn("cargo install cargo-nextest --version 0.9.133 --locked", testing_doc) self.assertIn("cargo llvm-cov clean --workspace", testing_doc) self.assertIn('eval "$(cargo llvm-cov show-env --sh)"', testing_doc) self.assertIn('export CARGO_TARGET_DIR="${CARGO_LLVM_COV_TARGET_DIR}/llvm-cov-target"', testing_doc) self.assertIn("make sync && uv run maturin develop", testing_doc) - self.assertIn("cargo test", testing_doc) + self.assertIn("cargo llvm-cov nextest --no-report", testing_doc) self.assertIn("make test-integration", testing_doc) self.assertIn("env -u CARGO_TARGET_DIR", testing_doc) self.assertIn("coverage/cobertura.xml", testing_doc) self.assertIn("python3 tools/plugin_catalog.py coverage-check . coverage/cobertura.xml 90.00", testing_doc) + self.assertIn("Nextest does not run Rust doctests", testing_doc) + self.assertIn("cargo nextest run --benches -E 'kind(bench)' --no-run", testing_doc) + self.assertIn("cargo install cargo-nextest --version 0.9.133 --locked", testing_doc) + self.assertIn("cargo install cargo-mutants --version 27.0.0 --locked", testing_doc) + self.assertIn("make plugin-mutants PLUGIN=retry_with_backoff", testing_doc) + self.assertIn('cargo mutants "${cargo_args[@]}"', testing_doc) + self.assertIn('test_tool = "nextest"', mutants_config) + self.assertIn('additional_cargo_test_args = ["--profile=mutants"]', mutants_config) + self.assertIn("cap_lints = false", mutants_config) + self.assertIn('nextest-version = "0.9.133"', nextest_config) + self.assertIn("[profile.ci]", nextest_config) + self.assertIn("fail-fast = false", nextest_config) + self.assertIn('failure-output = "immediate-final"', nextest_config) + self.assertIn("[profile.mutants]", nextest_config) + self.assertIn("fail-fast = true", nextest_config) def test_python_integration_tests_live_under_repo_plugins_tests(self) -> None: plugin_root = REPO_ROOT / "plugins" / "rust" / "python-package" @@ -2363,6 +2829,8 @@ def test_python_integration_tests_live_under_repo_plugins_tests(self) -> None: self.assertIn("test: test-unit test-integration", makefile) self.assertIn("test-all: test", makefile) self.assertIn("check-all: fmt-check clippy test-unit", makefile) + self.assertIn(f"CARGO_PACKAGE := {slug}", makefile) + self.assertIn("NEXTEST_PROFILE ?= default", makefile) self.assertNotIn("[tool.pytest.ini_options]", pyproject) result = subprocess.run( @@ -2373,7 +2841,7 @@ def test_python_integration_tests_live_under_repo_plugins_tests(self) -> None: check=False, ) self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("cargo test", result.stdout) + self.assertIn(f"cargo nextest run --profile default -p {slug}", result.stdout) self.assertIn(f"../../../tests/{slug}", result.stdout) def test_coverage_check_reports_per_plugin_percentages(self) -> None: @@ -2988,11 +3456,65 @@ def test_pii_benchmark_script_runs_with_package_surface(self) -> None: self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("ops_per_sec", result.stdout) + def test_bench_no_run_uses_nextest_benchmark_test_mode(self) -> None: + plugin_root = REPO_ROOT / "plugins" / "rust" / "python-package" + for slug in ( + "encoded_exfil_detection", + "pii_filter", + "rate_limiter", + "secrets_detection", + "url_reputation", + ): + makefile = (plugin_root / slug / "Makefile").read_text() + self.assertIn("bench-no-run:", makefile) + self.assertIn("$(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --benches -E 'kind(bench)' --no-run", makefile) + self.assertNotIn("$(CARGO) bench --no-run", makefile) + + def test_nextest_make_targets_dry_run(self) -> None: + plugin_root = REPO_ROOT / "plugins" / "rust" / "python-package" + cases = [ + ("encoded_exfil_detection", "test-unit", "cargo nextest run --profile default -p encoded_exfil_detection"), + ("encoded_exfil_detection", "bench-no-run", "cargo nextest run --profile default -p encoded_exfil_detection --benches -E 'kind(bench)' --no-run"), + ("pii_filter", "test-unit", "cargo nextest run --profile default -p pii_filter"), + ("pii_filter", "bench-no-run", "cargo nextest run --profile default -p pii_filter --benches -E 'kind(bench)' --no-run"), + ("rate_limiter", "test-unit", "cargo nextest run --profile default -p rate_limiter"), + ("rate_limiter", "bench-no-run", "cargo nextest run --profile default -p rate_limiter --benches -E 'kind(bench)' --no-run"), + ("retry_with_backoff", "test-unit", "cargo nextest run --profile default -p retry_with_backoff"), + ("secrets_detection", "test-unit", "cargo nextest run --profile default -p secrets_detection"), + ("secrets_detection", "bench-no-run", "cargo nextest run --profile default -p secrets_detection --benches -E 'kind(bench)' --no-run"), + ("url_reputation", "test-unit", "cargo nextest run --profile default -p url_reputation"), + ("url_reputation", "bench-no-run", "cargo nextest run --profile default -p url_reputation --benches -E 'kind(bench)' --no-run"), + ] + for slug, target, expected in cases: + with self.subTest(slug=slug, target=target): + result = subprocess.run( + ["make", "-n", target], + cwd=plugin_root / slug, + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(expected, result.stdout) + + def test_scaffold_benchmark_template_uses_nextest_benchmark_test_mode(self) -> None: + makefile_template = (REPO_ROOT / "tools" / "templates" / "plugin" / "Makefile.j2").read_text() + self.assertIn("bench-no-run:", makefile_template) + self.assertIn("$(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --benches -E 'kind(bench)' --no-run", makefile_template) + self.assertIn("ci: check-all verify-stubs build{% if include_benchmarks %} bench-no-run{% endif %} install-wheel test-python", makefile_template) + def test_root_plugin_test_uses_plugin_ci_target(self) -> None: makefile = (REPO_ROOT / "Makefile").read_text() self.assertIn("make ci", makefile) self.assertNotIn("make install && make test-all", makefile) + def test_root_makefile_exposes_plugin_mutants_targets(self) -> None: + makefile = (REPO_ROOT / "Makefile").read_text() + self.assertIn("plugin-mutants PLUGIN=", makefile) + self.assertIn("plugin-mutants-list PLUGIN=", makefile) + self.assertIn('cargo mutants -p "$(PLUGIN)"', makefile) + self.assertIn('cargo mutants --list -p "$(PLUGIN)"', makefile) + def test_retry_make_ci_enforces_local_coverage_floor(self) -> None: plugin_dir = REPO_ROOT / "plugins" / "rust" / "python-package" / "retry_with_backoff" makefile = (plugin_dir / "Makefile").read_text() @@ -3000,6 +3522,7 @@ def test_retry_make_ci_enforces_local_coverage_floor(self) -> None: self.assertIn("rustup component add llvm-tools-preview", makefile) self.assertIn("cargo install cargo-llvm-cov --version $(CARGO_LLVM_COV_VERSION) --locked", makefile) self.assertIn("cargo llvm-cov clean --workspace", makefile) + self.assertIn("$(CARGO) llvm-cov nextest --no-report -p $(CARGO_PACKAGE) -P $(NEXTEST_PROFILE)", makefile) self.assertIn("cargo llvm-cov report -p $(CARGO_PACKAGE)", makefile) self.assertIn( "python3 $(REPO_ROOT)/tools/plugin_catalog.py coverage-check $(REPO_ROOT) $(COVERAGE_REPORT) $(COVERAGE_MIN)", diff --git a/tools/plugin_catalog.py b/tools/plugin_catalog.py index 95240d1..d663431 100644 --- a/tools/plugin_catalog.py +++ b/tools/plugin_catalog.py @@ -19,6 +19,8 @@ REPOSITORY_URL = "https://github.com/IBM/cpex-plugins" SHARED_PATH_PREFIXES = ( "Makefile", + ".cargo/", + ".config/", ".github/workflows/", "Cargo.toml", "Cargo.lock", @@ -29,7 +31,6 @@ "TESTING.md", "tools/", ) - ENTRY_POINT_PATTERN = re.compile( r"^(?P[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*):(?P[A-Za-z_][A-Za-z0-9_]*)$" ) @@ -487,14 +488,13 @@ def _git_changed_paths(root: Path, base: str, head: str) -> list[str]: def changed_plugins(root: Path, base: str, head: str) -> list[str]: plugins = discover_plugins(root) - return _changed_plugins_for_records(root, plugins, base, head) + return _changed_plugins_for_records(root, plugins, _git_changed_paths(root, base, head)) def _changed_plugins_for_records( - root: Path, plugins: list[PluginRecord], base: str, head: str + root: Path, plugins: list[PluginRecord], changed_paths: list[str] ) -> list[str]: plugin_lookup = {record.slug: record for record in plugins} - changed_paths = _git_changed_paths(root, base, head) if any( path == prefix.rstrip("/") or path.startswith(prefix) @@ -522,21 +522,76 @@ def _changed_plugins_for_records( return sorted(changed) +def _plugin_depends_on_crate(root: Path, record: PluginRecord, crate_name: str) -> bool: + manifest_path = root / record.path / "Cargo.toml" + with manifest_path.open("rb") as handle: + manifest = tomllib.load(handle) + return crate_name in manifest.get("dependencies", {}) + + +def _mutation_jobs_for_records( + root: Path, plugins: list[PluginRecord], changed_paths: list[str] +) -> list[dict[str, object]]: + plugin_lookup = {record.slug: record for record in plugins} + jobs: dict[str, dict[str, object]] = {} + managed_prefix = f"{MANAGED_ROOT.as_posix()}/" + + def add_job(cargo_package: str, *, in_diff: bool, test_packages: list[str] | None = None) -> None: + jobs[cargo_package] = { + "cargo_package": cargo_package, + "in_diff": in_diff, + "test_packages": test_packages or [], + } + + for path in changed_paths: + if not path.endswith(".rs"): + continue + if path.startswith("crates/framework_bridge/"): + test_packages: list[str] = [] + for record in plugins: + if _plugin_depends_on_crate(root, record, "cpex_framework_bridge"): + test_packages.append(record.cargo_package_name) + add_job("cpex_framework_bridge", in_diff=True, test_packages=sorted(test_packages)) + continue + if not path.startswith(managed_prefix): + continue + relative = path[len(managed_prefix):] + slug = relative.split("/", maxsplit=1)[0] + if slug in plugin_lookup: + add_job(plugin_lookup[slug].cargo_package_name, in_diff=True) + + return [jobs[key] for key in sorted(jobs)] + + def ci_selection(root: Path, mode: str, base: str | None = None, head: str | None = None) -> dict: plugins = discover_plugins(root) plugin_lookup = {plugin.slug: plugin for plugin in plugins} if mode == "all": selected = sorted(plugin.slug for plugin in plugins) + mutation_jobs = [ + { + "cargo_package": plugin_lookup[slug].cargo_package_name, + "in_diff": False, + "test_packages": [], + } + for slug in selected + ] else: if base is None or head is None: raise CatalogError("ci-selection diff mode requires base and head revisions") - selected = _changed_plugins_for_records(root, plugins, base, head) + changed_paths = _git_changed_paths(root, base, head) + selected = _changed_plugins_for_records(root, plugins, changed_paths) + mutation_jobs = _mutation_jobs_for_records(root, plugins, changed_paths) cargo_packages = [plugin_lookup[slug].cargo_package_name for slug in selected] + mutation_cargo_packages = [str(job["cargo_package"]) for job in mutation_jobs] return { "plugins": selected, "has_plugins": bool(selected), "plugin_count": len(selected), "cargo_packages": cargo_packages, + "mutation_cargo_packages": mutation_cargo_packages, + "has_mutation_cargo_packages": bool(mutation_cargo_packages), + "mutation_jobs": mutation_jobs, } @@ -782,6 +837,9 @@ def build_parser() -> argparse.ArgumentParser: "has_plugins", "plugin_count", "cargo_packages", + "mutation_cargo_packages", + "has_mutation_cargo_packages", + "mutation_jobs", ), ) diff --git a/tools/templates/plugin/Makefile.j2 b/tools/templates/plugin/Makefile.j2 index cfaf5ab..cb7c3ce 100644 --- a/tools/templates/plugin/Makefile.j2 +++ b/tools/templates/plugin/Makefile.j2 @@ -5,6 +5,8 @@ help: PACKAGE_NAME := {{ package_name }} WHEEL_PREFIX := {{ module_name }} CARGO := cargo +CARGO_PACKAGE := {{ plugin_name }} +NEXTEST_PROFILE ?= default STUB_FILES := {{ module_name }}/__init__.pyi {{ module_name }}/{{ rust_lib_name }}/__init__.pyi WHEEL_DIR := ../../../../target/wheels @@ -38,11 +40,11 @@ sync: test: @echo "$(GREEN)Running {{ plugin_name }} Rust tests...$(NC)" - $(CARGO) test + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) test-verbose: @echo "$(GREEN)Running {{ plugin_name }} Rust tests (verbose)...$(NC)" - $(CARGO) test -- --nocapture + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --no-capture test-python: @echo "$(GREEN)Running Python tests...$(NC)" @@ -84,12 +86,12 @@ uninstall: @uv pip uninstall -y $(PACKAGE_NAME) 2>/dev/null || true {% if include_benchmarks -%} -# help: bench-no-run - Compile benchmarks without running them +# help: bench-no-run - Compile benchmark targets with nextest .PHONY: bench-no-run bench-no-run: - @echo "$(GREEN)Compiling benchmarks without running them...$(NC)" - $(CARGO) bench --no-run + @echo "$(GREEN)Compiling Criterion benchmark targets with nextest...$(NC)" + $(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --benches -E 'kind(bench)' --no-run {% endif -%} .PHONY: clean clean-all diff --git a/tools/templates/plugin/README.md.j2 b/tools/templates/plugin/README.md.j2 index f8e418e..706cfc5 100644 --- a/tools/templates/plugin/README.md.j2 +++ b/tools/templates/plugin/README.md.j2 @@ -99,7 +99,7 @@ TODO: Add test coverage information **Run tests:** ```bash -cargo test --lib # Run Rust unit tests +cargo nextest run -p {{ plugin_name }} # Run Rust unit tests pytest tests/ # Run Python tests ``` diff --git a/tools/validate_ci_selection.py b/tools/validate_ci_selection.py new file mode 100644 index 0000000..63d7387 --- /dev/null +++ b/tools/validate_ci_selection.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Validate CI plugin selection payload shape and normalize output.""" + +from __future__ import annotations + +import json +import re +import sys + +SLUG_RE = re.compile(r"^[a-z0-9_]+$") + + +def _assert_slug_list(value: object, field_name: str) -> list[str]: + if not isinstance(value, list) or any( + not isinstance(item, str) or SLUG_RE.fullmatch(item) is None for item in value + ): + raise AssertionError(f"{field_name} must be a slug string list") + return value + + +def _assert_mutation_jobs(value: object) -> list[dict[str, object]]: + if not isinstance(value, list): + raise AssertionError("mutation_jobs must be a list") + for job in value: + if not isinstance(job, dict): + raise AssertionError("mutation_jobs entries must be objects") + cargo_package = job.get("cargo_package") + in_diff = job.get("in_diff") + test_packages = job.get("test_packages") + if not isinstance(cargo_package, str) or SLUG_RE.fullmatch(cargo_package) is None: + raise AssertionError("mutation_jobs.cargo_package must be a slug") + if not isinstance(in_diff, bool): + raise AssertionError("mutation_jobs.in_diff must be bool") + if not isinstance(test_packages, list) or any( + not isinstance(item, str) or SLUG_RE.fullmatch(item) is None + for item in test_packages + ): + raise AssertionError("mutation_jobs.test_packages must be a slug string list") + return value + + +def main() -> int: + payload = json.load(sys.stdin) + plugins = _assert_slug_list(payload.get("plugins"), "plugins") + cargo_packages = _assert_slug_list(payload.get("cargo_packages"), "cargo_packages") + mutation_cargo_packages = _assert_slug_list( + payload.get("mutation_cargo_packages"), "mutation_cargo_packages" + ) + mutation_jobs = _assert_mutation_jobs(payload.get("mutation_jobs")) + has_plugins = payload.get("has_plugins") + plugin_count = payload.get("plugin_count") + + if not isinstance(has_plugins, bool): + raise AssertionError("has_plugins must be bool") + if not isinstance(plugin_count, int) or plugin_count != len(plugins): + raise AssertionError("plugin_count must equal len(plugins)") + + print( + json.dumps( + { + "plugins": plugins, + "has_plugins": has_plugins, + "plugin_count": plugin_count, + "cargo_packages": cargo_packages, + "mutation_cargo_packages": mutation_cargo_packages, + "mutation_jobs": mutation_jobs, + "has_mutation_cargo_packages": bool(mutation_cargo_packages), + } + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())