From 4e2092f4384ebda7331b6911d330464492bc603b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 11:50:06 +0000 Subject: [PATCH 1/5] Build e2e test binaries once and tier PM legs The e2e matrix recompiled the CLI and its test binary in each of its 176 legs. e2e-build now compiles every CLI test target once per OS and the legs run the downloaded binaries from the same checkout path. That compile also replaces the --all-features --no-run pass in test and test-release, so each of those compiles one feature set. PR runs keep every named version boundary, the oldest and newest release of each tool and every vlt era. The 31 middle e2e rows, 2 yarn-berry releases and 10 cargo toolchain x lock cells move to *-full jobs that run on main pushes, a new nightly schedule and dispatch. - e2e-docker (a subset of coverage-docker) runs nightly only. - Dockerfile.base is built once per run and loaded by each docker leg. - The hermetic maven/nuget crawl tests run in `test`; their rows and e2e_composer's are gone. e2e_safety_cargo_build rides cargo-vex. - pdm-compat builds the capstone once, skips the 7 cells ci.yml runs, drops the Windows native rows (they never ran), and fails a cell whose bootstrap fails or whose rows all skip. - vlt-compat install-proof leaves ci.yml's identical cells to it. - npm/pnpm compatibility are path-filtered on PRs; the disarmed vlt serve watchdog runs daily instead of every 6 hours. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HK6zFsizxHRUtyxPvdhM4c --- .github/workflows/ci.yml | 458 ++++++++++++++------- .github/workflows/npm-compatibility.yml | 19 + .github/workflows/pdm-compatibility.yml | 72 +++- .github/workflows/pnpm-compatibility.yml | 18 + .github/workflows/vlt-compatibility.yml | 12 + .github/workflows/vlt-serve-watchdog.yml | 5 +- crates/socket-patch-cli/tests/e2e_maven.rs | 4 +- crates/socket-patch-cli/tests/e2e_nuget.rs | 4 +- docs/testing/pdm-compatibility.md | 7 +- docs/testing/vlt-compatibility.md | 3 +- docs/testing/yarn-berry-compatibility.md | 3 + scripts/backtest-pdm.py | 6 +- scripts/ci-e2e-bundle.py | 83 ++++ scripts/ci-vlt-proof-suites.py | 81 ++++ scripts/tests/test_ci_e2e_tiers.py | 166 ++++++++ scripts/tests/test_ci_vlt_rows.py | 11 +- tests/docker/README.md | 5 +- 17 files changed, 780 insertions(+), 177 deletions(-) create mode 100644 scripts/ci-e2e-bundle.py create mode 100644 scripts/ci-vlt-proof-suites.py create mode 100644 scripts/tests/test_ci_e2e_tiers.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb2d4722..10f65db9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,10 @@ on: push: branches: [main] pull_request: + schedule: + # Nightly on main: the `full` tier (e2e-full, cargo-vex-matrix-full, + # yarn-berry-full) and e2e-docker, which pull_request runs skip. + - cron: '41 5 * * *' workflow_dispatch: inputs: hosted_e2e: @@ -261,7 +265,7 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - name: Build - run: cargo build --workspace --all-features + run: cargo build --workspace - name: Install Go (for vexctl) # The `vex` subcommand emits OpenVEX documents; tests/e2e_vex.rs @@ -310,13 +314,12 @@ jobs: echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" - name: Run tests - # `--all-features` would also RUN the docker-e2e / setup-e2e suites, - # which soft-skip as "ok" in this job (no images are built here, and - # macOS/Windows have no Docker at all) — dozens of fake greens per OS - # that would hide a broken skip-guard behind a passing checkmark. - # Build them with --all-features (compile rot is real coverage), but - # run only the default-feature suites; the dedicated e2e-docker and - # setup-matrix jobs run the gated suites for real. + # Default features only: `--all-features` would also RUN the + # docker-e2e / setup-e2e suites, which soft-skip as "ok" here (no + # images, and macOS/Windows have no Docker) — fake greens hiding a + # broken skip-guard. Their compile rot is caught on every OS by + # e2e-build, which compiles every CLI test target with + # --all-features; one feature set here means one compile. # # `--no-fail-fast`: without it cargo stops at the first failing test # BINARY, so one bad file hides every later binary's result on that @@ -329,18 +332,13 @@ jobs: SOCKET_PATCH_GO_E2E_REQUIRED: '1' SOCKET_PATCH_GO_E2E_VERSION: '1.24' run: | - set -euo pipefail - cargo test --workspace --all-features --no-run cargo test --workspace --no-fail-fast test-release: runs-on: ubuntu-latest - # Every tests/ target is its own optimized link, and the two cargo - # invocations below build the graph twice (--all-features, then the - # default features): ~25m on main with ~240 test binaries, so 30m left - # no headroom as suites grow. The manifest-less VEX suites share two - # multi-module binaries (tests/e2e_vex_lockfile/, tests/e2e_vex_build/) - # to keep the count down; the extra 10m covers the rest. + # Every tests/ target is its own optimized link (~240 test binaries). + # The manifest-less VEX suites share two multi-module binaries + # (tests/e2e_vex_lockfile/, tests/e2e_vex_build/) to keep the count down. timeout-minutes: 40 steps: - name: Checkout @@ -366,13 +364,9 @@ jobs: # `ci-release` = [profile.release] minus the full-LTO link (see the # profile's comment in Cargo.toml). Same opt-level/debug-assertion # semantics this job exists to validate; ~23m of LTO relinking gone. - # Build/run split for the same reason as the `test` job: the gated - # docker-e2e / setup-e2e suites only soft-skip here — compile them, - # don't count their skips as passes. - run: | - set -euo pipefail - cargo test --workspace --all-features --profile ci-release --no-run - cargo test --workspace --profile ci-release + # Default features for the same reason as the `test` job; the + # feature-gated suites' compile rot is e2e-build's. + run: cargo test --workspace --profile ci-release coverage: # Code coverage via cargo-llvm-cov (LLVM source-based instrumentation). @@ -454,6 +448,45 @@ jobs: if-no-files-found: error retention-days: 30 + # Dockerfile.base compiles the full-LTO release binary inside Docker, with + # no cache. Build it once per run and hand the image to every docker leg. + docker-base: + runs-on: ubuntu-22.04 + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Docker Buildx + # `driver: docker`: see coverage-docker's matching step. + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + with: + driver: docker + + - name: Build base image + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + with: + context: . + file: tests/docker/Dockerfile.base + tags: socket-patch-test-base:latest + load: true + + - name: Save base image + run: | + mkdir -p docker-base + docker save --output docker-base/socket-patch-test-base.tar socket-patch-test-base:latest + + - uses: ./.github/actions/upload-artifact + with: + name: docker-base-image + path: docker-base/socket-patch-test-base.tar + if-no-files-found: error + retention-days: 3 + coverage-docker: # Per-ecosystem coverage for the Docker-driven e2e suite. Mirrors # the e2e-docker matrix but builds an instrumented socket-patch @@ -463,7 +496,9 @@ jobs: # # Hooks: docker_e2e_.rs reads SOCKET_PATCH_COV_BIN + # SOCKET_PATCH_COV_PROFRAW_DIR. Both unset is the no-op default - # (used by the e2e-docker matrix below). + # (used by the e2e-docker matrix below). Every per-push docker_e2e + # suite runs here; e2e-docker is the nightly run of the same suites + # against the base image's full-LTO release binary. # # Pin to ubuntu-22.04 (glibc 2.35) instead of ubuntu-latest # (currently 24.04, glibc 2.39). The instrumented binary built @@ -471,6 +506,7 @@ jobs: # (glibc 2.36); a binary linked against a newer glibc than the # container ships fails to load. ubuntu-22.04's older glibc is # the highest base that's forward-compatible with debian:12. + needs: docker-base runs-on: ubuntu-22.04 timeout-minutes: 30 permissions: @@ -516,13 +552,15 @@ jobs: # (a PR-poisoned cargo cache could compromise the instrumented # binary we mount into the container). - - name: Build base image - uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + - name: Download base image + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - context: . - file: tests/docker/Dockerfile.base - tags: socket-patch-test-base:latest - load: true + pattern: docker-base-image* + merge-multiple: true + path: docker-base + + - name: Load base image + run: docker load --input docker-base/socket-patch-test-base.tar - name: Build ${{ matrix.ecosystem }} image uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 @@ -674,8 +712,57 @@ jobs: - name: Run pypi dispatch tests run: python pypi/socket-patch/test_dispatch.py + # Compiles the CLI and every CLI test target once per OS (--all-features, + # so this is also the feature-gated suites' compile-rot check) and uploads + # the binaries the e2e, e2e-full and cargo-vex legs run. The legs run the + # test binaries directly from the same checkout path, so `CARGO_BIN_EXE_*` + # and `CARGO_MANIFEST_DIR` resolve as they did under `cargo test`. + e2e-build: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + env: + CARGO_PROFILE_DEV_DEBUG: '0' + CARGO_INCREMENTAL: '0' + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install Rust + run: rustup show + + - name: Cache cargo + uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + key: e2e-build + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Compile the CLI and every CLI test target + shell: bash + run: | + set -euo pipefail + cargo test --locked -p socket-patch-cli --all-features --tests --no-run --message-format=json > target-build.json + + - name: Bundle the binaries the legs run + shell: bash + env: + BUNDLE_OS: ${{ matrix.os }} + run: python3 scripts/ci-e2e-bundle.py --os "$BUNDLE_OS" --cargo-json target-build.json --dest target/e2e-bin + + - uses: ./.github/actions/upload-artifact + with: + name: e2e-bin-${{ matrix.os }} + path: target/e2e-bin/ + if-no-files-found: error + retention-days: 3 + e2e: - needs: test + needs: [test, e2e-build] strategy: fail-fast: false matrix: @@ -684,18 +771,9 @@ jobs: suite: e2e_cargo - os: ubuntu-latest suite: e2e_golang - - os: ubuntu-latest - suite: e2e_maven - - os: ubuntu-latest - suite: e2e_composer - # composer is a shipped ecosystem, so e2e_composer's tests are - # NOT `#[ignore]`-gated the way the live-registry maven/nuget - # suites are — the matrix default `--ignored` filter - # selected zero tests here and the leg passed vacuously. - # `--include-ignored` runs them, plus any capstone added later. - test_filter: --include-ignored - - os: ubuntu-latest - suite: e2e_nuget + # (No e2e_maven / e2e_nuget / e2e_composer rows: their tests are + # hermetic and not `#[ignore]`d, so the `test` job runs them on + # every OS.) # Host build-proof capstones: fresh-checkout install + revert # against the REAL composer/bundler toolchains, each ending in the # manifest-less VEX matrix. `#[ignore]`-gated (the unpinned `test` @@ -723,13 +801,13 @@ jobs: # 1.17/2.1 merged GEM section, 2.2 separate sections, 2.5 last # pre-CHECKSUMS, 2.6 CHECKSUMS, 4.0.15/4.0.21 before/after the # strict frozen check (rubygems#9750). bundler <= 2.2 needs - # Ruby <= 3.3 and 1.17-2.1 need Ruby <= 3.1 (`untaint`). + # Ruby <= 3.3 and 1.17-2.1 need Ruby <= 3.1 (`untaint`). 2.7.2 + # (no boundary of its own) is in e2e-full. - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.1', bundler: '1.17.3'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.1', bundler: '2.1.4'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.1', bundler: '2.2.33'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.3', bundler: '2.5.23'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.3', bundler: '2.6.9'} - - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.3', bundler: '2.7.2'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.4', bundler: '4.0.15'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.4', bundler: '4.0.21'} - {os: ubuntu-latest, suite: e2e_vendor_gem_build, ruby: '3.1', bundler: '1.17.3'} @@ -737,7 +815,6 @@ jobs: - {os: ubuntu-latest, suite: e2e_vendor_gem_build, ruby: '3.1', bundler: '2.2.33'} - {os: ubuntu-latest, suite: e2e_vendor_gem_build, ruby: '3.3', bundler: '2.5.23'} - {os: ubuntu-latest, suite: e2e_vendor_gem_build, ruby: '3.3', bundler: '2.6.9'} - - {os: ubuntu-latest, suite: e2e_vendor_gem_build, ruby: '3.3', bundler: '2.7.2'} - {os: ubuntu-latest, suite: e2e_vendor_gem_build, ruby: '3.4', bundler: '4.0.15'} - {os: ubuntu-latest, suite: e2e_vendor_gem_build, ruby: '3.4', bundler: '4.0.21'} # not #[ignore]-gated -> --include-ignored is mandatory @@ -767,14 +844,9 @@ jobs: # Safety-hardening e2e suites. The fast non-ignored ones # (e2e_safety_lock, e2e_safety_yarn_pnp) run via the # standard `test` job above on all three platforms, so no - # matrix entry is needed for them. The two below need real - # toolchains and are #[ignore]-gated. - - os: ubuntu-latest - suite: e2e_safety_cargo_build - - os: macos-latest - suite: e2e_safety_cargo_build - - os: windows-latest - suite: e2e_safety_cargo_build + # matrix entry is needed for them. e2e_safety_pnpm below needs a + # real pnpm and is #[ignore]-gated; e2e_safety_cargo_build runs in + # every cargo-vex-matrix leg (ubuntu, macOS and Windows). - os: ubuntu-latest suite: e2e_safety_pnpm - os: macos-latest @@ -964,39 +1036,18 @@ jobs: - {os: macos-latest, suite: e2e_redirect_pnpm_build, test_filter: '--ignored --skip pnpm_pinned_matrix'} - {os: windows-latest, suite: e2e_redirect_pnpm_build, test_filter: '--ignored --skip pnpm_pinned_matrix'} # Real-uv hosted/vendored capstones ending in manifest-less VEX: - # one leg per uv 0.N line + the 0.5.x boundary (0.5.4 still + # the oldest and newest line + the 0.5.x boundary (0.5.4 still # re-resolves a transitive override / rejects a repointed - # constraint under --locked; 0.5.5 keeps both). + # constraint under --locked; 0.5.5 keeps both). The other 0.N + # lines and 0.5.3/0.5.6 are in e2e-full. - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.1.45'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.2.37'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.3.5'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.4.30'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.5.3'} - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.5.4'} - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.5.5'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.5.6'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.6.17'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.7.22'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.8.24'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.9.30'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.10.12'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.11.33'} - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.12.17'} - {os: macos-latest, suite: e2e_redirect_uv_build, uv: '0.12.17'} - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.1.45', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.2.37', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.3.5', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.4.30', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.5.3', test_filter: --include-ignored} - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.5.4', test_filter: --include-ignored} - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.5.5', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.5.6', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.6.17', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.7.22', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.8.24', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.9.30', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.10.12', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.11.33', test_filter: --include-ignored} - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.12.17', test_filter: --include-ignored} - {os: macos-latest, suite: e2e_vendor_pypi_build, uv: '0.12.17', test_filter: --include-ignored} # The Poetry / PDM / Hatch / Pipenv / pip / deno manifest-less VEX @@ -1006,11 +1057,11 @@ jobs: # `--ignored`. # Real-Poetry hosted + vendored capstones (#[ignore]-gated, # unix-only). One leg per major / lock format: 1.0 (lock 1.0), - # 1.1 (lock 1.1), 1.8 (lock 2.0), 2.x (lock 2.1). + # 1.1 (lock 1.1), 1.8 (lock 2.0), 2.x (lock 2.1; 2.0.1 is in + # e2e-full). - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'poetry:: --ignored', poetry: '1.0.10'} - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'poetry:: --ignored', poetry: '1.1.15'} - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'poetry:: --ignored', poetry: '1.8.5'} - - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'poetry:: --ignored', poetry: '2.0.1'} - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'poetry:: --ignored', poetry: '2.4.3'} - {os: macos-latest, suite: e2e_vex_build, test_filter: 'poetry:: --ignored', poetry: '2.4.3'} # Real PDM / Hatch capstones (wiremock Socket API that also serves @@ -1034,11 +1085,9 @@ jobs: - {os: macos-latest, suite: e2e_vex_build, test_filter: 'hatch:: --ignored', hatch: '1.18.1'} # Real Pipenv / pip capstones (mock patch server; the tools are # bootstrapped from PyPI). `pipenv:` / `pip:` hold one or more - # space-separated releases the suite loops over. + # space-separated releases the suite loops over. The middle + # Pipenv releases are in e2e-full. - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2022.12.19'} - - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2023.12.1'} - - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2024.4.1'} - - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2025.1.3'} - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2026.8.0'} - {os: macos-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2022.12.19 2026.8.0'} - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pip:: --ignored', pip: '22 23 24 25 26'} @@ -1057,11 +1106,9 @@ jobs: - {os: ubuntu-latest, suite: e2e_vendor_maven_build, maven: '4.0.0-rc-6'} - {os: macos-latest, suite: e2e_vendor_maven_build, maven: '3.9.16'} # Real .NET SDK capstones: hosted + vendored nuget, one leg per SDK - # major (the suite pins the major through a sandbox global.json). + # major (the suite pins the major through a sandbox global.json): + # the oldest and newest here, 7-9 on ubuntu in e2e-full. - {os: ubuntu-latest, suite: e2e_nuget_dotnet_build, dotnet: '6'} - - {os: ubuntu-latest, suite: e2e_nuget_dotnet_build, dotnet: '7'} - - {os: ubuntu-latest, suite: e2e_nuget_dotnet_build, dotnet: '8'} - - {os: ubuntu-latest, suite: e2e_nuget_dotnet_build, dotnet: '9'} - {os: ubuntu-latest, suite: e2e_nuget_dotnet_build, dotnet: '10'} - {os: macos-latest, suite: e2e_nuget_dotnet_build, dotnet: '8'} # Real deno negative capstone (no hosted/vendored wiring exists for @@ -1073,32 +1120,31 @@ jobs: # pipenv) or bootstrap a tool from PyPI before the suite (poetry, pdm, # hatch), hence more than the 25 minutes the older legs needed. timeout-minutes: 40 - steps: + # e2e-full runs these same steps over its rows. + steps: &e2e-steps - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - name: Install Rust - # rustup is pre-installed on GitHub-hosted runners. `rustup show` - # reads rust-toolchain.toml in the repo root, then installs the - # pinned channel + listed components if missing. No third-party - # action dependency needed for toolchain setup. - run: rustup show - - - name: Cache cargo - # Swatinem/rust-cache, main-only saves: see the first `Cache cargo` - # step in this file. - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - name: Download the e2e binaries + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - # Matrix suites otherwise collide on one key: only one of the ~9 - # same-OS legs wins the cache reserve and the rest fail to save. - # Several suites run one leg per pinned toolchain release (bun, uv, - # poetry, pdm, hatch, pipenv, pip, bundler, composer, maven, dotnet, - # deno, vlt), so the release is part of the key too, plus the vlt - # store linker of the two ubuntu e2e_safety_vlt legs. - key: ${{ matrix.suite }}-${{ matrix.vlt || matrix.bun || matrix.uv || matrix.poetry || matrix.pdm || matrix.hatch || matrix.pipenv || matrix.pip || matrix.bundler || matrix.composer || matrix.maven || matrix.dotnet || matrix.deno || 'default' }}${{ matrix.vlt_store_linker && format('-{0}', matrix.vlt_store_linker) || '' }} - save-if: ${{ github.ref == 'refs/heads/main' }} + pattern: e2e-bin-${{ matrix.os }}* + merge-multiple: true + path: target/e2e-bin + + - name: Stage the CLI where the test binaries expect it + # `CARGO_BIN_EXE_socket-patch` was baked in at compile time as + # /target/debug/socket-patch; CARGO_TARGET_TMPDIR likewise. + shell: bash + run: | + set -euo pipefail + exe='' + if [ "$RUNNER_OS" = Windows ]; then exe=.exe; fi + chmod +x target/e2e-bin/* || true + mkdir -p target/debug target/tmp + cp "target/e2e-bin/socket-patch$exe" "target/debug/socket-patch$exe" - name: Setup Node.js if: matrix.suite == 'e2e_npm' || matrix.suite == 'e2e_scan' || matrix.suite == 'e2e_safety_pnpm' @@ -1354,7 +1400,18 @@ jobs: SOCKET_PATCH_DOTNET_E2E_VERSION: ${{ matrix.dotnet }} SOCKET_PATCH_DENO_E2E_REQUIRED: ${{ matrix.deno != '' && '1' || '' }} SOCKET_PATCH_DENO_E2E_VERSION: ${{ matrix.deno }} - run: cargo test -p socket-patch-cli --all-features --test ${{ matrix.suite }} -- ${{ matrix.test_filter || '--ignored' }} + E2E_SUITE: ${{ matrix.suite }} + E2E_TEST_FILTER: ${{ matrix.test_filter || '--ignored' }} + shell: bash + # Runs from the package root with CARGO_MANIFEST_DIR set, as + # `cargo test` would. + run: | + exe='' + if [ "$RUNNER_OS" = Windows ]; then exe=.exe; fi + cd crates/socket-patch-cli + export CARGO_MANIFEST_DIR="$PWD" + # shellcheck disable=SC2086 # the filter is several libtest arguments + "../../target/e2e-bin/$E2E_SUITE$exe" $E2E_TEST_FILTER - name: Run vlt e2e tests if: matrix.vlt != '' @@ -1369,12 +1426,66 @@ jobs: run: | set -uo pipefail status=0 - # shellcheck disable=SC2086 # the filter is several libtest arguments - cargo test -p socket-patch-cli --all-features --test "$VLT_SUITE" -- $VLT_TEST_FILTER 2>&1 | tee vlt-leg.log || status=1 + exe='' + if [ "$RUNNER_OS" = Windows ]; then exe=.exe; fi + ( + cd crates/socket-patch-cli + export CARGO_MANIFEST_DIR="$PWD" + # shellcheck disable=SC2086 # the filter is several libtest arguments + "../../target/e2e-bin/$VLT_SUITE$exe" $VLT_TEST_FILTER + ) 2>&1 | tee vlt-leg.log || status=1 py=$(command -v python3 || command -v python) - "$py" scripts/check-vlt-legs.py --manifest crates/socket-patch-cli/tests/vlt-leg-manifest.json vlt-leg.log || status=1 + # No cargo `Running` line when the binary runs directly: name it. + "$py" scripts/check-vlt-legs.py --binary "$VLT_SUITE" --manifest crates/socket-patch-cli/tests/vlt-leg-manifest.json vlt-leg.log || status=1 exit "$status" + # The PM-version legs with no boundary of their own: the middle uv lines, + # Pipenv releases and .NET SDK majors, Poetry 2.0 and bundler 2.7. Skipped + # on pull_request (the `e2e` rows keep every named boundary, the oldest + # and newest release of each tool and every vlt era there); main pushes, + # the nightly schedule and dispatch run them with the `e2e` steps. + e2e-full: + if: github.event_name != 'pull_request' + needs: [test, e2e-build] + strategy: + fail-fast: false + matrix: + include: + - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.3', bundler: '2.7.2'} + - {os: ubuntu-latest, suite: e2e_vendor_gem_build, ruby: '3.3', bundler: '2.7.2'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.2.37'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.3.5'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.4.30'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.5.3'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.5.6'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.6.17'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.7.22'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.8.24'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.9.30'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.10.12'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.11.33'} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.2.37', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.3.5', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.4.30', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.5.3', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.5.6', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.6.17', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.7.22', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.8.24', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.9.30', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.10.12', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.11.33', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'poetry:: --ignored', poetry: '2.0.1'} + - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2023.12.1'} + - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2024.4.1'} + - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2025.1.3'} + - {os: ubuntu-latest, suite: e2e_nuget_dotnet_build, dotnet: '7'} + - {os: ubuntu-latest, suite: e2e_nuget_dotnet_build, dotnet: '8'} + - {os: ubuntu-latest, suite: e2e_nuget_dotnet_build, dotnet: '9'} + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + steps: *e2e-steps + # ---------------------------------------------------------------------- # Docker-driven real-package e2e suite. # @@ -1385,11 +1496,16 @@ jobs: # managers and run socket-patch against a wiremock-served fixture — # no real Socket API contact. Hermetic, reproducible. # - # Triggered on every PR. The `e2e` job above runs the - # `#[ignore]`-gated real-toolchain capstones; the live-API smoke suites - # are run by hand (see the note in its matrix). + # Nightly and on dispatch only: coverage-docker runs every one of these + # suites on each push (with an instrumented binary mounted in); this job + # is the one run of them against the base image's full-LTO release + # binary. The `e2e` job above runs the `#[ignore]`-gated real-toolchain + # capstones; the live-API smoke suites are run by hand (see the note in + # its matrix). # ---------------------------------------------------------------------- e2e-docker: + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + needs: docker-base runs-on: ubuntu-latest timeout-minutes: 35 permissions: @@ -1419,13 +1535,15 @@ jobs: # No `actions/cache` here intentionally. This job builds Docker # images and would be flagged by zizmor's cache-poisoning audit. - - name: Build base image - uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + - name: Download base image + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - context: . - file: tests/docker/Dockerfile.base - tags: socket-patch-test-base:latest - load: true + pattern: docker-base-image* + merge-multiple: true + path: docker-base + + - name: Load base image + run: docker load --input docker-base/socket-patch-test-base.tar - name: Build ${{ matrix.ecosystem }} image uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 @@ -1500,15 +1618,17 @@ jobs: fail-fast: false matrix: # 4.0.2 bare-hex checksum writer (4.0.0-4.0.2), 4.1.0 first - # `10c0/` writer, then a spread up to current. - os: [ubuntu-latest] - yarn: ['4.0.2', '4.1.0', '4.6.0', '4.12.0', '4.18.0'] + # `10c0/` writer, current, and 4.12.0 on macOS / Windows. The rest + # of the spread (4.6.0, 4.12.0 on ubuntu) is yarn-berry-full. include: + - {os: ubuntu-latest, yarn: '4.0.2'} + - {os: ubuntu-latest, yarn: '4.1.0'} + - {os: ubuntu-latest, yarn: '4.18.0'} - {os: macos-latest, yarn: '4.12.0'} - {os: windows-latest, yarn: '4.12.0'} runs-on: ${{ matrix.os }} timeout-minutes: 30 - steps: + steps: &yarn-berry-steps - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -1530,28 +1650,54 @@ jobs: YARN_BERRY_RELEASE: ${{ matrix.yarn }} run: scripts/yarn-berry-vex-matrix.sh "$YARN_BERRY_RELEASE" + # Skipped on pull_request, like e2e-full. + yarn-berry-full: + name: yarn-berry ${{ matrix.yarn }} (${{ matrix.os }}) + if: github.event_name != 'pull_request' + needs: test + strategy: + fail-fast: false + matrix: + include: + - {os: ubuntu-latest, yarn: '4.6.0'} + - {os: ubuntu-latest, yarn: '4.12.0'} + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + steps: *yarn-berry-steps + + # Every toolchain and every lock re-encoding at least once on PRs (the + # toolchain's own lock is re-encoded as v1-v4 before socket-patch touches + # it, the committed-lockfile case; '' keeps the toolchain's own format), + # plus macOS and Windows. cargo-vex-matrix-full runs the rest of the + # toolchain x lock cross off pull_request. Each leg also runs + # e2e_safety_cargo_build (its headline test honours the knobs). cargo-vex-matrix: name: cargo ${{ matrix.toolchain }} lock-v${{ matrix.lock || 'own' }} (${{ matrix.os }}) - needs: test + needs: [test, e2e-build] runs-on: ${{ matrix.os }} timeout-minutes: 40 strategy: fail-fast: false matrix: - # The toolchain's own lock is re-encoded as v1-v4 before - # socket-patch touches it (the committed-lockfile case); '' keeps the - # toolchain's own format. - os: [ubuntu-latest] - toolchain: ['1.82.0', '1.93.1', 'stable'] - lock: ['', '1', '2', '3', '4'] include: + - {os: ubuntu-latest, toolchain: '1.82.0', lock: ''} + - {os: ubuntu-latest, toolchain: '1.93.1', lock: '1'} + - {os: ubuntu-latest, toolchain: stable, lock: '2'} + - {os: ubuntu-latest, toolchain: '1.82.0', lock: '3'} + - {os: ubuntu-latest, toolchain: '1.93.1', lock: '4'} - {os: macos-latest, toolchain: stable, lock: '1'} - {os: windows-latest, toolchain: stable, lock: '1'} - steps: + steps: &cargo-vex-steps - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + - name: Download the e2e binaries + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: e2e-bin-${{ matrix.os }}* + merge-multiple: true + path: target/e2e-bin - name: Install Rust run: rustup show - name: Install the cargo under test @@ -1562,12 +1708,9 @@ jobs: env: CARGO_TEST_TOOLCHAIN: ${{ matrix.toolchain }} run: rustup toolchain install "$CARGO_TEST_TOOLCHAIN" --profile minimal - - name: Cache cargo - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - with: - key: cargo-vex-${{ matrix.toolchain }} - save-if: ${{ github.ref == 'refs/heads/main' }} - name: Real-cargo hosted/vendored + manifest-less VEX + # The e2e-build binaries, run from the package root as `cargo test` + # would (see the e2e job's staging step). shell: bash env: SOCKET_PATCH_CARGO_E2E_REQUIRED: '1' @@ -1575,8 +1718,43 @@ jobs: SOCKET_PATCH_CARGO_E2E_LOCK_VERSION: ${{ matrix.lock }} run: | set -euo pipefail - cargo test -p socket-patch-cli --test e2e_redirect_cargo_build --test e2e_redirect_cargo_shapes --test e2e_vendor_cargo_build --test mode_migration_cargo - cargo test -p socket-patch-cli --test e2e_safety_cargo_build -- --ignored + exe='' + if [ "$RUNNER_OS" = Windows ]; then exe=.exe; fi + chmod +x target/e2e-bin/* || true + mkdir -p target/debug target/tmp + cp "target/e2e-bin/socket-patch$exe" "target/debug/socket-patch$exe" + cd crates/socket-patch-cli + export CARGO_MANIFEST_DIR="$PWD" + status=0 + for suite in e2e_redirect_cargo_build e2e_redirect_cargo_shapes e2e_vendor_cargo_build mode_migration_cargo; do + echo "::group::$suite" + "../../target/e2e-bin/$suite$exe" || status=1 + echo "::endgroup::" + done + "../../target/e2e-bin/e2e_safety_cargo_build$exe" --ignored || status=1 + exit "$status" + + cargo-vex-matrix-full: + name: cargo ${{ matrix.toolchain }} lock-v${{ matrix.lock || 'own' }} (${{ matrix.os }}) + if: github.event_name != 'pull_request' + needs: [test, e2e-build] + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + include: + - {os: ubuntu-latest, toolchain: '1.82.0', lock: '1'} + - {os: ubuntu-latest, toolchain: '1.82.0', lock: '2'} + - {os: ubuntu-latest, toolchain: '1.82.0', lock: '4'} + - {os: ubuntu-latest, toolchain: '1.93.1', lock: ''} + - {os: ubuntu-latest, toolchain: '1.93.1', lock: '2'} + - {os: ubuntu-latest, toolchain: '1.93.1', lock: '3'} + - {os: ubuntu-latest, toolchain: stable, lock: ''} + - {os: ubuntu-latest, toolchain: stable, lock: '1'} + - {os: ubuntu-latest, toolchain: stable, lock: '3'} + - {os: ubuntu-latest, toolchain: stable, lock: '4'} + steps: *cargo-vex-steps # Manifest `[patch]` + the tagged detached lock (the v5 vendored cargo # wiring) on cargo 1.41 and 1.56 — below / at the 1.56 floor of diff --git a/.github/workflows/npm-compatibility.yml b/.github/workflows/npm-compatibility.yml index 45ad4759..a4377699 100644 --- a/.github/workflows/npm-compatibility.yml +++ b/.github/workflows/npm-compatibility.yml @@ -7,7 +7,26 @@ name: npm hosted/vendored compatibility # installs one pinned npm and runs them. See docs/testing/npm-compatibility.md. on: + # PRs: any crate source, but only the test files these capstones + # compile (a later `!` pattern excludes, a later plain one re-includes). + # Main pushes stay unfiltered. pull_request: + paths: + - '.github/actions/upload-artifact/**' + - 'Cargo.lock' + - 'Cargo.toml' + - 'rust-toolchain.toml' + - '.github/workflows/npm-compatibility.yml' + - 'docs/testing/npm-compatibility.md' + - 'crates/**' + - '!crates/**/*.md' + - '!crates/socket-patch-node/**' + - '!crates/socket-patch-core/tests/**' + - '!crates/socket-patch-cli/tests/**' + - 'crates/socket-patch-cli/tests/vex_e2e_common/**' + - 'crates/socket-patch-cli/tests/e2e_redirect_npm_build.rs' + - 'crates/socket-patch-cli/tests/e2e_vendor_npm_build.rs' + - 'crates/socket-patch-cli/tests/npm_e2e_common/**' push: branches: [main] workflow_dispatch: diff --git a/.github/workflows/pdm-compatibility.yml b/.github/workflows/pdm-compatibility.yml index ef4401b6..424cf9be 100644 --- a/.github/workflows/pdm-compatibility.yml +++ b/.github/workflows/pdm-compatibility.yml @@ -1,10 +1,11 @@ name: PDM patch compatibility -# Native PDM installer matrix: builds the CLI once per OS, bootstraps pinned -# PDM releases with uv, and runs `scripts/backtest-pdm.py` — hosted, vendored -# and agent mode against the public urllib3 free patch, verifying the -# INSTALLED bytes, lock/manifest stability, hash rejection and rollback. No -# Socket API token is needed. See docs/testing/pdm-compatibility.md. +# Native PDM installer matrix: builds the CLI and the capstone test binary +# once per OS, bootstraps pinned PDM releases with uv, and runs +# `scripts/backtest-pdm.py` — hosted, vendored and agent mode against the +# public urllib3 free patch, verifying the INSTALLED bytes, lock/manifest +# stability, hash rejection and rollback. No Socket API token is needed. See +# docs/testing/pdm-compatibility.md. on: pull_request: @@ -60,7 +61,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: @@ -70,13 +71,25 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: key: pdm-compat - - run: cargo build --locked -p socket-patch-cli + - name: Compile the CLI and the capstone once + run: | + set -euo pipefail + cargo test --locked -p socket-patch-cli --test e2e_vex_build --no-run --message-format=json > target-build.json + python3 - <<'PY' + import json, pathlib, shutil + dest = pathlib.Path('target/pdm-e2e') + dest.mkdir(parents=True, exist_ok=True) + shutil.copy2('target/debug/socket-patch', dest / 'socket-patch') + for line in pathlib.Path('target-build.json').read_text().splitlines(): + item = json.loads(line) + if item.get('target', {}).get('name') == 'e2e_vex_build' and item.get('executable'): + shutil.copy2(item['executable'], dest / 'e2e_vex_build') + assert (dest / 'e2e_vex_build').is_file() + PY - uses: ./.github/actions/upload-artifact with: name: pdm-cli-${{ matrix.os }} - path: | - target/debug/socket-patch - target/debug/socket-patch.exe + path: target/pdm-e2e/ if-no-files-found: error retention-days: 7 @@ -86,8 +99,10 @@ jobs: fail-fast: false matrix: # Every stable PDM major family (0.x, 1.x, 2.x) and each 2.x lock-format - # boundary, on Linux and Windows; macOS samples the ends of the range. - os: [ubuntu-latest, windows-latest] + # boundary on Linux; macOS samples the ends of the range. No Windows: + # the harness bootstraps PDM through a POSIX venv layout (bin/pdm), so + # every Windows cell skipped; backtest-pdm.py now fails such a cell. + os: [ubuntu-latest] pdm: ['0.12.3', '1.15.5', '2.0.3', '2.1.5', '2.3.4', '2.6.1', '2.7.4', '2.8.2', '2.9.3', '2.10.4', '2.11.2', '2.17.3', '2.20.1', '2.22.4', '2.25.9', '2.29.2'] include: - { os: macos-latest, pdm: '0.12.3' } @@ -152,25 +167,37 @@ jobs: # The hermetic Rust capstone (wiremock Socket API that also serves the # hosted wheel) over every PDM release the backtest covers: real hosted + # vendored flows ending in the manifest-less VEX matrix; refused lock - # formats (3.1, 4.0-4.2) must attest nothing. + # formats (3.1, 4.0-4.2) must attest nothing. The cells ci.yml's `e2e` + # job runs on every PR and main push are excluded here + # (scripts/tests/test_ci_e2e_tiers.py keeps the two lists in step). capstone: + needs: build strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] pdm: ['0.12.3', '1.0.0', '1.4.5', '1.8.5', '1.15.5', '2.0.3', '2.7.4', '2.8.2', '2.10.4', '2.11.2', '2.17.3', '2.20.1', '2.22.4', '2.25.9', '2.29.2'] + exclude: + - {os: ubuntu-latest, pdm: '1.4.5'} + - {os: ubuntu-latest, pdm: '1.15.5'} + - {os: ubuntu-latest, pdm: '2.7.4'} + - {os: ubuntu-latest, pdm: '2.8.2'} + - {os: ubuntu-latest, pdm: '2.25.9'} + - {os: ubuntu-latest, pdm: '2.29.2'} + - {os: macos-latest, pdm: '2.29.2'} runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: + # The binaries resolve fixtures through the build job's checkout path, + # which is the same on every runner of one OS. - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - key: pdm-vex-capstone - # Only main writes the cache: 30 PR matrix cells saving would churn - # the repo's 10 GiB budget (ci.yml's rust-cache note). - save-if: ${{ github.ref == 'refs/heads/main' }} + pattern: pdm-cli-${{ matrix.os }}* + merge-multiple: true + path: target/pdm-e2e - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.12' @@ -179,4 +206,11 @@ jobs: env: SOCKET_PATCH_PDM_E2E_REQUIRED: '1' SOCKET_PATCH_PDM_E2E_VERSION: ${{ matrix.pdm }} - run: cargo test --locked -p socket-patch-cli --test e2e_vex_build -- 'pdm::' --ignored + run: | + set -euo pipefail + chmod +x target/pdm-e2e/* + mkdir -p target/debug target/tmp + cp target/pdm-e2e/socket-patch target/debug/socket-patch + cd crates/socket-patch-cli + export CARGO_MANIFEST_DIR="$PWD" + ../../target/pdm-e2e/e2e_vex_build 'pdm::' --ignored diff --git a/.github/workflows/pnpm-compatibility.yml b/.github/workflows/pnpm-compatibility.yml index 2aaa8dd9..31e9d908 100644 --- a/.github/workflows/pnpm-compatibility.yml +++ b/.github/workflows/pnpm-compatibility.yml @@ -1,7 +1,25 @@ name: pnpm hosted compatibility on: + # PRs: any crate source, but only the test files these capstones + # compile (a later `!` pattern excludes, a later plain one re-includes). + # Main pushes stay unfiltered. pull_request: + paths: + - '.github/actions/upload-artifact/**' + - 'Cargo.lock' + - 'Cargo.toml' + - 'rust-toolchain.toml' + - '.github/workflows/pnpm-compatibility.yml' + - 'crates/**' + - '!crates/**/*.md' + - '!crates/socket-patch-node/**' + - '!crates/socket-patch-core/tests/**' + - '!crates/socket-patch-cli/tests/**' + - 'crates/socket-patch-cli/tests/vex_e2e_common/**' + - 'crates/socket-patch-cli/tests/e2e_redirect_pnpm_build.rs' + - 'crates/socket-patch-cli/tests/e2e_vendor_pnpm_build.rs' + - 'crates/socket-patch-cli/tests/common/cache_env.rs' push: branches: [main] workflow_dispatch: diff --git a/.github/workflows/vlt-compatibility.yml b/.github/workflows/vlt-compatibility.yml index 6fd588bd..64a29d41 100644 --- a/.github/workflows/vlt-compatibility.yml +++ b/.github/workflows/vlt-compatibility.yml @@ -54,6 +54,7 @@ on: - 'scripts/install-vlt.sh' - 'scripts/vlt-historical-integrity.json' - 'scripts/gen-vlt-collation-golden.mjs' + - 'scripts/ci-vlt-proof-suites.py' push: branches: [main] paths: @@ -91,6 +92,7 @@ on: - 'scripts/install-vlt.sh' - 'scripts/vlt-historical-integrity.json' - 'scripts/gen-vlt-collation-golden.mjs' + - 'scripts/ci-vlt-proof-suites.py' schedule: # Nightly: vlt releases and the production service drift with no PR open. - cron: '17 4 * * *' @@ -313,6 +315,8 @@ jobs: SOCKET_PATCH_VLT_E2E_STORE_LINKER: ${{ matrix.linker }} SOCKET_PATCH_VLT_E2E_CACHE_ROOT: ${{ matrix.cache_root }} VLT_SUITES: ${{ matrix.suites || 'e2e_redirect_vlt_build e2e_vendor_vlt_build mode_migration_vlt e2e_safety_vlt e2e_vlt' }} + MATRIX_OS: ${{ matrix.os }} + NODE_PIN: ${{ matrix.node }} run: | set -uo pipefail exe='' @@ -328,6 +332,14 @@ jobs: mkdir -p "$SOCKET_PATCH_VLT_E2E_CACHE_ROOT" fi py=$(command -v python3 || command -v python) + # Cells ci.yml's e2e rows run identically (every PR, main push and + # nightly) are left to them; a dispatch runs every cell. + if [ "$GITHUB_EVENT_NAME" != workflow_dispatch ]; then + # shellcheck disable=SC2086 # VLT_SUITES is a word list + VLT_SUITES=$("$py" scripts/ci-vlt-proof-suites.py --os "$MATRIX_OS" --vlt "$SOCKET_PATCH_VLT_E2E_VERSION" \ + --node "$NODE_PIN" --linker "${SOCKET_PATCH_VLT_E2E_STORE_LINKER:-}" \ + --cache-root "${SOCKET_PATCH_VLT_E2E_CACHE_ROOT:-}" $VLT_SUITES | tr -d '\r') || exit 1 + fi status=0 for suite in $VLT_SUITES; do echo "::group::$suite" diff --git a/.github/workflows/vlt-serve-watchdog.yml b/.github/workflows/vlt-serve-watchdog.yml index 28f4af5c..2bb63e4d 100644 --- a/.github/workflows/vlt-serve-watchdog.yml +++ b/.github/workflows/vlt-serve-watchdog.yml @@ -12,10 +12,11 @@ name: vlt serve watchdog # Like installer-drift.yml it checks a deployed service, not the diff. It is # `continue-on-error` until the serve fix (`Cache-Control: no-transform`) is # verified in production; removing that line arms it (DESIGN §8.4, the depscan -# rollout's last step). +# rollout's last step). Until then it cannot alert, so it runs once a day to +# record the probe; go back to every 6 hours when arming it. on: schedule: - - cron: '23 */6 * * *' + - cron: '23 4 * * *' workflow_dispatch: permissions: diff --git a/crates/socket-patch-cli/tests/e2e_maven.rs b/crates/socket-patch-cli/tests/e2e_maven.rs index 63f53e27..006c8a1f 100644 --- a/crates/socket-patch-cli/tests/e2e_maven.rs +++ b/crates/socket-patch-cli/tests/e2e_maven.rs @@ -10,7 +10,7 @@ //! //! # Running //! ```sh -//! cargo test -p socket-patch-cli --test e2e_maven -- --ignored +//! cargo test -p socket-patch-cli --test e2e_maven //! ``` use std::path::{Path, PathBuf}; @@ -99,7 +99,6 @@ async fn assert_proxy_served_scans(server: &MockServer, scans: usize) { /// Verify that `socket-patch scan` discovers artifacts in a fake Maven local repo. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "opt-in maven crawl e2e; run with --ignored"] async fn scan_discovers_maven_artifacts() { let server = start_proxy().await; let proxy_url = server.uri(); @@ -226,7 +225,6 @@ async fn scan_discovers_maven_artifacts() { /// Verify that `socket-patch scan` discovers Gradle project artifacts. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "opt-in maven crawl e2e; run with --ignored"] async fn scan_discovers_gradle_project_artifacts() { let server = start_proxy().await; let proxy_url = server.uri(); diff --git a/crates/socket-patch-cli/tests/e2e_nuget.rs b/crates/socket-patch-cli/tests/e2e_nuget.rs index 93fc84b7..e9bd1551 100644 --- a/crates/socket-patch-cli/tests/e2e_nuget.rs +++ b/crates/socket-patch-cli/tests/e2e_nuget.rs @@ -10,7 +10,7 @@ //! //! # Running //! ```sh -//! cargo test -p socket-patch-cli --test e2e_nuget -- --ignored +//! cargo test -p socket-patch-cli --test e2e_nuget //! ``` use std::path::{Path, PathBuf}; @@ -179,7 +179,6 @@ async fn assert_proxy_served_scans(server: &MockServer, scans: usize) { /// Verify that `socket-patch scan` discovers packages in a fake global cache layout. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "opt-in nuget crawl e2e; run with --ignored"] async fn scan_discovers_global_cache_packages() { let server = start_proxy().await; let proxy_url = server.uri(); @@ -247,7 +246,6 @@ async fn scan_discovers_global_cache_packages() { /// Verify that `socket-patch scan` discovers packages in a fake legacy packages/ layout. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "opt-in nuget crawl e2e; run with --ignored"] async fn scan_discovers_legacy_packages() { let server = start_proxy().await; let proxy_url = server.uri(); diff --git a/docs/testing/pdm-compatibility.md b/docs/testing/pdm-compatibility.md index 78a5a8c9..8090e4bb 100644 --- a/docs/testing/pdm-compatibility.md +++ b/docs/testing/pdm-compatibility.md @@ -105,8 +105,11 @@ the patched `urllib3/response.py` (git-blob SHA-256 matches the ledger `afterHash`), ordinary `pdm install` keeps the lock stable, a tampered hash is rejected, a relock-then-rescan keeps rollback invertible, and rollback restores the lock and `pyproject.toml` byte for byte. `.github/workflows/pdm-compatibility.yml` -runs it on Linux, Windows and macOS across every PDM major family. The matrix -needs no Socket API token (the `urllib3@1.26.18` patch is a free tier). +runs it on Linux and macOS across every PDM major family. It does not run on +Windows: the harness bootstraps PDM through a POSIX venv layout (`bin/pdm`), so +every Windows cell used to skip, and a run whose cells all skip or whose PDM +bootstrap fails is now an error. The matrix needs no Socket API token (the +`urllib3@1.26.18` patch is a free tier). > **Note (v5.0):** the "refused vendored scan still writes a `.socket/manifest.json` > record" observation in the notes column below describes the 4.0.0 binary the run diff --git a/docs/testing/vlt-compatibility.md b/docs/testing/vlt-compatibility.md index 3129c0d3..4a930f5f 100644 --- a/docs/testing/vlt-compatibility.md +++ b/docs/testing/vlt-compatibility.md @@ -73,7 +73,8 @@ asserted and each named test or row exists. `install-proof` (every capstone on 31 Linux, 11 macOS and 15 Windows releases, the Node floors 22.22.0 / 22.13.0 / 22.7.0 / 22.0.0 with the collation golden, and the store linkers auto / hardlink / copy / unpack / a `/dev/shm` - cache root); `native` (the backtest against production, artifacts + cache root; a cell `ci.yml`'s `e2e` rows run identically is left to them, + see `scripts/ci-vlt-proof-suites.py`, except on dispatch); `native` (the backtest against production, artifacts `vlt-results--` in depscan's capture `result.json` shape); `lock-diff` (the same cell's `vlt-lock.json` must be byte-identical on Linux, macOS and Windows); `matrix-coverage` (every era × suite × OS). diff --git a/docs/testing/yarn-berry-compatibility.md b/docs/testing/yarn-berry-compatibility.md index 4ea50420..d3e5380e 100644 --- a/docs/testing/yarn-berry-compatibility.md +++ b/docs/testing/yarn-berry-compatibility.md @@ -23,6 +23,9 @@ The `yarn-berry-e2e` job in `.github/workflows/ci.yml` runs | macos-latest | 4.12.0 | | windows-latest | 4.12.0 | +Pull requests skip ubuntu 4.6.0 and 4.12.0 (the `yarn-berry-full` job, which +runs on main pushes, nightly and dispatch). + Each release drives four real-yarn suites — `e2e_redirect_yarn_berry_build`, `e2e_vendor_yarn_berry_build`, `e2e_yarn4_pnpm_linker_build` and `e2e_yarn4_workspaces_build` — each ending in the manifest-less VEX matrix of diff --git a/scripts/backtest-pdm.py b/scripts/backtest-pdm.py index c2fda379..c952e6cb 100644 --- a/scripts/backtest-pdm.py +++ b/scripts/backtest-pdm.py @@ -1362,7 +1362,7 @@ def uninstall(log): for s in args.shapes: for m in args.modes: if wanted(v, s, m): - results.append({"pdm": v, "python": python_for(v), "shape": s, "mode": m, "outcome": "SKIP", "passed": None, "checks": {}, "info": {"skip": "tool bootstrap failed: " + tool_environments.get(v, {}).get("error", "?")[-300:]}}) + results.append({"pdm": v, "python": python_for(v), "shape": s, "mode": m, "outcome": "ERROR", "passed": False, "checks": {}, "info": {"error": "tool bootstrap failed: " + tool_environments.get(v, {}).get("error", "?")[-300:]}}) jobs = [j for j in jobs if tool_environments.get(j[0], {}).get("ok")] def persist(): @@ -1394,6 +1394,10 @@ def persist(): say(render_matrix(summary)) bad = [r for r in summary["results"] if r["outcome"] in ("FAIL", "ERROR")] say(f"{len(summary['results'])} rows: " + ", ".join(f"{o} {sum(1 for r in summary['results'] if r['outcome'] == o)}" for o in ("PASS", "REFUSED-EXPECTED", "UNSUPPORTED", "SKIP", "FAIL", "ERROR"))) + # A cell whose every row skipped exercised nothing; it must not read as green. + if summary["results"] and all(r["outcome"] == "SKIP" for r in summary["results"]): + say("every row SKIPPED: this cell exercised nothing") + sys.exit(1) if bad or errors: sys.exit(1) diff --git a/scripts/ci-e2e-bundle.py b/scripts/ci-e2e-bundle.py new file mode 100644 index 00000000..8212429a --- /dev/null +++ b/scripts/ci-e2e-bundle.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Copy the CLI and the test binaries one OS's CI legs run into one directory. + +ci.yml's e2e-build job compiles every CLI test target once per OS +(`cargo test --no-run --message-format=json`); this picks out the binaries +the `e2e` and `e2e-full` rows name for that OS, plus the suites every +cargo-vex-matrix leg runs, so the legs download them instead of compiling. +""" + +import argparse +import importlib.util +import json +import shutil +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +CI = ROOT / ".github" / "workflows" / "ci.yml" +MATRIX_JOBS = ("e2e", "e2e-full") +# The binaries cargo-vex-matrix and cargo-vex-matrix-full run on every OS. +CARGO_VEX_SUITES = ( + "e2e_redirect_cargo_build", + "e2e_redirect_cargo_shapes", + "e2e_vendor_cargo_build", + "mode_migration_cargo", + "e2e_safety_cargo_build", +) + + +def load_reader(): + path = ROOT / "scripts" / "tests" / "test_ci_vlt_rows.py" + spec = importlib.util.spec_from_file_location("ci_rows", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def suites_for(os_name, text=None): + reader = load_reader() + jobs = reader.jobs(text if text is not None else CI.read_text(encoding="utf-8")) + suites = set(CARGO_VEX_SUITES) + for job in MATRIX_JOBS: + for row in reader.matrix_include(jobs[job]): + if row["os"] == os_name: + suites.add(row["suite"]) + return sorted(suites) + + +def executables(cargo_json): + found = {} + for line in cargo_json.splitlines(): + if not line.startswith("{"): + continue + item = json.loads(line) + target = item.get("target", {}) + if item.get("executable") and item.get("profile", {}).get("test") and "test" in target.get("kind", []): + found[target["name"]] = Path(item["executable"]) + return found + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--os", required=True, help="the runner label the rows use, e.g. ubuntu-latest") + ap.add_argument("--cargo-json", required=True, type=Path) + ap.add_argument("--dest", required=True, type=Path) + args = ap.parse_args(argv) + exe = ".exe" if sys.platform == "win32" else "" + built = executables(args.cargo_json.read_text(encoding="utf-8")) + wanted = suites_for(args.os) + missing = [s for s in wanted if s not in built] + if missing: + print(f"ci-e2e-bundle: no test binary for {missing}", file=sys.stderr) + return 1 + args.dest.mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / "target" / "debug" / f"socket-patch{exe}", args.dest / f"socket-patch{exe}") + for suite in wanted: + shutil.copy2(built[suite], args.dest / f"{suite}{exe}") + print(f"ci-e2e-bundle: {len(wanted)} test binaries for {args.os}: {' '.join(wanted)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci-vlt-proof-suites.py b/scripts/ci-vlt-proof-suites.py new file mode 100644 index 00000000..bebb8dfe --- /dev/null +++ b/scripts/ci-vlt-proof-suites.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Print the capstones one vlt-compatibility install-proof row still runs. + +A suite is left out when ci.yml's `e2e` job (which runs on every pull +request, main push and the nightly schedule) has a row for the identical +cell: same suite, OS and vlt release, the default Node, the same store +linker, no cache root and, for mode_migration_vlt (the one suite that reads +it), the same upgrade vlt. Everything else runs here. +""" + +import argparse +import importlib.util +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +CI = ROOT / ".github" / "workflows" / "ci.yml" +UPGRADE_SUITE = "mode_migration_vlt" + + +def load_reader(): + path = ROOT / "scripts" / "tests" / "test_ci_vlt_rows.py" + spec = importlib.util.spec_from_file_location("ci_rows", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def proof_upgrade(vlt, node): + """The upgrade vlt install-proof gives a row (its `Install vlt` step).""" + if node: + return "" + m = re.fullmatch(r"0\.0\.0-(\d+)|1\.0\.0-rc\.(\d+)", vlt) + if m and (int(m.group(1)) >= 19 if m.group(1) else int(m.group(2)) <= 14): + return "1.2.0" + return "" + + +def ci_cells(text=None): + reader = load_reader() + rows = reader.matrix_include(reader.jobs(text if text is not None else CI.read_text(encoding="utf-8"))["e2e"]) + return {(r["suite"], r["os"], r["vlt"], r.get("vlt_store_linker", ""), r.get("vlt_upgrade", "")) + for r in rows if r.get("vlt")} + + +def remaining(suites, os_name, vlt, node="", linker="", cache_root="", text=None): + if node or cache_root: + return list(suites) + cells = ci_cells(text) + upgrade = proof_upgrade(vlt, node) + keep = [] + for suite in suites: + want_upgrade = upgrade if suite == UPGRADE_SUITE else None + covered = any(s == suite and o == os_name and v == vlt and lk == linker + and (want_upgrade is None or up == want_upgrade) + for s, o, v, lk, up in cells) + if not covered: + keep.append(suite) + return keep + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--os", required=True) + ap.add_argument("--vlt", required=True) + ap.add_argument("--node", default="") + ap.add_argument("--linker", default="") + ap.add_argument("--cache-root", default="") + ap.add_argument("suites", nargs="+") + args = ap.parse_args(argv) + keep = remaining(args.suites, args.os, args.vlt, args.node, args.linker, args.cache_root) + for suite in args.suites: + if suite not in keep: + print(f"{suite}: run by ci.yml's e2e row for this cell", file=sys.stderr) + print(" ".join(keep)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_ci_e2e_tiers.py b/scripts/tests/test_ci_e2e_tiers.py new file mode 100644 index 00000000..94e752e4 --- /dev/null +++ b/scripts/tests/test_ci_e2e_tiers.py @@ -0,0 +1,166 @@ +"""ci.yml's build-once e2e fan-out and its PR / full tiers: every row names a +real test target that e2e-build bundles, the off-PR tiers only add releases +to suites the PR tier already runs on that OS, and the cargo toolchain x +lock cross is exactly split between the two cargo-vex jobs.""" + +import importlib.util +import itertools +import re +import unittest +from pathlib import Path + +ROOT = Path(__file__).parents[2] +CI = ROOT / ".github" / "workflows" / "ci.yml" +PDM = ROOT / ".github" / "workflows" / "pdm-compatibility.yml" +TESTS = ROOT / "crates" / "socket-patch-cli" / "tests" + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +rows_mod = load("ci_rows", Path(__file__).parent / "test_ci_vlt_rows.py") +bundle = load("ci_e2e_bundle", ROOT / "scripts" / "ci-e2e-bundle.py") +proof = load("ci_vlt_proof_suites", ROOT / "scripts" / "ci-vlt-proof-suites.py") +COMPAT = ROOT / ".github" / "workflows" / "vlt-compatibility.yml" +TEXT = CI.read_text(encoding="utf-8") +JOBS = rows_mod.jobs(TEXT) + + +def rows(job): + return rows_mod.matrix_include(JOBS[job]) + + +def job_text(job): + return "\n".join(JOBS[job]) + + +class Tiers(unittest.TestCase): + def test_every_row_is_a_test_target(self): + for job in ("e2e", "e2e-full"): + for row in rows(job): + with self.subTest(job=job, row=row): + suite = row["suite"] + self.assertTrue((TESTS / f"{suite}.rs").is_file() or (TESTS / suite / "main.rs").is_file(), + f"no test target {suite}") + + def test_full_rows_only_add_releases_to_pr_suites(self): + pr = {(r["suite"], r["os"], r.get("test_filter", "")) for r in rows("e2e")} + for row in rows("e2e-full"): + with self.subTest(row=row): + self.assertIn((row["suite"], row["os"], row.get("test_filter", "")), pr) + pr_rows = [tuple(sorted(r.items())) for r in rows("e2e")] + full_rows = [tuple(sorted(r.items())) for r in rows("e2e-full")] + self.assertFalse(set(pr_rows) & set(full_rows), "a row in both tiers runs twice") + self.assertEqual(len(pr_rows + full_rows), len(set(pr_rows + full_rows)), "duplicate rows") + + def test_full_jobs_skip_pull_requests_and_share_steps(self): + for job, anchor in (("e2e-full", "e2e-steps"), ("yarn-berry-full", "yarn-berry-steps"), + ("cargo-vex-matrix-full", "cargo-vex-steps")): + with self.subTest(job=job): + text = job_text(job) + self.assertIn("if: github.event_name != 'pull_request'", text) + self.assertIn(f"steps: *{anchor}", text) + self.assertIn(f"steps: &{anchor}", TEXT) + + def test_nightly_schedule_runs_the_full_tier(self): + self.assertRegex(TEXT, r"(?m)^ schedule:\n(?: #.*\n)* - cron: '[^']+'$") + self.assertIn("if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'", + job_text("e2e-docker")) + + def test_cargo_cross_is_split_exactly(self): + pr = [(r["os"], r["toolchain"], r.get("lock", "")) for r in rows("cargo-vex-matrix")] + full = [(r["os"], r["toolchain"], r.get("lock", "")) for r in rows("cargo-vex-matrix-full")] + want = {("ubuntu-latest", t, l) for t, l in itertools.product(("1.82.0", "1.93.1", "stable"), + ("", "1", "2", "3", "4"))} + want |= {("macos-latest", "stable", "1"), ("windows-latest", "stable", "1")} + self.assertEqual(len(pr + full), len(want)) + self.assertEqual(set(pr) | set(full), want) + ubuntu = [c for c in pr if c[0] == "ubuntu-latest"] + self.assertEqual({c[1] for c in ubuntu}, {"1.82.0", "1.93.1", "stable"}, "every toolchain on PRs") + self.assertEqual({c[2] for c in ubuntu}, {"", "1", "2", "3", "4"}, "every lock on PRs") + + def test_cargo_vex_runs_the_bundled_suites(self): + text = job_text("cargo-vex-matrix") + ran = set(re.findall(r"\b(e2e_[a-z_]+|mode_migration_[a-z_]+)\b", text.split("Real-cargo hosted")[1])) + self.assertEqual(ran, set(bundle.CARGO_VEX_SUITES)) + + def test_bundle_covers_every_os(self): + for os_name in ("ubuntu-latest", "macos-latest", "windows-latest"): + with self.subTest(os=os_name): + suites = bundle.suites_for(os_name, TEXT) + for job in ("e2e", "e2e-full"): + for row in rows(job): + if row["os"] == os_name: + self.assertIn(row["suite"], suites) + + def test_bundle_reads_cargo_json(self): + json_lines = "\n".join([ + '{"reason":"compiler-artifact","target":{"name":"e2e_vlt","kind":["test"]},' + '"profile":{"test":true},"executable":"/t/deps/e2e_vlt-abc"}', + '{"reason":"compiler-artifact","target":{"name":"socket_patch_cli","kind":["lib"]},' + '"profile":{"test":true},"executable":"/t/deps/socket_patch_cli-abc"}', + '{"reason":"compiler-artifact","target":{"name":"socket-patch","kind":["bin"]},' + '"profile":{"test":false},"executable":"/t/debug/socket-patch"}', + "not json", + ]) + self.assertEqual({k: str(v) for k, v in bundle.executables(json_lines).items()}, + {"e2e_vlt": "/t/deps/e2e_vlt-abc"}) + + +class PdmCapstone(unittest.TestCase): + job = rows_mod.jobs(PDM.read_text(encoding="utf-8"))["capstone"] + + def test_excludes_exactly_the_cells_ci_runs_on_every_pr(self): + excluded = rows_mod.matrix_include([l.replace("exclude:", "include:") for l in self.job]) + ci = {(r["os"], r["pdm"]) for r in rows("e2e") if "pdm" in r} + self.assertEqual({(r["os"], r["pdm"]) for r in excluded}, ci) + self.assertEqual(len(excluded), len(ci)) + for row in rows("e2e"): + if "pdm" in row: + self.assertEqual(row.get("test_filter"), "pdm:: --ignored") + self.assertFalse(any("pdm" in r for r in rows("e2e-full")), + "a pdm row off the PR tier would leave its cell unrun on PRs") + versions = re.search(r"pdm: \[([^\]]*)\]", "\n".join(self.job)).group(1) + for _, version in ci: + self.assertIn(f"'{version}'", versions) + + +class VltProofDedupe(unittest.TestCase): + suites = ["e2e_redirect_vlt_build", "e2e_vendor_vlt_build", "mode_migration_vlt", "e2e_safety_vlt", + "e2e_vlt"] + + def test_only_identical_ci_cells_are_left_out(self): + cells = proof.ci_cells(TEXT) + compat = rows_mod.jobs(COMPAT.read_text(encoding="utf-8")) + for row in rows_mod.matrix_include(compat["install-proof"]): + suites = row.get("suites", " ".join(self.suites)).split() + keep = proof.remaining(suites, row["os"], row["vlt"], row.get("node", ""), row.get("linker", ""), + row.get("cache_root", ""), TEXT) + for suite in set(suites) - set(keep): + with self.subTest(row=row, suite=suite): + self.assertFalse(row.get("node") or row.get("cache_root")) + upgrade = proof.proof_upgrade(row["vlt"], "") if suite == proof.UPGRADE_SUITE else None + self.assertTrue(any(c[:4] == (suite, row["os"], row["vlt"], row.get("linker", "")) + and (upgrade is None or c[4] == upgrade) for c in cells)) + + def test_upgrade_rule_matches_the_install_step(self): + text = "\n".join(rows_mod.jobs(COMPAT.read_text(encoding="utf-8"))["install-proof"]) + self.assertIn("Number(m[1]) >= 19 : Number(m[2]) <= 14", text) + self.assertIn("scripts/install-vlt.sh 1.2.0", text) + self.assertEqual(proof.proof_upgrade("0.0.0-19", ""), "1.2.0") + self.assertEqual(proof.proof_upgrade("0.0.0-18", ""), "") + self.assertEqual(proof.proof_upgrade("1.0.0-rc.14", ""), "1.2.0") + self.assertEqual(proof.proof_upgrade("1.0.0-rc.15", ""), "") + self.assertEqual(proof.proof_upgrade("1.0.0-rc.14", "22.13.0"), "") + + def test_lv0_mode_migration_without_ci_upgrade_stays(self): + self.assertIn("mode_migration_vlt", proof.remaining(self.suites, "windows-latest", "1.0.0-rc.14", text=TEXT)) + self.assertNotIn("mode_migration_vlt", proof.remaining(self.suites, "ubuntu-latest", "1.0.0-rc.14", text=TEXT)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_ci_vlt_rows.py b/scripts/tests/test_ci_vlt_rows.py index 94ddb8fc..567900f5 100644 --- a/scripts/tests/test_ci_vlt_rows.py +++ b/scripts/tests/test_ci_vlt_rows.py @@ -200,8 +200,8 @@ def test_rows_pin_supported_releases(self): def test_the_steps_install_vlt_and_check_the_legs(self): e2e = "\n".join(self.ci["e2e"]) - self.assertIn("key: ${{ matrix.suite }}-${{ matrix.vlt || ", e2e, - "vlt releases share a suite, so the release is part of the cache key") + self.assertIn("pattern: e2e-bin-${{ matrix.os }}*", e2e, + "the legs run the binaries e2e-build compiled once per OS") setup = step(self.ci["e2e"], "Setup vlt") self.assertIn("if: matrix.vlt != ''", setup) self.assertIn("scripts/install-vlt.sh", setup) @@ -215,6 +215,8 @@ def test_the_steps_install_vlt_and_check_the_legs(self): self.assertIn("if: matrix.vlt != ''", run) self.assertIn("SOCKET_PATCH_VLT_E2E_REQUIRED: ${{ matrix.vlt != '' && '1' || '' }}", run) self.assertIn("scripts/check-vlt-legs.py", run) + self.assertIn('--binary "$VLT_SUITE"', run, + "a directly run binary prints no cargo `Running` line to name it") self.assertIn("vlt-leg-manifest.json", run) other = step(self.ci["e2e"], "Run e2e tests") self.assertIn("if: matrix.vlt == ''", other) @@ -304,8 +306,9 @@ def test_jobs_and_triggers(self): def test_watchdog(self): text = WATCHDOG.read_text(encoding="utf-8") - self.assertIn("cron: '23 */6 * * *'", text) - self.assertIn("continue-on-error: true", text) + # Daily while disarmed; every 6 hours once continue-on-error goes. + armed = "continue-on-error: true" not in text + self.assertIn("cron: '23 */6 * * *'" if armed else "cron: '23 4 * * *'", text) self.assertIn("scripts/backtest-vlt.py --serve-probe", text) diff --git a/tests/docker/README.md b/tests/docker/README.md index 67611479..75ebaf77 100644 --- a/tests/docker/README.md +++ b/tests/docker/README.md @@ -67,8 +67,9 @@ the real package managers, each in its ecosystem's image: | `docker_e2e_vendor_nuget` | `nuget` | .NET SDK 8.0, Newtonsoft.Json 13.0.3 | | `docker_e2e_vendor_pypi_pm` | `pypi` | poetry, pdm, pipenv on six 1.16.0 | -CI's `e2e-docker` job runs the composer, nuget and pypi_pm capstones; -`coverage-docker` runs all five. Unlike the scan→apply suites they are MULTI-STAGE: a host +CI's `coverage-docker` job runs all five on every push; the nightly +`e2e-docker` job runs the composer, nuget and pypi_pm capstones against +the release binary. Unlike the scan→apply suites they are MULTI-STAGE: a host tempdir is bind-mounted at `/workspace` and shared across three `docker run`s (networked fixture install + offline `socket-patch vendor`; then a fresh-checkout install under `--network none` with cold caches; then From 233fa178538b102e8dc6ac78d0a122d6e456ecb3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 11:58:54 +0000 Subject: [PATCH 2/5] Address review: env parity, filters, PR cells - The legs that run test binaries directly set SOCKET_NO_CONFIG and SOCKET_NO_UPDATE_CHECK, which cargo's [env] gave `cargo test`. - cargo 1.93.1 with its own lock (the pinned toolchain the removed e2e_safety_cargo_build rows ran) stays on PRs; 1.82.0 own-lock moves to the full tier. - Poetry 2.0.1, the first lock 2.1 writer, stays on PRs. - e2e-build gets 60 minutes on Windows. - npm/pnpm filters also watch .cargo/config.toml and cache_env.rs. - vlt install-proof notes a cell left entirely to ci.yml. - pdm-compat saves its build cache from main only. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HK6zFsizxHRUtyxPvdhM4c --- .github/workflows/ci.yml | 38 +++++++++++++++++++----- .github/workflows/npm-compatibility.yml | 2 ++ .github/workflows/pdm-compatibility.yml | 1 + .github/workflows/pnpm-compatibility.yml | 1 + .github/workflows/vlt-compatibility.yml | 3 ++ 5 files changed, 37 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10f65db9..970ba249 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -723,7 +723,8 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} - timeout-minutes: 45 + # Windows compiles the same ~240 targets ~1.6x slower (see `test`). + timeout-minutes: ${{ matrix.os == 'windows-latest' && 60 || 45 }} env: CARGO_PROFILE_DEV_DEBUG: '0' CARGO_INCREMENTAL: '0' @@ -1057,11 +1058,12 @@ jobs: # `--ignored`. # Real-Poetry hosted + vendored capstones (#[ignore]-gated, # unix-only). One leg per major / lock format: 1.0 (lock 1.0), - # 1.1 (lock 1.1), 1.8 (lock 2.0), 2.x (lock 2.1; 2.0.1 is in - # e2e-full). + # 1.1 (lock 1.1), 1.8 (lock 2.0), 2.0 (first lock 2.1 writer), + # current. - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'poetry:: --ignored', poetry: '1.0.10'} - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'poetry:: --ignored', poetry: '1.1.15'} - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'poetry:: --ignored', poetry: '1.8.5'} + - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'poetry:: --ignored', poetry: '2.0.1'} - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'poetry:: --ignored', poetry: '2.4.3'} - {os: macos-latest, suite: e2e_vex_build, test_filter: 'poetry:: --ignored', poetry: '2.4.3'} # Real PDM / Hatch capstones (wiremock Socket API that also serves @@ -1120,6 +1122,11 @@ jobs: # pipenv) or bootstrap a tool from PyPI before the suite (poetry, pdm, # hatch), hence more than the 25 minutes the older legs needed. timeout-minutes: 40 + # What .cargo/config.toml's [env] gives processes cargo launches; these + # legs launch the test binaries themselves. + env: + SOCKET_NO_CONFIG: '1' + SOCKET_NO_UPDATE_CHECK: '1' # e2e-full runs these same steps over its rows. steps: &e2e-steps - name: Checkout @@ -1440,7 +1447,7 @@ jobs: exit "$status" # The PM-version legs with no boundary of their own: the middle uv lines, - # Pipenv releases and .NET SDK majors, Poetry 2.0 and bundler 2.7. Skipped + # Pipenv releases and .NET SDK majors, and bundler 2.7. Skipped # on pull_request (the `e2e` rows keep every named boundary, the oldest # and newest release of each tool and every vlt era there); main pushes, # the nightly schedule and dispatch run them with the `e2e` steps. @@ -1475,7 +1482,6 @@ jobs: - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.9.30', test_filter: --include-ignored} - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.10.12', test_filter: --include-ignored} - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.11.33', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'poetry:: --ignored', poetry: '2.0.1'} - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2023.12.1'} - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2024.4.1'} - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2025.1.3'} @@ -1484,6 +1490,11 @@ jobs: - {os: ubuntu-latest, suite: e2e_nuget_dotnet_build, dotnet: '9'} runs-on: ${{ matrix.os }} timeout-minutes: 40 + # What .cargo/config.toml's [env] gives processes cargo launches; these + # legs launch the test binaries themselves. + env: + SOCKET_NO_CONFIG: '1' + SOCKET_NO_UPDATE_CHECK: '1' steps: *e2e-steps # ---------------------------------------------------------------------- @@ -1668,7 +1679,8 @@ jobs: # Every toolchain and every lock re-encoding at least once on PRs (the # toolchain's own lock is re-encoded as v1-v4 before socket-patch touches # it, the committed-lockfile case; '' keeps the toolchain's own format), - # plus macOS and Windows. cargo-vex-matrix-full runs the rest of the + # plus macOS and Windows; 1.93.1 with its own lock is the pinned + # rust-toolchain.toml cell. cargo-vex-matrix-full runs the rest of the # toolchain x lock cross off pull_request. Each leg also runs # e2e_safety_cargo_build (its headline test honours the knobs). cargo-vex-matrix: @@ -1676,11 +1688,16 @@ jobs: needs: [test, e2e-build] runs-on: ${{ matrix.os }} timeout-minutes: 40 + # What .cargo/config.toml's [env] gives processes cargo launches; these + # legs launch the test binaries themselves. + env: + SOCKET_NO_CONFIG: '1' + SOCKET_NO_UPDATE_CHECK: '1' strategy: fail-fast: false matrix: include: - - {os: ubuntu-latest, toolchain: '1.82.0', lock: ''} + - {os: ubuntu-latest, toolchain: '1.93.1', lock: ''} - {os: ubuntu-latest, toolchain: '1.93.1', lock: '1'} - {os: ubuntu-latest, toolchain: stable, lock: '2'} - {os: ubuntu-latest, toolchain: '1.82.0', lock: '3'} @@ -1740,6 +1757,11 @@ jobs: needs: [test, e2e-build] runs-on: ${{ matrix.os }} timeout-minutes: 40 + # What .cargo/config.toml's [env] gives processes cargo launches; these + # legs launch the test binaries themselves. + env: + SOCKET_NO_CONFIG: '1' + SOCKET_NO_UPDATE_CHECK: '1' strategy: fail-fast: false matrix: @@ -1747,7 +1769,7 @@ jobs: - {os: ubuntu-latest, toolchain: '1.82.0', lock: '1'} - {os: ubuntu-latest, toolchain: '1.82.0', lock: '2'} - {os: ubuntu-latest, toolchain: '1.82.0', lock: '4'} - - {os: ubuntu-latest, toolchain: '1.93.1', lock: ''} + - {os: ubuntu-latest, toolchain: '1.82.0', lock: ''} - {os: ubuntu-latest, toolchain: '1.93.1', lock: '2'} - {os: ubuntu-latest, toolchain: '1.93.1', lock: '3'} - {os: ubuntu-latest, toolchain: stable, lock: ''} diff --git a/.github/workflows/npm-compatibility.yml b/.github/workflows/npm-compatibility.yml index a4377699..4aa62e0f 100644 --- a/.github/workflows/npm-compatibility.yml +++ b/.github/workflows/npm-compatibility.yml @@ -16,6 +16,7 @@ on: - 'Cargo.lock' - 'Cargo.toml' - 'rust-toolchain.toml' + - '.cargo/config.toml' - '.github/workflows/npm-compatibility.yml' - 'docs/testing/npm-compatibility.md' - 'crates/**' @@ -27,6 +28,7 @@ on: - 'crates/socket-patch-cli/tests/e2e_redirect_npm_build.rs' - 'crates/socket-patch-cli/tests/e2e_vendor_npm_build.rs' - 'crates/socket-patch-cli/tests/npm_e2e_common/**' + - 'crates/socket-patch-cli/tests/common/cache_env.rs' push: branches: [main] workflow_dispatch: diff --git a/.github/workflows/pdm-compatibility.yml b/.github/workflows/pdm-compatibility.yml index 424cf9be..9f276f65 100644 --- a/.github/workflows/pdm-compatibility.yml +++ b/.github/workflows/pdm-compatibility.yml @@ -71,6 +71,7 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: key: pdm-compat + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Compile the CLI and the capstone once run: | set -euo pipefail diff --git a/.github/workflows/pnpm-compatibility.yml b/.github/workflows/pnpm-compatibility.yml index 31e9d908..11f06172 100644 --- a/.github/workflows/pnpm-compatibility.yml +++ b/.github/workflows/pnpm-compatibility.yml @@ -10,6 +10,7 @@ on: - 'Cargo.lock' - 'Cargo.toml' - 'rust-toolchain.toml' + - '.cargo/config.toml' - '.github/workflows/pnpm-compatibility.yml' - 'crates/**' - '!crates/**/*.md' diff --git a/.github/workflows/vlt-compatibility.yml b/.github/workflows/vlt-compatibility.yml index 64a29d41..61d4d251 100644 --- a/.github/workflows/vlt-compatibility.yml +++ b/.github/workflows/vlt-compatibility.yml @@ -340,6 +340,9 @@ jobs: --node "$NODE_PIN" --linker "${SOCKET_PATCH_VLT_E2E_STORE_LINKER:-}" \ --cache-root "${SOCKET_PATCH_VLT_E2E_CACHE_ROOT:-}" $VLT_SUITES | tr -d '\r') || exit 1 fi + if [ -z "$VLT_SUITES" ]; then + echo "::notice::every capstone of this cell runs in ci.yml's e2e rows; nothing left to run here" + fi status=0 for suite in $VLT_SUITES; do echo "::group::$suite" From b6d133b215d8aa9b4675263ca27927272349eca0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 11:59:58 +0000 Subject: [PATCH 3/5] Run the full tier on v5 pushes; review fixes - ci.yml also runs on pushes to release/v5-prerelease, whose PRs skip the full tier and which has no nightly; e2e-docker runs there too. - cargo 1.93.1 with its own lock runs on macOS and Windows on PRs, the cell the old e2e_safety_cargo_build rows ran there. - e2e-build and pdm-compat print rendered compile errors (json-render-diagnostics). - vlt-compat and pdm-compat also trigger on ci.yml changes, and the vlt dedupe only counts ci rows with the same test filter. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HK6zFsizxHRUtyxPvdhM4c --- .github/workflows/ci.yml | 18 ++++++++++++------ .github/workflows/pdm-compatibility.yml | 4 +++- .github/workflows/vlt-compatibility.yml | 4 ++++ scripts/ci-vlt-proof-suites.py | 2 +- scripts/tests/test_ci_e2e_tiers.py | 16 ++++++++++++++-- 5 files changed, 34 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 970ba249..9f70d67f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,9 @@ name: CI on: push: - branches: [main] + # The v5 integration branch too: PRs into it skip the full tier, and + # `schedule` only fires on main, so each landing runs the full tier. + branches: [main, release/v5-prerelease] pull_request: schedule: # Nightly on main: the `full` tier (e2e-full, cargo-vex-matrix-full, @@ -747,7 +749,7 @@ jobs: shell: bash run: | set -euo pipefail - cargo test --locked -p socket-patch-cli --all-features --tests --no-run --message-format=json > target-build.json + cargo test --locked -p socket-patch-cli --all-features --tests --no-run --message-format=json-render-diagnostics > target-build.json - name: Bundle the binaries the legs run shell: bash @@ -1507,7 +1509,7 @@ jobs: # managers and run socket-patch against a wiremock-served fixture — # no real Socket API contact. Hermetic, reproducible. # - # Nightly and on dispatch only: coverage-docker runs every one of these + # Nightly, on dispatch and on v5 pushes only: coverage-docker runs these # suites on each push (with an instrumented binary mounted in); this job # is the one run of them against the base image's full-LTO release # binary. The `e2e` job above runs the `#[ignore]`-gated real-toolchain @@ -1515,7 +1517,8 @@ jobs: # its matrix). # ---------------------------------------------------------------------- e2e-docker: - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + # The v5 branch has no nightly of its own, so its pushes run it too. + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/release/v5-prerelease' needs: docker-base runs-on: ubuntu-latest timeout-minutes: 35 @@ -1679,8 +1682,9 @@ jobs: # Every toolchain and every lock re-encoding at least once on PRs (the # toolchain's own lock is re-encoded as v1-v4 before socket-patch touches # it, the committed-lockfile case; '' keeps the toolchain's own format), - # plus macOS and Windows; 1.93.1 with its own lock is the pinned - # rust-toolchain.toml cell. cargo-vex-matrix-full runs the rest of the + # plus macOS and Windows; 1.93.1 with its own lock (on every OS) is the + # pinned rust-toolchain.toml cell `cargo test` used to give + # e2e_safety_cargo_build. cargo-vex-matrix-full runs the rest of the # toolchain x lock cross off pull_request. Each leg also runs # e2e_safety_cargo_build (its headline test honours the knobs). cargo-vex-matrix: @@ -1704,6 +1708,8 @@ jobs: - {os: ubuntu-latest, toolchain: '1.93.1', lock: '4'} - {os: macos-latest, toolchain: stable, lock: '1'} - {os: windows-latest, toolchain: stable, lock: '1'} + - {os: macos-latest, toolchain: '1.93.1', lock: ''} + - {os: windows-latest, toolchain: '1.93.1', lock: ''} steps: &cargo-vex-steps - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/.github/workflows/pdm-compatibility.yml b/.github/workflows/pdm-compatibility.yml index 9f276f65..bb812f51 100644 --- a/.github/workflows/pdm-compatibility.yml +++ b/.github/workflows/pdm-compatibility.yml @@ -26,6 +26,8 @@ on: - 'crates/socket-patch-cli/tests/e2e_vex_build/main.rs' - 'crates/socket-patch-cli/tests/e2e_vex_build/pdm.rs' - 'crates/socket-patch-cli/tests/vex_pypi_real_common/**' + # The capstone skips the cells ci.yml's e2e rows run. + - '.github/workflows/ci.yml' push: branches: [main] paths: @@ -75,7 +77,7 @@ jobs: - name: Compile the CLI and the capstone once run: | set -euo pipefail - cargo test --locked -p socket-patch-cli --test e2e_vex_build --no-run --message-format=json > target-build.json + cargo test --locked -p socket-patch-cli --test e2e_vex_build --no-run --message-format=json-render-diagnostics > target-build.json python3 - <<'PY' import json, pathlib, shutil dest = pathlib.Path('target/pdm-e2e') diff --git a/.github/workflows/vlt-compatibility.yml b/.github/workflows/vlt-compatibility.yml index 61d4d251..14d0a03f 100644 --- a/.github/workflows/vlt-compatibility.yml +++ b/.github/workflows/vlt-compatibility.yml @@ -55,6 +55,8 @@ on: - 'scripts/vlt-historical-integrity.json' - 'scripts/gen-vlt-collation-golden.mjs' - 'scripts/ci-vlt-proof-suites.py' + - '.github/workflows/ci.yml' + - 'scripts/tests/test_ci_vlt_rows.py' push: branches: [main] paths: @@ -93,6 +95,8 @@ on: - 'scripts/vlt-historical-integrity.json' - 'scripts/gen-vlt-collation-golden.mjs' - 'scripts/ci-vlt-proof-suites.py' + - '.github/workflows/ci.yml' + - 'scripts/tests/test_ci_vlt_rows.py' schedule: # Nightly: vlt releases and the production service drift with no PR open. - cron: '17 4 * * *' diff --git a/scripts/ci-vlt-proof-suites.py b/scripts/ci-vlt-proof-suites.py index bebb8dfe..8a119d68 100644 --- a/scripts/ci-vlt-proof-suites.py +++ b/scripts/ci-vlt-proof-suites.py @@ -41,7 +41,7 @@ def ci_cells(text=None): reader = load_reader() rows = reader.matrix_include(reader.jobs(text if text is not None else CI.read_text(encoding="utf-8"))["e2e"]) return {(r["suite"], r["os"], r["vlt"], r.get("vlt_store_linker", ""), r.get("vlt_upgrade", "")) - for r in rows if r.get("vlt")} + for r in rows if r.get("vlt") and r.get("test_filter") == "--include-ignored vlt_pinned_matrix"} def remaining(suites, os_name, vlt, node="", linker="", cache_root="", text=None): diff --git a/scripts/tests/test_ci_e2e_tiers.py b/scripts/tests/test_ci_e2e_tiers.py index 94e752e4..360dc2ba 100644 --- a/scripts/tests/test_ci_e2e_tiers.py +++ b/scripts/tests/test_ci_e2e_tiers.py @@ -68,7 +68,7 @@ def test_full_jobs_skip_pull_requests_and_share_steps(self): def test_nightly_schedule_runs_the_full_tier(self): self.assertRegex(TEXT, r"(?m)^ schedule:\n(?: #.*\n)* - cron: '[^']+'$") - self.assertIn("if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'", + self.assertIn("if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' ||", job_text("e2e-docker")) def test_cargo_cross_is_split_exactly(self): @@ -76,9 +76,12 @@ def test_cargo_cross_is_split_exactly(self): full = [(r["os"], r["toolchain"], r.get("lock", "")) for r in rows("cargo-vex-matrix-full")] want = {("ubuntu-latest", t, l) for t, l in itertools.product(("1.82.0", "1.93.1", "stable"), ("", "1", "2", "3", "4"))} - want |= {("macos-latest", "stable", "1"), ("windows-latest", "stable", "1")} + want |= {("macos-latest", "stable", "1"), ("windows-latest", "stable", "1"), + ("macos-latest", "1.93.1", ""), ("windows-latest", "1.93.1", "")} self.assertEqual(len(pr + full), len(want)) self.assertEqual(set(pr) | set(full), want) + for os_name in ("ubuntu-latest", "macos-latest", "windows-latest"): + self.assertIn((os_name, "1.93.1", ""), pr, "the pinned toolchain's own lock on every OS") ubuntu = [c for c in pr if c[0] == "ubuntu-latest"] self.assertEqual({c[1] for c in ubuntu}, {"1.82.0", "1.93.1", "stable"}, "every toolchain on PRs") self.assertEqual({c[2] for c in ubuntu}, {"", "1", "2", "3", "4"}, "every lock on PRs") @@ -147,6 +150,15 @@ def test_only_identical_ci_cells_are_left_out(self): self.assertTrue(any(c[:4] == (suite, row["os"], row["vlt"], row.get("linker", "")) and (upgrade is None or c[4] == upgrade) for c in cells)) + def test_ci_vlt_rows_match_the_proof_invocation(self): + for row in rows("e2e"): + if row.get("vlt"): + self.assertEqual(row["test_filter"], "--include-ignored vlt_pinned_matrix", row) + ci_node = rows_mod.step(JOBS["e2e"], "Setup Node.js 24 (vlt legs)") + proof_text = "\n".join(rows_mod.jobs(COMPAT.read_text(encoding="utf-8"))["install-proof"]) + node = re.search(r"node-version: '([^']+)'", ci_node).group(1) + self.assertIn(f"node-version: ${{{{ matrix.node || '{node}' }}}}", proof_text) + def test_upgrade_rule_matches_the_install_step(self): text = "\n".join(rows_mod.jobs(COMPAT.read_text(encoding="utf-8"))["install-proof"]) self.assertIn("Number(m[1]) >= 19 : Number(m[2]) <= 14", text) From 4d0ced4ed3b4adfa94389c5c79137f120f1ba761 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 05:09:37 +0000 Subject: [PATCH 4/5] Run the vlt agent get test in agent mode get now defaults to hosted mode (5e5f5ed), so the real-vlt get_and_remove leg ran a hosted get and found the installed copy unpatched. It now passes --mode agent, like the other agent-mode fixtures that commit updated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01R11RZRvYFL3fzmEnkFAU4A (cherry picked from commit cc5f1b6cc7f98d861283c136565faa002ab84cb1) --- crates/socket-patch-cli/tests/e2e_vlt.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/socket-patch-cli/tests/e2e_vlt.rs b/crates/socket-patch-cli/tests/e2e_vlt.rs index dc9e45e5..7d1dcb0e 100644 --- a/crates/socket-patch-cli/tests/e2e_vlt.rs +++ b/crates/socket-patch-cli/tests/e2e_vlt.rs @@ -151,7 +151,7 @@ async fn vlt_pinned_matrix_agent_get_and_remove() { return; }; let fx = Fixture::build(leg, Shape::with_bystander().warm()).await; - let out = socket_api(&fx.proj, &fx.svc, &["get", UUID], &[]); + let out = socket_api(&fx.proj, &fx.svc, &["get", UUID, "--mode", "agent"], &[]); assert_eq!(out.code, 0, "{out}"); assert_eq!(state(&fx.proj, fx.t()), State::Patched); let purl = fx.t().purl(); From 36a9e82c5cdf760f2144a47eb2d115569b82be48 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 16:37:53 +0000 Subject: [PATCH 5/5] Give the nightly CI run its own concurrency group A concurrency group holds one pending run. Sharing main's group let a queued main push and the nightly cancel each other, and the nightly is the only automatic run of e2e-docker. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HK6zFsizxHRUtyxPvdhM4c --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a66ff348..a1856535 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,9 +26,11 @@ permissions: # A newer push to the same PR supersedes its older run; nothing else is # cancelled. Push, dispatch and schedule runs always finish: main runs are # the ONLY rust-cache writers (save-if) and must not die mid-save, and a -# dispatched base-branch run is the base's only CI verdict. +# dispatched base-branch run is the base's only CI verdict. The nightly +# gets its own group: a group holds one pending run, so sharing main's would +# let a queued push and the nightly cancel each other. concurrency: - group: ci-${{ github.event.pull_request.number || github.ref }} + group: ci-${{ github.event.pull_request.number || github.ref }}${{ github.event_name == 'schedule' && '-nightly' || '' }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: