diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f11064b5b..cea21be2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,8 +2,14 @@ name: CI on: push: - branches: [main] + # The v5 integration branch too: PRs into it skip the full tier, and + # `schedule` only fires on main, so each landing runs the full tier. + branches: [main, release/v5-prerelease] pull_request: + schedule: + # Nightly on main: the `full` tier (e2e-full, cargo-vex-matrix-full, + # yarn-berry-full) and e2e-docker, which pull_request runs skip. + - cron: '41 5 * * *' workflow_dispatch: inputs: hosted_e2e: @@ -20,9 +26,11 @@ permissions: # A newer push to the same PR supersedes its older run; nothing else is # cancelled. Push, dispatch and schedule runs always finish: main runs are # the ONLY rust-cache writers (save-if) and must not die mid-save, and a -# dispatched base-branch run is the base's only CI verdict. +# dispatched base-branch run is the base's only CI verdict. The nightly +# gets its own group: a group holds one pending run, so sharing main's would +# let a queued push and the nightly cancel each other. concurrency: - group: ci-${{ github.event.pull_request.number || github.ref }} + group: ci-${{ github.event.pull_request.number || github.ref }}${{ github.event_name == 'schedule' && '-nightly' || '' }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: @@ -240,7 +248,7 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - name: Build - run: cargo build --workspace --all-features + run: cargo build --workspace - name: Install Go (for vexctl) # The `vex` subcommand emits OpenVEX documents; tests/e2e_vex.rs @@ -289,13 +297,12 @@ jobs: echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" - name: Run tests - # `--all-features` would also RUN the docker-e2e suites, - # which soft-skip as "ok" in this job (no images are built here, and - # macOS/Windows have no Docker at all) — dozens of fake greens per OS - # that would hide a broken skip-guard behind a passing checkmark. - # Build them with --all-features (compile rot is real coverage), but - # run only the default-feature suites; the dedicated e2e-docker - # job runs the gated suites for real. + # Default features only: `--all-features` would also RUN the + # docker-e2e suites, which soft-skip as "ok" here (no images, and + # macOS/Windows have no Docker) — fake greens hiding a broken + # skip-guard. Their compile rot is caught on every OS by e2e-build, + # which compiles every CLI test target with --all-features; one + # feature set here means one compile. # # `--no-fail-fast`: without it cargo stops at the first failing test # BINARY, so one bad file hides every later binary's result on that @@ -308,18 +315,13 @@ jobs: SOCKET_PATCH_GO_E2E_REQUIRED: '1' SOCKET_PATCH_GO_E2E_VERSION: '1.24' run: | - set -euo pipefail - cargo test --workspace --all-features --no-run cargo test --workspace --no-fail-fast test-release: runs-on: ubuntu-latest - # Every tests/ target is its own optimized link, and the two cargo - # invocations below build the graph twice (--all-features, then the - # default features): ~25m on main with ~240 test binaries, so 30m left - # no headroom as suites grow. The manifest-less VEX suites share two - # multi-module binaries (tests/e2e_vex_lockfile/, tests/e2e_vex_build/) - # to keep the count down; the extra 10m covers the rest. + # Every tests/ target is its own optimized link (~240 test binaries). + # The manifest-less VEX suites share two multi-module binaries + # (tests/e2e_vex_lockfile/, tests/e2e_vex_build/) to keep the count down. timeout-minutes: 40 steps: - name: Checkout @@ -345,13 +347,9 @@ jobs: # `ci-release` = [profile.release] minus the full-LTO link (see the # profile's comment in Cargo.toml). Same opt-level/debug-assertion # semantics this job exists to validate; ~23m of LTO relinking gone. - # Build/run split for the same reason as the `test` job: the gated - # docker-e2e suites only soft-skip here — compile them, - # don't count their skips as passes. - run: | - set -euo pipefail - cargo test --workspace --all-features --profile ci-release --no-run - cargo test --workspace --profile ci-release + # Default features for the same reason as the `test` job; the + # feature-gated suites' compile rot is e2e-build's. + run: cargo test --workspace --profile ci-release coverage: # Code coverage via cargo-llvm-cov (LLVM source-based instrumentation). @@ -433,6 +431,45 @@ jobs: if-no-files-found: error retention-days: 30 + # Dockerfile.base compiles the full-LTO release binary inside Docker, with + # no cache. Build it once per run and hand the image to every docker leg. + docker-base: + runs-on: ubuntu-22.04 + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Docker Buildx + # `driver: docker`: see coverage-docker's matching step. + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + with: + driver: docker + + - name: Build base image + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + with: + context: . + file: tests/docker/Dockerfile.base + tags: socket-patch-test-base:latest + load: true + + - name: Save base image + run: | + mkdir -p docker-base + docker save --output docker-base/socket-patch-test-base.tar socket-patch-test-base:latest + + - uses: ./.github/actions/upload-artifact + with: + name: docker-base-image + path: docker-base/socket-patch-test-base.tar + if-no-files-found: error + retention-days: 3 + coverage-docker: # Per-ecosystem coverage for the Docker-driven e2e suite. Mirrors # the e2e-docker matrix but builds an instrumented socket-patch @@ -442,7 +479,9 @@ jobs: # # Hooks: docker_e2e_.rs reads SOCKET_PATCH_COV_BIN + # SOCKET_PATCH_COV_PROFRAW_DIR. Both unset is the no-op default - # (used by the e2e-docker matrix below). + # (used by the e2e-docker matrix below). Every per-push docker_e2e + # suite runs here; e2e-docker is the nightly run of the same suites + # against the base image's full-LTO release binary. # # Pin to ubuntu-22.04 (glibc 2.35) instead of ubuntu-latest # (currently 24.04, glibc 2.39). The instrumented binary built @@ -450,6 +489,7 @@ jobs: # (glibc 2.36); a binary linked against a newer glibc than the # container ships fails to load. ubuntu-22.04's older glibc is # the highest base that's forward-compatible with debian:12. + needs: docker-base runs-on: ubuntu-22.04 timeout-minutes: 30 permissions: @@ -495,13 +535,15 @@ jobs: # (a PR-poisoned cargo cache could compromise the instrumented # binary we mount into the container). - - name: Build base image - uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + - name: Download base image + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - context: . - file: tests/docker/Dockerfile.base - tags: socket-patch-test-base:latest - load: true + pattern: docker-base-image* + merge-multiple: true + path: docker-base + + - name: Load base image + run: docker load --input docker-base/socket-patch-test-base.tar - name: Build ${{ matrix.ecosystem }} image uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 @@ -653,28 +695,67 @@ jobs: - name: Run pypi dispatch tests run: python pypi/socket-patch/test_dispatch.py + # Compiles the CLI and every CLI test target once per OS (--all-features, + # so this is also the feature-gated suites' compile-rot check) and uploads + # the binaries the e2e, e2e-full and cargo-vex legs run. The legs run the + # test binaries directly from the same checkout path, so `CARGO_BIN_EXE_*` + # and `CARGO_MANIFEST_DIR` resolve as they did under `cargo test`. + e2e-build: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + # Windows compiles the same ~240 targets ~1.6x slower (see `test`). + timeout-minutes: ${{ matrix.os == 'windows-latest' && 60 || 45 }} + env: + CARGO_PROFILE_DEV_DEBUG: '0' + CARGO_INCREMENTAL: '0' + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install Rust + run: rustup show + + - name: Cache cargo + uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + key: e2e-build + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Compile the CLI and every CLI test target + shell: bash + run: | + set -euo pipefail + cargo test --locked -p socket-patch-cli --all-features --tests --no-run --message-format=json-render-diagnostics > target-build.json + + - name: Bundle the binaries the legs run + shell: bash + env: + BUNDLE_OS: ${{ matrix.os }} + run: python3 scripts/ci-e2e-bundle.py --os "$BUNDLE_OS" --cargo-json target-build.json --dest target/e2e-bin + + - uses: ./.github/actions/upload-artifact + with: + name: e2e-bin-${{ matrix.os }} + path: target/e2e-bin/ + if-no-files-found: error + retention-days: 3 + e2e: - needs: test + needs: [test, e2e-build] strategy: fail-fast: false matrix: include: # (No e2e_cargo / e2e_golang rows: neither suite has an - # `#[ignore]`-gated test or needs a real toolchain, so the - # unfiltered `test` job already runs all of them and the rows' - # `--ignored` legs selected zero tests.) - - os: ubuntu-latest - suite: e2e_maven - - os: ubuntu-latest - suite: e2e_composer - # composer is a shipped ecosystem, so e2e_composer's tests are - # NOT `#[ignore]`-gated the way the live-registry maven/nuget - # suites are — the matrix default `--ignored` filter - # selected zero tests here and the leg passed vacuously. - # `--include-ignored` runs them, plus any capstone added later. - test_filter: --include-ignored - - os: ubuntu-latest - suite: e2e_nuget + # `#[ignore]`-gated test or needs a real toolchain, so the `test` + # job already runs all of them. No e2e_maven / e2e_nuget / + # e2e_composer rows either: their tests are hermetic and not + # `#[ignore]`d, so `test` runs them on every OS.) # Host build-proof capstones: fresh-checkout install + revert # against the REAL composer/bundler toolchains, each ending in the # manifest-less VEX matrix. `#[ignore]`-gated (the unpinned `test` @@ -699,13 +780,13 @@ jobs: # 1.17/2.1 merged GEM section, 2.2 separate sections, 2.5 last # pre-CHECKSUMS, 2.6 CHECKSUMS, 4.0.15/4.0.21 before/after the # strict frozen check (rubygems#9750). bundler <= 2.2 needs - # Ruby <= 3.3 and 1.17-2.1 need Ruby <= 3.1 (`untaint`). + # Ruby <= 3.3 and 1.17-2.1 need Ruby <= 3.1 (`untaint`). 2.7.2 + # (no boundary of its own) is in e2e-full. - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.1', bundler: '1.17.3'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.1', bundler: '2.1.4'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.1', bundler: '2.2.33'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.3', bundler: '2.5.23'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.3', bundler: '2.6.9'} - - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.3', bundler: '2.7.2'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.4', bundler: '4.0.15'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.4', bundler: '4.0.21'} - {os: ubuntu-latest, suite: e2e_vendor_gem_build, ruby: '3.1', bundler: '1.17.3'} @@ -713,7 +794,6 @@ jobs: - {os: ubuntu-latest, suite: e2e_vendor_gem_build, ruby: '3.1', bundler: '2.2.33'} - {os: ubuntu-latest, suite: e2e_vendor_gem_build, ruby: '3.3', bundler: '2.5.23'} - {os: ubuntu-latest, suite: e2e_vendor_gem_build, ruby: '3.3', bundler: '2.6.9'} - - {os: ubuntu-latest, suite: e2e_vendor_gem_build, ruby: '3.3', bundler: '2.7.2'} - {os: ubuntu-latest, suite: e2e_vendor_gem_build, ruby: '3.4', bundler: '4.0.15'} - {os: ubuntu-latest, suite: e2e_vendor_gem_build, ruby: '3.4', bundler: '4.0.21'} # The live-API smoke suites (e2e_npm, e2e_pypi, e2e_gem, @@ -741,14 +821,9 @@ jobs: # Safety-hardening e2e suites. The fast non-ignored ones # (e2e_safety_lock, e2e_safety_yarn_pnp) run via the # standard `test` job above on all three platforms, so no - # matrix entry is needed for them. The two below need real - # toolchains and are #[ignore]-gated. - - os: ubuntu-latest - suite: e2e_safety_cargo_build - - os: macos-latest - suite: e2e_safety_cargo_build - - os: windows-latest - suite: e2e_safety_cargo_build + # matrix entry is needed for them. e2e_safety_pnpm below needs a + # real pnpm and is #[ignore]-gated; e2e_safety_cargo_build runs in + # every cargo-vex-matrix leg (ubuntu, macOS and Windows). - os: ubuntu-latest suite: e2e_safety_pnpm - os: macos-latest @@ -938,39 +1013,18 @@ jobs: - {os: macos-latest, suite: e2e_redirect_pnpm_build, test_filter: '--ignored --skip pnpm_pinned_matrix'} - {os: windows-latest, suite: e2e_redirect_pnpm_build, test_filter: '--ignored --skip pnpm_pinned_matrix'} # Real-uv hosted/vendored capstones ending in manifest-less VEX: - # one leg per uv 0.N line + the 0.5.x boundary (0.5.4 still + # the oldest and newest line + the 0.5.x boundary (0.5.4 still # re-resolves a transitive override / rejects a repointed - # constraint under --locked; 0.5.5 keeps both). + # constraint under --locked; 0.5.5 keeps both). The other 0.N + # lines and 0.5.3/0.5.6 are in e2e-full. - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.1.45'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.2.37'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.3.5'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.4.30'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.5.3'} - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.5.4'} - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.5.5'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.5.6'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.6.17'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.7.22'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.8.24'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.9.30'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.10.12'} - - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.11.33'} - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.12.17'} - {os: macos-latest, suite: e2e_redirect_uv_build, uv: '0.12.17'} - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.1.45', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.2.37', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.3.5', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.4.30', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.5.3', test_filter: --include-ignored} - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.5.4', test_filter: --include-ignored} - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.5.5', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.5.6', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.6.17', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.7.22', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.8.24', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.9.30', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.10.12', test_filter: --include-ignored} - - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.11.33', test_filter: --include-ignored} - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.12.17', test_filter: --include-ignored} - {os: macos-latest, suite: e2e_vendor_pypi_build, uv: '0.12.17', test_filter: --include-ignored} # The Poetry / PDM / Hatch / Pipenv / pip / deno manifest-less VEX @@ -980,7 +1034,8 @@ jobs: # `--ignored`. # Real-Poetry hosted + vendored capstones (#[ignore]-gated, # unix-only). One leg per major / lock format: 1.0 (lock 1.0), - # 1.1 (lock 1.1), 1.8 (lock 2.0), 2.x (lock 2.1). + # 1.1 (lock 1.1), 1.8 (lock 2.0), 2.0 (first lock 2.1 writer), + # current. - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'poetry:: --ignored', poetry: '1.0.10'} - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'poetry:: --ignored', poetry: '1.1.15'} - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'poetry:: --ignored', poetry: '1.8.5'} @@ -1008,11 +1063,9 @@ jobs: - {os: macos-latest, suite: e2e_vex_build, test_filter: 'hatch:: --ignored', hatch: '1.18.1'} # Real Pipenv / pip capstones (mock patch server; the tools are # bootstrapped from PyPI). `pipenv:` / `pip:` hold one or more - # space-separated releases the suite loops over. + # space-separated releases the suite loops over. The middle + # Pipenv releases are in e2e-full. - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2022.12.19'} - - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2023.12.1'} - - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2024.4.1'} - - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2025.1.3'} - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2026.8.0'} - {os: macos-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2022.12.19 2026.8.0'} - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pip:: --ignored', pip: '22 23 24 25 26'} @@ -1031,11 +1084,9 @@ jobs: - {os: ubuntu-latest, suite: e2e_vendor_maven_build, maven: '4.0.0-rc-6'} - {os: macos-latest, suite: e2e_vendor_maven_build, maven: '3.9.16'} # Real .NET SDK capstones: hosted + vendored nuget, one leg per SDK - # major (the suite pins the major through a sandbox global.json). + # major (the suite pins the major through a sandbox global.json): + # the oldest and newest here, 7-9 on ubuntu in e2e-full. - {os: ubuntu-latest, suite: e2e_nuget_dotnet_build, dotnet: '6'} - - {os: ubuntu-latest, suite: e2e_nuget_dotnet_build, dotnet: '7'} - - {os: ubuntu-latest, suite: e2e_nuget_dotnet_build, dotnet: '8'} - - {os: ubuntu-latest, suite: e2e_nuget_dotnet_build, dotnet: '9'} - {os: ubuntu-latest, suite: e2e_nuget_dotnet_build, dotnet: '10'} - {os: macos-latest, suite: e2e_nuget_dotnet_build, dotnet: '8'} # Real deno negative capstone (no hosted/vendored wiring exists for @@ -1047,32 +1098,36 @@ jobs: # pipenv) or bootstrap a tool from PyPI before the suite (poetry, pdm, # hatch), hence more than the 25 minutes the older legs needed. timeout-minutes: 40 - steps: + # What .cargo/config.toml's [env] gives processes cargo launches; these + # legs launch the test binaries themselves. + env: + SOCKET_NO_CONFIG: '1' + SOCKET_NO_UPDATE_CHECK: '1' + # e2e-full runs these same steps over its rows. + steps: &e2e-steps - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - name: Install Rust - # rustup is pre-installed on GitHub-hosted runners. `rustup show` - # reads rust-toolchain.toml in the repo root, then installs the - # pinned channel + listed components if missing. No third-party - # action dependency needed for toolchain setup. - run: rustup show - - - name: Cache cargo - # Swatinem/rust-cache, main-only saves: see the first `Cache cargo` - # step in this file. - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - name: Download the e2e binaries + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - # Matrix suites otherwise collide on one key: only one of the ~9 - # same-OS legs wins the cache reserve and the rest fail to save. - # Several suites run one leg per pinned toolchain release (bun, uv, - # poetry, pdm, hatch, pipenv, pip, bundler, composer, maven, dotnet, - # deno, vlt), so the release is part of the key too, plus the vlt - # store linker of the two ubuntu e2e_safety_vlt legs. - key: ${{ matrix.suite }}-${{ matrix.vlt || matrix.bun || matrix.uv || matrix.poetry || matrix.pdm || matrix.hatch || matrix.pipenv || matrix.pip || matrix.bundler || matrix.composer || matrix.maven || matrix.dotnet || matrix.deno || 'default' }}${{ matrix.vlt_store_linker && format('-{0}', matrix.vlt_store_linker) || '' }} - save-if: ${{ github.ref == 'refs/heads/main' }} + pattern: e2e-bin-${{ matrix.os }}* + merge-multiple: true + path: target/e2e-bin + + - name: Stage the CLI where the test binaries expect it + # `CARGO_BIN_EXE_socket-patch` was baked in at compile time as + # /target/debug/socket-patch; CARGO_TARGET_TMPDIR likewise. + shell: bash + run: | + set -euo pipefail + exe='' + if [ "$RUNNER_OS" = Windows ]; then exe=.exe; fi + chmod +x target/e2e-bin/* || true + mkdir -p target/debug target/tmp + cp "target/e2e-bin/socket-patch$exe" "target/debug/socket-patch$exe" - name: Setup Node.js if: matrix.suite == 'e2e_npm' || matrix.suite == 'e2e_scan' || matrix.suite == 'e2e_safety_pnpm' @@ -1328,7 +1383,18 @@ jobs: SOCKET_PATCH_DOTNET_E2E_VERSION: ${{ matrix.dotnet }} SOCKET_PATCH_DENO_E2E_REQUIRED: ${{ matrix.deno != '' && '1' || '' }} SOCKET_PATCH_DENO_E2E_VERSION: ${{ matrix.deno }} - run: cargo test -p socket-patch-cli --all-features --test ${{ matrix.suite }} -- ${{ matrix.test_filter || '--ignored' }} + E2E_SUITE: ${{ matrix.suite }} + E2E_TEST_FILTER: ${{ matrix.test_filter || '--ignored' }} + shell: bash + # Runs from the package root with CARGO_MANIFEST_DIR set, as + # `cargo test` would. + run: | + exe='' + if [ "$RUNNER_OS" = Windows ]; then exe=.exe; fi + cd crates/socket-patch-cli + export CARGO_MANIFEST_DIR="$PWD" + # shellcheck disable=SC2086 # the filter is several libtest arguments + "../../target/e2e-bin/$E2E_SUITE$exe" $E2E_TEST_FILTER - name: Run vlt e2e tests if: matrix.vlt != '' @@ -1343,12 +1409,70 @@ jobs: run: | set -uo pipefail status=0 - # shellcheck disable=SC2086 # the filter is several libtest arguments - cargo test -p socket-patch-cli --all-features --test "$VLT_SUITE" -- $VLT_TEST_FILTER 2>&1 | tee vlt-leg.log || status=1 + exe='' + if [ "$RUNNER_OS" = Windows ]; then exe=.exe; fi + ( + cd crates/socket-patch-cli + export CARGO_MANIFEST_DIR="$PWD" + # shellcheck disable=SC2086 # the filter is several libtest arguments + "../../target/e2e-bin/$VLT_SUITE$exe" $VLT_TEST_FILTER + ) 2>&1 | tee vlt-leg.log || status=1 py=$(command -v python3 || command -v python) - "$py" scripts/check-vlt-legs.py --manifest crates/socket-patch-cli/tests/vlt-leg-manifest.json vlt-leg.log || status=1 + # No cargo `Running` line when the binary runs directly: name it. + "$py" scripts/check-vlt-legs.py --binary "$VLT_SUITE" --manifest crates/socket-patch-cli/tests/vlt-leg-manifest.json vlt-leg.log || status=1 exit "$status" + # The PM-version legs with no boundary of their own: the middle uv lines, + # Pipenv releases and .NET SDK majors, and bundler 2.7. Skipped + # on pull_request (the `e2e` rows keep every named boundary, the oldest + # and newest release of each tool and every vlt era there); main pushes, + # the nightly schedule and dispatch run them with the `e2e` steps. + e2e-full: + if: github.event_name != 'pull_request' + needs: [test, e2e-build] + strategy: + fail-fast: false + matrix: + include: + - {os: ubuntu-latest, suite: e2e_redirect_gem_build, ruby: '3.3', bundler: '2.7.2'} + - {os: ubuntu-latest, suite: e2e_vendor_gem_build, ruby: '3.3', bundler: '2.7.2'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.2.37'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.3.5'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.4.30'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.5.3'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.5.6'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.6.17'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.7.22'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.8.24'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.9.30'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.10.12'} + - {os: ubuntu-latest, suite: e2e_redirect_uv_build, uv: '0.11.33'} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.2.37', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.3.5', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.4.30', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.5.3', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.5.6', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.6.17', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.7.22', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.8.24', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.9.30', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.10.12', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vendor_pypi_build, uv: '0.11.33', test_filter: --include-ignored} + - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2023.12.1'} + - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2024.4.1'} + - {os: ubuntu-latest, suite: e2e_vex_build, test_filter: 'pipenv:: --ignored', pipenv: '2025.1.3'} + - {os: ubuntu-latest, suite: e2e_nuget_dotnet_build, dotnet: '7'} + - {os: ubuntu-latest, suite: e2e_nuget_dotnet_build, dotnet: '8'} + - {os: ubuntu-latest, suite: e2e_nuget_dotnet_build, dotnet: '9'} + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + # What .cargo/config.toml's [env] gives processes cargo launches; these + # legs launch the test binaries themselves. + env: + SOCKET_NO_CONFIG: '1' + SOCKET_NO_UPDATE_CHECK: '1' + steps: *e2e-steps + # ---------------------------------------------------------------------- # Docker-driven real-package e2e suite. # @@ -1359,11 +1483,17 @@ jobs: # managers and run socket-patch against a wiremock-served fixture — # no real Socket API contact. Hermetic, reproducible. # - # Triggered on every PR. The `e2e` job above runs the - # `#[ignore]`-gated real-toolchain capstones; the live-API smoke suites - # are run by hand (see the note in its matrix). + # Nightly, on dispatch and on v5 pushes only: coverage-docker runs these + # suites on each push (with an instrumented binary mounted in); this job + # is the one run of them against the base image's full-LTO release + # binary. The `e2e` job above runs the `#[ignore]`-gated real-toolchain + # capstones; the live-API smoke suites are run by hand (see the note in + # its matrix). # ---------------------------------------------------------------------- e2e-docker: + # The v5 branch has no nightly of its own, so its pushes run it too. + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/release/v5-prerelease' + needs: docker-base runs-on: ubuntu-latest timeout-minutes: 35 permissions: @@ -1393,13 +1523,15 @@ jobs: # No `actions/cache` here intentionally. This job builds Docker # images and would be flagged by zizmor's cache-poisoning audit. - - name: Build base image - uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + - name: Download base image + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - context: . - file: tests/docker/Dockerfile.base - tags: socket-patch-test-base:latest - load: true + pattern: docker-base-image* + merge-multiple: true + path: docker-base + + - name: Load base image + run: docker load --input docker-base/socket-patch-test-base.tar - name: Build ${{ matrix.ecosystem }} image uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 @@ -1474,15 +1606,17 @@ jobs: fail-fast: false matrix: # 4.0.2 bare-hex checksum writer (4.0.0-4.0.2), 4.1.0 first - # `10c0/` writer, then a spread up to current. - os: [ubuntu-latest] - yarn: ['4.0.2', '4.1.0', '4.6.0', '4.12.0', '4.18.0'] + # `10c0/` writer, current, and 4.12.0 on macOS / Windows. The rest + # of the spread (4.6.0, 4.12.0 on ubuntu) is yarn-berry-full. include: + - {os: ubuntu-latest, yarn: '4.0.2'} + - {os: ubuntu-latest, yarn: '4.1.0'} + - {os: ubuntu-latest, yarn: '4.18.0'} - {os: macos-latest, yarn: '4.12.0'} - {os: windows-latest, yarn: '4.12.0'} runs-on: ${{ matrix.os }} timeout-minutes: 30 - steps: + steps: &yarn-berry-steps - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -1504,28 +1638,63 @@ jobs: YARN_BERRY_RELEASE: ${{ matrix.yarn }} run: scripts/yarn-berry-vex-matrix.sh "$YARN_BERRY_RELEASE" + # Skipped on pull_request, like e2e-full. + yarn-berry-full: + name: yarn-berry ${{ matrix.yarn }} (${{ matrix.os }}) + if: github.event_name != 'pull_request' + needs: test + strategy: + fail-fast: false + matrix: + include: + - {os: ubuntu-latest, yarn: '4.6.0'} + - {os: ubuntu-latest, yarn: '4.12.0'} + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + steps: *yarn-berry-steps + + # Every toolchain and every lock re-encoding at least once on PRs (the + # toolchain's own lock is re-encoded as v1-v4 before socket-patch touches + # it, the committed-lockfile case; '' keeps the toolchain's own format), + # plus macOS and Windows; 1.93.1 with its own lock (on every OS) is the + # pinned rust-toolchain.toml cell `cargo test` used to give + # e2e_safety_cargo_build. cargo-vex-matrix-full runs the rest of the + # toolchain x lock cross off pull_request. Each leg also runs + # e2e_safety_cargo_build (its headline test honours the knobs). cargo-vex-matrix: name: cargo ${{ matrix.toolchain }} lock-v${{ matrix.lock || 'own' }} (${{ matrix.os }}) - needs: test + needs: [test, e2e-build] runs-on: ${{ matrix.os }} timeout-minutes: 40 + # What .cargo/config.toml's [env] gives processes cargo launches; these + # legs launch the test binaries themselves. + env: + SOCKET_NO_CONFIG: '1' + SOCKET_NO_UPDATE_CHECK: '1' strategy: fail-fast: false matrix: - # The toolchain's own lock is re-encoded as v1-v4 before - # socket-patch touches it (the committed-lockfile case); '' keeps the - # toolchain's own format. - os: [ubuntu-latest] - toolchain: ['1.82.0', '1.93.1', 'stable'] - lock: ['', '1', '2', '3', '4'] include: + - {os: ubuntu-latest, toolchain: '1.93.1', lock: ''} + - {os: ubuntu-latest, toolchain: '1.93.1', lock: '1'} + - {os: ubuntu-latest, toolchain: stable, lock: '2'} + - {os: ubuntu-latest, toolchain: '1.82.0', lock: '3'} + - {os: ubuntu-latest, toolchain: '1.93.1', lock: '4'} - {os: macos-latest, toolchain: stable, lock: '1'} - {os: windows-latest, toolchain: stable, lock: '1'} - steps: + - {os: macos-latest, toolchain: '1.93.1', lock: ''} + - {os: windows-latest, toolchain: '1.93.1', lock: ''} + steps: &cargo-vex-steps - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + - name: Download the e2e binaries + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: e2e-bin-${{ matrix.os }}* + merge-multiple: true + path: target/e2e-bin - name: Install Rust run: rustup show - name: Install the cargo under test @@ -1536,12 +1705,9 @@ jobs: env: CARGO_TEST_TOOLCHAIN: ${{ matrix.toolchain }} run: rustup toolchain install "$CARGO_TEST_TOOLCHAIN" --profile minimal - - name: Cache cargo - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - with: - key: cargo-vex-${{ matrix.toolchain }} - save-if: ${{ github.ref == 'refs/heads/main' }} - name: Real-cargo hosted/vendored + manifest-less VEX + # The e2e-build binaries, run from the package root as `cargo test` + # would (see the e2e job's staging step). shell: bash env: SOCKET_PATCH_CARGO_E2E_REQUIRED: '1' @@ -1549,8 +1715,48 @@ jobs: SOCKET_PATCH_CARGO_E2E_LOCK_VERSION: ${{ matrix.lock }} run: | set -euo pipefail - cargo test -p socket-patch-cli --test e2e_redirect_cargo_build --test e2e_redirect_cargo_shapes --test e2e_vendor_cargo_build --test mode_migration_cargo - cargo test -p socket-patch-cli --test e2e_safety_cargo_build -- --ignored + exe='' + if [ "$RUNNER_OS" = Windows ]; then exe=.exe; fi + chmod +x target/e2e-bin/* || true + mkdir -p target/debug target/tmp + cp "target/e2e-bin/socket-patch$exe" "target/debug/socket-patch$exe" + cd crates/socket-patch-cli + export CARGO_MANIFEST_DIR="$PWD" + status=0 + for suite in e2e_redirect_cargo_build e2e_redirect_cargo_shapes e2e_vendor_cargo_build mode_migration_cargo; do + echo "::group::$suite" + "../../target/e2e-bin/$suite$exe" || status=1 + echo "::endgroup::" + done + "../../target/e2e-bin/e2e_safety_cargo_build$exe" --ignored || status=1 + exit "$status" + + cargo-vex-matrix-full: + name: cargo ${{ matrix.toolchain }} lock-v${{ matrix.lock || 'own' }} (${{ matrix.os }}) + if: github.event_name != 'pull_request' + needs: [test, e2e-build] + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + # What .cargo/config.toml's [env] gives processes cargo launches; these + # legs launch the test binaries themselves. + env: + SOCKET_NO_CONFIG: '1' + SOCKET_NO_UPDATE_CHECK: '1' + strategy: + fail-fast: false + matrix: + include: + - {os: ubuntu-latest, toolchain: '1.82.0', lock: '1'} + - {os: ubuntu-latest, toolchain: '1.82.0', lock: '2'} + - {os: ubuntu-latest, toolchain: '1.82.0', lock: '4'} + - {os: ubuntu-latest, toolchain: '1.82.0', lock: ''} + - {os: ubuntu-latest, toolchain: '1.93.1', lock: '2'} + - {os: ubuntu-latest, toolchain: '1.93.1', lock: '3'} + - {os: ubuntu-latest, toolchain: stable, lock: ''} + - {os: ubuntu-latest, toolchain: stable, lock: '1'} + - {os: ubuntu-latest, toolchain: stable, lock: '3'} + - {os: ubuntu-latest, toolchain: stable, lock: '4'} + steps: *cargo-vex-steps # Manifest `[patch]` + the tagged detached lock (the v5 vendored cargo # wiring) on cargo 1.41 and 1.56 — below / at the 1.56 floor of diff --git a/.github/workflows/npm-compatibility.yml b/.github/workflows/npm-compatibility.yml index 45ad47596..4aa62e0ff 100644 --- a/.github/workflows/npm-compatibility.yml +++ b/.github/workflows/npm-compatibility.yml @@ -7,7 +7,28 @@ name: npm hosted/vendored compatibility # installs one pinned npm and runs them. See docs/testing/npm-compatibility.md. on: + # PRs: any crate source, but only the test files these capstones + # compile (a later `!` pattern excludes, a later plain one re-includes). + # Main pushes stay unfiltered. pull_request: + paths: + - '.github/actions/upload-artifact/**' + - 'Cargo.lock' + - 'Cargo.toml' + - 'rust-toolchain.toml' + - '.cargo/config.toml' + - '.github/workflows/npm-compatibility.yml' + - 'docs/testing/npm-compatibility.md' + - 'crates/**' + - '!crates/**/*.md' + - '!crates/socket-patch-node/**' + - '!crates/socket-patch-core/tests/**' + - '!crates/socket-patch-cli/tests/**' + - 'crates/socket-patch-cli/tests/vex_e2e_common/**' + - 'crates/socket-patch-cli/tests/e2e_redirect_npm_build.rs' + - 'crates/socket-patch-cli/tests/e2e_vendor_npm_build.rs' + - 'crates/socket-patch-cli/tests/npm_e2e_common/**' + - 'crates/socket-patch-cli/tests/common/cache_env.rs' push: branches: [main] workflow_dispatch: diff --git a/.github/workflows/pdm-compatibility.yml b/.github/workflows/pdm-compatibility.yml index b417cb72c..6af960cea 100644 --- a/.github/workflows/pdm-compatibility.yml +++ b/.github/workflows/pdm-compatibility.yml @@ -1,10 +1,11 @@ name: PDM patch compatibility -# Native PDM installer matrix: builds the CLI once per OS, bootstraps pinned -# PDM releases with uv, and runs `scripts/backtest-pdm.py` — hosted, vendored -# and agent mode against the public urllib3 free patch, verifying the -# INSTALLED bytes, lock/manifest stability, hash rejection and rollback. No -# Socket API token is needed. See docs/testing/pdm-compatibility.md. +# Native PDM installer matrix: builds the CLI and the capstone test binary +# once per OS, bootstraps pinned PDM releases with uv, and runs +# `scripts/backtest-pdm.py` — hosted, vendored and agent mode against the +# public urllib3 free patch, verifying the INSTALLED bytes, lock/manifest +# stability, hash rejection and rollback. No Socket API token is needed. See +# docs/testing/pdm-compatibility.md. on: pull_request: @@ -25,6 +26,8 @@ on: - 'crates/socket-patch-cli/tests/e2e_vex_build/main.rs' - 'crates/socket-patch-cli/tests/e2e_vex_build/pdm.rs' - 'crates/socket-patch-cli/tests/vex_pypi_real_common/**' + # The capstone skips the cells ci.yml's e2e rows run. + - '.github/workflows/ci.yml' push: branches: [main] paths: @@ -60,7 +63,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: @@ -70,13 +73,26 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: key: pdm-compat - - run: cargo build --locked -p socket-patch-cli + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Compile the CLI and the capstone once + run: | + set -euo pipefail + cargo test --locked -p socket-patch-cli --test e2e_vex_build --no-run --message-format=json-render-diagnostics > target-build.json + python3 - <<'PY' + import json, pathlib, shutil + dest = pathlib.Path('target/pdm-e2e') + dest.mkdir(parents=True, exist_ok=True) + shutil.copy2('target/debug/socket-patch', dest / 'socket-patch') + for line in pathlib.Path('target-build.json').read_text().splitlines(): + item = json.loads(line) + if item.get('target', {}).get('name') == 'e2e_vex_build' and item.get('executable'): + shutil.copy2(item['executable'], dest / 'e2e_vex_build') + assert (dest / 'e2e_vex_build').is_file() + PY - uses: ./.github/actions/upload-artifact with: name: pdm-cli-${{ matrix.os }} - path: | - target/debug/socket-patch - target/debug/socket-patch.exe + path: target/pdm-e2e/ if-no-files-found: error retention-days: 7 @@ -86,8 +102,10 @@ jobs: fail-fast: false matrix: # Every stable PDM major family (0.x, 1.x, 2.x) and each 2.x lock-format - # boundary, on Linux and Windows; macOS samples the ends of the range. - os: [ubuntu-latest, windows-latest] + # boundary on Linux; macOS samples the ends of the range. No Windows: + # the harness bootstraps PDM through a POSIX venv layout (bin/pdm), so + # every Windows cell skipped; backtest-pdm.py now fails such a cell. + os: [ubuntu-latest] pdm: ['0.12.3', '1.15.5', '2.0.3', '2.1.5', '2.3.4', '2.6.1', '2.7.4', '2.8.2', '2.9.3', '2.10.4', '2.11.2', '2.17.3', '2.20.1', '2.22.4', '2.25.9', '2.29.2'] include: - { os: macos-latest, pdm: '0.12.3' } @@ -152,25 +170,37 @@ jobs: # The hermetic Rust capstone (wiremock Socket API that also serves the # hosted wheel) over every PDM release the backtest covers: real hosted + # vendored flows ending in the manifest-less VEX matrix; refused lock - # formats (3.1, 4.0-4.2) must attest nothing. + # formats (3.1, 4.0-4.2) must attest nothing. The cells ci.yml's `e2e` + # job runs on every PR and main push are excluded here + # (scripts/tests/test_ci_e2e_tiers.py keeps the two lists in step). capstone: + needs: build strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] pdm: ['0.12.3', '1.0.0', '1.4.5', '1.8.5', '1.15.5', '2.0.3', '2.7.4', '2.8.2', '2.10.4', '2.11.2', '2.17.3', '2.20.1', '2.22.4', '2.25.9', '2.29.2'] + exclude: + - {os: ubuntu-latest, pdm: '1.4.5'} + - {os: ubuntu-latest, pdm: '1.15.5'} + - {os: ubuntu-latest, pdm: '2.7.4'} + - {os: ubuntu-latest, pdm: '2.8.2'} + - {os: ubuntu-latest, pdm: '2.25.9'} + - {os: ubuntu-latest, pdm: '2.29.2'} + - {os: macos-latest, pdm: '2.29.2'} runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: + # The binaries resolve fixtures through the build job's checkout path, + # which is the same on every runner of one OS. - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - key: pdm-vex-capstone - # Only main writes the cache: 30 PR matrix cells saving would churn - # the repo's 10 GiB budget (ci.yml's rust-cache note). - save-if: ${{ github.ref == 'refs/heads/main' }} + pattern: pdm-cli-${{ matrix.os }}* + merge-multiple: true + path: target/pdm-e2e - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.12' @@ -179,4 +209,11 @@ jobs: env: SOCKET_PATCH_PDM_E2E_REQUIRED: '1' SOCKET_PATCH_PDM_E2E_VERSION: ${{ matrix.pdm }} - run: cargo test --locked -p socket-patch-cli --test e2e_vex_build -- 'pdm::' --ignored + run: | + set -euo pipefail + chmod +x target/pdm-e2e/* + mkdir -p target/debug target/tmp + cp target/pdm-e2e/socket-patch target/debug/socket-patch + cd crates/socket-patch-cli + export CARGO_MANIFEST_DIR="$PWD" + ../../target/pdm-e2e/e2e_vex_build 'pdm::' --ignored diff --git a/.github/workflows/pnpm-compatibility.yml b/.github/workflows/pnpm-compatibility.yml index 2aaa8dd9b..11f061728 100644 --- a/.github/workflows/pnpm-compatibility.yml +++ b/.github/workflows/pnpm-compatibility.yml @@ -1,7 +1,26 @@ name: pnpm hosted compatibility on: + # PRs: any crate source, but only the test files these capstones + # compile (a later `!` pattern excludes, a later plain one re-includes). + # Main pushes stay unfiltered. pull_request: + paths: + - '.github/actions/upload-artifact/**' + - 'Cargo.lock' + - 'Cargo.toml' + - 'rust-toolchain.toml' + - '.cargo/config.toml' + - '.github/workflows/pnpm-compatibility.yml' + - 'crates/**' + - '!crates/**/*.md' + - '!crates/socket-patch-node/**' + - '!crates/socket-patch-core/tests/**' + - '!crates/socket-patch-cli/tests/**' + - 'crates/socket-patch-cli/tests/vex_e2e_common/**' + - 'crates/socket-patch-cli/tests/e2e_redirect_pnpm_build.rs' + - 'crates/socket-patch-cli/tests/e2e_vendor_pnpm_build.rs' + - 'crates/socket-patch-cli/tests/common/cache_env.rs' push: branches: [main] workflow_dispatch: diff --git a/.github/workflows/vlt-compatibility.yml b/.github/workflows/vlt-compatibility.yml index 9a90e9ac4..738a2f548 100644 --- a/.github/workflows/vlt-compatibility.yml +++ b/.github/workflows/vlt-compatibility.yml @@ -53,6 +53,9 @@ on: - 'scripts/install-vlt.sh' - 'scripts/vlt-historical-integrity.json' - 'scripts/gen-vlt-collation-golden.mjs' + - 'scripts/ci-vlt-proof-suites.py' + - '.github/workflows/ci.yml' + - 'scripts/tests/test_ci_vlt_rows.py' push: branches: [main] paths: @@ -89,6 +92,9 @@ on: - 'scripts/install-vlt.sh' - 'scripts/vlt-historical-integrity.json' - 'scripts/gen-vlt-collation-golden.mjs' + - 'scripts/ci-vlt-proof-suites.py' + - '.github/workflows/ci.yml' + - 'scripts/tests/test_ci_vlt_rows.py' schedule: # Nightly: vlt releases and the production service drift with no PR open. - cron: '17 4 * * *' @@ -311,6 +317,8 @@ jobs: SOCKET_PATCH_VLT_E2E_STORE_LINKER: ${{ matrix.linker }} SOCKET_PATCH_VLT_E2E_CACHE_ROOT: ${{ matrix.cache_root }} VLT_SUITES: ${{ matrix.suites || 'e2e_redirect_vlt_build e2e_vendor_vlt_build mode_migration_vlt e2e_safety_vlt e2e_vlt' }} + MATRIX_OS: ${{ matrix.os }} + NODE_PIN: ${{ matrix.node }} run: | set -uo pipefail exe='' @@ -326,6 +334,17 @@ jobs: mkdir -p "$SOCKET_PATCH_VLT_E2E_CACHE_ROOT" fi py=$(command -v python3 || command -v python) + # Cells ci.yml's e2e rows run identically (every PR, main push and + # nightly) are left to them; a dispatch runs every cell. + if [ "$GITHUB_EVENT_NAME" != workflow_dispatch ]; then + # shellcheck disable=SC2086 # VLT_SUITES is a word list + VLT_SUITES=$("$py" scripts/ci-vlt-proof-suites.py --os "$MATRIX_OS" --vlt "$SOCKET_PATCH_VLT_E2E_VERSION" \ + --node "$NODE_PIN" --linker "${SOCKET_PATCH_VLT_E2E_STORE_LINKER:-}" \ + --cache-root "${SOCKET_PATCH_VLT_E2E_CACHE_ROOT:-}" $VLT_SUITES | tr -d '\r') || exit 1 + fi + if [ -z "$VLT_SUITES" ]; then + echo "::notice::every capstone of this cell runs in ci.yml's e2e rows; nothing left to run here" + fi status=0 for suite in $VLT_SUITES; do echo "::group::$suite" diff --git a/.github/workflows/vlt-serve-watchdog.yml b/.github/workflows/vlt-serve-watchdog.yml index 28f4af5c7..2bb63e4d7 100644 --- a/.github/workflows/vlt-serve-watchdog.yml +++ b/.github/workflows/vlt-serve-watchdog.yml @@ -12,10 +12,11 @@ name: vlt serve watchdog # Like installer-drift.yml it checks a deployed service, not the diff. It is # `continue-on-error` until the serve fix (`Cache-Control: no-transform`) is # verified in production; removing that line arms it (DESIGN §8.4, the depscan -# rollout's last step). +# rollout's last step). Until then it cannot alert, so it runs once a day to +# record the probe; go back to every 6 hours when arming it. on: schedule: - - cron: '23 */6 * * *' + - cron: '23 4 * * *' workflow_dispatch: permissions: diff --git a/crates/socket-patch-cli/tests/e2e_maven.rs b/crates/socket-patch-cli/tests/e2e_maven.rs index 63f53e27d..006c8a1f6 100644 --- a/crates/socket-patch-cli/tests/e2e_maven.rs +++ b/crates/socket-patch-cli/tests/e2e_maven.rs @@ -10,7 +10,7 @@ //! //! # Running //! ```sh -//! cargo test -p socket-patch-cli --test e2e_maven -- --ignored +//! cargo test -p socket-patch-cli --test e2e_maven //! ``` use std::path::{Path, PathBuf}; @@ -99,7 +99,6 @@ async fn assert_proxy_served_scans(server: &MockServer, scans: usize) { /// Verify that `socket-patch scan` discovers artifacts in a fake Maven local repo. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "opt-in maven crawl e2e; run with --ignored"] async fn scan_discovers_maven_artifacts() { let server = start_proxy().await; let proxy_url = server.uri(); @@ -226,7 +225,6 @@ async fn scan_discovers_maven_artifacts() { /// Verify that `socket-patch scan` discovers Gradle project artifacts. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "opt-in maven crawl e2e; run with --ignored"] async fn scan_discovers_gradle_project_artifacts() { let server = start_proxy().await; let proxy_url = server.uri(); diff --git a/crates/socket-patch-cli/tests/e2e_nuget.rs b/crates/socket-patch-cli/tests/e2e_nuget.rs index 93fc84b7f..e9bd15517 100644 --- a/crates/socket-patch-cli/tests/e2e_nuget.rs +++ b/crates/socket-patch-cli/tests/e2e_nuget.rs @@ -10,7 +10,7 @@ //! //! # Running //! ```sh -//! cargo test -p socket-patch-cli --test e2e_nuget -- --ignored +//! cargo test -p socket-patch-cli --test e2e_nuget //! ``` use std::path::{Path, PathBuf}; @@ -179,7 +179,6 @@ async fn assert_proxy_served_scans(server: &MockServer, scans: usize) { /// Verify that `socket-patch scan` discovers packages in a fake global cache layout. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "opt-in nuget crawl e2e; run with --ignored"] async fn scan_discovers_global_cache_packages() { let server = start_proxy().await; let proxy_url = server.uri(); @@ -247,7 +246,6 @@ async fn scan_discovers_global_cache_packages() { /// Verify that `socket-patch scan` discovers packages in a fake legacy packages/ layout. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "opt-in nuget crawl e2e; run with --ignored"] async fn scan_discovers_legacy_packages() { let server = start_proxy().await; let proxy_url = server.uri(); diff --git a/docs/testing/pdm-compatibility.md b/docs/testing/pdm-compatibility.md index 78a5a8c9d..8090e4bbb 100644 --- a/docs/testing/pdm-compatibility.md +++ b/docs/testing/pdm-compatibility.md @@ -105,8 +105,11 @@ the patched `urllib3/response.py` (git-blob SHA-256 matches the ledger `afterHash`), ordinary `pdm install` keeps the lock stable, a tampered hash is rejected, a relock-then-rescan keeps rollback invertible, and rollback restores the lock and `pyproject.toml` byte for byte. `.github/workflows/pdm-compatibility.yml` -runs it on Linux, Windows and macOS across every PDM major family. The matrix -needs no Socket API token (the `urllib3@1.26.18` patch is a free tier). +runs it on Linux and macOS across every PDM major family. It does not run on +Windows: the harness bootstraps PDM through a POSIX venv layout (`bin/pdm`), so +every Windows cell used to skip, and a run whose cells all skip or whose PDM +bootstrap fails is now an error. The matrix needs no Socket API token (the +`urllib3@1.26.18` patch is a free tier). > **Note (v5.0):** the "refused vendored scan still writes a `.socket/manifest.json` > record" observation in the notes column below describes the 4.0.0 binary the run diff --git a/docs/testing/vlt-compatibility.md b/docs/testing/vlt-compatibility.md index 9778689b0..c5b46d97e 100644 --- a/docs/testing/vlt-compatibility.md +++ b/docs/testing/vlt-compatibility.md @@ -73,7 +73,8 @@ asserted and each named test or row exists. `install-proof` (every capstone on 31 Linux, 11 macOS and 15 Windows releases, the Node floors 22.22.0 / 22.13.0 / 22.7.0 / 22.0.0 with the collation golden, and the store linkers auto / hardlink / copy / unpack / a `/dev/shm` - cache root); `native` (the backtest against production, artifacts + cache root; a cell `ci.yml`'s `e2e` rows run identically is left to them, + see `scripts/ci-vlt-proof-suites.py`, except on dispatch); `native` (the backtest against production, artifacts `vlt-results--` in depscan's capture `result.json` shape); `lock-diff` (the same cell's `vlt-lock.json` must be byte-identical on Linux, macOS and Windows); `matrix-coverage` (every era × suite × OS). diff --git a/docs/testing/yarn-berry-compatibility.md b/docs/testing/yarn-berry-compatibility.md index a968f430c..7c36cb69b 100644 --- a/docs/testing/yarn-berry-compatibility.md +++ b/docs/testing/yarn-berry-compatibility.md @@ -23,6 +23,9 @@ The `yarn-berry-e2e` job in `.github/workflows/ci.yml` runs | macos-latest | 4.12.0 | | windows-latest | 4.12.0 | +Pull requests skip ubuntu 4.6.0 and 4.12.0 (the `yarn-berry-full` job, which +runs on main pushes, nightly and dispatch). + Each release drives four real-yarn suites — `e2e_redirect_yarn_berry_build`, `e2e_vendor_yarn_berry_build`, `e2e_yarn4_pnpm_linker_build` and `e2e_yarn4_workspaces_build` — each ending in the manifest-less VEX matrix of diff --git a/scripts/backtest-pdm.py b/scripts/backtest-pdm.py index c2fda3796..c952e6cb8 100644 --- a/scripts/backtest-pdm.py +++ b/scripts/backtest-pdm.py @@ -1362,7 +1362,7 @@ def uninstall(log): for s in args.shapes: for m in args.modes: if wanted(v, s, m): - results.append({"pdm": v, "python": python_for(v), "shape": s, "mode": m, "outcome": "SKIP", "passed": None, "checks": {}, "info": {"skip": "tool bootstrap failed: " + tool_environments.get(v, {}).get("error", "?")[-300:]}}) + results.append({"pdm": v, "python": python_for(v), "shape": s, "mode": m, "outcome": "ERROR", "passed": False, "checks": {}, "info": {"error": "tool bootstrap failed: " + tool_environments.get(v, {}).get("error", "?")[-300:]}}) jobs = [j for j in jobs if tool_environments.get(j[0], {}).get("ok")] def persist(): @@ -1394,6 +1394,10 @@ def persist(): say(render_matrix(summary)) bad = [r for r in summary["results"] if r["outcome"] in ("FAIL", "ERROR")] say(f"{len(summary['results'])} rows: " + ", ".join(f"{o} {sum(1 for r in summary['results'] if r['outcome'] == o)}" for o in ("PASS", "REFUSED-EXPECTED", "UNSUPPORTED", "SKIP", "FAIL", "ERROR"))) + # A cell whose every row skipped exercised nothing; it must not read as green. + if summary["results"] and all(r["outcome"] == "SKIP" for r in summary["results"]): + say("every row SKIPPED: this cell exercised nothing") + sys.exit(1) if bad or errors: sys.exit(1) diff --git a/scripts/ci-e2e-bundle.py b/scripts/ci-e2e-bundle.py new file mode 100644 index 000000000..8212429a5 --- /dev/null +++ b/scripts/ci-e2e-bundle.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Copy the CLI and the test binaries one OS's CI legs run into one directory. + +ci.yml's e2e-build job compiles every CLI test target once per OS +(`cargo test --no-run --message-format=json`); this picks out the binaries +the `e2e` and `e2e-full` rows name for that OS, plus the suites every +cargo-vex-matrix leg runs, so the legs download them instead of compiling. +""" + +import argparse +import importlib.util +import json +import shutil +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +CI = ROOT / ".github" / "workflows" / "ci.yml" +MATRIX_JOBS = ("e2e", "e2e-full") +# The binaries cargo-vex-matrix and cargo-vex-matrix-full run on every OS. +CARGO_VEX_SUITES = ( + "e2e_redirect_cargo_build", + "e2e_redirect_cargo_shapes", + "e2e_vendor_cargo_build", + "mode_migration_cargo", + "e2e_safety_cargo_build", +) + + +def load_reader(): + path = ROOT / "scripts" / "tests" / "test_ci_vlt_rows.py" + spec = importlib.util.spec_from_file_location("ci_rows", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def suites_for(os_name, text=None): + reader = load_reader() + jobs = reader.jobs(text if text is not None else CI.read_text(encoding="utf-8")) + suites = set(CARGO_VEX_SUITES) + for job in MATRIX_JOBS: + for row in reader.matrix_include(jobs[job]): + if row["os"] == os_name: + suites.add(row["suite"]) + return sorted(suites) + + +def executables(cargo_json): + found = {} + for line in cargo_json.splitlines(): + if not line.startswith("{"): + continue + item = json.loads(line) + target = item.get("target", {}) + if item.get("executable") and item.get("profile", {}).get("test") and "test" in target.get("kind", []): + found[target["name"]] = Path(item["executable"]) + return found + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--os", required=True, help="the runner label the rows use, e.g. ubuntu-latest") + ap.add_argument("--cargo-json", required=True, type=Path) + ap.add_argument("--dest", required=True, type=Path) + args = ap.parse_args(argv) + exe = ".exe" if sys.platform == "win32" else "" + built = executables(args.cargo_json.read_text(encoding="utf-8")) + wanted = suites_for(args.os) + missing = [s for s in wanted if s not in built] + if missing: + print(f"ci-e2e-bundle: no test binary for {missing}", file=sys.stderr) + return 1 + args.dest.mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / "target" / "debug" / f"socket-patch{exe}", args.dest / f"socket-patch{exe}") + for suite in wanted: + shutil.copy2(built[suite], args.dest / f"{suite}{exe}") + print(f"ci-e2e-bundle: {len(wanted)} test binaries for {args.os}: {' '.join(wanted)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci-vlt-proof-suites.py b/scripts/ci-vlt-proof-suites.py new file mode 100644 index 000000000..8a119d680 --- /dev/null +++ b/scripts/ci-vlt-proof-suites.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Print the capstones one vlt-compatibility install-proof row still runs. + +A suite is left out when ci.yml's `e2e` job (which runs on every pull +request, main push and the nightly schedule) has a row for the identical +cell: same suite, OS and vlt release, the default Node, the same store +linker, no cache root and, for mode_migration_vlt (the one suite that reads +it), the same upgrade vlt. Everything else runs here. +""" + +import argparse +import importlib.util +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +CI = ROOT / ".github" / "workflows" / "ci.yml" +UPGRADE_SUITE = "mode_migration_vlt" + + +def load_reader(): + path = ROOT / "scripts" / "tests" / "test_ci_vlt_rows.py" + spec = importlib.util.spec_from_file_location("ci_rows", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def proof_upgrade(vlt, node): + """The upgrade vlt install-proof gives a row (its `Install vlt` step).""" + if node: + return "" + m = re.fullmatch(r"0\.0\.0-(\d+)|1\.0\.0-rc\.(\d+)", vlt) + if m and (int(m.group(1)) >= 19 if m.group(1) else int(m.group(2)) <= 14): + return "1.2.0" + return "" + + +def ci_cells(text=None): + reader = load_reader() + rows = reader.matrix_include(reader.jobs(text if text is not None else CI.read_text(encoding="utf-8"))["e2e"]) + return {(r["suite"], r["os"], r["vlt"], r.get("vlt_store_linker", ""), r.get("vlt_upgrade", "")) + for r in rows if r.get("vlt") and r.get("test_filter") == "--include-ignored vlt_pinned_matrix"} + + +def remaining(suites, os_name, vlt, node="", linker="", cache_root="", text=None): + if node or cache_root: + return list(suites) + cells = ci_cells(text) + upgrade = proof_upgrade(vlt, node) + keep = [] + for suite in suites: + want_upgrade = upgrade if suite == UPGRADE_SUITE else None + covered = any(s == suite and o == os_name and v == vlt and lk == linker + and (want_upgrade is None or up == want_upgrade) + for s, o, v, lk, up in cells) + if not covered: + keep.append(suite) + return keep + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--os", required=True) + ap.add_argument("--vlt", required=True) + ap.add_argument("--node", default="") + ap.add_argument("--linker", default="") + ap.add_argument("--cache-root", default="") + ap.add_argument("suites", nargs="+") + args = ap.parse_args(argv) + keep = remaining(args.suites, args.os, args.vlt, args.node, args.linker, args.cache_root) + for suite in args.suites: + if suite not in keep: + print(f"{suite}: run by ci.yml's e2e row for this cell", file=sys.stderr) + print(" ".join(keep)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_ci_e2e_tiers.py b/scripts/tests/test_ci_e2e_tiers.py new file mode 100644 index 000000000..360dc2ba1 --- /dev/null +++ b/scripts/tests/test_ci_e2e_tiers.py @@ -0,0 +1,178 @@ +"""ci.yml's build-once e2e fan-out and its PR / full tiers: every row names a +real test target that e2e-build bundles, the off-PR tiers only add releases +to suites the PR tier already runs on that OS, and the cargo toolchain x +lock cross is exactly split between the two cargo-vex jobs.""" + +import importlib.util +import itertools +import re +import unittest +from pathlib import Path + +ROOT = Path(__file__).parents[2] +CI = ROOT / ".github" / "workflows" / "ci.yml" +PDM = ROOT / ".github" / "workflows" / "pdm-compatibility.yml" +TESTS = ROOT / "crates" / "socket-patch-cli" / "tests" + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +rows_mod = load("ci_rows", Path(__file__).parent / "test_ci_vlt_rows.py") +bundle = load("ci_e2e_bundle", ROOT / "scripts" / "ci-e2e-bundle.py") +proof = load("ci_vlt_proof_suites", ROOT / "scripts" / "ci-vlt-proof-suites.py") +COMPAT = ROOT / ".github" / "workflows" / "vlt-compatibility.yml" +TEXT = CI.read_text(encoding="utf-8") +JOBS = rows_mod.jobs(TEXT) + + +def rows(job): + return rows_mod.matrix_include(JOBS[job]) + + +def job_text(job): + return "\n".join(JOBS[job]) + + +class Tiers(unittest.TestCase): + def test_every_row_is_a_test_target(self): + for job in ("e2e", "e2e-full"): + for row in rows(job): + with self.subTest(job=job, row=row): + suite = row["suite"] + self.assertTrue((TESTS / f"{suite}.rs").is_file() or (TESTS / suite / "main.rs").is_file(), + f"no test target {suite}") + + def test_full_rows_only_add_releases_to_pr_suites(self): + pr = {(r["suite"], r["os"], r.get("test_filter", "")) for r in rows("e2e")} + for row in rows("e2e-full"): + with self.subTest(row=row): + self.assertIn((row["suite"], row["os"], row.get("test_filter", "")), pr) + pr_rows = [tuple(sorted(r.items())) for r in rows("e2e")] + full_rows = [tuple(sorted(r.items())) for r in rows("e2e-full")] + self.assertFalse(set(pr_rows) & set(full_rows), "a row in both tiers runs twice") + self.assertEqual(len(pr_rows + full_rows), len(set(pr_rows + full_rows)), "duplicate rows") + + def test_full_jobs_skip_pull_requests_and_share_steps(self): + for job, anchor in (("e2e-full", "e2e-steps"), ("yarn-berry-full", "yarn-berry-steps"), + ("cargo-vex-matrix-full", "cargo-vex-steps")): + with self.subTest(job=job): + text = job_text(job) + self.assertIn("if: github.event_name != 'pull_request'", text) + self.assertIn(f"steps: *{anchor}", text) + self.assertIn(f"steps: &{anchor}", TEXT) + + def test_nightly_schedule_runs_the_full_tier(self): + self.assertRegex(TEXT, r"(?m)^ schedule:\n(?: #.*\n)* - cron: '[^']+'$") + self.assertIn("if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' ||", + job_text("e2e-docker")) + + def test_cargo_cross_is_split_exactly(self): + pr = [(r["os"], r["toolchain"], r.get("lock", "")) for r in rows("cargo-vex-matrix")] + full = [(r["os"], r["toolchain"], r.get("lock", "")) for r in rows("cargo-vex-matrix-full")] + want = {("ubuntu-latest", t, l) for t, l in itertools.product(("1.82.0", "1.93.1", "stable"), + ("", "1", "2", "3", "4"))} + want |= {("macos-latest", "stable", "1"), ("windows-latest", "stable", "1"), + ("macos-latest", "1.93.1", ""), ("windows-latest", "1.93.1", "")} + self.assertEqual(len(pr + full), len(want)) + self.assertEqual(set(pr) | set(full), want) + for os_name in ("ubuntu-latest", "macos-latest", "windows-latest"): + self.assertIn((os_name, "1.93.1", ""), pr, "the pinned toolchain's own lock on every OS") + ubuntu = [c for c in pr if c[0] == "ubuntu-latest"] + self.assertEqual({c[1] for c in ubuntu}, {"1.82.0", "1.93.1", "stable"}, "every toolchain on PRs") + self.assertEqual({c[2] for c in ubuntu}, {"", "1", "2", "3", "4"}, "every lock on PRs") + + def test_cargo_vex_runs_the_bundled_suites(self): + text = job_text("cargo-vex-matrix") + ran = set(re.findall(r"\b(e2e_[a-z_]+|mode_migration_[a-z_]+)\b", text.split("Real-cargo hosted")[1])) + self.assertEqual(ran, set(bundle.CARGO_VEX_SUITES)) + + def test_bundle_covers_every_os(self): + for os_name in ("ubuntu-latest", "macos-latest", "windows-latest"): + with self.subTest(os=os_name): + suites = bundle.suites_for(os_name, TEXT) + for job in ("e2e", "e2e-full"): + for row in rows(job): + if row["os"] == os_name: + self.assertIn(row["suite"], suites) + + def test_bundle_reads_cargo_json(self): + json_lines = "\n".join([ + '{"reason":"compiler-artifact","target":{"name":"e2e_vlt","kind":["test"]},' + '"profile":{"test":true},"executable":"/t/deps/e2e_vlt-abc"}', + '{"reason":"compiler-artifact","target":{"name":"socket_patch_cli","kind":["lib"]},' + '"profile":{"test":true},"executable":"/t/deps/socket_patch_cli-abc"}', + '{"reason":"compiler-artifact","target":{"name":"socket-patch","kind":["bin"]},' + '"profile":{"test":false},"executable":"/t/debug/socket-patch"}', + "not json", + ]) + self.assertEqual({k: str(v) for k, v in bundle.executables(json_lines).items()}, + {"e2e_vlt": "/t/deps/e2e_vlt-abc"}) + + +class PdmCapstone(unittest.TestCase): + job = rows_mod.jobs(PDM.read_text(encoding="utf-8"))["capstone"] + + def test_excludes_exactly_the_cells_ci_runs_on_every_pr(self): + excluded = rows_mod.matrix_include([l.replace("exclude:", "include:") for l in self.job]) + ci = {(r["os"], r["pdm"]) for r in rows("e2e") if "pdm" in r} + self.assertEqual({(r["os"], r["pdm"]) for r in excluded}, ci) + self.assertEqual(len(excluded), len(ci)) + for row in rows("e2e"): + if "pdm" in row: + self.assertEqual(row.get("test_filter"), "pdm:: --ignored") + self.assertFalse(any("pdm" in r for r in rows("e2e-full")), + "a pdm row off the PR tier would leave its cell unrun on PRs") + versions = re.search(r"pdm: \[([^\]]*)\]", "\n".join(self.job)).group(1) + for _, version in ci: + self.assertIn(f"'{version}'", versions) + + +class VltProofDedupe(unittest.TestCase): + suites = ["e2e_redirect_vlt_build", "e2e_vendor_vlt_build", "mode_migration_vlt", "e2e_safety_vlt", + "e2e_vlt"] + + def test_only_identical_ci_cells_are_left_out(self): + cells = proof.ci_cells(TEXT) + compat = rows_mod.jobs(COMPAT.read_text(encoding="utf-8")) + for row in rows_mod.matrix_include(compat["install-proof"]): + suites = row.get("suites", " ".join(self.suites)).split() + keep = proof.remaining(suites, row["os"], row["vlt"], row.get("node", ""), row.get("linker", ""), + row.get("cache_root", ""), TEXT) + for suite in set(suites) - set(keep): + with self.subTest(row=row, suite=suite): + self.assertFalse(row.get("node") or row.get("cache_root")) + upgrade = proof.proof_upgrade(row["vlt"], "") if suite == proof.UPGRADE_SUITE else None + self.assertTrue(any(c[:4] == (suite, row["os"], row["vlt"], row.get("linker", "")) + and (upgrade is None or c[4] == upgrade) for c in cells)) + + def test_ci_vlt_rows_match_the_proof_invocation(self): + for row in rows("e2e"): + if row.get("vlt"): + self.assertEqual(row["test_filter"], "--include-ignored vlt_pinned_matrix", row) + ci_node = rows_mod.step(JOBS["e2e"], "Setup Node.js 24 (vlt legs)") + proof_text = "\n".join(rows_mod.jobs(COMPAT.read_text(encoding="utf-8"))["install-proof"]) + node = re.search(r"node-version: '([^']+)'", ci_node).group(1) + self.assertIn(f"node-version: ${{{{ matrix.node || '{node}' }}}}", proof_text) + + def test_upgrade_rule_matches_the_install_step(self): + text = "\n".join(rows_mod.jobs(COMPAT.read_text(encoding="utf-8"))["install-proof"]) + self.assertIn("Number(m[1]) >= 19 : Number(m[2]) <= 14", text) + self.assertIn("scripts/install-vlt.sh 1.2.0", text) + self.assertEqual(proof.proof_upgrade("0.0.0-19", ""), "1.2.0") + self.assertEqual(proof.proof_upgrade("0.0.0-18", ""), "") + self.assertEqual(proof.proof_upgrade("1.0.0-rc.14", ""), "1.2.0") + self.assertEqual(proof.proof_upgrade("1.0.0-rc.15", ""), "") + self.assertEqual(proof.proof_upgrade("1.0.0-rc.14", "22.13.0"), "") + + def test_lv0_mode_migration_without_ci_upgrade_stays(self): + self.assertIn("mode_migration_vlt", proof.remaining(self.suites, "windows-latest", "1.0.0-rc.14", text=TEXT)) + self.assertNotIn("mode_migration_vlt", proof.remaining(self.suites, "ubuntu-latest", "1.0.0-rc.14", text=TEXT)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_ci_vlt_rows.py b/scripts/tests/test_ci_vlt_rows.py index 94ddb8fc2..567900f50 100644 --- a/scripts/tests/test_ci_vlt_rows.py +++ b/scripts/tests/test_ci_vlt_rows.py @@ -200,8 +200,8 @@ def test_rows_pin_supported_releases(self): def test_the_steps_install_vlt_and_check_the_legs(self): e2e = "\n".join(self.ci["e2e"]) - self.assertIn("key: ${{ matrix.suite }}-${{ matrix.vlt || ", e2e, - "vlt releases share a suite, so the release is part of the cache key") + self.assertIn("pattern: e2e-bin-${{ matrix.os }}*", e2e, + "the legs run the binaries e2e-build compiled once per OS") setup = step(self.ci["e2e"], "Setup vlt") self.assertIn("if: matrix.vlt != ''", setup) self.assertIn("scripts/install-vlt.sh", setup) @@ -215,6 +215,8 @@ def test_the_steps_install_vlt_and_check_the_legs(self): self.assertIn("if: matrix.vlt != ''", run) self.assertIn("SOCKET_PATCH_VLT_E2E_REQUIRED: ${{ matrix.vlt != '' && '1' || '' }}", run) self.assertIn("scripts/check-vlt-legs.py", run) + self.assertIn('--binary "$VLT_SUITE"', run, + "a directly run binary prints no cargo `Running` line to name it") self.assertIn("vlt-leg-manifest.json", run) other = step(self.ci["e2e"], "Run e2e tests") self.assertIn("if: matrix.vlt == ''", other) @@ -304,8 +306,9 @@ def test_jobs_and_triggers(self): def test_watchdog(self): text = WATCHDOG.read_text(encoding="utf-8") - self.assertIn("cron: '23 */6 * * *'", text) - self.assertIn("continue-on-error: true", text) + # Daily while disarmed; every 6 hours once continue-on-error goes. + armed = "continue-on-error: true" not in text + self.assertIn("cron: '23 */6 * * *'" if armed else "cron: '23 4 * * *'", text) self.assertIn("scripts/backtest-vlt.py --serve-probe", text) diff --git a/tests/docker/README.md b/tests/docker/README.md index fbf066842..dc2b4c8de 100644 --- a/tests/docker/README.md +++ b/tests/docker/README.md @@ -67,8 +67,9 @@ the real package managers, each in its ecosystem's image: | `docker_e2e_vendor_nuget` | `nuget` | .NET SDK 8.0, Newtonsoft.Json 13.0.3 | | `docker_e2e_vendor_pypi_pm` | `pypi` | poetry, pdm, pipenv on six 1.16.0 | -CI's `e2e-docker` job runs the composer, nuget and pypi_pm capstones; -`coverage-docker` runs all five. Unlike the scan→apply suites they are MULTI-STAGE: a host +CI's `coverage-docker` job runs all five on every push; the nightly +`e2e-docker` job runs the composer, nuget and pypi_pm capstones against +the release binary. Unlike the scan→apply suites they are MULTI-STAGE: a host tempdir is bind-mounted at `/workspace` and shared across three `docker run`s (networked fixture install + offline `socket-patch vendor`; then a fresh-checkout install under `--network none` with cold caches; then