Add a scan benchmark suite and a CI performance gate

.github/workflows/bench.yml

@@ -0,0 +1,191 @@
+name: Benchmarks
+
+# Performance gate for `socket-patch scan` (crates/socket-patch-bench; its
+# README has the details). Builds the pull request's base and head with the
+# same profile on one runner, times both on a synthetic project per package
+# manager against a local patch API (interleaved: base, head, head, base,
+# ...), and fails when the head
+#   - is significantly slower (wall or CPU time; median of the paired
+#     head/base ratios past the threshold with its 95% interval above 1,
+#     confirmed by a second round of pairs),
+#   - uses significantly more memory,
+#   - makes more API requests, or
+#   - no longer does a scenario's work (scan counts, redirected patches,
+#     rewritten files).
+# The job summary carries the full table; results.json is uploaded.
+#
+# An intentional slowdown: label the PR `performance-regression-accepted`.
+# The comparison still runs and reports, but a regression no longer fails
+# the job (a head that fails a scenario's validation still does).
+#
+# On main, each push is compared against the commit before it and recorded
+# (an artifact per push) without failing on regressions.
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled, unlabeled]
+    paths:
+      - '.github/workflows/bench.yml'
+      - '.cargo/**'
+      - 'Cargo.lock'
+      - 'Cargo.toml'
+      - 'rust-toolchain.toml'
+      - 'crates/socket-patch-bench/**'
+      - 'crates/socket-patch-cli/Cargo.toml'
+      - 'crates/socket-patch-cli/build.rs'
+      - 'crates/socket-patch-cli/src/**'
+      - 'crates/socket-patch-core/Cargo.toml'
+      - 'crates/socket-patch-core/src/**'
+  push:
+    branches: [main]
+    paths:
+      - '.github/workflows/bench.yml'
+      - '.cargo/**'
+      - 'Cargo.lock'
+      - 'Cargo.toml'
+      - 'rust-toolchain.toml'
+      - 'crates/socket-patch-bench/**'
+      - 'crates/socket-patch-cli/Cargo.toml'
+      - 'crates/socket-patch-cli/build.rs'
+      - 'crates/socket-patch-cli/src/**'
+      - 'crates/socket-patch-core/Cargo.toml'
+      - 'crates/socket-patch-core/src/**'
+  workflow_dispatch:
+    inputs:
+      base:
+        description: 'Commit or ref to compare HEAD against (default: the parent commit)'
+        required: false
+        default: ''
+      filter:
+        description: 'Only scenarios matching this regex (see `socket-patch-bench list`)'
+        required: false
+        default: ''
+
+permissions:
+  contents: read
+
+jobs:
+  scan:
+    name: scan performance
+    # Labels re-trigger the workflow; only this one changes the outcome.
+    if: >-
+      (github.event.action != 'labeled' && github.event.action != 'unlabeled')
+      || github.event.label.name == 'performance-regression-accepted'
+    # Job-level, so an unrelated label (skipped above) cannot cancel a run
+    # in progress. A newer push to the same PR supersedes the older run.
+    concurrency:
+      group: bench-${{ github.event.pull_request.number || github.ref }}
+      cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      # Both sides build with these, so a base that predates the `perf`
+      # profile (or carries an older copy of it) builds the same way.
+      CARGO_PROFILE_PERF_INHERITS: release
+      CARGO_PROFILE_PERF_LTO: thin
+      CARGO_PROFILE_PERF_STRIP: none
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+          # The parent is the baseline: for a pull request, the merge
+          # commit's first parent is exactly the base it was merged onto.
+          fetch-depth: 2
+
+      - name: Install Rust
+        # rustup is pre-installed; `rustup show` installs the
+        # rust-toolchain.toml channel.
+        run: rustup show
+
+      - name: Cache cargo
+        # Swatinem/rust-cache, main-only saves: see ci.yml.
+        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
+        with:
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+
+      - name: Resolve the baseline commit
+        id: base
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          PUSH_BEFORE: ${{ github.event.before }}
+          INPUT_BASE: ${{ inputs.base }}
+        run: |
+          set -euo pipefail
+          case "$EVENT_NAME" in
+            push) ref="$PUSH_BEFORE" ;;
+            workflow_dispatch) ref="${INPUT_BASE:-HEAD^1}" ;;
+            *) ref="HEAD^1" ;;
+          esac
+          if ! git rev-parse --verify --quiet "$ref^{commit}" >/dev/null; then
+            git fetch --quiet --depth 1 origin "$ref"
+            ref=FETCH_HEAD
+          fi
+          sha="$(git rev-parse "$ref^{commit}")"
+          echo "Baseline: $sha"
+          echo "sha=$sha" >> "$GITHUB_OUTPUT"
+
+      - name: Build head
+        run: |
+          set -euo pipefail
+          BENCH="$RUNNER_TEMP/bench"
+          cargo build --locked --profile perf -p socket-patch-cli -p socket-patch-bench
+          mkdir -p "$BENCH/head"
+          cp target/perf/socket-patch "$BENCH/head/"
+          cp target/perf/socket-patch-bench "$BENCH/"
+
+      - name: Build base
+        env:
+          BASE_SHA: ${{ steps.base.outputs.sha }}
+        run: |
+          set -euo pipefail
+          BENCH="$RUNNER_TEMP/bench"
+          git worktree add --detach "$BENCH/base-src" "$BASE_SHA"
+          # Same target directory: the dependencies are already built, only
+          # the workspace crates compile again (unless the base pins another
+          # toolchain, which `rustup show` installs first).
+          (cd "$BENCH/base-src" && rustup show >/dev/null && \
+            CARGO_TARGET_DIR="$GITHUB_WORKSPACE/target" \
+            cargo build --locked --profile perf -p socket-patch-cli)
+          mkdir -p "$BENCH/base"
+          cp target/perf/socket-patch "$BENCH/base/"
+          "$BENCH/base/socket-patch" --version
+          "$BENCH/head/socket-patch" --version
+
+      - name: Compare
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          ACCEPTED: ${{ contains(github.event.pull_request.labels.*.name, 'performance-regression-accepted') }}
+          FILTER: ${{ inputs.filter }}
+        run: |
+          set -euo pipefail
+          BENCH="$RUNNER_TEMP/bench"
+          args=()
+          if [ "$EVENT_NAME" != pull_request ] || [ "$ACCEPTED" = true ]; then
+            args+=(--no-fail)
+          fi
+          if [ -n "${FILTER:-}" ]; then
+            args+=(--filter "$FILTER")
+          fi
+          "$BENCH/socket-patch-bench" compare \
+            --base "$BENCH/base/socket-patch" \
+            --head "$BENCH/head/socket-patch" \
+            --work-dir "$BENCH/work" \
+            --out "$BENCH/out" \
+            ${args[@]+"${args[@]}"}
+
+      - name: Job summary
+        if: ${{ !cancelled() }}
+        run: |
+          if [ -f "$RUNNER_TEMP/bench/out/summary.md" ]; then
+            cat "$RUNNER_TEMP/bench/out/summary.md" >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Upload results
+        if: ${{ !cancelled() }}
+        uses: ./.github/actions/upload-artifact
+        with:
+          name: bench-scan
+          path: ${{ runner.temp }}/bench/out/
+          if-no-files-found: warn
+          retention-days: 90

Cargo.toml

@@ -3,6 +3,7 @@ members = [
     "crates/socket-patch-core",
     "crates/socket-patch-cli",
     "crates/socket-patch-node",
+    "crates/socket-patch-bench",
 ]
 # Bare `cargo build` (release.yml's per-target builds) skips the Node addon,
 # which no release artifact ships; `--workspace` and `-p` still build it.
@@ -78,6 +79,16 @@ inherits = "release"
 lto = false
 strip = "none" # unstripped test binaries => usable backtraces on failure
 
+# The benchmark gate's build (crates/socket-patch-bench, .github/workflows/
+# bench.yml): the shipped profile's codegen (opt-level = "s", no debug
+# assertions) so timings track what users run, with thin instead of full
+# LTO so CI can afford to build both sides of a comparison. Unstripped, so
+# a profiler can symbolize it.
+[profile.perf]
+inherits = "release"
+lto = "thin"
+strip = "none"
+
 # The Node addon (crates/socket-patch-node) for hosts that load it
 # in-process: release semantics without the slow full-LTO link.
 [profile.addon]

crates/socket-patch-bench/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "socket-patch-bench"
+description = "Benchmark harness for `socket-patch scan`: synthetic projects, a local patch API, and A/B regression checks"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+publish = false
+
+[[bin]]
+name = "socket-patch-bench"
+path = "src/main.rs"
+
+# Only crates the workspace already pins: the harness adds nothing to
+# Cargo.lock that the CLI's own tests do not already pull in.
+[dependencies]
+clap = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+sha2 = { workspace = true }
+sha1 = { workspace = true }
+hex = { workspace = true }
+base64 = { workspace = true }
+tempfile = { workspace = true }
+regex = { workspace = true }
+# uv's hosted rewrite reads the patched wheel's METADATA, so the mock
+# serves real (tiny) wheels.
+zip = { workspace = true }
+
+[target.'cfg(unix)'.dependencies]
+libc = { workspace = true }