test(fspy): benchmark tracking overhead

.github/workflows/ci.yml

@@ -353,6 +353,10 @@ jobs:
           tools: cargo-shear@1.11.1,cargo-autoinherit@0.1.6
           components: clippy rust-docs rustfmt
 
+      # fspy_benchmark has a x86_64-unknown-linux-musl artifact dependency, so
+      # documenting the workspace needs the musl standard library.
+      - run: rustup target add x86_64-unknown-linux-musl
+
       - uses: oxc-project/setup-node@4c588e9266bd930b6ddc34307df0659ed511d187 # v1.3.1
       - run: pnpm exec vp fmt --check
       - run: cargo autoinherit && git diff --exit-code

.github/workflows/fspy-benchmark.yml

@@ -0,0 +1,167 @@
+name: fspy benchmark
+
+permissions: {}
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, reopened, synchronize]
+    # Only changes that can move fspy's cost: fspy and benchmark crates, their
+    # locked dependencies, the toolchain, and this workflow.
+    paths:
+      - 'crates/fspy*/**'
+      - 'Cargo.lock'
+      - 'rust-toolchain.toml'
+      - '.github/workflows/fspy-benchmark.yml'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+# Benchmark results are never compared across runs: runner instances disagree
+# by more than a regression worth catching. Instead, each pull request job
+# builds the benchmark launcher twice — against the fspy under review and
+# against the fspy of the merge base — and the harness interleaves both builds
+# on the same machine. See crates/fspy_benchmark/README.md.
+jobs:
+  benchmark:
+    name: Benchmark (${{ matrix.platform }})
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux
+            os: namespace-profile-linux-x64-default
+            static: true
+          - platform: macos
+            os: namespace-profile-mac-default
+            static: false
+          - platform: windows
+            os: namespace-profile-windows-4c-8g
+            static: false
+    runs-on: ${{ matrix.os }}
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
+      BASE_DIR: ${{ github.workspace }}/../fspy-benchmark-base
+    steps:
+      - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2
+
+      - name: Update Detours submodule
+        if: matrix.platform == 'windows'
+        run: git submodule update --init --recursive
+
+      - uses: oxc-project/setup-rust@3d6fb132fbe7cdcb66bf8ec193911c2945369d12 # v1.0.17
+        with:
+          save-cache: true
+          cache-key: fspy-benchmark-${{ matrix.platform }}
+
+      - name: Install static target
+        if: matrix.static
+        run: rustup target add x86_64-unknown-linux-musl
+
+      - name: Check out merge base
+        if: github.event_name == 'pull_request'
+        run: |
+          # The checkout is the pull request merged into the base branch, so
+          # its first parent is exactly the base tip the merge was built on.
+          git fetch --depth 2 origin "refs/pull/${{ github.event.pull_request.number }}/merge"
+          git worktree add "$BASE_DIR" "$(git rev-parse FETCH_HEAD^1)"
+
+      - name: Update baseline Detours submodule
+        if: github.event_name == 'pull_request' && matrix.platform == 'windows'
+        working-directory: ${{ env.BASE_DIR }}
+        run: git submodule update --init --recursive
+
+      - name: Build baseline launcher
+        if: github.event_name == 'pull_request'
+        working-directory: ${{ env.BASE_DIR }}
+        env:
+          CARGO_TARGET_DIR: ${{ github.workspace }}/target/fspy-base
+        run: |
+          # Both fspy revisions must be measured by identical code, and the
+          # baseline may predate the launcher crate, so overlay the head's
+          # launcher source before building it against the baseline fspy.
+          rm -rf crates/fspy_benchmark_launcher
+          cp -R "$GITHUB_WORKSPACE/crates/fspy_benchmark_launcher" crates/fspy_benchmark_launcher
+          cargo build --release -p fspy_benchmark_launcher
+
+      - name: Run benchmark
+        run: |
+          exe=""
+          [[ "$RUNNER_OS" == "Windows" ]] && exe=.exe
+          args=()
+          if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
+            args=(--base-launcher "$CARGO_TARGET_DIR/fspy-base/release/fspy_benchmark_launcher$exe")
+          fi
+          mkdir -p .benchmark-results
+          set -o pipefail
+          cargo run --release -p fspy_benchmark -- "${args[@]}" |
+            tee ".benchmark-results/${{ matrix.platform }}.txt"
+
+      - name: Add job summary
+        if: ${{ !cancelled() }}
+        run: |
+          {
+            echo '```text'
+            cat ".benchmark-results/${{ matrix.platform }}.txt"
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload benchmark report
+        if: ${{ !cancelled() && github.event_name == 'pull_request' }}
+        # Matrix jobs have isolated filesystems, so persist each report for the
+        # comment job to combine into one sticky comment after all platforms finish.
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: fspy-benchmark-report-${{ matrix.platform }}
+          path: .benchmark-results/${{ matrix.platform }}.txt
+          if-no-files-found: error
+          overwrite: true
+          retention-days: 90
+
+  comment:
+    name: Report benchmark
+    if: >-
+      !cancelled() &&
+      github.event_name == 'pull_request' &&
+      github.event.pull_request.head.repo.full_name == github.repository
+    needs: benchmark
+    runs-on: namespace-profile-linux-x64-default
+    permissions:
+      actions: read
+      pull-requests: write
+    steps:
+      - name: Download benchmark reports
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: fspy-benchmark-report-*
+          path: .benchmark-results
+          merge-multiple: true
+
+      - name: Assemble comment
+        run: |
+          {
+            echo '## fspy benchmark'
+            echo
+            for platform in linux macos windows; do
+              echo "### $platform"
+              echo
+              echo '```text'
+              cat ".benchmark-results/$platform.txt" ||
+                echo "no report; the $platform benchmark job failed before producing one"
+              echo '```'
+              echo
+            done
+          } > .benchmark-results/comment.md
+
+      - name: Update PR comment
+        uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v3.0.4
+        with:
+          header: fspy-benchmark
+          path: .benchmark-results/comment.md

Cargo.toml

@@ -72,6 +72,8 @@ materialized_artifact = { path = "crates/materialized_artifact" }
 materialized_artifact_build = { path = "crates/materialized_artifact_build" }
 flate2 = "1.0.35"
 fspy = { path = "crates/fspy" }
+fspy_benchmark_launcher = { path = "crates/fspy_benchmark_launcher", artifact = "bin" }
+fspy_benchmark_target = { path = "crates/fspy_benchmark_target", artifact = "bin" }
 fspy_detours_sys = { path = "crates/fspy_detours_sys" }
 fspy_preload_unix = { path = "crates/fspy_preload_unix", artifact = "cdylib", target = "target" }
 fspy_preload_windows = { path = "crates/fspy_preload_windows", artifact = "cdylib", target = "target" }
@@ -177,6 +179,7 @@ zstd = "0.13"
 [workspace.metadata.cargo-shear]
 ignored = [
   # These are artifact dependencies. They are not directly `use`d in Rust code.
+  "fspy_benchmark_target",
   "fspy_preload_unix",
   "fspy_preload_windows",
   "vite_task_client_napi",

crates/fspy_benchmark/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "fspy_benchmark"
+version = "0.0.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish = false
+rust-version.workspace = true
+
+[lints]
+workspace = true
+
+[[bin]]
+name = "fspy_benchmark"
+test = false
+doctest = false
+
+[dependencies]
+fspy_benchmark_launcher = { workspace = true }
+fspy_benchmark_target = { workspace = true }
+
+[target.'cfg(all(target_os = "linux", target_arch = "x86_64"))'.dependencies]
+fspy_benchmark_static_target = { path = "../fspy_benchmark_static_target", artifact = "bin", target = "x86_64-unknown-linux-musl" }
+
+[package.metadata.cargo-shear]
+ignored = ["fspy_benchmark_launcher", "fspy_benchmark_target", "fspy_benchmark_static_target"]

crates/fspy_benchmark/README.md

@@ -0,0 +1,59 @@
+# fspy benchmark
+
+Measures what fspy costs a process it tracks, and whether a change to fspy moved that cost. The
+cost is two separable things, and each is measured apart from the other so that neither dilutes it:
+
+- `launch` is the wall clock of a launch that opens nothing, so it prices starting a process under
+  tracking: injection, session setup, and teardown.
+- `access` is how long a batch of opens takes, timed by the thread that runs them, so it prices
+  interception itself rather than the launch around it.
+- `access-tail` is the slow end of the same samples, which moves when accesses start to scatter
+  rather than to cost more.
+- `access-concurrent` is `access` with a second thread intercepting at the same time, each thread
+  opening a path of its own.
+
+Linux measures a dynamically linked target that exercises `LD_PRELOAD` plus a
+`x86_64-unknown-linux-musl` target that exercises seccomp user notification. macOS measures
+`DYLD_INSERT_LIBRARIES` and Windows measures Detours injection.
+
+Run the overhead report locally with:
+
+```sh
+just benchmark-fspy
+```
+
+## How regressions are caught
+
+Comparing a pull request run with a baseline recorded by an earlier run on `main` cannot work,
+and every attempt to make it work failed the same way: two runs land on two runner instances, and
+identically configured instances disagree — on absolute times by tens of percent, and still by up
+to ten percent after normalizing every measurement against an untracked launch on the same
+machine. A threshold above that noise is a threshold above any regression worth catching.
+
+So nothing is ever compared across runs. The pieces are split so that the two fspy revisions being
+compared can run side by side on one machine:
+
+- `fspy_benchmark_target` (and its static twin) is the workload: it opens missing paths from a
+  configurable number of threads and reports how long its batches took on stdout. It does not link
+  fspy.
+- `fspy_benchmark_launcher` runs the target once — tracked through `fspy::Command`, or untracked
+  for context — and prints the launch wall clock plus the target's numbers. It is the only piece
+  that links fspy.
+- `fspy_benchmark` is the harness. CI builds the launcher twice, against the fspy under review and
+  against the merge-base fspy (from the same launcher source, so both revisions are measured by
+  identical code), then the harness launches both builds interleaved, back to back, cycling
+  every ordering. Whatever the runner instance is doing to the numbers, it is doing to both arms.
+
+Each iteration yields the ratio of the head launch to the base launch per metric; the reported
+change is the median of those ratios. An untracked arm runs in the same rotation and prices
+tracking itself — that overhead number is context, not a gate. A row whose median change exceeds
+its threshold fails the job.
+
+Locally there is no second fspy build, so `just benchmark-fspy` reports only the overhead column.
+Point `--base-launcher` at another build of `fspy_benchmark_launcher` to compare two revisions by
+hand.
+
+Because the launcher is compiled against both revisions from the head's source, a pull request
+that changes the parts of the `fspy::Command` API the launcher uses can fail the baseline build.
+Keep the launcher's API surface minimal; if the API must change incompatibly, the benchmark run
+for that one pull request is expected to fail its baseline build.