From 7e752520e950ca4984a83be31e6d2cadf79e1396 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 00:52:31 +0800 Subject: [PATCH 01/35] test(fspy): benchmark tracking overhead Add a single multithreaded open-and-close workload, a Linux static variant, and three-platform Namespace reporting with PR/main baselines. Co-authored-by: GPT-5.6 Codex --- .github/scripts/fspy-benchmark.mjs | 264 ++++++++++++++++++ .github/scripts/fspy-benchmark.test.mjs | 38 +++ .../workflows/fspy-benchmark-fork-report.yml | 88 ++++++ .github/workflows/fspy-benchmark.yml | 203 ++++++++++++++ Cargo.lock | 112 +++++++- Cargo.toml | 3 + crates/fspy_benchmark/Cargo.toml | 36 +++ crates/fspy_benchmark/README.md | 26 ++ crates/fspy_benchmark/benches/fspy.rs | 144 ++++++++++ crates/fspy_benchmark/src/lib.rs | 1 + .../fspy_benchmark_static_target/Cargo.toml | 23 ++ .../fspy_benchmark_static_target/src/main.rs | 1 + crates/fspy_benchmark_target/Cargo.toml | 20 ++ crates/fspy_benchmark_target/src/main.rs | 53 ++++ justfile | 3 + 15 files changed, 1013 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/fspy-benchmark.mjs create mode 100644 .github/scripts/fspy-benchmark.test.mjs create mode 100644 .github/workflows/fspy-benchmark-fork-report.yml create mode 100644 .github/workflows/fspy-benchmark.yml create mode 100644 crates/fspy_benchmark/Cargo.toml create mode 100644 crates/fspy_benchmark/README.md create mode 100644 crates/fspy_benchmark/benches/fspy.rs create mode 100644 crates/fspy_benchmark/src/lib.rs create mode 100644 crates/fspy_benchmark_static_target/Cargo.toml create mode 100644 crates/fspy_benchmark_static_target/src/main.rs create mode 100644 crates/fspy_benchmark_target/Cargo.toml create mode 100644 crates/fspy_benchmark_target/src/main.rs diff --git a/.github/scripts/fspy-benchmark.mjs b/.github/scripts/fspy-benchmark.mjs new file mode 100644 index 000000000..67cf184e2 --- /dev/null +++ b/.github/scripts/fspy-benchmark.mjs @@ -0,0 +1,264 @@ +#!/usr/bin/env node + +import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises'; +import { arch, cpus, platform as osPlatform, release } from 'node:os'; +import { dirname, join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const THREAD_COUNT = 4; +const FILES_PER_THREAD = 2048; + +function parseArgs(args) { + const parsed = new Map(); + for (let index = 0; index < args.length; index += 2) { + const flag = args[index]; + const value = args[index + 1]; + if (!flag?.startsWith('--') || value === undefined) { + throw new Error(`invalid arguments near ${flag ?? ''}`); + } + parsed.set(flag.slice(2), value); + } + return parsed; +} + +function required(args, name) { + const value = args.get(name); + if (value === undefined || value === '') { + throw new Error(`missing --${name}`); + } + return value; +} + +async function githubJson(path) { + const token = requiredEnv('GITHUB_TOKEN'); + const response = await fetch(`https://api.github.com${path}`, { + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'vite-task-fspy-benchmark', + }, + }); + if (!response.ok) { + throw new Error(`GitHub API ${response.status}: ${await response.text()}`); + } + return response.json(); +} + +function requiredEnv(name) { + const value = process.env[name]; + if (!value) throw new Error(`missing ${name}`); + return value; +} + +async function findLatestArtifact(name, currentRunId) { + const repository = requiredEnv('GITHUB_REPOSITORY'); + const response = await githubJson( + `/repos/${repository}/actions/artifacts?name=${encodeURIComponent(name)}&per_page=100`, + ); + return response.artifacts + .filter( + (artifact) => !artifact.expired && String(artifact.workflow_run?.id) !== String(currentRunId), + ) + .sort((left, right) => Date.parse(right.created_at) - Date.parse(left.created_at))[0]; +} + +async function resolveBaseline(args) { + const outputPath = requiredEnv('GITHUB_OUTPUT'); + const platform = required(args, 'platform'); + const context = required(args, 'context'); + const currentRunId = requiredEnv('GITHUB_RUN_ID'); + const currentArtifactName = `fspy-benchmark-${platform}-${context}`; + const candidates = [currentArtifactName]; + if (context !== 'main') candidates.push(`fspy-benchmark-${platform}-main`); + + let artifact; + let kind; + for (const candidate of candidates) { + artifact = await findLatestArtifact(candidate, currentRunId); + if (artifact) { + kind = candidate === currentArtifactName ? 'previous PR run' : 'main'; + break; + } + } + + const outputs = [ + `current-artifact-name=${currentArtifactName}`, + `found=${artifact ? 'true' : 'false'}`, + ]; + if (artifact) { + outputs.push( + `artifact-name=${artifact.name}`, + `run-id=${artifact.workflow_run.id}`, + `kind=${kind}`, + ); + } + await appendFile(outputPath, `${outputs.join('\n')}\n`); +} + +async function readEstimate(criterionRoot, target, mode) { + const path = join(criterionRoot, target, mode, 'new', 'estimates.json'); + const estimates = JSON.parse(await readFile(path, 'utf8')); + return { + meanNs: estimates.mean.point_estimate, + meanLowerNs: estimates.mean.confidence_interval.lower_bound, + meanUpperNs: estimates.mean.confidence_interval.upper_bound, + medianNs: estimates.median.point_estimate, + }; +} + +async function collect(args) { + const criterionRoot = required(args, 'criterion-root'); + const output = required(args, 'output'); + const platform = required(args, 'platform'); + const expectStatic = args.get('expect-static') === 'true'; + const benchmarks = { + 'dynamic/untracked': await readEstimate(criterionRoot, 'dynamic', 'untracked'), + 'dynamic/tracked': await readEstimate(criterionRoot, 'dynamic', 'tracked'), + }; + if (expectStatic) { + benchmarks['static/untracked'] = await readEstimate(criterionRoot, 'static', 'untracked'); + benchmarks['static/tracked'] = await readEstimate(criterionRoot, 'static', 'tracked'); + } + + const result = { + schemaVersion: 1, + platform, + architecture: process.env.RUNNER_ARCH ?? arch(), + os: { + platform: osPlatform(), + release: release(), + cpu: cpus()[0]?.model ?? 'unknown', + }, + runner: args.get('runner') ?? '', + commit: required(args, 'commit'), + runId: required(args, 'run-id'), + runAttempt: args.get('run-attempt') ?? '1', + event: required(args, 'event'), + pullRequest: args.get('pull-request') ?? '', + workload: { + threadCount: THREAD_COUNT, + filesPerThread: FILES_PER_THREAD, + totalOpens: THREAD_COUNT * FILES_PER_THREAD, + }, + benchmarks, + }; + + await mkdir(dirname(output), { recursive: true }); + await writeFile(output, `${JSON.stringify(result, null, 2)}\n`); +} + +function ratio(result, target) { + return ( + result.benchmarks[`${target}/tracked`].meanNs / result.benchmarks[`${target}/untracked`].meanNs + ); +} + +function percentage(value) { + const sign = value > 0 ? '+' : ''; + return `${sign}${(value * 100).toFixed(2)}%`; +} + +function duration(nanoseconds) { + if (nanoseconds >= 1e9) return `${(nanoseconds / 1e9).toFixed(3)} s`; + if (nanoseconds >= 1e6) return `${(nanoseconds / 1e6).toFixed(3)} ms`; + if (nanoseconds >= 1e3) return `${(nanoseconds / 1e3).toFixed(3)} µs`; + return `${nanoseconds.toFixed(1)} ns`; +} + +function resultLink(result) { + const repository = process.env.GITHUB_REPOSITORY; + if (!repository || !result.runId) return ''; + return `https://github.com/${repository}/actions/runs/${result.runId}`; +} + +export function renderReport(current, baseline, baselineKind = '') { + const compatible = + baseline && + current.platform === baseline.platform && + current.architecture === baseline.architecture; + const lines = [`### ${current.platform[0].toUpperCase()}${current.platform.slice(1)}`, '']; + + if (compatible) { + const link = resultLink(baseline); + const description = link + ? `[${baselineKind || 'baseline'}](${link})` + : baselineKind || 'baseline'; + lines.push(`Compared with ${description} at \`${baseline.commit.slice(0, 8)}\`.`, ''); + } else if (baseline) { + lines.push( + `No comparison: the baseline architecture is \`${baseline.architecture}\`, current is \`${current.architecture}\`.`, + '', + ); + } else { + lines.push('No previous result was available for this runner.', ''); + } + + lines.push( + '| Target | Untracked | Tracked | fspy overhead | Normalized change |', + '| --- | ---: | ---: | ---: | ---: |', + ); + + const targets = Object.hasOwn(current.benchmarks, 'static/tracked') + ? ['dynamic', 'static'] + : ['dynamic']; + for (const target of targets) { + const untracked = current.benchmarks[`${target}/untracked`].meanNs; + const tracked = current.benchmarks[`${target}/tracked`].meanNs; + const overhead = ratio(current, target) - 1; + const normalizedChange = + compatible && Object.hasOwn(baseline.benchmarks, `${target}/tracked`) + ? ratio(current, target) / ratio(baseline, target) - 1 + : undefined; + lines.push( + `| ${target} | ${duration(untracked)} | ${duration(tracked)} | ${percentage(overhead)} | ${normalizedChange === undefined ? '—' : percentage(normalizedChange)} |`, + ); + } + + lines.push( + '', + `Workload: ${current.workload.threadCount} threads, ${current.workload.filesPerThread.toLocaleString('en-US')} files per thread, ${current.workload.totalOpens.toLocaleString('en-US')} total open-and-close operations.`, + '', + `\`${current.architecture}\` · ${current.os.cpu} · run [${current.runId}](${resultLink(current)})`, + '', + ); + return `${lines.join('\n')}\n`; +} + +async function compare(args) { + const current = JSON.parse(await readFile(required(args, 'current'), 'utf8')); + const baselinePath = args.get('baseline'); + let baseline; + if (baselinePath) { + try { + baseline = JSON.parse(await readFile(baselinePath, 'utf8')); + } catch (error) { + if (error.code !== 'ENOENT') throw error; + } + } + const report = renderReport(current, baseline, args.get('baseline-kind') ?? ''); + await writeFile(required(args, 'output'), report); + process.stdout.write(report); +} + +async function main() { + const command = process.argv[2]; + const args = parseArgs(process.argv.slice(3)); + if (command === 'resolve-baseline') { + await resolveBaseline(args); + } else if (command === 'collect') { + await collect(args); + } else if (command === 'compare') { + await compare(args); + } else { + throw new Error(`unknown command: ${command ?? ''}`); + } +} + +const invokedPath = process.argv[1] ? pathToFileURL(process.argv[1]).href : ''; +if (import.meta.url === invokedPath) { + main().catch((error) => { + process.stderr.write(`${error.stack ?? error}\n`); + process.exitCode = 1; + }); +} diff --git a/.github/scripts/fspy-benchmark.test.mjs b/.github/scripts/fspy-benchmark.test.mjs new file mode 100644 index 000000000..d219767dc --- /dev/null +++ b/.github/scripts/fspy-benchmark.test.mjs @@ -0,0 +1,38 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { renderReport } from './fspy-benchmark.mjs'; + +function result(platform, architecture, runId, untracked, tracked) { + return { + platform, + architecture, + os: { cpu: 'Test CPU' }, + commit: '1234567890abcdef', + runId, + workload: { threadCount: 4, filesPerThread: 2048, totalOpens: 8192 }, + benchmarks: { + 'dynamic/untracked': { meanNs: untracked }, + 'dynamic/tracked': { meanNs: tracked }, + }, + }; +} + +test('renders normalized change against a compatible baseline', () => { + const baseline = result('linux', 'X64', '10', 100, 120); + const current = result('linux', 'X64', '20', 100, 132); + const report = renderReport(current, baseline, 'previous PR run'); + + assert.match(report, /fspy overhead \| Normalized change/); + assert.match(report, /\+32\.00% \| \+10\.00%/); + assert.match(report, /previous PR run/); +}); + +test('does not compare different architectures', () => { + const baseline = result('macos', 'X64', '10', 100, 120); + const current = result('macos', 'ARM64', '20', 100, 132); + const report = renderReport(current, baseline, 'main'); + + assert.match(report, /No comparison/); + assert.match(report, /\+32\.00% \| —/); +}); diff --git a/.github/workflows/fspy-benchmark-fork-report.yml b/.github/workflows/fspy-benchmark-fork-report.yml new file mode 100644 index 000000000..4127d262b --- /dev/null +++ b/.github/workflows/fspy-benchmark-fork-report.yml @@ -0,0 +1,88 @@ +name: Fspy Benchmark Fork Report + +permissions: {} + +on: + workflow_run: + workflows: [Fspy Benchmark] + types: [completed] + +jobs: + comment: + name: Report fork benchmark + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.head_repository.full_name != github.repository && + github.event.workflow_run.pull_requests[0].number != null + runs-on: namespace-profile-linux-x64-default + permissions: + actions: read + contents: read + issues: write + steps: + - name: Download benchmark results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: fspy-benchmark-*-pr-${{ github.event.workflow_run.pull_requests[0].number }} + path: .benchmark-results + merge-multiple: true + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Build PR comment + env: + BENCHMARK_SHA: ${{ github.event.workflow_run.head_sha }} + run: | + { + echo '' + echo '## fspy benchmark' + echo + echo "Results for commit \`$BENCHMARK_SHA\`. Each platform uses its latest result from this PR, falling back to \`main\`." + echo + for platform in linux macos windows; do + cat ".benchmark-results/$platform.md" + done + echo + echo 'This comment is updated by the Fspy Benchmark workflow.' + } > benchmark-comment.md + + - name: Update PR comment + env: + GITHUB_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} + run: | + node --input-type=module <<'EOF' + import { readFile } from "node:fs/promises"; + + const marker = ""; + const repository = process.env.GITHUB_REPOSITORY; + const token = process.env.GITHUB_TOKEN; + const pullRequest = process.env.PR_NUMBER; + const headers = { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + }; + const commentsResponse = await fetch( + `https://api.github.com/repos/${repository}/issues/${pullRequest}/comments?per_page=100`, + { headers }, + ); + if (!commentsResponse.ok) throw new Error(await commentsResponse.text()); + const comments = await commentsResponse.json(); + const existing = comments.find( + (comment) => comment.user?.type === "Bot" && comment.body?.includes(marker), + ); + const body = await readFile("benchmark-comment.md", "utf8"); + const url = existing + ? `https://api.github.com/repos/${repository}/issues/comments/${existing.id}` + : `https://api.github.com/repos/${repository}/issues/${pullRequest}/comments`; + const response = await fetch(url, { + method: existing ? "PATCH" : "POST", + headers, + body: JSON.stringify({ body }), + }); + if (!response.ok) throw new Error(await response.text()); + EOF diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml new file mode 100644 index 000000000..b475bd886 --- /dev/null +++ b/.github/workflows/fspy-benchmark.yml @@ -0,0 +1,203 @@ +name: Fspy Benchmark + +permissions: {} + +on: + workflow_dispatch: + pull_request: + types: [opened, reopened, synchronize] + push: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.ref_name != 'main' }} + +defaults: + run: + shell: bash + +jobs: + benchmark: + name: Benchmark (${{ matrix.platform }}) + permissions: + actions: read + contents: read + strategy: + fail-fast: false + matrix: + include: + - platform: linux + os: namespace-profile-linux-x64-default + static: true + - platform: macos + os: namespace-profile-mac-default + static: false + - platform: windows + os: namespace-profile-windows-4c-8g + static: false + runs-on: ${{ matrix.os }} + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + steps: + - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 + + - name: Update Detours submodule + if: matrix.platform == 'windows' + run: git submodule update --init --recursive + + - uses: oxc-project/setup-rust@3d6fb132fbe7cdcb66bf8ec193911c2945369d12 # v1.0.17 + with: + save-cache: ${{ github.ref_name == 'main' }} + cache-key: fspy-benchmark-${{ matrix.platform }} + + - name: Install static target + if: matrix.static + run: rustup target add x86_64-unknown-linux-musl + + - name: Set benchmark context + id: context + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then + context="pr-$PR_NUMBER" + pull_request="$PR_NUMBER" + else + context="main" + pull_request="none" + fi + echo "name=$context" >> "$GITHUB_OUTPUT" + echo "pull-request=$pull_request" >> "$GITHUB_OUTPUT" + + - name: Find comparison baseline + id: baseline + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node .github/scripts/fspy-benchmark.mjs resolve-baseline + --platform "${{ matrix.platform }}" + --context "${{ steps.context.outputs.name }}" + + - name: Download comparison baseline + if: steps.baseline.outputs.found == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ steps.baseline.outputs.artifact-name }} + path: .benchmark-baseline + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ steps.baseline.outputs.run-id }} + + - name: Run benchmark + run: cargo bench --locked -p fspy_benchmark --bench fspy + + - name: Collect benchmark results + run: >- + node .github/scripts/fspy-benchmark.mjs collect + --criterion-root "$CARGO_TARGET_DIR/criterion/fspy" + --output ".benchmark-results/${{ matrix.platform }}.json" + --platform "${{ matrix.platform }}" + --expect-static "${{ matrix.static }}" + --runner "${{ matrix.os }}" + --commit "$GITHUB_SHA" + --run-id "$GITHUB_RUN_ID" + --run-attempt "$GITHUB_RUN_ATTEMPT" + --event "$GITHUB_EVENT_NAME" + --pull-request "${{ steps.context.outputs.pull-request }}" + + - name: Compare benchmark results + run: >- + node .github/scripts/fspy-benchmark.mjs compare + --current ".benchmark-results/${{ matrix.platform }}.json" + --baseline ".benchmark-baseline/${{ matrix.platform }}.json" + --baseline-kind "${{ steps.baseline.outputs.kind }}" + --output ".benchmark-results/${{ matrix.platform }}.md" + + - name: Add job summary + run: cat ".benchmark-results/${{ matrix.platform }}.md" >> "$GITHUB_STEP_SUMMARY" + + - name: Upload benchmark result + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.baseline.outputs.current-artifact-name }} + path: .benchmark-results/${{ matrix.platform }}.* + if-no-files-found: error + overwrite: true + retention-days: 90 + + comment: + name: Report benchmark + if: >- + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + needs: benchmark + runs-on: namespace-profile-linux-x64-default + permissions: + actions: read + contents: read + issues: write + steps: + - name: Download current results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: fspy-benchmark-*-pr-${{ github.event.pull_request.number }} + path: .benchmark-results + merge-multiple: true + + - name: Build PR comment + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + { + echo '' + echo '## fspy benchmark' + echo + echo "Results for commit \`$GITHUB_SHA\`. Each platform uses its latest result from this PR, falling back to \`main\`." + echo + for platform in linux macos windows; do + cat ".benchmark-results/$platform.md" + done + echo + echo 'This comment is updated by the Fspy Benchmark workflow.' + } > benchmark-comment.md + + - name: Update PR comment + env: + GITHUB_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + node --input-type=module <<'EOF' + import { readFile } from "node:fs/promises"; + + const marker = ""; + const repository = process.env.GITHUB_REPOSITORY; + const token = process.env.GITHUB_TOKEN; + const pullRequest = process.env.PR_NUMBER; + const headers = { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + }; + const commentsResponse = await fetch( + `https://api.github.com/repos/${repository}/issues/${pullRequest}/comments?per_page=100`, + { headers }, + ); + if (!commentsResponse.ok) throw new Error(await commentsResponse.text()); + const comments = await commentsResponse.json(); + const existing = comments.find( + (comment) => comment.user?.type === "Bot" && comment.body?.includes(marker), + ); + const body = await readFile("benchmark-comment.md", "utf8"); + const url = existing + ? `https://api.github.com/repos/${repository}/issues/comments/${existing.id}` + : `https://api.github.com/repos/${repository}/issues/${pullRequest}/comments`; + const response = await fetch(url, { + method: existing ? "PATCH" : "POST", + headers, + body: JSON.stringify({ body }), + }); + if (!response.ok) throw new Error(await response.text()); + EOF diff --git a/Cargo.lock b/Cargo.lock index fd72b2ce0..f5a4464cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -317,6 +317,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "bpaf" +version = "0.9.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b86829876e7e200161a5aa6ea688d46c32d64d70ee3100127790dde84688d6e" + [[package]] name = "brush-parser" version = "0.4.0" @@ -411,6 +417,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ade8366b8bd5ba243f0a58f036cc0ca8a2f069cff1a2351ef1cac6b083e16fc0" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "castaway" version = "0.2.4" @@ -459,6 +471,33 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clang-sys" version = "1.8.1" @@ -670,6 +709,21 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion2" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b961b72d7eaf7444d1eb98d353d5c2aa08e2443d77fef6bff0e6e97064162482" +dependencies = [ + "bpaf", + "cast", + "ciborium", + "num-traits", + "oorandom", + "serde", + "serde_json", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -723,6 +777,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -1239,6 +1299,37 @@ dependencies = [ "winsafe 0.0.27", ] +[[package]] +name = "fspy_benchmark" +version = "0.0.0" +dependencies = [ + "criterion2", + "fspy", + "fspy_benchmark_static_target", + "fspy_benchmark_target", + "tempfile", + "tokio", + "tokio-util", + "vite_path", + "vite_str", +] + +[[package]] +name = "fspy_benchmark_static_target" +version = "0.0.0" +dependencies = [ + "vite_path", + "vite_str", +] + +[[package]] +name = "fspy_benchmark_target" +version = "0.0.0" +dependencies = [ + "vite_path", + "vite_str", +] + [[package]] name = "fspy_detours_sys" version = "0.0.0" @@ -1537,6 +1628,17 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -2394,6 +2496,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "option-ext" version = "0.2.0" @@ -3251,9 +3359,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap", "itoa", diff --git a/Cargo.toml b/Cargo.toml index 804541230..818b45454 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,7 @@ crossterm = { version = "0.29.0", features = ["event-stream"] } csv-async = { version = "1.3.1", features = ["tokio"] } ctor = "1.0" ctrlc = "3.5.2" +criterion2 = { version = "3.0.4", default-features = false } derive_more = "2.0.1" diff-struct = "0.5.3" directories = "6.0.0" @@ -72,6 +73,7 @@ materialized_artifact = { path = "crates/materialized_artifact" } materialized_artifact_build = { path = "crates/materialized_artifact_build" } flate2 = "1.0.35" fspy = { path = "crates/fspy" } +fspy_benchmark_target = { path = "crates/fspy_benchmark_target", artifact = "bin" } fspy_detours_sys = { path = "crates/fspy_detours_sys" } fspy_preload_unix = { path = "crates/fspy_preload_unix", artifact = "cdylib", target = "target" } fspy_preload_windows = { path = "crates/fspy_preload_windows", artifact = "cdylib", target = "target" } @@ -177,6 +179,7 @@ zstd = "0.13" [workspace.metadata.cargo-shear] ignored = [ # These are artifact dependencies. They are not directly `use`d in Rust code. + "fspy_benchmark_target", "fspy_preload_unix", "fspy_preload_windows", "vite_task_client_napi", diff --git a/crates/fspy_benchmark/Cargo.toml b/crates/fspy_benchmark/Cargo.toml new file mode 100644 index 000000000..18330d1b1 --- /dev/null +++ b/crates/fspy_benchmark/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "fspy_benchmark" +version = "0.0.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +publish = false +rust-version.workspace = true + +[lints] +workspace = true + +[lib] +test = false +bench = false +doctest = false + +[[bench]] +name = "fspy" +harness = false + +[dev-dependencies] +criterion2 = { workspace = true } +fspy = { workspace = true } +fspy_benchmark_target = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "process", "rt-multi-thread"] } +tokio-util = { workspace = true } +vite_path = { workspace = true } +vite_str = { workspace = true } + +[target.'cfg(all(target_os = "linux", target_arch = "x86_64"))'.dev-dependencies] +fspy_benchmark_static_target = { path = "../fspy_benchmark_static_target", artifact = "bin", target = "x86_64-unknown-linux-musl" } + +[package.metadata.cargo-shear] +ignored = ["fspy_benchmark_target", "fspy_benchmark_static_target"] diff --git a/crates/fspy_benchmark/README.md b/crates/fspy_benchmark/README.md new file mode 100644 index 000000000..fcda70746 --- /dev/null +++ b/crates/fspy_benchmark/README.md @@ -0,0 +1,26 @@ +# fspy benchmark + +This benchmark measures the wall-clock overhead fspy adds to a filesystem-heavy child process. +The target starts four threads. Each thread opens 2,048 distinct, pre-created files for reading +and drops every handle immediately. + +The fixture is created before Criterion starts measuring. Tracked and untracked runs execute the +same target with the same arguments, empty environment, working directory, and standard streams. +The timer covers spawning the target through process termination and fspy finalization. A tracked +warmup verifies that all fixture accesses were captured outside the timed section. + +On Linux, the suite measures two targets: + +- `dynamic` is dynamically linked and exercises `LD_PRELOAD`. +- `static` is linked for `x86_64-unknown-linux-musl` and exercises seccomp user notification. + +macOS measures `DYLD_INSERT_LIBRARIES`, and Windows measures Detours injection. + +Run the benchmark with: + +```sh +just benchmark-fspy +``` + +CI runs it on the Linux, macOS, and Windows Namespace runner profiles. Each result is compared with +the latest result from the same pull request, falling back to the latest `main` result. diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs new file mode 100644 index 000000000..91dbc11b3 --- /dev/null +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -0,0 +1,144 @@ +use std::{ + ffi::OsStr, + fs::File, + process::{ExitStatus, Stdio}, + time::Duration, +}; + +use criterion::{ + BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, criterion_group, criterion_main, + measurement::WallTime, +}; +use fspy::{ChildTermination, Command}; +use tempfile::TempDir; +use tokio::runtime::{Builder, Runtime}; +use tokio_util::sync::CancellationToken; +use vite_path::{AbsolutePath, AbsolutePathBuf}; + +const THREAD_COUNT: usize = 4; +const FILES_PER_THREAD: usize = 2048; +const TOTAL_FILE_COUNT: usize = THREAD_COUNT * FILES_PER_THREAD; +const THREAD_COUNT_ARG: &str = "4"; +const FILES_PER_THREAD_ARG: &str = "2048"; +const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); + +fn benchmark(criterion: &mut Criterion) { + let (_fixture, fixture_root) = create_fixture(); + let runtime = Builder::new_multi_thread().worker_threads(2).enable_all().build().unwrap(); + let mut group = criterion.benchmark_group("fspy"); + group.sampling_mode(SamplingMode::Flat); + + benchmark_target(&mut group, &runtime, "dynamic", DYNAMIC_TARGET, &fixture_root); + + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + benchmark_target(&mut group, &runtime, "static", STATIC_TARGET, &fixture_root); + + group.finish(); +} + +fn benchmark_target( + group: &mut BenchmarkGroup<'_, WallTime>, + runtime: &Runtime, + target_name: &str, + target: &str, + fixture: &AbsolutePath, +) { + validate_tracked_run(runtime, target, fixture); + + group.bench_with_input( + BenchmarkId::new(target_name, "untracked"), + &target, + |bencher, target| { + bencher.iter_with_setup_wrapper(|runner| { + let status = runner.run(|| runtime.block_on(run_untracked(target, fixture))); + assert!(status.success(), "untracked benchmark target failed: {status}"); + }); + }, + ); + + group.bench_with_input(BenchmarkId::new(target_name, "tracked"), &target, |bencher, target| { + bencher.iter_with_setup_wrapper(|runner| { + let termination = runner.run(|| runtime.block_on(run_tracked(target, fixture))); + assert!( + termination.status.success(), + "tracked benchmark target failed: {}", + termination.status + ); + }); + }); +} + +fn create_fixture() -> (TempDir, AbsolutePathBuf) { + let fixture = tempfile::tempdir().expect("failed to create benchmark fixture"); + let fixture_root = AbsolutePathBuf::new(fixture.path().to_path_buf()) + .expect("temporary path must be absolute"); + for index in 0..TOTAL_FILE_COUNT { + let path = fixture_root.join(vite_str::format!("file-{index:05}.txt")); + File::create(&path).unwrap_or_else(|error| { + panic!("failed to create {}: {error}", path.as_absolute_path()) + }); + } + (fixture, fixture_root) +} + +fn validate_tracked_run(runtime: &Runtime, target: &str, fixture: &AbsolutePath) { + let termination = runtime.block_on(run_tracked(target, fixture)); + assert!(termination.status.success(), "benchmark target failed: {}", termination.status); + let fixture_access_count = termination + .path_accesses + .iter() + .filter(|access| access.path.strip_path_prefix(fixture, |result| result.is_ok())) + .count(); + assert!( + fixture_access_count >= TOTAL_FILE_COUNT, + "expected at least {TOTAL_FILE_COUNT} fixture accesses, captured {fixture_access_count}" + ); +} + +async fn run_untracked(target: &str, fixture: &AbsolutePath) -> ExitStatus { + let mut command = tokio::process::Command::new(target); + command + .env_clear() + .args(benchmark_args(fixture)) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + command.status().await.expect("failed to run untracked benchmark target") +} + +async fn run_tracked(target: &str, fixture: &AbsolutePath) -> ChildTermination { + let mut command = Command::new(target); + command + .args(benchmark_args(fixture)) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + command + .spawn(CancellationToken::new()) + .await + .expect("failed to spawn tracked benchmark target") + .wait_handle + .await + .expect("failed to wait for tracked benchmark target") +} + +fn benchmark_args(fixture: &AbsolutePath) -> [&OsStr; 3] { + [fixture.as_path().as_os_str(), OsStr::new(THREAD_COUNT_ARG), OsStr::new(FILES_PER_THREAD_ARG)] +} + +fn criterion_config() -> Criterion { + Criterion::default() + .sample_size(20) + .warm_up_time(Duration::from_secs(2)) + .measurement_time(Duration::from_secs(10)) +} + +criterion_group! { + name = benches; + config = criterion_config(); + targets = benchmark +} +criterion_main!(benches); diff --git a/crates/fspy_benchmark/src/lib.rs b/crates/fspy_benchmark/src/lib.rs new file mode 100644 index 000000000..6d118a163 --- /dev/null +++ b/crates/fspy_benchmark/src/lib.rs @@ -0,0 +1 @@ +//! Benchmarks for the overhead fspy adds to tracked processes. diff --git a/crates/fspy_benchmark_static_target/Cargo.toml b/crates/fspy_benchmark_static_target/Cargo.toml new file mode 100644 index 000000000..b296739b1 --- /dev/null +++ b/crates/fspy_benchmark_static_target/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "fspy_benchmark_static_target" +version = "0.0.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +publish = false +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +vite_path = { workspace = true } +vite_str = { workspace = true } + +[package.metadata.cargo-shear] +ignored = ["vite_path", "vite_str"] + +[[bin]] +name = "fspy_benchmark_static_target" +test = false +doctest = false diff --git a/crates/fspy_benchmark_static_target/src/main.rs b/crates/fspy_benchmark_static_target/src/main.rs new file mode 100644 index 000000000..6e96ceda0 --- /dev/null +++ b/crates/fspy_benchmark_static_target/src/main.rs @@ -0,0 +1 @@ +include!("../../fspy_benchmark_target/src/main.rs"); diff --git a/crates/fspy_benchmark_target/Cargo.toml b/crates/fspy_benchmark_target/Cargo.toml new file mode 100644 index 000000000..c5d917b47 --- /dev/null +++ b/crates/fspy_benchmark_target/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "fspy_benchmark_target" +version = "0.0.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +publish = false +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +vite_path = { workspace = true } +vite_str = { workspace = true } + +[[bin]] +name = "fspy_benchmark_target" +test = false +doctest = false diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs new file mode 100644 index 000000000..7bc6b50a7 --- /dev/null +++ b/crates/fspy_benchmark_target/src/main.rs @@ -0,0 +1,53 @@ +use std::{ + env, + fs::File, + sync::{Arc, Barrier}, + thread, +}; + +use vite_path::{AbsolutePath, AbsolutePathBuf}; + +fn main() { + let mut args = env::args_os(); + let _program = args.next(); + let root = AbsolutePathBuf::new(args.next().expect("missing fixture root").into()) + .expect("fixture root must be absolute"); + let thread_count = parse_usize(args.next(), "thread count"); + let files_per_thread = parse_usize(args.next(), "files per thread"); + assert!(args.next().is_none(), "unexpected extra arguments"); + + run(&root, thread_count, files_per_thread); +} + +fn parse_usize(value: Option, name: &str) -> usize { + value + .unwrap_or_else(|| panic!("missing {name}")) + .into_string() + .unwrap_or_else(|_| panic!("{name} is not UTF-8")) + .parse() + .unwrap_or_else(|_| panic!("invalid {name}")) +} + +fn run(root: &AbsolutePath, thread_count: usize, files_per_thread: usize) { + assert!(thread_count > 1, "benchmark requires multiple threads"); + assert!(files_per_thread > 0, "benchmark requires at least one file per thread"); + + let paths = (0..thread_count * files_per_thread) + .map(|index| root.join(vite_str::format!("file-{index:05}.txt"))) + .collect::>(); + let barrier = Arc::new(Barrier::new(thread_count)); + + thread::scope(|scope| { + for thread_paths in paths.chunks_exact(files_per_thread) { + let barrier = Arc::clone(&barrier); + scope.spawn(move || { + barrier.wait(); + for path in thread_paths { + drop(File::open(path).unwrap_or_else(|error| { + panic!("failed to open {}: {error}", path.as_absolute_path()) + })); + } + }); + } + }); +} diff --git a/justfile b/justfile index f4b8b865d..bd2d114f2 100644 --- a/justfile +++ b/justfile @@ -47,6 +47,9 @@ lint-linux: lint-windows: cargo-xwin clippy --workspace --all-targets --all-features --target x86_64-pc-windows-msvc -- --deny warnings +benchmark-fspy: + cargo bench -p fspy_benchmark --bench fspy + [unix] doc: RUSTDOCFLAGS='-D warnings' cargo doc --no-deps --document-private-items From d960ba4cefad4504137017172593be2e1306472e Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 00:57:21 +0800 Subject: [PATCH 02/35] ci: document trusted benchmark reporter Make the reporter shell explicit and document why its text-only workflow_run boundary is intentionally safe. Co-authored-by: GPT-5.6 Codex --- .github/workflows/fspy-benchmark-fork-report.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fspy-benchmark-fork-report.yml b/.github/workflows/fspy-benchmark-fork-report.yml index 4127d262b..2f2b5e5de 100644 --- a/.github/workflows/fspy-benchmark-fork-report.yml +++ b/.github/workflows/fspy-benchmark-fork-report.yml @@ -2,11 +2,15 @@ name: Fspy Benchmark Fork Report permissions: {} -on: +on: # zizmor: ignore[dangerous-triggers] PR artifacts are read as text and never executed. workflow_run: workflows: [Fspy Benchmark] types: [completed] +defaults: + run: + shell: bash + jobs: comment: name: Report fork benchmark From e2b478a61fd5df7d1ed5c5cbc7a5713a0eb54885 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 00:58:32 +0800 Subject: [PATCH 03/35] ci: upload hidden benchmark results Allow upload-artifact to include the dot-prefixed benchmark results directory. Co-authored-by: GPT-5.6 Codex --- .github/workflows/fspy-benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index b475bd886..e9dc47ef3 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -124,6 +124,7 @@ jobs: name: ${{ steps.baseline.outputs.current-artifact-name }} path: .benchmark-results/${{ matrix.platform }}.* if-no-files-found: error + include-hidden-files: true overwrite: true retention-days: 90 From 804077e1c44ee0d5916bd144d977804b4b66bb19 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 01:03:23 +0800 Subject: [PATCH 04/35] ci: bound fspy benchmark process launches Use a shorter process-level sampling window and inherit child stderr so runtime failures remain diagnosable. Co-authored-by: GPT-5.6 Codex --- crates/fspy_benchmark/benches/fspy.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 91dbc11b3..53e8d518c 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -105,7 +105,7 @@ async fn run_untracked(target: &str, fixture: &AbsolutePath) -> ExitStatus { .args(benchmark_args(fixture)) .stdin(Stdio::null()) .stdout(Stdio::null()) - .stderr(Stdio::null()); + .stderr(Stdio::inherit()); command.status().await.expect("failed to run untracked benchmark target") } @@ -115,7 +115,7 @@ async fn run_tracked(target: &str, fixture: &AbsolutePath) -> ChildTermination { .args(benchmark_args(fixture)) .stdin(Stdio::null()) .stdout(Stdio::null()) - .stderr(Stdio::null()); + .stderr(Stdio::inherit()); command .spawn(CancellationToken::new()) .await @@ -131,9 +131,9 @@ fn benchmark_args(fixture: &AbsolutePath) -> [&OsStr; 3] { fn criterion_config() -> Criterion { Criterion::default() - .sample_size(20) - .warm_up_time(Duration::from_secs(2)) - .measurement_time(Duration::from_secs(10)) + .sample_size(10) + .warm_up_time(Duration::from_millis(500)) + .measurement_time(Duration::from_secs(1)) } criterion_group! { From 93e970fcaf5bf467af63033034c155e800069573 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 01:06:46 +0800 Subject: [PATCH 05/35] ci: grant benchmark PR comment access Use pull-request write permission for the sticky benchmark report endpoint. Co-authored-by: GPT-5.6 Codex --- .github/workflows/fspy-benchmark-fork-report.yml | 2 +- .github/workflows/fspy-benchmark.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fspy-benchmark-fork-report.yml b/.github/workflows/fspy-benchmark-fork-report.yml index 2f2b5e5de..35ff6d7da 100644 --- a/.github/workflows/fspy-benchmark-fork-report.yml +++ b/.github/workflows/fspy-benchmark-fork-report.yml @@ -23,7 +23,7 @@ jobs: permissions: actions: read contents: read - issues: write + pull-requests: write steps: - name: Download benchmark results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index e9dc47ef3..d86162339 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -138,7 +138,7 @@ jobs: permissions: actions: read contents: read - issues: write + pull-requests: write steps: - name: Download current results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 From dd350d26226f368e81bfa53ce3dc53d079c52cc5 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 10:29:03 +0800 Subject: [PATCH 06/35] ci: compare fspy benchmarks against main only Preferring an earlier run of the same PR as the baseline masked cumulative regressions, so the baseline is now always the latest main artifact. Gate the main baseline artifact name on refs/heads/main so a workflow_dispatch run from another branch cannot pollute it. Remove the fork report workflow: workflow_run events carry an empty pull_requests array for PRs from forks, so its trigger condition could never match. Fork PRs keep job summaries and artifacts. Fold the PR comment assembly and upsert into fspy-benchmark.mjs. Co-Authored-By: Claude Fable 5 --- .github/scripts/fspy-benchmark.mjs | 79 +++++++++++----- .github/scripts/fspy-benchmark.test.mjs | 6 +- .../workflows/fspy-benchmark-fork-report.yml | 92 ------------------- .github/workflows/fspy-benchmark.yml | 67 +++----------- crates/fspy_benchmark/README.md | 2 +- 5 files changed, 72 insertions(+), 174 deletions(-) delete mode 100644 .github/workflows/fspy-benchmark-fork-report.yml diff --git a/.github/scripts/fspy-benchmark.mjs b/.github/scripts/fspy-benchmark.mjs index 67cf184e2..3d68f97db 100644 --- a/.github/scripts/fspy-benchmark.mjs +++ b/.github/scripts/fspy-benchmark.mjs @@ -29,14 +29,16 @@ function required(args, name) { return value; } -async function githubJson(path) { +async function githubJson(path, init = {}) { const token = requiredEnv('GITHUB_TOKEN'); const response = await fetch(`https://api.github.com${path}`, { + ...init, headers: { Accept: 'application/vnd.github+json', Authorization: `Bearer ${token}`, 'X-GitHub-Api-Version': '2022-11-28', 'User-Agent': 'vite-task-fspy-benchmark', + ...(init.body === undefined ? {} : { 'Content-Type': 'application/json' }), }, }); if (!response.ok) { @@ -58,7 +60,10 @@ async function findLatestArtifact(name, currentRunId) { ); return response.artifacts .filter( - (artifact) => !artifact.expired && String(artifact.workflow_run?.id) !== String(currentRunId), + (artifact) => + !artifact.expired && + artifact.workflow_run?.id != null && + String(artifact.workflow_run.id) !== String(currentRunId), ) .sort((left, right) => Date.parse(right.created_at) - Date.parse(left.created_at))[0]; } @@ -69,29 +74,14 @@ async function resolveBaseline(args) { const context = required(args, 'context'); const currentRunId = requiredEnv('GITHUB_RUN_ID'); const currentArtifactName = `fspy-benchmark-${platform}-${context}`; - const candidates = [currentArtifactName]; - if (context !== 'main') candidates.push(`fspy-benchmark-${platform}-main`); - - let artifact; - let kind; - for (const candidate of candidates) { - artifact = await findLatestArtifact(candidate, currentRunId); - if (artifact) { - kind = candidate === currentArtifactName ? 'previous PR run' : 'main'; - break; - } - } + const artifact = await findLatestArtifact(`fspy-benchmark-${platform}-main`, currentRunId); const outputs = [ `current-artifact-name=${currentArtifactName}`, `found=${artifact ? 'true' : 'false'}`, ]; if (artifact) { - outputs.push( - `artifact-name=${artifact.name}`, - `run-id=${artifact.workflow_run.id}`, - `kind=${kind}`, - ); + outputs.push(`artifact-name=${artifact.name}`, `run-id=${artifact.workflow_run.id}`); } await appendFile(outputPath, `${outputs.join('\n')}\n`); } @@ -172,7 +162,7 @@ function resultLink(result) { return `https://github.com/${repository}/actions/runs/${result.runId}`; } -export function renderReport(current, baseline, baselineKind = '') { +export function renderReport(current, baseline) { const compatible = baseline && current.platform === baseline.platform && @@ -181,9 +171,7 @@ export function renderReport(current, baseline, baselineKind = '') { if (compatible) { const link = resultLink(baseline); - const description = link - ? `[${baselineKind || 'baseline'}](${link})` - : baselineKind || 'baseline'; + const description = link ? `[main](${link})` : 'main'; lines.push(`Compared with ${description} at \`${baseline.commit.slice(0, 8)}\`.`, ''); } else if (baseline) { lines.push( @@ -191,7 +179,7 @@ export function renderReport(current, baseline, baselineKind = '') { '', ); } else { - lines.push('No previous result was available for this runner.', ''); + lines.push('No `main` result was available for this runner.', ''); } lines.push( @@ -236,11 +224,50 @@ async function compare(args) { if (error.code !== 'ENOENT') throw error; } } - const report = renderReport(current, baseline, args.get('baseline-kind') ?? ''); + const report = renderReport(current, baseline); await writeFile(required(args, 'output'), report); process.stdout.write(report); } +const COMMENT_MARKER = ''; + +async function comment(args) { + const repository = requiredEnv('GITHUB_REPOSITORY'); + const pullRequest = required(args, 'pull-request'); + const commit = required(args, 'commit'); + const resultsDir = required(args, 'results-dir'); + + const sections = []; + for (const platform of ['linux', 'macos', 'windows']) { + sections.push(await readFile(join(resultsDir, `${platform}.md`), 'utf8')); + } + const body = [ + COMMENT_MARKER, + '## fspy benchmark', + '', + `Results for commit \`${commit}\`. Each platform is compared with the latest \`main\` result.`, + '', + ...sections, + 'This comment is updated by the Fspy Benchmark workflow.', + '', + ].join('\n'); + + const comments = await githubJson( + `/repos/${repository}/issues/${pullRequest}/comments?per_page=100`, + ); + const existing = comments.find( + (existingComment) => + existingComment.user?.type === 'Bot' && existingComment.body?.includes(COMMENT_MARKER), + ); + const path = existing + ? `/repos/${repository}/issues/comments/${existing.id}` + : `/repos/${repository}/issues/${pullRequest}/comments`; + await githubJson(path, { + method: existing ? 'PATCH' : 'POST', + body: JSON.stringify({ body }), + }); +} + async function main() { const command = process.argv[2]; const args = parseArgs(process.argv.slice(3)); @@ -250,6 +277,8 @@ async function main() { await collect(args); } else if (command === 'compare') { await compare(args); + } else if (command === 'comment') { + await comment(args); } else { throw new Error(`unknown command: ${command ?? ''}`); } diff --git a/.github/scripts/fspy-benchmark.test.mjs b/.github/scripts/fspy-benchmark.test.mjs index d219767dc..486fed4a4 100644 --- a/.github/scripts/fspy-benchmark.test.mjs +++ b/.github/scripts/fspy-benchmark.test.mjs @@ -21,17 +21,17 @@ function result(platform, architecture, runId, untracked, tracked) { test('renders normalized change against a compatible baseline', () => { const baseline = result('linux', 'X64', '10', 100, 120); const current = result('linux', 'X64', '20', 100, 132); - const report = renderReport(current, baseline, 'previous PR run'); + const report = renderReport(current, baseline); assert.match(report, /fspy overhead \| Normalized change/); assert.match(report, /\+32\.00% \| \+10\.00%/); - assert.match(report, /previous PR run/); + assert.match(report, /Compared with main at `12345678`/); }); test('does not compare different architectures', () => { const baseline = result('macos', 'X64', '10', 100, 120); const current = result('macos', 'ARM64', '20', 100, 132); - const report = renderReport(current, baseline, 'main'); + const report = renderReport(current, baseline); assert.match(report, /No comparison/); assert.match(report, /\+32\.00% \| —/); diff --git a/.github/workflows/fspy-benchmark-fork-report.yml b/.github/workflows/fspy-benchmark-fork-report.yml deleted file mode 100644 index 35ff6d7da..000000000 --- a/.github/workflows/fspy-benchmark-fork-report.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: Fspy Benchmark Fork Report - -permissions: {} - -on: # zizmor: ignore[dangerous-triggers] PR artifacts are read as text and never executed. - workflow_run: - workflows: [Fspy Benchmark] - types: [completed] - -defaults: - run: - shell: bash - -jobs: - comment: - name: Report fork benchmark - if: >- - github.event.workflow_run.conclusion == 'success' && - github.event.workflow_run.event == 'pull_request' && - github.event.workflow_run.head_repository.full_name != github.repository && - github.event.workflow_run.pull_requests[0].number != null - runs-on: namespace-profile-linux-x64-default - permissions: - actions: read - contents: read - pull-requests: write - steps: - - name: Download benchmark results - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: fspy-benchmark-*-pr-${{ github.event.workflow_run.pull_requests[0].number }} - path: .benchmark-results - merge-multiple: true - github-token: ${{ github.token }} - repository: ${{ github.repository }} - run-id: ${{ github.event.workflow_run.id }} - - - name: Build PR comment - env: - BENCHMARK_SHA: ${{ github.event.workflow_run.head_sha }} - run: | - { - echo '' - echo '## fspy benchmark' - echo - echo "Results for commit \`$BENCHMARK_SHA\`. Each platform uses its latest result from this PR, falling back to \`main\`." - echo - for platform in linux macos windows; do - cat ".benchmark-results/$platform.md" - done - echo - echo 'This comment is updated by the Fspy Benchmark workflow.' - } > benchmark-comment.md - - - name: Update PR comment - env: - GITHUB_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} - run: | - node --input-type=module <<'EOF' - import { readFile } from "node:fs/promises"; - - const marker = ""; - const repository = process.env.GITHUB_REPOSITORY; - const token = process.env.GITHUB_TOKEN; - const pullRequest = process.env.PR_NUMBER; - const headers = { - Accept: "application/vnd.github+json", - Authorization: `Bearer ${token}`, - "Content-Type": "application/json", - "X-GitHub-Api-Version": "2022-11-28", - }; - const commentsResponse = await fetch( - `https://api.github.com/repos/${repository}/issues/${pullRequest}/comments?per_page=100`, - { headers }, - ); - if (!commentsResponse.ok) throw new Error(await commentsResponse.text()); - const comments = await commentsResponse.json(); - const existing = comments.find( - (comment) => comment.user?.type === "Bot" && comment.body?.includes(marker), - ); - const body = await readFile("benchmark-comment.md", "utf8"); - const url = existing - ? `https://api.github.com/repos/${repository}/issues/comments/${existing.id}` - : `https://api.github.com/repos/${repository}/issues/${pullRequest}/comments`; - const response = await fetch(url, { - method: existing ? "PATCH" : "POST", - headers, - body: JSON.stringify({ body }), - }); - if (!response.ok) throw new Error(await response.text()); - EOF diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index d86162339..f981223e3 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -61,12 +61,17 @@ jobs: env: PR_NUMBER: ${{ github.event.pull_request.number }} run: | + # Only real main-branch runs may publish the main baseline artifact; + # a workflow_dispatch run from another branch must not pollute it. if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then context="pr-$PR_NUMBER" pull_request="$PR_NUMBER" - else + elif [[ "$GITHUB_REF" == "refs/heads/main" ]]; then context="main" pull_request="none" + else + context="dispatch" + pull_request="none" fi echo "name=$context" >> "$GITHUB_OUTPUT" echo "pull-request=$pull_request" >> "$GITHUB_OUTPUT" @@ -112,7 +117,6 @@ jobs: node .github/scripts/fspy-benchmark.mjs compare --current ".benchmark-results/${{ matrix.platform }}.json" --baseline ".benchmark-baseline/${{ matrix.platform }}.json" - --baseline-kind "${{ steps.baseline.outputs.kind }}" --output ".benchmark-results/${{ matrix.platform }}.md" - name: Add job summary @@ -140,6 +144,8 @@ jobs: contents: read pull-requests: write steps: + - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 + - name: Download current results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -147,58 +153,13 @@ jobs: path: .benchmark-results merge-multiple: true - - name: Build PR comment - env: - PR_NUMBER: ${{ github.event.pull_request.number }} - run: | - { - echo '' - echo '## fspy benchmark' - echo - echo "Results for commit \`$GITHUB_SHA\`. Each platform uses its latest result from this PR, falling back to \`main\`." - echo - for platform in linux macos windows; do - cat ".benchmark-results/$platform.md" - done - echo - echo 'This comment is updated by the Fspy Benchmark workflow.' - } > benchmark-comment.md - - name: Update PR comment env: GITHUB_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} - run: | - node --input-type=module <<'EOF' - import { readFile } from "node:fs/promises"; - - const marker = ""; - const repository = process.env.GITHUB_REPOSITORY; - const token = process.env.GITHUB_TOKEN; - const pullRequest = process.env.PR_NUMBER; - const headers = { - Accept: "application/vnd.github+json", - Authorization: `Bearer ${token}`, - "Content-Type": "application/json", - "X-GitHub-Api-Version": "2022-11-28", - }; - const commentsResponse = await fetch( - `https://api.github.com/repos/${repository}/issues/${pullRequest}/comments?per_page=100`, - { headers }, - ); - if (!commentsResponse.ok) throw new Error(await commentsResponse.text()); - const comments = await commentsResponse.json(); - const existing = comments.find( - (comment) => comment.user?.type === "Bot" && comment.body?.includes(marker), - ); - const body = await readFile("benchmark-comment.md", "utf8"); - const url = existing - ? `https://api.github.com/repos/${repository}/issues/comments/${existing.id}` - : `https://api.github.com/repos/${repository}/issues/${pullRequest}/comments`; - const response = await fetch(url, { - method: existing ? "PATCH" : "POST", - headers, - body: JSON.stringify({ body }), - }); - if (!response.ok) throw new Error(await response.text()); - EOF + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: >- + node .github/scripts/fspy-benchmark.mjs comment + --results-dir .benchmark-results + --pull-request "$PR_NUMBER" + --commit "$HEAD_SHA" diff --git a/crates/fspy_benchmark/README.md b/crates/fspy_benchmark/README.md index fcda70746..137393eb4 100644 --- a/crates/fspy_benchmark/README.md +++ b/crates/fspy_benchmark/README.md @@ -23,4 +23,4 @@ just benchmark-fspy ``` CI runs it on the Linux, macOS, and Windows Namespace runner profiles. Each result is compared with -the latest result from the same pull request, falling back to the latest `main` result. +the latest `main` result. From cac68068dfdd359b874eb2096f411ce61f6d7177 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 11:47:42 +0800 Subject: [PATCH 07/35] ci: convert fspy benchmark script to TypeScript Node runs .mts files directly by stripping types, so the script stays build-free while gaining typed structures for the result JSON, Criterion estimates, and GitHub API payloads. The new tsconfig gives the scripts directory Node types and enforces erasable-only syntax. Comment the non-obvious constraints: baseline selection, re-run exclusion, ratio normalization, and the workload constants mirrored from the Criterion bench. Co-Authored-By: Claude Fable 5 --- ...{fspy-benchmark.mjs => fspy-benchmark.mts} | 211 +++++++++++++----- ...hmark.test.mjs => fspy-benchmark.test.mts} | 14 +- .github/scripts/tsconfig.json | 14 ++ .github/workflows/fspy-benchmark.yml | 8 +- 4 files changed, 184 insertions(+), 63 deletions(-) rename .github/scripts/{fspy-benchmark.mjs => fspy-benchmark.mts} (55%) rename .github/scripts/{fspy-benchmark.test.mjs => fspy-benchmark.test.mts} (76%) create mode 100644 .github/scripts/tsconfig.json diff --git a/.github/scripts/fspy-benchmark.mjs b/.github/scripts/fspy-benchmark.mts similarity index 55% rename from .github/scripts/fspy-benchmark.mjs rename to .github/scripts/fspy-benchmark.mts index 3d68f97db..c59a5f89f 100644 --- a/.github/scripts/fspy-benchmark.mjs +++ b/.github/scripts/fspy-benchmark.mts @@ -1,15 +1,65 @@ #!/usr/bin/env node +// Support script for the Fspy Benchmark workflow (.github/workflows/fspy-benchmark.yml). +// Executed directly with `node`, which strips the type annotations at load time, +// so only erasable TypeScript syntax is allowed (no enums, namespaces, etc.). + import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises'; import { arch, cpus, platform as osPlatform, release } from 'node:os'; import { dirname, join } from 'node:path'; import { pathToFileURL } from 'node:url'; +// Mirrors THREAD_COUNT and FILES_PER_THREAD in crates/fspy_benchmark/benches/fspy.rs. +// Only used to describe the workload in reports. const THREAD_COUNT = 4; const FILES_PER_THREAD = 2048; -function parseArgs(args) { - const parsed = new Map(); +type Args = Map; + +/** One Criterion estimate, in nanoseconds. */ +interface Estimate { + meanNs: number; + meanLowerNs: number; + meanUpperNs: number; + medianNs: number; +} + +/** The subset of a stored benchmark result that reports are rendered from. */ +interface ReportInput { + platform: string; + architecture: string; + os: { cpu: string }; + commit: string; + runId: string; + workload: { threadCount: number; filesPerThread: number; totalOpens: number }; + benchmarks: Record; +} + +/** The full result JSON produced by `collect` and uploaded as an artifact. */ +interface BenchmarkResult extends ReportInput { + schemaVersion: number; + os: { platform: string; release: string; cpu: string }; + runner: string; + runAttempt: string; + event: string; + pullRequest: string; + benchmarks: Record; +} + +interface BaselineArtifact { + name: string; + runId: string; + createdAt: string; +} + +interface IssueComment { + id: number; + body?: string | null; + user?: { type?: string } | null; +} + +function parseArgs(args: string[]): Args { + const parsed = new Map(); for (let index = 0; index < args.length; index += 2) { const flag = args[index]; const value = args[index + 1]; @@ -21,7 +71,7 @@ function parseArgs(args) { return parsed; } -function required(args, name) { +function required(args: Args, name: string): string { const value = args.get(name); if (value === undefined || value === '') { throw new Error(`missing --${name}`); @@ -29,7 +79,13 @@ function required(args, name) { return value; } -async function githubJson(path, init = {}) { +function requiredEnv(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`missing ${name}`); + return value; +} + +async function githubJson(path: string, init: RequestInit = {}): Promise { const token = requiredEnv('GITHUB_TOKEN'); const response = await fetch(`https://api.github.com${path}`, { ...init, @@ -44,31 +100,48 @@ async function githubJson(path, init = {}) { if (!response.ok) { throw new Error(`GitHub API ${response.status}: ${await response.text()}`); } - return response.json(); + return response.json() as Promise; } -function requiredEnv(name) { - const value = process.env[name]; - if (!value) throw new Error(`missing ${name}`); - return value; -} - -async function findLatestArtifact(name, currentRunId) { +async function findLatestArtifact( + name: string, + currentRunId: string, +): Promise { const repository = requiredEnv('GITHUB_REPOSITORY'); - const response = await githubJson( - `/repos/${repository}/actions/artifacts?name=${encodeURIComponent(name)}&per_page=100`, + // The name filter is applied server-side, so one page covers the recent history. + const response = await githubJson<{ + artifacts: { + name: string; + expired: boolean; + created_at: string; + workflow_run?: { id: number } | null; + }[]; + }>(`/repos/${repository}/actions/artifacts?name=${encodeURIComponent(name)}&per_page=100`); + return ( + response.artifacts + .flatMap((artifact) => + !artifact.expired && artifact.workflow_run?.id != null + ? [ + { + name: artifact.name, + runId: String(artifact.workflow_run.id), + createdAt: artifact.created_at, + }, + ] + : [], + ) + // Skip the current run so a re-run attempt does not compare against the + // artifact its own previous attempt uploaded. + .filter((artifact) => artifact.runId !== currentRunId) + .sort((left, right) => Date.parse(right.createdAt) - Date.parse(left.createdAt))[0] ); - return response.artifacts - .filter( - (artifact) => - !artifact.expired && - artifact.workflow_run?.id != null && - String(artifact.workflow_run.id) !== String(currentRunId), - ) - .sort((left, right) => Date.parse(right.created_at) - Date.parse(left.created_at))[0]; } -async function resolveBaseline(args) { +// Emits step outputs consumed by the workflow's baseline download and result +// upload steps. The baseline is always the latest `main` artifact: comparing +// against earlier runs of the same PR would mask regressions that accumulate +// over the PR's lifetime. +async function resolveBaseline(args: Args): Promise { const outputPath = requiredEnv('GITHUB_OUTPUT'); const platform = required(args, 'platform'); const context = required(args, 'context'); @@ -81,14 +154,26 @@ async function resolveBaseline(args) { `found=${artifact ? 'true' : 'false'}`, ]; if (artifact) { - outputs.push(`artifact-name=${artifact.name}`, `run-id=${artifact.workflow_run.id}`); + outputs.push(`artifact-name=${artifact.name}`, `run-id=${artifact.runId}`); } await appendFile(outputPath, `${outputs.join('\n')}\n`); } -async function readEstimate(criterionRoot, target, mode) { +// Criterion writes the estimates of its most recent run to +// ///new/estimates.json. +async function readEstimate( + criterionRoot: string, + target: string, + mode: string, +): Promise { const path = join(criterionRoot, target, mode, 'new', 'estimates.json'); - const estimates = JSON.parse(await readFile(path, 'utf8')); + const estimates = JSON.parse(await readFile(path, 'utf8')) as { + mean: { + point_estimate: number; + confidence_interval: { lower_bound: number; upper_bound: number }; + }; + median: { point_estimate: number }; + }; return { meanNs: estimates.mean.point_estimate, meanLowerNs: estimates.mean.confidence_interval.lower_bound, @@ -97,12 +182,14 @@ async function readEstimate(criterionRoot, target, mode) { }; } -async function collect(args) { +// Combines Criterion's output with run metadata into the JSON document that is +// uploaded as an artifact and later consumed as a comparison baseline. +async function collect(args: Args): Promise { const criterionRoot = required(args, 'criterion-root'); const output = required(args, 'output'); const platform = required(args, 'platform'); const expectStatic = args.get('expect-static') === 'true'; - const benchmarks = { + const benchmarks: Record = { 'dynamic/untracked': await readEstimate(criterionRoot, 'dynamic', 'untracked'), 'dynamic/tracked': await readEstimate(criterionRoot, 'dynamic', 'tracked'), }; @@ -111,10 +198,10 @@ async function collect(args) { benchmarks['static/tracked'] = await readEstimate(criterionRoot, 'static', 'tracked'); } - const result = { + const result: BenchmarkResult = { schemaVersion: 1, platform, - architecture: process.env.RUNNER_ARCH ?? arch(), + architecture: process.env['RUNNER_ARCH'] ?? arch(), os: { platform: osPlatform(), release: release(), @@ -138,36 +225,40 @@ async function collect(args) { await writeFile(output, `${JSON.stringify(result, null, 2)}\n`); } -function ratio(result, target) { - return ( - result.benchmarks[`${target}/tracked`].meanNs / result.benchmarks[`${target}/untracked`].meanNs - ); +/** Wall-clock cost of tracking, as tracked time over untracked time. */ +function ratio(result: ReportInput, target: string): number { + const tracked = result.benchmarks[`${target}/tracked`]; + const untracked = result.benchmarks[`${target}/untracked`]; + if (!tracked || !untracked) throw new Error(`missing benchmark data for ${target}`); + return tracked.meanNs / untracked.meanNs; } -function percentage(value) { +function percentage(value: number): string { const sign = value > 0 ? '+' : ''; return `${sign}${(value * 100).toFixed(2)}%`; } -function duration(nanoseconds) { +function duration(nanoseconds: number): string { if (nanoseconds >= 1e9) return `${(nanoseconds / 1e9).toFixed(3)} s`; if (nanoseconds >= 1e6) return `${(nanoseconds / 1e6).toFixed(3)} ms`; if (nanoseconds >= 1e3) return `${(nanoseconds / 1e3).toFixed(3)} µs`; return `${nanoseconds.toFixed(1)} ns`; } -function resultLink(result) { - const repository = process.env.GITHUB_REPOSITORY; +function resultLink(result: ReportInput): string { + const repository = process.env['GITHUB_REPOSITORY']; if (!repository || !result.runId) return ''; return `https://github.com/${repository}/actions/runs/${result.runId}`; } -export function renderReport(current, baseline) { +export function renderReport(current: ReportInput, baseline?: ReportInput): string { + // Overhead ratios are only comparable within one architecture; a runner + // profile can move to different hardware over time. const compatible = - baseline && + baseline !== undefined && current.platform === baseline.platform && current.architecture === baseline.architecture; - const lines = [`### ${current.platform[0].toUpperCase()}${current.platform.slice(1)}`, '']; + const lines = [`### ${current.platform.charAt(0).toUpperCase()}${current.platform.slice(1)}`, '']; if (compatible) { const link = resultLink(baseline); @@ -191,15 +282,19 @@ export function renderReport(current, baseline) { ? ['dynamic', 'static'] : ['dynamic']; for (const target of targets) { - const untracked = current.benchmarks[`${target}/untracked`].meanNs; - const tracked = current.benchmarks[`${target}/tracked`].meanNs; + const untracked = current.benchmarks[`${target}/untracked`]; + const tracked = current.benchmarks[`${target}/tracked`]; + if (!untracked || !tracked) throw new Error(`missing benchmark data for ${target}`); const overhead = ratio(current, target) - 1; + // Change of the overhead ratio relative to the baseline. Comparing ratios + // instead of absolute times cancels machine-speed differences between + // runner instances. const normalizedChange = compatible && Object.hasOwn(baseline.benchmarks, `${target}/tracked`) ? ratio(current, target) / ratio(baseline, target) - 1 : undefined; lines.push( - `| ${target} | ${duration(untracked)} | ${duration(tracked)} | ${percentage(overhead)} | ${normalizedChange === undefined ? '—' : percentage(normalizedChange)} |`, + `| ${target} | ${duration(untracked.meanNs)} | ${duration(tracked.meanNs)} | ${percentage(overhead)} | ${normalizedChange === undefined ? '—' : percentage(normalizedChange)} |`, ); } @@ -213,15 +308,17 @@ export function renderReport(current, baseline) { return `${lines.join('\n')}\n`; } -async function compare(args) { - const current = JSON.parse(await readFile(required(args, 'current'), 'utf8')); +async function compare(args: Args): Promise { + const current = JSON.parse(await readFile(required(args, 'current'), 'utf8')) as BenchmarkResult; const baselinePath = args.get('baseline'); - let baseline; + let baseline: BenchmarkResult | undefined; if (baselinePath) { try { - baseline = JSON.parse(await readFile(baselinePath, 'utf8')); + baseline = JSON.parse(await readFile(baselinePath, 'utf8')) as BenchmarkResult; } catch (error) { - if (error.code !== 'ENOENT') throw error; + // The workflow always passes --baseline; the file is absent when no + // baseline artifact was found. + if ((error as { code?: string }).code !== 'ENOENT') throw error; } } const report = renderReport(current, baseline); @@ -231,13 +328,15 @@ async function compare(args) { const COMMENT_MARKER = ''; -async function comment(args) { +// Creates or updates the sticky PR comment. The invisible marker identifies +// the comment, so later runs edit it in place instead of posting a new one. +async function comment(args: Args): Promise { const repository = requiredEnv('GITHUB_REPOSITORY'); const pullRequest = required(args, 'pull-request'); const commit = required(args, 'commit'); const resultsDir = required(args, 'results-dir'); - const sections = []; + const sections: string[] = []; for (const platform of ['linux', 'macos', 'windows']) { sections.push(await readFile(join(resultsDir, `${platform}.md`), 'utf8')); } @@ -252,7 +351,7 @@ async function comment(args) { '', ].join('\n'); - const comments = await githubJson( + const comments = await githubJson( `/repos/${repository}/issues/${pullRequest}/comments?per_page=100`, ); const existing = comments.find( @@ -268,7 +367,7 @@ async function comment(args) { }); } -async function main() { +async function main(): Promise { const command = process.argv[2]; const args = parseArgs(process.argv.slice(3)); if (command === 'resolve-baseline') { @@ -284,10 +383,12 @@ async function main() { } } +// Run only when executed directly, so the test file can import renderReport. const invokedPath = process.argv[1] ? pathToFileURL(process.argv[1]).href : ''; if (import.meta.url === invokedPath) { - main().catch((error) => { - process.stderr.write(`${error.stack ?? error}\n`); + main().catch((error: unknown) => { + const message = error instanceof Error ? (error.stack ?? error.message) : String(error); + process.stderr.write(`${message}\n`); process.exitCode = 1; }); } diff --git a/.github/scripts/fspy-benchmark.test.mjs b/.github/scripts/fspy-benchmark.test.mts similarity index 76% rename from .github/scripts/fspy-benchmark.test.mjs rename to .github/scripts/fspy-benchmark.test.mts index 486fed4a4..33d24c4ee 100644 --- a/.github/scripts/fspy-benchmark.test.mjs +++ b/.github/scripts/fspy-benchmark.test.mts @@ -1,9 +1,15 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { renderReport } from './fspy-benchmark.mjs'; +import { renderReport } from './fspy-benchmark.mts'; -function result(platform, architecture, runId, untracked, tracked) { +function result( + platform: string, + architecture: string, + runId: string, + untracked: number, + tracked: number, +) { return { platform, architecture, @@ -18,7 +24,7 @@ function result(platform, architecture, runId, untracked, tracked) { }; } -test('renders normalized change against a compatible baseline', () => { +void test('renders normalized change against a compatible baseline', () => { const baseline = result('linux', 'X64', '10', 100, 120); const current = result('linux', 'X64', '20', 100, 132); const report = renderReport(current, baseline); @@ -28,7 +34,7 @@ test('renders normalized change against a compatible baseline', () => { assert.match(report, /Compared with main at `12345678`/); }); -test('does not compare different architectures', () => { +void test('does not compare different architectures', () => { const baseline = result('macos', 'X64', '10', 100, 120); const current = result('macos', 'ARM64', '20', 100, 132); const report = renderReport(current, baseline); diff --git a/.github/scripts/tsconfig.json b/.github/scripts/tsconfig.json new file mode 100644 index 000000000..07f8c9b9b --- /dev/null +++ b/.github/scripts/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "@tsconfig/strictest/tsconfig.json", + "compilerOptions": { + "allowImportingTsExtensions": true, + "erasableSyntaxOnly": true, + "module": "NodeNext", + "moduleResolution": "NodeNext", + "noEmit": true, + "target": "ES2022", + "lib": ["ES2022"], + "types": ["node"] + }, + "include": ["*.mts"] +} diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index f981223e3..a27ba9762 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -81,7 +81,7 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: >- - node .github/scripts/fspy-benchmark.mjs resolve-baseline + node .github/scripts/fspy-benchmark.mts resolve-baseline --platform "${{ matrix.platform }}" --context "${{ steps.context.outputs.name }}" @@ -100,7 +100,7 @@ jobs: - name: Collect benchmark results run: >- - node .github/scripts/fspy-benchmark.mjs collect + node .github/scripts/fspy-benchmark.mts collect --criterion-root "$CARGO_TARGET_DIR/criterion/fspy" --output ".benchmark-results/${{ matrix.platform }}.json" --platform "${{ matrix.platform }}" @@ -114,7 +114,7 @@ jobs: - name: Compare benchmark results run: >- - node .github/scripts/fspy-benchmark.mjs compare + node .github/scripts/fspy-benchmark.mts compare --current ".benchmark-results/${{ matrix.platform }}.json" --baseline ".benchmark-baseline/${{ matrix.platform }}.json" --output ".benchmark-results/${{ matrix.platform }}.md" @@ -159,7 +159,7 @@ jobs: PR_NUMBER: ${{ github.event.pull_request.number }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: >- - node .github/scripts/fspy-benchmark.mjs comment + node .github/scripts/fspy-benchmark.mts comment --results-dir .benchmark-results --pull-request "$PR_NUMBER" --commit "$HEAD_SHA" From d43ef68c8d4a66fe8ba039044788df8f9900ae22 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 11:50:06 +0800 Subject: [PATCH 08/35] ci: set up pinned Node for the fspy benchmark script The Linux and Windows runner images preinstall Node 20, which cannot run .mts files; type stripping needs the pinned 22.19. Co-Authored-By: Claude Fable 5 --- .github/workflows/fspy-benchmark.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index a27ba9762..f7a90af24 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -52,6 +52,11 @@ jobs: save-cache: ${{ github.ref_name == 'main' }} cache-key: fspy-benchmark-${{ matrix.platform }} + # The support script is TypeScript; running it needs the .node-version + # Node, which strips types natively. Runner-preinstalled Node may be + # older. + - uses: oxc-project/setup-node@4c588e9266bd930b6ddc34307df0659ed511d187 # v1.3.1 + - name: Install static target if: matrix.static run: rustup target add x86_64-unknown-linux-musl @@ -146,6 +151,8 @@ jobs: steps: - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 + - uses: oxc-project/setup-node@4c588e9266bd930b6ddc34307df0659ed511d187 # v1.3.1 + - name: Download current results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: From 0745520c42525559d0a61409b0053a7079a077d1 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 13:39:47 +0800 Subject: [PATCH 09/35] test(fspy): amplify benchmark workload over process startup With a single pass over the fixture, the tracked/untracked delta on the preload paths (2-8 ms) is the same order as spawn-time variance on CI runners: overhead swung 11 points between identical runs, and on Windows the confidence interval exceeded the delta. Repeat the open loop 16 times so per-open interception cost dominates, extend the measurement window to five seconds, and render each mean's confidence interval in the report. Bump the result schema and skip comparisons across schema versions, since ratios from different workloads are not comparable. Co-Authored-By: Claude Fable 5 --- .github/scripts/fspy-benchmark.mts | 49 +++++++++++++++++------- .github/scripts/fspy-benchmark.test.mts | 22 +++++++++-- crates/fspy_benchmark/README.md | 6 ++- crates/fspy_benchmark/benches/fspy.rs | 19 +++++++-- crates/fspy_benchmark_target/src/main.rs | 16 +++++--- 5 files changed, 83 insertions(+), 29 deletions(-) diff --git a/.github/scripts/fspy-benchmark.mts b/.github/scripts/fspy-benchmark.mts index c59a5f89f..ca8493b23 100644 --- a/.github/scripts/fspy-benchmark.mts +++ b/.github/scripts/fspy-benchmark.mts @@ -9,35 +9,45 @@ import { arch, cpus, platform as osPlatform, release } from 'node:os'; import { dirname, join } from 'node:path'; import { pathToFileURL } from 'node:url'; -// Mirrors THREAD_COUNT and FILES_PER_THREAD in crates/fspy_benchmark/benches/fspy.rs. -// Only used to describe the workload in reports. +// Mirrors THREAD_COUNT, FILES_PER_THREAD, and PASSES in +// crates/fspy_benchmark/benches/fspy.rs. Only used to describe the workload in +// reports. const THREAD_COUNT = 4; const FILES_PER_THREAD = 2048; +const PASSES = 16; + +// Bump when the workload or result shape changes; results from different +// schema versions are not comparable. +const SCHEMA_VERSION = 2; type Args = Map; -/** One Criterion estimate, in nanoseconds. */ -interface Estimate { +/** Mean with its confidence interval, in nanoseconds. */ +interface EstimateSummary { meanNs: number; meanLowerNs: number; meanUpperNs: number; +} + +/** One Criterion estimate, in nanoseconds. */ +interface Estimate extends EstimateSummary { medianNs: number; } /** The subset of a stored benchmark result that reports are rendered from. */ interface ReportInput { + schemaVersion: number; platform: string; architecture: string; os: { cpu: string }; commit: string; runId: string; - workload: { threadCount: number; filesPerThread: number; totalOpens: number }; - benchmarks: Record; + workload: { threadCount: number; filesPerThread: number; passes: number; totalOpens: number }; + benchmarks: Record; } /** The full result JSON produced by `collect` and uploaded as an artifact. */ interface BenchmarkResult extends ReportInput { - schemaVersion: number; os: { platform: string; release: string; cpu: string }; runner: string; runAttempt: string; @@ -199,7 +209,7 @@ async function collect(args: Args): Promise { } const result: BenchmarkResult = { - schemaVersion: 1, + schemaVersion: SCHEMA_VERSION, platform, architecture: process.env['RUNNER_ARCH'] ?? arch(), os: { @@ -216,7 +226,8 @@ async function collect(args: Args): Promise { workload: { threadCount: THREAD_COUNT, filesPerThread: FILES_PER_THREAD, - totalOpens: THREAD_COUNT * FILES_PER_THREAD, + passes: PASSES, + totalOpens: THREAD_COUNT * FILES_PER_THREAD * PASSES, }, benchmarks, }; @@ -245,6 +256,12 @@ function duration(nanoseconds: number): string { return `${nanoseconds.toFixed(1)} ns`; } +/** Mean with the half-width of its confidence interval, e.g. `12.3 ms ±1.4%`. */ +function durationWithUncertainty(estimate: EstimateSummary): string { + const halfWidth = (estimate.meanUpperNs - estimate.meanLowerNs) / 2 / estimate.meanNs; + return `${duration(estimate.meanNs)} ±${(halfWidth * 100).toFixed(1)}%`; +} + function resultLink(result: ReportInput): string { const repository = process.env['GITHUB_REPOSITORY']; if (!repository || !result.runId) return ''; @@ -252,10 +269,12 @@ function resultLink(result: ReportInput): string { } export function renderReport(current: ReportInput, baseline?: ReportInput): string { - // Overhead ratios are only comparable within one architecture; a runner - // profile can move to different hardware over time. + // Overhead ratios are only comparable within one architecture and one + // workload: a runner profile can move to different hardware over time, and + // schema bumps change what is measured. const compatible = baseline !== undefined && + current.schemaVersion === baseline.schemaVersion && current.platform === baseline.platform && current.architecture === baseline.architecture; const lines = [`### ${current.platform.charAt(0).toUpperCase()}${current.platform.slice(1)}`, '']; @@ -266,7 +285,9 @@ export function renderReport(current: ReportInput, baseline?: ReportInput): stri lines.push(`Compared with ${description} at \`${baseline.commit.slice(0, 8)}\`.`, ''); } else if (baseline) { lines.push( - `No comparison: the baseline architecture is \`${baseline.architecture}\`, current is \`${current.architecture}\`.`, + baseline.schemaVersion === current.schemaVersion + ? `No comparison: the baseline architecture is \`${baseline.architecture}\`, current is \`${current.architecture}\`.` + : `No comparison: the baseline uses result schema ${baseline.schemaVersion}, current is ${current.schemaVersion}.`, '', ); } else { @@ -294,13 +315,13 @@ export function renderReport(current: ReportInput, baseline?: ReportInput): stri ? ratio(current, target) / ratio(baseline, target) - 1 : undefined; lines.push( - `| ${target} | ${duration(untracked.meanNs)} | ${duration(tracked.meanNs)} | ${percentage(overhead)} | ${normalizedChange === undefined ? '—' : percentage(normalizedChange)} |`, + `| ${target} | ${durationWithUncertainty(untracked)} | ${durationWithUncertainty(tracked)} | ${percentage(overhead)} | ${normalizedChange === undefined ? '—' : percentage(normalizedChange)} |`, ); } lines.push( '', - `Workload: ${current.workload.threadCount} threads, ${current.workload.filesPerThread.toLocaleString('en-US')} files per thread, ${current.workload.totalOpens.toLocaleString('en-US')} total open-and-close operations.`, + `Workload: ${current.workload.threadCount} threads × ${current.workload.filesPerThread.toLocaleString('en-US')} files × ${current.workload.passes.toLocaleString('en-US')} passes, ${current.workload.totalOpens.toLocaleString('en-US')} total open-and-close operations.`, '', `\`${current.architecture}\` · ${current.os.cpu} · run [${current.runId}](${resultLink(current)})`, '', diff --git a/.github/scripts/fspy-benchmark.test.mts b/.github/scripts/fspy-benchmark.test.mts index 33d24c4ee..4b527790f 100644 --- a/.github/scripts/fspy-benchmark.test.mts +++ b/.github/scripts/fspy-benchmark.test.mts @@ -3,6 +3,10 @@ import test from 'node:test'; import { renderReport } from './fspy-benchmark.mts'; +function estimate(meanNs: number) { + return { meanNs, meanLowerNs: meanNs * 0.98, meanUpperNs: meanNs * 1.02 }; +} + function result( platform: string, architecture: string, @@ -11,15 +15,16 @@ function result( tracked: number, ) { return { + schemaVersion: 2, platform, architecture, os: { cpu: 'Test CPU' }, commit: '1234567890abcdef', runId, - workload: { threadCount: 4, filesPerThread: 2048, totalOpens: 8192 }, + workload: { threadCount: 4, filesPerThread: 2048, passes: 16, totalOpens: 131072 }, benchmarks: { - 'dynamic/untracked': { meanNs: untracked }, - 'dynamic/tracked': { meanNs: tracked }, + 'dynamic/untracked': estimate(untracked), + 'dynamic/tracked': estimate(tracked), }, }; } @@ -30,7 +35,7 @@ void test('renders normalized change against a compatible baseline', () => { const report = renderReport(current, baseline); assert.match(report, /fspy overhead \| Normalized change/); - assert.match(report, /\+32\.00% \| \+10\.00%/); + assert.match(report, /100\.0 ns ±2\.0% \| 132\.0 ns ±2\.0% \| \+32\.00% \| \+10\.00%/); assert.match(report, /Compared with main at `12345678`/); }); @@ -42,3 +47,12 @@ void test('does not compare different architectures', () => { assert.match(report, /No comparison/); assert.match(report, /\+32\.00% \| —/); }); + +void test('does not compare different schema versions', () => { + const baseline = { ...result('linux', 'X64', '10', 100, 120), schemaVersion: 1 }; + const current = result('linux', 'X64', '20', 100, 132); + const report = renderReport(current, baseline); + + assert.match(report, /No comparison: the baseline uses result schema 1, current is 2\./); + assert.match(report, /\+32\.00% \| —/); +}); diff --git a/crates/fspy_benchmark/README.md b/crates/fspy_benchmark/README.md index 137393eb4..bb9d50758 100644 --- a/crates/fspy_benchmark/README.md +++ b/crates/fspy_benchmark/README.md @@ -1,8 +1,10 @@ # fspy benchmark This benchmark measures the wall-clock overhead fspy adds to a filesystem-heavy child process. -The target starts four threads. Each thread opens 2,048 distinct, pre-created files for reading -and drops every handle immediately. +The target starts four threads. Each thread opens 2,048 distinct, pre-created files for reading, +dropping every handle immediately, and repeats this for 16 passes — 131,072 tracked opens per +run. The passes keep per-open interception cost dominant over process startup, whose variance on +CI runners would otherwise drown the tracked/untracked delta. The fixture is created before Criterion starts measuring. Tracked and untracked runs execute the same target with the same arguments, empty environment, working directory, and standard streams. diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 53e8d518c..ef709347b 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -15,11 +15,17 @@ use tokio::runtime::{Builder, Runtime}; use tokio_util::sync::CancellationToken; use vite_path::{AbsolutePath, AbsolutePathBuf}; +// Mirrored by .github/scripts/fspy-benchmark.mts for report rendering. +// The passes multiplier makes per-open interception cost dominate process +// startup in the measurement; with a single pass the tracked/untracked delta +// is the same order as spawn-time variance on CI runners. const THREAD_COUNT: usize = 4; const FILES_PER_THREAD: usize = 2048; +const PASSES: usize = 16; const TOTAL_FILE_COUNT: usize = THREAD_COUNT * FILES_PER_THREAD; const THREAD_COUNT_ARG: &str = "4"; const FILES_PER_THREAD_ARG: &str = "2048"; +const PASSES_ARG: &str = "16"; const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); #[cfg(all(target_os = "linux", target_arch = "x86_64"))] @@ -92,6 +98,8 @@ fn validate_tracked_run(runtime: &Runtime, target: &str, fixture: &AbsolutePath) .iter() .filter(|access| access.path.strip_path_prefix(fixture, |result| result.is_ok())) .count(); + // Requires every distinct fixture file to be observed, independent of + // whether repeated accesses to the same path are deduplicated. assert!( fixture_access_count >= TOTAL_FILE_COUNT, "expected at least {TOTAL_FILE_COUNT} fixture accesses, captured {fixture_access_count}" @@ -125,15 +133,20 @@ async fn run_tracked(target: &str, fixture: &AbsolutePath) -> ChildTermination { .expect("failed to wait for tracked benchmark target") } -fn benchmark_args(fixture: &AbsolutePath) -> [&OsStr; 3] { - [fixture.as_path().as_os_str(), OsStr::new(THREAD_COUNT_ARG), OsStr::new(FILES_PER_THREAD_ARG)] +fn benchmark_args(fixture: &AbsolutePath) -> [&OsStr; 4] { + [ + fixture.as_path().as_os_str(), + OsStr::new(THREAD_COUNT_ARG), + OsStr::new(FILES_PER_THREAD_ARG), + OsStr::new(PASSES_ARG), + ] } fn criterion_config() -> Criterion { Criterion::default() .sample_size(10) .warm_up_time(Duration::from_millis(500)) - .measurement_time(Duration::from_secs(1)) + .measurement_time(Duration::from_secs(5)) } criterion_group! { diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs index 7bc6b50a7..786fa5fe0 100644 --- a/crates/fspy_benchmark_target/src/main.rs +++ b/crates/fspy_benchmark_target/src/main.rs @@ -14,9 +14,10 @@ fn main() { .expect("fixture root must be absolute"); let thread_count = parse_usize(args.next(), "thread count"); let files_per_thread = parse_usize(args.next(), "files per thread"); + let passes = parse_usize(args.next(), "passes"); assert!(args.next().is_none(), "unexpected extra arguments"); - run(&root, thread_count, files_per_thread); + run(&root, thread_count, files_per_thread, passes); } fn parse_usize(value: Option, name: &str) -> usize { @@ -28,9 +29,10 @@ fn parse_usize(value: Option, name: &str) -> usize { .unwrap_or_else(|_| panic!("invalid {name}")) } -fn run(root: &AbsolutePath, thread_count: usize, files_per_thread: usize) { +fn run(root: &AbsolutePath, thread_count: usize, files_per_thread: usize, passes: usize) { assert!(thread_count > 1, "benchmark requires multiple threads"); assert!(files_per_thread > 0, "benchmark requires at least one file per thread"); + assert!(passes > 0, "benchmark requires at least one pass"); let paths = (0..thread_count * files_per_thread) .map(|index| root.join(vite_str::format!("file-{index:05}.txt"))) @@ -42,10 +44,12 @@ fn run(root: &AbsolutePath, thread_count: usize, files_per_thread: usize) { let barrier = Arc::clone(&barrier); scope.spawn(move || { barrier.wait(); - for path in thread_paths { - drop(File::open(path).unwrap_or_else(|error| { - panic!("failed to open {}: {error}", path.as_absolute_path()) - })); + for _ in 0..passes { + for path in thread_paths { + drop(File::open(path).unwrap_or_else(|error| { + panic!("failed to open {}: {error}", path.as_absolute_path()) + })); + } } }); } From a413732ce9f701f0ba34645105cbcfffc5a0010d Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 13:41:17 +0800 Subject: [PATCH 10/35] test(fspy): report per-open throughput via criterion Co-Authored-By: Claude Fable 5 --- crates/fspy_benchmark/benches/fspy.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index ef709347b..1fd01ef37 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -6,8 +6,8 @@ use std::{ }; use criterion::{ - BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, criterion_group, criterion_main, - measurement::WallTime, + BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, Throughput, criterion_group, + criterion_main, measurement::WallTime, }; use fspy::{ChildTermination, Command}; use tempfile::TempDir; @@ -36,6 +36,8 @@ fn benchmark(criterion: &mut Criterion) { let runtime = Builder::new_multi_thread().worker_threads(2).enable_all().build().unwrap(); let mut group = criterion.benchmark_group("fspy"); group.sampling_mode(SamplingMode::Flat); + // Criterion reports per-open time alongside wall time. + group.throughput(Throughput::Elements((TOTAL_FILE_COUNT * PASSES) as u64)); benchmark_target(&mut group, &runtime, "dynamic", DYNAMIC_TARGET, &fixture_root); From 0d215de8659bfa9b6089bf96d9ed0bd6378166ae Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 17:14:31 +0800 Subject: [PATCH 11/35] test(fspy): simplify overhead benchmark Co-authored-by: GPT-5 Codex --- .github/scripts/fspy-benchmark.mts | 415 ------------------ .github/scripts/fspy-benchmark.test.mts | 58 --- .github/scripts/tsconfig.json | 14 - .github/workflows/fspy-benchmark.yml | 153 +++---- Cargo.lock | 11 - crates/fspy_benchmark/Cargo.toml | 3 - crates/fspy_benchmark/README.md | 22 +- crates/fspy_benchmark/benches/fspy.rs | 145 +++--- .../fspy_benchmark_static_target/Cargo.toml | 7 - crates/fspy_benchmark_target/Cargo.toml | 4 - crates/fspy_benchmark_target/src/main.rs | 50 +-- 11 files changed, 136 insertions(+), 746 deletions(-) delete mode 100644 .github/scripts/fspy-benchmark.mts delete mode 100644 .github/scripts/fspy-benchmark.test.mts delete mode 100644 .github/scripts/tsconfig.json diff --git a/.github/scripts/fspy-benchmark.mts b/.github/scripts/fspy-benchmark.mts deleted file mode 100644 index ca8493b23..000000000 --- a/.github/scripts/fspy-benchmark.mts +++ /dev/null @@ -1,415 +0,0 @@ -#!/usr/bin/env node - -// Support script for the Fspy Benchmark workflow (.github/workflows/fspy-benchmark.yml). -// Executed directly with `node`, which strips the type annotations at load time, -// so only erasable TypeScript syntax is allowed (no enums, namespaces, etc.). - -import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises'; -import { arch, cpus, platform as osPlatform, release } from 'node:os'; -import { dirname, join } from 'node:path'; -import { pathToFileURL } from 'node:url'; - -// Mirrors THREAD_COUNT, FILES_PER_THREAD, and PASSES in -// crates/fspy_benchmark/benches/fspy.rs. Only used to describe the workload in -// reports. -const THREAD_COUNT = 4; -const FILES_PER_THREAD = 2048; -const PASSES = 16; - -// Bump when the workload or result shape changes; results from different -// schema versions are not comparable. -const SCHEMA_VERSION = 2; - -type Args = Map; - -/** Mean with its confidence interval, in nanoseconds. */ -interface EstimateSummary { - meanNs: number; - meanLowerNs: number; - meanUpperNs: number; -} - -/** One Criterion estimate, in nanoseconds. */ -interface Estimate extends EstimateSummary { - medianNs: number; -} - -/** The subset of a stored benchmark result that reports are rendered from. */ -interface ReportInput { - schemaVersion: number; - platform: string; - architecture: string; - os: { cpu: string }; - commit: string; - runId: string; - workload: { threadCount: number; filesPerThread: number; passes: number; totalOpens: number }; - benchmarks: Record; -} - -/** The full result JSON produced by `collect` and uploaded as an artifact. */ -interface BenchmarkResult extends ReportInput { - os: { platform: string; release: string; cpu: string }; - runner: string; - runAttempt: string; - event: string; - pullRequest: string; - benchmarks: Record; -} - -interface BaselineArtifact { - name: string; - runId: string; - createdAt: string; -} - -interface IssueComment { - id: number; - body?: string | null; - user?: { type?: string } | null; -} - -function parseArgs(args: string[]): Args { - const parsed = new Map(); - for (let index = 0; index < args.length; index += 2) { - const flag = args[index]; - const value = args[index + 1]; - if (!flag?.startsWith('--') || value === undefined) { - throw new Error(`invalid arguments near ${flag ?? ''}`); - } - parsed.set(flag.slice(2), value); - } - return parsed; -} - -function required(args: Args, name: string): string { - const value = args.get(name); - if (value === undefined || value === '') { - throw new Error(`missing --${name}`); - } - return value; -} - -function requiredEnv(name: string): string { - const value = process.env[name]; - if (!value) throw new Error(`missing ${name}`); - return value; -} - -async function githubJson(path: string, init: RequestInit = {}): Promise { - const token = requiredEnv('GITHUB_TOKEN'); - const response = await fetch(`https://api.github.com${path}`, { - ...init, - headers: { - Accept: 'application/vnd.github+json', - Authorization: `Bearer ${token}`, - 'X-GitHub-Api-Version': '2022-11-28', - 'User-Agent': 'vite-task-fspy-benchmark', - ...(init.body === undefined ? {} : { 'Content-Type': 'application/json' }), - }, - }); - if (!response.ok) { - throw new Error(`GitHub API ${response.status}: ${await response.text()}`); - } - return response.json() as Promise; -} - -async function findLatestArtifact( - name: string, - currentRunId: string, -): Promise { - const repository = requiredEnv('GITHUB_REPOSITORY'); - // The name filter is applied server-side, so one page covers the recent history. - const response = await githubJson<{ - artifacts: { - name: string; - expired: boolean; - created_at: string; - workflow_run?: { id: number } | null; - }[]; - }>(`/repos/${repository}/actions/artifacts?name=${encodeURIComponent(name)}&per_page=100`); - return ( - response.artifacts - .flatMap((artifact) => - !artifact.expired && artifact.workflow_run?.id != null - ? [ - { - name: artifact.name, - runId: String(artifact.workflow_run.id), - createdAt: artifact.created_at, - }, - ] - : [], - ) - // Skip the current run so a re-run attempt does not compare against the - // artifact its own previous attempt uploaded. - .filter((artifact) => artifact.runId !== currentRunId) - .sort((left, right) => Date.parse(right.createdAt) - Date.parse(left.createdAt))[0] - ); -} - -// Emits step outputs consumed by the workflow's baseline download and result -// upload steps. The baseline is always the latest `main` artifact: comparing -// against earlier runs of the same PR would mask regressions that accumulate -// over the PR's lifetime. -async function resolveBaseline(args: Args): Promise { - const outputPath = requiredEnv('GITHUB_OUTPUT'); - const platform = required(args, 'platform'); - const context = required(args, 'context'); - const currentRunId = requiredEnv('GITHUB_RUN_ID'); - const currentArtifactName = `fspy-benchmark-${platform}-${context}`; - const artifact = await findLatestArtifact(`fspy-benchmark-${platform}-main`, currentRunId); - - const outputs = [ - `current-artifact-name=${currentArtifactName}`, - `found=${artifact ? 'true' : 'false'}`, - ]; - if (artifact) { - outputs.push(`artifact-name=${artifact.name}`, `run-id=${artifact.runId}`); - } - await appendFile(outputPath, `${outputs.join('\n')}\n`); -} - -// Criterion writes the estimates of its most recent run to -// ///new/estimates.json. -async function readEstimate( - criterionRoot: string, - target: string, - mode: string, -): Promise { - const path = join(criterionRoot, target, mode, 'new', 'estimates.json'); - const estimates = JSON.parse(await readFile(path, 'utf8')) as { - mean: { - point_estimate: number; - confidence_interval: { lower_bound: number; upper_bound: number }; - }; - median: { point_estimate: number }; - }; - return { - meanNs: estimates.mean.point_estimate, - meanLowerNs: estimates.mean.confidence_interval.lower_bound, - meanUpperNs: estimates.mean.confidence_interval.upper_bound, - medianNs: estimates.median.point_estimate, - }; -} - -// Combines Criterion's output with run metadata into the JSON document that is -// uploaded as an artifact and later consumed as a comparison baseline. -async function collect(args: Args): Promise { - const criterionRoot = required(args, 'criterion-root'); - const output = required(args, 'output'); - const platform = required(args, 'platform'); - const expectStatic = args.get('expect-static') === 'true'; - const benchmarks: Record = { - 'dynamic/untracked': await readEstimate(criterionRoot, 'dynamic', 'untracked'), - 'dynamic/tracked': await readEstimate(criterionRoot, 'dynamic', 'tracked'), - }; - if (expectStatic) { - benchmarks['static/untracked'] = await readEstimate(criterionRoot, 'static', 'untracked'); - benchmarks['static/tracked'] = await readEstimate(criterionRoot, 'static', 'tracked'); - } - - const result: BenchmarkResult = { - schemaVersion: SCHEMA_VERSION, - platform, - architecture: process.env['RUNNER_ARCH'] ?? arch(), - os: { - platform: osPlatform(), - release: release(), - cpu: cpus()[0]?.model ?? 'unknown', - }, - runner: args.get('runner') ?? '', - commit: required(args, 'commit'), - runId: required(args, 'run-id'), - runAttempt: args.get('run-attempt') ?? '1', - event: required(args, 'event'), - pullRequest: args.get('pull-request') ?? '', - workload: { - threadCount: THREAD_COUNT, - filesPerThread: FILES_PER_THREAD, - passes: PASSES, - totalOpens: THREAD_COUNT * FILES_PER_THREAD * PASSES, - }, - benchmarks, - }; - - await mkdir(dirname(output), { recursive: true }); - await writeFile(output, `${JSON.stringify(result, null, 2)}\n`); -} - -/** Wall-clock cost of tracking, as tracked time over untracked time. */ -function ratio(result: ReportInput, target: string): number { - const tracked = result.benchmarks[`${target}/tracked`]; - const untracked = result.benchmarks[`${target}/untracked`]; - if (!tracked || !untracked) throw new Error(`missing benchmark data for ${target}`); - return tracked.meanNs / untracked.meanNs; -} - -function percentage(value: number): string { - const sign = value > 0 ? '+' : ''; - return `${sign}${(value * 100).toFixed(2)}%`; -} - -function duration(nanoseconds: number): string { - if (nanoseconds >= 1e9) return `${(nanoseconds / 1e9).toFixed(3)} s`; - if (nanoseconds >= 1e6) return `${(nanoseconds / 1e6).toFixed(3)} ms`; - if (nanoseconds >= 1e3) return `${(nanoseconds / 1e3).toFixed(3)} µs`; - return `${nanoseconds.toFixed(1)} ns`; -} - -/** Mean with the half-width of its confidence interval, e.g. `12.3 ms ±1.4%`. */ -function durationWithUncertainty(estimate: EstimateSummary): string { - const halfWidth = (estimate.meanUpperNs - estimate.meanLowerNs) / 2 / estimate.meanNs; - return `${duration(estimate.meanNs)} ±${(halfWidth * 100).toFixed(1)}%`; -} - -function resultLink(result: ReportInput): string { - const repository = process.env['GITHUB_REPOSITORY']; - if (!repository || !result.runId) return ''; - return `https://github.com/${repository}/actions/runs/${result.runId}`; -} - -export function renderReport(current: ReportInput, baseline?: ReportInput): string { - // Overhead ratios are only comparable within one architecture and one - // workload: a runner profile can move to different hardware over time, and - // schema bumps change what is measured. - const compatible = - baseline !== undefined && - current.schemaVersion === baseline.schemaVersion && - current.platform === baseline.platform && - current.architecture === baseline.architecture; - const lines = [`### ${current.platform.charAt(0).toUpperCase()}${current.platform.slice(1)}`, '']; - - if (compatible) { - const link = resultLink(baseline); - const description = link ? `[main](${link})` : 'main'; - lines.push(`Compared with ${description} at \`${baseline.commit.slice(0, 8)}\`.`, ''); - } else if (baseline) { - lines.push( - baseline.schemaVersion === current.schemaVersion - ? `No comparison: the baseline architecture is \`${baseline.architecture}\`, current is \`${current.architecture}\`.` - : `No comparison: the baseline uses result schema ${baseline.schemaVersion}, current is ${current.schemaVersion}.`, - '', - ); - } else { - lines.push('No `main` result was available for this runner.', ''); - } - - lines.push( - '| Target | Untracked | Tracked | fspy overhead | Normalized change |', - '| --- | ---: | ---: | ---: | ---: |', - ); - - const targets = Object.hasOwn(current.benchmarks, 'static/tracked') - ? ['dynamic', 'static'] - : ['dynamic']; - for (const target of targets) { - const untracked = current.benchmarks[`${target}/untracked`]; - const tracked = current.benchmarks[`${target}/tracked`]; - if (!untracked || !tracked) throw new Error(`missing benchmark data for ${target}`); - const overhead = ratio(current, target) - 1; - // Change of the overhead ratio relative to the baseline. Comparing ratios - // instead of absolute times cancels machine-speed differences between - // runner instances. - const normalizedChange = - compatible && Object.hasOwn(baseline.benchmarks, `${target}/tracked`) - ? ratio(current, target) / ratio(baseline, target) - 1 - : undefined; - lines.push( - `| ${target} | ${durationWithUncertainty(untracked)} | ${durationWithUncertainty(tracked)} | ${percentage(overhead)} | ${normalizedChange === undefined ? '—' : percentage(normalizedChange)} |`, - ); - } - - lines.push( - '', - `Workload: ${current.workload.threadCount} threads × ${current.workload.filesPerThread.toLocaleString('en-US')} files × ${current.workload.passes.toLocaleString('en-US')} passes, ${current.workload.totalOpens.toLocaleString('en-US')} total open-and-close operations.`, - '', - `\`${current.architecture}\` · ${current.os.cpu} · run [${current.runId}](${resultLink(current)})`, - '', - ); - return `${lines.join('\n')}\n`; -} - -async function compare(args: Args): Promise { - const current = JSON.parse(await readFile(required(args, 'current'), 'utf8')) as BenchmarkResult; - const baselinePath = args.get('baseline'); - let baseline: BenchmarkResult | undefined; - if (baselinePath) { - try { - baseline = JSON.parse(await readFile(baselinePath, 'utf8')) as BenchmarkResult; - } catch (error) { - // The workflow always passes --baseline; the file is absent when no - // baseline artifact was found. - if ((error as { code?: string }).code !== 'ENOENT') throw error; - } - } - const report = renderReport(current, baseline); - await writeFile(required(args, 'output'), report); - process.stdout.write(report); -} - -const COMMENT_MARKER = ''; - -// Creates or updates the sticky PR comment. The invisible marker identifies -// the comment, so later runs edit it in place instead of posting a new one. -async function comment(args: Args): Promise { - const repository = requiredEnv('GITHUB_REPOSITORY'); - const pullRequest = required(args, 'pull-request'); - const commit = required(args, 'commit'); - const resultsDir = required(args, 'results-dir'); - - const sections: string[] = []; - for (const platform of ['linux', 'macos', 'windows']) { - sections.push(await readFile(join(resultsDir, `${platform}.md`), 'utf8')); - } - const body = [ - COMMENT_MARKER, - '## fspy benchmark', - '', - `Results for commit \`${commit}\`. Each platform is compared with the latest \`main\` result.`, - '', - ...sections, - 'This comment is updated by the Fspy Benchmark workflow.', - '', - ].join('\n'); - - const comments = await githubJson( - `/repos/${repository}/issues/${pullRequest}/comments?per_page=100`, - ); - const existing = comments.find( - (existingComment) => - existingComment.user?.type === 'Bot' && existingComment.body?.includes(COMMENT_MARKER), - ); - const path = existing - ? `/repos/${repository}/issues/comments/${existing.id}` - : `/repos/${repository}/issues/${pullRequest}/comments`; - await githubJson(path, { - method: existing ? 'PATCH' : 'POST', - body: JSON.stringify({ body }), - }); -} - -async function main(): Promise { - const command = process.argv[2]; - const args = parseArgs(process.argv.slice(3)); - if (command === 'resolve-baseline') { - await resolveBaseline(args); - } else if (command === 'collect') { - await collect(args); - } else if (command === 'compare') { - await compare(args); - } else if (command === 'comment') { - await comment(args); - } else { - throw new Error(`unknown command: ${command ?? ''}`); - } -} - -// Run only when executed directly, so the test file can import renderReport. -const invokedPath = process.argv[1] ? pathToFileURL(process.argv[1]).href : ''; -if (import.meta.url === invokedPath) { - main().catch((error: unknown) => { - const message = error instanceof Error ? (error.stack ?? error.message) : String(error); - process.stderr.write(`${message}\n`); - process.exitCode = 1; - }); -} diff --git a/.github/scripts/fspy-benchmark.test.mts b/.github/scripts/fspy-benchmark.test.mts deleted file mode 100644 index 4b527790f..000000000 --- a/.github/scripts/fspy-benchmark.test.mts +++ /dev/null @@ -1,58 +0,0 @@ -import assert from 'node:assert/strict'; -import test from 'node:test'; - -import { renderReport } from './fspy-benchmark.mts'; - -function estimate(meanNs: number) { - return { meanNs, meanLowerNs: meanNs * 0.98, meanUpperNs: meanNs * 1.02 }; -} - -function result( - platform: string, - architecture: string, - runId: string, - untracked: number, - tracked: number, -) { - return { - schemaVersion: 2, - platform, - architecture, - os: { cpu: 'Test CPU' }, - commit: '1234567890abcdef', - runId, - workload: { threadCount: 4, filesPerThread: 2048, passes: 16, totalOpens: 131072 }, - benchmarks: { - 'dynamic/untracked': estimate(untracked), - 'dynamic/tracked': estimate(tracked), - }, - }; -} - -void test('renders normalized change against a compatible baseline', () => { - const baseline = result('linux', 'X64', '10', 100, 120); - const current = result('linux', 'X64', '20', 100, 132); - const report = renderReport(current, baseline); - - assert.match(report, /fspy overhead \| Normalized change/); - assert.match(report, /100\.0 ns ±2\.0% \| 132\.0 ns ±2\.0% \| \+32\.00% \| \+10\.00%/); - assert.match(report, /Compared with main at `12345678`/); -}); - -void test('does not compare different architectures', () => { - const baseline = result('macos', 'X64', '10', 100, 120); - const current = result('macos', 'ARM64', '20', 100, 132); - const report = renderReport(current, baseline); - - assert.match(report, /No comparison/); - assert.match(report, /\+32\.00% \| —/); -}); - -void test('does not compare different schema versions', () => { - const baseline = { ...result('linux', 'X64', '10', 100, 120), schemaVersion: 1 }; - const current = result('linux', 'X64', '20', 100, 132); - const report = renderReport(current, baseline); - - assert.match(report, /No comparison: the baseline uses result schema 1, current is 2\./); - assert.match(report, /\+32\.00% \| —/); -}); diff --git a/.github/scripts/tsconfig.json b/.github/scripts/tsconfig.json deleted file mode 100644 index 07f8c9b9b..000000000 --- a/.github/scripts/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "extends": "@tsconfig/strictest/tsconfig.json", - "compilerOptions": { - "allowImportingTsExtensions": true, - "erasableSyntaxOnly": true, - "module": "NodeNext", - "moduleResolution": "NodeNext", - "noEmit": true, - "target": "ES2022", - "lib": ["ES2022"], - "types": ["node"] - }, - "include": ["*.mts"] -} diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index f7a90af24..29707b726 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -40,6 +40,7 @@ jobs: runs-on: ${{ matrix.os }} env: CARGO_TARGET_DIR: ${{ github.workspace }}/target + CRITERION_HOME: ${{ github.workspace }}/target/criterion steps: - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 @@ -52,88 +53,67 @@ jobs: save-cache: ${{ github.ref_name == 'main' }} cache-key: fspy-benchmark-${{ matrix.platform }} - # The support script is TypeScript; running it needs the .node-version - # Node, which strips types natively. Runner-preinstalled Node may be - # older. - - uses: oxc-project/setup-node@4c588e9266bd930b6ddc34307df0659ed511d187 # v1.3.1 - - name: Install static target if: matrix.static run: rustup target add x86_64-unknown-linux-musl - - name: Set benchmark context - id: context + - name: Download main baseline + id: main-baseline + if: github.event_name != 'push' + uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21 + with: + github_token: ${{ github.token }} + workflow: fspy-benchmark.yml + workflow_conclusion: success + branch: main + event: push + name: fspy-benchmark-baseline-v1-${{ matrix.platform }} + path: ${{ env.CRITERION_HOME }}/fspy + search_artifacts: true + if_no_artifact_found: warn + + - name: Run benchmark env: - PR_NUMBER: ${{ github.event.pull_request.number }} + BASELINE_FOUND: ${{ steps.main-baseline.outputs.found_artifact }} run: | - # Only real main-branch runs may publish the main baseline artifact; - # a workflow_dispatch run from another branch must not pollute it. - if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then - context="pr-$PR_NUMBER" - pull_request="$PR_NUMBER" - elif [[ "$GITHUB_REF" == "refs/heads/main" ]]; then - context="main" - pull_request="none" + mkdir -p .benchmark-results + if [[ "$GITHUB_EVENT_NAME" == "push" || "$BASELINE_FOUND" != "true" ]]; then + baseline=(--save-baseline main) else - context="dispatch" - pull_request="none" + baseline=(--baseline main) fi - echo "name=$context" >> "$GITHUB_OUTPUT" - echo "pull-request=$pull_request" >> "$GITHUB_OUTPUT" - - - name: Find comparison baseline - id: baseline - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - node .github/scripts/fspy-benchmark.mts resolve-baseline - --platform "${{ matrix.platform }}" - --context "${{ steps.context.outputs.name }}" - - - name: Download comparison baseline - if: steps.baseline.outputs.found == 'true' - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: ${{ steps.baseline.outputs.artifact-name }} - path: .benchmark-baseline - github-token: ${{ github.token }} - repository: ${{ github.repository }} - run-id: ${{ steps.baseline.outputs.run-id }} - - - name: Run benchmark - run: cargo bench --locked -p fspy_benchmark --bench fspy - - - name: Collect benchmark results - run: >- - node .github/scripts/fspy-benchmark.mts collect - --criterion-root "$CARGO_TARGET_DIR/criterion/fspy" - --output ".benchmark-results/${{ matrix.platform }}.json" - --platform "${{ matrix.platform }}" - --expect-static "${{ matrix.static }}" - --runner "${{ matrix.os }}" - --commit "$GITHUB_SHA" - --run-id "$GITHUB_RUN_ID" - --run-attempt "$GITHUB_RUN_ATTEMPT" - --event "$GITHUB_EVENT_NAME" - --pull-request "${{ steps.context.outputs.pull-request }}" - - - name: Compare benchmark results - run: >- - node .github/scripts/fspy-benchmark.mts compare - --current ".benchmark-results/${{ matrix.platform }}.json" - --baseline ".benchmark-baseline/${{ matrix.platform }}.json" - --output ".benchmark-results/${{ matrix.platform }}.md" + set -o pipefail + cargo bench --locked -p fspy_benchmark --bench fspy -- \ + --color never -n "${baseline[@]}" | + tee ".benchmark-results/${{ matrix.platform }}.txt" - name: Add job summary - run: cat ".benchmark-results/${{ matrix.platform }}.md" >> "$GITHUB_STEP_SUMMARY" + run: | + { + echo '```text' + cat ".benchmark-results/${{ matrix.platform }}.txt" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload main baseline + if: github.event_name == 'push' && github.ref_name == 'main' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: fspy-benchmark-baseline-v1-${{ matrix.platform }} + path: ${{ env.CRITERION_HOME }}/fspy + if-no-files-found: error + overwrite: true + retention-days: 90 - - name: Upload benchmark result + - name: Upload benchmark report + if: github.event_name == 'pull_request' + # Matrix jobs have isolated filesystems, so persist each report for the + # comment job to combine into one sticky comment after all platforms finish. uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: ${{ steps.baseline.outputs.current-artifact-name }} - path: .benchmark-results/${{ matrix.platform }}.* + name: fspy-benchmark-report-${{ matrix.platform }} + path: .benchmark-results/${{ matrix.platform }}.txt if-no-files-found: error - include-hidden-files: true overwrite: true retention-days: 90 @@ -146,27 +126,32 @@ jobs: runs-on: namespace-profile-linux-x64-default permissions: actions: read - contents: read pull-requests: write steps: - - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 - - - uses: oxc-project/setup-node@4c588e9266bd930b6ddc34307df0659ed511d187 # v1.3.1 - - - name: Download current results + - name: Download benchmark reports uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - pattern: fspy-benchmark-*-pr-${{ github.event.pull_request.number }} + pattern: fspy-benchmark-report-* path: .benchmark-results merge-multiple: true + - name: Assemble comment + run: | + { + echo '## fspy benchmark' + echo + for platform in linux macos windows; do + echo "### $platform" + echo + echo '```text' + cat ".benchmark-results/$platform.txt" + echo '```' + echo + done + } > .benchmark-results/comment.md + - name: Update PR comment - env: - GITHUB_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ github.event.pull_request.number }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: >- - node .github/scripts/fspy-benchmark.mts comment - --results-dir .benchmark-results - --pull-request "$PR_NUMBER" - --commit "$HEAD_SHA" + uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v3.0.4 + with: + header: fspy-benchmark + path: .benchmark-results/comment.md diff --git a/Cargo.lock b/Cargo.lock index f5a4464cd..1a71e1516 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1307,28 +1307,17 @@ dependencies = [ "fspy", "fspy_benchmark_static_target", "fspy_benchmark_target", - "tempfile", "tokio", "tokio-util", - "vite_path", - "vite_str", ] [[package]] name = "fspy_benchmark_static_target" version = "0.0.0" -dependencies = [ - "vite_path", - "vite_str", -] [[package]] name = "fspy_benchmark_target" version = "0.0.0" -dependencies = [ - "vite_path", - "vite_str", -] [[package]] name = "fspy_detours_sys" diff --git a/crates/fspy_benchmark/Cargo.toml b/crates/fspy_benchmark/Cargo.toml index 18330d1b1..c6fbe7b67 100644 --- a/crates/fspy_benchmark/Cargo.toml +++ b/crates/fspy_benchmark/Cargo.toml @@ -23,11 +23,8 @@ harness = false criterion2 = { workspace = true } fspy = { workspace = true } fspy_benchmark_target = { workspace = true } -tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "process", "rt-multi-thread"] } tokio-util = { workspace = true } -vite_path = { workspace = true } -vite_str = { workspace = true } [target.'cfg(all(target_os = "linux", target_arch = "x86_64"))'.dev-dependencies] fspy_benchmark_static_target = { path = "../fspy_benchmark_static_target", artifact = "bin", target = "x86_64-unknown-linux-musl" } diff --git a/crates/fspy_benchmark/README.md b/crates/fspy_benchmark/README.md index bb9d50758..00d39f68f 100644 --- a/crates/fspy_benchmark/README.md +++ b/crates/fspy_benchmark/README.md @@ -1,28 +1,12 @@ # fspy benchmark -This benchmark measures the wall-clock overhead fspy adds to a filesystem-heavy child process. -The target starts four threads. Each thread opens 2,048 distinct, pre-created files for reading, -dropping every handle immediately, and repeats this for 16 passes — 131,072 tracked opens per -run. The passes keep per-open interception cost dominant over process startup, whose variance on -CI runners would otherwise drown the tracked/untracked delta. +Measures fspy's wall-clock overhead while multiple threads repeatedly open a nonexistent path. +Criterion reports the nonnegative difference between tracked and untracked runs. -The fixture is created before Criterion starts measuring. Tracked and untracked runs execute the -same target with the same arguments, empty environment, working directory, and standard streams. -The timer covers spawning the target through process termination and fspy finalization. A tracked -warmup verifies that all fixture accesses were captured outside the timed section. - -On Linux, the suite measures two targets: - -- `dynamic` is dynamically linked and exercises `LD_PRELOAD`. -- `static` is linked for `x86_64-unknown-linux-musl` and exercises seccomp user notification. - -macOS measures `DYLD_INSERT_LIBRARIES`, and Windows measures Detours injection. +Linux measures dynamic and static targets. macOS and Windows measure their preload implementations. Run the benchmark with: ```sh just benchmark-fspy ``` - -CI runs it on the Linux, macOS, and Windows Namespace runner profiles. Each result is compared with -the latest `main` result. diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 1fd01ef37..7929f917a 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -1,48 +1,35 @@ use std::{ - ffi::OsStr, - fs::File, process::{ExitStatus, Stdio}, - time::Duration, + time::{Duration, Instant}, }; use criterion::{ - BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, Throughput, criterion_group, - criterion_main, measurement::WallTime, + BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, criterion_group, criterion_main, + measurement::WallTime, }; use fspy::{ChildTermination, Command}; -use tempfile::TempDir; use tokio::runtime::{Builder, Runtime}; use tokio_util::sync::CancellationToken; -use vite_path::{AbsolutePath, AbsolutePathBuf}; - -// Mirrored by .github/scripts/fspy-benchmark.mts for report rendering. -// The passes multiplier makes per-open interception cost dominate process -// startup in the measurement; with a single pass the tracked/untracked delta -// is the same order as spawn-time variance on CI runners. -const THREAD_COUNT: usize = 4; -const FILES_PER_THREAD: usize = 2048; -const PASSES: usize = 16; -const TOTAL_FILE_COUNT: usize = THREAD_COUNT * FILES_PER_THREAD; -const THREAD_COUNT_ARG: &str = "4"; -const FILES_PER_THREAD_ARG: &str = "2048"; -const PASSES_ARG: &str = "16"; + const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); #[cfg(all(target_os = "linux", target_arch = "x86_64"))] const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); +#[cfg(unix)] +const MISSING_PATH: &str = "/.fspy-benchmark-missing"; +#[cfg(windows)] +const MISSING_PATH: &str = r"C:\.fspy-benchmark-missing"; + fn benchmark(criterion: &mut Criterion) { - let (_fixture, fixture_root) = create_fixture(); let runtime = Builder::new_multi_thread().worker_threads(2).enable_all().build().unwrap(); let mut group = criterion.benchmark_group("fspy"); group.sampling_mode(SamplingMode::Flat); - // Criterion reports per-open time alongside wall time. - group.throughput(Throughput::Elements((TOTAL_FILE_COUNT * PASSES) as u64)); - benchmark_target(&mut group, &runtime, "dynamic", DYNAMIC_TARGET, &fixture_root); + benchmark_target(&mut group, &runtime, "dynamic", DYNAMIC_TARGET); #[cfg(all(target_os = "linux", target_arch = "x86_64"))] - benchmark_target(&mut group, &runtime, "static", STATIC_TARGET, &fixture_root); + benchmark_target(&mut group, &runtime, "static", STATIC_TARGET); group.finish(); } @@ -52,80 +39,63 @@ fn benchmark_target( runtime: &Runtime, target_name: &str, target: &str, - fixture: &AbsolutePath, ) { - validate_tracked_run(runtime, target, fixture); - - group.bench_with_input( - BenchmarkId::new(target_name, "untracked"), - &target, - |bencher, target| { - bencher.iter_with_setup_wrapper(|runner| { - let status = runner.run(|| runtime.block_on(run_untracked(target, fixture))); - assert!(status.success(), "untracked benchmark target failed: {status}"); - }); - }, - ); - - group.bench_with_input(BenchmarkId::new(target_name, "tracked"), &target, |bencher, target| { - bencher.iter_with_setup_wrapper(|runner| { - let termination = runner.run(|| runtime.block_on(run_tracked(target, fixture))); - assert!( - termination.status.success(), - "tracked benchmark target failed: {}", - termination.status - ); + validate_tracked_run(runtime, target); + + group.bench_with_input(BenchmarkId::from_parameter(target_name), &target, |bencher, target| { + bencher.iter_custom(|iterations| { + let mut tracked = Duration::ZERO; + let mut untracked = Duration::ZERO; + for _ in 0..iterations { + tracked += measure_tracked(runtime, target); + untracked += measure_untracked(runtime, target); + } + // Keep clamped samples reportable at a stable 1 ns per-iteration floor. + tracked.saturating_sub(untracked).max(Duration::from_nanos(iterations)) }); }); } -fn create_fixture() -> (TempDir, AbsolutePathBuf) { - let fixture = tempfile::tempdir().expect("failed to create benchmark fixture"); - let fixture_root = AbsolutePathBuf::new(fixture.path().to_path_buf()) - .expect("temporary path must be absolute"); - for index in 0..TOTAL_FILE_COUNT { - let path = fixture_root.join(vite_str::format!("file-{index:05}.txt")); - File::create(&path).unwrap_or_else(|error| { - panic!("failed to create {}: {error}", path.as_absolute_path()) - }); - } - (fixture, fixture_root) +fn validate_tracked_run(runtime: &Runtime, target: &str) { + let termination = runtime.block_on(run_tracked(target)); + assert!(termination.status.success(), "benchmark target failed: {}", termination.status); + let captured_missing_access = termination.path_accesses.iter().any(|access| { + access.path.strip_path_prefix(MISSING_PATH, |result| { + result.is_ok_and(|path| path.as_os_str().is_empty()) + }) + }); + assert!(captured_missing_access, "failed to capture access to {MISSING_PATH}"); } -fn validate_tracked_run(runtime: &Runtime, target: &str, fixture: &AbsolutePath) { - let termination = runtime.block_on(run_tracked(target, fixture)); - assert!(termination.status.success(), "benchmark target failed: {}", termination.status); - let fixture_access_count = termination - .path_accesses - .iter() - .filter(|access| access.path.strip_path_prefix(fixture, |result| result.is_ok())) - .count(); - // Requires every distinct fixture file to be observed, independent of - // whether repeated accesses to the same path are deduplicated. +fn measure_untracked(runtime: &Runtime, target: &str) -> Duration { + let start = Instant::now(); + let status = runtime.block_on(run_untracked(target)); + let elapsed = start.elapsed(); + assert!(status.success(), "untracked benchmark target failed: {status}"); + elapsed +} + +fn measure_tracked(runtime: &Runtime, target: &str) -> Duration { + let start = Instant::now(); + let termination = runtime.block_on(run_tracked(target)); + let elapsed = start.elapsed(); assert!( - fixture_access_count >= TOTAL_FILE_COUNT, - "expected at least {TOTAL_FILE_COUNT} fixture accesses, captured {fixture_access_count}" + termination.status.success(), + "tracked benchmark target failed: {}", + termination.status ); + elapsed } -async fn run_untracked(target: &str, fixture: &AbsolutePath) -> ExitStatus { +async fn run_untracked(target: &str) -> ExitStatus { let mut command = tokio::process::Command::new(target); - command - .env_clear() - .args(benchmark_args(fixture)) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::inherit()); + command.env_clear().stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::inherit()); command.status().await.expect("failed to run untracked benchmark target") } -async fn run_tracked(target: &str, fixture: &AbsolutePath) -> ChildTermination { +async fn run_tracked(target: &str) -> ChildTermination { let mut command = Command::new(target); - command - .args(benchmark_args(fixture)) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::inherit()); + command.stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::inherit()); command .spawn(CancellationToken::new()) .await @@ -135,20 +105,11 @@ async fn run_tracked(target: &str, fixture: &AbsolutePath) -> ChildTermination { .expect("failed to wait for tracked benchmark target") } -fn benchmark_args(fixture: &AbsolutePath) -> [&OsStr; 4] { - [ - fixture.as_path().as_os_str(), - OsStr::new(THREAD_COUNT_ARG), - OsStr::new(FILES_PER_THREAD_ARG), - OsStr::new(PASSES_ARG), - ] -} - fn criterion_config() -> Criterion { Criterion::default() .sample_size(10) .warm_up_time(Duration::from_millis(500)) - .measurement_time(Duration::from_secs(5)) + .measurement_time(Duration::from_secs(60)) } criterion_group! { diff --git a/crates/fspy_benchmark_static_target/Cargo.toml b/crates/fspy_benchmark_static_target/Cargo.toml index b296739b1..d8daccfe0 100644 --- a/crates/fspy_benchmark_static_target/Cargo.toml +++ b/crates/fspy_benchmark_static_target/Cargo.toml @@ -10,13 +10,6 @@ rust-version.workspace = true [lints] workspace = true -[dependencies] -vite_path = { workspace = true } -vite_str = { workspace = true } - -[package.metadata.cargo-shear] -ignored = ["vite_path", "vite_str"] - [[bin]] name = "fspy_benchmark_static_target" test = false diff --git a/crates/fspy_benchmark_target/Cargo.toml b/crates/fspy_benchmark_target/Cargo.toml index c5d917b47..ea5536db2 100644 --- a/crates/fspy_benchmark_target/Cargo.toml +++ b/crates/fspy_benchmark_target/Cargo.toml @@ -10,10 +10,6 @@ rust-version.workspace = true [lints] workspace = true -[dependencies] -vite_path = { workspace = true } -vite_str = { workspace = true } - [[bin]] name = "fspy_benchmark_target" test = false diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs index 786fa5fe0..9ce595970 100644 --- a/crates/fspy_benchmark_target/src/main.rs +++ b/crates/fspy_benchmark_target/src/main.rs @@ -1,55 +1,27 @@ use std::{ - env, fs::File, sync::{Arc, Barrier}, thread, }; -use vite_path::{AbsolutePath, AbsolutePathBuf}; +const THREAD_COUNT: usize = 4; +const OPEN_COUNT_PER_THREAD: usize = 32_768; -fn main() { - let mut args = env::args_os(); - let _program = args.next(); - let root = AbsolutePathBuf::new(args.next().expect("missing fixture root").into()) - .expect("fixture root must be absolute"); - let thread_count = parse_usize(args.next(), "thread count"); - let files_per_thread = parse_usize(args.next(), "files per thread"); - let passes = parse_usize(args.next(), "passes"); - assert!(args.next().is_none(), "unexpected extra arguments"); - - run(&root, thread_count, files_per_thread, passes); -} +#[cfg(unix)] +const MISSING_PATH: &str = "/.fspy-benchmark-missing"; +#[cfg(windows)] +const MISSING_PATH: &str = r"C:\.fspy-benchmark-missing"; -fn parse_usize(value: Option, name: &str) -> usize { - value - .unwrap_or_else(|| panic!("missing {name}")) - .into_string() - .unwrap_or_else(|_| panic!("{name} is not UTF-8")) - .parse() - .unwrap_or_else(|_| panic!("invalid {name}")) -} - -fn run(root: &AbsolutePath, thread_count: usize, files_per_thread: usize, passes: usize) { - assert!(thread_count > 1, "benchmark requires multiple threads"); - assert!(files_per_thread > 0, "benchmark requires at least one file per thread"); - assert!(passes > 0, "benchmark requires at least one pass"); - - let paths = (0..thread_count * files_per_thread) - .map(|index| root.join(vite_str::format!("file-{index:05}.txt"))) - .collect::>(); - let barrier = Arc::new(Barrier::new(thread_count)); +fn main() { + let barrier = Arc::new(Barrier::new(THREAD_COUNT)); thread::scope(|scope| { - for thread_paths in paths.chunks_exact(files_per_thread) { + for _ in 0..THREAD_COUNT { let barrier = Arc::clone(&barrier); scope.spawn(move || { barrier.wait(); - for _ in 0..passes { - for path in thread_paths { - drop(File::open(path).unwrap_or_else(|error| { - panic!("failed to open {}: {error}", path.as_absolute_path()) - })); - } + for _ in 0..OPEN_COUNT_PER_THREAD { + drop(File::open(MISSING_PATH)); } }); } From 86cd596dcee5de65d0ae8afa3c732010062d51ef Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 17:21:18 +0800 Subject: [PATCH 12/35] test(fspy): stabilize benchmark sampling Co-authored-by: GPT-5 Codex --- .github/workflows/fspy-benchmark.yml | 60 +++++++++++++++++++++++++-- crates/fspy_benchmark/benches/fspy.rs | 4 +- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index 29707b726..55a70fff2 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -72,15 +72,40 @@ jobs: search_artifacts: true if_no_artifact_found: warn + - name: Download pull request baseline + id: pr-baseline + if: >- + github.event_name == 'pull_request' && + steps.main-baseline.outputs.found_artifact != 'true' && + github.event.pull_request.head.repo.full_name == github.repository + uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21 + with: + github_token: ${{ github.token }} + workflow: fspy-benchmark.yml + workflow_conclusion: success + branch: ${{ github.event.pull_request.head.ref }} + event: pull_request + name: fspy-benchmark-baseline-pr-v1-${{ github.event.pull_request.number }}-${{ matrix.platform }} + path: ${{ env.CRITERION_HOME }}/fspy + search_artifacts: true + if_no_artifact_found: warn + - name: Run benchmark env: - BASELINE_FOUND: ${{ steps.main-baseline.outputs.found_artifact }} + MAIN_BASELINE_FOUND: ${{ steps.main-baseline.outputs.found_artifact }} + PR_BASELINE_FOUND: ${{ steps.pr-baseline.outputs.found_artifact }} run: | mkdir -p .benchmark-results - if [[ "$GITHUB_EVENT_NAME" == "push" || "$BASELINE_FOUND" != "true" ]]; then + if [[ "$GITHUB_EVENT_NAME" == "push" ]]; then baseline=(--save-baseline main) - else + elif [[ "$MAIN_BASELINE_FOUND" == "true" ]]; then baseline=(--baseline main) + elif [[ "$PR_BASELINE_FOUND" == "true" ]]; then + baseline=(--baseline pr) + elif [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then + baseline=(--save-baseline pr) + else + baseline=(--save-baseline main) fi set -o pipefail cargo bench --locked -p fspy_benchmark --bench fspy -- \ @@ -95,6 +120,20 @@ jobs: echo '```' } >> "$GITHUB_STEP_SUMMARY" + - name: Save pull request baseline + if: >- + github.event_name == 'pull_request' && + steps.main-baseline.outputs.found_artifact != 'true' && + github.event.pull_request.head.repo.full_name == github.repository + run: | + for target in dynamic static; do + source="$CRITERION_HOME/fspy/$target/new" + if [[ -d "$source" ]]; then + mkdir -p "$CRITERION_HOME/fspy/$target/pr" + cp -R "$source/." "$CRITERION_HOME/fspy/$target/pr/" + fi + done + - name: Upload main baseline if: github.event_name == 'push' && github.ref_name == 'main' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -105,6 +144,21 @@ jobs: overwrite: true retention-days: 90 + - name: Upload pull request baseline + if: >- + github.event_name == 'pull_request' && + steps.main-baseline.outputs.found_artifact != 'true' && + github.event.pull_request.head.repo.full_name == github.repository + # Until main has a baseline, keep Criterion's data so the next commit + # in this pull request can compare against its latest run. + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: fspy-benchmark-baseline-pr-v1-${{ github.event.pull_request.number }}-${{ matrix.platform }} + path: ${{ env.CRITERION_HOME }}/fspy + if-no-files-found: error + overwrite: true + retention-days: 90 + - name: Upload benchmark report if: github.event_name == 'pull_request' # Matrix jobs have isolated filesystems, so persist each report for the diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 7929f917a..30d2d4138 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -108,8 +108,8 @@ async fn run_tracked(target: &str) -> ChildTermination { fn criterion_config() -> Criterion { Criterion::default() .sample_size(10) - .warm_up_time(Duration::from_millis(500)) - .measurement_time(Duration::from_secs(60)) + .warm_up_time(Duration::from_secs(5)) + .measurement_time(Duration::from_secs(180)) } criterion_group! { From 0fdcc35a2dc15691c9cee50b5d5b030e5ab52bac Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 17:30:12 +0800 Subject: [PATCH 13/35] test(fspy): compare benchmarks with main only Co-authored-by: GPT-5 Codex --- .github/workflows/fspy-benchmark.yml | 56 +--------------------------- 1 file changed, 1 insertion(+), 55 deletions(-) diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index 55a70fff2..8812add2e 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -72,38 +72,13 @@ jobs: search_artifacts: true if_no_artifact_found: warn - - name: Download pull request baseline - id: pr-baseline - if: >- - github.event_name == 'pull_request' && - steps.main-baseline.outputs.found_artifact != 'true' && - github.event.pull_request.head.repo.full_name == github.repository - uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21 - with: - github_token: ${{ github.token }} - workflow: fspy-benchmark.yml - workflow_conclusion: success - branch: ${{ github.event.pull_request.head.ref }} - event: pull_request - name: fspy-benchmark-baseline-pr-v1-${{ github.event.pull_request.number }}-${{ matrix.platform }} - path: ${{ env.CRITERION_HOME }}/fspy - search_artifacts: true - if_no_artifact_found: warn - - name: Run benchmark env: MAIN_BASELINE_FOUND: ${{ steps.main-baseline.outputs.found_artifact }} - PR_BASELINE_FOUND: ${{ steps.pr-baseline.outputs.found_artifact }} run: | mkdir -p .benchmark-results - if [[ "$GITHUB_EVENT_NAME" == "push" ]]; then - baseline=(--save-baseline main) - elif [[ "$MAIN_BASELINE_FOUND" == "true" ]]; then + if [[ "$GITHUB_EVENT_NAME" != "push" && "$MAIN_BASELINE_FOUND" == "true" ]]; then baseline=(--baseline main) - elif [[ "$PR_BASELINE_FOUND" == "true" ]]; then - baseline=(--baseline pr) - elif [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then - baseline=(--save-baseline pr) else baseline=(--save-baseline main) fi @@ -120,20 +95,6 @@ jobs: echo '```' } >> "$GITHUB_STEP_SUMMARY" - - name: Save pull request baseline - if: >- - github.event_name == 'pull_request' && - steps.main-baseline.outputs.found_artifact != 'true' && - github.event.pull_request.head.repo.full_name == github.repository - run: | - for target in dynamic static; do - source="$CRITERION_HOME/fspy/$target/new" - if [[ -d "$source" ]]; then - mkdir -p "$CRITERION_HOME/fspy/$target/pr" - cp -R "$source/." "$CRITERION_HOME/fspy/$target/pr/" - fi - done - - name: Upload main baseline if: github.event_name == 'push' && github.ref_name == 'main' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -144,21 +105,6 @@ jobs: overwrite: true retention-days: 90 - - name: Upload pull request baseline - if: >- - github.event_name == 'pull_request' && - steps.main-baseline.outputs.found_artifact != 'true' && - github.event.pull_request.head.repo.full_name == github.repository - # Until main has a baseline, keep Criterion's data so the next commit - # in this pull request can compare against its latest run. - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: fspy-benchmark-baseline-pr-v1-${{ github.event.pull_request.number }}-${{ matrix.platform }} - path: ${{ env.CRITERION_HOME }}/fspy - if-no-files-found: error - overwrite: true - retention-days: 90 - - name: Upload benchmark report if: github.event_name == 'pull_request' # Matrix jobs have isolated filesystems, so persist each report for the From 6e8a4762c83e3012e554375a5a5466532bf41bf4 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 17:37:56 +0800 Subject: [PATCH 14/35] test(fspy): stabilize detailed benchmark report Co-authored-by: GPT-5 Codex --- .github/workflows/fspy-benchmark.yml | 11 ++++++++--- crates/fspy_benchmark/benches/fspy.rs | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index 8812add2e..a0761fb16 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -1,4 +1,4 @@ -name: Fspy Benchmark +name: fspy benchmark permissions: {} @@ -83,15 +83,19 @@ jobs: baseline=(--save-baseline main) fi set -o pipefail + result_file=".benchmark-results/${{ matrix.platform }}.txt" cargo bench --locked -p fspy_benchmark --bench fspy -- \ - --color never -n "${baseline[@]}" | - tee ".benchmark-results/${{ matrix.platform }}.txt" + --color never --verbose -n "${baseline[@]}" | + tee "$result_file" + result="$(<"$result_file")" + printf '%s' "$result" > "$result_file" - name: Add job summary run: | { echo '```text' cat ".benchmark-results/${{ matrix.platform }}.txt" + echo echo '```' } >> "$GITHUB_STEP_SUMMARY" @@ -145,6 +149,7 @@ jobs: echo echo '```text' cat ".benchmark-results/$platform.txt" + echo echo '```' echo done diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 30d2d4138..eb6934579 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -47,8 +47,8 @@ fn benchmark_target( let mut tracked = Duration::ZERO; let mut untracked = Duration::ZERO; for _ in 0..iterations { - tracked += measure_tracked(runtime, target); untracked += measure_untracked(runtime, target); + tracked += measure_tracked(runtime, target); } // Keep clamped samples reportable at a stable 1 ns per-iteration floor. tracked.saturating_sub(untracked).max(Duration::from_nanos(iterations)) From 2be63d6f108a933fca6dd7b755c48ae75ca41270 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 17:48:46 +0800 Subject: [PATCH 15/35] test(fspy): reduce benchmark measurement noise Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark/benches/fspy.rs | 2 +- crates/fspy_benchmark_target/src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index eb6934579..3c868075e 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -109,7 +109,7 @@ fn criterion_config() -> Criterion { Criterion::default() .sample_size(10) .warm_up_time(Duration::from_secs(5)) - .measurement_time(Duration::from_secs(180)) + .measurement_time(Duration::from_secs(60)) } criterion_group! { diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs index 9ce595970..deb04d3e3 100644 --- a/crates/fspy_benchmark_target/src/main.rs +++ b/crates/fspy_benchmark_target/src/main.rs @@ -5,7 +5,7 @@ use std::{ }; const THREAD_COUNT: usize = 4; -const OPEN_COUNT_PER_THREAD: usize = 32_768; +const OPEN_COUNT_PER_THREAD: usize = 8_192; #[cfg(unix)] const MISSING_PATH: &str = "/.fspy-benchmark-missing"; From 7bef7e47a8f6fbcfb8b4550e411be71f18eecd8d Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 17:54:09 +0800 Subject: [PATCH 16/35] test(fspy): bound dynamic benchmark sampling Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark/benches/fspy.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 3c868075e..7d67753e4 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -12,6 +12,7 @@ use tokio::runtime::{Builder, Runtime}; use tokio_util::sync::CancellationToken; const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); +const MIN_DYNAMIC_ITERATIONS: u64 = 100; #[cfg(all(target_os = "linux", target_arch = "x86_64"))] const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); @@ -41,17 +42,22 @@ fn benchmark_target( target: &str, ) { validate_tracked_run(runtime, target); + let minimum_iterations = if target_name == "dynamic" { MIN_DYNAMIC_ITERATIONS } else { 0 }; group.bench_with_input(BenchmarkId::from_parameter(target_name), &target, |bencher, target| { bencher.iter_custom(|iterations| { + let measured_iterations = iterations.max(minimum_iterations); let mut tracked = Duration::ZERO; let mut untracked = Duration::ZERO; - for _ in 0..iterations { + for _ in 0..measured_iterations { untracked += measure_untracked(runtime, target); tracked += measure_tracked(runtime, target); } - // Keep clamped samples reportable at a stable 1 ns per-iteration floor. - tracked.saturating_sub(untracked).max(Duration::from_nanos(iterations)) + tracked + .saturating_sub(untracked) + .div_f64(measured_iterations as f64) + .max(Duration::from_nanos(1)) + .mul_f64(iterations as f64) }); }); } From 835453c9d9b8eb942821136eafb0f20cb43222b3 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 18:03:57 +0800 Subject: [PATCH 17/35] test(fspy): increase dynamic process samples Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark/benches/fspy.rs | 5 +++-- crates/fspy_benchmark_target/src/main.rs | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 7d67753e4..711fa1282 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -12,7 +12,7 @@ use tokio::runtime::{Builder, Runtime}; use tokio_util::sync::CancellationToken; const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); -const MIN_DYNAMIC_ITERATIONS: u64 = 100; +const MIN_DYNAMIC_PROCESS_PAIRS_PER_SAMPLE: u64 = 500; #[cfg(all(target_os = "linux", target_arch = "x86_64"))] const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); @@ -42,7 +42,8 @@ fn benchmark_target( target: &str, ) { validate_tracked_run(runtime, target); - let minimum_iterations = if target_name == "dynamic" { MIN_DYNAMIC_ITERATIONS } else { 0 }; + let minimum_iterations = + if target_name == "dynamic" { MIN_DYNAMIC_PROCESS_PAIRS_PER_SAMPLE } else { 0 }; group.bench_with_input(BenchmarkId::from_parameter(target_name), &target, |bencher, target| { bencher.iter_custom(|iterations| { diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs index deb04d3e3..f9548576f 100644 --- a/crates/fspy_benchmark_target/src/main.rs +++ b/crates/fspy_benchmark_target/src/main.rs @@ -5,7 +5,7 @@ use std::{ }; const THREAD_COUNT: usize = 4; -const OPEN_COUNT_PER_THREAD: usize = 8_192; +const OPEN_COUNT_PER_THREAD: usize = 2_048; #[cfg(unix)] const MISSING_PATH: &str = "/.fspy-benchmark-missing"; From 4beff361ecd049bfff567c4440624f3c1dc547f3 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 18:11:45 +0800 Subject: [PATCH 18/35] test(fspy): stabilize static benchmark samples Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark/benches/fspy.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 711fa1282..d4430d428 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -12,7 +12,7 @@ use tokio::runtime::{Builder, Runtime}; use tokio_util::sync::CancellationToken; const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); -const MIN_DYNAMIC_PROCESS_PAIRS_PER_SAMPLE: u64 = 500; +const MIN_PROCESS_PAIRS_PER_SAMPLE: u64 = 500; #[cfg(all(target_os = "linux", target_arch = "x86_64"))] const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); @@ -42,12 +42,10 @@ fn benchmark_target( target: &str, ) { validate_tracked_run(runtime, target); - let minimum_iterations = - if target_name == "dynamic" { MIN_DYNAMIC_PROCESS_PAIRS_PER_SAMPLE } else { 0 }; group.bench_with_input(BenchmarkId::from_parameter(target_name), &target, |bencher, target| { bencher.iter_custom(|iterations| { - let measured_iterations = iterations.max(minimum_iterations); + let measured_iterations = iterations.max(MIN_PROCESS_PAIRS_PER_SAMPLE); let mut tracked = Duration::ZERO; let mut untracked = Duration::ZERO; for _ in 0..measured_iterations { From 6678675391d331a697c06841a88366b32faf75ed Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 18:35:45 +0800 Subject: [PATCH 19/35] test(fspy): reduce runner sensitivity Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark_target/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs index f9548576f..b7e62b707 100644 --- a/crates/fspy_benchmark_target/src/main.rs +++ b/crates/fspy_benchmark_target/src/main.rs @@ -5,7 +5,7 @@ use std::{ }; const THREAD_COUNT: usize = 4; -const OPEN_COUNT_PER_THREAD: usize = 2_048; +const OPEN_COUNT_PER_THREAD: usize = 512; #[cfg(unix)] const MISSING_PATH: &str = "/.fspy-benchmark-missing"; From 7e51d71e62f69edc6649bbe885d54c91805c0be7 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 18:43:02 +0800 Subject: [PATCH 20/35] test(fspy): warm benchmark runners before sampling Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark/benches/fspy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index d4430d428..5efb9b65e 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -113,7 +113,7 @@ async fn run_tracked(target: &str) -> ChildTermination { fn criterion_config() -> Criterion { Criterion::default() .sample_size(10) - .warm_up_time(Duration::from_secs(5)) + .warm_up_time(Duration::from_secs(30)) .measurement_time(Duration::from_secs(60)) } From f1fc4cd31de5b866f3bdfde3d9236ecff4126329 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 18:47:12 +0800 Subject: [PATCH 21/35] test(fspy): fix process pairs per sample Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark/benches/fspy.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 5efb9b65e..dfb9cf521 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -12,7 +12,7 @@ use tokio::runtime::{Builder, Runtime}; use tokio_util::sync::CancellationToken; const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); -const MIN_PROCESS_PAIRS_PER_SAMPLE: u64 = 500; +const PROCESS_PAIRS_PER_SAMPLE: u64 = 500; #[cfg(all(target_os = "linux", target_arch = "x86_64"))] const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); @@ -45,7 +45,8 @@ fn benchmark_target( group.bench_with_input(BenchmarkId::from_parameter(target_name), &target, |bencher, target| { bencher.iter_custom(|iterations| { - let measured_iterations = iterations.max(MIN_PROCESS_PAIRS_PER_SAMPLE); + // Keep run averages comparable when the delta misleads Criterion's calibration. + let measured_iterations = PROCESS_PAIRS_PER_SAMPLE; let mut tracked = Duration::ZERO; let mut untracked = Duration::ZERO; for _ in 0..measured_iterations { From eb97891b47bbe11402acad599089c107a3517e63 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 19:04:41 +0800 Subject: [PATCH 22/35] test(fspy): keep benchmark warmup short Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark/benches/fspy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index dfb9cf521..0462354c2 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -114,7 +114,7 @@ async fn run_tracked(target: &str) -> ChildTermination { fn criterion_config() -> Criterion { Criterion::default() .sample_size(10) - .warm_up_time(Duration::from_secs(30)) + .warm_up_time(Duration::from_secs(5)) .measurement_time(Duration::from_secs(60)) } From 4c39433089a5a5899ce2c13481883584d40b5092 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 19:18:05 +0800 Subject: [PATCH 23/35] test(fspy): simplify benchmark sampling Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark/benches/fspy.rs | 18 ++++++------------ crates/fspy_benchmark_target/src/main.rs | 2 +- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 0462354c2..c5feda70a 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -12,7 +12,6 @@ use tokio::runtime::{Builder, Runtime}; use tokio_util::sync::CancellationToken; const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); -const PROCESS_PAIRS_PER_SAMPLE: u64 = 500; #[cfg(all(target_os = "linux", target_arch = "x86_64"))] const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); @@ -45,19 +44,14 @@ fn benchmark_target( group.bench_with_input(BenchmarkId::from_parameter(target_name), &target, |bencher, target| { bencher.iter_custom(|iterations| { - // Keep run averages comparable when the delta misleads Criterion's calibration. - let measured_iterations = PROCESS_PAIRS_PER_SAMPLE; let mut tracked = Duration::ZERO; let mut untracked = Duration::ZERO; - for _ in 0..measured_iterations { - untracked += measure_untracked(runtime, target); + for _ in 0..iterations { tracked += measure_tracked(runtime, target); + untracked += measure_untracked(runtime, target); } - tracked - .saturating_sub(untracked) - .div_f64(measured_iterations as f64) - .max(Duration::from_nanos(1)) - .mul_f64(iterations as f64) + // Keep clamped samples reportable at a stable 1 ns per-iteration floor. + tracked.saturating_sub(untracked).max(Duration::from_nanos(iterations)) }); }); } @@ -114,8 +108,8 @@ async fn run_tracked(target: &str) -> ChildTermination { fn criterion_config() -> Criterion { Criterion::default() .sample_size(10) - .warm_up_time(Duration::from_secs(5)) - .measurement_time(Duration::from_secs(60)) + .warm_up_time(Duration::from_millis(500)) + .measurement_time(Duration::from_secs(5)) } criterion_group! { diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs index b7e62b707..9ce595970 100644 --- a/crates/fspy_benchmark_target/src/main.rs +++ b/crates/fspy_benchmark_target/src/main.rs @@ -5,7 +5,7 @@ use std::{ }; const THREAD_COUNT: usize = 4; -const OPEN_COUNT_PER_THREAD: usize = 512; +const OPEN_COUNT_PER_THREAD: usize = 32_768; #[cfg(unix)] const MISSING_PATH: &str = "/.fspy-benchmark-missing"; From b68b54b7d040ca0a0c096155df489189943f32eb Mon Sep 17 00:00:00 2001 From: wan9chi Date: Sun, 26 Jul 2026 17:15:56 +0800 Subject: [PATCH 24/35] test(fspy): make benchmark results comparable across runs The reported number was the difference between a tracked and an untracked launch, which is not reproducible: identically configured runners disagree on absolute launch times by tens of percent, so the same commit measured 23.3 ms and 26.8 ms for the same workload on Linux, and on macOS the difference kept landing below zero and was clamped to 1 ns. A baseline saved on main cannot tell a regression from that. Measure what fspy actually costs a process instead, as two separate things, each in the way that reproduces best. Starting a process under tracking stays a wall-clock measurement, because that is the quantity, reported as a share of an untracked launch measured back to back. Normalizing that way cancels most of the difference between runner instances, and reporting a share rather than a difference keeps a given slowdown moving the number by the same amount on every platform. Interception is now timed from inside the target, which reports how long its batches of opens took. A launch is one number covering milliseconds, so anything the runner does during it lands in the result; batches are a few microseconds long and there are hundreds per launch, so the median discards the ones the scheduler spoiled. That took the spread on macOS from 4% to 0.7%, and it made the benchmark faster, because a handful of launches now carries more information than hundreds did. Runs agree within about 1% to 4%, measured by benchmarking the same commit on six runner instances per platform. Two rows are left out where they do not reproduce: concurrent accesses on macOS, where a second busy thread makes every open cost about three times as much by an amount that changes from launch to launch, and the tail of seccomp accesses, which is dominated by how quickly the runner wakes the supervisor. Co-authored-by: Claude Opus 4.5 --- crates/fspy_benchmark/README.md | 54 ++- crates/fspy_benchmark/benches/fspy.rs | 401 ++++++++++++++++++++--- crates/fspy_benchmark_target/src/main.rs | 72 +++- 3 files changed, 471 insertions(+), 56 deletions(-) diff --git a/crates/fspy_benchmark/README.md b/crates/fspy_benchmark/README.md index 00d39f68f..1d869c9bd 100644 --- a/crates/fspy_benchmark/README.md +++ b/crates/fspy_benchmark/README.md @@ -1,12 +1,60 @@ # fspy benchmark -Measures fspy's wall-clock overhead while multiple threads repeatedly open a nonexistent path. -Criterion reports the nonnegative difference between tracked and untracked runs. +Measures what fspy costs a process it tracks. That cost is two separable things, and each is +measured apart from the other so that neither dilutes it: -Linux measures dynamic and static targets. macOS and Windows measure their preload implementations. +- `launch` is the wall clock of a launch that opens nothing, so it prices starting a process under + tracking: injection, session setup, and teardown. +- `access` is how long a batch of opens takes, timed by the thread that runs them, so it prices + interception itself rather than the launch around it. +- `access-tail` is the slow end of the same samples, which moves when accesses start to scatter + rather than to cost more. +- `access-concurrent` is `access` with a second thread intercepting at the same time, each thread + opening a path of its own. + +Every sample runs tracked and untracked launches as interleaved pairs, alternating which one goes +first, and reports the median per-pair overhead: `tracked / untracked - 1`. + +Linux measures a dynamically linked target that exercises `LD_PRELOAD` plus a +`x86_64-unknown-linux-musl` target that exercises seccomp user notification. macOS measures +`DYLD_INSERT_LIBRARIES` and Windows measures Detours injection. Run the benchmark with: ```sh just benchmark-fspy ``` + +## Comparing runs + +CI stores the results of the latest `main` run and compares pull request runs against them, so the +numbers have to be reproducible on whichever runner instance a pull request happens to get. + +Absolute times are not reproducible: identically configured runners disagree by tens of percent, +which is far more than the overhead being measured. Two things bring that back under control. + +Overhead is measured against an untracked launch that ran back to back, and reported as a share of +it rather than as a difference, so that a given slowdown in fspy moves the number by the same amount +everywhere, however cheap tracking is compared with the work the target does. + +Accesses are timed from inside the target rather than around the launch. A wall-clock launch is one +number covering milliseconds, so anything the runner does during it lands in the result; timing each +batch of eight opens instead produces hundreds of samples per launch, each a few microseconds long, +and the median discards the ones the scheduler spoiled. That took the spread from 4% to 0.7% on +macOS, and it made the benchmark faster, because a handful of launches now carries more information +than hundreds did. + +What remains is a spread of about 1% to 4%, measured by benchmarking the same commit on six runner +instances per platform. The noise threshold is set above it, and Criterion still prints the change it +measured when it stays below the threshold. `launch` is the widest of them, at times 7% on Linux: it +is the one row that has to be a whole-process wall-clock measurement, so it is also the one most +exposed to the runner. + +Two rows are left out where they do not reproduce: + +- `access-concurrent` on macOS. A second busy thread there makes every open cost about three times + as much, by an amount that changes from launch to launch, which moves the result by 25% between + runner instances against 3% for one thread. The same measurement reproduces within 4% on Linux and + Windows. +- `access-tail` on the seccomp target, whose slow samples are dominated by how quickly the runner + wakes the supervisor and move by 10% between instances. diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index c5feda70a..4f415f0e3 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -1,14 +1,21 @@ use std::{ - process::{ExitStatus, Stdio}, + cell::RefCell, + collections::VecDeque, + process::Stdio, time::{Duration, Instant}, }; use criterion::{ - BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, criterion_group, criterion_main, - measurement::WallTime, + BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, Throughput, criterion_group, + criterion_main, + measurement::{Measurement, ValueFormatter}, }; use fspy::{ChildTermination, Command}; -use tokio::runtime::{Builder, Runtime}; +use tokio::{ + io::AsyncReadExt as _, + process::ChildStdout, + runtime::{Builder, Runtime}, +}; use tokio_util::sync::CancellationToken; const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); @@ -21,43 +28,246 @@ const MISSING_PATH: &str = "/.fspy-benchmark-missing"; #[cfg(windows)] const MISSING_PATH: &str = r"C:\.fspy-benchmark-missing"; -fn benchmark(criterion: &mut Criterion) { +struct Workload { + threads: &'static str, + opens: &'static str, + /// Launch pairs behind every Criterion sample. Fixed so that a sample always + /// averages the same amount of work, whatever the runner speed is. + pairs_per_sample: u32, +} + +/// Concurrent launches scatter far more than single threaded ones, and by how +/// much depends on how many cores the runner has to spare, so samples have to +/// average enough pairs for the median to settle. Unix launches are cheap +/// enough to afford four times the base count; Windows launches cost several +/// times more wall clock and reproduce just as well without it. +#[cfg(windows)] +const PAIR_MULTIPLIER: u32 = 1; +#[cfg(not(windows))] +const PAIR_MULTIPLIER: u32 = 4; + +/// Opens one batch, so that capturing can be validated before measuring. Never +/// sampled, so its pair count does not matter. +const VALIDATION_WORKLOAD: Workload = Workload { threads: "1", opens: "8", pairs_per_sample: 0 }; + +/// Opens nothing, so the whole launch is the cost of starting a tracked +/// process. Two threads, because a process that fspy tracks rarely has one. +const LAUNCH_WORKLOAD: Workload = Workload { threads: "2", opens: "0", pairs_per_sample: 200 }; + +/// Accesses timed one thread at a time. Its samples come from inside the +/// target, so a handful of launches already carries hundreds of them. +const ACCESS_WORKLOAD: Workload = Workload { threads: "1", opens: "4096", pairs_per_sample: 25 }; + +/// The same access count spread over two threads, each opening a path of its +/// own so that they exercise interception rather than one shared lookup. +#[cfg(not(target_os = "macos"))] +const CONCURRENT_ACCESS_WORKLOAD: Workload = + Workload { threads: "2", opens: "2048", pairs_per_sample: 25 }; + +/// fspy costs a process two separable things: a one-off cost to start it under +/// tracking, and a cost on every access it makes. They are measured apart so +/// that neither dilutes the other, and each in the way that reproduces best. +#[derive(Clone, Copy)] +enum Metric { + /// Wall clock of a launch that opens nothing. + Launch = 0, + /// The typical time a batch of opens took, timed by the thread running it. + Access = 1, + /// The slow end of the same samples, which moves when accesses start to + /// scatter rather than to cost more. + AccessTail = 2, + /// The typical sample once a second thread is intercepting as well. + ConcurrentAccess = 3, +} + +impl Metric { + const fn name(self) -> &'static str { + match self { + Self::Launch => "launch", + Self::Access => "access", + Self::AccessTail => "access-tail", + Self::ConcurrentAccess => "access-concurrent", + } + } + + const fn of(self, launch: &Launch) -> Duration { + match self { + Self::Launch => launch.wall, + Self::Access | Self::ConcurrentAccess => launch.access, + Self::AccessTail => launch.access_tail, + } + } +} + +/// A workload and the metrics read out of its launches. +struct Case { + workload: &'static Workload, + metrics: &'static [Metric], +} + +/// Two threads racing to be intercepted only reproduce where the runner can be +/// counted on to run them at once. On macOS the same measurement moves by 25% +/// between runner instances, against 3% for one thread, because a second busy +/// thread there makes every open cost about three times as much by an amount +/// that changes from launch to launch. +#[cfg(target_os = "macos")] +const CONCURRENT_ACCESS_CASES: &[Case] = &[]; +#[cfg(not(target_os = "macos"))] +const CONCURRENT_ACCESS_CASES: &[Case] = + &[Case { workload: &CONCURRENT_ACCESS_WORKLOAD, metrics: &[Metric::ConcurrentAccess] }]; + +const ACCESS_CASES: &[Case] = + &[Case { workload: &ACCESS_WORKLOAD, metrics: &[Metric::Access, Metric::AccessTail] }]; + +const LAUNCH_CASES: &[Case] = &[Case { workload: &LAUNCH_WORKLOAD, metrics: &[Metric::Launch] }]; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +const STATIC_ACCESS_CASES: &[Case] = + &[Case { workload: &ACCESS_WORKLOAD, metrics: &[Metric::Access] }]; + +fn benchmark(criterion: &mut Criterion) { let runtime = Builder::new_multi_thread().worker_threads(2).enable_all().build().unwrap(); + let mut group = criterion.benchmark_group("fspy"); group.sampling_mode(SamplingMode::Flat); - benchmark_target(&mut group, &runtime, "dynamic", DYNAMIC_TARGET); + for cases in [LAUNCH_CASES, ACCESS_CASES, CONCURRENT_ACCESS_CASES] { + benchmark_target(&mut group, &runtime, "dynamic", DYNAMIC_TARGET, cases); + } + // Setting up seccomp user notification costs too differently between runner + // instances to compare, so the static target only measures accesses. Its + // tail is left out too: waiting for the supervisor to wake up scatters + // enough that the slow end moves by 10% between runner instances. #[cfg(all(target_os = "linux", target_arch = "x86_64"))] - benchmark_target(&mut group, &runtime, "static", STATIC_TARGET); + for cases in [STATIC_ACCESS_CASES, CONCURRENT_ACCESS_CASES] { + benchmark_target(&mut group, &runtime, "static", STATIC_TARGET, cases); + } group.finish(); } fn benchmark_target( - group: &mut BenchmarkGroup<'_, WallTime>, + group: &mut BenchmarkGroup<'_, Overhead>, runtime: &Runtime, target_name: &str, target: &str, + cases: &[Case], ) { validate_tracked_run(runtime, target); - group.bench_with_input(BenchmarkId::from_parameter(target_name), &target, |bencher, target| { - bencher.iter_custom(|iterations| { - let mut tracked = Duration::ZERO; - let mut untracked = Duration::ZERO; - for _ in 0..iterations { - tracked += measure_tracked(runtime, target); - untracked += measure_untracked(runtime, target); - } - // Keep clamped samples reportable at a stable 1 ns per-iteration floor. - tracked.saturating_sub(untracked).max(Duration::from_nanos(iterations)) + for case in cases { + benchmark_case(group, runtime, target_name, target, case); + } +} + +fn benchmark_case( + group: &mut BenchmarkGroup<'_, Overhead>, + runtime: &Runtime, + target_name: &str, + target: &str, + case: &Case, +) { + let workload = case.workload; + let metrics = case.metrics; + + // Criterion reports one number per benchmark and runs benchmarks one after + // another, so the first one measures every metric of the case and leaves + // the samples of the metrics after it here. + let handed_over: RefCell> = RefCell::new(VecDeque::new()); + + for (position, &metric) in metrics.iter().enumerate() { + group.bench_function(BenchmarkId::new(target_name, metric.name()), |bencher| { + bencher.iter_custom(|iterations| { + let overhead = if position == 0 { + let overheads = measure_overheads(runtime, target, workload); + let mut handed_over = handed_over.borrow_mut(); + for later in &metrics[1..] { + handed_over.push_back(overheads[*later as usize]); + } + overheads[metric as usize] + } else { + let sample = handed_over.borrow_mut().pop_front(); + sample.unwrap_or_else(|| { + measure_overheads(runtime, target, workload)[metric as usize] + }) + }; + // The sample always covers `pairs_per_sample` launch pairs, so + // report the per-pair overhead scaled to whatever iteration + // count Criterion asked for. + #[expect(clippy::cast_precision_loss, reason = "iteration counts stay small")] + let iterations = iterations as f64; + overhead * iterations + }); }); - }); + } +} + +/// Interleaves tracked and untracked launches, alternating which one goes first +/// so that ordering cannot bias a pair, and reports the median per-pair +/// overhead of every metric. +/// +/// Absolute launch times differ by tens of percent between runner instances, +/// while the overhead of a tracked launch relative to an untracked launch +/// measured back to back stays comparable, which is what makes results from +/// different runs comparable. Overhead is reported relative to the untracked +/// launch instead of as a tracked/untracked ratio so that a given slowdown in +/// fspy moves the number by the same amount on every platform, however cheap +/// tracking is compared with the work the target does. +fn measure_overheads(runtime: &Runtime, target: &str, workload: &Workload) -> [f64; 4] { + let pairs = workload.pairs_per_sample * PAIR_MULTIPLIER; + let mut overheads = [ + Vec::with_capacity(pairs as usize), + Vec::with_capacity(pairs as usize), + Vec::with_capacity(pairs as usize), + Vec::with_capacity(pairs as usize), + ]; + for index in 0..pairs { + let (tracked, untracked) = if index % 2 == 0 { + let untracked = measure(runtime, target, workload, Tracking::Off); + (measure(runtime, target, workload, Tracking::On), untracked) + } else { + let tracked = measure(runtime, target, workload, Tracking::On); + (tracked, measure(runtime, target, workload, Tracking::Off)) + }; + for (metric, samples) in + [Metric::Launch, Metric::Access, Metric::AccessTail, Metric::ConcurrentAccess] + .into_iter() + .zip(&mut overheads) + { + samples.push(overhead(metric, &tracked, &untracked)); + } + } + overheads.map(|mut samples| median(&mut samples)) +} + +fn overhead(metric: Metric, tracked: &Launch, untracked: &Launch) -> f64 { + metric.of(tracked).as_secs_f64() / metric.of(untracked).as_secs_f64() - 1.0 +} + +fn median(overheads: &mut [f64]) -> f64 { + overheads.sort_unstable_by(f64::total_cmp); + overheads[overheads.len() / 2] +} + +#[derive(Clone, Copy)] +enum Tracking { + On, + Off, +} + +/// What one launch measured. +struct Launch { + /// Wall clock of the launch, from spawning it to reaping it. + wall: Duration, + /// The typical batch of opens, as timed inside the target. + access: Duration, + /// The slow end of those batches. + access_tail: Duration, } fn validate_tracked_run(runtime: &Runtime, target: &str) { - let termination = runtime.block_on(run_tracked(target)); + let termination = runtime.block_on(run_tracked(target, &VALIDATION_WORKLOAD)); assert!(termination.status.success(), "benchmark target failed: {}", termination.status); let captured_missing_access = termination.path_accesses.iter().any(|access| { access.path.strip_path_prefix(MISSING_PATH, |result| { @@ -67,36 +277,64 @@ fn validate_tracked_run(runtime: &Runtime, target: &str) { assert!(captured_missing_access, "failed to capture access to {MISSING_PATH}"); } -fn measure_untracked(runtime: &Runtime, target: &str) -> Duration { - let start = Instant::now(); - let status = runtime.block_on(run_untracked(target)); - let elapsed = start.elapsed(); +fn measure(runtime: &Runtime, target: &str, workload: &Workload, tracking: Tracking) -> Launch { + runtime.block_on(async { + let start = Instant::now(); + let mut reported = match tracking { + Tracking::Off => run_untracked(target, workload).await, + Tracking::On => run_tracked_reporting(target, workload).await, + }; + let wall = start.elapsed(); + let [access, access_tail] = read_reported_samples(&mut reported).await; + Launch { wall, access, access_tail } + }) +} + +/// Reads the samples the target timed for itself, which it writes to its +/// standard output once it is done. +async fn read_reported_samples(reported: &mut ChildStdout) -> [Duration; 2] { + let mut output = Vec::new(); + reported.read_to_end(&mut output).await.expect("failed to read the reported samples"); + let output = str::from_utf8(&output).expect("the reported samples are not utf-8"); + let mut samples = output + .split_whitespace() + .map(|sample| Duration::from_nanos(sample.parse().expect("unreadable reported sample"))); + let typical = samples.next().expect("the target reported no samples"); + let tail = samples.next().expect("the target reported no tail sample"); + [typical, tail] +} + +async fn run_untracked(target: &str, workload: &Workload) -> ChildStdout { + let mut child = tokio::process::Command::new(target) + .args([workload.threads, workload.opens]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .expect("failed to run untracked benchmark target"); + let stdout = child.stdout.take().expect("untracked benchmark target has no stdout"); + let status = child.wait().await.expect("failed to wait for untracked benchmark target"); assert!(status.success(), "untracked benchmark target failed: {status}"); - elapsed + stdout } -fn measure_tracked(runtime: &Runtime, target: &str) -> Duration { - let start = Instant::now(); - let termination = runtime.block_on(run_tracked(target)); - let elapsed = start.elapsed(); +async fn run_tracked_reporting(target: &str, workload: &Workload) -> ChildStdout { + let mut child = tracked_command(target, workload, Stdio::piped()) + .spawn(CancellationToken::new()) + .await + .expect("failed to spawn tracked benchmark target"); + let stdout = child.stdout.take().expect("tracked benchmark target has no stdout"); + let termination = child.wait_handle.await.expect("failed to wait for tracked target"); assert!( termination.status.success(), "tracked benchmark target failed: {}", termination.status ); - elapsed + stdout } -async fn run_untracked(target: &str) -> ExitStatus { - let mut command = tokio::process::Command::new(target); - command.env_clear().stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::inherit()); - command.status().await.expect("failed to run untracked benchmark target") -} - -async fn run_tracked(target: &str) -> ChildTermination { - let mut command = Command::new(target); - command.stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::inherit()); - command +async fn run_tracked(target: &str, workload: &Workload) -> ChildTermination { + tracked_command(target, workload, Stdio::null()) .spawn(CancellationToken::new()) .await .expect("failed to spawn tracked benchmark target") @@ -105,11 +343,90 @@ async fn run_tracked(target: &str) -> ChildTermination { .expect("failed to wait for tracked benchmark target") } -fn criterion_config() -> Criterion { +fn tracked_command(target: &str, workload: &Workload, stdout: Stdio) -> Command { + let mut command = Command::new(target); + command + .args([workload.threads, workload.opens]) + .stdin(Stdio::null()) + .stdout(stdout) + .stderr(Stdio::inherit()); + command +} + +/// Measures the share of extra wall-clock time a tracked launch takes over an +/// untracked one. +struct Overhead; + +impl Measurement for Overhead { + type Intermediate = (); + type Value = f64; + + fn start(&self) -> Self::Intermediate {} + + fn end(&self, _intermediate: Self::Intermediate) -> Self::Value { + // Samples come from `iter_custom`, which reports shares directly. + 0.0 + } + + fn add(&self, v1: &Self::Value, v2: &Self::Value) -> Self::Value { + v1 + v2 + } + + fn zero(&self) -> Self::Value { + 0.0 + } + + fn to_f64(&self, value: &Self::Value) -> f64 { + *value + } + + fn formatter(&self) -> &dyn ValueFormatter { + &PercentFormatter + } +} + +struct PercentFormatter; + +impl PercentFormatter { + fn to_percent(values: &mut [f64]) -> &'static str { + for value in values { + *value *= 100.0; + } + "%" + } +} + +impl ValueFormatter for PercentFormatter { + fn scale_values(&self, _typical_value: f64, values: &mut [f64]) -> &'static str { + Self::to_percent(values) + } + + fn scale_throughputs( + &self, + _typical_value: f64, + _throughput: &Throughput, + values: &mut [f64], + ) -> &'static str { + Self::to_percent(values) + } + + fn scale_for_machines(&self, values: &mut [f64]) -> &'static str { + Self::to_percent(values) + } +} + +fn criterion_config() -> Criterion { Criterion::default() + .with_measurement(Overhead) .sample_size(10) - .warm_up_time(Duration::from_millis(500)) + // A single call already covers a full sample of launch pairs, and the + // benchmark that reads handed over samples must not spend its warmup + // draining them. + .warm_up_time(Duration::from_nanos(1)) .measurement_time(Duration::from_secs(5)) + // Runner instances still disagree by a few percent, so only report + // larger moves as changes. + .noise_threshold(0.1) } criterion_group! { diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs index 9ce595970..da3b63e26 100644 --- a/crates/fspy_benchmark_target/src/main.rs +++ b/crates/fspy_benchmark_target/src/main.rs @@ -1,29 +1,79 @@ use std::{ + env, fs::File, sync::{Arc, Barrier}, thread, + time::{Duration, Instant}, }; -const THREAD_COUNT: usize = 4; -const OPEN_COUNT_PER_THREAD: usize = 32_768; +const DEFAULT_THREAD_COUNT: usize = 4; +const DEFAULT_OPEN_COUNT_PER_THREAD: usize = 512; + +/// Opens behind one timing sample. Timing every open on its own would leave the +/// sample close to the clock's granularity, which is 42 ns on Apple silicon and +/// 100 ns on Windows, while a batch this small still keeps a sample short +/// enough that the scheduler only ever spoils a few of them. +const OPENS_PER_SAMPLE: usize = 8; #[cfg(unix)] const MISSING_PATH: &str = "/.fspy-benchmark-missing"; #[cfg(windows)] const MISSING_PATH: &str = r"C:\.fspy-benchmark-missing"; +/// Threads open distinct paths so that they do not contend for the same lookup. +const THREAD_SUFFIXES: [&str; 8] = ["", "1", "2", "3", "4", "5", "6", "7"]; + fn main() { - let barrier = Arc::new(Barrier::new(THREAD_COUNT)); + let mut args = env::args().skip(1); + let thread_count = + args.next().and_then(|arg| arg.parse().ok()).unwrap_or(DEFAULT_THREAD_COUNT).max(1); + let open_count_per_thread = + args.next().and_then(|arg| arg.parse().ok()).unwrap_or(DEFAULT_OPEN_COUNT_PER_THREAD); + let barrier = Arc::new(Barrier::new(thread_count)); + let mut samples = Vec::new(); thread::scope(|scope| { - for _ in 0..THREAD_COUNT { - let barrier = Arc::clone(&barrier); - scope.spawn(move || { - barrier.wait(); - for _ in 0..OPEN_COUNT_PER_THREAD { - drop(File::open(MISSING_PATH)); - } - }); + let workers: Vec<_> = (0..thread_count) + .map(|index| { + let barrier = Arc::clone(&barrier); + let mut path = MISSING_PATH.to_owned(); + path.push_str(THREAD_SUFFIXES[index % THREAD_SUFFIXES.len()]); + scope.spawn(move || time_opens(&barrier, &path, open_count_per_thread)) + }) + .collect(); + for worker in workers { + samples.extend(worker.join().expect("benchmark thread panicked")); } }); + + report(&mut samples); +} + +/// Times how long batches of opens take, from the thread that runs them, so +/// that the measurement covers the opens themselves rather than the launch +/// around them. +fn time_opens(barrier: &Barrier, path: &str, open_count: usize) -> Vec { + let sample_count = open_count / OPENS_PER_SAMPLE; + let mut samples = Vec::with_capacity(sample_count); + barrier.wait(); + for _ in 0..sample_count { + let started = Instant::now(); + for _ in 0..OPENS_PER_SAMPLE { + drop(File::open(path)); + } + samples.push(started.elapsed()); + } + samples +} + +/// Reports the typical and the slow sample, so that a change in how long opens +/// take can be told apart from a change in how much they scatter. +fn report(samples: &mut [Duration]) { + samples.sort_unstable(); + let percentile = + |percent: usize| samples.get(samples.len() * percent / 100).map_or(0, Duration::as_nanos); + #[expect(clippy::print_stdout, reason = "the benchmark reads the samples from stdout")] + { + println!("{} {}", percentile(50), percentile(90)); + } } From 79e9b779f669fa085243787caf7c8289b7c16f42 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Tue, 28 Jul 2026 10:07:44 +0800 Subject: [PATCH 25/35] test(fspy): compare fspy revisions on one runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every attempt to compare a pull request run with a baseline recorded by an earlier run on main fought the same fact: two runs land on two runner instances, and identically configured instances disagree on absolute times by tens of percent, and still by up to ten percent after normalizing each measurement against an untracked launch on the same machine. A threshold above that noise is a threshold above any regression worth catching. Stop comparing across runs. The launcher — the only benchmark piece that links fspy — is now built twice per pull request job, against the fspy under review and against the merge-base fspy, and the harness interleaves launches of both builds back to back on the same machine, with an untracked arm in the same rotation pricing tracking itself as context. Each iteration yields the head/base ratio per metric, the reported change is the median of those ratios, and a row that moves past its threshold fails the job. Criterion is gone with the cross-run baselines: medians of interleaved ratio pairs never fit its model of timing a closure, and everything it still provided — warmup, sample counts, change detection — is a handful of lines without it. So are the saved baseline artifacts, the main-push runs that recorded them, and the download action that fetched them. Co-Authored-By: Claude Fable 5 --- .github/workflows/fspy-benchmark.yml | 94 +++-- Cargo.lock | 87 +--- Cargo.toml | 2 +- crates/fspy_benchmark/Cargo.toml | 19 +- crates/fspy_benchmark/README.md | 77 ++-- crates/fspy_benchmark/benches/fspy.rs | 437 --------------------- crates/fspy_benchmark/src/lib.rs | 1 - crates/fspy_benchmark/src/main.rs | 297 ++++++++++++++ crates/fspy_benchmark_launcher/Cargo.toml | 21 + crates/fspy_benchmark_launcher/src/main.rs | 124 ++++++ justfile | 2 +- 11 files changed, 542 insertions(+), 619 deletions(-) delete mode 100644 crates/fspy_benchmark/benches/fspy.rs delete mode 100644 crates/fspy_benchmark/src/lib.rs create mode 100644 crates/fspy_benchmark/src/main.rs create mode 100644 crates/fspy_benchmark_launcher/Cargo.toml create mode 100644 crates/fspy_benchmark_launcher/src/main.rs diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index a0761fb16..3fbbe8d3d 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -6,23 +6,24 @@ on: workflow_dispatch: pull_request: types: [opened, reopened, synchronize] - push: - branches: - - main concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.ref_name != 'main' }} + cancel-in-progress: true defaults: run: shell: bash +# Benchmark results are never compared across runs: runner instances disagree +# by more than a regression worth catching. Instead, each pull request job +# builds the benchmark launcher twice — against the fspy under review and +# against the fspy of the merge base — and the harness interleaves both builds +# on the same machine. See crates/fspy_benchmark/README.md. jobs: benchmark: name: Benchmark (${{ matrix.platform }}) permissions: - actions: read contents: read strategy: fail-fast: false @@ -40,7 +41,7 @@ jobs: runs-on: ${{ matrix.os }} env: CARGO_TARGET_DIR: ${{ github.workspace }}/target - CRITERION_HOME: ${{ github.workspace }}/target/criterion + BASE_DIR: ${{ github.workspace }}/../fspy-benchmark-base steps: - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 @@ -50,67 +51,63 @@ jobs: - uses: oxc-project/setup-rust@3d6fb132fbe7cdcb66bf8ec193911c2945369d12 # v1.0.17 with: - save-cache: ${{ github.ref_name == 'main' }} + save-cache: true cache-key: fspy-benchmark-${{ matrix.platform }} - name: Install static target if: matrix.static run: rustup target add x86_64-unknown-linux-musl - - name: Download main baseline - id: main-baseline - if: github.event_name != 'push' - uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21 - with: - github_token: ${{ github.token }} - workflow: fspy-benchmark.yml - workflow_conclusion: success - branch: main - event: push - name: fspy-benchmark-baseline-v1-${{ matrix.platform }} - path: ${{ env.CRITERION_HOME }}/fspy - search_artifacts: true - if_no_artifact_found: warn + - name: Check out merge base + if: github.event_name == 'pull_request' + run: | + # The checkout is the pull request merged into the base branch, so + # its first parent is exactly the base tip the merge was built on. + git fetch --depth 2 origin "refs/pull/${{ github.event.pull_request.number }}/merge" + git worktree add "$BASE_DIR" "$(git rev-parse FETCH_HEAD^1)" + + - name: Update baseline Detours submodule + if: github.event_name == 'pull_request' && matrix.platform == 'windows' + working-directory: ${{ env.BASE_DIR }} + run: git submodule update --init --recursive - - name: Run benchmark + - name: Build baseline launcher + if: github.event_name == 'pull_request' + working-directory: ${{ env.BASE_DIR }} env: - MAIN_BASELINE_FOUND: ${{ steps.main-baseline.outputs.found_artifact }} + CARGO_TARGET_DIR: ${{ github.workspace }}/target/fspy-base run: | - mkdir -p .benchmark-results - if [[ "$GITHUB_EVENT_NAME" != "push" && "$MAIN_BASELINE_FOUND" == "true" ]]; then - baseline=(--baseline main) - else - baseline=(--save-baseline main) + # Both fspy revisions must be measured by identical code, and the + # baseline may predate the launcher crate, so overlay the head's + # launcher source before building it against the baseline fspy. + rm -rf crates/fspy_benchmark_launcher + cp -R "$GITHUB_WORKSPACE/crates/fspy_benchmark_launcher" crates/fspy_benchmark_launcher + cargo build --release -p fspy_benchmark_launcher + + - name: Run benchmark + run: | + exe="" + [[ "$RUNNER_OS" == "Windows" ]] && exe=.exe + args=() + if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then + args=(--base-launcher "$CARGO_TARGET_DIR/fspy-base/release/fspy_benchmark_launcher$exe") fi + mkdir -p .benchmark-results set -o pipefail - result_file=".benchmark-results/${{ matrix.platform }}.txt" - cargo bench --locked -p fspy_benchmark --bench fspy -- \ - --color never --verbose -n "${baseline[@]}" | - tee "$result_file" - result="$(<"$result_file")" - printf '%s' "$result" > "$result_file" + cargo run --release -p fspy_benchmark -- "${args[@]}" | + tee ".benchmark-results/${{ matrix.platform }}.txt" - name: Add job summary + if: ${{ !cancelled() }} run: | { echo '```text' cat ".benchmark-results/${{ matrix.platform }}.txt" - echo echo '```' } >> "$GITHUB_STEP_SUMMARY" - - name: Upload main baseline - if: github.event_name == 'push' && github.ref_name == 'main' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: fspy-benchmark-baseline-v1-${{ matrix.platform }} - path: ${{ env.CRITERION_HOME }}/fspy - if-no-files-found: error - overwrite: true - retention-days: 90 - - name: Upload benchmark report - if: github.event_name == 'pull_request' + if: ${{ !cancelled() && github.event_name == 'pull_request' }} # Matrix jobs have isolated filesystems, so persist each report for the # comment job to combine into one sticky comment after all platforms finish. uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -124,6 +121,7 @@ jobs: comment: name: Report benchmark if: >- + !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository needs: benchmark @@ -148,8 +146,8 @@ jobs: echo "### $platform" echo echo '```text' - cat ".benchmark-results/$platform.txt" - echo + cat ".benchmark-results/$platform.txt" || + echo "no report; the $platform benchmark job failed before producing one" echo '```' echo done diff --git a/Cargo.lock b/Cargo.lock index 1a71e1516..351ca9422 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -317,12 +317,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "bpaf" -version = "0.9.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b86829876e7e200161a5aa6ea688d46c32d64d70ee3100127790dde84688d6e" - [[package]] name = "brush-parser" version = "0.4.0" @@ -417,12 +411,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ade8366b8bd5ba243f0a58f036cc0ca8a2f069cff1a2351ef1cac6b083e16fc0" -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - [[package]] name = "castaway" version = "0.2.4" @@ -471,33 +459,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" -[[package]] -name = "ciborium" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" - -[[package]] -name = "ciborium-ll" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" -dependencies = [ - "ciborium-io", - "half", -] - [[package]] name = "clang-sys" version = "1.8.1" @@ -709,21 +670,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "criterion2" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b961b72d7eaf7444d1eb98d353d5c2aa08e2443d77fef6bff0e6e97064162482" -dependencies = [ - "bpaf", - "cast", - "ciborium", - "num-traits", - "oorandom", - "serde", - "serde_json", -] - [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -777,12 +723,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - [[package]] name = "crypto-common" version = "0.1.7" @@ -1303,10 +1243,16 @@ dependencies = [ name = "fspy_benchmark" version = "0.0.0" dependencies = [ - "criterion2", - "fspy", + "fspy_benchmark_launcher", "fspy_benchmark_static_target", "fspy_benchmark_target", +] + +[[package]] +name = "fspy_benchmark_launcher" +version = "0.0.0" +dependencies = [ + "fspy", "tokio", "tokio-util", ] @@ -1617,17 +1563,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "half" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" -dependencies = [ - "cfg-if", - "crunchy", - "zerocopy", -] - [[package]] name = "hashbrown" version = "0.15.5" @@ -2485,12 +2420,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "oorandom" -version = "11.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" - [[package]] name = "option-ext" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 818b45454..002cea9be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,6 @@ crossterm = { version = "0.29.0", features = ["event-stream"] } csv-async = { version = "1.3.1", features = ["tokio"] } ctor = "1.0" ctrlc = "3.5.2" -criterion2 = { version = "3.0.4", default-features = false } derive_more = "2.0.1" diff-struct = "0.5.3" directories = "6.0.0" @@ -73,6 +72,7 @@ materialized_artifact = { path = "crates/materialized_artifact" } materialized_artifact_build = { path = "crates/materialized_artifact_build" } flate2 = "1.0.35" fspy = { path = "crates/fspy" } +fspy_benchmark_launcher = { path = "crates/fspy_benchmark_launcher", artifact = "bin" } fspy_benchmark_target = { path = "crates/fspy_benchmark_target", artifact = "bin" } fspy_detours_sys = { path = "crates/fspy_detours_sys" } fspy_preload_unix = { path = "crates/fspy_preload_unix", artifact = "cdylib", target = "target" } diff --git a/crates/fspy_benchmark/Cargo.toml b/crates/fspy_benchmark/Cargo.toml index c6fbe7b67..01e640e6d 100644 --- a/crates/fspy_benchmark/Cargo.toml +++ b/crates/fspy_benchmark/Cargo.toml @@ -10,24 +10,17 @@ rust-version.workspace = true [lints] workspace = true -[lib] +[[bin]] +name = "fspy_benchmark" test = false -bench = false doctest = false -[[bench]] -name = "fspy" -harness = false - -[dev-dependencies] -criterion2 = { workspace = true } -fspy = { workspace = true } +[dependencies] +fspy_benchmark_launcher = { workspace = true } fspy_benchmark_target = { workspace = true } -tokio = { workspace = true, features = ["macros", "process", "rt-multi-thread"] } -tokio-util = { workspace = true } -[target.'cfg(all(target_os = "linux", target_arch = "x86_64"))'.dev-dependencies] +[target.'cfg(all(target_os = "linux", target_arch = "x86_64"))'.dependencies] fspy_benchmark_static_target = { path = "../fspy_benchmark_static_target", artifact = "bin", target = "x86_64-unknown-linux-musl" } [package.metadata.cargo-shear] -ignored = ["fspy_benchmark_target", "fspy_benchmark_static_target"] +ignored = ["fspy_benchmark_launcher", "fspy_benchmark_target", "fspy_benchmark_static_target"] diff --git a/crates/fspy_benchmark/README.md b/crates/fspy_benchmark/README.md index 1d869c9bd..c60f5f2aa 100644 --- a/crates/fspy_benchmark/README.md +++ b/crates/fspy_benchmark/README.md @@ -1,7 +1,7 @@ # fspy benchmark -Measures what fspy costs a process it tracks. That cost is two separable things, and each is -measured apart from the other so that neither dilutes it: +Measures what fspy costs a process it tracks, and whether a change to fspy moved that cost. The +cost is two separable things, and each is measured apart from the other so that neither dilutes it: - `launch` is the wall clock of a launch that opens nothing, so it prices starting a process under tracking: injection, session setup, and teardown. @@ -12,49 +12,48 @@ measured apart from the other so that neither dilutes it: - `access-concurrent` is `access` with a second thread intercepting at the same time, each thread opening a path of its own. -Every sample runs tracked and untracked launches as interleaved pairs, alternating which one goes -first, and reports the median per-pair overhead: `tracked / untracked - 1`. - Linux measures a dynamically linked target that exercises `LD_PRELOAD` plus a `x86_64-unknown-linux-musl` target that exercises seccomp user notification. macOS measures `DYLD_INSERT_LIBRARIES` and Windows measures Detours injection. -Run the benchmark with: +Run the overhead report locally with: ```sh just benchmark-fspy ``` -## Comparing runs - -CI stores the results of the latest `main` run and compares pull request runs against them, so the -numbers have to be reproducible on whichever runner instance a pull request happens to get. - -Absolute times are not reproducible: identically configured runners disagree by tens of percent, -which is far more than the overhead being measured. Two things bring that back under control. - -Overhead is measured against an untracked launch that ran back to back, and reported as a share of -it rather than as a difference, so that a given slowdown in fspy moves the number by the same amount -everywhere, however cheap tracking is compared with the work the target does. - -Accesses are timed from inside the target rather than around the launch. A wall-clock launch is one -number covering milliseconds, so anything the runner does during it lands in the result; timing each -batch of eight opens instead produces hundreds of samples per launch, each a few microseconds long, -and the median discards the ones the scheduler spoiled. That took the spread from 4% to 0.7% on -macOS, and it made the benchmark faster, because a handful of launches now carries more information -than hundreds did. - -What remains is a spread of about 1% to 4%, measured by benchmarking the same commit on six runner -instances per platform. The noise threshold is set above it, and Criterion still prints the change it -measured when it stays below the threshold. `launch` is the widest of them, at times 7% on Linux: it -is the one row that has to be a whole-process wall-clock measurement, so it is also the one most -exposed to the runner. - -Two rows are left out where they do not reproduce: - -- `access-concurrent` on macOS. A second busy thread there makes every open cost about three times - as much, by an amount that changes from launch to launch, which moves the result by 25% between - runner instances against 3% for one thread. The same measurement reproduces within 4% on Linux and - Windows. -- `access-tail` on the seccomp target, whose slow samples are dominated by how quickly the runner - wakes the supervisor and move by 10% between instances. +## How regressions are caught + +Comparing a pull request run with a baseline recorded by an earlier run on `main` cannot work, +and every attempt to make it work failed the same way: two runs land on two runner instances, and +identically configured instances disagree — on absolute times by tens of percent, and still by up +to ten percent after normalizing every measurement against an untracked launch on the same +machine. A threshold above that noise is a threshold above any regression worth catching. + +So nothing is ever compared across runs. The pieces are split so that the two fspy revisions being +compared can run side by side on one machine: + +- `fspy_benchmark_target` (and its static twin) is the workload: it opens missing paths from a + configurable number of threads and reports how long its batches took on stdout. It does not link + fspy. +- `fspy_benchmark_launcher` runs the target once — tracked through `fspy::Command`, or untracked + for context — and prints the launch wall clock plus the target's numbers. It is the only piece + that links fspy. +- `fspy_benchmark` is the harness. CI builds the launcher twice, against the fspy under review and + against the merge-base fspy (from the same launcher source, so both revisions are measured by + identical code), then the harness launches both builds interleaved, back to back, in rotating + order. Whatever the runner instance is doing to the numbers, it is doing to both arms. + +Each iteration yields the ratio of the head launch to the base launch per metric; the reported +change is the median of those ratios. An untracked arm runs in the same rotation and prices +tracking itself — that overhead number is context, not a gate. A row whose median change exceeds +its threshold fails the job. + +Locally there is no second fspy build, so `just benchmark-fspy` reports only the overhead column. +Point `--base-launcher` at another build of `fspy_benchmark_launcher` to compare two revisions by +hand. + +Because the launcher is compiled against both revisions from the head's source, a pull request +that changes the parts of the `fspy::Command` API the launcher uses can fail the baseline build. +Keep the launcher's API surface minimal; if the API must change incompatibly, the benchmark run +for that one pull request is expected to fail its baseline build. diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs deleted file mode 100644 index 4f415f0e3..000000000 --- a/crates/fspy_benchmark/benches/fspy.rs +++ /dev/null @@ -1,437 +0,0 @@ -use std::{ - cell::RefCell, - collections::VecDeque, - process::Stdio, - time::{Duration, Instant}, -}; - -use criterion::{ - BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, Throughput, criterion_group, - criterion_main, - measurement::{Measurement, ValueFormatter}, -}; -use fspy::{ChildTermination, Command}; -use tokio::{ - io::AsyncReadExt as _, - process::ChildStdout, - runtime::{Builder, Runtime}, -}; -use tokio_util::sync::CancellationToken; - -const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); - -#[cfg(all(target_os = "linux", target_arch = "x86_64"))] -const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); - -#[cfg(unix)] -const MISSING_PATH: &str = "/.fspy-benchmark-missing"; -#[cfg(windows)] -const MISSING_PATH: &str = r"C:\.fspy-benchmark-missing"; - -struct Workload { - threads: &'static str, - opens: &'static str, - /// Launch pairs behind every Criterion sample. Fixed so that a sample always - /// averages the same amount of work, whatever the runner speed is. - pairs_per_sample: u32, -} - -/// Concurrent launches scatter far more than single threaded ones, and by how -/// much depends on how many cores the runner has to spare, so samples have to -/// average enough pairs for the median to settle. Unix launches are cheap -/// enough to afford four times the base count; Windows launches cost several -/// times more wall clock and reproduce just as well without it. -#[cfg(windows)] -const PAIR_MULTIPLIER: u32 = 1; -#[cfg(not(windows))] -const PAIR_MULTIPLIER: u32 = 4; - -/// Opens one batch, so that capturing can be validated before measuring. Never -/// sampled, so its pair count does not matter. -const VALIDATION_WORKLOAD: Workload = Workload { threads: "1", opens: "8", pairs_per_sample: 0 }; - -/// Opens nothing, so the whole launch is the cost of starting a tracked -/// process. Two threads, because a process that fspy tracks rarely has one. -const LAUNCH_WORKLOAD: Workload = Workload { threads: "2", opens: "0", pairs_per_sample: 200 }; - -/// Accesses timed one thread at a time. Its samples come from inside the -/// target, so a handful of launches already carries hundreds of them. -const ACCESS_WORKLOAD: Workload = Workload { threads: "1", opens: "4096", pairs_per_sample: 25 }; - -/// The same access count spread over two threads, each opening a path of its -/// own so that they exercise interception rather than one shared lookup. -#[cfg(not(target_os = "macos"))] -const CONCURRENT_ACCESS_WORKLOAD: Workload = - Workload { threads: "2", opens: "2048", pairs_per_sample: 25 }; - -/// fspy costs a process two separable things: a one-off cost to start it under -/// tracking, and a cost on every access it makes. They are measured apart so -/// that neither dilutes the other, and each in the way that reproduces best. -#[derive(Clone, Copy)] -enum Metric { - /// Wall clock of a launch that opens nothing. - Launch = 0, - /// The typical time a batch of opens took, timed by the thread running it. - Access = 1, - /// The slow end of the same samples, which moves when accesses start to - /// scatter rather than to cost more. - AccessTail = 2, - /// The typical sample once a second thread is intercepting as well. - ConcurrentAccess = 3, -} - -impl Metric { - const fn name(self) -> &'static str { - match self { - Self::Launch => "launch", - Self::Access => "access", - Self::AccessTail => "access-tail", - Self::ConcurrentAccess => "access-concurrent", - } - } - - const fn of(self, launch: &Launch) -> Duration { - match self { - Self::Launch => launch.wall, - Self::Access | Self::ConcurrentAccess => launch.access, - Self::AccessTail => launch.access_tail, - } - } -} - -/// A workload and the metrics read out of its launches. -struct Case { - workload: &'static Workload, - metrics: &'static [Metric], -} - -/// Two threads racing to be intercepted only reproduce where the runner can be -/// counted on to run them at once. On macOS the same measurement moves by 25% -/// between runner instances, against 3% for one thread, because a second busy -/// thread there makes every open cost about three times as much by an amount -/// that changes from launch to launch. -#[cfg(target_os = "macos")] -const CONCURRENT_ACCESS_CASES: &[Case] = &[]; -#[cfg(not(target_os = "macos"))] -const CONCURRENT_ACCESS_CASES: &[Case] = - &[Case { workload: &CONCURRENT_ACCESS_WORKLOAD, metrics: &[Metric::ConcurrentAccess] }]; - -const ACCESS_CASES: &[Case] = - &[Case { workload: &ACCESS_WORKLOAD, metrics: &[Metric::Access, Metric::AccessTail] }]; - -const LAUNCH_CASES: &[Case] = &[Case { workload: &LAUNCH_WORKLOAD, metrics: &[Metric::Launch] }]; - -#[cfg(all(target_os = "linux", target_arch = "x86_64"))] -const STATIC_ACCESS_CASES: &[Case] = - &[Case { workload: &ACCESS_WORKLOAD, metrics: &[Metric::Access] }]; - -fn benchmark(criterion: &mut Criterion) { - let runtime = Builder::new_multi_thread().worker_threads(2).enable_all().build().unwrap(); - - let mut group = criterion.benchmark_group("fspy"); - group.sampling_mode(SamplingMode::Flat); - - for cases in [LAUNCH_CASES, ACCESS_CASES, CONCURRENT_ACCESS_CASES] { - benchmark_target(&mut group, &runtime, "dynamic", DYNAMIC_TARGET, cases); - } - - // Setting up seccomp user notification costs too differently between runner - // instances to compare, so the static target only measures accesses. Its - // tail is left out too: waiting for the supervisor to wake up scatters - // enough that the slow end moves by 10% between runner instances. - #[cfg(all(target_os = "linux", target_arch = "x86_64"))] - for cases in [STATIC_ACCESS_CASES, CONCURRENT_ACCESS_CASES] { - benchmark_target(&mut group, &runtime, "static", STATIC_TARGET, cases); - } - - group.finish(); -} - -fn benchmark_target( - group: &mut BenchmarkGroup<'_, Overhead>, - runtime: &Runtime, - target_name: &str, - target: &str, - cases: &[Case], -) { - validate_tracked_run(runtime, target); - - for case in cases { - benchmark_case(group, runtime, target_name, target, case); - } -} - -fn benchmark_case( - group: &mut BenchmarkGroup<'_, Overhead>, - runtime: &Runtime, - target_name: &str, - target: &str, - case: &Case, -) { - let workload = case.workload; - let metrics = case.metrics; - - // Criterion reports one number per benchmark and runs benchmarks one after - // another, so the first one measures every metric of the case and leaves - // the samples of the metrics after it here. - let handed_over: RefCell> = RefCell::new(VecDeque::new()); - - for (position, &metric) in metrics.iter().enumerate() { - group.bench_function(BenchmarkId::new(target_name, metric.name()), |bencher| { - bencher.iter_custom(|iterations| { - let overhead = if position == 0 { - let overheads = measure_overheads(runtime, target, workload); - let mut handed_over = handed_over.borrow_mut(); - for later in &metrics[1..] { - handed_over.push_back(overheads[*later as usize]); - } - overheads[metric as usize] - } else { - let sample = handed_over.borrow_mut().pop_front(); - sample.unwrap_or_else(|| { - measure_overheads(runtime, target, workload)[metric as usize] - }) - }; - // The sample always covers `pairs_per_sample` launch pairs, so - // report the per-pair overhead scaled to whatever iteration - // count Criterion asked for. - #[expect(clippy::cast_precision_loss, reason = "iteration counts stay small")] - let iterations = iterations as f64; - overhead * iterations - }); - }); - } -} - -/// Interleaves tracked and untracked launches, alternating which one goes first -/// so that ordering cannot bias a pair, and reports the median per-pair -/// overhead of every metric. -/// -/// Absolute launch times differ by tens of percent between runner instances, -/// while the overhead of a tracked launch relative to an untracked launch -/// measured back to back stays comparable, which is what makes results from -/// different runs comparable. Overhead is reported relative to the untracked -/// launch instead of as a tracked/untracked ratio so that a given slowdown in -/// fspy moves the number by the same amount on every platform, however cheap -/// tracking is compared with the work the target does. -fn measure_overheads(runtime: &Runtime, target: &str, workload: &Workload) -> [f64; 4] { - let pairs = workload.pairs_per_sample * PAIR_MULTIPLIER; - let mut overheads = [ - Vec::with_capacity(pairs as usize), - Vec::with_capacity(pairs as usize), - Vec::with_capacity(pairs as usize), - Vec::with_capacity(pairs as usize), - ]; - for index in 0..pairs { - let (tracked, untracked) = if index % 2 == 0 { - let untracked = measure(runtime, target, workload, Tracking::Off); - (measure(runtime, target, workload, Tracking::On), untracked) - } else { - let tracked = measure(runtime, target, workload, Tracking::On); - (tracked, measure(runtime, target, workload, Tracking::Off)) - }; - for (metric, samples) in - [Metric::Launch, Metric::Access, Metric::AccessTail, Metric::ConcurrentAccess] - .into_iter() - .zip(&mut overheads) - { - samples.push(overhead(metric, &tracked, &untracked)); - } - } - overheads.map(|mut samples| median(&mut samples)) -} - -fn overhead(metric: Metric, tracked: &Launch, untracked: &Launch) -> f64 { - metric.of(tracked).as_secs_f64() / metric.of(untracked).as_secs_f64() - 1.0 -} - -fn median(overheads: &mut [f64]) -> f64 { - overheads.sort_unstable_by(f64::total_cmp); - overheads[overheads.len() / 2] -} - -#[derive(Clone, Copy)] -enum Tracking { - On, - Off, -} - -/// What one launch measured. -struct Launch { - /// Wall clock of the launch, from spawning it to reaping it. - wall: Duration, - /// The typical batch of opens, as timed inside the target. - access: Duration, - /// The slow end of those batches. - access_tail: Duration, -} - -fn validate_tracked_run(runtime: &Runtime, target: &str) { - let termination = runtime.block_on(run_tracked(target, &VALIDATION_WORKLOAD)); - assert!(termination.status.success(), "benchmark target failed: {}", termination.status); - let captured_missing_access = termination.path_accesses.iter().any(|access| { - access.path.strip_path_prefix(MISSING_PATH, |result| { - result.is_ok_and(|path| path.as_os_str().is_empty()) - }) - }); - assert!(captured_missing_access, "failed to capture access to {MISSING_PATH}"); -} - -fn measure(runtime: &Runtime, target: &str, workload: &Workload, tracking: Tracking) -> Launch { - runtime.block_on(async { - let start = Instant::now(); - let mut reported = match tracking { - Tracking::Off => run_untracked(target, workload).await, - Tracking::On => run_tracked_reporting(target, workload).await, - }; - let wall = start.elapsed(); - let [access, access_tail] = read_reported_samples(&mut reported).await; - Launch { wall, access, access_tail } - }) -} - -/// Reads the samples the target timed for itself, which it writes to its -/// standard output once it is done. -async fn read_reported_samples(reported: &mut ChildStdout) -> [Duration; 2] { - let mut output = Vec::new(); - reported.read_to_end(&mut output).await.expect("failed to read the reported samples"); - let output = str::from_utf8(&output).expect("the reported samples are not utf-8"); - let mut samples = output - .split_whitespace() - .map(|sample| Duration::from_nanos(sample.parse().expect("unreadable reported sample"))); - let typical = samples.next().expect("the target reported no samples"); - let tail = samples.next().expect("the target reported no tail sample"); - [typical, tail] -} - -async fn run_untracked(target: &str, workload: &Workload) -> ChildStdout { - let mut child = tokio::process::Command::new(target) - .args([workload.threads, workload.opens]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::inherit()) - .spawn() - .expect("failed to run untracked benchmark target"); - let stdout = child.stdout.take().expect("untracked benchmark target has no stdout"); - let status = child.wait().await.expect("failed to wait for untracked benchmark target"); - assert!(status.success(), "untracked benchmark target failed: {status}"); - stdout -} - -async fn run_tracked_reporting(target: &str, workload: &Workload) -> ChildStdout { - let mut child = tracked_command(target, workload, Stdio::piped()) - .spawn(CancellationToken::new()) - .await - .expect("failed to spawn tracked benchmark target"); - let stdout = child.stdout.take().expect("tracked benchmark target has no stdout"); - let termination = child.wait_handle.await.expect("failed to wait for tracked target"); - assert!( - termination.status.success(), - "tracked benchmark target failed: {}", - termination.status - ); - stdout -} - -async fn run_tracked(target: &str, workload: &Workload) -> ChildTermination { - tracked_command(target, workload, Stdio::null()) - .spawn(CancellationToken::new()) - .await - .expect("failed to spawn tracked benchmark target") - .wait_handle - .await - .expect("failed to wait for tracked benchmark target") -} - -fn tracked_command(target: &str, workload: &Workload, stdout: Stdio) -> Command { - let mut command = Command::new(target); - command - .args([workload.threads, workload.opens]) - .stdin(Stdio::null()) - .stdout(stdout) - .stderr(Stdio::inherit()); - command -} - -/// Measures the share of extra wall-clock time a tracked launch takes over an -/// untracked one. -struct Overhead; - -impl Measurement for Overhead { - type Intermediate = (); - type Value = f64; - - fn start(&self) -> Self::Intermediate {} - - fn end(&self, _intermediate: Self::Intermediate) -> Self::Value { - // Samples come from `iter_custom`, which reports shares directly. - 0.0 - } - - fn add(&self, v1: &Self::Value, v2: &Self::Value) -> Self::Value { - v1 + v2 - } - - fn zero(&self) -> Self::Value { - 0.0 - } - - fn to_f64(&self, value: &Self::Value) -> f64 { - *value - } - - fn formatter(&self) -> &dyn ValueFormatter { - &PercentFormatter - } -} - -struct PercentFormatter; - -impl PercentFormatter { - fn to_percent(values: &mut [f64]) -> &'static str { - for value in values { - *value *= 100.0; - } - "%" - } -} - -impl ValueFormatter for PercentFormatter { - fn scale_values(&self, _typical_value: f64, values: &mut [f64]) -> &'static str { - Self::to_percent(values) - } - - fn scale_throughputs( - &self, - _typical_value: f64, - _throughput: &Throughput, - values: &mut [f64], - ) -> &'static str { - Self::to_percent(values) - } - - fn scale_for_machines(&self, values: &mut [f64]) -> &'static str { - Self::to_percent(values) - } -} - -fn criterion_config() -> Criterion { - Criterion::default() - .with_measurement(Overhead) - .sample_size(10) - // A single call already covers a full sample of launch pairs, and the - // benchmark that reads handed over samples must not spend its warmup - // draining them. - .warm_up_time(Duration::from_nanos(1)) - .measurement_time(Duration::from_secs(5)) - // Runner instances still disagree by a few percent, so only report - // larger moves as changes. - .noise_threshold(0.1) -} - -criterion_group! { - name = benches; - config = criterion_config(); - targets = benchmark -} -criterion_main!(benches); diff --git a/crates/fspy_benchmark/src/lib.rs b/crates/fspy_benchmark/src/lib.rs deleted file mode 100644 index 6d118a163..000000000 --- a/crates/fspy_benchmark/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ -//! Benchmarks for the overhead fspy adds to tracked processes. diff --git a/crates/fspy_benchmark/src/main.rs b/crates/fspy_benchmark/src/main.rs new file mode 100644 index 000000000..57486890c --- /dev/null +++ b/crates/fspy_benchmark/src/main.rs @@ -0,0 +1,297 @@ +//! Compares what two fspy builds cost the processes they track. +//! +//! Every earlier attempt at this benchmark compared a pull request run with a +//! baseline recorded by an earlier run on `main`, and kept fighting the same +//! fact: two runs land on two runner instances, and identically configured +//! instances disagree on every absolute and relative measurement by more than +//! a regression worth catching. This harness never compares across runs. +//! Instead the workflow builds the launcher twice — once against the fspy +//! under review and once against the baseline fspy — and this harness +//! interleaves launches of both builds on the same machine, in the same run, +//! as neighboring processes. Whatever the runner instance is doing to the +//! numbers, it is doing to both sides. +//! +//! Each iteration launches every arm — baseline-tracked, head-tracked, and +//! untracked — back to back in rotating order, and yields the ratio of head +//! to baseline per metric. The reported change is the median of those ratios, +//! so it prices exactly one thing: what the fspy change under review costs a +//! tracked process. The untracked arm prices tracking itself, as context. + +use std::{ + env, + ffi::{OsStr, OsString}, + process::{Command, Stdio}, +}; + +const HEAD_LAUNCHER: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_LAUNCHER"); +const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); + +/// Opens one small batch, only to prove that capturing works before measuring. +const VALIDATION_ARGS: [&str; 2] = ["1", "8"]; + +struct Workload { + /// Thread count the target spawns, passed through to it. + threads: &'static str, + /// Opens per target thread, passed through to it. + opens: &'static str, + /// Measured iterations. Each one launches every arm once. + iterations: usize, + /// Unmeasured iterations run first, to fill caches and settle the runner. + warmup: usize, +} + +/// Launches cost several times more wall clock on Windows, so it affords +/// fewer of them in the same time. +#[cfg(windows)] +const LAUNCH_ITERATIONS: usize = 150; +#[cfg(not(windows))] +const LAUNCH_ITERATIONS: usize = 300; + +/// Opens nothing, so the whole launch is the cost of starting a tracked +/// process. Two threads, because a process that fspy tracks rarely has one. +const LAUNCH_WORKLOAD: Workload = + Workload { threads: "2", opens: "0", iterations: LAUNCH_ITERATIONS, warmup: 5 }; + +/// Accesses timed one thread at a time, from inside the target, so they price +/// interception rather than the launch around it. +const ACCESS_WORKLOAD: Workload = + Workload { threads: "1", opens: "4096", iterations: 100, warmup: 3 }; + +/// The same access count spread over two threads, each opening a path of its +/// own, so that they exercise concurrent interception. +const CONCURRENT_WORKLOAD: Workload = + Workload { threads: "2", opens: "2048", iterations: 100, warmup: 3 }; + +/// What a row reads out of the launches of its workload. +#[derive(Clone, Copy)] +enum Metric { + /// Wall clock of the launch, from spawn to reap. + Wall, + /// The typical batch of opens, as timed inside the target. + Typical, + /// The slow end of those batches, which moves when accesses start to + /// scatter rather than to cost more. + Tail, +} + +struct Row { + name: &'static str, + metric: Metric, + /// The change past which the row fails the run, as a share. Set from the + /// spread observed when benchmarking one commit on many runner instances. + threshold: f64, +} + +struct Suite { + workload: &'static Workload, + rows: &'static [Row], +} + +const LAUNCH_SUITE: Suite = Suite { + workload: &LAUNCH_WORKLOAD, + rows: &[Row { name: "launch", metric: Metric::Wall, threshold: 0.05 }], +}; + +const ACCESS_SUITE: Suite = Suite { + workload: &ACCESS_WORKLOAD, + rows: &[ + Row { name: "access", metric: Metric::Typical, threshold: 0.05 }, + Row { name: "access-tail", metric: Metric::Tail, threshold: 0.08 }, + ], +}; + +const CONCURRENT_SUITE: Suite = Suite { + workload: &CONCURRENT_WORKLOAD, + rows: &[Row { name: "access-concurrent", metric: Metric::Typical, threshold: 0.08 }], +}; + +struct Backend { + name: &'static str, + target: &'static str, +} + +fn main() { + let base_launcher = parse_base_launcher(); + + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + let backends = [ + Backend { name: "dynamic", target: DYNAMIC_TARGET }, + Backend { name: "static", target: STATIC_TARGET }, + ]; + #[cfg(not(all(target_os = "linux", target_arch = "x86_64")))] + let backends = [Backend { name: "dynamic", target: DYNAMIC_TARGET }]; + + let mut regressed = false; + for backend in &backends { + validate(HEAD_LAUNCHER.as_ref(), backend.target); + if let Some(base_launcher) = &base_launcher { + validate(base_launcher, backend.target); + } + for suite in [&LAUNCH_SUITE, &ACCESS_SUITE, &CONCURRENT_SUITE] { + regressed |= run_suite(backend, suite, base_launcher.as_deref()); + } + } + + if regressed { + std::process::exit(1); + } +} + +fn parse_base_launcher() -> Option { + let mut args = env::args_os().skip(1); + let base_launcher = match args.next() { + None => None, + Some(flag) if flag == "--base-launcher" => { + Some(args.next().expect("--base-launcher needs a path")) + } + Some(flag) => { + panic!("unknown argument {}, expected --base-launcher PATH", flag.to_string_lossy()) + } + }; + assert!(args.next().is_none(), "unexpected extra arguments"); + base_launcher +} + +/// What one launch measured, in nanoseconds. +#[derive(Clone, Copy, Default)] +struct Sample { + wall: f64, + typical: f64, + tail: f64, +} + +impl Sample { + const fn metric(&self, metric: Metric) -> f64 { + match metric { + Metric::Wall => self.wall, + Metric::Typical => self.typical, + Metric::Tail => self.tail, + } + } +} + +/// One iteration's launches, one per arm. +#[derive(Default)] +struct Iteration { + base: Sample, + head: Sample, + untracked: Sample, +} + +/// Runs a suite's workload and reports its rows. Returns whether any row +/// regressed past its threshold. +fn run_suite(backend: &Backend, suite: &Suite, base_launcher: Option<&OsStr>) -> bool { + let workload = suite.workload; + let arm_count = if base_launcher.is_some() { 3 } else { 2 }; + let mut iterations = Vec::with_capacity(workload.iterations); + for index in 0..workload.warmup + workload.iterations { + let mut iteration = Iteration::default(); + // Rotate which arm goes first, so that no arm always pays for waking + // the machine up or always benefits from the caches the previous arm + // warmed. + for position in 0..arm_count { + match (index + position) % arm_count { + 0 => iteration.head = launch(HEAD_LAUNCHER.as_ref(), None, backend, workload), + 1 => { + iteration.untracked = + launch(HEAD_LAUNCHER.as_ref(), Some("--untracked"), backend, workload); + } + _ => { + iteration.base = launch(base_launcher.unwrap(), None, backend, workload); + } + } + } + if index >= workload.warmup { + iterations.push(iteration); + } + } + + let mut regressed = false; + for row in suite.rows { + regressed |= report_row(backend, row, &iterations, base_launcher.is_some()); + } + regressed +} + +/// Prints one row and returns whether it regressed past its threshold. +#[expect(clippy::print_stdout, reason = "the report is the benchmark's output")] +fn report_row(backend: &Backend, row: &Row, iterations: &[Iteration], has_base: bool) -> bool { + let name = [backend.name, "/", row.name].concat(); + let overhead = median( + iterations + .iter() + .map(|it| it.head.metric(row.metric) / it.untracked.metric(row.metric) - 1.0), + ); + + if has_base { + let mut changes: Vec = iterations + .iter() + .map(|it| it.head.metric(row.metric) / it.base.metric(row.metric) - 1.0) + .collect(); + changes.sort_unstable_by(f64::total_cmp); + let change = changes[changes.len() / 2]; + let low = changes[changes.len() / 4]; + let high = changes[changes.len() * 3 / 4]; + let verdict = if change > row.threshold { + " REGRESSED" + } else if change < -row.threshold { + " improved" + } else { + "" + }; + println!( + "{name:<26} change {:>+6.2}% [{:>+6.2}% .. {:>+6.2}%] threshold {:>4.1}% overhead {:>+8.2}%{verdict}", + change * 100.0, + low * 100.0, + high * 100.0, + row.threshold * 100.0, + overhead * 100.0, + ); + change > row.threshold + } else { + println!("{name:<26} overhead {:>+8.2}%", overhead * 100.0); + false + } +} + +fn median(values: impl Iterator) -> f64 { + let mut values: Vec = values.collect(); + values.sort_unstable_by(f64::total_cmp); + values[values.len() / 2] +} + +/// Launches one arm and reads the three numbers its launcher printed. +fn launch(launcher: &OsStr, mode: Option<&str>, backend: &Backend, workload: &Workload) -> Sample { + let mut command = Command::new(launcher); + if let Some(mode) = mode { + command.arg(mode); + } + let output = command + .args([backend.target, workload.threads, workload.opens]) + .stdin(Stdio::null()) + .stderr(Stdio::inherit()) + .output() + .expect("failed to run the benchmark launcher"); + assert!(output.status.success(), "benchmark launcher failed: {}", output.status); + let stdout = str::from_utf8(&output.stdout).expect("launcher output is not utf-8"); + let mut numbers = stdout + .split_whitespace() + .map(|number| number.parse().expect("unreadable launcher measurement")); + let mut next = || numbers.next().expect("launcher reported too few measurements"); + Sample { wall: next(), typical: next(), tail: next() } +} + +fn validate(launcher: &OsStr, target: &str) { + let status = Command::new(launcher) + .arg("--validate") + .arg(target) + .args(VALIDATION_ARGS) + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .status() + .expect("failed to run the benchmark launcher"); + assert!(status.success(), "launcher validation failed for {}", launcher.to_string_lossy()); +} diff --git a/crates/fspy_benchmark_launcher/Cargo.toml b/crates/fspy_benchmark_launcher/Cargo.toml new file mode 100644 index 000000000..ab6abb5c7 --- /dev/null +++ b/crates/fspy_benchmark_launcher/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "fspy_benchmark_launcher" +version = "0.0.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +publish = false +rust-version.workspace = true + +[lints] +workspace = true + +[[bin]] +name = "fspy_benchmark_launcher" +test = false +doctest = false + +[dependencies] +fspy = { workspace = true } +tokio = { workspace = true, features = ["macros", "process", "rt-multi-thread"] } +tokio-util = { workspace = true } diff --git a/crates/fspy_benchmark_launcher/src/main.rs b/crates/fspy_benchmark_launcher/src/main.rs new file mode 100644 index 000000000..91126f6ac --- /dev/null +++ b/crates/fspy_benchmark_launcher/src/main.rs @@ -0,0 +1,124 @@ +//! Launches the benchmark target once and reports what the launch cost. +//! +//! This is the only benchmark piece that links fspy. The harness compares two +//! builds of this binary — one against the fspy under review and one against +//! the baseline fspy — so both revisions must be measured by identical code: +//! the harness always builds this crate from the same source for both. +//! +//! Prints one line to stdout: the wall clock of the launch in nanoseconds, +//! followed by the two batch timings the target reported for itself. + +use std::{env, ffi::OsString, process::Stdio, time::Instant}; + +use fspy::Command; +use tokio::{io::AsyncReadExt as _, process::ChildStdout, runtime::Builder}; +use tokio_util::sync::CancellationToken; + +#[cfg(unix)] +const MISSING_PATH: &str = "/.fspy-benchmark-missing"; +#[cfg(windows)] +const MISSING_PATH: &str = r"C:\.fspy-benchmark-missing"; + +fn main() { + let mut args = env::args_os().skip(1).collect::>(); + let mode = match args.first().map(OsString::as_os_str) { + Some(arg) if arg == "--untracked" => { + args.remove(0); + Mode::Untracked + } + Some(arg) if arg == "--validate" => { + args.remove(0); + Mode::Validate + } + _ => Mode::Tracked, + }; + let (target, target_args) = + args.split_first().expect("usage: fspy_benchmark_launcher [MODE] TARGET ARGS..."); + + let runtime = Builder::new_multi_thread().worker_threads(2).enable_all().build().unwrap(); + runtime.block_on(async { + match mode { + Mode::Tracked => report(run_tracked(target, target_args).await).await, + Mode::Untracked => report(run_untracked(target, target_args).await).await, + Mode::Validate => validate(target, target_args).await, + } + }); +} + +enum Mode { + Tracked, + Untracked, + Validate, +} + +struct Launch { + wall_nanos: u128, + stdout: ChildStdout, +} + +/// Times the launch from just before the spawn to just after the wait, so +/// that a tracked launch covers session setup, injection, and teardown, and +/// nothing of this launcher's own startup. +async fn run_tracked(target: &OsString, target_args: &[OsString]) -> Launch { + let mut command = Command::new(target); + command.args(target_args).stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::inherit()); + let start = Instant::now(); + let mut child = command + .spawn(CancellationToken::new()) + .await + .expect("failed to spawn tracked benchmark target"); + let stdout = child.stdout.take().expect("tracked benchmark target has no stdout"); + let termination = child.wait_handle.await.expect("failed to wait for tracked target"); + let wall_nanos = start.elapsed().as_nanos(); + assert!( + termination.status.success(), + "tracked benchmark target failed: {}", + termination.status + ); + Launch { wall_nanos, stdout } +} + +async fn run_untracked(target: &OsString, target_args: &[OsString]) -> Launch { + let mut command = tokio::process::Command::new(target); + command.args(target_args).stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::inherit()); + let start = Instant::now(); + let mut child = command.spawn().expect("failed to spawn untracked benchmark target"); + let stdout = child.stdout.take().expect("untracked benchmark target has no stdout"); + let status = child.wait().await.expect("failed to wait for untracked target"); + let wall_nanos = start.elapsed().as_nanos(); + assert!(status.success(), "untracked benchmark target failed: {status}"); + Launch { wall_nanos, stdout } +} + +/// Relays the samples the target timed for itself after the wall clock, so the +/// harness reads every number from one line. +async fn report(mut launch: Launch) { + let mut samples = Vec::new(); + launch.stdout.read_to_end(&mut samples).await.expect("failed to read reported samples"); + let samples = str::from_utf8(&samples).expect("reported samples are not utf-8"); + #[expect(clippy::print_stdout, reason = "the harness reads the measurement from stdout")] + { + println!("{} {}", launch.wall_nanos, samples.trim()); + } +} + +/// Runs the target tracked and asserts that its accesses were captured, so +/// that the harness never benchmarks tracking that silently stopped working. +async fn validate(target: &OsString, target_args: &[OsString]) { + let mut command = Command::new(target); + command.args(target_args).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::inherit()); + let termination = command + .spawn(CancellationToken::new()) + .await + .expect("failed to spawn tracked benchmark target") + .wait_handle + .await + .expect("failed to wait for tracked target"); + assert!(termination.status.success(), "benchmark target failed: {}", termination.status); + let captured_missing_access = termination.path_accesses.iter().any(|access| { + access.path.strip_path_prefix(MISSING_PATH, |result| { + result.is_ok_and(|path| path.as_os_str().is_empty()) + }) + }); + assert!(captured_missing_access, "failed to capture access to {MISSING_PATH}"); +} diff --git a/justfile b/justfile index bd2d114f2..628d8123d 100644 --- a/justfile +++ b/justfile @@ -48,7 +48,7 @@ lint-windows: cargo-xwin clippy --workspace --all-targets --all-features --target x86_64-pc-windows-msvc -- --deny warnings benchmark-fspy: - cargo bench -p fspy_benchmark --bench fspy + cargo run --release -p fspy_benchmark [unix] doc: From 1f3247cb7009174905a3e2d233b97781581ef1e1 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Tue, 28 Jul 2026 10:14:55 +0800 Subject: [PATCH 26/35] test(fspy): cycle every launch order in the benchmark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rotating one fixed cycle keeps every arm's neighbors the same forever — the baseline always ran right behind the untracked launch — and the leftovers a launch inherits from its neighbor do not cancel out, which biased the A/A change on the launch row by almost a percent. Cycle all orderings instead, so position and neighbors both even out. Co-Authored-By: Claude Fable 5 --- crates/fspy_benchmark/README.md | 4 +-- crates/fspy_benchmark/src/main.rs | 50 ++++++++++++++++++++++--------- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/crates/fspy_benchmark/README.md b/crates/fspy_benchmark/README.md index c60f5f2aa..3f30a01e7 100644 --- a/crates/fspy_benchmark/README.md +++ b/crates/fspy_benchmark/README.md @@ -41,8 +41,8 @@ compared can run side by side on one machine: that links fspy. - `fspy_benchmark` is the harness. CI builds the launcher twice, against the fspy under review and against the merge-base fspy (from the same launcher source, so both revisions are measured by - identical code), then the harness launches both builds interleaved, back to back, in rotating - order. Whatever the runner instance is doing to the numbers, it is doing to both arms. + identical code), then the harness launches both builds interleaved, back to back, cycling + every ordering. Whatever the runner instance is doing to the numbers, it is doing to both arms. Each iteration yields the ratio of the head launch to the base launch per metric; the reported change is the median of those ratios. An untracked arm runs in the same rotation and prices diff --git a/crates/fspy_benchmark/src/main.rs b/crates/fspy_benchmark/src/main.rs index 57486890c..3e9496db6 100644 --- a/crates/fspy_benchmark/src/main.rs +++ b/crates/fspy_benchmark/src/main.rs @@ -12,8 +12,8 @@ //! numbers, it is doing to both sides. //! //! Each iteration launches every arm — baseline-tracked, head-tracked, and -//! untracked — back to back in rotating order, and yields the ratio of head -//! to baseline per metric. The reported change is the median of those ratios, +//! untracked — back to back, cycling every ordering, and yields the ratio of +//! head to baseline per metric. The reported change is the median of those ratios, //! so it prices exactly one thing: what the fspy change under review costs a //! tracked process. The untracked arm prices tracking itself, as context. @@ -36,7 +36,8 @@ struct Workload { threads: &'static str, /// Opens per target thread, passed through to it. opens: &'static str, - /// Measured iterations. Each one launches every arm once. + /// Measured iterations. Each one launches every arm once. A multiple of + /// every order-cycle length, so each ordering runs equally often. iterations: usize, /// Unmeasured iterations run first, to fill caches and settle the runner. warmup: usize, @@ -57,12 +58,12 @@ const LAUNCH_WORKLOAD: Workload = /// Accesses timed one thread at a time, from inside the target, so they price /// interception rather than the launch around it. const ACCESS_WORKLOAD: Workload = - Workload { threads: "1", opens: "4096", iterations: 100, warmup: 3 }; + Workload { threads: "1", opens: "4096", iterations: 102, warmup: 3 }; /// The same access count spread over two threads, each opening a path of its /// own, so that they exercise concurrent interception. const CONCURRENT_WORKLOAD: Workload = - Workload { threads: "2", opens: "2048", iterations: 100, warmup: 3 }; + Workload { threads: "2", opens: "2048", iterations: 102, warmup: 3 }; /// What a row reads out of the launches of its workload. #[derive(Clone, Copy)] @@ -180,25 +181,46 @@ struct Iteration { untracked: Sample, } +/// Every ordering of the arms of an iteration. Cycling through all of them — +/// rather than rotating one fixed cycle, which keeps every arm's neighbors +/// the same forever — evens out both what position an arm runs in and which +/// arm's leftovers it runs after. +const ORDERS_WITHOUT_BASE: &[&[Arm]] = + &[&[Arm::Head, Arm::Untracked], &[Arm::Untracked, Arm::Head]]; +const ORDERS_WITH_BASE: &[&[Arm]] = &[ + &[Arm::Head, Arm::Untracked, Arm::Base], + &[Arm::Untracked, Arm::Base, Arm::Head], + &[Arm::Base, Arm::Head, Arm::Untracked], + &[Arm::Base, Arm::Untracked, Arm::Head], + &[Arm::Untracked, Arm::Head, Arm::Base], + &[Arm::Head, Arm::Base, Arm::Untracked], +]; + +#[derive(Clone, Copy)] +enum Arm { + Head, + Untracked, + Base, +} + /// Runs a suite's workload and reports its rows. Returns whether any row /// regressed past its threshold. fn run_suite(backend: &Backend, suite: &Suite, base_launcher: Option<&OsStr>) -> bool { let workload = suite.workload; - let arm_count = if base_launcher.is_some() { 3 } else { 2 }; + let orders = if base_launcher.is_some() { ORDERS_WITH_BASE } else { ORDERS_WITHOUT_BASE }; let mut iterations = Vec::with_capacity(workload.iterations); for index in 0..workload.warmup + workload.iterations { let mut iteration = Iteration::default(); - // Rotate which arm goes first, so that no arm always pays for waking - // the machine up or always benefits from the caches the previous arm - // warmed. - for position in 0..arm_count { - match (index + position) % arm_count { - 0 => iteration.head = launch(HEAD_LAUNCHER.as_ref(), None, backend, workload), - 1 => { + for &arm in orders[index % orders.len()] { + match arm { + Arm::Head => { + iteration.head = launch(HEAD_LAUNCHER.as_ref(), None, backend, workload); + } + Arm::Untracked => { iteration.untracked = launch(HEAD_LAUNCHER.as_ref(), Some("--untracked"), backend, workload); } - _ => { + Arm::Base => { iteration.base = launch(base_launcher.unwrap(), None, backend, workload); } } From 8714c777811e9757078c3cf143599105ac656189 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Tue, 28 Jul 2026 10:18:24 +0800 Subject: [PATCH 27/35] ci: run the fspy benchmark only when it can move Nothing outside the fspy and benchmark crates, the locked dependencies, and the toolchain can change what fspy costs, so skip the benchmark for every other pull request. Co-Authored-By: Claude Fable 5 --- .github/workflows/fspy-benchmark.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index 3fbbe8d3d..13151f867 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -6,6 +6,13 @@ on: workflow_dispatch: pull_request: types: [opened, reopened, synchronize] + # Only changes that can move fspy's cost: fspy and benchmark crates, their + # locked dependencies, the toolchain, and this workflow. + paths: + - 'crates/fspy*/**' + - 'Cargo.lock' + - 'rust-toolchain.toml' + - '.github/workflows/fspy-benchmark.yml' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} From af07265f1c6a7e6c11c21622cd7df763f8c8cad3 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Tue, 28 Jul 2026 10:31:53 +0800 Subject: [PATCH 28/35] ci: install the musl target for the doc build The benchmark harness has a x86_64-unknown-linux-musl artifact dependency among its regular dependencies, so documenting the workspace now builds the static benchmark target and needs the musl standard library, like the clippy and test jobs already do. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 594e60e07..38bbe2b3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -353,6 +353,10 @@ jobs: tools: cargo-shear@1.11.1,cargo-autoinherit@0.1.6 components: clippy rust-docs rustfmt + # fspy_benchmark has a x86_64-unknown-linux-musl artifact dependency, so + # documenting the workspace needs the musl standard library. + - run: rustup target add x86_64-unknown-linux-musl + - uses: oxc-project/setup-node@4c588e9266bd930b6ddc34307df0659ed511d187 # v1.3.1 - run: pnpm exec vp fmt --check - run: cargo autoinherit && git diff --exit-code From 46c02d2a8fd00b31be4d5013dfc1efe72bce2b24 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Tue, 28 Jul 2026 15:46:57 +0800 Subject: [PATCH 29/35] test(fspy): report only launch and concurrent access Four rows per backend made the report harder to read than the two questions it answers: what starting a tracked process costs, and what an access costs while tracking. Keep one row for each. The access row keeps the two-thread workload, because a process that fspy tracks rarely accesses files from one thread, and its 8% threshold, set from the spread that concurrent interception shows between runner instances. The single-threaded row and the tail percentile were finer instruments for telling contention and scatter apart from cost, at the price of two more rows to explain; a red access row now just leaves that bisection to the investigation. Co-Authored-By: Claude Fable 5 --- crates/fspy_benchmark/README.md | 14 +++++------ crates/fspy_benchmark/src/main.rs | 29 +++++----------------- crates/fspy_benchmark_launcher/src/main.rs | 4 +-- crates/fspy_benchmark_target/src/main.rs | 9 +++---- 4 files changed, 18 insertions(+), 38 deletions(-) diff --git a/crates/fspy_benchmark/README.md b/crates/fspy_benchmark/README.md index 3f30a01e7..b10554f87 100644 --- a/crates/fspy_benchmark/README.md +++ b/crates/fspy_benchmark/README.md @@ -5,12 +5,9 @@ cost is two separable things, and each is measured apart from the other so that - `launch` is the wall clock of a launch that opens nothing, so it prices starting a process under tracking: injection, session setup, and teardown. -- `access` is how long a batch of opens takes, timed by the thread that runs them, so it prices - interception itself rather than the launch around it. -- `access-tail` is the slow end of the same samples, which moves when accesses start to scatter - rather than to cost more. -- `access-concurrent` is `access` with a second thread intercepting at the same time, each thread - opening a path of its own. +- `access` is how long a batch of opens takes, timed by each of two threads opening a path of + their own, so it prices interception under the concurrency a tracked process actually has, + rather than the launch around it. Linux measures a dynamically linked target that exercises `LD_PRELOAD` plus a `x86_64-unknown-linux-musl` target that exercises seccomp user notification. macOS measures @@ -34,8 +31,9 @@ So nothing is ever compared across runs. The pieces are split so that the two fs compared can run side by side on one machine: - `fspy_benchmark_target` (and its static twin) is the workload: it opens missing paths from a - configurable number of threads and reports how long its batches took on stdout. It does not link - fspy. + configurable number of threads and reports the median batch time on stdout. Batches are a few + microseconds long, so the median falls on batches the scheduler left alone; a whole-run wall + clock would sum every disturbance instead. It does not link fspy. - `fspy_benchmark_launcher` runs the target once — tracked through `fspy::Command`, or untracked for context — and prints the launch wall clock plus the target's numbers. It is the only piece that links fspy. diff --git a/crates/fspy_benchmark/src/main.rs b/crates/fspy_benchmark/src/main.rs index 3e9496db6..4b834e20d 100644 --- a/crates/fspy_benchmark/src/main.rs +++ b/crates/fspy_benchmark/src/main.rs @@ -55,14 +55,10 @@ const LAUNCH_ITERATIONS: usize = 300; const LAUNCH_WORKLOAD: Workload = Workload { threads: "2", opens: "0", iterations: LAUNCH_ITERATIONS, warmup: 5 }; -/// Accesses timed one thread at a time, from inside the target, so they price -/// interception rather than the launch around it. +/// Opens timed from inside the target, so they price interception rather than +/// the launch around it. Two threads, each opening a path of its own, because +/// a process that fspy tracks rarely accesses files from one thread. const ACCESS_WORKLOAD: Workload = - Workload { threads: "1", opens: "4096", iterations: 102, warmup: 3 }; - -/// The same access count spread over two threads, each opening a path of its -/// own, so that they exercise concurrent interception. -const CONCURRENT_WORKLOAD: Workload = Workload { threads: "2", opens: "2048", iterations: 102, warmup: 3 }; /// What a row reads out of the launches of its workload. @@ -72,9 +68,6 @@ enum Metric { Wall, /// The typical batch of opens, as timed inside the target. Typical, - /// The slow end of those batches, which moves when accesses start to - /// scatter rather than to cost more. - Tail, } struct Row { @@ -97,15 +90,7 @@ const LAUNCH_SUITE: Suite = Suite { const ACCESS_SUITE: Suite = Suite { workload: &ACCESS_WORKLOAD, - rows: &[ - Row { name: "access", metric: Metric::Typical, threshold: 0.05 }, - Row { name: "access-tail", metric: Metric::Tail, threshold: 0.08 }, - ], -}; - -const CONCURRENT_SUITE: Suite = Suite { - workload: &CONCURRENT_WORKLOAD, - rows: &[Row { name: "access-concurrent", metric: Metric::Typical, threshold: 0.08 }], + rows: &[Row { name: "access", metric: Metric::Typical, threshold: 0.08 }], }; struct Backend { @@ -130,7 +115,7 @@ fn main() { if let Some(base_launcher) = &base_launcher { validate(base_launcher, backend.target); } - for suite in [&LAUNCH_SUITE, &ACCESS_SUITE, &CONCURRENT_SUITE] { + for suite in [&LAUNCH_SUITE, &ACCESS_SUITE] { regressed |= run_suite(backend, suite, base_launcher.as_deref()); } } @@ -160,7 +145,6 @@ fn parse_base_launcher() -> Option { struct Sample { wall: f64, typical: f64, - tail: f64, } impl Sample { @@ -168,7 +152,6 @@ impl Sample { match metric { Metric::Wall => self.wall, Metric::Typical => self.typical, - Metric::Tail => self.tail, } } } @@ -302,7 +285,7 @@ fn launch(launcher: &OsStr, mode: Option<&str>, backend: &Backend, workload: &Wo .split_whitespace() .map(|number| number.parse().expect("unreadable launcher measurement")); let mut next = || numbers.next().expect("launcher reported too few measurements"); - Sample { wall: next(), typical: next(), tail: next() } + Sample { wall: next(), typical: next() } } fn validate(launcher: &OsStr, target: &str) { diff --git a/crates/fspy_benchmark_launcher/src/main.rs b/crates/fspy_benchmark_launcher/src/main.rs index 91126f6ac..c1eb2e64f 100644 --- a/crates/fspy_benchmark_launcher/src/main.rs +++ b/crates/fspy_benchmark_launcher/src/main.rs @@ -6,7 +6,7 @@ //! the harness always builds this crate from the same source for both. //! //! Prints one line to stdout: the wall clock of the launch in nanoseconds, -//! followed by the two batch timings the target reported for itself. +//! followed by the batch timing the target reported for itself. use std::{env, ffi::OsString, process::Stdio, time::Instant}; @@ -90,7 +90,7 @@ async fn run_untracked(target: &OsString, target_args: &[OsString]) -> Launch { Launch { wall_nanos, stdout } } -/// Relays the samples the target timed for itself after the wall clock, so the +/// Relays the sample the target timed for itself after the wall clock, so the /// harness reads every number from one line. async fn report(mut launch: Launch) { let mut samples = Vec::new(); diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs index da3b63e26..6d9886d82 100644 --- a/crates/fspy_benchmark_target/src/main.rs +++ b/crates/fspy_benchmark_target/src/main.rs @@ -66,14 +66,13 @@ fn time_opens(barrier: &Barrier, path: &str, open_count: usize) -> Vec samples } -/// Reports the typical and the slow sample, so that a change in how long opens -/// take can be told apart from a change in how much they scatter. +/// Reports the typical sample. The median discards the batches the scheduler +/// spoiled, which a whole-run wall clock would sum up instead. fn report(samples: &mut [Duration]) { samples.sort_unstable(); - let percentile = - |percent: usize| samples.get(samples.len() * percent / 100).map_or(0, Duration::as_nanos); + let median = samples.get(samples.len() / 2).map_or(0, Duration::as_nanos); #[expect(clippy::print_stdout, reason = "the benchmark reads the samples from stdout")] { - println!("{} {}", percentile(50), percentile(90)); + println!("{median}"); } } From 9e5c392d8c8e300b52cba21c946ea21eedd9cb66 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Tue, 28 Jul 2026 16:04:43 +0800 Subject: [PATCH 30/35] test(fspy): simplify the benchmark after review Collapse the workload/row/suite triple into one suite struct now that every suite has exactly one row, and read the suite's metric straight out of the launcher's line instead of storing both numbers and selecting later. Round iterations up to whole order cycles instead of hand-picking multiples. Make the launcher the one owner of the missing path, passed to the target as an argument, so the two crates cannot drift apart on it. Require the target's arguments instead of silently substituting defaults nothing uses, derive per-thread suffixes instead of wrapping an eight-entry table, and share the barrier by scoped borrow instead of an Arc. Declare the tokio macros feature on fspy itself, which uses select!, instead of relying on whichever dependent enables it; the launcher then declares only what it uses. Drop the workspace-level cargo-shear ignore that the package-level list already covers, and document why the static target is a separate package over the same source. Co-Authored-By: Claude Fable 5 --- Cargo.toml | 1 - crates/fspy/Cargo.toml | 2 +- crates/fspy_benchmark/src/main.rs | 190 ++++++++---------- crates/fspy_benchmark_launcher/Cargo.toml | 2 +- crates/fspy_benchmark_launcher/src/main.rs | 32 ++- .../fspy_benchmark_static_target/src/main.rs | 4 + crates/fspy_benchmark_target/src/main.rs | 37 ++-- 7 files changed, 127 insertions(+), 141 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 002cea9be..7e1382b6b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -179,7 +179,6 @@ zstd = "0.13" [workspace.metadata.cargo-shear] ignored = [ # These are artifact dependencies. They are not directly `use`d in Rust code. - "fspy_benchmark_target", "fspy_preload_unix", "fspy_preload_windows", "vite_task_client_napi", diff --git a/crates/fspy/Cargo.toml b/crates/fspy/Cargo.toml index 236a0f7bd..38b9ffb51 100644 --- a/crates/fspy/Cargo.toml +++ b/crates/fspy/Cargo.toml @@ -18,7 +18,7 @@ ouroboros = { workspace = true } rustc-hash = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = ["net", "process", "io-util", "sync", "rt"] } +tokio = { workspace = true, features = ["macros", "net", "process", "io-util", "sync", "rt"] } tokio-util = { workspace = true } which = { workspace = true, features = ["tracing"] } diff --git a/crates/fspy_benchmark/src/main.rs b/crates/fspy_benchmark/src/main.rs index 4b834e20d..2e1f45ad5 100644 --- a/crates/fspy_benchmark/src/main.rs +++ b/crates/fspy_benchmark/src/main.rs @@ -13,9 +13,10 @@ //! //! Each iteration launches every arm — baseline-tracked, head-tracked, and //! untracked — back to back, cycling every ordering, and yields the ratio of -//! head to baseline per metric. The reported change is the median of those ratios, -//! so it prices exactly one thing: what the fspy change under review costs a -//! tracked process. The untracked arm prices tracking itself, as context. +//! head to baseline per metric. The reported change is the median of those +//! ratios, so it prices exactly one thing: what the fspy change under review +//! costs a tracked process. The untracked arm prices tracking itself, as +//! context. use std::{ env, @@ -28,40 +29,11 @@ const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); #[cfg(all(target_os = "linux", target_arch = "x86_64"))] const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); -/// Opens one small batch, only to prove that capturing works before measuring. -const VALIDATION_ARGS: [&str; 2] = ["1", "8"]; +/// Threads the target runs, passed through to it. Two, because a process that +/// fspy tracks rarely accesses files from one thread. +const THREADS: &str = "2"; -struct Workload { - /// Thread count the target spawns, passed through to it. - threads: &'static str, - /// Opens per target thread, passed through to it. - opens: &'static str, - /// Measured iterations. Each one launches every arm once. A multiple of - /// every order-cycle length, so each ordering runs equally often. - iterations: usize, - /// Unmeasured iterations run first, to fill caches and settle the runner. - warmup: usize, -} - -/// Launches cost several times more wall clock on Windows, so it affords -/// fewer of them in the same time. -#[cfg(windows)] -const LAUNCH_ITERATIONS: usize = 150; -#[cfg(not(windows))] -const LAUNCH_ITERATIONS: usize = 300; - -/// Opens nothing, so the whole launch is the cost of starting a tracked -/// process. Two threads, because a process that fspy tracks rarely has one. -const LAUNCH_WORKLOAD: Workload = - Workload { threads: "2", opens: "0", iterations: LAUNCH_ITERATIONS, warmup: 5 }; - -/// Opens timed from inside the target, so they price interception rather than -/// the launch around it. Two threads, each opening a path of its own, because -/// a process that fspy tracks rarely accesses files from one thread. -const ACCESS_WORKLOAD: Workload = - Workload { threads: "2", opens: "2048", iterations: 102, warmup: 3 }; - -/// What a row reads out of the launches of its workload. +/// What a suite reads out of its launches. #[derive(Clone, Copy)] enum Metric { /// Wall clock of the launch, from spawn to reap. @@ -70,27 +42,41 @@ enum Metric { Typical, } -struct Row { +struct Suite { name: &'static str, + /// Opens per target thread, passed through to it. + opens: &'static str, + /// Measured iterations. Each one launches every arm once. + iterations: usize, + /// Unmeasured iterations run first, to fill caches and settle the runner. + warmup: usize, metric: Metric, - /// The change past which the row fails the run, as a share. Set from the + /// The change past which the suite fails the run, as a share. Set from the /// spread observed when benchmarking one commit on many runner instances. threshold: f64, } -struct Suite { - workload: &'static Workload, - rows: &'static [Row], -} - +/// Opens nothing, so the whole launch is the cost of starting a tracked +/// process. Launches cost several times more wall clock on Windows, so it +/// affords fewer of them in the same time. const LAUNCH_SUITE: Suite = Suite { - workload: &LAUNCH_WORKLOAD, - rows: &[Row { name: "launch", metric: Metric::Wall, threshold: 0.05 }], + name: "launch", + opens: "0", + iterations: if cfg!(windows) { 150 } else { 300 }, + warmup: 5, + metric: Metric::Wall, + threshold: 0.05, }; +/// Opens timed from inside the target, so they price interception rather than +/// the launch around it. const ACCESS_SUITE: Suite = Suite { - workload: &ACCESS_WORKLOAD, - rows: &[Row { name: "access", metric: Metric::Typical, threshold: 0.08 }], + name: "access", + opens: "2048", + iterations: 102, + warmup: 3, + metric: Metric::Typical, + threshold: 0.08, }; struct Backend { @@ -140,28 +126,12 @@ fn parse_base_launcher() -> Option { base_launcher } -/// What one launch measured, in nanoseconds. -#[derive(Clone, Copy, Default)] -struct Sample { - wall: f64, - typical: f64, -} - -impl Sample { - const fn metric(&self, metric: Metric) -> f64 { - match metric { - Metric::Wall => self.wall, - Metric::Typical => self.typical, - } - } -} - -/// One iteration's launches, one per arm. +/// One iteration's measurements, one per arm, in nanoseconds. #[derive(Default)] struct Iteration { - base: Sample, - head: Sample, - untracked: Sample, + base: f64, + head: f64, + untracked: f64, } /// Every ordering of the arms of an iteration. Cycling through all of them — @@ -186,62 +156,52 @@ enum Arm { Base, } -/// Runs a suite's workload and reports its rows. Returns whether any row -/// regressed past its threshold. +/// Runs a suite and reports its row. Returns whether it regressed past its +/// threshold. fn run_suite(backend: &Backend, suite: &Suite, base_launcher: Option<&OsStr>) -> bool { - let workload = suite.workload; let orders = if base_launcher.is_some() { ORDERS_WITH_BASE } else { ORDERS_WITHOUT_BASE }; - let mut iterations = Vec::with_capacity(workload.iterations); - for index in 0..workload.warmup + workload.iterations { + // Whole cycles only, so that each ordering runs equally often. + let iterations = suite.iterations.next_multiple_of(orders.len()); + let mut measured = Vec::with_capacity(iterations); + for index in 0..suite.warmup + iterations { let mut iteration = Iteration::default(); for &arm in orders[index % orders.len()] { match arm { Arm::Head => { - iteration.head = launch(HEAD_LAUNCHER.as_ref(), None, backend, workload); + iteration.head = launch(HEAD_LAUNCHER.as_ref(), None, backend, suite); } Arm::Untracked => { iteration.untracked = - launch(HEAD_LAUNCHER.as_ref(), Some("--untracked"), backend, workload); + launch(HEAD_LAUNCHER.as_ref(), Some("--untracked"), backend, suite); } Arm::Base => { - iteration.base = launch(base_launcher.unwrap(), None, backend, workload); + iteration.base = launch(base_launcher.unwrap(), None, backend, suite); } } } - if index >= workload.warmup { - iterations.push(iteration); + if index >= suite.warmup { + measured.push(iteration); } } - let mut regressed = false; - for row in suite.rows { - regressed |= report_row(backend, row, &iterations, base_launcher.is_some()); - } - regressed + report(backend, suite, &measured, base_launcher.is_some()) } -/// Prints one row and returns whether it regressed past its threshold. +/// Prints the suite's row and returns whether it regressed past its threshold. #[expect(clippy::print_stdout, reason = "the report is the benchmark's output")] -fn report_row(backend: &Backend, row: &Row, iterations: &[Iteration], has_base: bool) -> bool { - let name = [backend.name, "/", row.name].concat(); - let overhead = median( - iterations - .iter() - .map(|it| it.head.metric(row.metric) / it.untracked.metric(row.metric) - 1.0), - ); +fn report(backend: &Backend, suite: &Suite, iterations: &[Iteration], has_base: bool) -> bool { + let name = [backend.name, "/", suite.name].concat(); + let overheads = sorted(iterations.iter().map(|it| it.head / it.untracked - 1.0)); + let overhead = quantile(&overheads, 1, 2); if has_base { - let mut changes: Vec = iterations - .iter() - .map(|it| it.head.metric(row.metric) / it.base.metric(row.metric) - 1.0) - .collect(); - changes.sort_unstable_by(f64::total_cmp); - let change = changes[changes.len() / 2]; - let low = changes[changes.len() / 4]; - let high = changes[changes.len() * 3 / 4]; - let verdict = if change > row.threshold { + let changes = sorted(iterations.iter().map(|it| it.head / it.base - 1.0)); + let change = quantile(&changes, 1, 2); + let low = quantile(&changes, 1, 4); + let high = quantile(&changes, 3, 4); + let verdict = if change > suite.threshold { " REGRESSED" - } else if change < -row.threshold { + } else if change < -suite.threshold { " improved" } else { "" @@ -251,30 +211,35 @@ fn report_row(backend: &Backend, row: &Row, iterations: &[Iteration], has_base: change * 100.0, low * 100.0, high * 100.0, - row.threshold * 100.0, + suite.threshold * 100.0, overhead * 100.0, ); - change > row.threshold + change > suite.threshold } else { println!("{name:<26} overhead {:>+8.2}%", overhead * 100.0); false } } -fn median(values: impl Iterator) -> f64 { +fn sorted(values: impl Iterator) -> Vec { let mut values: Vec = values.collect(); values.sort_unstable_by(f64::total_cmp); - values[values.len() / 2] + values +} + +fn quantile(sorted: &[f64], numerator: usize, denominator: usize) -> f64 { + sorted[sorted.len() * numerator / denominator] } -/// Launches one arm and reads the three numbers its launcher printed. -fn launch(launcher: &OsStr, mode: Option<&str>, backend: &Backend, workload: &Workload) -> Sample { +/// Launches one arm and reads the metric the suite names out of the two +/// numbers its launcher printed. +fn launch(launcher: &OsStr, mode: Option<&str>, backend: &Backend, suite: &Suite) -> f64 { let mut command = Command::new(launcher); if let Some(mode) = mode { command.arg(mode); } let output = command - .args([backend.target, workload.threads, workload.opens]) + .args([backend.target, THREADS, suite.opens]) .stdin(Stdio::null()) .stderr(Stdio::inherit()) .output() @@ -285,14 +250,21 @@ fn launch(launcher: &OsStr, mode: Option<&str>, backend: &Backend, workload: &Wo .split_whitespace() .map(|number| number.parse().expect("unreadable launcher measurement")); let mut next = || numbers.next().expect("launcher reported too few measurements"); - Sample { wall: next(), typical: next() } + let [wall, typical] = [next(), next()]; + match suite.metric { + Metric::Wall => wall, + Metric::Typical => typical, + } } fn validate(launcher: &OsStr, target: &str) { let status = Command::new(launcher) .arg("--validate") .arg(target) - .args(VALIDATION_ARGS) + // One thread opening one batch: the count matches the target's + // OPENS_PER_SAMPLE. Fewer would open nothing, which validation + // catches by failing to capture the access. + .args(["1", "8"]) .stdin(Stdio::null()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) diff --git a/crates/fspy_benchmark_launcher/Cargo.toml b/crates/fspy_benchmark_launcher/Cargo.toml index ab6abb5c7..d380c8050 100644 --- a/crates/fspy_benchmark_launcher/Cargo.toml +++ b/crates/fspy_benchmark_launcher/Cargo.toml @@ -17,5 +17,5 @@ doctest = false [dependencies] fspy = { workspace = true } -tokio = { workspace = true, features = ["macros", "process", "rt-multi-thread"] } +tokio = { workspace = true, features = ["io-util", "process", "rt-multi-thread"] } tokio-util = { workspace = true } diff --git a/crates/fspy_benchmark_launcher/src/main.rs b/crates/fspy_benchmark_launcher/src/main.rs index c1eb2e64f..8a0a6c70e 100644 --- a/crates/fspy_benchmark_launcher/src/main.rs +++ b/crates/fspy_benchmark_launcher/src/main.rs @@ -14,6 +14,9 @@ use fspy::Command; use tokio::{io::AsyncReadExt as _, process::ChildStdout, runtime::Builder}; use tokio_util::sync::CancellationToken; +/// The base path the target opens, appended to its arguments so that only this +/// crate names it: the target derives its per-thread paths from it, and +/// validation asserts the bare path was captured. #[cfg(unix)] const MISSING_PATH: &str = "/.fspy-benchmark-missing"; #[cfg(windows)] @@ -61,7 +64,12 @@ struct Launch { /// nothing of this launcher's own startup. async fn run_tracked(target: &OsString, target_args: &[OsString]) -> Launch { let mut command = Command::new(target); - command.args(target_args).stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::inherit()); + command + .args(target_args) + .arg(MISSING_PATH) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()); let start = Instant::now(); let mut child = command .spawn(CancellationToken::new()) @@ -80,7 +88,12 @@ async fn run_tracked(target: &OsString, target_args: &[OsString]) -> Launch { async fn run_untracked(target: &OsString, target_args: &[OsString]) -> Launch { let mut command = tokio::process::Command::new(target); - command.args(target_args).stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::inherit()); + command + .args(target_args) + .arg(MISSING_PATH) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()); let start = Instant::now(); let mut child = command.spawn().expect("failed to spawn untracked benchmark target"); let stdout = child.stdout.take().expect("untracked benchmark target has no stdout"); @@ -93,12 +106,12 @@ async fn run_untracked(target: &OsString, target_args: &[OsString]) -> Launch { /// Relays the sample the target timed for itself after the wall clock, so the /// harness reads every number from one line. async fn report(mut launch: Launch) { - let mut samples = Vec::new(); - launch.stdout.read_to_end(&mut samples).await.expect("failed to read reported samples"); - let samples = str::from_utf8(&samples).expect("reported samples are not utf-8"); + let mut sample = Vec::new(); + launch.stdout.read_to_end(&mut sample).await.expect("failed to read the reported sample"); + let sample = str::from_utf8(&sample).expect("the reported sample is not utf-8"); #[expect(clippy::print_stdout, reason = "the harness reads the measurement from stdout")] { - println!("{} {}", launch.wall_nanos, samples.trim()); + println!("{} {}", launch.wall_nanos, sample.trim()); } } @@ -106,7 +119,12 @@ async fn report(mut launch: Launch) { /// that the harness never benchmarks tracking that silently stopped working. async fn validate(target: &OsString, target_args: &[OsString]) { let mut command = Command::new(target); - command.args(target_args).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::inherit()); + command + .args(target_args) + .arg(MISSING_PATH) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()); let termination = command .spawn(CancellationToken::new()) .await diff --git a/crates/fspy_benchmark_static_target/src/main.rs b/crates/fspy_benchmark_static_target/src/main.rs index 6e96ceda0..8f25ad746 100644 --- a/crates/fspy_benchmark_static_target/src/main.rs +++ b/crates/fspy_benchmark_static_target/src/main.rs @@ -1 +1,5 @@ +// A second package over the same source, because cargo names artifact +// dependency environment variables after the package: the harness needs the +// host build and the x86_64-unknown-linux-musl build of the target at once, +// and one package cannot be an artifact dependency for two triples. include!("../../fspy_benchmark_target/src/main.rs"); diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs index 6d9886d82..a17488dea 100644 --- a/crates/fspy_benchmark_target/src/main.rs +++ b/crates/fspy_benchmark_target/src/main.rs @@ -1,44 +1,37 @@ use std::{ env, fs::File, - sync::{Arc, Barrier}, + sync::Barrier, thread, time::{Duration, Instant}, }; -const DEFAULT_THREAD_COUNT: usize = 4; -const DEFAULT_OPEN_COUNT_PER_THREAD: usize = 512; - /// Opens behind one timing sample. Timing every open on its own would leave the /// sample close to the clock's granularity, which is 42 ns on Apple silicon and /// 100 ns on Windows, while a batch this small still keeps a sample short /// enough that the scheduler only ever spoils a few of them. const OPENS_PER_SAMPLE: usize = 8; -#[cfg(unix)] -const MISSING_PATH: &str = "/.fspy-benchmark-missing"; -#[cfg(windows)] -const MISSING_PATH: &str = r"C:\.fspy-benchmark-missing"; - -/// Threads open distinct paths so that they do not contend for the same lookup. -const THREAD_SUFFIXES: [&str; 8] = ["", "1", "2", "3", "4", "5", "6", "7"]; - fn main() { let mut args = env::args().skip(1); - let thread_count = - args.next().and_then(|arg| arg.parse().ok()).unwrap_or(DEFAULT_THREAD_COUNT).max(1); - let open_count_per_thread = - args.next().and_then(|arg| arg.parse().ok()).unwrap_or(DEFAULT_OPEN_COUNT_PER_THREAD); - let barrier = Arc::new(Barrier::new(thread_count)); + let mut next = || args.next().expect("usage: fspy_benchmark_target THREADS OPENS PATH"); + let thread_count: usize = next().parse().expect("unreadable thread count"); + let open_count_per_thread: usize = next().parse().expect("unreadable open count"); + let base_path = next(); - let mut samples = Vec::new(); + let barrier = &Barrier::new(thread_count); + let mut samples = Vec::with_capacity(thread_count * (open_count_per_thread / OPENS_PER_SAMPLE)); thread::scope(|scope| { let workers: Vec<_> = (0..thread_count) .map(|index| { - let barrier = Arc::clone(&barrier); - let mut path = MISSING_PATH.to_owned(); - path.push_str(THREAD_SUFFIXES[index % THREAD_SUFFIXES.len()]); - scope.spawn(move || time_opens(&barrier, &path, open_count_per_thread)) + // Threads open distinct paths so that they do not contend for + // the same lookup. The first thread opens the bare path, which + // is the one the launcher validates was captured. + let mut path = base_path.clone(); + if index > 0 { + path.push_str(&index.to_string()); + } + scope.spawn(move || time_opens(barrier, &path, open_count_per_thread)) }) .collect(); for worker in workers { From fc030eab98a594e824d6d00864ad93d1719b1805 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Tue, 28 Jul 2026 16:04:44 +0800 Subject: [PATCH 31/35] ci: trim the benchmark workflow Share the head build's target directory with the baseline launcher build and set the binary aside, instead of compiling the whole dependency graph a second time into a separate directory that the cache then carries. Save the cache only on main, where every other run can restore it, and add a main push trigger so something actually seeds it and the overhead history stays recorded. Skip uploading reports nothing consumes: fork pull requests get no comment, and the comment job reads the artifacts minutes after they upload, not in ninety days. Documenting the workspace no longer installs a musl standard library on every run just for the benchmark's static target; the doc build skips the benchmark crate instead. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 8 ++--- .github/workflows/fspy-benchmark.yml | 44 ++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38bbe2b3b..519c91518 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -353,16 +353,14 @@ jobs: tools: cargo-shear@1.11.1,cargo-autoinherit@0.1.6 components: clippy rust-docs rustfmt - # fspy_benchmark has a x86_64-unknown-linux-musl artifact dependency, so - # documenting the workspace needs the musl standard library. - - run: rustup target add x86_64-unknown-linux-musl - - uses: oxc-project/setup-node@4c588e9266bd930b6ddc34307df0659ed511d187 # v1.3.1 - run: pnpm exec vp fmt --check - run: cargo autoinherit && git diff --exit-code - run: cargo shear --deny-warnings - run: cargo fmt --check - - run: RUSTDOCFLAGS='-D warnings' cargo doc --no-deps --document-private-items + # --exclude fspy_benchmark: documenting it would build its musl artifact + # dependency, which needs a rust-std this job otherwise has no use for. + - run: RUSTDOCFLAGS='-D warnings' cargo doc --workspace --exclude fspy_benchmark --no-deps --document-private-items - uses: crate-ci/typos@bee27e3a4fd1ea2111cf90ab89cd076c870fce14 # v1.48.0 with: diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index 13151f867..91ad2ed8a 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -4,10 +4,18 @@ permissions: {} on: workflow_dispatch: + # Only changes that can move fspy's cost: fspy and benchmark crates, their + # locked dependencies, the toolchain, and this workflow. Runs on main only + # keep the build cache warm and record the overhead history. pull_request: - types: [opened, reopened, synchronize] - # Only changes that can move fspy's cost: fspy and benchmark crates, their - # locked dependencies, the toolchain, and this workflow. + paths: + - 'crates/fspy*/**' + - 'Cargo.lock' + - 'rust-toolchain.toml' + - '.github/workflows/fspy-benchmark.yml' + push: + branches: + - main paths: - 'crates/fspy*/**' - 'Cargo.lock' @@ -16,7 +24,7 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.ref_name != 'main' }} defaults: run: @@ -47,7 +55,6 @@ jobs: static: false runs-on: ${{ matrix.os }} env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target BASE_DIR: ${{ github.workspace }}/../fspy-benchmark-base steps: - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 @@ -58,7 +65,9 @@ jobs: - uses: oxc-project/setup-rust@3d6fb132fbe7cdcb66bf8ec193911c2945369d12 # v1.0.17 with: - save-cache: true + # Caches saved by pull request runs are scoped to their branch, so + # only runs on main seed the cache every other run restores. + save-cache: ${{ github.ref_name == 'main' }} cache-key: fspy-benchmark-${{ matrix.platform }} - name: Install static target @@ -82,7 +91,10 @@ jobs: if: github.event_name == 'pull_request' working-directory: ${{ env.BASE_DIR }} env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target/fspy-base + # Share the head build's target directory: the crates.io graph is + # fingerprinted by package and profile, not by workspace, so only the + # fspy path crates build twice. + CARGO_TARGET_DIR: ${{ github.workspace }}/target run: | # Both fspy revisions must be measured by identical code, and the # baseline may predate the launcher crate, so overlay the head's @@ -90,6 +102,10 @@ jobs: rm -rf crates/fspy_benchmark_launcher cp -R "$GITHUB_WORKSPACE/crates/fspy_benchmark_launcher" crates/fspy_benchmark_launcher cargo build --release -p fspy_benchmark_launcher + # Set the binary aside before the head build overwrites it. + exe="" + [[ "$RUNNER_OS" == "Windows" ]] && exe=.exe + cp "$CARGO_TARGET_DIR/release/fspy_benchmark_launcher$exe" "$RUNNER_TEMP/fspy-base-launcher$exe" - name: Run benchmark run: | @@ -97,7 +113,7 @@ jobs: [[ "$RUNNER_OS" == "Windows" ]] && exe=.exe args=() if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then - args=(--base-launcher "$CARGO_TARGET_DIR/fspy-base/release/fspy_benchmark_launcher$exe") + args=(--base-launcher "$RUNNER_TEMP/fspy-base-launcher$exe") fi mkdir -p .benchmark-results set -o pipefail @@ -114,16 +130,20 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" - name: Upload benchmark report - if: ${{ !cancelled() && github.event_name == 'pull_request' }} # Matrix jobs have isolated filesystems, so persist each report for the - # comment job to combine into one sticky comment after all platforms finish. + # comment job to combine into one sticky comment after all platforms + # finish. Fork pull requests get no comment, so nothing consumes their + # reports; the job summary already carries them. + if: >- + !cancelled() && + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: fspy-benchmark-report-${{ matrix.platform }} path: .benchmark-results/${{ matrix.platform }}.txt if-no-files-found: error - overwrite: true - retention-days: 90 + retention-days: 1 comment: name: Report benchmark From afc6aeb991c1a993628e31de9c6b867a3ca6438f Mon Sep 17 00:00:00 2001 From: wan9chi Date: Tue, 28 Jul 2026 16:08:59 +0800 Subject: [PATCH 32/35] ci: benchmark every pull request The benchmark finishes in a third of the CI critical path once the cache is warm, so path-filtering it saves nothing anyone waits for, and an unfiltered workflow can become a required check one day, which a path-filtered one cannot. On pull requests that leave fspy alone the change column compares two identical builds and reads zero; that is the price of not maintaining a list of every path that can move fspy's codegen. Co-Authored-By: Claude Fable 5 --- .github/workflows/fspy-benchmark.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index 91ad2ed8a..8d68c09c6 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -4,23 +4,11 @@ permissions: {} on: workflow_dispatch: - # Only changes that can move fspy's cost: fspy and benchmark crates, their - # locked dependencies, the toolchain, and this workflow. Runs on main only - # keep the build cache warm and record the overhead history. pull_request: - paths: - - 'crates/fspy*/**' - - 'Cargo.lock' - - 'rust-toolchain.toml' - - '.github/workflows/fspy-benchmark.yml' + # Runs on main keep the build cache warm and record the overhead history. push: branches: - main - paths: - - 'crates/fspy*/**' - - 'Cargo.lock' - - 'rust-toolchain.toml' - - '.github/workflows/fspy-benchmark.yml' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} From 213b889fdc0bad88e6fcfd0fb2fdc294c00e81d1 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Tue, 28 Jul 2026 16:10:38 +0800 Subject: [PATCH 33/35] test(fspy): build the launcher against merge-base fspy again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The launcher dropped the tokio macros feature as unused, which it is — by the launcher. fspy uses select! without declaring the feature, and older revisions of it only compile when a dependent enables it, which the launcher does on every benchmark run when it is built against the merge-base fspy. Linux hid the breakage because fspy pulls the seccomp crate there, which declares the feature itself; macOS and Windows have no such dependent. Keep fspy declaring the feature for itself, keep the launcher enabling it for the revisions that do not, and say why. Also stop failing the summary and upload steps a second time when the benchmark failed before writing a report; the comment job already prints a placeholder for a missing platform. Co-Authored-By: Claude Fable 5 --- .github/workflows/fspy-benchmark.yml | 7 +++++-- crates/fspy_benchmark_launcher/Cargo.toml | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index 8d68c09c6..dca071f44 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -113,7 +113,8 @@ jobs: run: | { echo '```text' - cat ".benchmark-results/${{ matrix.platform }}.txt" + cat ".benchmark-results/${{ matrix.platform }}.txt" || + echo 'no report; the benchmark failed before producing one' echo '```' } >> "$GITHUB_STEP_SUMMARY" @@ -130,7 +131,9 @@ jobs: with: name: fspy-benchmark-report-${{ matrix.platform }} path: .benchmark-results/${{ matrix.platform }}.txt - if-no-files-found: error + # A job that failed before producing a report has nothing to upload; + # the comment job prints its own placeholder for the platform. + if-no-files-found: warn retention-days: 1 comment: diff --git a/crates/fspy_benchmark_launcher/Cargo.toml b/crates/fspy_benchmark_launcher/Cargo.toml index d380c8050..740ad568e 100644 --- a/crates/fspy_benchmark_launcher/Cargo.toml +++ b/crates/fspy_benchmark_launcher/Cargo.toml @@ -17,5 +17,8 @@ doctest = false [dependencies] fspy = { workspace = true } -tokio = { workspace = true, features = ["io-util", "process", "rt-multi-thread"] } +# macros is fspy's, not ours: revisions of fspy that use select! without +# declaring the feature only compile when a dependent enables it, and this +# crate is built against the merge-base fspy on every benchmark run. +tokio = { workspace = true, features = ["io-util", "macros", "process", "rt-multi-thread"] } tokio-util = { workspace = true } From f1cc2f7454a17e928b03e1eb39ee0d83f8c7ec07 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Tue, 28 Jul 2026 16:27:09 +0800 Subject: [PATCH 34/35] docs(fspy): plain words for the benchmark README Co-Authored-By: Claude Fable 5 --- crates/fspy_benchmark/README.md | 63 ++++++++++----------------------- 1 file changed, 19 insertions(+), 44 deletions(-) diff --git a/crates/fspy_benchmark/README.md b/crates/fspy_benchmark/README.md index b10554f87..b4c8fecc3 100644 --- a/crates/fspy_benchmark/README.md +++ b/crates/fspy_benchmark/README.md @@ -1,17 +1,11 @@ # fspy benchmark -Measures what fspy costs a process it tracks, and whether a change to fspy moved that cost. The -cost is two separable things, and each is measured apart from the other so that neither dilutes it: +Measures what fspy costs a process it tracks, and whether a change to fspy moved that cost. It reports two rows: -- `launch` is the wall clock of a launch that opens nothing, so it prices starting a process under - tracking: injection, session setup, and teardown. -- `access` is how long a batch of opens takes, timed by each of two threads opening a path of - their own, so it prices interception under the concurrency a tracked process actually has, - rather than the launch around it. +- `launch`: the wall clock of a tracked launch that opens nothing. This is the cost of starting a process under tracking: injection, session setup, and teardown. +- `access`: how long a batch of opens takes, timed by two threads that each open their own path. This is the cost of interception itself, under the concurrency a tracked process normally has. -Linux measures a dynamically linked target that exercises `LD_PRELOAD` plus a -`x86_64-unknown-linux-musl` target that exercises seccomp user notification. macOS measures -`DYLD_INSERT_LIBRARIES` and Windows measures Detours injection. +Linux measures a dynamically linked target (`LD_PRELOAD`) and a `x86_64-unknown-linux-musl` target (seccomp user notification). macOS measures `DYLD_INSERT_LIBRARIES`. Windows measures Detours injection. Run the overhead report locally with: @@ -21,37 +15,18 @@ just benchmark-fspy ## How regressions are caught -Comparing a pull request run with a baseline recorded by an earlier run on `main` cannot work, -and every attempt to make it work failed the same way: two runs land on two runner instances, and -identically configured instances disagree — on absolute times by tens of percent, and still by up -to ten percent after normalizing every measurement against an untracked launch on the same -machine. A threshold above that noise is a threshold above any regression worth catching. - -So nothing is ever compared across runs. The pieces are split so that the two fspy revisions being -compared can run side by side on one machine: - -- `fspy_benchmark_target` (and its static twin) is the workload: it opens missing paths from a - configurable number of threads and reports the median batch time on stdout. Batches are a few - microseconds long, so the median falls on batches the scheduler left alone; a whole-run wall - clock would sum every disturbance instead. It does not link fspy. -- `fspy_benchmark_launcher` runs the target once — tracked through `fspy::Command`, or untracked - for context — and prints the launch wall clock plus the target's numbers. It is the only piece - that links fspy. -- `fspy_benchmark` is the harness. CI builds the launcher twice, against the fspy under review and - against the merge-base fspy (from the same launcher source, so both revisions are measured by - identical code), then the harness launches both builds interleaved, back to back, cycling - every ordering. Whatever the runner instance is doing to the numbers, it is doing to both arms. - -Each iteration yields the ratio of the head launch to the base launch per metric; the reported -change is the median of those ratios. An untracked arm runs in the same rotation and prices -tracking itself — that overhead number is context, not a gate. A row whose median change exceeds -its threshold fails the job. - -Locally there is no second fspy build, so `just benchmark-fspy` reports only the overhead column. -Point `--base-launcher` at another build of `fspy_benchmark_launcher` to compare two revisions by -hand. - -Because the launcher is compiled against both revisions from the head's source, a pull request -that changes the parts of the `fspy::Command` API the launcher uses can fail the baseline build. -Keep the launcher's API surface minimal; if the API must change incompatibly, the benchmark run -for that one pull request is expected to fail its baseline build. +Comparing a pull request run against a saved result from an earlier `main` run does not work. The two runs land on different runner instances, and identical instances disagree by more than any regression worth catching: tens of percent on absolute times, and still up to ten percent after normalizing against an untracked launch. A threshold above that noise is a threshold above every real regression. + +So nothing is ever compared across runs. Both fspy revisions run side by side on one machine, in one job. Three pieces make that possible: + +- `fspy_benchmark_target` (and its static twin) is the workload. It opens missing paths from several threads and prints the median batch time. It does not link fspy. Batches are a few microseconds long, so the median lands on batches the scheduler left alone; a whole-run wall clock would add up every disturbance instead. +- `fspy_benchmark_launcher` runs the target once, tracked through `fspy::Command` or untracked, and prints the launch wall clock plus the target's number. It is the only piece that links fspy. +- `fspy_benchmark` is the harness. CI builds the launcher twice: against the fspy under review, and against the merge-base fspy. Both builds use the same launcher source, so both revisions are measured by identical code. The harness then launches both builds back to back, cycling every ordering. Whatever the runner does to the numbers, it does to both. + +Each iteration gives one head/base ratio per row. The reported change is the median of those ratios. A row that moves past its threshold fails the job. + +An untracked launch runs in the same rotation. It prices tracking itself, and is reported as context, never gated. + +Locally there is no second fspy build, so `just benchmark-fspy` reports only the overhead column. Pass `--base-launcher` with another build of `fspy_benchmark_launcher` to compare two revisions by hand. + +One caveat: the launcher is compiled against both revisions from the head's source. A pull request that breaks the small part of the `fspy::Command` API the launcher uses will fail the baseline build. Keep that API surface small; if it must break, expect a red benchmark on that one pull request. From dc57cf22fd2f3e77cf138805028f0f025da4be07 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Tue, 28 Jul 2026 16:32:13 +0800 Subject: [PATCH 35/35] test(fspy): report changes without judging them The benchmark ran on thresholds set from measured noise and failed the job past them. Now that it runs on every pull request, most runs compare two identical fspy builds, where a threshold can only ever produce a false alarm. Print the change and its quartile spread and leave the judgment to the reader; the job stays green. Co-Authored-By: Claude Fable 5 --- crates/fspy_benchmark/README.md | 2 +- crates/fspy_benchmark/src/main.rs | 53 +++++++------------------------ 2 files changed, 13 insertions(+), 42 deletions(-) diff --git a/crates/fspy_benchmark/README.md b/crates/fspy_benchmark/README.md index b4c8fecc3..7d722b214 100644 --- a/crates/fspy_benchmark/README.md +++ b/crates/fspy_benchmark/README.md @@ -23,7 +23,7 @@ So nothing is ever compared across runs. Both fspy revisions run side by side on - `fspy_benchmark_launcher` runs the target once, tracked through `fspy::Command` or untracked, and prints the launch wall clock plus the target's number. It is the only piece that links fspy. - `fspy_benchmark` is the harness. CI builds the launcher twice: against the fspy under review, and against the merge-base fspy. Both builds use the same launcher source, so both revisions are measured by identical code. The harness then launches both builds back to back, cycling every ordering. Whatever the runner does to the numbers, it does to both. -Each iteration gives one head/base ratio per row. The reported change is the median of those ratios. A row that moves past its threshold fails the job. +Each iteration gives one head/base ratio per row. The reported change is the median of those ratios, printed with its quartile spread. The benchmark only reports; it never fails the job. Running the same fspy on both sides stays within a couple of percent, so read a change well past that as real. An untracked launch runs in the same rotation. It prices tracking itself, and is reported as context, never gated. diff --git a/crates/fspy_benchmark/src/main.rs b/crates/fspy_benchmark/src/main.rs index 2e1f45ad5..c4567104a 100644 --- a/crates/fspy_benchmark/src/main.rs +++ b/crates/fspy_benchmark/src/main.rs @@ -51,9 +51,6 @@ struct Suite { /// Unmeasured iterations run first, to fill caches and settle the runner. warmup: usize, metric: Metric, - /// The change past which the suite fails the run, as a share. Set from the - /// spread observed when benchmarking one commit on many runner instances. - threshold: f64, } /// Opens nothing, so the whole launch is the cost of starting a tracked @@ -65,19 +62,12 @@ const LAUNCH_SUITE: Suite = Suite { iterations: if cfg!(windows) { 150 } else { 300 }, warmup: 5, metric: Metric::Wall, - threshold: 0.05, }; /// Opens timed from inside the target, so they price interception rather than /// the launch around it. -const ACCESS_SUITE: Suite = Suite { - name: "access", - opens: "2048", - iterations: 102, - warmup: 3, - metric: Metric::Typical, - threshold: 0.08, -}; +const ACCESS_SUITE: Suite = + Suite { name: "access", opens: "2048", iterations: 102, warmup: 3, metric: Metric::Typical }; struct Backend { name: &'static str, @@ -95,20 +85,15 @@ fn main() { #[cfg(not(all(target_os = "linux", target_arch = "x86_64")))] let backends = [Backend { name: "dynamic", target: DYNAMIC_TARGET }]; - let mut regressed = false; for backend in &backends { validate(HEAD_LAUNCHER.as_ref(), backend.target); if let Some(base_launcher) = &base_launcher { validate(base_launcher, backend.target); } for suite in [&LAUNCH_SUITE, &ACCESS_SUITE] { - regressed |= run_suite(backend, suite, base_launcher.as_deref()); + run_suite(backend, suite, base_launcher.as_deref()); } } - - if regressed { - std::process::exit(1); - } } fn parse_base_launcher() -> Option { @@ -156,9 +141,8 @@ enum Arm { Base, } -/// Runs a suite and reports its row. Returns whether it regressed past its -/// threshold. -fn run_suite(backend: &Backend, suite: &Suite, base_launcher: Option<&OsStr>) -> bool { +/// Runs a suite and reports its row. +fn run_suite(backend: &Backend, suite: &Suite, base_launcher: Option<&OsStr>) { let orders = if base_launcher.is_some() { ORDERS_WITH_BASE } else { ORDERS_WITHOUT_BASE }; // Whole cycles only, so that each ordering runs equally often. let iterations = suite.iterations.next_multiple_of(orders.len()); @@ -184,40 +168,27 @@ fn run_suite(backend: &Backend, suite: &Suite, base_launcher: Option<&OsStr>) -> } } - report(backend, suite, &measured, base_launcher.is_some()) + report(backend, suite, &measured, base_launcher.is_some()); } -/// Prints the suite's row and returns whether it regressed past its threshold. +/// Prints the suite's row. #[expect(clippy::print_stdout, reason = "the report is the benchmark's output")] -fn report(backend: &Backend, suite: &Suite, iterations: &[Iteration], has_base: bool) -> bool { +fn report(backend: &Backend, suite: &Suite, iterations: &[Iteration], has_base: bool) { let name = [backend.name, "/", suite.name].concat(); let overheads = sorted(iterations.iter().map(|it| it.head / it.untracked - 1.0)); let overhead = quantile(&overheads, 1, 2); if has_base { let changes = sorted(iterations.iter().map(|it| it.head / it.base - 1.0)); - let change = quantile(&changes, 1, 2); - let low = quantile(&changes, 1, 4); - let high = quantile(&changes, 3, 4); - let verdict = if change > suite.threshold { - " REGRESSED" - } else if change < -suite.threshold { - " improved" - } else { - "" - }; println!( - "{name:<26} change {:>+6.2}% [{:>+6.2}% .. {:>+6.2}%] threshold {:>4.1}% overhead {:>+8.2}%{verdict}", - change * 100.0, - low * 100.0, - high * 100.0, - suite.threshold * 100.0, + "{name:<26} change {:>+6.2}% [{:>+6.2}% .. {:>+6.2}%] overhead {:>+8.2}%", + quantile(&changes, 1, 2) * 100.0, + quantile(&changes, 1, 4) * 100.0, + quantile(&changes, 3, 4) * 100.0, overhead * 100.0, ); - change > suite.threshold } else { println!("{name:<26} overhead {:>+8.2}%", overhead * 100.0); - false } }