diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 594e60e0..519c9151 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -358,7 +358,9 @@ jobs: - run: cargo autoinherit && git diff --exit-code - run: cargo shear --deny-warnings - run: cargo fmt --check - - run: RUSTDOCFLAGS='-D warnings' cargo doc --no-deps --document-private-items + # --exclude fspy_benchmark: documenting it would build its musl artifact + # dependency, which needs a rust-std this job otherwise has no use for. + - run: RUSTDOCFLAGS='-D warnings' cargo doc --workspace --exclude fspy_benchmark --no-deps --document-private-items - uses: crate-ci/typos@bee27e3a4fd1ea2111cf90ab89cd076c870fce14 # v1.48.0 with: diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml new file mode 100644 index 00000000..dca071f4 --- /dev/null +++ b/.github/workflows/fspy-benchmark.yml @@ -0,0 +1,178 @@ +name: fspy benchmark + +permissions: {} + +on: + workflow_dispatch: + pull_request: + # Runs on main keep the build cache warm and record the overhead history. + push: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.ref_name != 'main' }} + +defaults: + run: + shell: bash + +# Benchmark results are never compared across runs: runner instances disagree +# by more than a regression worth catching. Instead, each pull request job +# builds the benchmark launcher twice — against the fspy under review and +# against the fspy of the merge base — and the harness interleaves both builds +# on the same machine. See crates/fspy_benchmark/README.md. +jobs: + benchmark: + name: Benchmark (${{ matrix.platform }}) + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - platform: linux + os: namespace-profile-linux-x64-default + static: true + - platform: macos + os: namespace-profile-mac-default + static: false + - platform: windows + os: namespace-profile-windows-4c-8g + static: false + runs-on: ${{ matrix.os }} + env: + BASE_DIR: ${{ github.workspace }}/../fspy-benchmark-base + steps: + - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 + + - name: Update Detours submodule + if: matrix.platform == 'windows' + run: git submodule update --init --recursive + + - uses: oxc-project/setup-rust@3d6fb132fbe7cdcb66bf8ec193911c2945369d12 # v1.0.17 + with: + # Caches saved by pull request runs are scoped to their branch, so + # only runs on main seed the cache every other run restores. + save-cache: ${{ github.ref_name == 'main' }} + cache-key: fspy-benchmark-${{ matrix.platform }} + + - name: Install static target + if: matrix.static + run: rustup target add x86_64-unknown-linux-musl + + - name: Check out merge base + if: github.event_name == 'pull_request' + run: | + # The checkout is the pull request merged into the base branch, so + # its first parent is exactly the base tip the merge was built on. + git fetch --depth 2 origin "refs/pull/${{ github.event.pull_request.number }}/merge" + git worktree add "$BASE_DIR" "$(git rev-parse FETCH_HEAD^1)" + + - name: Update baseline Detours submodule + if: github.event_name == 'pull_request' && matrix.platform == 'windows' + working-directory: ${{ env.BASE_DIR }} + run: git submodule update --init --recursive + + - name: Build baseline launcher + if: github.event_name == 'pull_request' + working-directory: ${{ env.BASE_DIR }} + env: + # Share the head build's target directory: the crates.io graph is + # fingerprinted by package and profile, not by workspace, so only the + # fspy path crates build twice. + CARGO_TARGET_DIR: ${{ github.workspace }}/target + run: | + # Both fspy revisions must be measured by identical code, and the + # baseline may predate the launcher crate, so overlay the head's + # launcher source before building it against the baseline fspy. + rm -rf crates/fspy_benchmark_launcher + cp -R "$GITHUB_WORKSPACE/crates/fspy_benchmark_launcher" crates/fspy_benchmark_launcher + cargo build --release -p fspy_benchmark_launcher + # Set the binary aside before the head build overwrites it. + exe="" + [[ "$RUNNER_OS" == "Windows" ]] && exe=.exe + cp "$CARGO_TARGET_DIR/release/fspy_benchmark_launcher$exe" "$RUNNER_TEMP/fspy-base-launcher$exe" + + - name: Run benchmark + run: | + exe="" + [[ "$RUNNER_OS" == "Windows" ]] && exe=.exe + args=() + if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then + args=(--base-launcher "$RUNNER_TEMP/fspy-base-launcher$exe") + fi + mkdir -p .benchmark-results + set -o pipefail + cargo run --release -p fspy_benchmark -- "${args[@]}" | + tee ".benchmark-results/${{ matrix.platform }}.txt" + + - name: Add job summary + if: ${{ !cancelled() }} + run: | + { + echo '```text' + cat ".benchmark-results/${{ matrix.platform }}.txt" || + echo 'no report; the benchmark failed before producing one' + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload benchmark report + # Matrix jobs have isolated filesystems, so persist each report for the + # comment job to combine into one sticky comment after all platforms + # finish. Fork pull requests get no comment, so nothing consumes their + # reports; the job summary already carries them. + if: >- + !cancelled() && + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: fspy-benchmark-report-${{ matrix.platform }} + path: .benchmark-results/${{ matrix.platform }}.txt + # A job that failed before producing a report has nothing to upload; + # the comment job prints its own placeholder for the platform. + if-no-files-found: warn + retention-days: 1 + + comment: + name: Report benchmark + if: >- + !cancelled() && + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + needs: benchmark + runs-on: namespace-profile-linux-x64-default + permissions: + actions: read + pull-requests: write + steps: + - name: Download benchmark reports + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: fspy-benchmark-report-* + path: .benchmark-results + merge-multiple: true + + - name: Assemble comment + run: | + { + echo '## fspy benchmark' + echo + for platform in linux macos windows; do + echo "### $platform" + echo + echo '```text' + cat ".benchmark-results/$platform.txt" || + echo "no report; the $platform benchmark job failed before producing one" + echo '```' + echo + done + } > .benchmark-results/comment.md + + - name: Update PR comment + uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v3.0.4 + with: + header: fspy-benchmark + path: .benchmark-results/comment.md diff --git a/Cargo.lock b/Cargo.lock index fd72b2ce..351ca942 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1239,6 +1239,32 @@ dependencies = [ "winsafe 0.0.27", ] +[[package]] +name = "fspy_benchmark" +version = "0.0.0" +dependencies = [ + "fspy_benchmark_launcher", + "fspy_benchmark_static_target", + "fspy_benchmark_target", +] + +[[package]] +name = "fspy_benchmark_launcher" +version = "0.0.0" +dependencies = [ + "fspy", + "tokio", + "tokio-util", +] + +[[package]] +name = "fspy_benchmark_static_target" +version = "0.0.0" + +[[package]] +name = "fspy_benchmark_target" +version = "0.0.0" + [[package]] name = "fspy_detours_sys" version = "0.0.0" @@ -3251,9 +3277,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap", "itoa", diff --git a/Cargo.toml b/Cargo.toml index 80454123..7e1382b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,8 @@ materialized_artifact = { path = "crates/materialized_artifact" } materialized_artifact_build = { path = "crates/materialized_artifact_build" } flate2 = "1.0.35" fspy = { path = "crates/fspy" } +fspy_benchmark_launcher = { path = "crates/fspy_benchmark_launcher", artifact = "bin" } +fspy_benchmark_target = { path = "crates/fspy_benchmark_target", artifact = "bin" } fspy_detours_sys = { path = "crates/fspy_detours_sys" } fspy_preload_unix = { path = "crates/fspy_preload_unix", artifact = "cdylib", target = "target" } fspy_preload_windows = { path = "crates/fspy_preload_windows", artifact = "cdylib", target = "target" } diff --git a/crates/fspy/Cargo.toml b/crates/fspy/Cargo.toml index 236a0f7b..38b9ffb5 100644 --- a/crates/fspy/Cargo.toml +++ b/crates/fspy/Cargo.toml @@ -18,7 +18,7 @@ ouroboros = { workspace = true } rustc-hash = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = ["net", "process", "io-util", "sync", "rt"] } +tokio = { workspace = true, features = ["macros", "net", "process", "io-util", "sync", "rt"] } tokio-util = { workspace = true } which = { workspace = true, features = ["tracing"] } diff --git a/crates/fspy_benchmark/Cargo.toml b/crates/fspy_benchmark/Cargo.toml new file mode 100644 index 00000000..01e640e6 --- /dev/null +++ b/crates/fspy_benchmark/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "fspy_benchmark" +version = "0.0.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +publish = false +rust-version.workspace = true + +[lints] +workspace = true + +[[bin]] +name = "fspy_benchmark" +test = false +doctest = false + +[dependencies] +fspy_benchmark_launcher = { workspace = true } +fspy_benchmark_target = { workspace = true } + +[target.'cfg(all(target_os = "linux", target_arch = "x86_64"))'.dependencies] +fspy_benchmark_static_target = { path = "../fspy_benchmark_static_target", artifact = "bin", target = "x86_64-unknown-linux-musl" } + +[package.metadata.cargo-shear] +ignored = ["fspy_benchmark_launcher", "fspy_benchmark_target", "fspy_benchmark_static_target"] diff --git a/crates/fspy_benchmark/README.md b/crates/fspy_benchmark/README.md new file mode 100644 index 00000000..7d722b21 --- /dev/null +++ b/crates/fspy_benchmark/README.md @@ -0,0 +1,32 @@ +# fspy benchmark + +Measures what fspy costs a process it tracks, and whether a change to fspy moved that cost. It reports two rows: + +- `launch`: the wall clock of a tracked launch that opens nothing. This is the cost of starting a process under tracking: injection, session setup, and teardown. +- `access`: how long a batch of opens takes, timed by two threads that each open their own path. This is the cost of interception itself, under the concurrency a tracked process normally has. + +Linux measures a dynamically linked target (`LD_PRELOAD`) and a `x86_64-unknown-linux-musl` target (seccomp user notification). macOS measures `DYLD_INSERT_LIBRARIES`. Windows measures Detours injection. + +Run the overhead report locally with: + +```sh +just benchmark-fspy +``` + +## How regressions are caught + +Comparing a pull request run against a saved result from an earlier `main` run does not work. The two runs land on different runner instances, and identical instances disagree by more than any regression worth catching: tens of percent on absolute times, and still up to ten percent after normalizing against an untracked launch. A threshold above that noise is a threshold above every real regression. + +So nothing is ever compared across runs. Both fspy revisions run side by side on one machine, in one job. Three pieces make that possible: + +- `fspy_benchmark_target` (and its static twin) is the workload. It opens missing paths from several threads and prints the median batch time. It does not link fspy. Batches are a few microseconds long, so the median lands on batches the scheduler left alone; a whole-run wall clock would add up every disturbance instead. +- `fspy_benchmark_launcher` runs the target once, tracked through `fspy::Command` or untracked, and prints the launch wall clock plus the target's number. It is the only piece that links fspy. +- `fspy_benchmark` is the harness. CI builds the launcher twice: against the fspy under review, and against the merge-base fspy. Both builds use the same launcher source, so both revisions are measured by identical code. The harness then launches both builds back to back, cycling every ordering. Whatever the runner does to the numbers, it does to both. + +Each iteration gives one head/base ratio per row. The reported change is the median of those ratios, printed with its quartile spread. The benchmark only reports; it never fails the job. Running the same fspy on both sides stays within a couple of percent, so read a change well past that as real. + +An untracked launch runs in the same rotation. It prices tracking itself, and is reported as context, never gated. + +Locally there is no second fspy build, so `just benchmark-fspy` reports only the overhead column. Pass `--base-launcher` with another build of `fspy_benchmark_launcher` to compare two revisions by hand. + +One caveat: the launcher is compiled against both revisions from the head's source. A pull request that breaks the small part of the `fspy::Command` API the launcher uses will fail the baseline build. Keep that API surface small; if it must break, expect a red benchmark on that one pull request. diff --git a/crates/fspy_benchmark/src/main.rs b/crates/fspy_benchmark/src/main.rs new file mode 100644 index 00000000..c4567104 --- /dev/null +++ b/crates/fspy_benchmark/src/main.rs @@ -0,0 +1,245 @@ +//! Compares what two fspy builds cost the processes they track. +//! +//! Every earlier attempt at this benchmark compared a pull request run with a +//! baseline recorded by an earlier run on `main`, and kept fighting the same +//! fact: two runs land on two runner instances, and identically configured +//! instances disagree on every absolute and relative measurement by more than +//! a regression worth catching. This harness never compares across runs. +//! Instead the workflow builds the launcher twice — once against the fspy +//! under review and once against the baseline fspy — and this harness +//! interleaves launches of both builds on the same machine, in the same run, +//! as neighboring processes. Whatever the runner instance is doing to the +//! numbers, it is doing to both sides. +//! +//! Each iteration launches every arm — baseline-tracked, head-tracked, and +//! untracked — back to back, cycling every ordering, and yields the ratio of +//! head to baseline per metric. The reported change is the median of those +//! ratios, so it prices exactly one thing: what the fspy change under review +//! costs a tracked process. The untracked arm prices tracking itself, as +//! context. + +use std::{ + env, + ffi::{OsStr, OsString}, + process::{Command, Stdio}, +}; + +const HEAD_LAUNCHER: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_LAUNCHER"); +const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); + +/// Threads the target runs, passed through to it. Two, because a process that +/// fspy tracks rarely accesses files from one thread. +const THREADS: &str = "2"; + +/// What a suite reads out of its launches. +#[derive(Clone, Copy)] +enum Metric { + /// Wall clock of the launch, from spawn to reap. + Wall, + /// The typical batch of opens, as timed inside the target. + Typical, +} + +struct Suite { + name: &'static str, + /// Opens per target thread, passed through to it. + opens: &'static str, + /// Measured iterations. Each one launches every arm once. + iterations: usize, + /// Unmeasured iterations run first, to fill caches and settle the runner. + warmup: usize, + metric: Metric, +} + +/// Opens nothing, so the whole launch is the cost of starting a tracked +/// process. Launches cost several times more wall clock on Windows, so it +/// affords fewer of them in the same time. +const LAUNCH_SUITE: Suite = Suite { + name: "launch", + opens: "0", + iterations: if cfg!(windows) { 150 } else { 300 }, + warmup: 5, + metric: Metric::Wall, +}; + +/// Opens timed from inside the target, so they price interception rather than +/// the launch around it. +const ACCESS_SUITE: Suite = + Suite { name: "access", opens: "2048", iterations: 102, warmup: 3, metric: Metric::Typical }; + +struct Backend { + name: &'static str, + target: &'static str, +} + +fn main() { + let base_launcher = parse_base_launcher(); + + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + let backends = [ + Backend { name: "dynamic", target: DYNAMIC_TARGET }, + Backend { name: "static", target: STATIC_TARGET }, + ]; + #[cfg(not(all(target_os = "linux", target_arch = "x86_64")))] + let backends = [Backend { name: "dynamic", target: DYNAMIC_TARGET }]; + + for backend in &backends { + validate(HEAD_LAUNCHER.as_ref(), backend.target); + if let Some(base_launcher) = &base_launcher { + validate(base_launcher, backend.target); + } + for suite in [&LAUNCH_SUITE, &ACCESS_SUITE] { + run_suite(backend, suite, base_launcher.as_deref()); + } + } +} + +fn parse_base_launcher() -> Option { + let mut args = env::args_os().skip(1); + let base_launcher = match args.next() { + None => None, + Some(flag) if flag == "--base-launcher" => { + Some(args.next().expect("--base-launcher needs a path")) + } + Some(flag) => { + panic!("unknown argument {}, expected --base-launcher PATH", flag.to_string_lossy()) + } + }; + assert!(args.next().is_none(), "unexpected extra arguments"); + base_launcher +} + +/// One iteration's measurements, one per arm, in nanoseconds. +#[derive(Default)] +struct Iteration { + base: f64, + head: f64, + untracked: f64, +} + +/// Every ordering of the arms of an iteration. Cycling through all of them — +/// rather than rotating one fixed cycle, which keeps every arm's neighbors +/// the same forever — evens out both what position an arm runs in and which +/// arm's leftovers it runs after. +const ORDERS_WITHOUT_BASE: &[&[Arm]] = + &[&[Arm::Head, Arm::Untracked], &[Arm::Untracked, Arm::Head]]; +const ORDERS_WITH_BASE: &[&[Arm]] = &[ + &[Arm::Head, Arm::Untracked, Arm::Base], + &[Arm::Untracked, Arm::Base, Arm::Head], + &[Arm::Base, Arm::Head, Arm::Untracked], + &[Arm::Base, Arm::Untracked, Arm::Head], + &[Arm::Untracked, Arm::Head, Arm::Base], + &[Arm::Head, Arm::Base, Arm::Untracked], +]; + +#[derive(Clone, Copy)] +enum Arm { + Head, + Untracked, + Base, +} + +/// Runs a suite and reports its row. +fn run_suite(backend: &Backend, suite: &Suite, base_launcher: Option<&OsStr>) { + let orders = if base_launcher.is_some() { ORDERS_WITH_BASE } else { ORDERS_WITHOUT_BASE }; + // Whole cycles only, so that each ordering runs equally often. + let iterations = suite.iterations.next_multiple_of(orders.len()); + let mut measured = Vec::with_capacity(iterations); + for index in 0..suite.warmup + iterations { + let mut iteration = Iteration::default(); + for &arm in orders[index % orders.len()] { + match arm { + Arm::Head => { + iteration.head = launch(HEAD_LAUNCHER.as_ref(), None, backend, suite); + } + Arm::Untracked => { + iteration.untracked = + launch(HEAD_LAUNCHER.as_ref(), Some("--untracked"), backend, suite); + } + Arm::Base => { + iteration.base = launch(base_launcher.unwrap(), None, backend, suite); + } + } + } + if index >= suite.warmup { + measured.push(iteration); + } + } + + report(backend, suite, &measured, base_launcher.is_some()); +} + +/// Prints the suite's row. +#[expect(clippy::print_stdout, reason = "the report is the benchmark's output")] +fn report(backend: &Backend, suite: &Suite, iterations: &[Iteration], has_base: bool) { + let name = [backend.name, "/", suite.name].concat(); + let overheads = sorted(iterations.iter().map(|it| it.head / it.untracked - 1.0)); + let overhead = quantile(&overheads, 1, 2); + + if has_base { + let changes = sorted(iterations.iter().map(|it| it.head / it.base - 1.0)); + println!( + "{name:<26} change {:>+6.2}% [{:>+6.2}% .. {:>+6.2}%] overhead {:>+8.2}%", + quantile(&changes, 1, 2) * 100.0, + quantile(&changes, 1, 4) * 100.0, + quantile(&changes, 3, 4) * 100.0, + overhead * 100.0, + ); + } else { + println!("{name:<26} overhead {:>+8.2}%", overhead * 100.0); + } +} + +fn sorted(values: impl Iterator) -> Vec { + let mut values: Vec = values.collect(); + values.sort_unstable_by(f64::total_cmp); + values +} + +fn quantile(sorted: &[f64], numerator: usize, denominator: usize) -> f64 { + sorted[sorted.len() * numerator / denominator] +} + +/// Launches one arm and reads the metric the suite names out of the two +/// numbers its launcher printed. +fn launch(launcher: &OsStr, mode: Option<&str>, backend: &Backend, suite: &Suite) -> f64 { + let mut command = Command::new(launcher); + if let Some(mode) = mode { + command.arg(mode); + } + let output = command + .args([backend.target, THREADS, suite.opens]) + .stdin(Stdio::null()) + .stderr(Stdio::inherit()) + .output() + .expect("failed to run the benchmark launcher"); + assert!(output.status.success(), "benchmark launcher failed: {}", output.status); + let stdout = str::from_utf8(&output.stdout).expect("launcher output is not utf-8"); + let mut numbers = stdout + .split_whitespace() + .map(|number| number.parse().expect("unreadable launcher measurement")); + let mut next = || numbers.next().expect("launcher reported too few measurements"); + let [wall, typical] = [next(), next()]; + match suite.metric { + Metric::Wall => wall, + Metric::Typical => typical, + } +} + +fn validate(launcher: &OsStr, target: &str) { + let status = Command::new(launcher) + .arg("--validate") + .arg(target) + // One thread opening one batch: the count matches the target's + // OPENS_PER_SAMPLE. Fewer would open nothing, which validation + // catches by failing to capture the access. + .args(["1", "8"]) + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .status() + .expect("failed to run the benchmark launcher"); + assert!(status.success(), "launcher validation failed for {}", launcher.to_string_lossy()); +} diff --git a/crates/fspy_benchmark_launcher/Cargo.toml b/crates/fspy_benchmark_launcher/Cargo.toml new file mode 100644 index 00000000..740ad568 --- /dev/null +++ b/crates/fspy_benchmark_launcher/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "fspy_benchmark_launcher" +version = "0.0.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +publish = false +rust-version.workspace = true + +[lints] +workspace = true + +[[bin]] +name = "fspy_benchmark_launcher" +test = false +doctest = false + +[dependencies] +fspy = { workspace = true } +# macros is fspy's, not ours: revisions of fspy that use select! without +# declaring the feature only compile when a dependent enables it, and this +# crate is built against the merge-base fspy on every benchmark run. +tokio = { workspace = true, features = ["io-util", "macros", "process", "rt-multi-thread"] } +tokio-util = { workspace = true } diff --git a/crates/fspy_benchmark_launcher/src/main.rs b/crates/fspy_benchmark_launcher/src/main.rs new file mode 100644 index 00000000..8a0a6c70 --- /dev/null +++ b/crates/fspy_benchmark_launcher/src/main.rs @@ -0,0 +1,142 @@ +//! Launches the benchmark target once and reports what the launch cost. +//! +//! This is the only benchmark piece that links fspy. The harness compares two +//! builds of this binary — one against the fspy under review and one against +//! the baseline fspy — so both revisions must be measured by identical code: +//! the harness always builds this crate from the same source for both. +//! +//! Prints one line to stdout: the wall clock of the launch in nanoseconds, +//! followed by the batch timing the target reported for itself. + +use std::{env, ffi::OsString, process::Stdio, time::Instant}; + +use fspy::Command; +use tokio::{io::AsyncReadExt as _, process::ChildStdout, runtime::Builder}; +use tokio_util::sync::CancellationToken; + +/// The base path the target opens, appended to its arguments so that only this +/// crate names it: the target derives its per-thread paths from it, and +/// validation asserts the bare path was captured. +#[cfg(unix)] +const MISSING_PATH: &str = "/.fspy-benchmark-missing"; +#[cfg(windows)] +const MISSING_PATH: &str = r"C:\.fspy-benchmark-missing"; + +fn main() { + let mut args = env::args_os().skip(1).collect::>(); + let mode = match args.first().map(OsString::as_os_str) { + Some(arg) if arg == "--untracked" => { + args.remove(0); + Mode::Untracked + } + Some(arg) if arg == "--validate" => { + args.remove(0); + Mode::Validate + } + _ => Mode::Tracked, + }; + let (target, target_args) = + args.split_first().expect("usage: fspy_benchmark_launcher [MODE] TARGET ARGS..."); + + let runtime = Builder::new_multi_thread().worker_threads(2).enable_all().build().unwrap(); + runtime.block_on(async { + match mode { + Mode::Tracked => report(run_tracked(target, target_args).await).await, + Mode::Untracked => report(run_untracked(target, target_args).await).await, + Mode::Validate => validate(target, target_args).await, + } + }); +} + +enum Mode { + Tracked, + Untracked, + Validate, +} + +struct Launch { + wall_nanos: u128, + stdout: ChildStdout, +} + +/// Times the launch from just before the spawn to just after the wait, so +/// that a tracked launch covers session setup, injection, and teardown, and +/// nothing of this launcher's own startup. +async fn run_tracked(target: &OsString, target_args: &[OsString]) -> Launch { + let mut command = Command::new(target); + command + .args(target_args) + .arg(MISSING_PATH) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()); + let start = Instant::now(); + let mut child = command + .spawn(CancellationToken::new()) + .await + .expect("failed to spawn tracked benchmark target"); + let stdout = child.stdout.take().expect("tracked benchmark target has no stdout"); + let termination = child.wait_handle.await.expect("failed to wait for tracked target"); + let wall_nanos = start.elapsed().as_nanos(); + assert!( + termination.status.success(), + "tracked benchmark target failed: {}", + termination.status + ); + Launch { wall_nanos, stdout } +} + +async fn run_untracked(target: &OsString, target_args: &[OsString]) -> Launch { + let mut command = tokio::process::Command::new(target); + command + .args(target_args) + .arg(MISSING_PATH) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()); + let start = Instant::now(); + let mut child = command.spawn().expect("failed to spawn untracked benchmark target"); + let stdout = child.stdout.take().expect("untracked benchmark target has no stdout"); + let status = child.wait().await.expect("failed to wait for untracked target"); + let wall_nanos = start.elapsed().as_nanos(); + assert!(status.success(), "untracked benchmark target failed: {status}"); + Launch { wall_nanos, stdout } +} + +/// Relays the sample the target timed for itself after the wall clock, so the +/// harness reads every number from one line. +async fn report(mut launch: Launch) { + let mut sample = Vec::new(); + launch.stdout.read_to_end(&mut sample).await.expect("failed to read the reported sample"); + let sample = str::from_utf8(&sample).expect("the reported sample is not utf-8"); + #[expect(clippy::print_stdout, reason = "the harness reads the measurement from stdout")] + { + println!("{} {}", launch.wall_nanos, sample.trim()); + } +} + +/// Runs the target tracked and asserts that its accesses were captured, so +/// that the harness never benchmarks tracking that silently stopped working. +async fn validate(target: &OsString, target_args: &[OsString]) { + let mut command = Command::new(target); + command + .args(target_args) + .arg(MISSING_PATH) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()); + let termination = command + .spawn(CancellationToken::new()) + .await + .expect("failed to spawn tracked benchmark target") + .wait_handle + .await + .expect("failed to wait for tracked target"); + assert!(termination.status.success(), "benchmark target failed: {}", termination.status); + let captured_missing_access = termination.path_accesses.iter().any(|access| { + access.path.strip_path_prefix(MISSING_PATH, |result| { + result.is_ok_and(|path| path.as_os_str().is_empty()) + }) + }); + assert!(captured_missing_access, "failed to capture access to {MISSING_PATH}"); +} diff --git a/crates/fspy_benchmark_static_target/Cargo.toml b/crates/fspy_benchmark_static_target/Cargo.toml new file mode 100644 index 00000000..d8daccfe --- /dev/null +++ b/crates/fspy_benchmark_static_target/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "fspy_benchmark_static_target" +version = "0.0.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +publish = false +rust-version.workspace = true + +[lints] +workspace = true + +[[bin]] +name = "fspy_benchmark_static_target" +test = false +doctest = false diff --git a/crates/fspy_benchmark_static_target/src/main.rs b/crates/fspy_benchmark_static_target/src/main.rs new file mode 100644 index 00000000..8f25ad74 --- /dev/null +++ b/crates/fspy_benchmark_static_target/src/main.rs @@ -0,0 +1,5 @@ +// A second package over the same source, because cargo names artifact +// dependency environment variables after the package: the harness needs the +// host build and the x86_64-unknown-linux-musl build of the target at once, +// and one package cannot be an artifact dependency for two triples. +include!("../../fspy_benchmark_target/src/main.rs"); diff --git a/crates/fspy_benchmark_target/Cargo.toml b/crates/fspy_benchmark_target/Cargo.toml new file mode 100644 index 00000000..ea5536db --- /dev/null +++ b/crates/fspy_benchmark_target/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "fspy_benchmark_target" +version = "0.0.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +publish = false +rust-version.workspace = true + +[lints] +workspace = true + +[[bin]] +name = "fspy_benchmark_target" +test = false +doctest = false diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs new file mode 100644 index 00000000..a17488de --- /dev/null +++ b/crates/fspy_benchmark_target/src/main.rs @@ -0,0 +1,71 @@ +use std::{ + env, + fs::File, + sync::Barrier, + thread, + time::{Duration, Instant}, +}; + +/// Opens behind one timing sample. Timing every open on its own would leave the +/// sample close to the clock's granularity, which is 42 ns on Apple silicon and +/// 100 ns on Windows, while a batch this small still keeps a sample short +/// enough that the scheduler only ever spoils a few of them. +const OPENS_PER_SAMPLE: usize = 8; + +fn main() { + let mut args = env::args().skip(1); + let mut next = || args.next().expect("usage: fspy_benchmark_target THREADS OPENS PATH"); + let thread_count: usize = next().parse().expect("unreadable thread count"); + let open_count_per_thread: usize = next().parse().expect("unreadable open count"); + let base_path = next(); + + let barrier = &Barrier::new(thread_count); + let mut samples = Vec::with_capacity(thread_count * (open_count_per_thread / OPENS_PER_SAMPLE)); + thread::scope(|scope| { + let workers: Vec<_> = (0..thread_count) + .map(|index| { + // Threads open distinct paths so that they do not contend for + // the same lookup. The first thread opens the bare path, which + // is the one the launcher validates was captured. + let mut path = base_path.clone(); + if index > 0 { + path.push_str(&index.to_string()); + } + scope.spawn(move || time_opens(barrier, &path, open_count_per_thread)) + }) + .collect(); + for worker in workers { + samples.extend(worker.join().expect("benchmark thread panicked")); + } + }); + + report(&mut samples); +} + +/// Times how long batches of opens take, from the thread that runs them, so +/// that the measurement covers the opens themselves rather than the launch +/// around them. +fn time_opens(barrier: &Barrier, path: &str, open_count: usize) -> Vec { + let sample_count = open_count / OPENS_PER_SAMPLE; + let mut samples = Vec::with_capacity(sample_count); + barrier.wait(); + for _ in 0..sample_count { + let started = Instant::now(); + for _ in 0..OPENS_PER_SAMPLE { + drop(File::open(path)); + } + samples.push(started.elapsed()); + } + samples +} + +/// Reports the typical sample. The median discards the batches the scheduler +/// spoiled, which a whole-run wall clock would sum up instead. +fn report(samples: &mut [Duration]) { + samples.sort_unstable(); + let median = samples.get(samples.len() / 2).map_or(0, Duration::as_nanos); + #[expect(clippy::print_stdout, reason = "the benchmark reads the samples from stdout")] + { + println!("{median}"); + } +} diff --git a/justfile b/justfile index f4b8b865..628d8123 100644 --- a/justfile +++ b/justfile @@ -47,6 +47,9 @@ lint-linux: lint-windows: cargo-xwin clippy --workspace --all-targets --all-features --target x86_64-pc-windows-msvc -- --deny warnings +benchmark-fspy: + cargo run --release -p fspy_benchmark + [unix] doc: RUSTDOCFLAGS='-D warnings' cargo doc --no-deps --document-private-items