From 06f2d40a237f3e635c8ea736a01c27e103fcc34b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 04:39:31 +0000 Subject: [PATCH 1/2] Add a `scan` benchmark suite and a CI performance gate New crate crates/socket-patch-bench (publish = false, no new third-party dependencies) that benchmarks `socket-patch scan` end to end: - Synthetic, seed-deterministic projects for every package manager in its native lockfile format and install layout: npm, pnpm (isolated store), yarn classic, yarn berry, bun, vlt, pip requirements, uv, PEP 751 pylock, poetry, pipenv, pdm, bundler, composer, cargo, go, nuget and maven. Per-user caches live under the fixture's own HOME. - A std-only mock of the patch API, proxy and artifact host that answers from the scenario's catalog and counts requests per endpoint. - Each PM runs a hosted scan of a fresh project and a rescan of an already-redirected one; npm also runs --dry-run, the public proxy and 40 ms simulated latency (request concurrency). - Every run is validated (scanned/lockfile-only/patched counts, redirected patches, exact rewritten files that really changed on disk, allowed warnings, no unexpected requests, dry runs write nothing), so a scan that skips work fails instead of looking fast. Runs start from the pristine tree via snapshot diff/restore, with a cleared env. - `compare` interleaves base and head runs on one machine and fails on a wall/CPU slowdown (median paired ratio past 10% with a 95% sign-test interval above 1, confirmed by a second round), peak RSS growth, more API requests, or a head that fails validation. .github/workflows/bench.yml builds the PR's base and head with the new `perf` profile (release codegen, thin LTO) and runs the comparison on every PR touching Rust code; pushes to main are recorded without gating. The `performance-regression-accepted` label turns a regression into a report. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017hKPGJkJfyhsQrzXYf7RJZ --- .github/workflows/bench.yml | 191 ++++ Cargo.lock | 17 + Cargo.toml | 11 + crates/socket-patch-bench/Cargo.toml | 31 + crates/socket-patch-bench/README.md | 138 +++ crates/socket-patch-bench/src/engine.rs | 536 ++++++++++++ crates/socket-patch-bench/src/fixtures/gen.rs | 210 +++++ crates/socket-patch-bench/src/fixtures/mod.rs | 308 +++++++ crates/socket-patch-bench/src/fixtures/npm.rs | 793 +++++++++++++++++ .../socket-patch-bench/src/fixtures/other.rs | 824 ++++++++++++++++++ .../socket-patch-bench/src/fixtures/pypi.rs | 485 +++++++++++ crates/socket-patch-bench/src/main.rs | 623 +++++++++++++ crates/socket-patch-bench/src/mock.rs | 609 +++++++++++++ crates/socket-patch-bench/src/process.rs | 135 +++ crates/socket-patch-bench/src/report.rs | 280 ++++++ crates/socket-patch-bench/src/scenarios.rs | 142 +++ crates/socket-patch-bench/src/stats.rs | 230 +++++ crates/socket-patch-bench/src/tree.rs | 287 ++++++ docs/development.md | 7 +- docs/testing/README.md | 1 + scripts/perf/README.md | 4 + 21 files changed, 5861 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/bench.yml create mode 100644 crates/socket-patch-bench/Cargo.toml create mode 100644 crates/socket-patch-bench/README.md create mode 100644 crates/socket-patch-bench/src/engine.rs create mode 100644 crates/socket-patch-bench/src/fixtures/gen.rs create mode 100644 crates/socket-patch-bench/src/fixtures/mod.rs create mode 100644 crates/socket-patch-bench/src/fixtures/npm.rs create mode 100644 crates/socket-patch-bench/src/fixtures/other.rs create mode 100644 crates/socket-patch-bench/src/fixtures/pypi.rs create mode 100644 crates/socket-patch-bench/src/main.rs create mode 100644 crates/socket-patch-bench/src/mock.rs create mode 100644 crates/socket-patch-bench/src/process.rs create mode 100644 crates/socket-patch-bench/src/report.rs create mode 100644 crates/socket-patch-bench/src/scenarios.rs create mode 100644 crates/socket-patch-bench/src/stats.rs create mode 100644 crates/socket-patch-bench/src/tree.rs diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 000000000..7ae9f211e --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,191 @@ +name: Benchmarks + +# Performance gate for `socket-patch scan` (crates/socket-patch-bench; its +# README has the details). Builds the pull request's base and head with the +# same profile on one runner, times both on a synthetic project per package +# manager against a local patch API (interleaved: base, head, head, base, +# ...), and fails when the head +# - is significantly slower (wall or CPU time; median of the paired +# head/base ratios past the threshold with its 95% interval above 1, +# confirmed by a second round of pairs), +# - uses significantly more memory, +# - makes more API requests, or +# - no longer does a scenario's work (scan counts, redirected patches, +# rewritten files). +# The job summary carries the full table; results.json is uploaded. +# +# An intentional slowdown: label the PR `performance-regression-accepted`. +# The comparison still runs and reports, but a regression no longer fails +# the job (a head that fails a scenario's validation still does). +# +# On main, each push is compared against the commit before it and recorded +# (an artifact per push) without failing on regressions. + +on: + pull_request: + types: [opened, synchronize, reopened, labeled, unlabeled] + paths: + - '.github/workflows/bench.yml' + - '.cargo/**' + - 'Cargo.lock' + - 'Cargo.toml' + - 'rust-toolchain.toml' + - 'crates/socket-patch-bench/**' + - 'crates/socket-patch-cli/Cargo.toml' + - 'crates/socket-patch-cli/build.rs' + - 'crates/socket-patch-cli/src/**' + - 'crates/socket-patch-core/Cargo.toml' + - 'crates/socket-patch-core/src/**' + push: + branches: [main] + paths: + - '.github/workflows/bench.yml' + - '.cargo/**' + - 'Cargo.lock' + - 'Cargo.toml' + - 'rust-toolchain.toml' + - 'crates/socket-patch-bench/**' + - 'crates/socket-patch-cli/Cargo.toml' + - 'crates/socket-patch-cli/build.rs' + - 'crates/socket-patch-cli/src/**' + - 'crates/socket-patch-core/Cargo.toml' + - 'crates/socket-patch-core/src/**' + workflow_dispatch: + inputs: + base: + description: 'Commit or ref to compare HEAD against (default: the parent commit)' + required: false + default: '' + filter: + description: 'Only scenarios matching this regex (see `socket-patch-bench list`)' + required: false + default: '' + +permissions: + contents: read + +jobs: + scan: + name: scan performance + # Labels re-trigger the workflow; only this one changes the outcome. + if: >- + (github.event.action != 'labeled' && github.event.action != 'unlabeled') + || github.event.label.name == 'performance-regression-accepted' + # Job-level, so an unrelated label (skipped above) cannot cancel a run + # in progress. A newer push to the same PR supersedes the older run. + concurrency: + group: bench-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + # Both sides build with these, so a base that predates the `perf` + # profile (or carries an older copy of it) builds the same way. + CARGO_PROFILE_PERF_INHERITS: release + CARGO_PROFILE_PERF_LTO: thin + CARGO_PROFILE_PERF_STRIP: none + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + # The parent is the baseline: for a pull request, the merge + # commit's first parent is exactly the base it was merged onto. + fetch-depth: 2 + + - name: Install Rust + # rustup is pre-installed; `rustup show` installs the + # rust-toolchain.toml channel. + run: rustup show + + - name: Cache cargo + # Swatinem/rust-cache, main-only saves: see ci.yml. + uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Resolve the baseline commit + id: base + env: + EVENT_NAME: ${{ github.event_name }} + PUSH_BEFORE: ${{ github.event.before }} + INPUT_BASE: ${{ inputs.base }} + run: | + set -euo pipefail + case "$EVENT_NAME" in + push) ref="$PUSH_BEFORE" ;; + workflow_dispatch) ref="${INPUT_BASE:-HEAD^1}" ;; + *) ref="HEAD^1" ;; + esac + if ! git rev-parse --verify --quiet "$ref^{commit}" >/dev/null; then + git fetch --quiet --depth 1 origin "$ref" + ref=FETCH_HEAD + fi + sha="$(git rev-parse "$ref^{commit}")" + echo "Baseline: $sha" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + + - name: Build head + run: | + set -euo pipefail + BENCH="$RUNNER_TEMP/bench" + cargo build --locked --profile perf -p socket-patch-cli -p socket-patch-bench + mkdir -p "$BENCH/head" + cp target/perf/socket-patch "$BENCH/head/" + cp target/perf/socket-patch-bench "$BENCH/" + + - name: Build base + env: + BASE_SHA: ${{ steps.base.outputs.sha }} + run: | + set -euo pipefail + BENCH="$RUNNER_TEMP/bench" + git worktree add --detach "$BENCH/base-src" "$BASE_SHA" + # Same target directory: the dependencies are already built, only + # the workspace crates compile again (unless the base pins another + # toolchain, which `rustup show` installs first). + (cd "$BENCH/base-src" && rustup show >/dev/null && \ + CARGO_TARGET_DIR="$GITHUB_WORKSPACE/target" \ + cargo build --locked --profile perf -p socket-patch-cli) + mkdir -p "$BENCH/base" + cp target/perf/socket-patch "$BENCH/base/" + "$BENCH/base/socket-patch" --version + "$BENCH/head/socket-patch" --version + + - name: Compare + env: + EVENT_NAME: ${{ github.event_name }} + ACCEPTED: ${{ contains(github.event.pull_request.labels.*.name, 'performance-regression-accepted') }} + FILTER: ${{ inputs.filter }} + run: | + set -euo pipefail + BENCH="$RUNNER_TEMP/bench" + args=() + if [ "$EVENT_NAME" != pull_request ] || [ "$ACCEPTED" = true ]; then + args+=(--no-fail) + fi + if [ -n "${FILTER:-}" ]; then + args+=(--filter "$FILTER") + fi + "$BENCH/socket-patch-bench" compare \ + --base "$BENCH/base/socket-patch" \ + --head "$BENCH/head/socket-patch" \ + --work-dir "$BENCH/work" \ + --out "$BENCH/out" \ + ${args[@]+"${args[@]}"} + + - name: Job summary + if: ${{ !cancelled() }} + run: | + if [ -f "$RUNNER_TEMP/bench/out/summary.md" ]; then + cat "$RUNNER_TEMP/bench/out/summary.md" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload results + if: ${{ !cancelled() }} + uses: ./.github/actions/upload-artifact + with: + name: bench-scan + path: ${{ runner.temp }}/bench/out/ + if-no-files-found: warn + retention-days: 90 diff --git a/Cargo.lock b/Cargo.lock index efa5fd8b9..729f03296 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1875,6 +1875,23 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socket-patch-bench" +version = "4.0.0" +dependencies = [ + "base64", + "clap", + "hex", + "libc", + "regex", + "serde", + "serde_json", + "sha1", + "sha2", + "tempfile", + "zip", +] + [[package]] name = "socket-patch-cli" version = "4.0.0" diff --git a/Cargo.toml b/Cargo.toml index 2d241db05..045042fa4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "crates/socket-patch-core", "crates/socket-patch-cli", "crates/socket-patch-node", + "crates/socket-patch-bench", ] # Bare `cargo build` (release.yml's per-target builds) skips the Node addon, # which no release artifact ships; `--workspace` and `-p` still build it. @@ -78,6 +79,16 @@ inherits = "release" lto = false strip = "none" # unstripped test binaries => usable backtraces on failure +# The benchmark gate's build (crates/socket-patch-bench, .github/workflows/ +# bench.yml): the shipped profile's codegen (opt-level = "s", no debug +# assertions) so timings track what users run, with thin instead of full +# LTO so CI can afford to build both sides of a comparison. Unstripped, so +# a profiler can symbolize it. +[profile.perf] +inherits = "release" +lto = "thin" +strip = "none" + # The Node addon (crates/socket-patch-node) for hosts that load it # in-process: release semantics without the slow full-LTO link. [profile.addon] diff --git a/crates/socket-patch-bench/Cargo.toml b/crates/socket-patch-bench/Cargo.toml new file mode 100644 index 000000000..f2104d648 --- /dev/null +++ b/crates/socket-patch-bench/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "socket-patch-bench" +description = "Benchmark harness for `socket-patch scan`: synthetic projects, a local patch API, and A/B regression checks" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[[bin]] +name = "socket-patch-bench" +path = "src/main.rs" + +# Only crates the workspace already pins: the harness adds nothing to +# Cargo.lock that the CLI's own tests do not already pull in. +[dependencies] +clap = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +sha1 = { workspace = true } +hex = { workspace = true } +base64 = { workspace = true } +tempfile = { workspace = true } +regex = { workspace = true } +# uv's hosted rewrite reads the patched wheel's METADATA, so the mock +# serves real (tiny) wheels. +zip = { workspace = true } + +[target.'cfg(unix)'.dependencies] +libc = { workspace = true } diff --git a/crates/socket-patch-bench/README.md b/crates/socket-patch-bench/README.md new file mode 100644 index 000000000..083f5ff75 --- /dev/null +++ b/crates/socket-patch-bench/README.md @@ -0,0 +1,138 @@ +# socket-patch-bench + +Benchmarks for `socket-patch scan`, and the CI gate that keeps it from +getting slower. The harness generates a synthetic project per package +manager, serves the patch API from a local mock, runs the real +`socket-patch` binary on it, and checks every run did the work the project +calls for before keeping its timing. + +```sh +# Build the binary the way CI does (release semantics, thin LTO). +cargo build --locked --profile perf -p socket-patch-cli -p socket-patch-bench + +target/perf/socket-patch-bench list +target/perf/socket-patch-bench run --bin target/perf/socket-patch +target/perf/socket-patch-bench run --bin target/perf/socket-patch -f '^npm/' -f '^uv/' -v + +# A/B: interleaved base/head runs on this machine, same verdict as CI. +target/perf/socket-patch-bench compare --base /tmp/base/socket-patch --head target/perf/socket-patch +``` + +`run` and `compare` print a markdown table; `--out DIR` also writes +`results.json` (every sample, request counts and verdicts) and `summary.md`. + +## What is measured + +Each scenario is one project and one `scan` invocation: + +| scenario | what runs | +|---|---| +| `/hosted` | `socket-patch scan --json` (the default hosted mode) on a freshly installed project: crawl, lockfile inventory, hosted-pin discovery, batch query, per-package details, reference resolution, artifact checks where the ecosystem needs them, the rewrite, patch views and the writes | +| `/rescan` | the same scan on a project a previous scan already redirected — the steady state of a CI job that scans every build: pin discovery and update detection over rewritten lockfiles, and a no-op redirect | +| `npm/dry-run` | `scan --dry-run`: everything up to the rewrite, no views, no writes | +| `npm/public-proxy` | no API token: the public proxy's routes, batch size and concurrency cap | +| `npm/latency` | 40 ms per API request: request concurrency, not local work, decides the time | + +Package managers (``), each in its native lockfile format and install +layout: `npm`, `pnpm` (isolated `.pnpm` store with symlinks), `yarn-classic`, +`yarn-berry` (node-modules linker), `bun` (text `bun.lock`), `vlt` +(`.vlt` store), `pip` (hash-pinned `requirements.txt`), `uv`, `pylock` +(PEP 751), `poetry`, `pipenv`, `pdm`, `bundler`, `composer`, `cargo`, +`golang`, `nuget` and `maven`. Deno has no hosted rewrite and is not +benchmarked separately. + +Sizes are a large-but-ordinary project for the ecosystem (3000 npm-family +packages, 1500 for vlt, 400-1200 for the others, so every scan takes about +70 ms or more; `--scale` multiplies them), with 2-3% of packages patched. The graphs are generated from a fixed seed: +every run, on every machine, scans byte-identical projects. + +Per run the harness records wall time (spawn to exit), user+system CPU time +and peak RSS (from `wait4`, for exactly that child), and the mock's request +count per endpoint. + +### Making sure a timing means something + +A fast run that skipped work is a bug, not a speedup, so every run is +validated and an invalid run fails the scenario instead of contributing a +sample: + +- exit code 0 and one JSON document on stdout, `status: success`; +- `scannedPackages`, `lockfileOnlyPackages`, `packagesWithPatches` and + `totalPatches` equal what the generated project contains; +- hosted runs: `redirect.redirected` is every patch, `rewrittenFiles` is + exactly the expected set, nothing is skipped, every warning code is one + the scenario expects, and each reported file really changed on disk; +- dry runs: nothing on disk changed; +- rescans: nothing is rewritten and every patch is classified `already`; +- the CLI made no request the mock does not serve, and the same requests + on every run. + +Each run starts from the pristine project: wet runs rewrite lockfiles, so +the harness diffs the tree against a snapshot after every run and restores +whatever changed (cheap, and it also catches a run that writes somewhere +unexpected). A rescan's first, preparing scan is untimed. + +The environment is rebuilt from nothing for every run (`env -i`): `HOME`, +`XDG_*` and `TMPDIR` point into the fixture, so per-user caches +(`~/.cargo`, `~/go/pkg/mod`, `~/.nuget/packages`, `~/.m2`) are the +fixture's own and the runner's are never read; telemetry, the update check +and the persisted Socket login are off; and every proxy variable points at a +closed port, so a request to anything but the mock fails the run instead of +timing the internet. Python fixtures carry a project `.venv` and Ruby ones +`vendor/bundle`, as installed projects do — without them the crawlers would +spawn `python3` / `gem env` and read the machine's global packages. The +`pipenv` scenario sets `SOCKET_PIPENV_MAJOR` so the CLI does not spawn +`pipenv --version`. `strace -f -e trace=execve` on any scenario shows the +CLI spawns nothing. + +## The CI gate + +`.github/workflows/bench.yml` runs `compare` on every pull request that +touches Rust code. It builds the PR's base commit and the PR head with the +same profile, on the same runner, and interleaves their runs (base, head, +head, base, ...): runner-to-runner variance is larger than most regressions, +so only same-machine pairs are compared. + +A scenario fails the gate when any of these hold: + +- **wall or CPU time**: the median of the per-pair `head / base` ratios is + above `1 + --threshold` (default 10%) *and* its 95% sign-test confidence + interval lies entirely above 1 *and* the medians differ by at least 3 ms. + A scenario that first looks regressed is re-run with `--confirm-runs` + more pairs and judged on all of them, so one noisy burst cannot fail a PR; +- **peak RSS**: the same test at `--rss-threshold` (default 15%); +- **API requests**: head makes more requests than base (deterministic, so + any increase counts); +- **validation**: the head binary fails a scenario's checks. (A base that + fails one only loses that scenario's comparison — e.g. a PR that changes + the JSON output and updates the expectations here in the same PR.) + +The job summary has the full table; `results.json` is uploaded as an +artifact. A PR whose slowdown is intentional can carry the +`performance-regression-accepted` label: the comparison still runs and is +reported, but does not fail. + +On pushes to `main` the same comparison runs against the previous commit +and is recorded as an artifact without gating. + +## Profiling one scenario + +`serve` builds a scenario's project, starts its mock API and prints the exact +command line a run uses, then waits: + +```sh +target/perf/socket-patch-bench serve poetry/hosted --bin target/perf/socket-patch +# paste the printed `env -i ...` command under perf / samply / strace +``` + +A wet scan edits the project in place; rerun `serve` for a fresh copy. + +## Adding a scenario + +Fixtures live in `src/fixtures/` (`npm.rs` for the npm family, `pypi.rs`, +`other.rs`) and are registered in `fixtures::ALL`. A generator writes the +project under `project/` (and any per-user cache under `home/`), and returns +the patches the mock serves and an `Expect` describing what a correct scan +reports. Run it with `-v` until it validates; the error names the first +mismatch. The record/replay harness in `scripts/perf/` complements this for +measuring against real API traffic. diff --git a/crates/socket-patch-bench/src/engine.rs b/crates/socket-patch-bench/src/engine.rs new file mode 100644 index 000000000..7d8d3885e --- /dev/null +++ b/crates/socket-patch-bench/src/engine.rs @@ -0,0 +1,536 @@ +//! Run scenarios: build each fixture once, start its mock API, then time +//! the binaries on it with every run starting from the pristine tree. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use serde_json::Value; + +use crate::fixtures::{Fixture, PATCH_HOST}; +use crate::mock::{Catalog, MockApi, Stats}; +use crate::report::{BinaryResult, Sample, ScenarioResult}; +use crate::scenarios::{Auth, Kind, Scenario}; +use crate::tree::{self, Snapshot}; + +/// A binary under test. +#[derive(Debug, Clone)] +pub struct Binary { + pub label: String, + pub path: PathBuf, +} + +#[derive(Debug, Clone)] +pub struct Options { + pub warmup: usize, + /// Measured runs per binary. + pub runs: usize, + pub scale: f64, + pub work: PathBuf, + /// Log every run. + pub verbose: bool, +} + +/// One prepared scenario: fixture on disk, mock running. +pub struct Prepared<'a> { + pub scenario: &'a Scenario, + pub fixture: Fixture, + root: PathBuf, + pristine: PathBuf, + work: PathBuf, + snapshot: Snapshot, + mock: MockApi, + capture: PathBuf, +} + +pub const TOKEN: &str = "sktsec_benchbenchbenchbenchbenchbenchbenchbenchbenc_api"; + +impl<'a> Prepared<'a> { + pub fn new(scenario: &'a Scenario, opts: &Options) -> Result { + let root = opts.work.join(scenario.slug()); + if root.exists() { + std::fs::remove_dir_all(&root) + .map_err(|e| format!("clearing {}: {e}", root.display()))?; + } + let pristine = root.join("pristine"); + let work = root.join("work"); + let capture = root.join("capture"); + let tmp = root.join("tmp"); + for d in [&capture, &tmp] { + std::fs::create_dir_all(d).map_err(|e| e.to_string())?; + } + let size = scenario.size(opts.scale); + let mut t = crate::fixtures::gen::Tree::new(&pristine).map_err(|e| e.to_string())?; + let mut fixture = (scenario.pm.build)(&mut t, size) + .map_err(|e| format!("building the {} fixture: {e}", scenario.pm.name))?; + // A checkout, like every real project: the repository root (and so + // the socket.yml lookup) is the project itself. + t.mkdir(&format!("{}/.git", fixture.project)) + .map_err(|e| e.to_string())?; + + // Artifact URLs embed the mock's address, known only once it + // listens: rewrite the placeholder in the fixture and the catalog. + let mock = + MockApi::start(scenario.latency).map_err(|e| format!("starting the mock API: {e}"))?; + let host = mock.patch.uri(); + substitute(&pristine, PATCH_HOST, &host).map_err(|e| e.to_string())?; + for p in &mut fixture.patches { + replace_in_json(&mut p.reference, PATCH_HOST, &host); + replace_in_json(&mut p.view, PATCH_HOST, &host); + } + let mut catalog = Catalog::new(&fixture.patches, scenario.auth == Auth::Token); + for (path, body, ty) in &fixture.files { + catalog.serve_file(path, body.clone(), ty); + } + mock.set_catalog(catalog); + + tree::copy_tree(&pristine, &work).map_err(|e| e.to_string())?; + let snapshot = Snapshot::take(&work).map_err(|e| e.to_string())?; + Ok(Self { + scenario, + fixture, + root, + pristine, + work, + snapshot, + mock, + capture, + }) + } + + /// Stop the mock and delete the scenario's directory. + pub fn discard(self) { + let root = self.root.clone(); + drop(self); + let _ = std::fs::remove_dir_all(root); + } + + pub fn project_dir(&self) -> PathBuf { + self.work.join(self.fixture.project) + } + + /// The exact command a run executes (also printed by `serve`). + pub fn command(&self, bin: &Path, dry_run: bool) -> Command { + let mut cmd = Command::new(bin); + let project = self.project_dir(); + cmd.current_dir(&project); + cmd.env_clear(); + for (k, v) in self.env() { + cmd.env(k, v); + } + cmd.args(self.args(dry_run)); + cmd.arg("--cwd").arg(&project); + cmd + } + + pub fn args(&self, dry_run: bool) -> Vec { + let mut args: Vec = vec!["scan".into(), "--json".into()]; + if dry_run { + args.push("--dry-run".into()); + } + args.extend(self.scenario.extra_args.iter().map(|s| s.to_string())); + args + } + + pub fn env(&self) -> Vec<(String, String)> { + let home = self.work.join("home"); + let tmp = self.root.join("tmp"); + let mut env: Vec<(String, String)> = vec![ + // Only the variables below: nothing from the runner's + // environment (a stray SOCKET_*, npm_config_*, VIRTUAL_ENV, + // GOFLAGS, ...) can change what the CLI does. + ("PATH".into(), std::env::var("PATH").unwrap_or_default()), + ("HOME".into(), home.display().to_string()), + ("USERPROFILE".into(), home.display().to_string()), + ( + "XDG_CONFIG_HOME".into(), + home.join(".config").display().to_string(), + ), + ( + "XDG_CACHE_HOME".into(), + home.join(".cache").display().to_string(), + ), + ( + "XDG_DATA_HOME".into(), + home.join(".local/share").display().to_string(), + ), + ("TMPDIR".into(), tmp.display().to_string()), + // The policy lookup never walks out of the fixture (into a + // checkout the work dir happens to sit in). + ( + "GIT_CEILING_DIRECTORIES".into(), + self.work.display().to_string(), + ), + ("LANG".into(), "C.UTF-8".into()), + ("SOCKET_API_URL".into(), self.mock.api.uri()), + ("SOCKET_PROXY_URL".into(), self.mock.proxy.uri()), + ("SOCKET_PATCH_SERVER_URL".into(), self.mock.patch.uri()), + ("SOCKET_NO_CONFIG".into(), "1".into()), + ("SOCKET_NO_UPDATE_CHECK".into(), "1".into()), + ("SOCKET_TELEMETRY_DISABLED".into(), "1".into()), + // Anything that is not the mock goes to a closed port, so a + // run that reaches for the real network fails (and fails + // validation) instead of timing the internet. + ("HTTP_PROXY".into(), "http://127.0.0.1:9".into()), + ("HTTPS_PROXY".into(), "http://127.0.0.1:9".into()), + ("ALL_PROXY".into(), "http://127.0.0.1:9".into()), + ("NO_PROXY".into(), "127.0.0.1,localhost".into()), + ]; + if let Auth::Token = self.scenario.auth { + env.push(("SOCKET_API_TOKEN".into(), TOKEN.into())); + env.push(("SOCKET_ORG_SLUG".into(), crate::ORG.into())); + } + for (k, rel) in &self.fixture.env_paths { + env.push((k.to_string(), self.work.join(rel).display().to_string())); + } + for (k, v) in &self.fixture.env { + env.push((k.to_string(), v.clone())); + } + env + } + + /// One measured run of `bin`, from the pristine tree. Untimed + /// preparation (a rescan's first scan) and the restore afterwards are + /// outside the measurement. + pub fn run_once(&mut self, bin: &Path) -> Result<(Sample, Stats, Vec), String> { + if self.scenario.kind == Kind::Rescan { + let prep = crate::process::run(self.command(bin, false), &self.capture) + .map_err(|e| format!("spawning {}: {e}", bin.display()))?; + let stats = self.mock.take_stats(); + validate(self, &prep, &stats, Kind::Hosted) + .map_err(|e| format!("the rescan's preparatory scan failed validation: {e}"))?; + } + self.mock.take_stats(); + let dry = self.scenario.kind == Kind::DryRun; + let outcome = crate::process::run(self.command(bin, dry), &self.capture) + .map_err(|e| format!("spawning {}: {e}", bin.display()))?; + let stats = self.mock.take_stats(); + let checked = validate(self, &outcome, &stats, self.scenario.kind); + let drift = tree::restore(&self.work, &self.pristine, &mut self.snapshot) + .map_err(|e| format!("restoring the fixture: {e}"))?; + let codes = checked?; + check_drift(self, &drift)?; + let u = outcome.usage; + Ok(( + Sample { + wall_ms: u.wall.as_secs_f64() * 1000.0, + cpu_ms: u.cpu.map(|c| c.as_secs_f64() * 1000.0), + max_rss_kib: u.max_rss_kib, + }, + stats, + codes, + )) + } +} + +/// Rewrite `from` → `to` in every regular file under `root`. +fn substitute(root: &Path, from: &str, to: &str) -> std::io::Result<()> { + for entry in std::fs::read_dir(root)? { + let entry = entry?; + let ty = entry.file_type()?; + if ty.is_dir() { + substitute(&entry.path(), from, to)?; + } else if ty.is_file() { + let bytes = std::fs::read(entry.path())?; + if let Ok(s) = std::str::from_utf8(&bytes) { + if s.contains(from) { + std::fs::write(entry.path(), s.replace(from, to))?; + } + } + } + } + Ok(()) +} + +fn replace_in_json(v: &mut Value, from: &str, to: &str) { + match v { + Value::String(s) if s.contains(from) => *s = s.replace(from, to), + Value::Array(a) => a.iter_mut().for_each(|x| replace_in_json(x, from, to)), + Value::Object(o) => o.values_mut().for_each(|x| replace_in_json(x, from, to)), + _ => {} + } +} + +fn warning_codes(v: &Value) -> Vec { + v.as_array() + .map(|a| { + a.iter() + .filter_map(|w| w["code"].as_str().map(str::to_string)) + .collect() + }) + .unwrap_or_default() +} + +/// Check that a run did the work the fixture calls for. Any mismatch makes +/// the sample meaningless, so it is an error, not a footnote. +fn validate( + p: &Prepared<'_>, + o: &crate::process::Outcome, + stats: &Stats, + kind: Kind, +) -> Result, String> { + let stderr = String::from_utf8_lossy(&o.stderr); + let tail = |s: &str| -> String { + let lines: Vec<&str> = s.lines().collect(); + lines[lines.len().saturating_sub(15)..].join("\n") + }; + if !stats.unexpected.is_empty() { + let shown: Vec<&String> = stats.unexpected.iter().take(3).collect(); + return Err(format!( + "the CLI made {} request(s) the mock does not serve, e.g. {shown:?}", + stats.unexpected.len() + )); + } + if o.code != Some(0) { + return Err(format!( + "exit code {:?}\nstderr:\n{}\nstdout:\n{}", + o.code, + tail(&stderr), + tail(&String::from_utf8_lossy(&o.stdout)) + )); + } + let v: Value = serde_json::from_slice(&o.stdout).map_err(|e| { + format!( + "stdout is not one JSON document ({e})\nstderr:\n{}", + tail(&stderr) + ) + })?; + let e = &p.fixture.expect; + let want_patched: std::collections::BTreeSet<&str> = + p.fixture.patches.iter().map(|x| x.purl.as_str()).collect(); + let (scanned, lockfile_only) = if kind == Kind::Rescan { + (e.scanned + e.rescan_extra_scanned, e.rescan_lockfile_only) + } else { + (e.scanned, e.lockfile_only) + }; + let checks: [(&str, Value, Value); 5] = [ + ("status", v["status"].clone(), "success".into()), + ( + "scannedPackages", + v["scannedPackages"].clone(), + scanned.into(), + ), + ( + "lockfileOnlyPackages", + v["lockfileOnlyPackages"].clone(), + lockfile_only.into(), + ), + ( + "packagesWithPatches", + v["packagesWithPatches"].clone(), + want_patched.len().into(), + ), + ( + "totalPatches", + v["totalPatches"].clone(), + p.fixture.patches.len().into(), + ), + ]; + for (field, got, want) in checks { + if got != want { + return Err(format!("{field}: got {got}, want {want}")); + } + } + let mut codes = warning_codes(&v["warnings"]); + let redirect = &v["redirect"]; + codes.extend(warning_codes(&redirect["warnings"])); + let unexpected: Vec<&String> = codes + .iter() + .filter(|c| !e.allowed_warnings.contains(&c.as_str())) + .collect(); + if !unexpected.is_empty() { + return Err(format!( + "unexpected warnings {unexpected:?}: {}", + serde_json::to_string(&v["warnings"]).unwrap_or_default() + + &serde_json::to_string(&redirect["warnings"]).unwrap_or_default() + )); + } + let skipped = redirect["skipped"].as_array().map(Vec::len).unwrap_or(0); + if skipped > 0 { + return Err(format!( + "redirect skipped packages: {}", + redirect["skipped"] + )); + } + let rewritten: Vec = { + let mut r: Vec = redirect["rewrittenFiles"] + .as_array() + .map(|a| { + a.iter() + .filter_map(|f| f.as_str().map(str::to_string)) + .collect() + }) + .unwrap_or_default(); + r.sort(); + r + }; + let mut want_rewritten = e.rewritten.clone(); + want_rewritten.sort(); + match kind { + Kind::Hosted | Kind::DryRun => { + if redirect["redirected"] != e.redirected { + return Err(format!( + "redirect.redirected: got {}, want {}", + redirect["redirected"], e.redirected + )); + } + if rewritten != want_rewritten { + return Err(format!( + "redirect.rewrittenFiles: got {rewritten:?}, want {want_rewritten:?}" + )); + } + } + Kind::Rescan => { + // Everything is already pinned: a second scan finds the same + // patches and has nothing left to rewrite. + if !rewritten.is_empty() { + return Err(format!("a rescan rewrote {rewritten:?}")); + } + let already = &v["rollout"]["counts"]["already"]; + if *already != want_patched.len() { + return Err(format!( + "rollout.counts.already: got {already}, want {}", + want_patched.len() + )); + } + } + } + codes.sort(); + codes.dedup(); + Ok(codes) +} + +/// A dry run must not write; a hosted run must write exactly where it says. +fn check_drift(p: &Prepared<'_>, drift: &tree::Drift) -> Result<(), String> { + let project = Path::new(p.fixture.project); + let touched = drift.touched(); + match p.scenario.kind { + Kind::DryRun if !drift.is_empty() => Err(format!("a --dry-run changed {touched:?}")), + Kind::Hosted => { + for f in &p.fixture.expect.rewritten { + let rel = project.join(f); + if !touched.contains(&rel) { + return Err(format!( + "{} was reported rewritten but is unchanged", + rel.display() + )); + } + } + Ok(()) + } + _ => Ok(()), + } +} + +/// Run every binary on one scenario, interleaved, and collect results. +pub fn run_scenario( + scenario: &Scenario, + bins: &[Binary], + opts: &Options, + log: &mut dyn FnMut(&str), +) -> Result { + let setup = std::time::Instant::now(); + let mut p = Prepared::new(scenario, opts)?; + log(&format!( + "{}: {} packages, {} patched (fixture built in {:.1}s)", + scenario.name, + p.fixture.expect.scanned, + p.fixture.patches.len(), + setup.elapsed().as_secs_f64() + )); + let mut results: BTreeMap = bins + .iter() + .map(|b| (b.label.clone(), BinaryResult::default())) + .collect(); + + for _ in 0..opts.warmup { + for b in bins { + let r = results.get_mut(&b.label).unwrap(); + if r.invalid.is_some() { + continue; + } + if let Err(e) = p.run_once(&b.path) { + r.invalid = Some(e); + } + } + } + for i in 0..opts.runs { + // Alternate the order (ABBA...) so slow drift of the machine does + // not always favor whichever binary runs first. + let order: Vec<&Binary> = if i % 2 == 0 { + bins.iter().collect() + } else { + bins.iter().rev().collect() + }; + for b in order { + let r = results.get_mut(&b.label).unwrap(); + if r.invalid.is_some() { + continue; + } + match p.run_once(&b.path) { + Ok((sample, stats, codes)) => { + if r.samples.is_empty() { + r.requests = stats.by_kind.clone(); + r.max_inflight = stats.max_inflight; + if opts.verbose { + log(&format!( + " {:<5} requests {:?}, warnings {codes:?}", + b.label, stats.by_kind + )); + } + } else if r.requests != stats.by_kind { + r.invalid = Some(format!( + "request counts changed between runs: {:?} then {:?}", + r.requests, stats.by_kind + )); + continue; + } + if opts.verbose { + log(&format!( + " {:<5} run {:>2}: {:8.1} ms wall, {:8.1} ms cpu, {} requests", + b.label, + r.samples.len() + 1, + sample.wall_ms, + sample.cpu_ms.unwrap_or(f64::NAN), + stats.total() + )); + } + r.samples.push(sample); + } + Err(e) => r.invalid = Some(e), + } + } + } + let mut keep = false; + for (label, r) in &results { + if let Some(why) = &r.invalid { + log(&format!( + " {label}: INVALID: {}", + why.lines().next().unwrap_or("") + )); + keep = true; + } + } + let args = p.args(scenario.kind == Kind::DryRun); + let (packages, patched) = (p.fixture.expect.scanned, p.fixture.patches.len()); + // A finished scenario's tree is ~10^4 files; only a failed one is worth + // keeping (to rerun the logged command by hand). + if keep { + log(&format!( + " fixture kept for inspection: {}", + p.project_dir().display() + )); + } else { + p.discard(); + } + Ok(ScenarioResult { + name: scenario.name.clone(), + description: scenario.description(), + packages, + patched, + args, + latency_ms: scenario.latency.as_millis() as u64, + binaries: results, + comparison: None, + }) +} diff --git a/crates/socket-patch-bench/src/fixtures/gen.rs b/crates/socket-patch-bench/src/fixtures/gen.rs new file mode 100644 index 000000000..ab3f2c5b4 --- /dev/null +++ b/crates/socket-patch-bench/src/fixtures/gen.rs @@ -0,0 +1,210 @@ +//! Deterministic building blocks for synthetic projects: a seeded RNG, +//! package names and versions, and well-formed (fake) digests. The same +//! seed always yields byte-identical fixtures, so base and head scan the +//! same bytes and a run is reproducible from its scenario name. + +use std::path::{Path, PathBuf}; + +use base64::Engine as _; +use sha2::Digest as _; + +/// splitmix64: tiny, fast, and good enough for picking names. +#[derive(Debug, Clone)] +pub struct Rng(u64); + +impl Rng { + pub fn new(seed: &str) -> Self { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in seed.bytes() { + h ^= u64::from(b); + h = h.wrapping_mul(0x0100_0000_01b3); + } + Self(h) + } + + pub fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) + } + + /// Uniform in `0..n` (`n > 0`). + pub fn below(&mut self, n: usize) -> usize { + (self.next_u64() % n as u64) as usize + } + + /// True with probability `p`. + pub fn chance(&mut self, p: f64) -> bool { + ((self.next_u64() >> 11) as f64 / (1u64 << 53) as f64) < p + } +} + +const SYLLABLES: &[&str] = &[ + "ab", "ac", "al", "an", "ar", "as", "at", "ba", "be", "bi", "bo", "ca", "ce", "ch", "co", "da", + "de", "di", "do", "el", "en", "er", "es", "fa", "fe", "fi", "fo", "ga", "ge", "go", "ha", "he", + "hi", "in", "is", "ja", "jo", "ka", "ki", "la", "le", "li", "lo", "ma", "me", "mi", "mo", "na", + "ne", "ni", "no", "on", "or", "pa", "pe", "pi", "po", "qu", "ra", "re", "ri", "ro", "sa", "se", + "si", "so", "ta", "te", "ti", "to", "un", "ur", "va", "ve", "vi", "wa", "we", "xe", "ya", "zo", +]; + +/// A unique lowercase identifier: two to four syllables plus the index, +/// so names have realistic, varied lengths and never collide. +pub fn ident(rng: &mut Rng, index: usize) -> String { + let n = 2 + rng.below(3); + let mut s = String::new(); + for _ in 0..n { + s.push_str(SYLLABLES[rng.below(SYLLABLES.len())]); + } + format!("{s}{index}") +} + +/// A plausible semver version. +pub fn version(rng: &mut Rng) -> String { + let major = match rng.below(10) { + 0..=5 => rng.below(4), + 6..=8 => 4 + rng.below(8), + _ => 12 + rng.below(20), + }; + format!("{major}.{}.{}", rng.below(25), rng.below(15)) +} + +pub fn sha512_bytes(seed: &str) -> [u8; 64] { + sha2::Sha512::digest(seed.as_bytes()).into() +} + +/// An npm-style SRI string, `sha512-`. +pub fn sri_sha512(seed: &str) -> String { + format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(sha512_bytes(seed)) + ) +} + +pub fn sha512_hex(seed: &str) -> String { + hex::encode(sha512_bytes(seed)) +} + +pub fn sha256_hex(seed: &str) -> String { + hex::encode(sha2::Sha256::digest(seed.as_bytes())) +} + +pub fn sha1_hex(seed: &str) -> String { + hex::encode(sha1::Sha1::digest(seed.as_bytes())) +} + +pub fn sha256_base64(seed: &str) -> String { + base64::engine::general_purpose::STANDARD.encode(sha2::Sha256::digest(seed.as_bytes())) +} + +/// A deterministic v4-shaped UUID. +pub fn uuid(seed: &str) -> String { + let h = sha2::Sha256::digest(seed.as_bytes()); + let x = hex::encode(&h[..16]); + format!( + "{}-{}-4{}-8{}-{}", + &x[0..8], + &x[8..12], + &x[13..16], + &x[17..20], + &x[20..32] + ) +} + +/// Writes files under a root, creating parents as needed. +pub struct Tree { + root: PathBuf, +} + +impl Tree { + pub fn new(root: &Path) -> std::io::Result { + std::fs::create_dir_all(root)?; + Ok(Self { + root: root.to_path_buf(), + }) + } + + pub fn write(&mut self, rel: &str, contents: impl AsRef<[u8]>) -> std::io::Result<()> { + let p = self.root.join(rel); + if let Some(parent) = p.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(p, contents) + } + + pub fn mkdir(&mut self, rel: &str) -> std::io::Result<()> { + std::fs::create_dir_all(self.root.join(rel)) + } + + pub fn symlink(&mut self, target: &str, rel: &str) -> std::io::Result<()> { + let p = self.root.join(rel); + if let Some(parent) = p.parent() { + std::fs::create_dir_all(parent)?; + } + crate::tree::symlink(Path::new(target), &p) + } +} + +/// A small but non-trivial JS source file, so installed packages carry +/// some bytes the way real ones do (the crawl never reads them, but the +/// directory walk and any hashing would). +pub fn js_source(name: &str, rng: &mut Rng) -> String { + let fns = 2 + rng.below(6); + let mut s = format!("'use strict';\n// {name}\n"); + for i in 0..fns { + s.push_str(&format!( + "function f{i}(a, b) {{\n return (a ?? {i}) + (b ?? {}) * {};\n}}\nexports.f{i} = f{i};\n", + rng.below(100), + rng.below(1000) + )); + } + s +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rng_is_deterministic_per_seed() { + let a: Vec = (0..4) + .map({ + let mut r = Rng::new("npm"); + move |_| r.next_u64() + }) + .collect(); + let b: Vec = (0..4) + .map({ + let mut r = Rng::new("npm"); + move |_| r.next_u64() + }) + .collect(); + let c = Rng::new("pnpm").next_u64(); + assert_eq!(a, b); + assert_ne!(a[0], c); + } + + #[test] + fn idents_are_unique_and_lowercase() { + let mut rng = Rng::new("x"); + let names: std::collections::HashSet = + (0..5000).map(|i| ident(&mut rng, i)).collect(); + assert_eq!(names.len(), 5000); + assert!(names.iter().all(|n| n + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit()))); + } + + #[test] + fn digests_are_well_formed() { + assert!(sri_sha512("a").starts_with("sha512-")); + assert_eq!(sri_sha512("a").len(), 7 + 88); + assert_eq!(sha256_hex("a").len(), 64); + assert_eq!(sha1_hex("a").len(), 40); + let u = uuid("a"); + assert_eq!(u.len(), 36); + assert_eq!(&u[14..15], "4"); + assert_eq!(&u[19..20], "8"); + } +} diff --git a/crates/socket-patch-bench/src/fixtures/mod.rs b/crates/socket-patch-bench/src/fixtures/mod.rs new file mode 100644 index 000000000..eee4f751c --- /dev/null +++ b/crates/socket-patch-bench/src/fixtures/mod.rs @@ -0,0 +1,308 @@ +//! Synthetic projects, one generator per package manager. +//! +//! Each generator lays down what a real project of that kind has on disk +//! after an install — manifest, lockfile, installed package tree (and, for +//! ecosystems that install into a shared cache, that cache under the +//! fixture's own `home/`) — and returns what the mock API must serve for +//! it plus what a correct scan of it reports. The expectations are what +//! make a timing meaningful: a run that scans fewer packages, redirects +//! fewer patches or rewrites different files than expected is a failed +//! run, never a fast one. + +pub mod gen; + +use crate::mock::PatchSpec; + +/// Package counts for one fixture. +#[derive(Debug, Clone, Copy)] +pub struct Size { + /// Installed (or locked) packages the scan should report. + pub packages: usize, + /// How many of them have a patch. + pub patched: usize, +} + +impl Size { + pub fn scaled(packages: usize, patched: usize, scale: f64) -> Self { + let packages = ((packages as f64 * scale).round() as usize).max(4); + let patched = ((patched as f64 * scale).round() as usize).clamp(1, packages / 2); + Self { packages, patched } + } + + /// Spread the patched packages evenly over the index range, so they are + /// not all clustered at the start of a lockfile. + pub fn is_patched(&self, index: usize) -> bool { + let stride = (self.packages / self.patched).max(1); + index % stride == stride / 2 && index / stride < self.patched + } +} + +/// What a correct scan of a fixture reports. +#[derive(Debug, Clone, Default)] +pub struct Expect { + /// `scannedPackages`. + pub scanned: usize, + /// `lockfileOnlyPackages`. + pub lockfile_only: usize, + /// `redirect.redirected` for a hosted run from the pristine fixture. + pub redirected: usize, + /// `redirect.rewrittenFiles` for that run (project-relative). + pub rewritten: Vec, + /// Warning codes a correct run may emit (`redirect.warnings[].code`, + /// top-level `warnings[].code`). Any other code fails the run. + pub allowed_warnings: Vec<&'static str>, + /// `lockfileOnlyPackages` on a rescan, when the first scan's install + /// cleanup removes installed copies (vlt drops the patched store + /// entries so the next install fetches the patch). + pub rescan_lockfile_only: usize, + /// Extra `scannedPackages` on a rescan (a redirect that adds locked + /// entries: Go's `go.sum` gains the Socket module's lines). + pub rescan_extra_scanned: usize, +} + +/// Bodies the artifact host serves: `(URL path, bytes, content type)`. +pub type Served = Vec<(String, Vec, &'static str)>; + +/// A generated project. +#[derive(Debug, Clone)] +pub struct Fixture { + /// The project directory (`--cwd`), relative to the fixture root. + pub project: &'static str, + pub patches: Vec, + /// Bodies the artifact host serves, by URL path. + pub files: Served, + /// Environment variables whose values are paths relative to the + /// fixture root (caches the ecosystem would otherwise find in the + /// runner's real home). + pub env_paths: Vec<(&'static str, &'static str)>, + /// Other environment variables the ecosystem's real users have set. + pub env: Vec<(&'static str, String)>, + pub expect: Expect, +} + +/// The artifact host's base URL inside fixtures. The CLI is pointed at the +/// mock with `SOCKET_PATCH_SERVER_URL`; references carry absolute URLs, so +/// a fixture writes this placeholder and the engine rewrites it to the +/// mock's address when it builds the catalog. +pub const PATCH_HOST: &str = "http://socket-patch-bench.invalid"; + +pub mod npm; +pub mod other; +pub mod pypi; + +use crate::scenarios::Pm; + +/// Every package manager, in run order. Sizes are a large-but-ordinary +/// project for each ecosystem, and big enough that a scan takes ~100 ms or +/// more: shorter runs are dominated by process start-up and scheduler +/// noise. +pub static ALL: &[Pm] = &[ + Pm { + name: "npm", + description: "npm (package-lock.json v3, hoisted node_modules)", + packages: 3000, + patched: 60, + build: npm::build_npm, + }, + Pm { + name: "pnpm", + description: "pnpm (pnpm-lock.yaml 9.0, isolated .pnpm store)", + packages: 3000, + patched: 60, + build: npm::build_pnpm, + }, + Pm { + name: "yarn-classic", + description: "yarn classic (yarn.lock v1, hoisted node_modules)", + packages: 3000, + patched: 60, + build: npm::build_yarn_classic, + }, + Pm { + name: "yarn-berry", + description: "yarn berry (yarn.lock v8, node-modules linker)", + packages: 3000, + patched: 60, + build: npm::build_yarn_berry, + }, + Pm { + name: "bun", + description: "bun (text bun.lock v1, hoisted node_modules)", + packages: 3000, + patched: 60, + build: npm::build_bun, + }, + Pm { + name: "vlt", + description: "vlt (vlt-lock.json v1, node_modules/.vlt store)", + packages: 1500, + patched: 30, + build: npm::build_vlt, + }, + Pm { + name: "pip", + description: "pip (hash-pinned requirements.txt, .venv)", + packages: 1000, + patched: 25, + build: pypi::build_requirements, + }, + Pm { + name: "uv", + description: "uv (uv.lock + pyproject.toml, .venv)", + packages: 400, + patched: 12, + build: pypi::build_uv, + }, + Pm { + name: "pylock", + description: "PEP 751 pylock.toml, .venv", + packages: 400, + patched: 12, + build: pypi::build_pylock, + }, + Pm { + name: "poetry", + description: "poetry (poetry.lock 2.1, in-project .venv)", + packages: 400, + patched: 12, + build: pypi::build_poetry, + }, + Pm { + name: "pipenv", + description: "pipenv (Pipfile.lock spec 6, in-project .venv)", + packages: 400, + patched: 12, + build: pypi::build_pipenv, + }, + Pm { + name: "pdm", + description: "pdm (pdm.lock 4.5.1, .venv)", + packages: 400, + patched: 12, + build: pypi::build_pdm, + }, + Pm { + name: "bundler", + description: "RubyGems (Gemfile.lock with CHECKSUMS, vendor/bundle)", + packages: 800, + patched: 20, + build: other::build_gem, + }, + Pm { + name: "composer", + description: "Composer (composer.lock, vendor/)", + packages: 800, + patched: 20, + build: other::build_composer, + }, + Pm { + name: "cargo", + description: "Cargo (Cargo.lock v4, ~/.cargo/registry/src)", + packages: 600, + patched: 15, + build: other::build_cargo, + }, + Pm { + name: "golang", + description: "Go modules (go.mod + go.sum, ~/go/pkg/mod)", + packages: 1200, + patched: 30, + build: other::build_golang, + }, + Pm { + name: "nuget", + description: "NuGet (packages.lock.json + nuget.config, ~/.nuget/packages)", + packages: 900, + patched: 25, + build: other::build_nuget, + }, + Pm { + name: "maven", + description: "Maven (pom.xml, ~/.m2/repository)", + packages: 1000, + patched: 25, + build: other::build_maven, + }, +]; + +#[cfg(test)] +mod tests { + use super::*; + + /// Every file under `root` with its bytes (symlinks by target). + fn contents(root: &std::path::Path) -> std::collections::BTreeMap> { + fn walk( + root: &std::path::Path, + dir: &std::path::Path, + out: &mut std::collections::BTreeMap>, + ) { + for e in std::fs::read_dir(dir).unwrap() { + let e = e.unwrap(); + let rel = e.path().strip_prefix(root).unwrap().display().to_string(); + let ty = e.file_type().unwrap(); + if ty.is_symlink() { + out.insert( + rel, + std::fs::read_link(e.path()) + .unwrap() + .display() + .to_string() + .into_bytes(), + ); + } else if ty.is_dir() { + walk(root, &e.path(), out); + } else { + out.insert(rel, std::fs::read(e.path()).unwrap()); + } + } + } + let mut out = std::collections::BTreeMap::new(); + walk(root, root, &mut out); + out + } + + #[test] + fn every_fixture_is_deterministic_and_expects_real_work() { + let size = Size::scaled(60, 4, 1.0); + for pm in ALL { + let tmp = tempfile::tempdir().unwrap(); + let (a, b) = (tmp.path().join("a"), tmp.path().join("b")); + let fa = (pm.build)(&mut gen::Tree::new(&a).unwrap(), size).unwrap(); + let fb = (pm.build)(&mut gen::Tree::new(&b).unwrap(), size).unwrap(); + assert_eq!( + contents(&a), + contents(&b), + "{}: same seed, different bytes", + pm.name + ); + let uuids = |f: &Fixture| f.patches.iter().map(|p| p.uuid.clone()).collect::>(); + assert_eq!(uuids(&fa), uuids(&fb), "{}", pm.name); + assert_eq!(fa.expect.scanned, size.packages, "{}", pm.name); + assert_eq!(fa.patches.len(), size.patched, "{}", pm.name); + assert_eq!(fa.expect.redirected, size.patched, "{}", pm.name); + assert!(!fa.expect.rewritten.is_empty(), "{}", pm.name); + assert!(a.join(fa.project).is_dir(), "{}", pm.name); + for p in &fa.patches { + assert_eq!(p.reference["status"], "granted", "{}", pm.name); + assert!( + p.reference["url"].as_str().unwrap().starts_with(PATCH_HOST), + "{}", + pm.name + ); + } + } + } + + #[test] + fn patched_indices_are_spread_and_counted_exactly() { + for (n, k) in [(3000, 60), (400, 12), (60, 4), (7, 3)] { + let s = Size { + packages: n, + patched: k, + }; + let hits: Vec = (0..n).filter(|&i| s.is_patched(i)).collect(); + assert_eq!(hits.len(), k, "{n}/{k}"); + assert!(hits.last().unwrap() - hits[0] >= n / 2, "{n}/{k}: {hits:?}"); + } + } +} diff --git a/crates/socket-patch-bench/src/fixtures/npm.rs b/crates/socket-patch-bench/src/fixtures/npm.rs new file mode 100644 index 000000000..ba1e8189d --- /dev/null +++ b/crates/socket-patch-bench/src/fixtures/npm.rs @@ -0,0 +1,793 @@ +//! npm-family projects. One deterministic dependency graph is written out +//! as each package manager's lockfile and install layout, so the six +//! scenarios differ only in what each package manager puts on disk. + +use std::collections::BTreeMap; +use std::fmt::Write as _; + +use serde_json::{json, Value}; + +use super::gen::{self, Rng, Tree}; +use super::{Expect, Fixture, Size, PATCH_HOST}; +use crate::mock::PatchSpec; + +/// One installed npm package. +#[derive(Debug, Clone)] +pub struct Pkg { + /// `name` or `@scope/name`. + pub name: String, + pub version: String, + /// Hoisted install path (`node_modules/a`), or nested under its only + /// dependent (`node_modules/p/node_modules/a`) for a second version. + pub path: String, + /// The package's dependencies, by name, each resolving to another + /// package of the graph at that version. + pub deps: BTreeMap, + pub patched: bool, +} + +impl Pkg { + /// The crawler's spelling: a scope stays a literal `@`. + pub fn purl(&self) -> String { + format!("pkg:npm/{}@{}", self.name, self.version) + } + + pub fn basename(&self) -> &str { + self.name.rsplit('/').next().unwrap() + } + + pub fn is_nested(&self) -> bool { + self.path.matches("node_modules/").count() > 1 + } + + /// The hoisted parent's name for a nested package. + pub fn parent_name(&self) -> Option<&str> { + let inner = self.path.strip_prefix("node_modules/")?; + let (parent, _) = inner.split_once("/node_modules/")?; + Some(parent) + } + + pub fn registry_tarball(&self, registry: &str) -> String { + format!( + "{registry}/{}/-/{}-{}.tgz", + self.name, + self.basename(), + self.version + ) + } + + pub fn integrity(&self) -> String { + gen::sri_sha512(&format!("npm:{}@{}", self.name, self.version)) + } + + pub fn sha1(&self) -> String { + gen::sha1_hex(&format!("npm:{}@{}", self.name, self.version)) + } + + /// `@` with pnpm's `+` for the scope slash, as pnpm and + /// vlt spell store directories. + pub fn store_key(&self) -> String { + format!("{}@{}", self.name.replace('/', "+"), self.version) + } +} + +/// The project graph. +pub struct Graph { + pub pkgs: Vec, +} + +/// A deterministic graph of `size.packages` installed packages: about one +/// in fifteen scoped, one in twenty a second version of a hoisted package +/// installed nested under its dependent (the layout a version conflict +/// produces), and each package depending on up to four later packages (an +/// acyclic graph, like most real trees). The first tenth are the root's +/// direct dependencies. +pub fn graph(seed: &str, size: Size) -> Graph { + let mut rng = Rng::new(seed); + let n = size.packages; + let dups = n / 20; + let hoisted = n - dups; + let mut pkgs: Vec = Vec::with_capacity(n); + for i in 0..hoisted { + let base = gen::ident(&mut rng, i); + let name = if rng.chance(1.0 / 15.0) { + format!("@{}/{base}", gen::ident(&mut rng, i)) + } else { + base + }; + pkgs.push(Pkg { + path: format!("node_modules/{name}"), + name, + version: gen::version(&mut rng), + deps: BTreeMap::new(), + patched: false, + }); + } + for i in 0..hoisted { + for _ in 0..rng.below(5) { + if i + 1 >= hoisted { + break; + } + let j = i + 1 + rng.below((hoisted - i - 1).min(200)); + let (name, version) = (pkgs[j].name.clone(), pkgs[j].version.clone()); + pkgs[i].deps.entry(name).or_insert(version); + } + } + // Second versions: the copy of a later package that one earlier + // package needs at a different version, nested under that package. + for k in 0..dups { + let target = hoisted / 2 + (k * (hoisted / 2)) / dups.max(1); + let parent = rng.below(target.max(1)); + let mut version = gen::version(&mut rng); + if version == pkgs[target].version { + version.push_str("-next.1"); + } + if pkgs[parent].deps.contains_key(&pkgs[target].name) + || pkgs[hoisted..].iter().any(|d| d.name == pkgs[target].name) + { + // Keep one nested copy per name and parent; fill the slot with + // another unique package instead. + let name = format!("{}-alt{k}", pkgs[target].name); + pkgs.push(Pkg { + path: format!("{}/node_modules/{name}", pkgs[parent].path), + name: name.clone(), + version: version.clone(), + deps: BTreeMap::new(), + patched: false, + }); + pkgs[parent].deps.insert(name, version); + continue; + } + let name = pkgs[target].name.clone(); + pkgs.push(Pkg { + path: format!("{}/node_modules/{name}", pkgs[parent].path), + name: name.clone(), + version: version.clone(), + deps: BTreeMap::new(), + patched: false, + }); + pkgs[parent].deps.insert(name, version); + } + for (i, p) in pkgs.iter_mut().enumerate() { + p.patched = size.is_patched(i); + } + Graph { pkgs } +} + +impl Graph { + fn direct(&self) -> impl Iterator { + let n = (self.pkgs.len() / 10).max(1); + self.pkgs[..n].iter().filter(|p| !p.is_nested()) + } + + pub fn root_deps(&self) -> BTreeMap { + self.direct() + .map(|p| (p.name.clone(), format!("^{}", p.version))) + .collect() + } + + pub fn package_json(&self) -> String { + let v = json!({ + "name": "bench-app", + "version": "1.0.0", + "private": true, + "dependencies": self.root_deps(), + }); + serde_json::to_string_pretty(&v).unwrap() + "\n" + } + + /// Lay every package down at its npm path under `prefix`. + pub fn install_hoisted(&self, t: &mut Tree, prefix: &str) -> std::io::Result<()> { + let mut rng = Rng::new("npm-install"); + for p in &self.pkgs { + write_package(t, &format!("{prefix}{}", p.path), p, &mut rng)?; + } + Ok(()) + } + + /// Patches with npm-style artifact URLs. + pub fn patches(&self, berry: bool) -> Vec { + self.pkgs + .iter() + .filter(|p| p.patched) + .map(|p| npm_patch(p, berry)) + .collect() + } + + /// The package `name` resolves to from `from` (its nested copy if it + /// has one, else the hoisted one). + fn resolve<'a>(&'a self, from: &Pkg, name: &str, version: &str) -> Option<&'a Pkg> { + self.pkgs.iter().find(|p| { + p.name == name + && p.version == version + && (p.parent_name() == Some(&from.name) || !p.is_nested()) + }) + } +} + +pub fn write_package(t: &mut Tree, dir: &str, p: &Pkg, rng: &mut Rng) -> std::io::Result<()> { + let manifest = json!({ + "name": p.name, + "version": p.version, + "description": format!("synthetic package {}", p.name), + "main": "index.js", + "license": "MIT", + "dependencies": p.deps.iter().map(|(k, v)| (k.clone(), format!("^{v}"))).collect::>(), + }); + t.write( + &format!("{dir}/package.json"), + serde_json::to_string_pretty(&manifest).unwrap(), + )?; + t.write(&format!("{dir}/index.js"), gen::js_source(&p.name, rng))?; + if rng.chance(0.5) { + t.write( + &format!("{dir}/README.md"), + format!("# {}\n\nSynthetic.\n", p.name), + )?; + } + Ok(()) +} + +/// The bytes the artifact host serves for a patched package (only vlt's +/// preflight downloads them; the hash is what matters). +pub fn artifact_bytes(p: &Pkg) -> Vec { + format!("patched tarball for {}@{}\n", p.name, p.version).into_bytes() +} + +fn sri_of(bytes: &[u8]) -> String { + use base64::Engine as _; + use sha2::Digest as _; + format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(sha2::Sha512::digest(bytes)) + ) +} + +/// A patch for `p`, served at the canonical hosted URL shape +/// `…/patch/npm/////-.tgz`. +pub fn npm_patch(p: &Pkg, berry: bool) -> PatchSpec { + let uuid = gen::uuid(&format!("patch:{}", p.purl())); + let token = gen::uuid(&format!("grant:{uuid}")); + let url = format!("{PATCH_HOST}{}", artifact_path(p, &token, &uuid)); + let seed = format!("patched:{}@{}", p.name, p.version); + let mut artifacts = vec![json!({ + "kind": "tarball", + "url": url, + "integrity": { + "sha512": sri_of(&artifact_bytes(p)), + "sha1": gen::sha1_hex(&seed), + "sha256": gen::sha256_hex(&seed), + }, + })]; + if berry { + artifacts.push(json!({ + "kind": "yarn-berry-zip", + "url": null, + "integrity": { "yarnBerry10c0": format!("10c0/{}", gen::sha512_hex(&seed)) }, + })); + } + PatchSpec { + purl: p.purl(), + view: view_json(&uuid, &p.purl(), "package/index.js"), + reference: json!({ + "status": "granted", + "url": url, + "purl": p.purl(), + "artifacts": artifacts, + "registryOverride": null, + }), + uuid, + tier: "free", + severity: "high", + } +} + +pub fn artifact_path(p: &Pkg, token: &str, uuid: &str) -> String { + format!( + "/patch/npm/{}/{}/{token}/{uuid}/{}-{}.tgz", + p.name, + p.version, + p.basename(), + p.version + ) +} + +/// A `PatchResponse` for the view endpoint (one patched file). +pub fn view_json(uuid: &str, purl: &str, file: &str) -> Value { + json!({ + "uuid": uuid, + "purl": purl, + "publishedAt": "2024-06-01T00:00:00Z", + "files": { + file: { + "beforeHash": gen::sha256_hex(&format!("before:{purl}")), + "afterHash": gen::sha256_hex(&format!("after:{purl}")), + } + }, + "vulnerabilities": { + format!("GHSA-{}", &gen::sha1_hex(purl)[..14]): { + "cves": ["CVE-2024-0001"], + "summary": "synthetic vulnerability", + "severity": "high", + "description": "synthetic", + } + }, + "description": "synthetic patch", + "license": "MIT", + "tier": "free", + }) +} + +fn fixture( + g: &Graph, + patches: Vec, + rewritten: &[&str], + warnings: &[&'static str], +) -> Fixture { + Fixture { + project: "project", + expect: Expect { + scanned: g.pkgs.len(), + lockfile_only: 0, + redirected: patches.len(), + rewritten: rewritten.iter().map(|s| s.to_string()).collect(), + allowed_warnings: warnings.to_vec(), + rescan_lockfile_only: 0, + rescan_extra_scanned: 0, + }, + patches, + files: Vec::new(), + env_paths: Vec::new(), + env: Vec::new(), + } +} + +// ── npm ──────────────────────────────────────────────────────────────── + +/// `package-lock.json` (lockfileVersion 3). +pub fn package_lock(g: &Graph) -> String { + let mut packages = serde_json::Map::new(); + packages.insert( + String::new(), + json!({ "name": "bench-app", "version": "1.0.0", "dependencies": g.root_deps() }), + ); + let mut sorted: Vec<&Pkg> = g.pkgs.iter().collect(); + sorted.sort_by(|a, b| a.path.cmp(&b.path)); + for p in sorted { + let mut e = json!({ + "version": p.version, + "resolved": p.registry_tarball("https://registry.npmjs.org"), + "integrity": p.integrity(), + "license": "MIT", + }); + if !p.deps.is_empty() { + e["dependencies"] = json!(p + .deps + .iter() + .map(|(k, v)| (k.clone(), format!("^{v}"))) + .collect::>()); + } + packages.insert(p.path.clone(), e); + } + let lock = json!({ + "name": "bench-app", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": packages, + }); + serde_json::to_string_pretty(&lock).unwrap() + "\n" +} + +pub fn build_npm(t: &mut Tree, size: Size) -> std::io::Result { + let g = graph("npm", size); + t.write("project/package.json", g.package_json())?; + let lock = package_lock(&g); + t.write("project/package-lock.json", &lock)?; + // npm's hidden lockfile, as `npm install` leaves it. + t.write("project/node_modules/.package-lock.json", &lock)?; + g.install_hoisted(t, "project/")?; + t.mkdir("home")?; + Ok(fixture( + &g, + g.patches(false), + &["package-lock.json", ".npmrc"], + &["redirect_npm_allow_remote"], + )) +} + +// ── pnpm ─────────────────────────────────────────────────────────────── + +fn yaml_key(s: &str) -> String { + if s.starts_with('@') { + format!("'{s}'") + } else { + s.to_string() + } +} + +/// `pnpm-lock.yaml` (lockfileVersion 9.0, pnpm 9-10). +pub fn pnpm_lock(g: &Graph) -> String { + let mut s = String::from( + "lockfileVersion: '9.0'\n\nsettings:\n autoInstallPeers: true\n excludeLinksFromLockfile: false\n\nimporters:\n\n .:\n dependencies:\n", + ); + for p in g.direct() { + let _ = write!( + s, + " {}:\n specifier: ^{}\n version: {}\n", + yaml_key(&p.name), + p.version, + p.version + ); + } + let mut sorted: Vec<&Pkg> = g.pkgs.iter().collect(); + sorted.sort_by_key(|p| format!("{}@{}", p.name, p.version)); + s.push_str("\npackages:\n"); + for p in &sorted { + let _ = write!( + s, + "\n {}:\n resolution: {{integrity: {}}}\n", + yaml_key(&format!("{}@{}", p.name, p.version)), + p.integrity() + ); + if p.version.starts_with('0') { + s.push_str(" engines: {node: '>=12'}\n"); + } + } + s.push_str("\nsnapshots:\n"); + for p in &sorted { + let key = yaml_key(&format!("{}@{}", p.name, p.version)); + if p.deps.is_empty() { + let _ = writeln!(s, "\n {key}: {{}}"); + } else { + let _ = write!(s, "\n {key}:\n dependencies:\n"); + for (d, v) in &p.deps { + let _ = writeln!(s, " {}: {v}", yaml_key(d)); + } + } + } + s +} + +/// pnpm's isolated layout: every package is a real directory in the +/// virtual store, its dependencies are symlinks beside it, and only the +/// root's direct dependencies are linked into `node_modules/`. +pub fn build_pnpm(t: &mut Tree, size: Size) -> std::io::Result { + let g = graph("pnpm", size); + t.write("project/package.json", g.package_json())?; + t.write("project/pnpm-lock.yaml", pnpm_lock(&g))?; + let mut rng = Rng::new("pnpm-install"); + let mut lock_yaml = String::from("lockfileVersion: '9.0'\n"); + for p in &g.pkgs { + let entry = format!("project/node_modules/.pnpm/{}/node_modules", p.store_key()); + write_package(t, &format!("{entry}/{}", p.name), p, &mut rng)?; + for (d, v) in &p.deps { + let Some(dep) = g.resolve(p, d, v) else { + continue; + }; + let up = "../".repeat(1 + p.name.matches('/').count() + d.matches('/').count()); + let target = format!("{up}../{}/node_modules/{d}", dep.store_key()); + t.symlink(&target, &format!("{entry}/{d}"))?; + } + let _ = writeln!(lock_yaml, "# {}", p.store_key()); + } + for p in g.direct() { + let up = "../".repeat(p.name.matches('/').count()); + t.symlink( + &format!("{up}.pnpm/{}/node_modules/{}", p.store_key(), p.name), + &format!("project/node_modules/{}", p.name), + )?; + } + t.write( + "project/node_modules/.modules.yaml", + "layoutVersion: 5\nnodeLinker: isolated\npackageManager: pnpm@9.15.0\n", + )?; + t.write("project/node_modules/.pnpm/lock.yaml", pnpm_lock(&g))?; + t.mkdir("home")?; + Ok(fixture( + &g, + g.patches(false), + &["pnpm-lock.yaml", "pnpm-workspace.yaml"], + &["redirect_pnpm_trust_lockfile"], + )) +} + +// ── yarn classic ─────────────────────────────────────────────────────── + +fn yarn_key(name: &str, range: &str) -> String { + let k = format!("{name}@{range}"); + if name.starts_with('@') { + format!("\"{k}\"") + } else { + k + } +} + +/// `yarn.lock` v1. +pub fn yarn_classic_lock(g: &Graph) -> String { + let mut s = String::from( + "# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.\n# yarn lockfile v1\n\n", + ); + let mut sorted: Vec<&Pkg> = g.pkgs.iter().collect(); + sorted.sort_by_key(|p| format!("{}@{}", p.name, p.version)); + for p in sorted { + let _ = write!( + s, + "\n{}:\n version \"{}\"\n resolved \"{}#{}\"\n integrity {}\n", + yarn_key(&p.name, &format!("^{}", p.version)), + p.version, + p.registry_tarball("https://registry.yarnpkg.com"), + p.sha1(), + p.integrity() + ); + if !p.deps.is_empty() { + s.push_str(" dependencies:\n"); + for (d, v) in &p.deps { + let d = if d.starts_with('@') { + format!("\"{d}\"") + } else { + d.clone() + }; + let _ = writeln!(s, " {d} \"^{v}\""); + } + } + } + s +} + +pub fn build_yarn_classic(t: &mut Tree, size: Size) -> std::io::Result { + let g = graph("yarn-classic", size); + t.write("project/package.json", g.package_json())?; + t.write("project/yarn.lock", yarn_classic_lock(&g))?; + t.write( + "project/node_modules/.yarn-integrity", + "{\n \"systemParams\": \"linux-x64-127\",\n \"modulesFolders\": [\"node_modules\"]\n}\n", + )?; + g.install_hoisted(t, "project/")?; + t.mkdir("home")?; + Ok(fixture(&g, g.patches(false), &["yarn.lock"], &[])) +} + +// ── yarn berry ───────────────────────────────────────────────────────── + +/// `yarn.lock` for yarn 4 (`__metadata` version 8, cacheKey 10c0). +pub fn yarn_berry_lock(g: &Graph) -> String { + let mut s = String::from( + "# This file is generated by running \"yarn install\" inside your project.\n# Manual changes might be lost - proceed with caution!\n\n__metadata:\n version: 8\n cacheKey: 10c0\n", + ); + let mut entries: Vec<(String, String)> = Vec::new(); + for p in &g.pkgs { + let mut e = format!( + " version: {}\n resolution: \"{}@npm:{}\"\n", + p.version, p.name, p.version + ); + if !p.deps.is_empty() { + e.push_str(" dependencies:\n"); + for (d, v) in &p.deps { + let _ = writeln!( + e, + " {}: \"npm:^{v}\"", + if d.starts_with('@') { + format!("\"{d}\"") + } else { + d.clone() + } + ); + } + } + let _ = write!( + e, + " checksum: 10c0/{}\n languageName: node\n linkType: hard\n", + gen::sha512_hex(&format!("berry:{}@{}", p.name, p.version)) + ); + entries.push((format!("\"{}@npm:^{}\"", p.name, p.version), e)); + } + let mut root = String::from( + " version: 0.0.0-use.local\n resolution: \"bench-app@workspace:.\"\n dependencies:\n", + ); + for (d, r) in g.root_deps() { + let _ = writeln!( + root, + " {}: \"npm:{r}\"", + if d.starts_with('@') { + format!("\"{d}\"") + } else { + d + } + ); + } + root.push_str(" languageName: unknown\n linkType: soft\n"); + entries.push(("\"bench-app@workspace:.\"".into(), root)); + entries.sort(); + for (k, e) in entries { + let _ = write!(s, "\n{k}:\n{e}"); + } + s +} + +pub fn build_yarn_berry(t: &mut Tree, size: Size) -> std::io::Result { + let g = graph("yarn-berry", size); + t.write( + "project/package.json", + g.package_json().replacen( + "\"private\": true", + "\"private\": true,\n \"packageManager\": \"yarn@4.5.0\"", + 1, + ), + )?; + t.write("project/yarn.lock", yarn_berry_lock(&g))?; + t.write( + "project/.yarnrc.yml", + "nodeLinker: node-modules\nenableGlobalCache: false\n", + )?; + t.write("project/node_modules/.yarn-state.yml", "# Warning: This file is automatically generated. Removing it is fine, but will\n# cause your node_modules installation to become invalidated.\n\n__metadata:\n version: 1\n nmMode: classic\n")?; + g.install_hoisted(t, "project/")?; + t.mkdir("home")?; + Ok(fixture(&g, g.patches(true), &["yarn.lock"], &[])) +} + +// ── bun ──────────────────────────────────────────────────────────────── + +/// Text `bun.lock` (lockfileVersion 1, bun 1.2): JSONC with trailing +/// commas, one line per package, a blank line between packages. +pub fn bun_lock(g: &Graph) -> String { + let mut s = String::from( + "{\n \"lockfileVersion\": 1,\n \"workspaces\": {\n \"\": {\n \"name\": \"bench-app\",\n \"dependencies\": {\n", + ); + for (d, r) in g.root_deps() { + let _ = writeln!(s, " \"{d}\": \"{r}\","); + } + s.push_str(" },\n },\n },\n \"packages\": {\n"); + let mut entries: Vec<(String, String)> = g + .pkgs + .iter() + .map(|p| { + let key = match p.parent_name() { + Some(parent) => format!("{parent}/{}", p.name), + None => p.name.clone(), + }; + let meta = if p.deps.is_empty() { + "{}".to_string() + } else { + let deps: Vec = p + .deps + .iter() + .map(|(d, v)| format!("\"{d}\": \"^{v}\"")) + .collect(); + format!("{{ \"dependencies\": {{ {} }} }}", deps.join(", ")) + }; + ( + key.clone(), + format!( + " \"{key}\": [\"{}@{}\", \"\", {meta}, \"{}\"],\n", + p.name, + p.version, + p.integrity() + ), + ) + }) + .collect(); + entries.sort(); + let lines: Vec = entries.into_iter().map(|(_, l)| l).collect(); + s.push_str(&lines.join("\n")); + s.push_str(" }\n}\n"); + s +} + +pub fn build_bun(t: &mut Tree, size: Size) -> std::io::Result { + let g = graph("bun", size); + t.write("project/package.json", g.package_json())?; + t.write("project/bun.lock", bun_lock(&g))?; + g.install_hoisted(t, "project/")?; + t.mkdir("home")?; + Ok(fixture(&g, g.patches(false), &["bun.lock"], &[])) +} + +// ── vlt ──────────────────────────────────────────────────────────────── + +fn vlt_id(p: &Pkg) -> String { + format!("~npm~{}", p.store_key()) +} + +/// `vlt-lock.json` (lockfileVersion 1): one line per node, no spaces +/// between tuple elements. +pub fn vlt_lock(g: &Graph) -> String { + let mut nodes: Vec = g + .pkgs + .iter() + .map(|p| { + format!( + " \"{}\": [0,\"{}\",\"{}\",\"{}\"]", + vlt_id(p), + p.name, + p.integrity(), + p.registry_tarball("https://registry.npmjs.org") + ) + }) + .collect(); + nodes.sort(); + let mut edges: Vec = Vec::new(); + for p in g.direct() { + edges.push(format!( + " \"file~_d {}\": \"prod ^{} {}\"", + p.name, + p.version, + vlt_id(p) + )); + } + for p in &g.pkgs { + for (d, v) in &p.deps { + if let Some(dep) = g.resolve(p, d, v) { + edges.push(format!( + " \"{} {d}\": \"prod ^{v} {}\"", + vlt_id(p), + vlt_id(dep) + )); + } + } + } + edges.sort(); + format!( + "{{\n \"lockfileVersion\": 1,\n \"options\": {{\n \"registries\": {{\n \"npm\": \"https://registry.npmjs.org/\"\n }}\n }},\n \"nodes\": {{\n{}\n }},\n \"edges\": {{\n{}\n }}\n}}\n", + nodes.join(",\n"), + edges.join(",\n") + ) +} + +/// vlt's store layout: every package is a real directory under +/// `node_modules/.vlt//node_modules/`, dependencies are +/// symlinks beside it, the root's direct dependencies are linked into +/// `node_modules/`, and `node_modules/.vlt-lock.json` records the install. +pub fn build_vlt(t: &mut Tree, size: Size) -> std::io::Result { + let g = graph("vlt", size); + t.write("project/package.json", g.package_json())?; + t.write("project/vlt.json", "{}\n")?; + let lock = vlt_lock(&g); + t.write("project/vlt-lock.json", &lock)?; + t.write("project/node_modules/.vlt-lock.json", &lock)?; + let mut rng = Rng::new("vlt-install"); + for p in &g.pkgs { + let entry = format!("project/node_modules/.vlt/{}/node_modules", vlt_id(p)); + write_package(t, &format!("{entry}/{}", p.name), p, &mut rng)?; + for (d, v) in &p.deps { + let Some(dep) = g.resolve(p, d, v) else { + continue; + }; + let up = "../".repeat(1 + p.name.matches('/').count() + d.matches('/').count()); + t.symlink( + &format!("{up}../{}/node_modules/{d}", vlt_id(dep)), + &format!("{entry}/{d}"), + )?; + } + } + for p in g.direct() { + let up = "../".repeat(p.name.matches('/').count()); + t.symlink( + &format!("{up}.vlt/{}/node_modules/{}", vlt_id(p), p.name), + &format!("project/node_modules/{}", p.name), + )?; + } + t.mkdir("home")?; + let patches = g.patches(false); + let mut f = fixture( + &g, + patches, + &["vlt-lock.json"], + &["redirect_vlt_reinstall_required"], + ); + // The wet scan removes the patched packages' store entries (and the + // hidden lock) so the next `vlt install` fetches the patches; until + // then a rescan sees them in the lock only. + f.expect.rescan_lockfile_only = f.patches.len(); + // vlt's preflight downloads each artifact and checks its sha512. + for p in g.pkgs.iter().filter(|p| p.patched) { + let uuid = gen::uuid(&format!("patch:{}", p.purl())); + let token = gen::uuid(&format!("grant:{uuid}")); + f.files.push(( + artifact_path(p, &token, &uuid), + artifact_bytes(p), + "application/octet-stream", + )); + } + Ok(f) +} diff --git a/crates/socket-patch-bench/src/fixtures/other.rs b/crates/socket-patch-bench/src/fixtures/other.rs new file mode 100644 index 000000000..09adc07bb --- /dev/null +++ b/crates/socket-patch-bench/src/fixtures/other.rs @@ -0,0 +1,824 @@ +//! RubyGems, Composer, Cargo, Go, NuGet and Maven projects. +//! +//! Ecosystems that install into a per-user cache (Cargo, Go, NuGet, Maven) +//! get that cache under the fixture's `home/`, which is the run's `HOME`: +//! the crawlers find it exactly where they would find a developer's, and +//! never the CI runner's. + +use std::fmt::Write as _; + +use serde_json::json; + +use super::gen::{self, Rng, Tree}; +use super::npm::view_json; +use super::pypi::pretty4; +use super::{Expect, Fixture, Size, PATCH_HOST}; +use crate::mock::PatchSpec; + +/// A generic package of a non-npm ecosystem. +#[derive(Debug, Clone)] +struct Pkg { + name: String, + version: String, + deps: Vec, + direct: bool, + patched: bool, +} + +/// `size.packages` packages, the first `direct_share` of them direct. +/// Patched packages are always direct dependencies that nothing else +/// depends on: Cargo and Maven hosted mode only redirect direct +/// dependencies (a transitive one is refused, by design), and the other +/// ecosystems do not care. +fn universe( + seed: &str, + size: Size, + direct_share: f64, + mut name: impl FnMut(&mut Rng, usize) -> String, + mut version: impl FnMut(&mut Rng) -> String, +) -> Vec { + let mut rng = Rng::new(seed); + let n = size.packages; + let direct = ((n as f64 * direct_share) as usize) + .max(size.patched * 2) + .min(n); + let patched_size = Size { + packages: direct, + patched: size.patched, + }; + let mut out: Vec = (0..n) + .map(|i| Pkg { + name: name(&mut rng, i), + version: version(&mut rng), + deps: Vec::new(), + direct: i < direct, + patched: i < direct && patched_size.is_patched(i), + }) + .collect(); + for i in 0..n { + for _ in 0..rng.below(4) { + if i + 1 < n { + let j = i + 1 + rng.below((n - i - 1).min(120)); + if !out[j].patched && !out[i].deps.contains(&j) { + out[i].deps.push(j); + } + } + } + out[i].deps.sort_unstable(); + } + out +} + +fn grant(purl: &str) -> (String, String) { + let uuid = gen::uuid(&format!("patch:{purl}")); + let token = gen::uuid(&format!("grant:{uuid}")); + (uuid, token) +} + +fn spec(purl: String, uuid: String, view_file: &str, reference: serde_json::Value) -> PatchSpec { + PatchSpec { + view: view_json(&uuid, &purl, view_file), + purl, + uuid, + tier: "free", + severity: "high", + reference, + } +} + +fn fixture( + scanned: usize, + patches: Vec, + rewritten: &[&str], + warnings: &[&'static str], +) -> Fixture { + Fixture { + project: "project", + expect: Expect { + scanned, + lockfile_only: 0, + redirected: patches.len(), + rewritten: rewritten.iter().map(|s| s.to_string()).collect(), + allowed_warnings: warnings.to_vec(), + rescan_lockfile_only: 0, + rescan_extra_scanned: 0, + }, + patches, + files: Vec::new(), + env_paths: Vec::new(), + env: Vec::new(), + } +} + +// ── RubyGems (bundler) ───────────────────────────────────────────────── + +pub fn build_gem(t: &mut Tree, size: Size) -> std::io::Result { + let gems = universe( + "gem", + size, + 0.2, + |r, i| { + let a = gen::ident(r, i); + if r.chance(0.3) { + format!("{a}-{}", &gen::ident(r, i)[..4]) + } else { + a + } + }, + gen::version, + ); + let mut gemfile = String::from("source \"https://rubygems.org\"\n\nruby \"~> 3.3\"\n\n"); + for g in gems.iter().filter(|g| g.direct) { + let _ = writeln!(gemfile, "gem \"{}\", \"~> {}\"", g.name, g.version); + } + t.write("project/Gemfile", gemfile)?; + let mut sorted: Vec<&Pkg> = gems.iter().collect(); + sorted.sort_by(|a, b| a.name.cmp(&b.name)); + let mut lock = String::from("GEM\n remote: https://rubygems.org/\n specs:\n"); + for g in &sorted { + let _ = writeln!(lock, " {} ({})", g.name, g.version); + for &j in &g.deps { + let _ = writeln!(lock, " {} (>= {})", gems[j].name, gems[j].version); + } + } + lock.push_str("\nPLATFORMS\n ruby\n x86_64-linux\n\nDEPENDENCIES\n"); + let mut direct: Vec<&Pkg> = gems.iter().filter(|g| g.direct).collect(); + direct.sort_by(|a, b| a.name.cmp(&b.name)); + for g in direct { + let _ = writeln!(lock, " {} (~> {})", g.name, g.version); + } + lock.push_str("\nCHECKSUMS\n"); + for g in &sorted { + let _ = writeln!( + lock, + " {} ({}) sha256={}", + g.name, + g.version, + gen::sha256_hex(&format!("gem:{}", g.name)) + ); + } + lock.push_str("\nRUBY VERSION\n ruby 3.3.5p100\n\nBUNDLED WITH\n 2.6.2\n"); + t.write("project/Gemfile.lock", lock)?; + t.write( + "project/.bundle/config", + "---\nBUNDLE_PATH: \"vendor/bundle\"\n", + )?; + let root = "project/vendor/bundle/ruby/3.3.0"; + let mut rng = Rng::new("gem-install"); + for g in &gems { + let dir = format!("{root}/gems/{}-{}", g.name, g.version); + let module = g.name.replace('-', "_"); + t.write( + &format!("{dir}/lib/{module}.rb"), + gen::js_source(&g.name, &mut rng).replace("//", "#"), + )?; + t.write( + &format!("{dir}/lib/{module}/version.rb"), + format!("module X\n VERSION = \"{}\"\nend\n", g.version), + )?; + t.write(&format!("{dir}/README.md"), format!("# {}\n", g.name))?; + t.write( + &format!("{root}/specifications/{}-{}.gemspec", g.name, g.version), + format!("Gem::Specification.new do |s|\n s.name = \"{}\"\n s.version = \"{}\"\n s.files = [\"lib/{module}.rb\"]\nend\n", g.name, g.version), + )?; + } + t.mkdir("home")?; + let patches = gems + .iter() + .filter(|g| g.patched) + .map(|g| { + let purl = format!("pkg:gem/{}@{}", g.name, g.version); + let (uuid, token) = grant(&purl); + let url = format!("{PATCH_HOST}/patch/gem/{}/{}/{token}/{uuid}/{}-{}.gem", g.name, g.version, g.name, g.version); + let sha = gen::sha256_hex(&format!("patched-gem:{}", g.name)); + spec( + purl.clone(), + uuid.clone(), + &format!("package/lib/{}.rb", g.name.replace('-', "_")), + json!({ + "status": "granted", + "url": url, + "purl": purl, + "artifacts": [{ "kind": "tarball", "url": url, "integrity": { "sha256": sha } }], + "registryOverride": { + "kind": "rubygems-compact-index", + "indexUrl": format!("{PATCH_HOST}/patch-registry/gem/{token}/{uuid}/"), + "identifiers": { "name": g.name, "version": g.version, "gemChecksumSha256": sha }, + }, + }), + ) + }) + .collect(); + Ok(fixture( + gems.len(), + patches, + &["Gemfile", "Gemfile.lock"], + &["redirect_gem_stale_install"], + )) +} + +// ── Composer ─────────────────────────────────────────────────────────── + +pub fn build_composer(t: &mut Tree, size: Size) -> std::io::Result { + let pkgs = universe( + "composer", + size, + 0.2, + |r, i| { + format!( + "{}/{}", + &gen::ident(r, i)[..5].trim_end_matches(char::is_numeric), + gen::ident(r, i) + ) + }, + gen::version, + ); + let entry = |p: &Pkg, installed: bool| { + let reference = gen::sha1_hex(&format!("composer:{}", p.name)); + let mut e = json!({ + "name": p.name, + "version": p.version, + "source": { "type": "git", "url": format!("https://github.com/{}.git", p.name), "reference": reference }, + "dist": { + "type": "zip", + "url": format!("https://api.github.com/repos/{}/zipball/{reference}", p.name), + "reference": reference, + "shasum": "", + }, + "require": serde_json::Map::from_iter(p.deps.iter().map(|&j| (pkgs[j].name.clone(), json!(format!("^{}", pkgs[j].version))))), + "type": "library", + "autoload": { "psr-4": { format!("Bench\\{}\\", p.name.split('/').next_back().unwrap()): "src/" } }, + "license": ["MIT"], + "description": format!("Synthetic package {}", p.name), + "time": "2024-05-01T00:00:00+00:00", + }); + if installed { + e["install-path"] = json!(format!("../{}", p.name)); + } + e + }; + let mut sorted: Vec<&Pkg> = pkgs.iter().collect(); + sorted.sort_by(|a, b| a.name.cmp(&b.name)); + let lock = json!({ + "_readme": [ + "This file locks the dependencies of your project to a known state", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", + "This file is @generated automatically", + ], + "content-hash": gen::sha256_hex("composer-content")[..32], + "packages": sorted.iter().map(|p| entry(p, false)).collect::>(), + "packages-dev": [], + "aliases": [], + "minimum-stability": "stable", + "stability-flags": {}, + "prefer-stable": false, + "prefer-lowest": false, + "platform": { "php": ">=8.1" }, + "platform-dev": {}, + "plugin-api-version": "2.6.0", + }); + t.write("project/composer.lock", pretty4(&lock) + "\n")?; + let mut require = serde_json::Map::new(); + require.insert("php".into(), json!(">=8.1")); + for p in pkgs.iter().filter(|p| p.direct) { + require.insert(p.name.clone(), json!(format!("^{}", p.version))); + } + t.write( + "project/composer.json", + pretty4(&json!({ "name": "bench/app", "type": "project", "require": require })) + "\n", + )?; + let installed = json!({ + "packages": sorted.iter().map(|p| entry(p, true)).collect::>(), + "dev": true, + "dev-package-names": [], + }); + t.write( + "project/vendor/composer/installed.json", + pretty4(&installed) + "\n", + )?; + t.write( + "project/vendor/autoload.php", + " std::io::Result { + let crates = universe( + "cargo", + size, + 0.12, + |r, i| { + let a = gen::ident(r, i); + if r.chance(0.4) { + format!( + "{a}-{}", + ["core", "sys", "derive", "macros", "impl", "util"][r.below(6)] + ) + } else { + a + } + }, + gen::version, + ); + let mut toml = String::from("[package]\nname = \"bench-app\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[dependencies]\n"); + for c in crates.iter().filter(|c| c.direct) { + if c.name.len() % 3 == 0 { + let _ = writeln!( + toml, + "{} = {{ version = \"{}\", default-features = false }}", + c.name, c.version + ); + } else { + let _ = writeln!(toml, "{} = \"{}\"", c.name, c.version); + } + } + t.write("project/Cargo.toml", toml)?; + t.write("project/src/main.rs", "fn main() {}\n")?; + let mut lock = String::from("# This file is automatically @generated by Cargo.\n# It is not intended for manual editing.\nversion = 4\n"); + let mut entries: Vec<(String, String)> = crates + .iter() + .map(|c| { + let mut e = format!( + "\n[[package]]\nname = \"{}\"\nversion = \"{}\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"{}\"\n", + c.name, + c.version, + gen::sha256_hex(&format!("crate:{}", c.name)) + ); + if !c.deps.is_empty() { + let mut names: Vec<&str> = c.deps.iter().map(|&j| crates[j].name.as_str()).collect(); + names.sort_unstable(); + e.push_str("dependencies = [\n"); + for n in names { + let _ = writeln!(e, " \"{n}\","); + } + e.push_str("]\n"); + } + (c.name.clone(), e) + }) + .collect(); + let mut root = String::from( + "\n[[package]]\nname = \"bench-app\"\nversion = \"0.1.0\"\ndependencies = [\n", + ); + let mut direct: Vec<&str> = crates + .iter() + .filter(|c| c.direct) + .map(|c| c.name.as_str()) + .collect(); + direct.sort_unstable(); + for n in direct { + let _ = writeln!(root, " \"{n}\","); + } + root.push_str("]\n"); + entries.push(("bench-app".into(), root)); + entries.sort(); + for (_, e) in entries { + lock.push_str(&e); + } + t.write("project/Cargo.lock", lock)?; + let src = "home/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f"; + let mut rng = Rng::new("cargo-install"); + for c in &crates { + let dir = format!("{src}/{}-{}", c.name, c.version); + t.write( + &format!("{dir}/Cargo.toml"), + format!("[package]\nedition = \"2021\"\nname = \"{}\"\nversion = \"{}\"\nlicense = \"MIT\"\ndescription = \"synthetic\"\n", c.name, c.version), + )?; + t.write( + &format!("{dir}/src/lib.rs"), + gen::js_source(&c.name, &mut rng).replace("'use strict';", "//!"), + )?; + t.write(&format!("{dir}/.cargo-ok"), "{\"v\":1}")?; + } + let patches = crates + .iter() + .filter(|c| c.patched) + .map(|c| { + let purl = format!("pkg:cargo/{}@{}", c.name, c.version); + let (uuid, token) = grant(&purl); + let url = format!("{PATCH_HOST}/patch/cargo/{}/{}/{token}/{uuid}/{}-{}.crate", c.name, c.version, c.name, c.version); + let cksum = gen::sha256_hex(&format!("patched-crate:{}", c.name)); + spec( + purl.clone(), + uuid.clone(), + "package/src/lib.rs", + json!({ + "status": "granted", + "url": url, + "purl": purl, + "artifacts": [{ "kind": "tarball", "url": url, "integrity": { "sha256": cksum } }], + "registryOverride": { + "kind": "cargo-sparse", + "indexUrl": format!("sparse+{PATCH_HOST}/patch-registry/cargo/{token}/{uuid}/index/"), + "identifiers": { "name": c.name, "version": c.version, "cargoCksumSha256": cksum }, + }, + }), + ) + }) + .collect(); + Ok(fixture( + crates.len(), + patches, + &["Cargo.toml", "Cargo.lock", ".cargo/config.toml"], + &[], + )) +} + +// ── Go modules ───────────────────────────────────────────────────────── + +/// The module cache's case encoding (`Foo` → `!foo`). +fn go_escape(s: &str) -> String { + let mut out = String::new(); + for c in s.chars() { + if c.is_ascii_uppercase() { + out.push('!'); + out.push(c.to_ascii_lowercase()); + } else { + out.push(c); + } + } + out +} + +fn h1(seed: &str) -> String { + format!("h1:{}", gen::sha256_base64(seed)) +} + +pub fn build_golang(t: &mut Tree, size: Size) -> std::io::Result { + let mods = universe( + "golang", + size, + 0.15, + |r, i| { + let org = gen::ident(r, i); + let repo = gen::ident(r, i); + match r.below(10) { + 0 => format!("golang.org/x/{repo}"), + 1 => format!("go.uber.org/{repo}"), + // Mixed-case paths exercise the cache's `!` decoding. + 2 => { + let mut o = org.clone(); + o[..1].make_ascii_uppercase(); + format!("github.com/{o}/{repo}") + } + _ => format!("github.com/{org}/{repo}"), + } + }, + |r| format!("v{}.{}.{}", r.below(2), r.below(30), r.below(12)), + ); + let mut gomod = String::from("module example.com/bench/app\n\ngo 1.22\n\nrequire (\n"); + for m in mods.iter().filter(|m| m.direct) { + let _ = writeln!(gomod, "\t{} {}", m.name, m.version); + } + gomod.push_str(")\n\nrequire (\n"); + for m in mods.iter().filter(|m| !m.direct) { + let _ = writeln!(gomod, "\t{} {} // indirect", m.name, m.version); + } + gomod.push_str(")\n"); + t.write("project/go.mod", gomod)?; + let mut sum: Vec = Vec::new(); + for m in &mods { + sum.push(format!( + "{} {} {}", + m.name, + m.version, + h1(&format!("zip:{}", m.name)) + )); + sum.push(format!( + "{} {}/go.mod {}", + m.name, + m.version, + h1(&format!("mod:{}", m.name)) + )); + } + sum.sort(); + t.write("project/go.sum", sum.join("\n") + "\n")?; + t.write("project/main.go", "package main\n\nfunc main() {}\n")?; + let mut rng = Rng::new("go-install"); + for m in &mods { + let dir = format!("home/go/pkg/mod/{}@{}", go_escape(&m.name), m.version); + t.write( + &format!("{dir}/go.mod"), + format!("module {}\n\ngo 1.20\n", m.name), + )?; + let pkg = m.name.rsplit('/').next().unwrap().replace('-', "_"); + t.write( + &format!("{dir}/{pkg}.go"), + format!( + "package {pkg}\n\n{}", + gen::js_source(&m.name, &mut rng).replace("'use strict';", "") + ), + )?; + t.write(&format!("{dir}/LICENSE"), "MIT\n")?; + } + let patches = mods + .iter() + .filter(|m| m.patched) + .map(|m| { + let purl = format!("pkg:golang/{}@{}", m.name, m.version); + let (uuid, _) = grant(&purl); + let smod = format!("patch.socket.dev/gopatch/{uuid}"); + let sver = format!("{}-socketpatch.1", m.version); + let url = format!("{PATCH_HOST}/patch-registry/golang/{smod}/@v/{sver}.zip"); + spec( + purl.clone(), + uuid.clone(), + "package/patched.go", + json!({ + "status": "granted", + "url": url, + "purl": purl, + "artifacts": [{ "kind": "tarball", "url": url, "integrity": {} }], + "registryOverride": { + "kind": "goproxy", + "indexUrl": PATCH_HOST, + "identifiers": { + "name": m.name, + "version": m.version, + "goModulePath": smod, + "goModuleVersion": sver, + "goZipDirhashH1": h1(&format!("patched-zip:{}", m.name)), + "goModH1": h1(&format!("patched-mod:{}", m.name)), + }, + }, + }), + ) + }) + .collect(); + let mut f = fixture(mods.len(), patches, &["go.mod", "go.sum"], &[]); + // The rewrite adds each Socket module's go.sum lines, which a rescan's + // go.sum inventory reports as more (lockfile-only) modules. + f.expect.rescan_extra_scanned = f.patches.len(); + f.expect.rescan_lockfile_only = f.patches.len(); + Ok(f) +} + +// ── NuGet ────────────────────────────────────────────────────────────── + +pub fn build_nuget(t: &mut Tree, size: Size) -> std::io::Result { + let pkgs = universe( + "nuget", + size, + 0.25, + |r, i| { + let mut a = gen::ident(r, i); + a[..1].make_ascii_uppercase(); + let mut b = gen::ident(r, i + 7); + b[..1].make_ascii_uppercase(); + match r.below(3) { + 0 => format!("{a}.{b}"), + 1 => format!("{a}.{b}.Abstractions"), + _ => a, + } + }, + gen::version, + ); + let mut csproj = String::from("\n\n \n Exe\n net8.0\n true\n \n\n \n"); + for p in pkgs.iter().filter(|p| p.direct) { + let _ = writeln!( + csproj, + " ", + p.name, p.version + ); + } + csproj.push_str(" \n\n\n"); + t.write("project/Bench.App.csproj", csproj)?; + t.write( + "project/nuget.config", + "\n\n \n \n \n \n\n", + )?; + let mut deps = serde_json::Map::new(); + let mut sorted: Vec<&Pkg> = pkgs.iter().collect(); + sorted.sort_by_key(|p| p.name.to_ascii_lowercase()); + for p in sorted { + let mut e = serde_json::Map::new(); + e.insert( + "type".into(), + json!(if p.direct { "Direct" } else { "Transitive" }), + ); + if p.direct { + e.insert("requested".into(), json!(format!("[{}, )", p.version))); + } + e.insert("resolved".into(), json!(p.version)); + e.insert( + "contentHash".into(), + json!(gen::sri_sha512(&format!("nupkg:{}", p.name))[7..]), + ); + if !p.deps.is_empty() { + e.insert( + "dependencies".into(), + json!(serde_json::Map::from_iter( + p.deps + .iter() + .map(|&j| (pkgs[j].name.clone(), json!(pkgs[j].version))) + )), + ); + } + deps.insert(p.name.clone(), serde_json::Value::Object(e)); + } + let lock = json!({ "version": 1, "dependencies": { "net8.0": deps } }); + t.write( + "project/packages.lock.json", + serde_json::to_string_pretty(&lock).unwrap(), + )?; + t.write( + "project/Program.cs", + "System.Console.WriteLine(\"bench\");\n", + )?; + for p in &pkgs { + let id = p.name.to_ascii_lowercase(); + let dir = format!("home/.nuget/packages/{id}/{}", p.version); + t.write( + &format!("{dir}/{id}.nuspec"), + format!("\n\n \n {}\n {}\n bench\n \n\n", p.name, p.version), + )?; + t.write( + &format!("{dir}/{id}.{}.nupkg.sha512", p.version), + &gen::sri_sha512(&format!("nupkg:{}", p.name))[7..], + )?; + t.write(&format!("{dir}/.nupkg.metadata"), "{\n \"version\": 2,\n \"contentHash\": \"x\",\n \"source\": \"https://api.nuget.org/v3/index.json\"\n}")?; + t.write(&format!("{dir}/lib/net8.0/{}.dll", p.name), b"MZ synthetic")?; + } + let patches = pkgs + .iter() + .filter(|p| p.patched) + .map(|p| { + let id = p.name.to_ascii_lowercase(); + let purl = format!("pkg:nuget/{id}@{}", p.version); + let (uuid, token) = grant(&purl); + let url = format!("{PATCH_HOST}/patch-registry/nuget/{token}/{uuid}/flat/{id}/{}/{id}.{}.nupkg", p.version, p.version); + spec( + purl, + uuid.clone(), + &format!("package/lib/net8.0/{}.dll", p.name), + json!({ + "status": "granted", + "url": url, + "purl": format!("pkg:nuget/{}@{}", p.name, p.version), + "artifacts": [{ "kind": "tarball", "url": url, "integrity": { "sha512": gen::sri_sha512(&format!("patched-nupkg:{}", p.name)) } }], + "registryOverride": { + "kind": "nuget-v3", + "indexUrl": format!("{PATCH_HOST}/patch-registry/nuget/{token}/{uuid}/index.json"), + "identifiers": { "name": p.name, "version": p.version, "nugetIdLower": id, "nugetVersionNorm": p.version }, + }, + }), + ) + }) + .collect(); + Ok(fixture( + pkgs.len(), + patches, + &["nuget.config", "packages.lock.json"], + &[], + )) +} + +// ── Maven ────────────────────────────────────────────────────────────── + +pub fn build_maven(t: &mut Tree, size: Size) -> std::io::Result { + let arts = universe( + "maven", + size, + 0.2, + |r, i| { + let g = match r.below(3) { + 0 => format!("org.{}", gen::ident(r, i)), + 1 => format!("com.{}.{}", gen::ident(r, i), gen::ident(r, i + 3)), + _ => format!("io.{}", gen::ident(r, i)), + }; + let a = format!( + "{}-{}", + gen::ident(r, i), + ["core", "api", "client", "common"][r.below(4)] + ); + format!("{g}:{a}") + }, + gen::version, + ); + let ga = |p: &Pkg| -> (String, String) { + let (g, a) = p.name.split_once(':').unwrap(); + (g.to_string(), a.to_string()) + }; + let mut pom = String::from("\n\n 4.0.0\n dev.socket.bench\n bench-app\n 1.0.0\n jar\n\n \n 17\n \n\n \n"); + for p in arts.iter().filter(|p| p.direct) { + let (g, a) = ga(p); + let _ = write!(pom, " \n {g}\n {a}\n {}\n \n", p.version); + } + pom.push_str(" \n\n"); + t.write("project/pom.xml", pom)?; + t.write( + "project/src/main/java/App.java", + "public class App { public static void main(String[] a) {} }\n", + )?; + for p in &arts { + let (g, a) = ga(p); + let dir = format!( + "home/.m2/repository/{}/{a}/{}", + g.replace('.', "/"), + p.version + ); + let mut deps = String::new(); + for &j in &p.deps { + let (dg, da) = ga(&arts[j]); + let _ = write!(deps, " \n {dg}\n {da}\n {}\n \n", arts[j].version); + } + let pom = format!("\n\n 4.0.0\n {g}\n {a}\n {}\n \n{deps} \n\n", p.version); + t.write( + &format!("{dir}/{a}-{}.pom.sha1", p.version), + gen::sha1_hex(&pom), + )?; + t.write(&format!("{dir}/{a}-{}.pom", p.version), pom)?; + t.write(&format!("{dir}/{a}-{}.jar", p.version), b"PK synthetic")?; + t.write( + &format!("{dir}/_remote.repositories"), + format!( + "{a}-{}.jar>central=\n{a}-{}.pom>central=\n", + p.version, p.version + ), + )?; + } + let patches = arts + .iter() + .filter(|p| p.patched) + .map(|p| { + let (g, a) = ga(p); + let purl = format!("pkg:maven/{g}/{a}@{}", p.version); + let (uuid, token) = grant(&purl); + let suffixed = format!("{}-socket.{}", p.version, &uuid[..8]); + let url = format!("{PATCH_HOST}/patch/maven/{g}/{a}/{}/{token}/{uuid}/{a}-{suffixed}.jar", p.version); + spec( + purl.clone(), + uuid.clone(), + &format!("package/{a}.class"), + json!({ + "status": "granted", + "url": url, + "purl": purl, + "artifacts": [{ "kind": "tarball", "url": url, "integrity": { + "sha256": gen::sha256_hex(&format!("patched-jar:{}", p.name)), + "sha1": gen::sha1_hex(&format!("patched-jar:{}", p.name)), + } }], + "registryOverride": { + "kind": "maven2", + "indexUrl": format!("{PATCH_HOST}/patch-registry/maven/{token}/{uuid}/maven2"), + "identifiers": { + "name": format!("{g}/{a}"), + "version": p.version, + "mavenGroupId": g, + "mavenArtifactId": a, + "mavenSuffixedVersion": suffixed, + "mavenPomSha256": gen::sha256_hex(&format!("patched-pom:{}", p.name)), + }, + }, + }), + ) + }) + .collect(); + Ok(fixture( + arts.len(), + patches, + &[ + "pom.xml", + ".mvn/maven.config", + ".mvn/checksums/checksums.sha256", + ], + &[], + )) +} diff --git a/crates/socket-patch-bench/src/fixtures/pypi.rs b/crates/socket-patch-bench/src/fixtures/pypi.rs new file mode 100644 index 000000000..a785f6c3a --- /dev/null +++ b/crates/socket-patch-bench/src/fixtures/pypi.rs @@ -0,0 +1,485 @@ +//! PyPI projects: one installed virtualenv, locked by each Python tool. +//! +//! Every fixture has a project-local `.venv` (as `uv`, `poetry` with +//! in-project venvs, `pipenv` with `PIPENV_VENV_IN_PROJECT` and `pdm` +//! create): without one the crawler falls back to asking `python3` for the +//! machine's global site-packages, which would time the runner's Python +//! install instead of socket-patch. + +use std::fmt::Write as _; +use std::io::Write as _; + +use serde_json::json; + +use super::gen::{self, Rng, Tree}; +use super::{Expect, Fixture, Served, Size, PATCH_HOST}; +use crate::mock::PatchSpec; + +const SITE: &str = "project/.venv/lib/python3.12/site-packages"; + +#[derive(Debug, Clone)] +pub struct Dist { + /// PEP 503-normalized project name (`py-foo12`). + pub name: String, + pub version: String, + pub deps: Vec, + pub direct: bool, + pub patched: bool, +} + +impl Dist { + /// The wheel/dist-info spelling (`py_foo12`). + pub fn dist(&self) -> String { + self.name.replace('-', "_") + } + + pub fn purl(&self) -> String { + format!("pkg:pypi/{}@{}", self.name, self.version) + } + + pub fn wheel(&self) -> String { + format!("{}-{}-py3-none-any.whl", self.dist(), self.version) + } + + pub fn sdist(&self) -> String { + format!("{}-{}.tar.gz", self.dist(), self.version) + } + + pub fn hash(&self, what: &str) -> String { + gen::sha256_hex(&format!("pypi:{what}:{}@{}", self.name, self.version)) + } + + pub fn summary(&self) -> String { + format!("Synthetic distribution {}", self.name) + } +} + +pub fn dists(seed: &str, size: Size) -> Vec { + let mut rng = Rng::new(seed); + let n = size.packages; + let mut out: Vec = (0..n) + .map(|i| Dist { + name: format!("py-{}", gen::ident(&mut rng, i)), + version: gen::version(&mut rng), + deps: Vec::new(), + direct: i < (n / 6).max(1), + patched: size.is_patched(i), + }) + .collect(); + for (i, d) in out.iter_mut().enumerate() { + for _ in 0..rng.below(4) { + if i + 1 < n { + let j = i + 1 + rng.below((n - i - 1).min(80)); + if !d.deps.contains(&j) { + d.deps.push(j); + } + } + } + d.deps.sort_unstable(); + } + out +} + +/// A virtualenv holding every distribution: dist-info (`METADATA`, +/// `RECORD`, `INSTALLER`, `WHEEL`) plus the importable package. +pub fn install_venv(t: &mut Tree, ds: &[Dist]) -> std::io::Result<()> { + t.write( + "project/.venv/pyvenv.cfg", + "home = /usr/bin\ninclude-system-site-packages = false\nversion = 3.12.3\n", + )?; + let mut rng = Rng::new("venv"); + for d in ds { + let di = format!("{SITE}/{}-{}.dist-info", d.dist(), d.version); + let mut meta = format!( + "Metadata-Version: 2.1\nName: {}\nVersion: {}\nSummary: {}\nLicense: MIT\nRequires-Python: >=3.8\n", + d.name, + d.version, + d.summary() + ); + for &j in &d.deps { + let _ = writeln!(meta, "Requires-Dist: {}>={}", ds[j].name, ds[j].version); + } + meta.push_str("\nSynthetic long description.\n"); + t.write(&format!("{di}/METADATA"), meta)?; + t.write(&format!("{di}/INSTALLER"), "uv\n")?; + t.write( + &format!("{di}/WHEEL"), + "Wheel-Version: 1.0\nGenerator: bench\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + )?; + let init = format!( + "\"\"\"{}\"\"\"\n__version__ = \"{}\"\n{}", + d.name, + d.version, + gen::js_source(&d.name, &mut rng).replace("//", "#") + ); + t.write(&format!("{SITE}/{}/__init__.py", d.dist()), &init)?; + t.write( + &format!("{di}/RECORD"), + format!( + "{0}/__init__.py,sha256={1},{2}\n{0}-{3}.dist-info/METADATA,,\n{0}-{3}.dist-info/RECORD,,\n", + d.dist(), + gen::sha256_base64(&init).trim_end_matches('='), + init.len(), + d.version + ), + )?; + } + Ok(()) +} + +/// A minimal but real wheel: uv's rewrite reads the patched wheel's +/// `METADATA`, so the mock serves this and the reference pins its sha256. +pub fn wheel_bytes(d: &Dist) -> Vec { + let mut w = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + let opts = + zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + let di = format!("{}-{}.dist-info", d.dist(), d.version); + w.start_file(format!("{}/__init__.py", d.dist()), opts) + .unwrap(); + w.write_all(b"# patched\n").unwrap(); + w.start_file(format!("{di}/METADATA"), opts).unwrap(); + write!( + w, + "Metadata-Version: 2.1\nName: {}\nVersion: {}\nSummary: patched\n\n", + d.name, d.version + ) + .unwrap(); + w.start_file(format!("{di}/WHEEL"), opts).unwrap(); + w.write_all( + b"Wheel-Version: 1.0\nGenerator: bench\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ) + .unwrap(); + w.start_file(format!("{di}/RECORD"), opts).unwrap(); + w.write_all(b"").unwrap(); + w.finish().unwrap().into_inner() +} + +/// One patch per patched distribution, with the canonical hosted wheel URL +/// `…/patch/pypi/////`. Returns the specs and +/// the wheel bodies the artifact host serves. +pub fn patches(ds: &[Dist]) -> (Vec, Served) { + let mut specs = Vec::new(); + let mut files = Vec::new(); + for d in ds.iter().filter(|d| d.patched) { + let uuid = gen::uuid(&format!("patch:{}", d.purl())); + let token = gen::uuid(&format!("grant:{uuid}")); + let path = format!( + "/patch/pypi/{}/{}/{token}/{uuid}/{}", + d.name, + d.version, + d.wheel() + ); + let url = format!("{PATCH_HOST}{path}"); + let wheel = wheel_bytes(d); + let sha256 = { + use sha2::Digest as _; + hex::encode(sha2::Sha256::digest(&wheel)) + }; + files.push((path, wheel, "application/zip")); + let file = format!("{}/__init__.py", d.dist()); + specs.push(PatchSpec { + purl: d.purl(), + view: super::npm::view_json(&uuid, &d.purl(), &file), + reference: json!({ + "status": "granted", + "url": url, + "purl": d.purl(), + "artifacts": [{ "kind": "tarball", "url": url, "integrity": { "sha256": sha256 } }], + "registryOverride": null, + }), + uuid, + tier: "free", + severity: "high", + }); + } + (specs, files) +} + +fn fixture(ds: &[Dist], rewritten: &[&str], warnings: &[&'static str]) -> Fixture { + let (patches, files) = patches(ds); + Fixture { + project: "project", + expect: Expect { + scanned: ds.len(), + lockfile_only: 0, + redirected: patches.len(), + rewritten: rewritten.iter().map(|s| s.to_string()).collect(), + // The venv still holds the unpatched files after a hosted scan + // (a reinstall picks the patch up), and the wet run says so. + allowed_warnings: [&["redirect_pypi_stale_install"][..], warnings].concat(), + ..Expect::default() + }, + patches, + files, + env_paths: Vec::new(), + env: Vec::new(), + } +} + +fn pyproject(ds: &[Dist], extra: &str) -> String { + let mut s = String::from("[project]\nname = \"bench-app\"\nversion = \"1.0.0\"\nrequires-python = \">=3.9\"\ndependencies = [\n"); + for d in ds.iter().filter(|d| d.direct) { + let _ = writeln!(s, " \"{}=={}\",", d.name, d.version); + } + s.push_str("]\n"); + s.push_str(extra); + s +} + +// ── pip / requirements.txt ───────────────────────────────────────────── + +/// A `pip-compile --generate-hashes` style lock. +pub fn build_requirements(t: &mut Tree, size: Size) -> std::io::Result { + let ds = dists("pip", size); + let mut s = String::from("#\n# This file is autogenerated by pip-compile with Python 3.12\n# by the following command:\n#\n# pip-compile --generate-hashes requirements.in\n#\n"); + for (i, d) in ds.iter().enumerate() { + let _ = write!( + s, + "{}=={} \\\n --hash=sha256:{} \\\n --hash=sha256:{}\n", + d.name, + d.version, + d.hash("whl"), + d.hash("sdist") + ); + let via: Vec<&str> = ds[..i] + .iter() + .filter(|p| p.deps.contains(&i)) + .map(|p| p.name.as_str()) + .take(3) + .collect(); + if via.is_empty() { + s.push_str(" # via -r requirements.in\n"); + } else { + let _ = writeln!(s, " # via {}", via.join(", ")); + } + } + t.write("project/requirements.txt", s)?; + let direct: String = ds + .iter() + .filter(|d| d.direct) + .map(|d| format!("{}\n", d.name)) + .collect(); + t.write("project/requirements.in", direct)?; + install_venv(t, &ds)?; + t.mkdir("home")?; + Ok(fixture(&ds, &["requirements.txt"], &[])) +} + +// ── uv ───────────────────────────────────────────────────────────────── + +pub fn build_uv(t: &mut Tree, size: Size) -> std::io::Result { + let ds = dists("uv", size); + t.write("project/pyproject.toml", pyproject(&ds, ""))?; + let mut s = String::from("version = 1\nrevision = 3\nrequires-python = \">=3.9\"\n\n[[package]]\nname = \"bench-app\"\nversion = \"1.0.0\"\nsource = { virtual = \".\" }\ndependencies = [\n"); + for d in ds.iter().filter(|d| d.direct) { + let _ = writeln!(s, " {{ name = \"{}\" }},", d.name); + } + s.push_str("]\n\n[package.metadata]\nrequires-dist = ["); + let reqs: Vec = ds + .iter() + .filter(|d| d.direct) + .map(|d| { + format!( + "{{ name = \"{}\", specifier = \"=={}\" }}", + d.name, d.version + ) + }) + .collect(); + s.push_str(&reqs.join(", ")); + s.push_str("]\n"); + let mut sorted: Vec<&Dist> = ds.iter().collect(); + sorted.sort_by(|a, b| a.name.cmp(&b.name)); + for d in sorted { + let _ = write!( + s, + "\n[[package]]\nname = \"{}\"\nversion = \"{}\"\nsource = {{ registry = \"https://pypi.org/simple\" }}\n", + d.name, d.version + ); + if !d.deps.is_empty() { + s.push_str("dependencies = [\n"); + for &j in &d.deps { + let _ = writeln!(s, " {{ name = \"{}\" }},", ds[j].name); + } + s.push_str("]\n"); + } + let _ = write!( + s, + "sdist = {{ url = \"https://files.pythonhosted.org/packages/source/{0}/{1}\", hash = \"sha256:{2}\", size = 48213 }}\nwheels = [\n {{ url = \"https://files.pythonhosted.org/packages/py3/{0}/{3}\", hash = \"sha256:{4}\", size = 21877 }},\n]\n", + &d.name[..4], + d.sdist(), + d.hash("sdist"), + d.wheel(), + d.hash("whl") + ); + } + t.write("project/uv.lock", s)?; + install_venv(t, &ds)?; + t.mkdir("home")?; + Ok(fixture(&ds, &["pyproject.toml", "uv.lock"], &[])) +} + +// ── PEP 751 pylock.toml ──────────────────────────────────────────────── + +pub fn build_pylock(t: &mut Tree, size: Size) -> std::io::Result { + let ds = dists("pylock", size); + t.write("project/pyproject.toml", pyproject(&ds, ""))?; + let mut s = + String::from("lock-version = \"1.0\"\ncreated-by = \"uv\"\nrequires-python = \">=3.9\"\n"); + for d in &ds { + let _ = write!( + s, + "\n[[packages]]\nname = \"{0}\"\nversion = \"{1}\"\nindex = \"https://pypi.org/simple\"\nsdist = {{ url = \"https://files.pythonhosted.org/packages/source/{2}\", upload-time = 2024-05-01T00:00:00Z, size = 48213, hashes = {{ sha256 = \"{3}\" }} }}\nwheels = [{{ url = \"https://files.pythonhosted.org/packages/py3/{4}\", upload-time = 2024-05-01T00:00:00Z, size = 21877, hashes = {{ sha256 = \"{5}\" }} }}]\n", + d.name, + d.version, + d.sdist(), + d.hash("sdist"), + d.wheel(), + d.hash("whl") + ); + } + t.write("project/pylock.toml", s)?; + install_venv(t, &ds)?; + t.mkdir("home")?; + Ok(fixture(&ds, &["pylock.toml"], &[])) +} + +// ── poetry ───────────────────────────────────────────────────────────── + +pub fn build_poetry(t: &mut Tree, size: Size) -> std::io::Result { + let ds = dists("poetry", size); + let mut py = String::from("[tool.poetry]\nname = \"bench-app\"\nversion = \"1.0.0\"\ndescription = \"\"\nauthors = []\npackage-mode = false\n\n[tool.poetry.dependencies]\npython = \"^3.9\"\n"); + for d in ds.iter().filter(|d| d.direct) { + let _ = writeln!(py, "{} = \"{}\"", d.name, d.version); + } + py.push_str("\n[build-system]\nrequires = [\"poetry-core>=2.0.0\"]\nbuild-backend = \"poetry.core.masonry.api\"\n"); + t.write("project/pyproject.toml", py)?; + t.write("project/poetry.toml", "[virtualenvs]\nin-project = true\n")?; + let mut s = String::from("# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand.\n"); + let mut sorted: Vec<&Dist> = ds.iter().collect(); + sorted.sort_by(|a, b| a.name.cmp(&b.name)); + for d in sorted { + let _ = write!( + s, + "\n[[package]]\nname = \"{}\"\nversion = \"{}\"\ndescription = \"{}\"\noptional = false\npython-versions = \">=3.8\"\ngroups = [\"main\"]\nfiles = [\n {{file = \"{}\", hash = \"sha256:{}\"}},\n {{file = \"{}\", hash = \"sha256:{}\"}},\n]\n", + d.name, + d.version, + d.summary(), + d.wheel(), + d.hash("whl"), + d.sdist(), + d.hash("sdist") + ); + if !d.deps.is_empty() { + s.push_str("\n[package.dependencies]\n"); + for &j in &d.deps { + let _ = writeln!(s, "{} = \">={}\"", ds[j].name, ds[j].version); + } + } + } + let _ = write!( + s, + "\n[metadata]\nlock-version = \"2.1\"\npython-versions = \"^3.9\"\ncontent-hash = \"{}\"\n", + gen::sha256_hex("poetry-content") + ); + t.write("project/poetry.lock", s)?; + install_venv(t, &ds)?; + t.mkdir("home")?; + Ok(fixture(&ds, &["poetry.lock"], &[])) +} + +// ── pipenv ───────────────────────────────────────────────────────────── + +pub fn build_pipenv(t: &mut Tree, size: Size) -> std::io::Result { + let ds = dists("pipenv", size); + let mut pipfile = String::from("[[source]]\nurl = \"https://pypi.org/simple\"\nverify_ssl = true\nname = \"pypi\"\n\n[packages]\n"); + for d in ds.iter().filter(|d| d.direct) { + let _ = writeln!(pipfile, "{} = \"=={}\"", d.name, d.version); + } + pipfile.push_str("\n[dev-packages]\n\n[requires]\npython_version = \"3.12\"\n"); + t.write("project/Pipfile", pipfile)?; + let mut default = serde_json::Map::new(); + for d in &ds { + default.insert( + d.name.clone(), + json!({ + "hashes": [format!("sha256:{}", d.hash("whl")), format!("sha256:{}", d.hash("sdist"))], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": format!("=={}", d.version), + }), + ); + } + let lock = json!({ + "_meta": { + "hash": { "sha256": gen::sha256_hex("pipfile") }, + "pipfile-spec": 6, + "requires": { "python_version": "3.12" }, + "sources": [{ "name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": true }], + }, + "default": default, + "develop": {}, + }); + t.write("project/Pipfile.lock", pretty4(&lock) + "\n")?; + install_venv(t, &ds)?; + t.mkdir("home")?; + let mut f = fixture(&ds, &["Pipfile.lock"], &[]); + // Names the installer generation instead of spawning `pipenv + // --version` (a Python start-up per scan that would time the runner's + // pipenv, not socket-patch). + f.env.push(("SOCKET_PIPENV_MAJOR", "2026".into())); + Ok(f) +} + +/// JSON with four-space indentation (Pipenv's and Composer's spelling). +pub fn pretty4(v: &serde_json::Value) -> String { + use serde::Serialize as _; + let mut out = Vec::new(); + let fmt = serde_json::ser::PrettyFormatter::with_indent(b" "); + let mut ser = serde_json::Serializer::with_formatter(&mut out, fmt); + v.serialize(&mut ser).unwrap(); + String::from_utf8(out).unwrap() +} + +// ── pdm ──────────────────────────────────────────────────────────────── + +pub fn build_pdm(t: &mut Tree, size: Size) -> std::io::Result { + let ds = dists("pdm", size); + t.write( + "project/pyproject.toml", + pyproject(&ds, "\n[tool.pdm]\ndistribution = false\n"), + )?; + let mut s = format!( + "# This file is @generated by PDM.\n# It is not intended for manual editing.\n\n[metadata]\ngroups = [\"default\"]\nstrategy = [\"inherit_metadata\"]\nlock_version = \"4.5.1\"\ncontent_hash = \"sha256:{}\"\n\n[[metadata.targets]]\nrequires_python = \">=3.9\"\n", + gen::sha256_hex("pdm-content") + ); + let mut sorted: Vec<&Dist> = ds.iter().collect(); + sorted.sort_by(|a, b| a.name.cmp(&b.name)); + for d in sorted { + let _ = write!( + s, + "\n[[package]]\nname = \"{}\"\nversion = \"{}\"\nrequires_python = \">=3.8\"\nsummary = \"{}\"\ngroups = [\"default\"]\n", + d.name, + d.version, + d.summary() + ); + if !d.deps.is_empty() { + s.push_str("dependencies = [\n"); + for &j in &d.deps { + let _ = writeln!(s, " \"{}>={}\",", ds[j].name, ds[j].version); + } + s.push_str("]\n"); + } + let _ = write!( + s, + "files = [\n {{file = \"{}\", hash = \"sha256:{}\"}},\n {{file = \"{}\", hash = \"sha256:{}\"}},\n]\n", + d.wheel(), + d.hash("whl"), + d.sdist(), + d.hash("sdist") + ); + } + t.write("project/pdm.lock", s)?; + install_venv(t, &ds)?; + t.mkdir("home")?; + Ok(fixture(&ds, &["pdm.lock"], &[])) +} diff --git a/crates/socket-patch-bench/src/main.rs b/crates/socket-patch-bench/src/main.rs new file mode 100644 index 000000000..441f4bb6a --- /dev/null +++ b/crates/socket-patch-bench/src/main.rs @@ -0,0 +1,623 @@ +//! `socket-patch-bench`: benchmark `socket-patch scan` on synthetic +//! projects against a local patch API, and compare two builds. +//! +//! ```text +//! socket-patch-bench list +//! socket-patch-bench run --bin target/perf/socket-patch +//! socket-patch-bench compare --base /tmp/base/socket-patch --head target/perf/socket-patch +//! socket-patch-bench serve npm/hosted --bin target/perf/socket-patch +//! ``` +//! +//! See README.md for what is measured and how a regression is decided. + +mod engine; +mod fixtures; +mod mock; +mod process; +mod report; +mod scenarios; +mod stats; +mod tree; + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use clap::{Args, Parser, Subcommand}; + +use engine::{Binary, Options}; +use report::{Report, ScenarioComparison, ScenarioResult}; +use stats::{Gate, Verdict}; + +/// The org slug every authenticated scenario uses. +pub const ORG: &str = "bench-org"; + +#[derive(Parser)] +#[command(name = "socket-patch-bench", about = "Benchmark `socket-patch scan`")] +struct Cli { + #[command(subcommand)] + command: Cmd, +} + +#[derive(Subcommand)] +enum Cmd { + /// List the scenarios. + List, + /// Time one binary on every selected scenario. + Run { + /// The `socket-patch` binary to measure. + #[arg(long)] + bin: PathBuf, + #[command(flatten)] + common: Common, + }, + /// Compare two binaries, interleaved on the same machine, and exit 1 on + /// a regression or a failed validation. + Compare { + /// The baseline binary (e.g. built from the PR's base commit). + #[arg(long)] + base: PathBuf, + /// The candidate binary. + #[arg(long)] + head: PathBuf, + /// Relative slowdown that counts as a regression (0.10 = 10%). + #[arg(long, default_value_t = 0.10)] + threshold: f64, + /// Relative peak-RSS growth that counts as a regression. + #[arg(long, default_value_t = 0.15)] + rss_threshold: f64, + /// Extra pairs to run for a scenario that first looks regressed, + /// before the verdict stands (0 disables the confirmation round). + #[arg(long, default_value_t = 10)] + confirm_runs: usize, + /// Report regressions but exit 0. + #[arg(long)] + no_fail: bool, + #[command(flatten)] + common: Common, + }, + /// Build one scenario's fixture, start its mock API, print the command + /// that scans it, and wait (for profiling a run by hand). + Serve { + /// Scenario name (see `list`). + scenario: String, + /// Binary named in the printed command. + #[arg(long, default_value = "target/perf/socket-patch")] + bin: PathBuf, + #[arg(long, default_value_t = 1.0)] + scale: f64, + /// Where to build the fixture (default: a temp dir). + #[arg(long)] + work_dir: Option, + }, +} + +#[derive(Args)] +struct Common { + /// Only scenarios whose name matches this regex (repeatable). + #[arg(long = "filter", short = 'f')] + filters: Vec, + /// Untimed runs per binary before measuring (warm the page cache). + #[arg(long, default_value_t = 2)] + warmup: usize, + /// Measured runs per binary (`compare` pairs one base run with one + /// head run). + #[arg(long, default_value_t = 15)] + runs: usize, + /// Multiply every fixture's package count. + #[arg(long, default_value_t = 1.0)] + scale: f64, + /// Where fixtures are built (default: a temp dir, removed at exit). + #[arg(long)] + work_dir: Option, + /// Write `results.json` and `summary.md` here. + #[arg(long)] + out: Option, + /// Print every run. + #[arg(long, short)] + verbose: bool, +} + +fn select(filters: &[String]) -> Result, String> { + let res: Vec = filters + .iter() + .map(|f| regex::Regex::new(f).map_err(|e| format!("--filter {f}: {e}"))) + .collect::>()?; + let all = scenarios::all(); + let picked: Vec<_> = all + .into_iter() + .filter(|s| res.is_empty() || res.iter().any(|r| r.is_match(&s.name))) + .collect(); + if picked.is_empty() { + return Err("no scenario matches the filters (see `socket-patch-bench list`)".into()); + } + Ok(picked) +} + +fn absolute(p: &PathBuf) -> Result { + std::fs::canonicalize(p).map_err(|e| format!("{}: {e}", p.display())) +} + +struct WorkDir { + path: PathBuf, + _temp: Option, +} + +fn work_dir(given: Option<&PathBuf>) -> Result { + match given { + Some(p) => { + std::fs::create_dir_all(p).map_err(|e| e.to_string())?; + Ok(WorkDir { + path: absolute(p)?, + _temp: None, + }) + } + None => { + let t = tempfile::Builder::new() + .prefix("socket-patch-bench-") + .tempdir() + .map_err(|e| e.to_string())?; + Ok(WorkDir { + path: absolute(&t.path().to_path_buf())?, + _temp: Some(t), + }) + } + } +} + +fn log(line: &str) { + eprintln!("{line}"); +} + +fn write_outputs(report: &Report, out: Option<&PathBuf>) -> Result<(), String> { + let md = report.markdown(); + println!("{md}"); + if let Some(dir) = out { + std::fs::create_dir_all(dir).map_err(|e| e.to_string())?; + let json = serde_json::to_string_pretty(report).map_err(|e| e.to_string())?; + std::fs::write(dir.join("results.json"), json).map_err(|e| e.to_string())?; + std::fs::write(dir.join("summary.md"), md).map_err(|e| e.to_string())?; + } + Ok(()) +} + +fn compare_result(s: &ScenarioResult, gate: Gate, rss_gate: Gate) -> Option { + let base = s.binaries.get("base")?; + let head = s.binaries.get("head")?; + if base.invalid.is_some() || head.invalid.is_some() || base.samples.len() != head.samples.len() + { + return None; + } + let wall = stats::compare_paired(&base.wall(), &head.wall(), gate); + let cpu = match (base.cpu(), head.cpu()) { + (Some(b), Some(h)) => Some(stats::compare_paired(&b, &h, gate)), + _ => None, + }; + let rss = match (base.rss(), head.rss()) { + (Some(b), Some(h)) => Some(stats::compare_paired(&b, &h, rss_gate)), + _ => None, + }; + let request_delta = head.total_requests() as i64 - base.total_requests() as i64; + let mut reasons = Vec::new(); + if wall.verdict == Verdict::Regression { + reasons.push(format!("wall time {:+.1}%", (wall.ratio - 1.0) * 100.0)); + } + if let Some(c) = cpu.as_ref().filter(|c| c.verdict == Verdict::Regression) { + reasons.push(format!("CPU time {:+.1}%", (c.ratio - 1.0) * 100.0)); + } + if let Some(c) = rss.as_ref().filter(|c| c.verdict == Verdict::Regression) { + reasons.push(format!("peak RSS {:+.1}%", (c.ratio - 1.0) * 100.0)); + } + if request_delta > 0 { + let mut by_kind = Vec::new(); + let kinds: std::collections::BTreeSet<&String> = + base.requests.keys().chain(head.requests.keys()).collect(); + for k in kinds { + let b = base.requests.get(k).copied().unwrap_or(0); + let h = head.requests.get(k).copied().unwrap_or(0); + if b != h { + by_kind.push(format!("{k} {b}→{h}")); + } + } + reasons.push(format!( + "{request_delta:+} API requests ({})", + by_kind.join(", ") + )); + } + let verdict = if !reasons.is_empty() { + Verdict::Regression + } else if wall.verdict == Verdict::Improvement { + Verdict::Improvement + } else if wall.verdict == Verdict::Inconclusive { + Verdict::Inconclusive + } else { + Verdict::Unchanged + }; + Some(ScenarioComparison { + wall, + cpu, + rss, + request_delta, + verdict, + reasons, + }) +} + +fn main_inner() -> Result { + let cli = Cli::parse(); + match cli.command { + Cmd::List => { + for s in scenarios::all() { + let size = s.size(1.0); + println!( + "{:<22} {:>5} pkgs {:>3} patched {}", + s.name, + size.packages, + size.patched, + s.description() + ); + } + Ok(0) + } + Cmd::Run { bin, common } => { + let picked = select(&common.filters)?; + let work = work_dir(common.work_dir.as_ref())?; + let opts = Options { + warmup: common.warmup, + runs: common.runs, + scale: common.scale, + work: work.path.clone(), + verbose: common.verbose, + }; + let bins = [Binary { + label: "bin".into(), + path: absolute(&bin)?, + }]; + let mut results = Vec::new(); + for s in &picked { + results.push(engine::run_scenario(s, &bins, &opts, &mut log)?); + } + let report = Report { + schema: 1, + mode: "run", + scale: common.scale, + threshold: None, + binaries: BTreeMap::from([("bin".to_string(), bins[0].path.display().to_string())]), + scenarios: results, + }; + write_outputs(&report, common.out.as_ref())?; + Ok(if report.invalid().is_empty() { 0 } else { 1 }) + } + Cmd::Compare { + base, + head, + threshold, + rss_threshold, + confirm_runs, + no_fail, + common, + } => { + let picked = select(&common.filters)?; + let work = work_dir(common.work_dir.as_ref())?; + let mut opts = Options { + warmup: common.warmup, + runs: common.runs, + scale: common.scale, + work: work.path.clone(), + verbose: common.verbose, + }; + let bins = [ + Binary { + label: "base".into(), + path: absolute(&base)?, + }, + Binary { + label: "head".into(), + path: absolute(&head)?, + }, + ]; + let gate = Gate { + threshold, + min_abs_delta: 3.0, + confidence: 0.95, + }; + let rss_gate = Gate { + threshold: rss_threshold, + min_abs_delta: 1024.0, + confidence: 0.95, + }; + let mut results = Vec::new(); + for s in &picked { + let mut r = engine::run_scenario(s, &bins, &opts, &mut log)?; + r.comparison = compare_result(&r, gate, rss_gate); + // A first-round regression gets a second round, and the + // verdict is taken on all pairs together: one noisy burst + // on a shared runner must not fail a PR on its own. + let timing_only = r + .comparison + .as_ref() + .is_some_and(|c| c.verdict == Verdict::Regression && c.request_delta <= 0); + if timing_only && confirm_runs > 0 { + log(&format!( + " {}: looks regressed; confirming with {confirm_runs} more pairs", + s.name + )); + opts.runs = confirm_runs; + let more = engine::run_scenario( + s, + &bins, + &Options { + warmup: 1, + ..opts.clone() + }, + &mut log, + )?; + opts.runs = common.runs; + for (label, extra) in more.binaries { + let entry = r.binaries.get_mut(&label).unwrap(); + if entry.invalid.is_none() { + entry.invalid = extra.invalid; + } + entry.samples.extend(extra.samples); + } + r.comparison = compare_result(&r, gate, rss_gate); + } + if let Some(c) = &r.comparison { + log(&format!( + " {}: wall {:+.1}% (base {:.1} ms, head {:.1} ms) → {:?}", + s.name, + (c.wall.ratio - 1.0) * 100.0, + c.wall.base_median, + c.wall.head_median, + c.verdict + )); + } + results.push(r); + } + let report = Report { + schema: 1, + mode: "compare", + scale: common.scale, + threshold: Some(threshold), + binaries: bins + .iter() + .map(|b| (b.label.clone(), b.path.display().to_string())) + .collect(), + scenarios: results, + }; + write_outputs(&report, common.out.as_ref())?; + // A head that fails validation is a broken scan (or a scenario + // the PR must update); a base that fails it only loses that + // scenario's comparison. + let head_invalid = report.invalid().iter().any(|(_, bin, _)| *bin == "head"); + let regressed = !report.regressions().is_empty(); + if head_invalid { + log("FAIL: the head binary failed scenario validation (see summary)"); + } + if regressed { + log(&format!( + "{}: {} scenario(s) regressed", + if no_fail { "WARN" } else { "FAIL" }, + report.regressions().len() + )); + } + Ok(if head_invalid || (regressed && !no_fail) { + 1 + } else { + 0 + }) + } + Cmd::Serve { + scenario, + bin, + scale, + work_dir: given, + } => { + let all = scenarios::all(); + let s = all + .iter() + .find(|s| s.name == scenario) + .ok_or_else(|| format!("no scenario named {scenario} (see `list`)"))?; + let work = work_dir(given.as_ref())?; + let opts = Options { + warmup: 0, + runs: 0, + scale, + work: work.path.clone(), + verbose: false, + }; + let p = engine::Prepared::new(s, &opts)?; + let cmd = p.command(&bin, s.kind == scenarios::Kind::DryRun); + println!("# {}", s.description()); + println!("# fixture: {}", p.project_dir().display()); + println!("# the mock API serves until Ctrl-C; a wet scan edits the fixture in place."); + print!("env -i"); + for (k, v) in cmd.get_envs() { + if let Some(v) = v { + print!( + " {}={}", + k.to_string_lossy(), + shell_quote(&v.to_string_lossy()) + ); + } + } + print!(" {}", shell_quote(&cmd.get_program().to_string_lossy())); + for a in cmd.get_args() { + print!(" {}", shell_quote(&a.to_string_lossy())); + } + println!(); + // Serve until interrupted (Ctrl-C ends the process). + loop { + std::thread::park(); + } + } + } +} + +fn shell_quote(s: &str) -> String { + if s.chars() + .all(|c| c.is_ascii_alphanumeric() || "/._-=:,+@".contains(c)) + { + s.to_string() + } else { + format!("'{}'", s.replace('\'', "'\\''")) + } +} + +fn main() { + let code = match main_inner() { + Ok(code) => code, + Err(e) => { + eprintln!("error: {e}"); + 2 + } + }; + std::process::exit(code); +} + +#[cfg(test)] +mod tests { + use super::*; + use report::{BinaryResult, Sample}; + + const GATE: Gate = Gate { + threshold: 0.10, + min_abs_delta: 3.0, + confidence: 0.95, + }; + const RSS_GATE: Gate = Gate { + threshold: 0.15, + min_abs_delta: 1024.0, + confidence: 0.95, + }; + + fn binary(wall: &[f64], rss_kib: u64, requests: u64) -> BinaryResult { + BinaryResult { + samples: wall + .iter() + .map(|w| Sample { + wall_ms: *w, + cpu_ms: Some(*w * 0.9), + max_rss_kib: Some(rss_kib), + }) + .collect(), + requests: BTreeMap::from([("batch".to_string(), requests)]), + max_inflight: 1, + invalid: None, + } + } + + fn scenario(base: BinaryResult, head: BinaryResult) -> ScenarioResult { + ScenarioResult { + name: "npm/hosted".into(), + description: String::new(), + packages: 1, + patched: 1, + args: Vec::new(), + latency_ms: 0, + binaries: BTreeMap::from([("base".to_string(), base), ("head".to_string(), head)]), + comparison: None, + } + } + + fn noisy(center: f64) -> Vec { + (0..15).map(|i| center + f64::from(i % 5) - 2.0).collect() + } + + #[test] + fn identical_timings_are_unchanged() { + let s = scenario( + binary(&noisy(200.0), 40_000, 6), + binary(&noisy(200.0), 40_000, 6), + ); + let c = compare_result(&s, GATE, RSS_GATE).unwrap(); + assert_eq!(c.verdict, Verdict::Unchanged); + assert!(c.reasons.is_empty()); + } + + #[test] + fn a_slower_head_regresses_on_wall_and_cpu() { + let s = scenario( + binary(&noisy(200.0), 40_000, 6), + binary(&noisy(260.0), 40_000, 6), + ); + let c = compare_result(&s, GATE, RSS_GATE).unwrap(); + assert_eq!(c.verdict, Verdict::Regression); + assert!( + c.reasons.iter().any(|r| r.starts_with("wall time +")), + "{:?}", + c.reasons + ); + assert!( + c.reasons.iter().any(|r| r.starts_with("CPU time +")), + "{:?}", + c.reasons + ); + } + + #[test] + fn more_requests_regress_even_when_timings_hold() { + let s = scenario( + binary(&noisy(200.0), 40_000, 6), + binary(&noisy(200.0), 40_000, 9), + ); + let c = compare_result(&s, GATE, RSS_GATE).unwrap(); + assert_eq!(c.verdict, Verdict::Regression); + assert_eq!(c.request_delta, 3); + assert_eq!(c.reasons, vec!["+3 API requests (batch 6→9)".to_string()]); + } + + #[test] + fn memory_growth_past_its_threshold_regresses() { + let s = scenario( + binary(&noisy(200.0), 40_000, 6), + binary(&noisy(200.0), 52_000, 6), + ); + let c = compare_result(&s, GATE, RSS_GATE).unwrap(); + assert_eq!(c.verdict, Verdict::Regression); + assert!(c.reasons[0].starts_with("peak RSS +"), "{:?}", c.reasons); + } + + #[test] + fn a_faster_head_is_an_improvement() { + let s = scenario( + binary(&noisy(200.0), 40_000, 6), + binary(&noisy(150.0), 40_000, 5), + ); + assert_eq!( + compare_result(&s, GATE, RSS_GATE).unwrap().verdict, + Verdict::Improvement + ); + } + + #[test] + fn an_invalid_side_has_no_comparison() { + let mut head = binary(&noisy(200.0), 40_000, 6); + head.invalid = Some("scannedPackages: got 1, want 2".into()); + assert!(compare_result( + &scenario(binary(&noisy(200.0), 40_000, 6), head), + GATE, + RSS_GATE + ) + .is_none()); + } + + #[test] + fn every_scenario_has_a_unique_filesystem_safe_name() { + let all = scenarios::all(); + let mut slugs: Vec = all.iter().map(|s| s.slug()).collect(); + slugs.sort(); + slugs.dedup(); + assert_eq!(slugs.len(), all.len()); + assert!(slugs.iter().all(|s| !s.contains('/'))); + // Every package manager runs a fresh and an already-redirected scan. + for pm in scenarios::package_managers() { + for kind in ["hosted", "rescan"] { + assert!( + all.iter().any(|s| s.name == format!("{}/{kind}", pm.name)), + "{} {kind}", + pm.name + ); + } + } + } +} diff --git a/crates/socket-patch-bench/src/mock.rs b/crates/socket-patch-bench/src/mock.rs new file mode 100644 index 000000000..f8ba757cc --- /dev/null +++ b/crates/socket-patch-bench/src/mock.rs @@ -0,0 +1,609 @@ +//! The local patch API every benchmark run talks to. +//! +//! One responder stands in for all three Socket hosts the CLI can reach +//! (`api.socket.dev`, the public proxy `patches-api.socket.dev`, and the +//! artifact host `patch.socket.dev`), each on its own port so connection +//! reuse behaves as it does against the real, separate hosts. Answers are +//! computed from the scenario's catalog, so a CLI that changes batch +//! sizes, chunking or request order still gets the same data. +//! +//! It is a deliberately small HTTP/1.1 server on std threads, one thread +//! per connection: the CLI's client speaks HTTP/1.1 only, so concurrent +//! requests arrive on concurrent connections, and a simulated latency +//! sleeps on each in parallel like a real network would. Nothing here +//! shares a runtime with the harness, so starting and stopping a server +//! cannot stall a run. +//! +//! Every request is classified and counted. A request the mock does not +//! recognize is answered `599` and recorded as unexpected, which fails the +//! run's validation: the benchmark must never silently measure an error +//! path because the CLI started calling something new. + +use std::collections::{BTreeMap, HashMap}; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::Duration; + +use serde_json::{json, Value}; + +/// One patch the catalog serves. +#[derive(Debug, Clone)] +pub struct PatchSpec { + /// The purl exactly as the CLI sends it in a batch request. + pub purl: String, + pub uuid: String, + pub tier: &'static str, + pub severity: &'static str, + /// The `POST .../patches/package` result for this uuid + /// (a `PackageVendorResult`). + pub reference: Value, + /// The `GET .../patches/view/` body (a `PatchResponse`). + pub view: Value, +} + +#[derive(Debug, Default)] +pub struct Catalog { + /// Batch answers (`BatchPatchInfo`) per purl. + by_purl: HashMap>, + /// Detail answers (`PatchSearchResult`) per purl. + search: HashMap>, + references: HashMap, + views: HashMap, + /// Raw bodies served by path on the artifact host (wheels, tarballs). + files: HashMap, &'static str)>, + pub can_access_paid: bool, +} + +impl Catalog { + pub fn new(patches: &[PatchSpec], can_access_paid: bool) -> Self { + let mut c = Catalog { + can_access_paid, + ..Default::default() + }; + for (i, p) in patches.iter().enumerate() { + let cve = format!("CVE-2024-{}", 10000 + i); + let ghsa = format!("GHSA-bnch-{:04x}-{:04x}", i / 0x10000, i % 0x10000); + c.by_purl.entry(p.purl.clone()).or_default().push(json!({ + "uuid": p.uuid, + "purl": p.purl, + "tier": p.tier, + "cveIds": [cve], + "ghsaIds": [ghsa], + "severity": p.severity, + "title": format!("Synthetic patch for {}", p.purl), + "publishedAt": "2024-06-01T00:00:00Z", + })); + c.search.entry(p.purl.clone()).or_default().push(json!({ + "uuid": p.uuid, + "purl": p.purl, + "publishedAt": "2024-06-01T00:00:00Z", + "description": format!("Synthetic patch for {}", p.purl), + "license": "MIT", + "tier": p.tier, + "vulnerabilities": { + ghsa: { + "cves": [cve], + "summary": "synthetic vulnerability", + "severity": p.severity, + "description": "synthetic vulnerability", + } + }, + })); + c.references.insert(p.uuid.clone(), p.reference.clone()); + c.views.insert(p.uuid.clone(), p.view.clone()); + } + c + } + + pub fn serve_file(&mut self, path: &str, body: Vec, content_type: &'static str) { + self.files.insert(path.to_string(), (body, content_type)); + } +} + +/// Request counts for one run. +#[derive(Debug, Default, Clone)] +pub struct Stats { + pub by_kind: BTreeMap, + pub unexpected: Vec, + /// Most requests being answered at once. + pub max_inflight: usize, + inflight: usize, +} + +impl Stats { + pub fn total(&self) -> u64 { + self.by_kind.values().sum() + } +} + +/// The routes, in one place, so classification and answers cannot drift. +#[derive(Debug, PartialEq, Eq)] +enum Route<'a> { + Batch, + References, + View(&'a str), + ByPackage(&'a str), + Organizations, + File(&'a str), +} + +fn route<'a>(method: &str, path: &'a str) -> Option> { + // Authenticated `/v0/orgs//patches/` and proxy `/patch/`. + let rest = path + .strip_prefix("/v0/orgs/") + .and_then(|p| p.split_once("/patches/").map(|(_, r)| r)) + .or_else(|| path.strip_prefix("/patch/")); + match (method, rest) { + ("POST", Some("batch")) => Some(Route::Batch), + ("POST", Some("package")) => Some(Route::References), + ("GET", Some(r)) if r.starts_with("view/") => Some(Route::View(&r[5..])), + ("GET", Some(r)) if r.starts_with("by-package/") => Some(Route::ByPackage(&r[11..])), + ("GET", None) if path == "/v0/organizations" => Some(Route::Organizations), + _ => None, + } +} + +impl Route<'_> { + fn kind(&self) -> &'static str { + match self { + Route::Batch => "batch", + Route::References => "references", + Route::View(_) => "view", + Route::ByPackage(_) => "by-package", + Route::Organizations => "organizations", + Route::File(_) => "artifact", + } + } +} + +fn percent_decode(s: &str) -> String { + let b = s.as_bytes(); + let mut out = Vec::with_capacity(b.len()); + let mut i = 0; + while i < b.len() { + if b[i] == b'%' && i + 3 <= b.len() { + if let Ok(v) = u8::from_str_radix(&s[i + 1..i + 3], 16) { + out.push(v); + i += 3; + continue; + } + } + out.push(b[i]); + i += 1; + } + String::from_utf8_lossy(&out).into_owned() +} + +struct Response { + status: u16, + content_type: &'static str, + body: Vec, +} + +impl Response { + fn json(v: Value) -> Self { + Self { + status: 200, + content_type: "application/json", + body: serde_json::to_vec(&v).unwrap(), + } + } +} + +/// Answer one request from the catalog (`None`: not something the mock +/// serves). +fn answer(c: &Catalog, route: &Route<'_>, body: &[u8]) -> Option { + Some(match route { + Route::Batch => { + let body: Value = serde_json::from_slice(body).ok()?; + let mut packages = Vec::new(); + for comp in body["components"].as_array()? { + let purl = comp["purl"].as_str()?; + if let Some(patches) = c.by_purl.get(purl) { + packages.push(json!({ "purl": purl, "patches": patches })); + } + } + Response::json( + json!({ "packages": packages, "canAccessPaidPatches": c.can_access_paid }), + ) + } + Route::References => { + let body: Value = serde_json::from_slice(body).ok()?; + let mut results = serde_json::Map::new(); + for uuid in body["uuids"].as_array()? { + let uuid = uuid.as_str()?; + let r = c + .references + .get(uuid) + .cloned() + .unwrap_or_else(|| json!({ "status": "not_found" })); + results.insert(uuid.to_string(), r); + } + Response::json(json!({ "results": results })) + } + Route::View(uuid) => Response::json(c.views.get(*uuid)?.clone()), + Route::ByPackage(encoded) => Response::json(json!({ + "patches": c.search.get(&percent_decode(encoded)).cloned().unwrap_or_default(), + "canAccessPaidPatches": c.can_access_paid, + })), + Route::Organizations => Response::json(json!({ + "organizations": { "bench": { + "id": "bench", "name": null, "image": null, "plan": "enterprise", "slug": crate::ORG, + } } + })), + Route::File(path) => { + let (body, ty) = c.files.get(*path)?; + Response { + status: 200, + content_type: ty, + body: body.clone(), + } + } + }) +} + +struct Shared { + catalog: RwLock, + stats: Mutex, + delay: Duration, + stop: AtomicBool, +} + +/// One parsed request. +struct Request { + method: String, + path: String, + body: Vec, + close: bool, +} + +fn read_request(r: &mut BufReader) -> std::io::Result> { + let mut line = String::new(); + if r.read_line(&mut line)? == 0 { + return Ok(None); + } + let mut parts = line.split_whitespace(); + let (Some(method), Some(target)) = (parts.next(), parts.next()) else { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "bad request line", + )); + }; + let (method, target) = (method.to_string(), target.to_string()); + let mut len = 0usize; + let mut chunked = false; + let mut close = false; + loop { + let mut h = String::new(); + if r.read_line(&mut h)? == 0 { + return Ok(None); + } + let h = h.trim_end(); + if h.is_empty() { + break; + } + if let Some((k, v)) = h.split_once(':') { + let v = v.trim(); + match k.trim().to_ascii_lowercase().as_str() { + "content-length" => len = v.parse().unwrap_or(0), + "transfer-encoding" => chunked = v.to_ascii_lowercase().contains("chunked"), + "connection" => close = v.eq_ignore_ascii_case("close"), + _ => {} + } + } + } + let mut body = Vec::new(); + if chunked { + loop { + let mut size = String::new(); + r.read_line(&mut size)?; + let hex = size.trim().split(';').next().unwrap_or("0"); + let n = usize::from_str_radix(hex, 16).unwrap_or(0); + let mut chunk = vec![0; n + 2]; + r.read_exact(&mut chunk)?; + if n == 0 { + break; + } + body.extend_from_slice(&chunk[..n]); + } + } else if len > 0 { + body.resize(len, 0); + r.read_exact(&mut body)?; + } + // Only the path routes; a query string never selects another answer. + let path = target.split('?').next().unwrap_or("").to_string(); + Ok(Some(Request { + method, + path, + body, + close, + })) +} + +fn respond(shared: &Shared, req: &Request) -> Response { + let catalog = shared.catalog.read().unwrap(); + // Artifact paths share the proxy's `/patch/` prefix: a served file + // wins over the API routes. + let routed = if req.method == "GET" && catalog.files.contains_key(&req.path) { + Some(Route::File(&req.path)) + } else { + route(&req.method, &req.path) + }; + let answered = routed.as_ref().and_then(|r| answer(&catalog, r, &req.body)); + let mut stats = shared.stats.lock().unwrap(); + match (routed, answered) { + (Some(r), Some(resp)) => { + *stats.by_kind.entry(r.kind().to_string()).or_default() += 1; + resp + } + _ => { + stats + .unexpected + .push(format!("{} {}", req.method, req.path)); + Response { + status: 599, + content_type: "text/plain", + body: b"socket-patch-bench: unexpected request".to_vec(), + } + } + } +} + +fn serve_connection(stream: TcpStream, shared: Arc) { + let _ = stream.set_nodelay(true); + let _ = stream.set_read_timeout(Some(Duration::from_secs(120))); + let Ok(mut writer) = stream.try_clone() else { + return; + }; + let mut reader = BufReader::new(stream); + while let Ok(Some(req)) = read_request(&mut reader) { + { + let mut stats = shared.stats.lock().unwrap(); + stats.inflight += 1; + stats.max_inflight = stats.max_inflight.max(stats.inflight); + } + let resp = respond(&shared, &req); + if !shared.delay.is_zero() { + std::thread::sleep(shared.delay); + } + let reason = if resp.status == 200 { + "OK" + } else { + "Unexpected" + }; + let head = format!( + "HTTP/1.1 {} {reason}\r\nContent-Type: {}\r\nContent-Length: {}\r\n{}\r\n", + resp.status, + resp.content_type, + resp.body.len(), + if req.close { + "Connection: close\r\n" + } else { + "" + } + ); + let wrote = writer + .write_all(head.as_bytes()) + .and_then(|_| writer.write_all(&resp.body)) + .and_then(|_| writer.flush()); + { + let mut stats = shared.stats.lock().unwrap(); + stats.inflight = stats.inflight.saturating_sub(1); + } + if wrote.is_err() || req.close { + return; + } + } +} + +/// One listening host. +pub struct Server { + addr: SocketAddr, +} + +impl Server { + pub fn uri(&self) -> String { + format!("http://{}", self.addr) + } +} + +fn listen(shared: Arc) -> std::io::Result { + let listener = TcpListener::bind("127.0.0.1:0")?; + let addr = listener.local_addr()?; + std::thread::spawn(move || { + for stream in listener.incoming() { + if shared.stop.load(Ordering::Relaxed) { + break; + } + if let Ok(stream) = stream { + let shared = shared.clone(); + std::thread::spawn(move || serve_connection(stream, shared)); + } + } + }); + Ok(Server { addr }) +} + +/// The three stand-in hosts. +pub struct MockApi { + pub api: Server, + pub proxy: Server, + pub patch: Server, + shared: Arc, +} + +impl MockApi { + /// Start the three hosts on an empty catalog (artifact URLs embed the + /// artifact host's port, so the catalog is built after this and handed + /// over with [`Self::set_catalog`]). + pub fn start(delay: Duration) -> std::io::Result { + let shared = Arc::new(Shared { + catalog: RwLock::new(Catalog::default()), + stats: Mutex::new(Stats::default()), + delay, + stop: AtomicBool::new(false), + }); + Ok(Self { + api: listen(shared.clone())?, + proxy: listen(shared.clone())?, + patch: listen(shared.clone())?, + shared, + }) + } + + pub fn set_catalog(&self, catalog: Catalog) { + *self.shared.catalog.write().unwrap() = catalog; + } + + /// Take the counts since the last call. + pub fn take_stats(&self) -> Stats { + std::mem::take(&mut *self.shared.stats.lock().unwrap()) + } +} + +impl Drop for MockApi { + fn drop(&mut self) { + // Wake each accept loop so its thread exits; connection threads end + // when the (already exited) CLI's sockets read EOF. + self.shared.stop.store(true, Ordering::Relaxed); + for s in [&self.api, &self.proxy, &self.patch] { + let _ = TcpStream::connect(s.addr); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn routes_cover_authenticated_and_proxy_paths() { + assert_eq!( + route("POST", "/v0/orgs/acme/patches/batch"), + Some(Route::Batch) + ); + assert_eq!(route("POST", "/patch/batch"), Some(Route::Batch)); + assert_eq!( + route("POST", "/v0/orgs/acme/patches/package"), + Some(Route::References) + ); + assert_eq!(route("GET", "/patch/view/abc"), Some(Route::View("abc"))); + assert_eq!( + route("GET", "/v0/organizations"), + Some(Route::Organizations) + ); + assert_eq!( + route( + "GET", + "/v0/orgs/acme/patches/by-package/pkg%3Anpm%2F%40s%2Fa%401.0.0" + ), + Some(Route::ByPackage("pkg%3Anpm%2F%40s%2Fa%401.0.0")) + ); + assert_eq!(route("DELETE", "/patch/batch"), None); + assert_eq!(route("GET", "/somewhere/else"), None); + } + + #[test] + fn percent_decoding_matches_the_cli_encoding() { + assert_eq!( + percent_decode("pkg%3Anpm%2F%40s%2Fa%401.0.0"), + "pkg:npm/@s/a@1.0.0" + ); + assert_eq!(percent_decode("100%"), "100%"); + assert_eq!(percent_decode("a%2"), "a%2"); + } + + fn http(uri: &str, req: &str) -> String { + let mut s = TcpStream::connect(uri.trim_start_matches("http://")).unwrap(); + s.write_all(req.as_bytes()).unwrap(); + let mut out = String::new(); + s.read_to_string(&mut out).unwrap(); + out + } + + #[test] + fn serves_every_route_and_counts_it() { + let spec = PatchSpec { + purl: "pkg:npm/@s/a@1.0.0".into(), + uuid: "u-1".into(), + tier: "free", + severity: "high", + reference: json!({ "status": "granted", "url": "x" }), + view: json!({ "uuid": "u-1" }), + }; + let mock = MockApi::start(Duration::ZERO).unwrap(); + let mut cat = Catalog::new(&[spec], false); + cat.serve_file( + "/patch/npm/a.tgz", + b"bytes".to_vec(), + "application/octet-stream", + ); + mock.set_catalog(cat); + + let batch = r#"{"components":[{"purl":"pkg:npm/@s/a@1.0.0"},{"purl":"pkg:npm/b@1.0.0"}]}"#; + let out = http( + &mock.api.uri(), + &format!( + "POST /v0/orgs/o/patches/batch HTTP/1.1\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{batch}", + batch.len() + ), + ); + assert!(out.starts_with("HTTP/1.1 200"), "{out}"); + assert!( + out.contains("\"uuid\":\"u-1\"") && !out.contains("pkg:npm/b@"), + "{out}" + ); + + let out = http( + &mock.proxy.uri(), + "GET /patch/by-package/pkg%3Anpm%2F%40s%2Fa%401.0.0 HTTP/1.1\r\nConnection: close\r\n\r\n", + ); + assert!(out.contains("\"vulnerabilities\""), "{out}"); + + let refs = r#"{"uuids":["u-1","u-2"]}"#; + let out = http( + &mock.api.uri(), + &format!( + "POST /patch/package HTTP/1.1\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n{:x}\r\n{refs}\r\n0\r\n\r\n", + refs.len() + ), + ); + assert!( + out.contains("\"granted\"") && out.contains("\"not_found\""), + "{out}" + ); + + let out = http( + &mock.patch.uri(), + "GET /patch/npm/a.tgz HTTP/1.1\r\nConnection: close\r\n\r\n", + ); + assert!(out.ends_with("bytes"), "{out}"); + + let out = http( + &mock.patch.uri(), + "GET /nope HTTP/1.1\r\nConnection: close\r\n\r\n", + ); + assert!(out.starts_with("HTTP/1.1 599"), "{out}"); + + let stats = mock.take_stats(); + assert_eq!(stats.by_kind.get("batch"), Some(&1)); + assert_eq!(stats.by_kind.get("by-package"), Some(&1)); + assert_eq!(stats.by_kind.get("references"), Some(&1)); + assert_eq!(stats.by_kind.get("artifact"), Some(&1)); + assert_eq!(stats.unexpected, vec!["GET /nope".to_string()]); + assert_eq!(mock.take_stats().total(), 0); + } + + #[test] + fn keep_alive_connections_serve_several_requests() { + let mock = MockApi::start(Duration::ZERO).unwrap(); + mock.set_catalog(Catalog::default()); + let req = "GET /v0/organizations HTTP/1.1\r\n\r\n"; + let last = "GET /v0/organizations HTTP/1.1\r\nConnection: close\r\n\r\n"; + let out = http(&mock.api.uri(), &format!("{req}{req}{last}")); + assert_eq!(out.matches("HTTP/1.1 200").count(), 3, "{out}"); + assert_eq!(mock.take_stats().by_kind.get("organizations"), Some(&3)); + } +} diff --git a/crates/socket-patch-bench/src/process.rs b/crates/socket-patch-bench/src/process.rs new file mode 100644 index 000000000..baf37c5f6 --- /dev/null +++ b/crates/socket-patch-bench/src/process.rs @@ -0,0 +1,135 @@ +//! Spawn one CLI run and measure it. +//! +//! Wall time is taken around spawn → reap, so it includes process start +//! and exit the way a user's shell sees them. CPU time and peak RSS come +//! from `wait4`'s rusage for exactly that child: unlike `RUSAGE_CHILDREN` +//! it is not polluted by other children, and unlike sampling `/proc` it +//! sees the whole lifetime. + +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +#[derive(Debug, Clone, Copy, Default, serde::Serialize)] +pub struct Usage { + pub wall: Duration, + /// User + system CPU time (`None` where `wait4` is unavailable). + pub cpu: Option, + /// Peak resident set size in KiB. + pub max_rss_kib: Option, +} + +pub struct Outcome { + pub code: Option, + pub usage: Usage, + pub stdout: Vec, + pub stderr: Vec, +} + +/// Run `cmd` to completion with stdout/stderr captured to files under +/// `capture_dir` (files, not pipes: a pipe the harness does not drain +/// would stall a chatty child and bill the stall to the CLI). +pub fn run(mut cmd: Command, capture_dir: &Path) -> std::io::Result { + let out_path = capture_dir.join("stdout"); + let err_path = capture_dir.join("stderr"); + cmd.stdin(Stdio::null()) + .stdout(std::fs::File::create(&out_path)?) + .stderr(std::fs::File::create(&err_path)?); + let start = Instant::now(); + let child = cmd.spawn()?; + let (code, cpu, max_rss_kib) = reap(child)?; + let wall = start.elapsed(); + Ok(Outcome { + code, + usage: Usage { + wall, + cpu, + max_rss_kib, + }, + stdout: std::fs::read(&out_path)?, + stderr: std::fs::read(&err_path)?, + }) +} + +#[cfg(unix)] +fn reap( + child: std::process::Child, +) -> std::io::Result<(Option, Option, Option)> { + let pid = child.id() as libc::pid_t; + // `wait4` reaps the child itself; `child` is only dropped afterwards + // (dropping a `Child` neither waits nor kills). + let mut status: libc::c_int = 0; + // SAFETY: an all-zero rusage is a valid value; wait4 fills it in. + let mut ru: libc::rusage = unsafe { std::mem::zeroed() }; + loop { + // SAFETY: valid out-pointers to locals, and `pid` is our own + // unreaped child. + let r = unsafe { libc::wait4(pid, &mut status, 0, &mut ru) }; + if r == pid { + break; + } + let err = std::io::Error::last_os_error(); + if err.kind() != std::io::ErrorKind::Interrupted { + return Err(err); + } + } + drop(child); + let code = if libc::WIFEXITED(status) { + Some(libc::WEXITSTATUS(status)) + } else { + None + }; + let tv = |t: libc::timeval| { + Duration::from_secs(t.tv_sec as u64) + Duration::from_micros(t.tv_usec as u64) + }; + let cpu = tv(ru.ru_utime) + tv(ru.ru_stime); + // Linux reports KiB, macOS bytes. + let rss = if cfg!(target_os = "macos") { + ru.ru_maxrss as u64 / 1024 + } else { + ru.ru_maxrss as u64 + }; + Ok((code, Some(cpu), Some(rss))) +} + +#[cfg(not(unix))] +fn reap( + mut child: std::process::Child, +) -> std::io::Result<(Option, Option, Option)> { + let status = child.wait()?; + Ok((status.code(), None, None)) +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + + #[test] + fn measures_a_child_and_captures_its_output() { + let tmp = tempfile::tempdir().unwrap(); + let mut cmd = Command::new("sh"); + cmd.args(["-c", "echo out; echo err >&2; exit 3"]); + let o = run(cmd, tmp.path()).unwrap(); + assert_eq!(o.code, Some(3)); + assert_eq!(o.stdout, b"out\n"); + assert_eq!(o.stderr, b"err\n"); + assert!(o.usage.cpu.is_some()); + assert!(o.usage.max_rss_kib.unwrap() > 0); + } + + #[test] + fn cpu_time_counts_the_child_only() { + let tmp = tempfile::tempdir().unwrap(); + // A busy child burns CPU; the harness's own CPU must not leak in. + let mut cmd = Command::new("sh"); + cmd.args(["-c", "i=0; while [ $i -lt 200000 ]; do i=$((i+1)); done"]); + let o = run(cmd, tmp.path()).unwrap(); + let cpu = o.usage.cpu.unwrap(); + assert!(cpu > Duration::from_millis(5), "{cpu:?}"); + assert!( + cpu <= o.usage.wall + Duration::from_millis(50), + "{cpu:?} vs {:?}", + o.usage.wall + ); + } +} diff --git a/crates/socket-patch-bench/src/report.rs b/crates/socket-patch-bench/src/report.rs new file mode 100644 index 000000000..a2c02c346 --- /dev/null +++ b/crates/socket-patch-bench/src/report.rs @@ -0,0 +1,280 @@ +//! Result files: `results.json` (everything, for artifacts and tooling) +//! and `summary.md` (the table CI writes to the job summary). + +use std::collections::BTreeMap; +use std::fmt::Write as _; + +use serde::Serialize; + +use crate::stats::{Comparison, Verdict}; + +/// One run of one binary on one scenario. +#[derive(Debug, Clone, Serialize)] +pub struct Sample { + pub wall_ms: f64, + pub cpu_ms: Option, + pub max_rss_kib: Option, +} + +/// Everything measured for one binary on one scenario. +#[derive(Debug, Clone, Default, Serialize)] +pub struct BinaryResult { + pub samples: Vec, + /// Mock-API requests per endpoint for one run (identical across runs; + /// a run that disagrees is a validation failure). + pub requests: BTreeMap, + /// Most requests the mock was answering at once (one run). + pub max_inflight: usize, + /// The first validation failure, if any (the samples are then not + /// comparable). + pub invalid: Option, +} + +impl BinaryResult { + pub fn wall(&self) -> Vec { + self.samples.iter().map(|s| s.wall_ms).collect() + } + + pub fn cpu(&self) -> Option> { + self.samples.iter().map(|s| s.cpu_ms).collect() + } + + pub fn rss(&self) -> Option> { + self.samples + .iter() + .map(|s| s.max_rss_kib.map(|k| k as f64)) + .collect() + } + + pub fn total_requests(&self) -> u64 { + self.requests.values().sum() + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct ScenarioResult { + pub name: String, + pub description: String, + pub packages: usize, + pub patched: usize, + pub args: Vec, + pub latency_ms: u64, + /// Keyed `base`/`head` for `compare`, `bin` for `run`. + pub binaries: BTreeMap, + pub comparison: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ScenarioComparison { + pub wall: Comparison, + pub cpu: Option, + pub rss: Option, + /// Requests head made beyond base (positive) or saved (negative). + pub request_delta: i64, + pub verdict: Verdict, + /// Why the verdict is a regression, for the summary. + pub reasons: Vec, +} + +#[derive(Debug, Serialize)] +pub struct Report { + pub schema: u32, + pub mode: &'static str, + pub scale: f64, + pub threshold: Option, + pub binaries: BTreeMap, + pub scenarios: Vec, +} + +fn fmt_ms(ms: f64) -> String { + if ms >= 1000.0 { + format!("{:.2} s", ms / 1000.0) + } else { + format!("{ms:.1} ms") + } +} + +fn fmt_ratio(c: &Comparison) -> String { + let pct = (c.ratio - 1.0) * 100.0; + match c.ci { + Some((lo, hi)) => format!( + "{pct:+.1}% [{:+.1}, {:+.1}]", + (lo - 1.0) * 100.0, + (hi - 1.0) * 100.0 + ), + None => format!("{pct:+.1}%"), + } +} + +fn verdict_cell(v: Verdict) -> &'static str { + match v { + Verdict::Regression => "❌ regression", + Verdict::Improvement => "✅ faster", + Verdict::Unchanged => "≈", + Verdict::Inconclusive => "? inconclusive", + } +} + +impl Report { + pub fn regressions(&self) -> Vec<&ScenarioResult> { + self.scenarios + .iter() + .filter(|s| { + s.comparison + .as_ref() + .is_some_and(|c| c.verdict == Verdict::Regression) + }) + .collect() + } + + pub fn invalid(&self) -> Vec<(&ScenarioResult, &str, &str)> { + let mut out = Vec::new(); + for s in &self.scenarios { + for (bin, r) in &s.binaries { + if let Some(why) = &r.invalid { + out.push((s, bin.as_str(), why.as_str())); + } + } + } + out + } + + pub fn markdown(&self) -> String { + let mut md = String::new(); + let _ = writeln!(md, "## `socket-patch scan` benchmarks\n"); + for (label, path) in &self.binaries { + let _ = writeln!(md, "- **{label}**: `{path}`"); + } + let _ = writeln!(md, "- scale: {}", self.scale); + if let Some(t) = self.threshold { + let _ = writeln!( + md, + "- gate: median of paired head/base ratios > +{:.0}% with its 95% interval above zero", + t * 100.0 + ); + } + md.push('\n'); + if self.mode == "compare" { + md.push_str( + "| scenario | pkgs | base wall | head wall | Δ wall [95% CI] | Δ CPU | Δ peak RSS | requests | verdict |\n\ + |---|---:|---:|---:|---:|---:|---:|---:|---|\n", + ); + for s in &self.scenarios { + let (Some(base), Some(head)) = (s.binaries.get("base"), s.binaries.get("head")) + else { + continue; + }; + let Some(c) = &s.comparison else { + let why = head + .invalid + .as_deref() + .or(base.invalid.as_deref()) + .unwrap_or(""); + let _ = writeln!( + md, + "| `{}` | {} | | | | | | | ⚠️ invalid: {} |", + s.name, + s.packages, + why.lines().next().unwrap_or("").replace('|', "\\|") + ); + continue; + }; + let reqs = if c.request_delta == 0 { + format!("{}", head.total_requests()) + } else { + format!( + "{} → {} ({:+})", + base.total_requests(), + head.total_requests(), + c.request_delta + ) + }; + let _ = writeln!( + md, + "| `{}` | {} | {} | {} | {} | {} | {} | {} | {} |", + s.name, + s.packages, + fmt_ms(c.wall.base_median), + fmt_ms(c.wall.head_median), + fmt_ratio(&c.wall), + c.cpu + .as_ref() + .map(|c| format!("{:+.1}%", (c.ratio - 1.0) * 100.0)) + .unwrap_or_default(), + c.rss + .as_ref() + .map(|c| format!("{:+.1}%", (c.ratio - 1.0) * 100.0)) + .unwrap_or_default(), + reqs, + verdict_cell(c.verdict), + ); + } + let regressions = self.regressions(); + if !regressions.is_empty() { + md.push_str("\n### Regressions\n\n"); + for s in regressions { + let reasons = s + .comparison + .as_ref() + .map(|c| c.reasons.join("; ")) + .unwrap_or_default(); + let _ = writeln!(md, "- `{}`: {reasons}", s.name); + } + } + } else { + md.push_str( + "| scenario | pkgs | patched | median wall | min | max | median CPU | peak RSS | requests |\n\ + |---|---:|---:|---:|---:|---:|---:|---:|---:|\n", + ); + for s in &self.scenarios { + let Some(r) = s.binaries.values().next() else { + continue; + }; + if let Some(why) = &r.invalid { + let _ = writeln!( + md, + "| `{}` | {} | {} | ⚠️ invalid: {} | | | | | |", + s.name, + s.packages, + s.patched, + why.lines().next().unwrap_or("").replace('|', "\\|") + ); + continue; + } + let wall = r.wall(); + let min = wall.iter().copied().fold(f64::INFINITY, f64::min); + let max = wall.iter().copied().fold(0.0, f64::max); + let _ = writeln!( + md, + "| `{}` | {} | {} | {} | {} | {} | {} | {} | {} |", + s.name, + s.packages, + s.patched, + fmt_ms(crate::stats::median(&wall)), + fmt_ms(min), + fmt_ms(max), + r.cpu() + .map(|c| fmt_ms(crate::stats::median(&c))) + .unwrap_or_default(), + r.rss() + .map(|c| format!("{:.1} MiB", crate::stats::median(&c) / 1024.0)) + .unwrap_or_default(), + r.total_requests(), + ); + } + } + let invalid = self.invalid(); + if !invalid.is_empty() { + md.push_str("\n### Invalid runs\n\n"); + for (s, bin, why) in invalid { + let _ = writeln!( + md, + "- `{}` ({bin}):\n\n```\n{}\n```", + s.name, + why.trim_end() + ); + } + } + md + } +} diff --git a/crates/socket-patch-bench/src/scenarios.rs b/crates/socket-patch-bench/src/scenarios.rs new file mode 100644 index 000000000..e53fe2bc3 --- /dev/null +++ b/crates/socket-patch-bench/src/scenarios.rs @@ -0,0 +1,142 @@ +//! The benchmark catalog: which projects are scanned, how. +//! +//! Every package manager gets two scenarios on the default (hosted) mode, +//! the one `socket-patch scan` runs with no flags: +//! +//! - `/hosted` — a fresh project: crawl, lockfile inventory, batch +//! query, reference resolution, patch views, and the lockfile rewrite. +//! - `/rescan` — the same project after a first scan pinned its +//! patches (the steady state of a CI job that scans on every build): +//! hosted-pin discovery, update detection, and a no-op redirect. +//! +//! The largest npm project additionally runs as a `--dry-run` (planning +//! without writes), through the public proxy (no token: the free-tier +//! path most users hit, with its own batch size and concurrency), and +//! with simulated network latency (where request concurrency, not local +//! work, decides the wall time). + +use std::time::Duration; + +use crate::fixtures::{Fixture, Size}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + /// `scan` (hosted, wet) from the pristine project. + Hosted, + /// `scan --dry-run` from the pristine project. + DryRun, + /// `scan` on a project a previous `scan` already redirected. + Rescan, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Auth { + /// `SOCKET_API_TOKEN` + org: the authenticated API. + Token, + /// No token: the public proxy. + PublicProxy, +} + +/// A package manager's fixture generator. +pub struct Pm { + pub name: &'static str, + pub description: &'static str, + /// Packages / patched packages at scale 1.0. + pub packages: usize, + pub patched: usize, + pub build: fn(&mut crate::fixtures::gen::Tree, Size) -> std::io::Result, +} + +pub struct Scenario { + pub name: String, + pub pm: &'static Pm, + pub kind: Kind, + pub auth: Auth, + pub latency: Duration, + pub extra_args: Vec<&'static str>, +} + +impl Scenario { + pub fn size(&self, scale: f64) -> Size { + Size::scaled(self.pm.packages, self.pm.patched, scale) + } + + /// A filesystem-safe name. + pub fn slug(&self) -> String { + self.name.replace('/', "-") + } + + pub fn description(&self) -> String { + let what = match self.kind { + Kind::Hosted => "hosted scan of a fresh project", + Kind::DryRun => "hosted --dry-run", + Kind::Rescan => "rescan of an already-redirected project", + }; + let mut d = format!("{}: {what}", self.pm.description); + if self.auth == Auth::PublicProxy { + d.push_str(", public proxy (no token)"); + } + if !self.latency.is_zero() { + d.push_str(&format!( + ", {} ms simulated latency", + self.latency.as_millis() + )); + } + d + } +} + +pub fn package_managers() -> &'static [Pm] { + crate::fixtures::ALL +} + +/// Every scenario, in run order. +pub fn all() -> Vec { + let mut out = Vec::new(); + for pm in package_managers() { + for kind in [Kind::Hosted, Kind::Rescan] { + let suffix = if kind == Kind::Hosted { + "hosted" + } else { + "rescan" + }; + out.push(Scenario { + name: format!("{}/{suffix}", pm.name), + pm, + kind, + auth: Auth::Token, + latency: Duration::ZERO, + extra_args: Vec::new(), + }); + } + } + let npm = package_managers() + .iter() + .find(|p| p.name == "npm") + .expect("the npm fixture"); + out.push(Scenario { + name: "npm/dry-run".into(), + pm: npm, + kind: Kind::DryRun, + auth: Auth::Token, + latency: Duration::ZERO, + extra_args: Vec::new(), + }); + out.push(Scenario { + name: "npm/public-proxy".into(), + pm: npm, + kind: Kind::Hosted, + auth: Auth::PublicProxy, + latency: Duration::ZERO, + extra_args: Vec::new(), + }); + out.push(Scenario { + name: "npm/latency".into(), + pm: npm, + kind: Kind::Hosted, + auth: Auth::Token, + latency: Duration::from_millis(40), + extra_args: Vec::new(), + }); + out +} diff --git a/crates/socket-patch-bench/src/stats.rs b/crates/socket-patch-bench/src/stats.rs new file mode 100644 index 000000000..0c71a528d --- /dev/null +++ b/crates/socket-patch-bench/src/stats.rs @@ -0,0 +1,230 @@ +//! Distribution-free statistics for interleaved A/B samples. +//! +//! A CI runner's speed drifts over a job, and two runners differ by more +//! than most regressions, so the comparison only ever pairs a base run with +//! the head run taken right after it on the same machine. Each pair yields +//! one `head / base` ratio; the verdict reads the median ratio and a +//! sign-test confidence interval for it, which assumes nothing about the +//! timing distribution (CI timings are skewed and heavy-tailed, so a +//! t-interval would be wrong). + +/// Median of `xs` (the mean of the middle pair for an even count). +/// `NaN` for an empty slice. +pub fn median(xs: &[f64]) -> f64 { + if xs.is_empty() { + return f64::NAN; + } + let mut v = xs.to_vec(); + v.sort_by(f64::total_cmp); + let mid = v.len() / 2; + if v.len() % 2 == 1 { + v[mid] + } else { + (v[mid - 1] + v[mid]) / 2.0 + } +} + +/// `P(X <= k)` for `X ~ Binomial(n, 1/2)`, summed in log space so large +/// sample counts neither overflow nor underflow. +fn binomial_half_cdf(n: usize, k: usize) -> f64 { + let mut ln_p = -(n as f64) * std::f64::consts::LN_2; + let mut total = ln_p.exp(); + for i in 1..=k.min(n) { + ln_p += ((n - i + 1) as f64).ln() - (i as f64).ln(); + total += ln_p.exp(); + } + total.min(1.0) +} + +/// The sign-test confidence interval for the median of `xs`: the order +/// statistics `[x(k), x(n+1-k)]` for the largest `k` whose coverage +/// `1 - 2 P(X < k)` is at least `confidence`. `None` when there are too +/// few samples to reach it (fewer than 6 for 95%). +pub fn median_ci(xs: &[f64], confidence: f64) -> Option<(f64, f64)> { + let n = xs.len(); + if n == 0 { + return None; + } + let mut k = 0usize; + while k < n / 2 && 1.0 - 2.0 * binomial_half_cdf(n, k) >= confidence { + k += 1; + } + if k == 0 { + return None; + } + let mut v = xs.to_vec(); + v.sort_by(f64::total_cmp); + Some((v[k - 1], v[n - k])) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Verdict { + /// Slower by more than the threshold, and the whole interval is above 1. + Regression, + /// Faster by more than the threshold, and the whole interval is below 1. + Improvement, + /// Within the threshold, or not significant. + Unchanged, + /// Too few pairs for an interval. + Inconclusive, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct Comparison { + /// Median of the per-pair `head / base` ratios. + pub ratio: f64, + /// Sign-test interval for that median. + pub ci: Option<(f64, f64)>, + pub base_median: f64, + pub head_median: f64, + pub verdict: Verdict, +} + +/// What counts as a change worth failing on. +#[derive(Debug, Clone, Copy)] +pub struct Gate { + /// Relative threshold: 0.10 flags a median ratio above 1.10 (or below + /// 1 / 1.10 as an improvement). + pub threshold: f64, + /// The medians must also differ by at least this much in absolute + /// terms, so a 2 ms scheduler hiccup on a 15 ms run is not a "13% + /// regression". + pub min_abs_delta: f64, + pub confidence: f64, +} + +/// Compare paired samples (`base[i]` was measured right before `head[i]`). +pub fn compare_paired(base: &[f64], head: &[f64], gate: Gate) -> Comparison { + assert_eq!(base.len(), head.len(), "paired samples"); + let ratios: Vec = base + .iter() + .zip(head) + .map(|(b, h)| if *b > 0.0 { h / b } else { 1.0 }) + .collect(); + let ratio = median(&ratios); + let ci = median_ci(&ratios, gate.confidence); + let base_median = median(base); + let head_median = median(head); + let abs_delta = (head_median - base_median).abs(); + let verdict = match ci { + None => Verdict::Inconclusive, + Some((lo, _)) + if ratio > 1.0 + gate.threshold && lo > 1.0 && abs_delta >= gate.min_abs_delta => + { + Verdict::Regression + } + Some((_, hi)) + if ratio < 1.0 / (1.0 + gate.threshold) + && hi < 1.0 + && abs_delta >= gate.min_abs_delta => + { + Verdict::Improvement + } + Some(_) => Verdict::Unchanged, + }; + Comparison { + ratio, + ci, + base_median, + head_median, + verdict, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const GATE: Gate = Gate { + threshold: 0.10, + min_abs_delta: 1.0, + confidence: 0.95, + }; + + #[test] + fn median_handles_odd_even_and_unsorted_input() { + assert_eq!(median(&[3.0, 1.0, 2.0]), 2.0); + assert_eq!(median(&[4.0, 1.0, 3.0, 2.0]), 2.5); + assert!(median(&[]).is_nan()); + } + + #[test] + fn binomial_cdf_matches_exact_values() { + // Bin(15, 1/2): P(X <= 3) = (1 + 15 + 105 + 455) / 2^15. + assert!((binomial_half_cdf(15, 3) - 576.0 / 32768.0).abs() < 1e-12); + assert!((binomial_half_cdf(4, 4) - 1.0).abs() < 1e-12); + // Large n stays finite. + let p = binomial_half_cdf(2000, 1000); + assert!(p > 0.5 && p < 0.52, "{p}"); + } + + #[test] + fn median_ci_uses_the_sign_test_order_statistics() { + let xs: Vec = (1..=15).map(f64::from).collect(); + // n = 15 at 95%: k = 4 (coverage 0.965), so [x(4), x(12)]. + assert_eq!(median_ci(&xs, 0.95), Some((4.0, 12.0))); + // Five samples cannot reach 95% (the widest interval covers 93.75%). + assert_eq!(median_ci(&xs[..5], 0.95), None); + assert_eq!(median_ci(&xs[..6], 0.95), Some((1.0, 6.0))); + } + + #[test] + fn a_consistent_slowdown_is_a_regression() { + let base: Vec = (0..15).map(|i| 100.0 + f64::from(i % 3)).collect(); + let head: Vec = base.iter().map(|b| b * 1.25).collect(); + let c = compare_paired(&base, &head, GATE); + assert_eq!(c.verdict, Verdict::Regression); + assert!((c.ratio - 1.25).abs() < 1e-9); + } + + #[test] + fn a_consistent_speedup_is_an_improvement() { + let base = vec![100.0; 12]; + let head = vec![70.0; 12]; + assert_eq!( + compare_paired(&base, &head, GATE).verdict, + Verdict::Improvement + ); + } + + #[test] + fn noise_around_one_is_unchanged() { + let base = vec![100.0; 15]; + let head: Vec = (0..15) + .map(|i| if i % 2 == 0 { 92.0 } else { 109.0 }) + .collect(); + assert_eq!( + compare_paired(&base, &head, GATE).verdict, + Verdict::Unchanged + ); + } + + #[test] + fn a_large_median_with_a_wide_interval_is_not_a_regression() { + // Most pairs are 1.2x but a third are 0.8x: the median is past the + // threshold, the interval is not wholly above 1. + let base = vec![100.0; 15]; + let head: Vec = (0..15) + .map(|i| if i % 3 == 0 { 80.0 } else { 120.0 }) + .collect(); + let c = compare_paired(&base, &head, GATE); + assert!(c.ratio > 1.1); + assert_eq!(c.verdict, Verdict::Unchanged); + } + + #[test] + fn a_tiny_absolute_delta_is_not_a_regression() { + let base = vec![2.0; 15]; + let head = vec![2.6; 15]; + let c = compare_paired(&base, &head, GATE); + assert!(c.ratio > 1.25); + assert_eq!(c.verdict, Verdict::Unchanged); + } + + #[test] + fn too_few_pairs_is_inconclusive() { + let c = compare_paired(&[1.0, 1.0], &[5.0, 5.0], GATE); + assert_eq!(c.verdict, Verdict::Inconclusive); + } +} diff --git a/crates/socket-patch-bench/src/tree.rs b/crates/socket-patch-bench/src/tree.rs new file mode 100644 index 000000000..4cf3368c3 --- /dev/null +++ b/crates/socket-patch-bench/src/tree.rs @@ -0,0 +1,287 @@ +//! Restore a fixture between runs without re-copying it. +//! +//! A wet scan rewrites lockfiles and configs, so every timed run must start +//! from the pristine fixture. Copying a few-thousand-package `node_modules` +//! per run would dominate the job, so the work tree is compared against a +//! metadata snapshot instead and only the entries a run added, removed or +//! modified are put back from the pristine copy. The check covers the +//! whole tree, installed packages included: a run that unexpectedly writes +//! into `node_modules` is still undone before the next one. + +use std::collections::BTreeMap; +use std::io; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +#[derive(Debug, Clone, PartialEq, Eq)] +enum Kind { + Dir, + File { len: u64, mtime: Option }, + Symlink(PathBuf), +} + +/// Every entry under a root, keyed by its path relative to that root. +#[derive(Debug, Clone, Default)] +pub struct Snapshot { + entries: BTreeMap, +} + +impl Snapshot { + pub fn take(root: &Path) -> io::Result { + let mut entries = BTreeMap::new(); + walk(root, Path::new(""), &mut entries)?; + Ok(Self { entries }) + } + + #[cfg(test)] + pub fn len(&self) -> usize { + self.entries.len() + } +} + +fn walk(root: &Path, rel: &Path, out: &mut BTreeMap) -> io::Result<()> { + for entry in std::fs::read_dir(root.join(rel))? { + let entry = entry?; + let rel = rel.join(entry.file_name()); + let ty = entry.file_type()?; + if ty.is_symlink() { + out.insert( + rel.clone(), + Kind::Symlink(std::fs::read_link(entry.path())?), + ); + } else if ty.is_dir() { + out.insert(rel.clone(), Kind::Dir); + walk(root, &rel, out)?; + } else { + let meta = entry.metadata()?; + out.insert( + rel, + Kind::File { + len: meta.len(), + mtime: meta.modified().ok(), + }, + ); + } + } + Ok(()) +} + +/// What one restore had to undo. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Drift { + pub added: Vec, + pub removed: Vec, + pub modified: Vec, +} + +impl Drift { + pub fn is_empty(&self) -> bool { + self.added.is_empty() && self.removed.is_empty() && self.modified.is_empty() + } + + /// Every path the run touched, sorted. + pub fn touched(&self) -> Vec { + let mut all: Vec = self + .added + .iter() + .chain(&self.removed) + .chain(&self.modified) + .cloned() + .collect(); + all.sort(); + all + } +} + +/// Bring `work` back to `pristine` (whose layout `snapshot` recorded for +/// `work` right after the last restore) and return what had drifted. The +/// snapshot is updated to the restored state. +pub fn restore(work: &Path, pristine: &Path, snapshot: &mut Snapshot) -> io::Result { + let now = Snapshot::take(work)?; + let mut drift = Drift::default(); + + // Remove what the run added (deepest first, so a new directory's + // contents go before it) and what it turned into another kind. + for (rel, kind) in now.entries.iter().rev() { + match snapshot.entries.get(rel) { + None => { + drift.added.push(rel.clone()); + remove(&work.join(rel), kind)?; + } + Some(old) if std::mem::discriminant(old) != std::mem::discriminant(kind) => { + drift.modified.push(rel.clone()); + remove(&work.join(rel), kind)?; + } + _ => {} + } + } + // Put back what is missing or changed (shallowest first). + for (rel, old) in &snapshot.entries { + let current = now.entries.get(rel); + let same_kind = + current.is_some_and(|k| std::mem::discriminant(k) == std::mem::discriminant(old)); + let changed = match (old, current) { + (_, None) => { + drift.removed.push(rel.clone()); + true + } + _ if !same_kind => true, // already counted and removed above + (Kind::Dir, _) => false, + (old, Some(cur)) if old != cur => { + drift.modified.push(rel.clone()); + true + } + _ => false, + }; + if changed { + copy_entry(&pristine.join(rel), &work.join(rel), old)?; + } + } + drift.added.sort(); + drift.removed.sort(); + drift.modified.sort(); + drift.modified.dedup(); + if !drift.is_empty() { + *snapshot = Snapshot::take(work)?; + } + Ok(drift) +} + +fn remove(path: &Path, kind: &Kind) -> io::Result<()> { + let result = match kind { + Kind::Dir => std::fs::remove_dir_all(path), + Kind::File { .. } | Kind::Symlink(_) => std::fs::remove_file(path), + }; + match result { + // A parent removed earlier took it already. + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + other => other, + } +} + +fn copy_entry(from: &Path, to: &Path, kind: &Kind) -> io::Result<()> { + if let Some(parent) = to.parent() { + std::fs::create_dir_all(parent)?; + } + match kind { + Kind::Dir => std::fs::create_dir_all(to), + Kind::File { .. } => { + // Replace, never write through: the run may have left a + // hardlink to the pristine copy. + let _ = std::fs::remove_file(to); + std::fs::copy(from, to).map(|_| ()) + } + Kind::Symlink(target) => { + let _ = std::fs::remove_file(to); + symlink(target, to) + } + } +} + +#[cfg(unix)] +pub fn symlink(target: &Path, link: &Path) -> io::Result<()> { + std::os::unix::fs::symlink(target, link) +} + +#[cfg(windows)] +pub fn symlink(target: &Path, link: &Path) -> io::Result<()> { + std::os::windows::fs::symlink_dir(target, link) +} + +/// Recursively copy `from` to `to` (symlinks are recreated, not followed). +pub fn copy_tree(from: &Path, to: &Path) -> io::Result<()> { + std::fs::create_dir_all(to)?; + for entry in std::fs::read_dir(from)? { + let entry = entry?; + let src = entry.path(); + let dst = to.join(entry.file_name()); + let ty = entry.file_type()?; + if ty.is_symlink() { + symlink(&std::fs::read_link(&src)?, &dst)?; + } else if ty.is_dir() { + copy_tree(&src, &dst)?; + } else { + std::fs::copy(&src, &dst)?; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn write(root: &Path, rel: &str, body: &str) { + let p = root.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, body).unwrap(); + } + + #[test] + fn restore_undoes_adds_removes_and_edits() { + let tmp = tempfile::tempdir().unwrap(); + let pristine = tmp.path().join("pristine"); + let work = tmp.path().join("work"); + write(&pristine, "package-lock.json", "{}"); + write(&pristine, "node_modules/a/package.json", "a"); + write(&pristine, "node_modules/b/package.json", "b"); + copy_tree(&pristine, &work).unwrap(); + let mut snap = Snapshot::take(&work).unwrap(); + assert_eq!(snap.len(), 6); + + // Nothing changed: nothing to do. + assert!(restore(&work, &pristine, &mut snap).unwrap().is_empty()); + + std::fs::write(work.join("package-lock.json"), "{\"rewritten\":true}").unwrap(); + write(&work, ".npmrc", "allow-remote=all\n"); + write(&work, ".socket/state/x.json", "{}"); + std::fs::remove_dir_all(work.join("node_modules/b")).unwrap(); + + let drift = restore(&work, &pristine, &mut snap).unwrap(); + assert_eq!( + drift.added, + vec![ + PathBuf::from(".npmrc"), + PathBuf::from(".socket"), + PathBuf::from(".socket/state"), + PathBuf::from(".socket/state/x.json"), + ] + ); + assert_eq!( + drift.removed, + vec![ + PathBuf::from("node_modules/b"), + PathBuf::from("node_modules/b/package.json"), + ] + ); + assert_eq!(drift.modified, vec![PathBuf::from("package-lock.json")]); + + assert_eq!( + std::fs::read_to_string(work.join("package-lock.json")).unwrap(), + "{}" + ); + assert_eq!( + std::fs::read_to_string(work.join("node_modules/b/package.json")).unwrap(), + "b" + ); + assert!(!work.join(".npmrc").exists()); + assert!(!work.join(".socket").exists()); + // And the refreshed snapshot sees a clean tree. + assert!(restore(&work, &pristine, &mut snap).unwrap().is_empty()); + } + + #[test] + fn restore_handles_a_file_replaced_by_a_directory() { + let tmp = tempfile::tempdir().unwrap(); + let pristine = tmp.path().join("pristine"); + let work = tmp.path().join("work"); + write(&pristine, "x", "file"); + copy_tree(&pristine, &work).unwrap(); + let mut snap = Snapshot::take(&work).unwrap(); + std::fs::remove_file(work.join("x")).unwrap(); + write(&work, "x/inner", "dir now"); + let drift = restore(&work, &pristine, &mut snap).unwrap(); + assert!(drift.added.contains(&PathBuf::from("x/inner"))); + assert_eq!(std::fs::read_to_string(work.join("x")).unwrap(), "file"); + } +} diff --git a/docs/development.md b/docs/development.md index 1dcb3e88f..8ab84bbcb 100644 --- a/docs/development.md +++ b/docs/development.md @@ -30,6 +30,7 @@ persisted login. | [`socket-patch-cli`](../crates/socket-patch-cli/src/) | Arguments, command orchestration, terminal output, and JSON responses | | [`socket-patch-core`](../crates/socket-patch-core/src/) | API access, discovery, selection policy, patching, format handling, vendoring, and VEX | | [`socket-patch-node`](../crates/socket-patch-node/) | Node bindings for the in-memory hosted engine | +| [`socket-patch-bench`](../crates/socket-patch-bench/) | `scan` benchmarks and the CI performance gate (not published) | | [`scripts/`](../scripts/) | Installers, release tooling, compatibility runners, and performance tools | | [`tests/docker/`](../tests/docker/README.md) | Container fixtures for native package-manager integration tests | @@ -93,6 +94,10 @@ particular, the vlt tables and `vlt-coverage.json` are inputs to validation scri keep them in sync with tests and workflows. Store individual experiment logs and full backtest results as run artifacts, not as new product documentation. -For performance work, use the [record/replay harness](../scripts/perf/README.md). +For performance work, use the [`scan` benchmarks](../crates/socket-patch-bench/README.md): +synthetic projects for every package manager against a local patch API, with each +run validated. CI compares every pull request against its base with them and fails +on a significant slowdown, more API requests, or a scan that stops doing its work. +To measure against real API traffic, use the [record/replay harness](../scripts/perf/README.md). For publishing, follow the [release runbook](releasing.md) and [installer hosting guide](installer-hosting.md). diff --git a/docs/testing/README.md b/docs/testing/README.md index 43438c44d..c4532b7f2 100644 --- a/docs/testing/README.md +++ b/docs/testing/README.md @@ -15,6 +15,7 @@ Rust and Python checks. | Production suites | [Hosted](hosted-production-e2e.md), [vendored](vendored-production-e2e.md) | Real patch-service responses and artifact delivery | | Release compatibility backtests | Package-manager guides below and `scripts/backtest-*.py` | Format and installer boundaries across published releases | | Container suites | [Docker guide](../../tests/docker/README.md) | Toolchain isolation and offline installs | +| Performance benchmarks | [`crates/socket-patch-bench`](../../crates/socket-patch-bench/README.md), `.github/workflows/bench.yml` | `scan` timings, memory and API request counts per package manager, compared against the base on every PR | Native suites require the tools named in their guide. Opt-in or unavailable-toolchain skips are not installation evidence. Use the suite's `*_REQUIRED` or `*_STRICT` diff --git a/scripts/perf/README.md b/scripts/perf/README.md index af571dc9c..d293b86f9 100644 --- a/scripts/perf/README.md +++ b/scripts/perf/README.md @@ -1,5 +1,9 @@ # Network benchmark harness +For regression benchmarks of `scan` on synthetic projects (run in CI on every +pull request), see [`crates/socket-patch-bench`](../../crates/socket-patch-bench/README.md). +This harness is for measuring a real project against recorded API traffic. + Most `scan` time is spent waiting on API round trips. Live timings are noisy and can't be repeated, so this harness records the API traffic of one real run and then replays it locally. That makes timing deterministic and gives a From 08936bd7f3a7d0d3ba1dbffc71e9e59f0962e463 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 18:16:39 +0000 Subject: [PATCH 2/2] Fix bench pnpm/vlt dep links for scoped parents; expect() over unwrap() Intra-store dependency symlinks live at {store}/{key}/node_modules/{dep}, so only the dependency name's scope adds a directory level. Counting the parent's slashes pointed every link under a scoped parent one directory too high, leaving those packages with a broken isolated layout. Also replace the bare unwraps on the per-binary result maps with expect() messages stating the invariant, per Bugbot review. Co-Authored-By: Claude --- crates/socket-patch-bench/src/engine.rs | 8 ++++++-- crates/socket-patch-bench/src/fixtures/npm.rs | 4 ++-- crates/socket-patch-bench/src/main.rs | 5 ++++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/crates/socket-patch-bench/src/engine.rs b/crates/socket-patch-bench/src/engine.rs index 7d8d3885e..4baf217da 100644 --- a/crates/socket-patch-bench/src/engine.rs +++ b/crates/socket-patch-bench/src/engine.rs @@ -445,7 +445,9 @@ pub fn run_scenario( for _ in 0..opts.warmup { for b in bins { - let r = results.get_mut(&b.label).unwrap(); + let r = results + .get_mut(&b.label) + .expect("results is keyed by every label in bins"); if r.invalid.is_some() { continue; } @@ -463,7 +465,9 @@ pub fn run_scenario( bins.iter().rev().collect() }; for b in order { - let r = results.get_mut(&b.label).unwrap(); + let r = results + .get_mut(&b.label) + .expect("results is keyed by every label in bins"); if r.invalid.is_some() { continue; } diff --git a/crates/socket-patch-bench/src/fixtures/npm.rs b/crates/socket-patch-bench/src/fixtures/npm.rs index ba1e8189d..1985f703a 100644 --- a/crates/socket-patch-bench/src/fixtures/npm.rs +++ b/crates/socket-patch-bench/src/fixtures/npm.rs @@ -465,7 +465,7 @@ pub fn build_pnpm(t: &mut Tree, size: Size) -> std::io::Result { let Some(dep) = g.resolve(p, d, v) else { continue; }; - let up = "../".repeat(1 + p.name.matches('/').count() + d.matches('/').count()); + let up = "../".repeat(1 + d.matches('/').count()); let target = format!("{up}../{}/node_modules/{d}", dep.store_key()); t.symlink(&target, &format!("{entry}/{d}"))?; } @@ -753,7 +753,7 @@ pub fn build_vlt(t: &mut Tree, size: Size) -> std::io::Result { let Some(dep) = g.resolve(p, d, v) else { continue; }; - let up = "../".repeat(1 + p.name.matches('/').count() + d.matches('/').count()); + let up = "../".repeat(1 + d.matches('/').count()); t.symlink( &format!("{up}../{}/node_modules/{d}", vlt_id(dep)), &format!("{entry}/{d}"), diff --git a/crates/socket-patch-bench/src/main.rs b/crates/socket-patch-bench/src/main.rs index 441f4bb6a..4a0454fc0 100644 --- a/crates/socket-patch-bench/src/main.rs +++ b/crates/socket-patch-bench/src/main.rs @@ -353,7 +353,10 @@ fn main_inner() -> Result { )?; opts.runs = common.runs; for (label, extra) in more.binaries { - let entry = r.binaries.get_mut(&label).unwrap(); + let entry = r + .binaries + .get_mut(&label) + .expect("both rounds run the same binaries"); if entry.invalid.is_none() { entry.invalid = extra.invalid; }