From 315391aebc383b301f5951b43c0eb0772932f7d2 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 12:36:18 -0400 Subject: [PATCH 001/237] perf(core): add an ordered, bounded concurrency helper for API loops `ordered_concurrent` / `map_ordered_concurrent` wrap `stream::iter(..).map(f).buffered(limit)`: at most `limit` requests in flight, results yielded in input order, nothing started until polled. The serial patch-API loops can adopt it and fold results exactly as before. `API_CONCURRENCY` (8) and `PROXY_API_CONCURRENCY` (4) carry the per-client caps. futures-util was already in the lock; it is now a direct dependency of core and the CLI. Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 2 + Cargo.toml | 1 + crates/socket-patch-cli/Cargo.toml | 1 + crates/socket-patch-core/Cargo.toml | 1 + .../socket-patch-core/src/utils/concurrent.rs | 178 ++++++++++++++++++ crates/socket-patch-core/src/utils/mod.rs | 1 + 6 files changed, 184 insertions(+) create mode 100644 crates/socket-patch-core/src/utils/concurrent.rs diff --git a/Cargo.lock b/Cargo.lock index 9dc76883..ee1214c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1664,6 +1664,7 @@ dependencies = [ "dialoguer", "flate2", "fs2", + "futures-util", "glob", "hex", "libc", @@ -1693,6 +1694,7 @@ dependencies = [ "base64", "flate2", "fs2", + "futures-util", "hex", "libc", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index c1e95483..71cf4491 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ sha1 = "=0.10.6" hex = "=0.4.3" reqwest = { version = "=0.12.28", features = ["rustls-tls", "json"], default-features = false } tokio = { version = "=1.50.0", features = ["full"] } +futures-util = { version = "=0.3.32", default-features = false, features = ["std"] } thiserror = "=2.0.18" walkdir = "=2.5.0" uuid = { version = "=1.21.0", features = ["v4"] } diff --git a/crates/socket-patch-cli/Cargo.toml b/crates/socket-patch-cli/Cargo.toml index 00046ebe..c5549df4 100644 --- a/crates/socket-patch-cli/Cargo.toml +++ b/crates/socket-patch-cli/Cargo.toml @@ -22,6 +22,7 @@ clap = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } +futures-util = { workspace = true } console = { workspace = true } dialoguer = { workspace = true } uuid = { workspace = true } diff --git a/crates/socket-patch-core/Cargo.toml b/crates/socket-patch-core/Cargo.toml index 26b97ca4..3ea57f07 100644 --- a/crates/socket-patch-core/Cargo.toml +++ b/crates/socket-patch-core/Cargo.toml @@ -15,6 +15,7 @@ sha1 = { workspace = true } hex = { workspace = true } reqwest = { workspace = true } tokio = { workspace = true } +futures-util = { workspace = true } thiserror = { workspace = true } walkdir = { workspace = true } uuid = { workspace = true } diff --git a/crates/socket-patch-core/src/utils/concurrent.rs b/crates/socket-patch-core/src/utils/concurrent.rs new file mode 100644 index 00000000..725d2f25 --- /dev/null +++ b/crates/socket-patch-core/src/utils/concurrent.rs @@ -0,0 +1,178 @@ +//! Ordered, bounded concurrency for independent network requests. +//! +//! The CLI's patch-API loops (batch discovery, per-package detail GETs, +//! hosted record views) used to await one request at a time, so a run +//! paid one round trip per request. Every consumer here must keep its +//! output byte-identical to the serial loop, so the only primitive on +//! offer is the ORDERED one: at most `limit` futures run at once, and +//! results come back in input order no matter which request finishes +//! first. Callers fold them exactly as the serial loop did (warnings, +//! failure lists, first-error rules), so nothing downstream can tell the +//! difference. Never swap in `buffer_unordered`. +//! +//! Everything runs on the caller's task (no `spawn`), so the futures may +//! borrow (`&ApiClient`) and need not be `Send`. + +use std::future::Future; + +use futures_util::stream::{self, Stream, StreamExt}; + +/// In-flight request cap for the authenticated patch API. Measured with +/// no 429s up to 32 in flight; 8 already makes the loops latency-flat. +pub const API_CONCURRENCY: usize = 8; + +/// In-flight request cap on the public patch proxy, which serializes +/// anonymous callers behind one shared server-side semaphore — stay +/// polite there. +pub const PROXY_API_CONCURRENCY: usize = 4; + +/// The in-flight cap for a client on the public proxy (`true`) or the +/// authenticated API (`false`). +pub fn api_concurrency(use_public_proxy: bool) -> usize { + if use_public_proxy { + PROXY_API_CONCURRENCY + } else { + API_CONCURRENCY + } +} + +/// Run `f` over `items` with at most `limit` futures in flight, yielding +/// results in INPUT order (item `i`'s result is always the `i`-th item, +/// even when a later request finishes first). A `limit` of 0 is treated +/// as 1. Futures are only started as the stream is polled, so dropping +/// the stream early cancels whatever is still in flight and never starts +/// the rest — the property a caller relies on to abandon a window and +/// replay it elsewhere. +pub fn ordered_concurrent( + items: I, + limit: usize, + f: F, +) -> impl Stream +where + I: IntoIterator, + F: FnMut(I::Item) -> Fut, + Fut: Future, +{ + stream::iter(items).map(f).buffered(limit.max(1)) +} + +/// [`ordered_concurrent`], collected: every result, in input order. +pub async fn map_ordered_concurrent(items: I, limit: usize, f: F) -> Vec +where + I: IntoIterator, + F: FnMut(I::Item) -> Fut, + Fut: Future, +{ + ordered_concurrent(items, limit, f).collect().await +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + use std::time::Duration; + + /// Later items finish first (reversed latencies); results still come + /// back in input order. + #[tokio::test(start_paused = true)] + async fn results_keep_input_order_under_reversed_latencies() { + let items: Vec = (0..10).collect(); + let out = map_ordered_concurrent(items.clone(), 4, |i| async move { + tokio::time::sleep(Duration::from_millis(100 * (10 - i))).await; + i * 2 + }) + .await; + assert_eq!(out, items.iter().map(|i| i * 2).collect::>()); + } + + /// Never more than `limit` futures in flight, and the limit is + /// actually reached (the loop is concurrent, not serial). + #[tokio::test(start_paused = true)] + async fn in_flight_count_never_exceeds_limit() { + let live = Cell::new(0usize); + let peak = Cell::new(0usize); + let out = map_ordered_concurrent(0..20u64, 3, |i| { + let (live, peak) = (&live, &peak); + async move { + live.set(live.get() + 1); + peak.set(peak.get().max(live.get())); + tokio::time::sleep(Duration::from_millis(10 + (i % 4) * 7)).await; + live.set(live.get() - 1); + i + } + }) + .await; + assert_eq!(out, (0..20).collect::>()); + assert_eq!(peak.get(), 3); + assert_eq!(live.get(), 0); + } + + /// Wall time is ~ceil(n/limit) round trips, not n. + #[tokio::test(start_paused = true)] + async fn runs_requests_concurrently() { + let start = tokio::time::Instant::now(); + map_ordered_concurrent(0..8u32, 4, |_| async { + tokio::time::sleep(Duration::from_millis(100)).await; + }) + .await; + assert_eq!(start.elapsed(), Duration::from_millis(200)); + } + + #[tokio::test] + async fn empty_input_yields_nothing_and_starts_nothing() { + let started = Cell::new(0usize); + let out: Vec = map_ordered_concurrent(Vec::::new(), 8, |x| { + started.set(started.get() + 1); + async move { x } + }) + .await; + assert!(out.is_empty()); + assert_eq!(started.get(), 0); + } + + /// A zero limit degrades to serial instead of stalling forever. + #[tokio::test(start_paused = true)] + async fn zero_limit_is_serial() { + let live = Cell::new(0usize); + let peak = Cell::new(0usize); + let out = map_ordered_concurrent(0..5u8, 0, |i| { + let (live, peak) = (&live, &peak); + async move { + live.set(live.get() + 1); + peak.set(peak.get().max(live.get())); + tokio::time::sleep(Duration::from_millis(5)).await; + live.set(live.get() - 1); + i + } + }) + .await; + assert_eq!(out, vec![0, 1, 2, 3, 4]); + assert_eq!(peak.get(), 1); + } + + /// Dropping the stream after item `k` never starts the items past the + /// window: at most `limit` futures were ever created beyond `k`. + #[tokio::test(start_paused = true)] + async fn dropping_the_stream_stops_starting_new_work() { + let started = Cell::new(0usize); + { + let mut s = Box::pin(ordered_concurrent(0..100u32, 4, |i| { + started.set(started.get() + 1); + async move { + tokio::time::sleep(Duration::from_millis(10)).await; + i + } + })); + assert_eq!(s.next().await, Some(0)); + assert_eq!(s.next().await, Some(1)); + } + // Items 0..=1 consumed, window of 4 refilled once per consumption. + assert!(started.get() <= 6, "started {}", started.get()); + } + + #[test] + fn concurrency_caps_per_client_kind() { + assert_eq!(api_concurrency(false), API_CONCURRENCY); + assert_eq!(api_concurrency(true), PROXY_API_CONCURRENCY); + } +} diff --git a/crates/socket-patch-core/src/utils/mod.rs b/crates/socket-patch-core/src/utils/mod.rs index 9c57a536..0da9f627 100644 --- a/crates/socket-patch-core/src/utils/mod.rs +++ b/crates/socket-patch-core/src/utils/mod.rs @@ -1,4 +1,5 @@ pub mod cargo_workspace; +pub mod concurrent; pub(crate) mod digest; pub mod env_compat; pub mod fs; From 148a32d2091655f176049625ed815dec4b98b069 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 12:45:11 -0400 Subject: [PATCH 002/237] test(scan): pin API-loop ordering under reversed latencies New subprocess suite for the three patch-API loops `scan` drives (batch POSTs, per-package detail GETs, hosted record views). Every mock answers later requests first, so an implementation that folds in completion order, or lets a discarded response leak in, changes the output: - batch: a 401 on the first chunk sends that chunk and all later ones to the proxy with one auth request and one warning; a 401 on chunk 3 of 6 folds 0-2 from the auth API and replays 3-5 on the proxy; per batch 500 warnings print in chunk order; the all-failed error carries the last chunk's error. - details: partial-failure warnings print in package order and the whole human preview equals a zero-latency run; the all-failed error names the last package. - hosted wet run: record_fetch_failed warnings keep confirmed order and stdout, lockfile and ledger equal a zero-latency run. The suite passes against the current serial loops (checked with the baseline binary) and is the oracle for making them concurrent. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/scan_ordered_concurrency_e2e.rs | 707 ++++++++++++++++++ 1 file changed, 707 insertions(+) create mode 100644 crates/socket-patch-cli/tests/scan_ordered_concurrency_e2e.rs diff --git a/crates/socket-patch-cli/tests/scan_ordered_concurrency_e2e.rs b/crates/socket-patch-cli/tests/scan_ordered_concurrency_e2e.rs new file mode 100644 index 00000000..00502519 --- /dev/null +++ b/crates/socket-patch-cli/tests/scan_ordered_concurrency_e2e.rs @@ -0,0 +1,707 @@ +//! `scan`'s patch-API loops run their requests concurrently (batch POSTs, +//! per-package detail GETs, hosted record views) but must stay +//! indistinguishable from the old serial loops: results fold in input +//! order, warnings print in input order, and the authenticated-to-proxy +//! fallback replays from the exact chunk that triggered it. +//! +//! Every test here makes the server answer LATER requests FIRST (reversed +//! latencies), so an implementation that folded in completion order — or +//! that let a discarded response leak in — produces visibly different +//! output. Where a pure oracle exists, the output is also compared with a +//! zero-latency run of the same fixture. +//! +//! Subprocess runs scrub the `SOCKET_*` environment (the +//! `in_process_redirect.rs::scrubbed_cli` pattern) so ambient +//! configuration cannot reroute the branch under test. + +use std::path::Path; +use std::process::Command; +use std::time::Duration; + +use wiremock::matchers::{body_string_contains, method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const ORG: &str = "test-org"; + +/// Package names: no name is a prefix of another, so a body/path match on +/// `"{name}@"` is unambiguous. +const NAMES: [&str; 6] = [ + "conc-alpha", + "conc-bravo", + "conc-charlie", + "conc-delta", + "conc-echo", + "conc-foxtrot", +]; +const VERSION: &str = "1.0.0"; + +fn purl(name: &str) -> String { + format!("pkg:npm/{name}@{VERSION}") +} + +/// Distinct uuid per (package index, source) so the output reveals which +/// server response was folded for each package. +fn uuid(idx: usize, source: u8) -> String { + format!("{source:08x}-0000-4000-8000-{idx:012x}") +} + +const AUTH: u8 = 0xa; +const PROXY: u8 = 0xb; + +fn encode_purl(purl: &str) -> String { + purl.replace(':', "%3A") + .replace('/', "%2F") + .replace('@', "%40") +} + +fn scrubbed_cli() -> Command { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_socket-patch")); + cmd.env("SOCKET_DRY_RUN", "true") + .env("SOCKET_ECOSYSTEMS", "cargo") + .env("SOCKET_MANIFEST_PATH", "/nonexistent/manifest.json") + .env_remove("SOCKET_DRY_RUN") + .env_remove("SOCKET_ECOSYSTEMS") + .env_remove("SOCKET_MANIFEST_PATH"); + for (key, _) in std::env::vars_os() { + let name = key.to_string_lossy(); + if name.starts_with("SOCKET_") && name != "SOCKET_NO_CONFIG" { + cmd.env_remove(&key); + } + } + cmd.env_remove("VIRTUAL_ENV"); + cmd.env("SOCKET_TELEMETRY_DISABLED", "1"); + cmd +} + +/// `socket-patch scan ` in `cwd` against `api` (authenticated) with +/// the public proxy pointed at `proxy`. +fn run_scan(cwd: &Path, api: &str, proxy: &str, args: &[&str]) -> (i32, String, String) { + let out = scrubbed_cli() + .arg("scan") + .args([ + "--cwd", + cwd.to_str().unwrap(), + "--api-url", + api, + "--api-token", + "fake-token-for-test", + "--org", + ORG, + ]) + .args(args) + .env("SOCKET_PROXY_URL", proxy) + .output() + .expect("run socket-patch scan"); + ( + out.status.code().unwrap_or(-1), + String::from_utf8_lossy(&out.stdout).into_owned(), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) +} + +/// An npm project with every name in `names` installed and locked. +fn write_project(root: &Path, names: &[&str]) { + let deps: Vec = names + .iter() + .map(|n| format!(r#""{n}": "{VERSION}""#)) + .collect(); + let deps = deps.join(", "); + std::fs::write( + root.join("package.json"), + format!(r#"{{ "name": "consumer", "version": "0.0.0", "dependencies": {{ {deps} }} }}"#), + ) + .unwrap(); + let mut entries = vec![format!( + r#" "": {{ "name": "consumer", "version": "0.0.0", "dependencies": {{ {deps} }} }}"# + )]; + for name in names { + let pkg = root.join("node_modules").join(name); + std::fs::create_dir_all(&pkg).unwrap(); + std::fs::write( + pkg.join("package.json"), + format!(r#"{{ "name": "{name}", "version": "{VERSION}" }}"#), + ) + .unwrap(); + std::fs::write(pkg.join("index.js"), b"module.exports = 1;\n").unwrap(); + entries.push(format!( + r#" "node_modules/{name}": {{ + "version": "{VERSION}", + "resolved": "https://registry.npmjs.org/{name}/-/{name}-{VERSION}.tgz", + "integrity": "sha512-UPSTREAM{name}==" + }}"# + )); + } + std::fs::write( + root.join("package-lock.json"), + format!( + "{{\n \"name\": \"consumer\",\n \"version\": \"0.0.0\",\n \"lockfileVersion\": 3,\n \ + \"requires\": true,\n \"packages\": {{\n{}\n }}\n}}\n", + entries.join(",\n") + ), + ) + .unwrap(); +} + +fn batch_entry(idx: usize, source: u8) -> serde_json::Value { + let p = purl(NAMES[idx]); + serde_json::json!({ + "purl": p, + "patches": [{ + "uuid": uuid(idx, source), + "purl": p, + "tier": "free", + "cveIds": [], + "ghsaIds": [], + "severity": "high", + "title": format!("patch for {}", NAMES[idx]), + }] + }) +} + +fn batch_body(entries: Vec) -> serde_json::Value { + serde_json::json!({ "packages": entries, "canAccessPaidPatches": false }) +} + +/// One batch mock per package on `route`, matched by the package in the +/// request body: answers `status` (a 200 carries that package's patch from +/// `source`) after `delay`. +async fn mount_batch_for( + server: &MockServer, + route: &str, + idx: usize, + source: u8, + status: u16, + delay: Duration, +) { + let template = if status == 200 { + ResponseTemplate::new(200).set_body_json(batch_body(vec![batch_entry(idx, source)])) + } else { + ResponseTemplate::new(status).set_body_string(format!("boom-{}", NAMES[idx])) + }; + Mock::given(method("POST")) + .and(path(route)) + .and(body_string_contains(format!("\"{}\"", purl(NAMES[idx])))) + .respond_with(template.set_delay(delay)) + .mount(server) + .await; +} + +fn auth_batch_route() -> String { + format!("/v0/orgs/{ORG}/patches/batch") +} + +const PROXY_BATCH_ROUTE: &str = "/patch/batch"; + +async fn batch_requests(server: &MockServer, route: &str) -> Vec { + server + .received_requests() + .await + .expect("wiremock records requests") + .into_iter() + .filter(|r| r.method.as_str() == "POST" && r.url.path() == route) + .map(|r| String::from_utf8_lossy(&r.body).into_owned()) + .collect() +} + +/// The crawl order of the fixture's packages — the order `scan` chunks +/// them in. Learned from one default-batch-size run: its single batch body +/// lists every purl in that order. (Crawl order is readdir order, which the +/// test must not assume.) +async fn crawl_order(root: &Path) -> Vec { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(auth_batch_route())) + .respond_with(ResponseTemplate::new(200).set_body_json(batch_body(vec![]))) + .mount(&server) + .await; + let (code, stdout, stderr) = run_scan(root, &server.uri(), &server.uri(), &["--json"]); + assert_eq!(code, 0, "order probe: stdout={stdout} stderr={stderr}"); + let bodies = batch_requests(&server, &auth_batch_route()).await; + assert_eq!(bodies.len(), 1, "one batch expected: {bodies:?}"); + let body: serde_json::Value = serde_json::from_str(&bodies[0]).unwrap(); + let order: Vec = body["components"] + .as_array() + .unwrap() + .iter() + .map(|c| { + let p = c["purl"].as_str().unwrap(); + NAMES.iter().position(|n| purl(n) == p).unwrap() + }) + .collect(); + assert_eq!(order.len(), NAMES.len(), "every package crawled: {order:?}"); + order +} + +/// `package idx → folded patch uuids` from a `scan --json` envelope. +fn folded_uuids(stdout: &str) -> Vec<(String, Vec)> { + let v: serde_json::Value = serde_json::from_str(stdout).expect("scan --json envelope"); + v["packages"] + .as_array() + .expect("packages array") + .iter() + .map(|pkg| { + let uuids = pkg["patches"] + .as_array() + .unwrap() + .iter() + .map(|p| p["uuid"].as_str().unwrap().to_string()) + .collect(); + (pkg["purl"].as_str().unwrap().to_string(), uuids) + }) + .collect() +} + +/// Delay for the `pos`-th chunk (crawl position): later chunks answer +/// first. +fn reversed(pos: usize) -> Duration { + Duration::from_millis(60 * (NAMES.len() - pos) as u64) +} + +// --------------------------------------------------------------------------- +// Batch POSTs (A2) +// --------------------------------------------------------------------------- + +/// A 401 on the very first chunk: that chunk and every later one go to the +/// proxy, the authenticated API sees exactly the one request it saw +/// before, and the downgrade warning prints once. +#[tokio::test] +async fn batch_fallback_on_first_chunk_sends_everything_after_to_proxy() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), &NAMES[..5]); + + let auth = MockServer::start().await; + let proxy = MockServer::start().await; + Mock::given(method("POST")) + .and(path(auth_batch_route())) + .respond_with(ResponseTemplate::new(401).set_body_string("invalid token")) + .expect(1) + .mount(&auth) + .await; + for idx in 0..5 { + mount_batch_for(&proxy, PROXY_BATCH_ROUTE, idx, PROXY, 200, reversed(idx)).await; + } + + let (code, stdout, stderr) = run_scan( + tmp.path(), + &auth.uri(), + &proxy.uri(), + &["--json", "--batch-size", "1"], + ); + assert_eq!(code, 0, "stdout={stdout} stderr={stderr}"); + assert_eq!( + stderr + .matches("falling back to public patch API proxy") + .count(), + 1, + "exactly one downgrade warning: {stderr}" + ); + assert_eq!(batch_requests(&auth, &auth_batch_route()).await.len(), 1); + assert_eq!(batch_requests(&proxy, PROXY_BATCH_ROUTE).await.len(), 5); + let folded = folded_uuids(&stdout); + assert_eq!(folded.len(), 5, "{stdout}"); + for (p, uuids) in folded { + let idx = NAMES.iter().position(|n| purl(n) == p).unwrap(); + assert_eq!(uuids, vec![uuid(idx, PROXY)], "{p}"); + } +} + +/// A 401 on chunk 3 of 6 (crawl order): chunks 0-2 fold the authenticated +/// answers, chunk 3 is retried on the proxy and 4-5 are proxy-only. The +/// authenticated answers for 4-5 arrive BEFORE chunk 3's 401 (reversed +/// latencies) and must be discarded, exactly as if they had never been +/// sent. +#[tokio::test] +async fn batch_fallback_mid_run_replays_from_the_failing_chunk() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), &NAMES); + let order = crawl_order(tmp.path()).await; + + let auth = MockServer::start().await; + let proxy = MockServer::start().await; + for (pos, &idx) in order.iter().enumerate() { + let status = if pos == 3 { 401 } else { 200 }; + mount_batch_for(&auth, &auth_batch_route(), idx, AUTH, status, reversed(pos)).await; + mount_batch_for(&proxy, PROXY_BATCH_ROUTE, idx, PROXY, 200, reversed(pos)).await; + } + + let (code, stdout, stderr) = run_scan( + tmp.path(), + &auth.uri(), + &proxy.uri(), + &["--json", "--batch-size", "1"], + ); + assert_eq!(code, 0, "stdout={stdout} stderr={stderr}"); + assert_eq!( + stderr + .matches("falling back to public patch API proxy") + .count(), + 1, + "exactly one downgrade warning: {stderr}" + ); + + let mut expected: Vec<(String, Vec)> = order + .iter() + .enumerate() + .map(|(pos, &idx)| { + let source = if pos < 3 { AUTH } else { PROXY }; + (purl(NAMES[idx]), vec![uuid(idx, source)]) + }) + .collect(); + expected.sort(); + assert_eq!(folded_uuids(&stdout), expected); + + // The proxy saw exactly the replayed tail, chunk 3 onward. + let mut proxied: Vec = batch_requests(&proxy, PROXY_BATCH_ROUTE) + .await + .iter() + .map(|b| { + let v: serde_json::Value = serde_json::from_str(b).unwrap(); + v["components"][0]["purl"].as_str().unwrap().to_string() + }) + .collect(); + proxied.sort(); + let mut tail: Vec = order[3..].iter().map(|&i| purl(NAMES[i])).collect(); + tail.sort(); + assert_eq!(proxied, tail); +} + +/// Mixed 500s in chunks 2 and 4 with reversed latencies: the per-batch +/// warnings print in chunk order, and the run still succeeds with the +/// other chunks' patches. +#[tokio::test] +async fn batch_failure_warnings_print_in_chunk_order() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), &NAMES); + let order = crawl_order(tmp.path()).await; + + let auth = MockServer::start().await; + for (pos, &idx) in order.iter().enumerate() { + let status = if pos == 2 || pos == 4 { 500 } else { 200 }; + mount_batch_for(&auth, &auth_batch_route(), idx, AUTH, status, reversed(pos)).await; + } + let all: Vec = (0..NAMES.len()).collect(); + mount_details(&auth, &all, &[], |_| Duration::ZERO).await; + + let (code, stdout, stderr) = run_scan( + tmp.path(), + &auth.uri(), + &auth.uri(), + &["--batch-size", "1", "--dry-run"], + ); + assert_eq!(code, 0, "stdout={stdout} stderr={stderr}"); + let warn2 = format!( + "Warning: API batch 3 of 6 failed: API request failed with status 500: boom-{}", + NAMES[order[2]] + ); + let warn4 = format!( + "Warning: API batch 5 of 6 failed: API request failed with status 500: boom-{}", + NAMES[order[4]] + ); + let at2 = stderr + .find(&warn2) + .unwrap_or_else(|| panic!("{warn2}\n{stderr}")); + let at4 = stderr + .find(&warn4) + .unwrap_or_else(|| panic!("{warn4}\n{stderr}")); + assert!(at2 < at4, "chunk-order warnings: {stderr}"); + assert_eq!(stderr.matches("Warning: API batch").count(), 2, "{stderr}"); +} + +/// Every chunk fails, the FIRST chunk slowest: the all-failed error still +/// carries the LAST chunk's error, as the serial loop's `last_batch_error` +/// did. +#[tokio::test] +async fn all_batches_failed_reports_the_last_chunks_error() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), &NAMES); + let order = crawl_order(tmp.path()).await; + + let auth = MockServer::start().await; + for (pos, &idx) in order.iter().enumerate() { + mount_batch_for(&auth, &auth_batch_route(), idx, AUTH, 500, reversed(pos)).await; + } + + let (code, stdout, stderr) = run_scan( + tmp.path(), + &auth.uri(), + &auth.uri(), + &["--json", "--batch-size", "1"], + ); + assert_eq!(code, 1, "stdout={stdout} stderr={stderr}"); + let v: serde_json::Value = serde_json::from_str(&stdout).unwrap(); + assert_eq!(v["status"], "error"); + assert_eq!( + v["error"].as_str().unwrap(), + format!( + "API request failed with status 500: boom-{}", + NAMES[order[5]] + ) + ); +} + +// --------------------------------------------------------------------------- +// Per-package detail GETs (A1) +// --------------------------------------------------------------------------- + +/// One batch answering every package in `idxs`, then per-package detail +/// GETs ([`mount_details`]). +async fn mount_discovery_with_details( + server: &MockServer, + idxs: &[usize], + failing: &[usize], + delay: impl Fn(usize) -> Duration, +) { + Mock::given(method("POST")) + .and(path(auth_batch_route())) + .respond_with(ResponseTemplate::new(200).set_body_json(batch_body( + idxs.iter().map(|&i| batch_entry(i, AUTH)).collect(), + ))) + .mount(server) + .await; + mount_details(server, idxs, failing, delay).await; +} + +/// Per-package detail GETs for `idxs`: `failing` ones answer 500, the rest +/// their patch; every answer is delayed by `delay(idx)`. +async fn mount_details( + server: &MockServer, + idxs: &[usize], + failing: &[usize], + delay: impl Fn(usize) -> Duration, +) { + for &idx in idxs { + let p = purl(NAMES[idx]); + let template = if failing.contains(&idx) { + ResponseTemplate::new(500).set_body_string(format!("detail-boom-{}", NAMES[idx])) + } else { + ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "patches": [{ + "uuid": uuid(idx, AUTH), "purl": p, + "publishedAt": "2024-01-01T00:00:00Z", + "description": format!("details for {}", NAMES[idx]), + "license": "MIT", "tier": "free", + "vulnerabilities": {} + }], + "canAccessPaidPatches": false, + })) + }; + Mock::given(method("GET")) + .and(path(format!( + "/v0/orgs/{ORG}/patches/by-package/{}", + encode_purl(&p) + ))) + .respond_with(template.set_delay(delay(idx))) + .mount(server) + .await; + } +} + +/// Partial detail-fetch failures with reversed latencies: the per-package +/// warnings print in package (purl-sorted) order, and the whole human +/// preview is byte-identical to a zero-latency run. +#[tokio::test] +async fn detail_fetch_warnings_and_preview_keep_package_order() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), &NAMES); + let all: Vec = (0..NAMES.len()).collect(); + let failing = [1usize, 4]; + + let run_with = |delay: fn(usize) -> Duration| { + let all = all.clone(); + let root = tmp.path().to_path_buf(); + async move { + let server = MockServer::start().await; + mount_discovery_with_details(&server, &all, &failing, delay).await; + run_scan(&root, &server.uri(), &server.uri(), &["--dry-run"]) + } + }; + // NAMES is already purl-sorted, so index order is the fold order. + let (code, stdout, stderr) = run_with(|i| Duration::from_millis(50 * (6 - i as u64))).await; + assert_eq!(code, 0, "stdout={stdout} stderr={stderr}"); + let w1 = format!("Warning: could not fetch details for {}: ", purl(NAMES[1])); + let w4 = format!("Warning: could not fetch details for {}: ", purl(NAMES[4])); + let at1 = stderr.find(&w1).unwrap_or_else(|| panic!("{w1}\n{stderr}")); + let at4 = stderr.find(&w4).unwrap_or_else(|| panic!("{w4}\n{stderr}")); + assert!(at1 < at4, "package-order warnings: {stderr}"); + + let (code0, stdout0, stderr0) = run_with(|_| Duration::ZERO).await; + assert_eq!(code0, code); + assert_eq!(stdout0, stdout, "latency must not change the preview"); + assert_eq!(stderr0, stderr, "latency must not change stderr"); +} + +/// Every detail fetch fails (reversed latencies): the one terminal error +/// names the LAST package's failure, as the serial loop's `failures.last()` +/// did. +#[tokio::test] +async fn all_detail_fetches_failed_reports_the_last_packages_error() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), &NAMES); + let all: Vec = (0..NAMES.len()).collect(); + + let server = MockServer::start().await; + mount_discovery_with_details(&server, &all, &all, |i| { + Duration::from_millis(50 * (6 - i as u64)) + }) + .await; + let (code, stdout, stderr) = run_scan( + tmp.path(), + &server.uri(), + &server.uri(), + &["--json", "--mode", "hosted", "--dry-run"], + ); + assert_eq!(code, 1, "stdout={stdout} stderr={stderr}"); + let v: serde_json::Value = serde_json::from_str(&stdout).unwrap(); + let err = v["error"].as_str().unwrap(); + assert!( + err.starts_with("all 6 patch-detail queries failed: ") + && err.ends_with(&format!("detail-boom-{}", NAMES[5])), + "{err}" + ); +} + +// --------------------------------------------------------------------------- +// Hosted record views (A3) +// --------------------------------------------------------------------------- + +fn hosted_url(idx: usize) -> String { + let name = NAMES[idx]; + format!( + "http://patch.test/patch/npm/{name}/{VERSION}/22222222-2222-4222-8222-222222222222/{}/{name}-{VERSION}.tgz", + uuid(idx, AUTH) + ) +} + +/// Discovery + references + views for a hosted wet run over every package; +/// the `failing` views answer 500. Every by-package / view answer is delayed +/// by `delay(idx)`. +async fn mount_hosted(server: &MockServer, failing: &[usize], delay: fn(usize) -> Duration) { + let all: Vec = (0..NAMES.len()).collect(); + mount_discovery_with_details(server, &all, &[], delay).await; + let mut results = serde_json::Map::new(); + for (idx, name) in NAMES.iter().enumerate() { + results.insert( + uuid(idx, AUTH), + serde_json::json!({ + "status": "granted", + "url": hosted_url(idx), + "purl": purl(name), + "artifacts": [{ + "kind": "tarball", + "url": hosted_url(idx), + "integrity": { "sha512": format!("sha512-PATCHED{idx}==") } + }], + "registryOverride": null + }), + ); + } + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/package"))) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({ "results": results })), + ) + .mount(server) + .await; + for (idx, name) in NAMES.iter().enumerate() { + let u = uuid(idx, AUTH); + let template = if failing.contains(&idx) { + ResponseTemplate::new(500) + } else { + ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "uuid": u, + "purl": purl(name), + "publishedAt": "2024-01-01T00:00:00Z", + "files": { + "package/index.js": { + "beforeHash": format!("{idx:064x}"), + "afterHash": format!("{:064x}", idx + 100), + } + }, + "vulnerabilities": { + format!("GHSA-conc-{idx:04}-aaaa"): { + "cves": [format!("CVE-2024-{idx:04}")], + "summary": "s", "severity": "high", "description": "d" + } + }, + "description": "x", "license": "MIT", "tier": "free" + })) + }; + Mock::given(method("GET")) + .and(path(format!("/v0/orgs/{ORG}/patches/view/{u}"))) + .respond_with(template.set_delay(delay(idx))) + .mount(server) + .await; + } +} + +/// The ledger with its run timestamps blanked, for a byte comparison. +fn ledger_without_timestamps(root: &Path) -> String { + let raw = std::fs::read_to_string(root.join(".socket/vendor/redirect-state.json")).unwrap(); + let mut v: serde_json::Value = serde_json::from_str(&raw).unwrap(); + fn blank(v: &mut serde_json::Value) { + match v { + serde_json::Value::Object(map) => { + for (k, val) in map.iter_mut() { + let key = k.to_ascii_lowercase(); + if key.ends_with("at") && val.is_string() { + *val = serde_json::Value::String("".into()); + } else { + blank(val); + } + } + } + serde_json::Value::Array(items) => items.iter_mut().for_each(blank), + _ => {} + } + } + blank(&mut v); + serde_json::to_string_pretty(&v).unwrap() +} + +/// A wet hosted run where 2 of 6 record views fail, with reversed +/// latencies: the `record_fetch_failed` warnings keep `confirmed` order, +/// and stdout, the rewritten lockfile and the ledger all equal a +/// zero-latency run's. +#[tokio::test] +async fn hosted_record_fetch_failures_keep_order_and_ledger_bytes() { + let failing = [1usize, 3]; + let slow: fn(usize) -> Duration = |i| Duration::from_millis(50 * (6 - i as u64)); + let fast: fn(usize) -> Duration = |_| Duration::ZERO; + + let mut outcomes = Vec::new(); + for delay in [slow, fast] { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), &NAMES); + let server = MockServer::start().await; + mount_hosted(&server, &failing, delay).await; + let (code, stdout, stderr) = run_scan( + tmp.path(), + &server.uri(), + &server.uri(), + &["--json", "--mode", "hosted", "--yes"], + ); + assert_eq!(code, 0, "stdout={stdout} stderr={stderr}"); + let lock = std::fs::read_to_string(tmp.path().join("package-lock.json")).unwrap(); + outcomes.push((stdout, lock, ledger_without_timestamps(tmp.path()))); + } + + let (stdout, lock, ledger) = &outcomes[0]; + let v: serde_json::Value = serde_json::from_str(stdout).unwrap(); + let warnings: Vec<&str> = v["redirect"]["warnings"] + .as_array() + .unwrap() + .iter() + .filter(|w| w["code"] == "record_fetch_failed") + .map(|w| w["detail"].as_str().unwrap()) + .collect(); + assert_eq!(warnings.len(), 2, "{stdout}"); + assert!(warnings[0].starts_with(&format!("{} redirected", purl(NAMES[1])))); + assert!(warnings[1].starts_with(&format!("{} redirected", purl(NAMES[3])))); + for idx in 0..NAMES.len() { + assert!(lock.contains(&hosted_url(idx)), "{lock}"); + let has_record = ledger.contains(&format!("GHSA-conc-{idx:04}-aaaa")); + assert_eq!(has_record, !failing.contains(&idx), "{ledger}"); + } + + assert_eq!(outcomes[0], outcomes[1], "latency must not change the run"); +} From 113df71b68639d6aca64233e3205758052f3f758 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 12:45:23 -0400 Subject: [PATCH 003/237] perf(scan): fetch per-package patch details concurrently `fetch_patch_details` awaited one `by-package` GET per package with patches (74 on depscan, ~10 s of serial round trips). The queries now run through `ordered_concurrent` (8 in flight, 4 on the public proxy) and are consumed in `packages` order, so `results`, `failures`, the warn-after loop and the all-failed rule see exactly what the serial loop produced. `ApiClient::uses_public_proxy` picks the cap. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../socket-patch-cli/src/commands/scan/mod.rs | 17 +++++++++++++++-- crates/socket-patch-core/src/api/client.rs | 7 +++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index 0e727c38..be671f27 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -7,6 +7,7 @@ //! small helpers shared across the submodules. use clap::Args; +use futures_util::StreamExt; use socket_patch_core::api::client::{ build_proxy_fallback_client, get_api_client_with_overrides, is_fallback_candidate, ApiClient, }; @@ -16,6 +17,7 @@ use socket_patch_core::crawlers::{CrawlerOptions, Ecosystem}; use socket_patch_core::manifest::operations::read_manifest; use socket_patch_core::manifest::schema::PatchManifest; use socket_patch_core::telemetry::{track_patch_scan_failed, track_patch_scanned}; +use socket_patch_core::utils::concurrent::{api_concurrency, ordered_concurrent}; use socket_patch_core::utils::purl::{normalize_purl, purl_name_version, strip_purl_qualifiers}; use socket_patch_core::vendor::VendorState; use socket_patch_core::vex::discover::{LedgerLiveness, WiringMode}; @@ -530,13 +532,24 @@ async fn fetch_patch_details( // `show_progress` off reads as `--json` to the status line: never // drawn. On, it is live only on a terminal; it never prints a result. let mut status = StatusLine::stderr(!show_progress, false); - for (i, pkg) in packages.iter().enumerate() { + // The queries run concurrently but come back in `packages` order, so + // `results` and `failures` fold exactly as the serial loop's did. The + // counter names the next result awaited. + let mut responses = std::pin::pin!(ordered_concurrent( + packages, + api_concurrency(api_client.uses_public_proxy()), + |pkg| async move { (pkg, api_client.search_patches_by_package(&pkg.purl).await) }, + )); + for i in 0..packages.len() { status.set(format!( "Fetching patch details... ({}/{})", i + 1, packages.len() )); - match api_client.search_patches_by_package(&pkg.purl).await { + let Some((pkg, response)) = responses.next().await else { + break; + }; + match response { Ok(response) => results.extend(response.patches), Err(e) => failures.push((pkg.purl.clone(), e.to_string())), } diff --git a/crates/socket-patch-core/src/api/client.rs b/crates/socket-patch-core/src/api/client.rs index ba3ff596..4f3bd71c 100644 --- a/crates/socket-patch-core/src/api/client.rs +++ b/crates/socket-patch-core/src/api/client.rs @@ -291,6 +291,13 @@ impl ApiClient { self.org_slug.as_ref() } + /// Whether this client talks to the public patch proxy (vs. the + /// authenticated org API) — picks the concurrency cap + /// ([`crate::utils::concurrent::api_concurrency`]). + pub fn uses_public_proxy(&self) -> bool { + self.use_public_proxy + } + // ── Internal helpers ────────────────────────────────────────────── /// Internal GET that deserialises JSON. Returns `Ok(None)` on 404. From 04a730c6d4c83001312cc0dadf4322d55ffa9d1b Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 12:45:23 -0400 Subject: [PATCH 004/237] perf(scan): run batch discovery concurrently with exact proxy fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The batch loop POSTed one chunk at a time (56 chunks on depscan). Chunks now run through `ordered_concurrent` and are consumed strictly in chunk order, so per-batch warnings, `batch_error_count`, `last_batch_error` and the paid-access flag fold as before. The authenticated-to-proxy downgrade keeps the serial loop's exact sequence: the first chunk goes alone (a stale token still costs the auth API one request), and at the first consumed chunk k whose error is a fallback candidate — any index — the window is dropped, responses for chunks past k are discarded unfolded, the same warning prints, chunk k is retried on the proxy and the rest continue there (4 in flight). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../socket-patch-cli/src/commands/scan/mod.rs | 136 +++++++++++------- 1 file changed, 86 insertions(+), 50 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index be671f27..81b008a4 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -10,8 +10,9 @@ use clap::Args; use futures_util::StreamExt; use socket_patch_core::api::client::{ build_proxy_fallback_client, get_api_client_with_overrides, is_fallback_candidate, ApiClient, + ApiError, }; -use socket_patch_core::api::types::{BatchPackagePatches, PatchSearchResult}; +use socket_patch_core::api::types::{BatchPackagePatches, BatchSearchResponse, PatchSearchResult}; use socket_patch_core::crawlers::ruby_crawler::config_path_ignored_warning; use socket_patch_core::crawlers::{CrawlerOptions, Ecosystem}; use socket_patch_core::manifest::operations::read_manifest; @@ -1867,63 +1868,98 @@ pub async fn run(mut args: ScanArgs) -> i32 { let mut batch_error_count = 0usize; let mut last_batch_error: Option = None; - for (batch_idx, chunk) in all_purls.chunks(batch_size).enumerate() { - status.set(format!( - "Querying API for patches... (batch {}/{total_batches})", - batch_idx + 1 - )); - - let mut result = api_client.search_patches_batch(chunk).await; - - // Fallback: a 401/403 against the authenticated endpoint can - // mean a stale/revoked token. Retry against the public proxy - // (free patches only) once, then continue the rest of the - // loop with the downgraded client. Only triggers on the - // first authenticated batch; subsequent iterations are - // already on the proxy. - if !use_public_proxy { - if let Err(ref e) = result { - if is_fallback_candidate(e) { - // Errors-only under --silent; --json keeps it on stderr - // (the envelope has no slot for a mid-run downgrade). - if !args.common.silent { - status.println(format!( - "Warning: authenticated API returned {e}; \ - falling back to public patch API proxy (free patches only)." - )); - } - api_client = build_proxy_fallback_client(&overrides); - use_public_proxy = true; - fallback_to_proxy = true; - result = api_client.search_patches_batch(chunk).await; + // Fold one batch outcome, in chunk order. Every caller below consumes + // outcomes strictly by chunk index, so the per-batch warnings, + // `batch_error_count` and `last_batch_error` come out exactly as the + // serial loop produced them. + let mut fold = |batch_idx: usize, + result: Result, + status: &mut StatusLine<_>| match result { + Ok(response) => { + if response.can_access_paid_patches { + can_access_paid_patches = true; + } + for pkg in response.packages { + if !pkg.patches.is_empty() { + all_packages_with_patches.push(pkg); } } } + Err(e) => { + batch_error_count += 1; + last_batch_error = Some(e.to_string()); + // Not fatal by itself: the scan goes on with the other + // batches. A one-batch scan says it once, below. + if !args.common.json && !args.common.silent && total_batches > 1 { + status.println(render::batch_failed_warning( + batch_idx + 1, + total_batches, + &e.to_string(), + )); + } + } + }; - match result { - Ok(response) => { - if response.can_access_paid_patches { - can_access_paid_patches = true; - } - for pkg in response.packages { - if !pkg.patches.is_empty() { - all_packages_with_patches.push(pkg); + // The batches run concurrently (at most `api_concurrency` in flight) + // but are CONSUMED in chunk order, one window at a time: + // + // - The first chunk goes alone, so a stale token costs the + // authenticated API one request before the downgrade, as it always + // did. + // - Fallback: a 401/403 against the authenticated endpoint can mean a + // stale/revoked token. At the first consumed chunk `k` whose error is + // a fallback candidate (any index, not just the first), the window is + // dropped — in-flight requests for chunks past `k` are cancelled and + // any responses already received for them are discarded, never + // folded — then chunk `k` is retried against the public proxy (free + // patches only) and the rest continues on the downgraded client. + // That is exactly the serial loop's sequence; on the proxy no further + // fallback applies. + let chunks: Vec<&[String]> = all_purls.chunks(batch_size).collect(); + let mut next = 0usize; + while next < total_batches { + let end = if next == 0 { 1 } else { total_batches }; + let mut fallback_error = None; + { + let client = &api_client; + let mut results = std::pin::pin!(ordered_concurrent( + &chunks[next..end], + api_concurrency(use_public_proxy), + |chunk| client.search_patches_batch(chunk), + )); + while next < end { + status.set(format!( + "Querying API for patches... (batch {}/{total_batches})", + next + 1 + )); + let Some(result) = results.next().await else { + break; + }; + match result { + Err(e) if !use_public_proxy && is_fallback_candidate(&e) => { + fallback_error = Some(e); + break; } + result => fold(next, result, &mut status), } + next += 1; } - Err(e) => { - batch_error_count += 1; - last_batch_error = Some(e.to_string()); - // Not fatal by itself: the scan goes on with the other - // batches. A one-batch scan says it once, below. - if !args.common.json && !args.common.silent && total_batches > 1 { - status.println(render::batch_failed_warning( - batch_idx + 1, - total_batches, - &e.to_string(), - )); - } + } + if let Some(e) = fallback_error { + // Errors-only under --silent; --json keeps it on stderr + // (the envelope has no slot for a mid-run downgrade). + if !args.common.silent { + status.println(format!( + "Warning: authenticated API returned {e}; \ + falling back to public patch API proxy (free patches only)." + )); } + api_client = build_proxy_fallback_client(&overrides); + use_public_proxy = true; + fallback_to_proxy = true; + let result = api_client.search_patches_batch(chunks[next]).await; + fold(next, result, &mut status); + next += 1; } } From cab7007208b215a14542d07eba39f9c37628986e Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 12:45:23 -0400 Subject: [PATCH 005/237] perf(hosted): fetch patch record views concurrently on wet runs A wet hosted run fetched `patches/view/{uuid}` for every confirmed redirect one at a time (74 on depscan, ~9 s). The views now run through `ordered_concurrent` and are consumed in `confirmed` order, so `records` (newest wins) and the `record_fetch_failed` warnings are unchanged. The ledger re-fetch on idempotent re-runs is deliberately kept. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commands/scan/hosted.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 2c218c53..c11c9a44 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -6,9 +6,11 @@ use std::path::Path; use std::time::Duration; +use futures_util::StreamExt; use socket_patch_core::api::types::BatchPackagePatches; use socket_patch_core::patch::apply_lock::LockGuard; use socket_patch_core::patch::redirect::DepOverride; +use socket_patch_core::utils::concurrent::{api_concurrency, ordered_concurrent}; use socket_patch_core::utils::purl::purl_parts; use crate::commands::vex::generate_vex_from_manifest_path; @@ -2476,9 +2478,20 @@ pub(crate) async fn run_redirect_selected( if !common.dry_run { let total = confirmed.len(); - for (i, (purl, uuid)) in confirmed.iter().enumerate() { + // The views are fetched concurrently but consumed in `confirmed` + // order, so `records` (newest wins) and `record_warnings` fold + // exactly as the serial loop's did. + let mut views = std::pin::pin!(ordered_concurrent( + confirmed.iter(), + api_concurrency(api_client.uses_public_proxy()), + |(_, uuid)| api_client.fetch_patch(uuid), + )); + for (i, (purl, _)) in confirmed.iter().enumerate() { status.set(format!("Fetching patch records... ({}/{total})", i + 1)); - match api_client.fetch_patch(uuid).await { + let Some(view) = views.next().await else { + break; + }; + match view { Ok(Some(resp)) => { let (rec_purl, record) = crate::commands::get::record_from_patch_response(&resp); From fc03c9ebde81a25b882acf003f7d3f2e3f010a2a Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 12:52:18 -0400 Subject: [PATCH 006/237] perf(scan): send telemetry off the critical path, flushed before exit Scan awaited each telemetry POST inline (150-300 ms typical, up to the 5 s budget on a bad network) before carrying on. Its three events now go through `spawn_patch_scanned` / `spawn_patch_scan_failed`: the event is built and its endpoint resolved where it fires (same body, timestamp, env reads and "Sending telemetry" debug line), and only the POST runs in a background task. `scan::run` awaits `PendingTelemetry::flush` before returning, so every event is still delivered, or given up on within the same 2 s connect / 5 s request budget, before the process exits. The inline trackers and every other command are unchanged. Tests: core unit tests pin that a background send posts the same bytes and headers as an inline one and that flush waits for it; telemetry_e2e pins that each scan terminal (success, empty crawl, all batches failed) delivers its one event and stays alive until the slow endpoint answers. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../socket-patch-cli/src/commands/scan/mod.rs | 34 +- .../socket-patch-cli/tests/telemetry_e2e.rs | 95 ++++++ crates/socket-patch-core/src/telemetry.rs | 290 ++++++++++++++++-- 3 files changed, 376 insertions(+), 43 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index 81b008a4..e48cdee9 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -17,7 +17,9 @@ use socket_patch_core::crawlers::ruby_crawler::config_path_ignored_warning; use socket_patch_core::crawlers::{CrawlerOptions, Ecosystem}; use socket_patch_core::manifest::operations::read_manifest; use socket_patch_core::manifest::schema::PatchManifest; -use socket_patch_core::telemetry::{track_patch_scan_failed, track_patch_scanned}; +use socket_patch_core::telemetry::{ + spawn_patch_scan_failed, spawn_patch_scanned, PendingTelemetry, +}; use socket_patch_core::utils::concurrent::{api_concurrency, ordered_concurrent}; use socket_patch_core::utils::purl::{normalize_purl, purl_name_version, strip_purl_qualifiers}; use socket_patch_core::vendor::VendorState; @@ -1448,7 +1450,17 @@ fn print_zero_error_envelope(err: &str, paths: &[String]) { print_json(&result); } -pub async fn run(mut args: ScanArgs) -> i32 { +pub async fn run(args: ScanArgs) -> i32 { + // Scan's telemetry sends run off the critical path (spawned where each + // event fires) and are all awaited here, before the command returns — + // so every event is still delivered before the process exits. + let mut telemetry = PendingTelemetry::new(); + let code = Box::pin(run_scan(args, &mut telemetry)).await; + telemetry.flush().await; + code +} + +async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { apply_env_toggles(&args.common); // Fold the legacy mode booleans into `args.mode` before anything reads @@ -1719,7 +1731,8 @@ pub async fn run(mut args: ScanArgs) -> i32 { } } // Telemetry: empty-scan still counts as a successful scan. - track_patch_scanned( + spawn_patch_scanned( + telemetry, 0, 0, 0, @@ -1732,8 +1745,7 @@ pub async fn run(mut args: ScanArgs) -> i32 { false, telemetry_token.as_deref(), telemetry_org.as_deref(), - ) - .await; + ); if args.common.json { // When the crawler finds nothing, GC is intentionally skipped // — pruning every manifest entry on the assumption that the @@ -1976,13 +1988,13 @@ pub async fn run(mut args: ScanArgs) -> i32 { if total_batches > 0 && batch_error_count == total_batches { status.finish(); let err = last_batch_error.unwrap_or_else(|| "all batches failed".to_string()); - track_patch_scan_failed( + spawn_patch_scan_failed( + telemetry, &err, fallback_to_proxy, telemetry_token.as_deref(), telemetry_org.as_deref(), - ) - .await; + ); // A scan in which *every* batch failed produced no trustworthy // patch data. Surfacing `status: "success"` / exit 0 here would be @@ -2046,7 +2058,8 @@ pub async fn run(mut args: ScanArgs) -> i32 { // per-tier counts. `fallback_to_proxy` is `true` iff the batch // loop downgraded from the authenticated endpoint to the public // proxy after a 401/403. - track_patch_scanned( + spawn_patch_scanned( + telemetry, package_count, free_patches, paid_patches, @@ -2059,8 +2072,7 @@ pub async fn run(mut args: ScanArgs) -> i32 { fallback_to_proxy, telemetry_token.as_deref(), telemetry_org.as_deref(), - ) - .await; + ); // Read existing manifest once for update detection. Used by both the // JSON-mode emission (always includes an `updates` array) and the diff --git a/crates/socket-patch-cli/tests/telemetry_e2e.rs b/crates/socket-patch-cli/tests/telemetry_e2e.rs index 05df12ba..eb950068 100644 --- a/crates/socket-patch-cli/tests/telemetry_e2e.rs +++ b/crates/socket-patch-cli/tests/telemetry_e2e.rs @@ -728,3 +728,98 @@ async fn list_skips_telemetry_in_airgap_mode() { let count = telemetry_post_count(&mock, None).await; assert_eq!(count, 0, "SOCKET_OFFLINE=1 must suppress patch_listed"); } + +// --------------------------------------------------------------------------- +// scan: background sends are flushed before exit +// --------------------------------------------------------------------------- + +/// Scan sends its telemetry off the critical path but must flush it before +/// the process exits. Against a telemetry endpoint that answers only after +/// `TELEMETRY_DELAY`, each scan terminal — success, empty crawl, and +/// all-batches-failed — must still deliver exactly its one event AND stay +/// alive until the response arrives (the lower bound on wall time cannot +/// flake under load: load only makes a run slower). +#[tokio::test] +async fn scan_flushes_background_telemetry_before_exit() { + const TELEMETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(1500); + + struct Case { + label: &'static str, + batch_status: u16, + install_package: bool, + want_event: &'static str, + want_code: i32, + } + let cases = [ + Case { + label: "success", + batch_status: 200, + install_package: true, + want_event: "patch_scanned", + want_code: 0, + }, + Case { + label: "empty crawl", + batch_status: 200, + install_package: false, + want_event: "patch_scanned", + want_code: 0, + }, + Case { + label: "all batches failed", + batch_status: 500, + install_package: true, + want_event: "patch_scan_failed", + want_code: 1, + }, + ]; + + for case in cases { + let mock = MockServer::start().await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/patches/batch"))) + .respond_with(ResponseTemplate::new(case.batch_status).set_body_json( + serde_json::json!({ "packages": [], "canAccessPaidPatches": false }), + )) + .mount(&mock) + .await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/telemetry"))) + .respond_with(ResponseTemplate::new(201).set_delay(TELEMETRY_DELAY)) + .mount(&mock) + .await; + + let tmp = tempfile::tempdir().expect("tempdir"); + write_root_package_json(tmp.path()); + if case.install_package { + write_npm_package(tmp.path(), "minimist", "1.2.2"); + } + + let started = std::time::Instant::now(); + let (code, stdout, stderr) = run_cmd(tmp.path(), &mock.uri(), "scan", &[], &[]); + let elapsed = started.elapsed(); + assert_eq!( + code, case.want_code, + "{}: stdout={stdout} stderr={stderr}", + case.label + ); + assert_eq!( + telemetry_post_count(&mock, Some(case.want_event)).await, + 1, + "{}: exactly one {} event must be delivered", + case.label, + case.want_event + ); + assert_eq!( + telemetry_post_count(&mock, None).await, + 1, + "{}: no other telemetry event", + case.label + ); + assert!( + elapsed >= TELEMETRY_DELAY, + "{}: scan exited after {elapsed:?}, before its telemetry send completed", + case.label + ); + } +} diff --git a/crates/socket-patch-core/src/telemetry.rs b/crates/socket-patch-core/src/telemetry.rs index f704f2fd..4788cf37 100644 --- a/crates/socket-patch-core/src/telemetry.rs +++ b/crates/socket-patch-core/src/telemetry.rs @@ -259,23 +259,47 @@ fn resolve_telemetry_endpoint(api_token: Option<&str>, org_slug: Option<&str>) - } } -/// Send a telemetry event to the API. -/// -/// This is fire-and-forget: errors are logged in debug mode but never -/// propagated. Uses `reqwest` with a 5-second request timeout and a -/// 2-second connect timeout: the send is awaited inline by every command -/// before it prints, so a network that blackholes the endpoint (dropped -/// SYNs, no RST) must give up on the handshake quickly rather than stall -/// even a read-only `scan --json` for the full request budget. -async fn send_telemetry_event( - event: &PatchTelemetryEvent, +/// A telemetry event with its destination resolved, ready to POST. Built +/// synchronously where the event fires (timestamp, env and config reads, +/// the "Sending telemetry" debug line), so sending it inline or from a +/// background task posts the very same request. +struct PreparedSend { + event: PatchTelemetryEvent, + url: String, + /// The bearer token to attach (authenticated endpoint only). + bearer: Option, +} + +/// Resolve `event`'s endpoint (see [`resolve_telemetry_endpoint`]). +fn prepare_send( + event: PatchTelemetryEvent, api_token: Option<&str>, org_slug: Option<&str>, -) { +) -> PreparedSend { let (url, use_auth) = resolve_telemetry_endpoint(api_token, org_slug); debug_log(&format!("Sending telemetry to {url}")); + let bearer = if use_auth { + api_token.map(str::to_string) + } else { + None + }; + PreparedSend { event, url, bearer } +} + +/// Send a telemetry event to the API. +/// +/// This is fire-and-forget: errors are logged in debug mode but never +/// propagated. Uses `reqwest` with a 5-second request timeout and a +/// 2-second connect timeout: a command awaits every send before it exits +/// (inline, or via [`PendingTelemetry::flush`]), so a network that +/// blackholes the endpoint (dropped SYNs, no RST) must give up on the +/// handshake quickly rather than stall even a read-only `scan --json` for +/// the full request budget. +async fn send_telemetry_event(prepared: PreparedSend) { + let PreparedSend { event, url, bearer } = prepared; + let client = match reqwest::Client::builder() .connect_timeout(std::time::Duration::from_secs(2)) .timeout(std::time::Duration::from_secs(5)) @@ -293,13 +317,11 @@ async fn send_telemetry_event( .header("Content-Type", "application/json") .header("User-Agent", USER_AGENT); - if use_auth { - if let Some(token) = api_token { - request = request.header("Authorization", format!("Bearer {token}")); - } + if let Some(token) = bearer { + request = request.header("Authorization", format!("Bearer {token}")); } - match request.json(event).send().await { + match request.json(&event).send().await { Ok(response) => { let status = response.status(); if status.is_success() { @@ -314,6 +336,37 @@ async fn send_telemetry_event( } } +/// Telemetry sends a command started off its critical path. Each send is +/// spawned where its event fires (the event is built right there, so its +/// body and timestamp are what an inline send would have posted) and the +/// command awaits [`Self::flush`] before it returns, so every event is +/// still delivered — or given up on within the same 2 s connect / 5 s +/// request budget — before the process exits. +#[derive(Debug, Default)] +pub struct PendingTelemetry { + sends: Vec>, +} + +impl PendingTelemetry { + pub fn new() -> Self { + Self::default() + } + + /// Await every send started so far, in start order. + pub async fn flush(self) { + for send in self.sends { + // A send never panics on its own; a JoinError here can only be + // a runtime shutting down, which leaves nothing to deliver. + let _ = send.await; + } + } + + fn spawn(&mut self, prepared: PreparedSend) { + self.sends + .push(tokio::spawn(send_telemetry_event(prepared))); + } +} + // --------------------------------------------------------------------------- // Per-event tracker wrappers (the public API) // @@ -321,24 +374,21 @@ async fn send_telemetry_event( // convenient (callers typically have `Option` and call `.as_deref()`). // --------------------------------------------------------------------------- -/// Shared fire-and-forget helper for the per-event tracker wrappers below. -/// -/// Non-blocking and never returns errors: telemetry failures are logged in -/// debug mode but do not affect CLI operation. Returns immediately when +/// Build the event the tracker wrappers below send, or `None` when /// telemetry is disabled via environment variables. `metadata` is a /// `serde_json::json!({...})` object; non-object / empty values are dropped /// to avoid `.unwrap()` noise at every call site. -async fn fire( +fn prepare( event_type: PatchTelemetryEventType, command: &'static str, metadata: serde_json::Value, error: Option, api_token: Option<&str>, org_slug: Option<&str>, -) { +) -> Option { if is_telemetry_disabled() { debug_log("Telemetry is disabled, skipping event"); - return; + return None; } let metadata = match metadata { @@ -347,7 +397,25 @@ async fn fire( }; let error = error.map(|e| ("Error".to_string(), e.to_string())); let event = build_telemetry_event(event_type, command, metadata, error); - send_telemetry_event(&event, api_token, org_slug).await; + Some(prepare_send(event, api_token, org_slug)) +} + +/// Shared fire-and-forget helper for the per-event tracker wrappers below. +/// +/// Never returns errors: telemetry failures are logged in debug mode but +/// do not affect CLI operation. Returns immediately when telemetry is +/// disabled via environment variables. +async fn fire( + event_type: PatchTelemetryEventType, + command: &'static str, + metadata: serde_json::Value, + error: Option, + api_token: Option<&str>, + org_slug: Option<&str>, +) { + if let Some(prepared) = prepare(event_type, command, metadata, error, api_token, org_slug) { + send_telemetry_event(prepared).await; + } } /// Track a successful patch application. @@ -497,6 +565,27 @@ pub async fn track_patch_rollback_failed( // Read-side trackers: scan + get // --------------------------------------------------------------------------- +/// The `patch_scanned` metadata: per-tier patch counts and whether the +/// call was downgraded to the public proxy after an auth-endpoint 401/403 +/// (`fallback_to_proxy`). +fn patch_scanned_metadata( + packages_scanned: usize, + free_patches: usize, + paid_patches: usize, + can_access_paid: bool, + ecosystems: &[String], + fallback_to_proxy: bool, +) -> serde_json::Value { + serde_json::json!({ + "packages_scanned": packages_scanned, + "free_patches": free_patches, + "paid_patches": paid_patches, + "can_access_paid": can_access_paid, + "ecosystems": ecosystems, + "fallback_to_proxy": fallback_to_proxy, + }) +} + /// Track a successful `scan`. Reports per-tier patch counts and whether /// the call was downgraded to the public proxy after an auth-endpoint /// 401/403 (`fallback_to_proxy`). @@ -519,14 +608,14 @@ pub async fn track_patch_scanned( fire( PatchTelemetryEventType::PatchScanned, "scan", - serde_json::json!({ - "packages_scanned": packages_scanned, - "free_patches": free_patches, - "paid_patches": paid_patches, - "can_access_paid": can_access_paid, - "ecosystems": ecosystems, - "fallback_to_proxy": fallback_to_proxy, - }), + patch_scanned_metadata( + packages_scanned, + free_patches, + paid_patches, + can_access_paid, + ecosystems, + fallback_to_proxy, + ), None::<&str>, api_token, org_slug, @@ -534,6 +623,40 @@ pub async fn track_patch_scanned( .await; } +/// [`track_patch_scanned`], sent in the background: the event is built +/// now and its send joins `pending`, which the command flushes before it +/// returns. +#[allow(clippy::too_many_arguments)] +pub fn spawn_patch_scanned( + pending: &mut PendingTelemetry, + packages_scanned: usize, + free_patches: usize, + paid_patches: usize, + can_access_paid: bool, + ecosystems: &[String], + fallback_to_proxy: bool, + api_token: Option<&str>, + org_slug: Option<&str>, +) { + if let Some(prepared) = prepare( + PatchTelemetryEventType::PatchScanned, + "scan", + patch_scanned_metadata( + packages_scanned, + free_patches, + paid_patches, + can_access_paid, + ecosystems, + fallback_to_proxy, + ), + None::<&str>, + api_token, + org_slug, + ) { + pending.spawn(prepared); + } +} + /// Track a failed `scan`. pub async fn track_patch_scan_failed( error: impl std::fmt::Display, @@ -552,6 +675,27 @@ pub async fn track_patch_scan_failed( .await; } +/// [`track_patch_scan_failed`], sent in the background (see +/// [`spawn_patch_scanned`]). +pub fn spawn_patch_scan_failed( + pending: &mut PendingTelemetry, + error: impl std::fmt::Display, + fallback_to_proxy: bool, + api_token: Option<&str>, + org_slug: Option<&str>, +) { + if let Some(prepared) = prepare( + PatchTelemetryEventType::PatchScanFailed, + "scan", + serde_json::json!({ "fallback_to_proxy": fallback_to_proxy }), + Some(error), + api_token, + org_slug, + ) { + pending.spawn(prepared); + } +} + /// Track a successful `get`. Reports patch identity + delivery mode and /// whether the call was downgraded to the public proxy after an /// auth-endpoint 401/403. @@ -725,6 +869,88 @@ pub async fn track_vex_failed( mod tests { use super::*; + /// A prepared event posted to `server`'s `/telemetry` route. + fn prepared_for(server: &wiremock::MockServer) -> PreparedSend { + let mut metadata = HashMap::new(); + metadata.insert("packages_scanned".to_string(), serde_json::json!(3)); + PreparedSend { + event: build_telemetry_event( + PatchTelemetryEventType::PatchScanned, + "scan", + Some(metadata), + None, + ), + url: format!("{}/telemetry", server.uri()), + bearer: Some("tok".to_string()), + } + } + + /// A background send posts exactly the request an inline send posts + /// (same body bytes and headers), and is delivered by `flush`. + #[tokio::test] + async fn background_send_posts_the_inline_request_and_flush_delivers_it() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/telemetry")) + .respond_with(ResponseTemplate::new(201)) + .mount(&server) + .await; + + let prepared = prepared_for(&server); + let twin = PreparedSend { + event: prepared.event.clone(), + url: prepared.url.clone(), + bearer: prepared.bearer.clone(), + }; + send_telemetry_event(prepared).await; + let mut pending = PendingTelemetry::new(); + pending.spawn(twin); + pending.flush().await; + + let reqs = server.received_requests().await.unwrap(); + assert_eq!(reqs.len(), 2); + assert_eq!(reqs[0].body, reqs[1].body); + for req in &reqs { + assert_eq!(req.headers.get("authorization").unwrap(), "Bearer tok"); + assert_eq!(req.headers.get("user-agent").unwrap(), USER_AGENT); + assert_eq!(req.headers.get("content-type").unwrap(), "application/json"); + } + } + + /// Spawning returns at once; `flush` waits for the slow responses. + #[tokio::test] + async fn spawn_does_not_block_and_flush_waits_for_the_response() { + use wiremock::matchers::method; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let delay = std::time::Duration::from_millis(400); + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(201).set_delay(delay)) + .mount(&server) + .await; + + let mut pending = PendingTelemetry::new(); + pending.spawn(prepared_for(&server)); + pending.spawn(prepared_for(&server)); + // Nothing was awaited yet: both sends are still in flight. + assert_eq!(pending.sends.len(), 2); + assert!(pending.sends.iter().all(|s| !s.is_finished())); + let started = std::time::Instant::now(); + pending.flush().await; + assert!(started.elapsed() >= delay, "flush must await the responses"); + assert_eq!(server.received_requests().await.unwrap().len(), 2); + } + + /// Nothing started, nothing to wait for. + #[tokio::test] + async fn flushing_nothing_returns() { + PendingTelemetry::new().flush().await; + } + /// Combined into a single test to avoid env-var races across parallel tests. /// Exercises the `SOCKET_TELEMETRY_DISABLED` name, the legacy /// `SOCKET_PATCH_TELEMETRY_DISABLED` shim, and the airgap gate via From cf6f20c95801cbf7bdf1bcdf9b8cd54d4a7060a0 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 13:48:59 -0400 Subject: [PATCH 007/237] fix(scan): deliver background telemetry before the first stdout write The background send was only awaited after run_scan returned, so a process killed after the event fired but before that flush lost it: `scan | head` / `scan | true` dies of SIGPIPE on its first result write (main restores SIG_DFL), and a Ctrl-C at a confirm prompt or a CI SIGTERM had the same effect. The inline send it replaced had always landed before any output. `PendingTelemetry::flush` now drains (`&mut self`), and scan flushes at the first output point after each event fires: right after the send on the empty-crawl and all-batches-failed terminals (they print at once), at the start of the human section (before the table, prompts and every human exit), before the plain `--json` envelope, and inside `discover_selected` right after the detail fetches (before its error line and whatever the `--apply`, hosted and vendored `--json` arms print next). The send still overlaps the by-package detail fetches on those arms; the flush at the end of `run` stays as the exit backstop. Under `--debug` this also puts the human path's "Telemetry sent" line back ahead of the per-package detail warnings, as in the inline order. Tests: telemetry_e2e runs each JSON terminal with stdout closed before the child writes and requires the event delivered (red on the previous commit: SIGPIPE, 0 events); a core unit test pins that flush drains and that sends started after it join the next flush. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commands/scan/hosted.rs | 4 + .../socket-patch-cli/src/commands/scan/mod.rs | 25 +++- .../src/commands/scan/vendor_flow.rs | 8 +- .../socket-patch-cli/tests/telemetry_e2e.rs | 133 +++++++++++++++++- crates/socket-patch-core/src/telemetry.rs | 43 +++++- 5 files changed, 197 insertions(+), 16 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index c11c9a44..def69ed8 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -991,6 +991,9 @@ pub(super) async fn run_redirect( // it so the hosted `--json` envelope stays schema-consistent with every // other scan; `.take()` at each terminal (error or success) folds it in. mut scan_result: Option, + // Scan's pending telemetry, flushed by `discover_selected` before + // anything below writes to stdout. + telemetry: &mut socket_patch_core::telemetry::PendingTelemetry, ) -> i32 { // Same discovery/selection as `--apply`/`--vendor`. let selected = match discover_selected( @@ -1000,6 +1003,7 @@ pub(super) async fn run_redirect( &args.common, false, false, + telemetry, ) .await { diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index e48cdee9..ef6ff472 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -454,9 +454,14 @@ async fn discover_selected( common: &GlobalArgs, show_progress: bool, warn: bool, + telemetry: &mut PendingTelemetry, ) -> Result, (i32, String)> { let (all_search_results, failures) = fetch_patch_details(api_client, packages, show_progress, warn).await; + // The scan event's send overlapped the detail fetches; every caller's + // next output (the error line below, a `--json` envelope, a prompt) + // must find it delivered. + telemetry.flush().await; let error_count = failures.len(); if error_count > 0 && error_count == packages.len() { let err = failures @@ -1451,9 +1456,11 @@ fn print_zero_error_envelope(err: &str, paths: &[String]) { } pub async fn run(args: ScanArgs) -> i32 { - // Scan's telemetry sends run off the critical path (spawned where each - // event fires) and are all awaited here, before the command returns — - // so every event is still delivered before the process exits. + // Scan's telemetry sends run off the critical path: each is spawned + // where its event fires and flushed before the first stdout write that + // follows it (so a closed pipe's SIGPIPE, or a Ctrl-C at a prompt, still + // finds it delivered, as with an inline send). The flush here is the + // backstop that keeps every event ahead of the process exit. let mut telemetry = PendingTelemetry::new(); let code = Box::pin(run_scan(args, &mut telemetry)).await; telemetry.flush().await; @@ -1746,6 +1753,8 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { telemetry_token.as_deref(), telemetry_org.as_deref(), ); + // The result prints right away: nothing to overlap the send with. + telemetry.flush().await; if args.common.json { // When the crawler finds nothing, GC is intentionally skipped // — pruning every manifest entry on the assumption that the @@ -1995,6 +2004,8 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { telemetry_token.as_deref(), telemetry_org.as_deref(), ); + // The failure prints right away: nothing to overlap the send with. + telemetry.flush().await; // A scan in which *every* batch failed produced no trustworthy // patch data. Surfacing `status: "success"` / exit 0 here would be @@ -2180,6 +2191,7 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { &all_packages_with_patches, can_access_paid_patches, Some(result), + telemetry, ) .await; } @@ -2224,6 +2236,7 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { &args.common, false, false, + telemetry, ) .await { @@ -2366,6 +2379,7 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { prune, telemetry_token.as_deref(), telemetry_org.as_deref(), + telemetry, ) .await; } @@ -2391,10 +2405,14 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { &mut result, ) .await; + telemetry.flush().await; print_json(&result); return final_code; } + // Every human exit below prints first; the scan event goes out before. + telemetry.flush().await; + let use_color = ui::stdout_color(); let verbose = args.common.verbose; let silent = args.common.silent; @@ -2610,6 +2628,7 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { &args.common, human, !silent, + telemetry, ) .await { diff --git a/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs b/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs index 662bc2ee..42050908 100644 --- a/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs +++ b/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs @@ -21,7 +21,7 @@ use socket_patch_core::api::client::ApiClient; use socket_patch_core::api::types::{BatchPackagePatches, PatchResponse, PatchSearchResult}; use socket_patch_core::manifest::operations::{read_manifest, write_manifest}; use socket_patch_core::manifest::schema::{PatchManifest, PatchRecord}; -use socket_patch_core::telemetry::track_patch_vendor_failed; +use socket_patch_core::telemetry::{track_patch_vendor_failed, PendingTelemetry}; use socket_patch_core::utils::purl::strip_purl_qualifiers; use socket_patch_core::vendor::{load_state, lookup_entry, save_state, VendorState}; use std::collections::{HashMap, HashSet}; @@ -457,6 +457,9 @@ async fn run_vendor_json_path( prune: bool, telemetry_token: Option<&str>, telemetry_org: Option<&str>, + // Scan's pending telemetry, flushed by `discover_selected` before + // anything below writes to stdout. + telemetry: &mut PendingTelemetry, ) -> i32 { // Same discovery as `--apply`. Vendored purls are NOT filtered here — // re-vendoring a stale uuid is the point of the flag (same-uuid re-runs @@ -468,6 +471,7 @@ async fn run_vendor_json_path( &args.common, false, false, + telemetry, ) .await { @@ -818,6 +822,7 @@ pub(super) fn boxed_vendor_json_path<'a>( prune: bool, telemetry_token: Option<&'a str>, telemetry_org: Option<&'a str>, + telemetry: &'a mut PendingTelemetry, ) -> std::pin::Pin + 'a>> { Box::pin(run_vendor_json_path( args, @@ -833,6 +838,7 @@ pub(super) fn boxed_vendor_json_path<'a>( prune, telemetry_token, telemetry_org, + telemetry, )) } diff --git a/crates/socket-patch-cli/tests/telemetry_e2e.rs b/crates/socket-patch-cli/tests/telemetry_e2e.rs index eb950068..6f7df9e1 100644 --- a/crates/socket-patch-cli/tests/telemetry_e2e.rs +++ b/crates/socket-patch-cli/tests/telemetry_e2e.rs @@ -46,6 +46,24 @@ fn run_cmd( extra_args: &[&str], extra_env: &[(&str, &str)], ) -> (i32, String, String) { + let out = build_cmd(cwd, api_url, subcommand, extra_args, extra_env) + .output() + .expect("run socket-patch"); + ( + out.status.code().unwrap_or(-1), + String::from_utf8_lossy(&out.stdout).to_string(), + String::from_utf8_lossy(&out.stderr).to_string(), + ) +} + +/// The [`run_cmd`] invocation, unstarted (for tests that wire its stdio). +fn build_cmd( + cwd: &Path, + api_url: &str, + subcommand: &str, + extra_args: &[&str], + extra_env: &[(&str, &str)], +) -> Command { let mut args = vec![ subcommand, "--json", @@ -109,12 +127,7 @@ fn run_cmd( for (k, v) in extra_env { cmd.env(k, v); } - let out = cmd.output().expect("run socket-patch"); - ( - out.status.code().unwrap_or(-1), - String::from_utf8_lossy(&out.stdout).to_string(), - String::from_utf8_lossy(&out.stderr).to_string(), - ) + cmd } /// Count POSTs the wiremock server received against the telemetry @@ -823,3 +836,111 @@ async fn scan_flushes_background_telemetry_before_exit() { ); } } + +/// A consumer that exits early (`scan | head`, `scan | true`) closes +/// stdout, and the CLI dies of SIGPIPE on its first result write (main +/// restores SIG_DFL). The background send must already be delivered by +/// then — flushed before that write, as the inline send it replaced was — +/// not lost with the process. Covers each JSON flush point: the empty-crawl +/// and all-batches-failed terminals, the plain envelope, and the hosted and +/// vendored arms (flushed at `discover_selected`). The first three print +/// right after the event fires, so they fail deterministically without the +/// flush; the hosted/vendored arms do enough work before printing that an +/// unflushed send usually wins the race anyway, so for them this is a +/// delivery check rather than a pin on the flush point. +#[tokio::test] +async fn scan_delivers_telemetry_before_writing_to_a_closed_stdout() { + const TELEMETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(800); + + struct Case { + label: &'static str, + batch_status: u16, + install_package: bool, + extra_args: &'static [&'static str], + want_event: &'static str, + } + let cases = [ + Case { + label: "plain envelope", + batch_status: 200, + install_package: true, + extra_args: &[], + want_event: "patch_scanned", + }, + Case { + label: "empty crawl", + batch_status: 200, + install_package: false, + extra_args: &[], + want_event: "patch_scanned", + }, + Case { + label: "all batches failed", + batch_status: 500, + install_package: true, + extra_args: &[], + want_event: "patch_scan_failed", + }, + Case { + label: "hosted", + batch_status: 200, + install_package: true, + extra_args: &["--mode", "hosted", "--dry-run"], + want_event: "patch_scanned", + }, + Case { + label: "vendored", + batch_status: 200, + install_package: true, + extra_args: &["--mode", "vendored", "--dry-run"], + want_event: "patch_scanned", + }, + ]; + + for case in cases { + let mock = MockServer::start().await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/patches/batch"))) + .respond_with(ResponseTemplate::new(case.batch_status).set_body_json( + serde_json::json!({ "packages": [], "canAccessPaidPatches": false }), + )) + .mount(&mock) + .await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/telemetry"))) + .respond_with(ResponseTemplate::new(201).set_delay(TELEMETRY_DELAY)) + .mount(&mock) + .await; + + let tmp = tempfile::tempdir().expect("tempdir"); + write_root_package_json(tmp.path()); + if case.install_package { + write_npm_package(tmp.path(), "minimist", "1.2.2"); + } + + let mut child = build_cmd(tmp.path(), &mock.uri(), "scan", case.extra_args, &[]) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("spawn socket-patch"); + // Close the read end before the child can write anything: its + // first stdout write now raises SIGPIPE. + drop(child.stdout.take()); + let status = child.wait().expect("wait socket-patch"); + + assert_eq!( + telemetry_post_count(&mock, Some(case.want_event)).await, + 1, + "{}: the {} event must be delivered before stdout is written \ + (exit status {status:?})", + case.label, + case.want_event + ); + assert_eq!( + telemetry_post_count(&mock, None).await, + 1, + "{}: no other telemetry event", + case.label + ); + } +} diff --git a/crates/socket-patch-core/src/telemetry.rs b/crates/socket-patch-core/src/telemetry.rs index 4788cf37..cc80be79 100644 --- a/crates/socket-patch-core/src/telemetry.rs +++ b/crates/socket-patch-core/src/telemetry.rs @@ -339,9 +339,11 @@ async fn send_telemetry_event(prepared: PreparedSend) { /// Telemetry sends a command started off its critical path. Each send is /// spawned where its event fires (the event is built right there, so its /// body and timestamp are what an inline send would have posted) and the -/// command awaits [`Self::flush`] before it returns, so every event is -/// still delivered — or given up on within the same 2 s connect / 5 s -/// request budget — before the process exits. +/// command awaits [`Self::flush`] before its first stdout write after that +/// point — the send overlaps only the work in between, and is delivered (or +/// given up on within the same 2 s connect / 5 s request budget) before +/// any output that could raise SIGPIPE, and before any prompt a Ctrl-C +/// could interrupt, exactly as an inline send was. #[derive(Debug, Default)] pub struct PendingTelemetry { sends: Vec>, @@ -352,9 +354,10 @@ impl PendingTelemetry { Self::default() } - /// Await every send started so far, in start order. - pub async fn flush(self) { - for send in self.sends { + /// Await every send started so far, in start order. Idempotent: a + /// second flush with nothing started since returns at once. + pub async fn flush(&mut self) { + for send in std::mem::take(&mut self.sends) { // A send never panics on its own; a JoinError here can only be // a runtime shutting down, which leaves nothing to deliver. let _ = send.await; @@ -951,6 +954,34 @@ mod tests { PendingTelemetry::new().flush().await; } + /// Scan flushes at each output point after a send fires, so `flush` + /// drains: a later flush waits only for sends started since, and the + /// exit backstop after an early flush has nothing left to wait for. + #[tokio::test] + async fn flush_drains_and_later_sends_join_the_next_flush() { + use wiremock::matchers::method; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(201)) + .mount(&server) + .await; + + let mut pending = PendingTelemetry::new(); + pending.spawn(prepared_for(&server)); + pending.flush().await; + assert!(pending.sends.is_empty()); + assert_eq!(server.received_requests().await.unwrap().len(), 1); + pending.flush().await; + assert_eq!(server.received_requests().await.unwrap().len(), 1); + + pending.spawn(prepared_for(&server)); + pending.flush().await; + assert!(pending.sends.is_empty()); + assert_eq!(server.received_requests().await.unwrap().len(), 2); + } + /// Combined into a single test to avoid env-var races across parallel tests. /// Exercises the `SOCKET_TELEMETRY_DISABLED` name, the legacy /// `SOCKET_PATCH_TELEMETRY_DISABLED` shim, and the airgap gate via From 0636854b679ce7aa784965a0493437c3fe6694bc Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 13:48:59 -0400 Subject: [PATCH 008/237] refactor(telemetry): share the patch_scan_failed metadata builder `track_patch_scan_failed` and `spawn_patch_scan_failed` each spelled out the `{"fallback_to_proxy": ...}` literal; build it in one place, as `patch_scanned_metadata` already is for the success event, so the inline and background paths cannot drift. The inline trackers stay: they are public API of the published core crate. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/socket-patch-core/src/telemetry.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/socket-patch-core/src/telemetry.rs b/crates/socket-patch-core/src/telemetry.rs index cc80be79..156c9106 100644 --- a/crates/socket-patch-core/src/telemetry.rs +++ b/crates/socket-patch-core/src/telemetry.rs @@ -660,6 +660,11 @@ pub fn spawn_patch_scanned( } } +/// The `patch_scan_failed` metadata (see [`patch_scanned_metadata`]). +fn patch_scan_failed_metadata(fallback_to_proxy: bool) -> serde_json::Value { + serde_json::json!({ "fallback_to_proxy": fallback_to_proxy }) +} + /// Track a failed `scan`. pub async fn track_patch_scan_failed( error: impl std::fmt::Display, @@ -670,7 +675,7 @@ pub async fn track_patch_scan_failed( fire( PatchTelemetryEventType::PatchScanFailed, "scan", - serde_json::json!({ "fallback_to_proxy": fallback_to_proxy }), + patch_scan_failed_metadata(fallback_to_proxy), Some(error), api_token, org_slug, @@ -690,7 +695,7 @@ pub fn spawn_patch_scan_failed( if let Some(prepared) = prepare( PatchTelemetryEventType::PatchScanFailed, "scan", - serde_json::json!({ "fallback_to_proxy": fallback_to_proxy }), + patch_scan_failed_metadata(fallback_to_proxy), Some(error), api_token, org_slug, From 590476922305ec30afa0f594e07168438f6d3690 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 13:49:00 -0400 Subject: [PATCH 009/237] refactor(core): keep the collecting concurrency helper test-only No production caller used `map_ordered_concurrent`: every API loop consumes `ordered_concurrent` directly. Move it into the tests module so it no longer ships as unused public API. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../socket-patch-core/src/utils/concurrent.rs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/socket-patch-core/src/utils/concurrent.rs b/crates/socket-patch-core/src/utils/concurrent.rs index 725d2f25..64c80e82 100644 --- a/crates/socket-patch-core/src/utils/concurrent.rs +++ b/crates/socket-patch-core/src/utils/concurrent.rs @@ -56,22 +56,22 @@ where stream::iter(items).map(f).buffered(limit.max(1)) } -/// [`ordered_concurrent`], collected: every result, in input order. -pub async fn map_ordered_concurrent(items: I, limit: usize, f: F) -> Vec -where - I: IntoIterator, - F: FnMut(I::Item) -> Fut, - Fut: Future, -{ - ordered_concurrent(items, limit, f).collect().await -} - #[cfg(test)] mod tests { use super::*; use std::cell::Cell; use std::time::Duration; + /// [`ordered_concurrent`], collected: every result, in input order. + async fn map_ordered_concurrent(items: I, limit: usize, f: F) -> Vec + where + I: IntoIterator, + F: FnMut(I::Item) -> Fut, + Fut: Future, + { + ordered_concurrent(items, limit, f).collect().await + } + /// Later items finish first (reversed latencies); results still come /// back in input order. #[tokio::test(start_paused = true)] From b3565c06667833abfd0761a52c65e3239f5d4264 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 12:55:42 -0400 Subject: [PATCH 010/237] perf(crawl): walk npm trees on the blocking pool with parallel gather + ordered merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `crawl_all` and the workspace roots walk made one `spawn_blocking` round trip per readdir, stat and package.json read, strictly in sequence. Both now run as one blocking-pool task: directory I/O is gathered in parallel (rayon, already in the dependency graph via qbsdiff) into per-root event trees that record the sequential visit order, and a single-threaded merge replays them so the order-dependent `seen` dedup and the store entries' `identity_seen` decisions see exactly the state the old walk saw — same packages, same paths, same order. Two probes are answered from listings the walk reads anyway, only where that is provably the same answer: - the roots walk skips the `is_dir(child/node_modules)` stat when the child's complete listing holds nothing that could alias `node_modules` on a case-insensitive filesystem (a listed dir still stats: a readable-but-unsearchable parent lists kinds while stats fail); - a store entry's `node_modules` existence probe is the readdir the scan needs next; a dir that does not open falls back to the stat. FIFO-safe package.json reads (read_regular_to_string_sync), the NESTED_STORE depth/dir caps (kept sequential: the budget order decides survivors), symlink-not-traversed rules and lossy-vs-raw name joins are unchanged. The previous async implementation is kept verbatim as a #[cfg(test)] oracle; a randomized fixture test (flat/nested/legacy stores, scoped, live/dangling/store symlinks, duplicate identities, aliases, broken/BOM/FIFO/dir package.json, unreadable and unsearchable dirs, node_modules case variants) plus a kitchen-sink tree assert identical roots, crawl output, find_by_purls results and store enumeration. Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 1 + Cargo.toml | 1 + crates/socket-patch-core/Cargo.toml | 1 + .../src/crawlers/npm_crawler.rs | 901 ++++++---- .../src/crawlers/npm_crawler/oracle.rs | 1555 +++++++++++++++++ crates/socket-patch-core/src/utils/fs.rs | 44 + 6 files changed, 2153 insertions(+), 350 deletions(-) create mode 100644 crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs diff --git a/Cargo.lock b/Cargo.lock index ee1214c8..e72f6d2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1699,6 +1699,7 @@ dependencies = [ "libc", "once_cell", "qbsdiff", + "rayon", "regex", "reqwest", "same-file", diff --git a/Cargo.toml b/Cargo.toml index 71cf4491..1636d842 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ glob = "=0.3.4" toml_edit = "=0.25.12" once_cell = "=1.21.3" qbsdiff = "=1.4.4" +rayon = "=1.12.0" tar = "=0.4.46" flate2 = "=1.1.9" zip = { version = "=8.6.0", default-features = false, features = ["deflate"] } diff --git a/crates/socket-patch-core/Cargo.toml b/crates/socket-patch-core/Cargo.toml index 3ea57f07..560cbe9e 100644 --- a/crates/socket-patch-core/Cargo.toml +++ b/crates/socket-patch-core/Cargo.toml @@ -23,6 +23,7 @@ regex = { workspace = true } toml_edit = { workspace = true } once_cell = { workspace = true } qbsdiff = { workspace = true } +rayon = { workspace = true } tar = { workspace = true } flate2 = { workspace = true } fs2 = { workspace = true } diff --git a/crates/socket-patch-core/src/crawlers/npm_crawler.rs b/crates/socket-patch-core/src/crawlers/npm_crawler.rs index cb186fbb..d34c24cf 100644 --- a/crates/socket-patch-core/src/crawlers/npm_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/npm_crawler.rs @@ -1,11 +1,17 @@ use std::collections::{HashMap, HashSet, VecDeque}; +use std::ffi::{OsStr, OsString}; +use std::fs::FileType; use std::path::{Path, PathBuf}; +use rayon::prelude::*; use serde::Deserialize; use super::types::{CrawledPackage, CrawlerOptions}; use crate::patch::path_safety; -use crate::utils::fs::is_dir; +use crate::utils::fs::{is_dir, is_dir_sync, read_dir_entries_sync, run_blocking}; + +#[cfg(test)] +mod oracle; use crate::utils::purl::{percent_decode_purl_component, strip_purl_qualifiers}; /// Directories to skip when searching for workspace node_modules. @@ -40,11 +46,25 @@ pub async fn read_package_json(pkg_json_path: &Path) -> Option<(String, String)> let content = crate::utils::fs::read_regular_to_string(pkg_json_path) .await .ok()?; + parse_package_json_identity(&content) +} + +/// Blocking twin of [`read_package_json`] for the walks that run whole on +/// the blocking pool: the same FIFO-safe `read_regular_to_string_sync` +/// open and the same parse. +fn read_package_json_sync(pkg_json_path: &Path) -> Option<(String, String)> { + let content = crate::utils::fs::read_regular_to_string_sync(pkg_json_path).ok()?; + parse_package_json_identity(&content) +} + +/// `(name, version)` from package.json text, if both are present and +/// non-empty. +fn parse_package_json_identity(content: &str) -> Option<(String, String)> { // npm and Node both tolerate a leading UTF-8 BOM in package.json // (Windows-authored packages ship them), but serde_json rejects it — // a BOM'd install would be invisible to scan and unpatchable. let pkg: PackageJsonPartial = - serde_json::from_str(crate::package_json::detect::strip_bom(&content)).ok()?; + serde_json::from_str(crate::package_json::detect::strip_bom(content)).ok()?; let name = pkg.name?; let version = pkg.version?; if name.is_empty() || version.is_empty() { @@ -187,6 +207,128 @@ fn is_legacy_pnpm_store_dir_name(name: &str) -> bool { name.starts_with(".registry.") } +// --------------------------------------------------------------------------- +// Blocking-pool walk primitives +// --------------------------------------------------------------------------- + +/// One entry of a directory listing read on the blocking pool. +struct ListedEntry { + /// The raw name — what the walks join wherever they always joined the + /// `OsString`. + name: OsString, + /// The lossy UTF-8 spelling every name test (and the joins that always + /// used it) goes through. + name_str: String, + /// The entry's own, symlink-unaware kind; `None` when that stat failed, + /// which every walk treats as "skip this entry". + file_type: Option, +} + +/// A directory listing plus whether it is known complete (see +/// [`read_dir_entries_sync`]). A directory that cannot be opened lists as +/// empty and incomplete. +#[derive(Default)] +struct Listing { + entries: Vec, + complete: bool, +} + +impl Listing { + fn from_entries(entries: Vec, complete: bool) -> Self { + let entries = entries + .into_iter() + .map(|entry| { + let name = entry.file_name(); + ListedEntry { + name_str: name.to_string_lossy().into_owned(), + file_type: entry.file_type().ok(), + name, + } + }) + .collect(); + Self { entries, complete } + } +} + +/// List `path` (empty when it cannot be read — the walks' long-standing +/// tolerate-and-skip contract). +fn list_dir_sync(path: &Path) -> Listing { + match read_dir_entries_sync(path) { + Some((entries, complete)) => Listing::from_entries(entries, complete), + None => Listing::default(), + } +} + +/// Whether `dir/node_modules` is a directory, following symlinks — the +/// workspace roots walk's `is_dir` probe — skipping the stat when `dir`'s +/// own listing (which the walk reads anyway) already proves the answer is +/// no: the listing is complete and holds no entry that could be +/// `node_modules`, even on a case-insensitive filesystem (APFS, NTFS; see +/// [`may_alias_ascii_name`]). The stat then could only have failed. +/// +/// Every other case stats, including a listed real-directory +/// `node_modules`: a directory readable but not searchable (mode `r--`) +/// lists its entries' kinds while a stat through it still fails, so a +/// positive answer is never taken from the listing. Only the negative +/// case is common (most walked dirs have no `node_modules`). +fn has_node_modules_dir(dir: &Path, listing: &Listing) -> bool { + if listing.complete + && !listing + .entries + .iter() + .any(|e| may_alias_ascii_name(&e.name, "node_modules")) + { + return false; + } + is_dir_sync(&dir.join("node_modules")) +} + +/// Whether a directory entry called `name` could be what a lookup of the +/// plain-ASCII `target` resolves to on a case-insensitive filesystem: an +/// ASCII-case-insensitive match — or, conservatively, any name that is not +/// plain ASCII (Unicode case folding and normalization can map non-ASCII +/// spellings such as `ſ` or the Kelvin sign onto ASCII letters) or not +/// UTF-8 at all. +fn may_alias_ascii_name(name: &OsStr, target: &str) -> bool { + match name.to_str() { + Some(name) if name.is_ascii() => name.eq_ignore_ascii_case(target), + _ => true, + } +} + +/// What the blocking-pool scan of one `node_modules` tree records, in the +/// exact order the sequential walk visits it; +/// [`NpmCrawler::merge_scan_events`] then replays the order-dependent +/// `seen` dedup single-threaded. +enum ScanEvent { + /// A `check_package` candidate: the package dir, its package.json + /// identity (`None` = unreadable/invalid), and — for a direct child of + /// a virtual-store entry's `node_modules` only — the dir's package key + /// (`name` or `@scope/name`), which the entry's `identity_seen` skip + /// compares against. + Package { + path: PathBuf, + identity: Option<(String, String)>, + entry_key: Option, + }, + /// One virtual-store entry: its dir name decoded, and the events of its + /// `node_modules` walked under the store-entry policy. + StoreEntry { + decoded: Option<(String, String)>, + events: Vec, + }, +} + +/// A virtual-store entry found by +/// [`NpmCrawler::list_pnpm_store_entries_sync`]: its flat (or synthesized +/// nested) name, its `node_modules`, and — when the caller asked for +/// listings and the dir opened — that `node_modules`' listing. +struct StoreEntryDir { + name: String, + node_modules: PathBuf, + listing: Option, +} + // --------------------------------------------------------------------------- // Global prefix detection helpers // --------------------------------------------------------------------------- @@ -394,28 +536,6 @@ struct Target { dir_key: String, } -/// Which kind of `node_modules` directory a scan pass is walking — the one -/// traversal-policy bit that differs between them. -#[derive(Clone, Copy)] -enum ScanPolicy<'a> { - /// An importer's or package's `node_modules`: symlinked entries are - /// recorded (pnpm links direct deps; `npm link` targets) but never - /// traversed into, and a `.pnpm` child is the virtual store, scanned - /// in a deferred pass. - Importer, - /// One pnpm virtual-store entry's `node_modules`: only REAL - /// directories are inventoried — a symlinked entry here is the - /// package's dependency pointing at a sibling `.pnpm` store entry, - /// which is inventoried via that entry; following it would record the - /// same package under a path owned by a different store entry. - /// `identity_seen` optionally carries the entry's own package name - /// (what the store dir name decodes to) when its name@version is - /// already inventoried — the importer pass wins the `seen` dedup for - /// every root-linked direct dep — so that child's package.json is not - /// read a second time; everything below it is still scanned. - StoreEntry { identity_seen: Option<&'a str> }, -} - impl NpmCrawler { /// Create a new `NpmCrawler`. pub fn new() -> Self { @@ -435,31 +555,36 @@ impl NpmCrawler { &self, options: &CrawlerOptions, ) -> Result, std::io::Error> { - if options.global || options.global_prefix.is_some() { - if let Some(ref custom) = options.global_prefix { - return Ok(vec![custom.clone()]); - } - return Ok(self.get_global_node_modules_paths()); - } - - Ok(self.find_local_node_modules_dirs(&options.cwd).await) + let options = options.clone(); + Ok(run_blocking(move || Self::node_modules_paths_sync(&options)).await) } /// Crawl all discovered `node_modules` and return every package found. + /// + /// The whole walk runs as ONE blocking-pool task (instead of one + /// `spawn_blocking` round trip per readdir/stat/read): directory I/O is + /// gathered in parallel into per-root [`ScanEvent`] trees that record + /// the sequential visit order, then [`Self::merge_scan_events`] replays + /// them single-threaded so the order-dependent `seen` dedup (and the + /// store entries' `identity_seen` decisions) see exactly the state the + /// sequential walk would have — same packages, same paths, same order. pub async fn crawl_all(&self, options: &CrawlerOptions) -> Vec { - let mut packages = Vec::new(); - let mut seen = HashSet::new(); + let options = options.clone(); + run_blocking(move || Self::crawl_all_sync(&options)).await + } - let nm_paths = self - .get_node_modules_paths(options) - .await - .unwrap_or_default(); + fn crawl_all_sync(options: &CrawlerOptions) -> Vec { + let nm_paths = Self::node_modules_paths_sync(options); + let gathered: Vec> = nm_paths + .par_iter() + .map(|nm_path| Self::gather_node_modules(nm_path, None, false)) + .collect(); - for nm_path in &nm_paths { - let found = Self::scan_node_modules(nm_path, &mut seen, ScanPolicy::Importer).await; - packages.extend(found); + let mut packages = Vec::new(); + let mut seen = HashSet::new(); + for events in gathered { + Self::merge_scan_events(events, None, &mut seen, &mut packages); } - packages } @@ -828,193 +953,337 @@ impl NpmCrawler { // Private helpers – local node_modules discovery // ------------------------------------------------------------------ + /// Blocking body of [`Self::get_node_modules_paths`]. + fn node_modules_paths_sync(options: &CrawlerOptions) -> Vec { + if options.global || options.global_prefix.is_some() { + if let Some(ref custom) = options.global_prefix { + return vec![custom.clone()]; + } + return NpmCrawler.get_global_node_modules_paths(); + } + + Self::find_local_node_modules_dirs(&options.cwd) + } + /// Find `node_modules` directories within the project root. /// Recursively searches for workspace `node_modules` but stays within the /// project. - async fn find_local_node_modules_dirs(&self, start_path: &Path) -> Vec { + fn find_local_node_modules_dirs(start_path: &Path) -> Vec { let mut results = Vec::new(); + let listing = list_dir_sync(start_path); // Direct node_modules in start_path - let direct = start_path.join("node_modules"); - if is_dir(&direct).await { - results.push(direct); + if has_node_modules_dir(start_path, &listing) { + results.push(start_path.join("node_modules")); } // Recursively search for workspace node_modules - Self::find_workspace_node_modules(start_path, &mut results).await; + results.extend(Self::find_workspace_node_modules(start_path, listing)); results } - /// Recursively find `node_modules` in subdirectories (for monorepos / workspaces). - /// Skips symlinks, hidden dirs, and well-known non-workspace dirs. - fn find_workspace_node_modules<'a>( - dir: &'a Path, - results: &'a mut Vec, - ) -> std::pin::Pin + 'a>> { - Box::pin(async move { - for entry in crate::utils::fs::list_dir_entries(dir).await { - let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { - continue; - }; - if !file_type.is_dir() { - continue; - } - - let name = entry.file_name(); - let name_str = name.to_string_lossy(); - - // Skip node_modules, hidden dirs, and well-known build dirs - if name_str == "node_modules" - || name_str.starts_with('.') - || SKIP_DIRS.contains(&name_str.as_ref()) - { - continue; - } - - let full_path = dir.join(&name); - + /// Recursively find `node_modules` in subdirectories (for monorepos / + /// workspaces), given `dir`'s own listing. Skips symlinks, hidden dirs, + /// and well-known non-workspace dirs. + /// + /// Subdirectories are walked in parallel; the per-child results are + /// concatenated in listing order, each child contributing its own + /// `node_modules` first and then its subtree's — the sequential + /// depth-first order. A child whose listing (which the walk needs + /// anyway) proves it has no `node_modules` skips the stat; see + /// [`has_node_modules_dir`]. + fn find_workspace_node_modules(dir: &Path, listing: Listing) -> Vec { + let children: Vec = listing + .entries + .into_iter() + .filter(|entry| { + // Skip non-dirs (symlinks included), node_modules, hidden + // dirs, and well-known build dirs + entry.file_type.is_some_and(|ft| ft.is_dir()) + && !(entry.name_str == "node_modules" + || entry.name_str.starts_with('.') + || SKIP_DIRS.contains(&entry.name_str.as_str())) + }) + .map(|entry| dir.join(&entry.name)) + .collect(); + + children + .into_par_iter() + .map(|full_path| { + let listing = list_dir_sync(&full_path); + let mut found = Vec::new(); // Check if this subdirectory has its own node_modules - let sub_nm = full_path.join("node_modules"); - if is_dir(&sub_nm).await { - results.push(sub_nm); + if has_node_modules_dir(&full_path, &listing) { + found.push(full_path.join("node_modules")); } - // Recurse - Self::find_workspace_node_modules(&full_path, results).await; - } - }) + found.extend(Self::find_workspace_node_modules(&full_path, listing)); + found + }) + .collect::>() + .into_iter() + .flatten() + .collect() } - // ------------------------------------------------------------------ // Private helpers – scanning // ------------------------------------------------------------------ - /// Scan a `node_modules` directory, returning all valid packages found. - /// Recurses into each package's own nested `node_modules`. The one - /// policy bit distinguishing an importer/package tree from a pnpm - /// virtual-store entry is carried by [`ScanPolicy`]. - fn scan_node_modules<'a>( - node_modules_path: &'a Path, - seen: &'a mut HashSet, - policy: ScanPolicy<'a>, - ) -> std::pin::Pin> + 'a>> { - Box::pin(async move { - let mut results = Vec::new(); - let mut pnpm_store: Option = None; - let mut legacy_stores: Vec = Vec::new(); - let (store_entry, identity_seen) = match policy { - ScanPolicy::Importer => (false, None), - ScanPolicy::StoreEntry { identity_seen } => (true, identity_seen), - }; - - for entry in crate::utils::fs::list_dir_entries(node_modules_path).await { - let name = entry.file_name(); - let name_str = name.to_string_lossy().to_string(); - - // pnpm's virtual store: under the isolated linker it is the - // ONLY physical home of transitive dependencies (the - // importer's node_modules symlinks direct deps only), so - // skipping it as just-another-hidden-dir leaves every - // transitive-only install invisible to scan. Deferred until - // after this loop so root-level entries are inventoried - // first and win the `seen` name@version dedup at their - // importer-root paths. (A store entry's own children never - // include a nested `.pnpm`; under `StoreEntry` policy the - // name falls through to the hidden-entry skip below.) - if !store_entry && name_str == ".pnpm" { - let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { - continue; - }; - if file_type.is_dir() { - pnpm_store = Some(node_modules_path.join(&name_str)); - } - continue; - } - - // pnpm <=3 virtual store (a hidden `.` dir; - // no `.pnpm` exists on those layouts): same - // transitive-only-home property, same deferred scan so - // root-level entries win the `seen` dedup. Must run before - // the hidden-entry skip below, which would otherwise leave - // every transitive-only install invisible to scan. - if !store_entry && is_legacy_pnpm_store_dir_name(&name_str) { - let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { - continue; - }; - if file_type.is_dir() { - legacy_stores.push(node_modules_path.join(&name_str)); - } - continue; + /// Gather one `node_modules` directory's scan into [`ScanEvent`]s, in + /// the exact order the sequential walk visits it: the directory's own + /// entries (each package followed by its nested `node_modules`), then + /// the deferred pnpm virtual store(s). `listing` is the directory's + /// listing when the caller already read it. `store_entry` selects the + /// one policy bit distinguishing an importer/package tree from a pnpm + /// virtual-store entry's `node_modules`: + /// - importer trees accept both directories and symlinks (pnpm links + /// direct deps; `npm link` targets) but never traverse a symlink, and + /// a `.pnpm` (or pnpm <=3 `.`) child is the virtual + /// store, walked after the loop; + /// - a store entry accepts REAL directories only — a symlinked entry + /// there is the package's dependency pointing at a sibling store + /// entry, inventoried via that entry — and its direct children carry + /// their package key so the merge can apply the entry's + /// `identity_seen` skip. + /// + /// Sibling packages (and their subtrees) are gathered in parallel; the + /// results are concatenated back in listing order. + fn gather_node_modules( + node_modules_path: &Path, + listing: Option, + store_entry: bool, + ) -> Vec { + let listing = listing.unwrap_or_else(|| list_dir_sync(node_modules_path)); + let mut pnpm_store: Option = None; + let mut legacy_stores: Vec = Vec::new(); + let mut children: Vec<(PathBuf, String, FileType)> = Vec::new(); + + for entry in listing.entries { + let name_str = entry.name_str; + + // pnpm's virtual store: under the isolated linker it is the + // ONLY physical home of transitive dependencies (the + // importer's node_modules symlinks direct deps only), so + // skipping it as just-another-hidden-dir leaves every + // transitive-only install invisible to scan. Deferred until + // after this loop so root-level entries are inventoried + // first and win the `seen` name@version dedup at their + // importer-root paths. (A store entry's own children never + // include a nested `.pnpm`; under the store-entry policy the + // name falls through to the hidden-entry skip below.) + if !store_entry && name_str == ".pnpm" { + if entry.file_type.is_some_and(|ft| ft.is_dir()) { + pnpm_store = Some(node_modules_path.join(&name_str)); } + continue; + } - // Skip hidden files and node_modules - if name_str.starts_with('.') || name_str == "node_modules" { - continue; + // pnpm <=3 virtual store (a hidden `.` dir; + // no `.pnpm` exists on those layouts): same + // transitive-only-home property, same deferred scan so + // root-level entries win the `seen` dedup. Must run before + // the hidden-entry skip below, which would otherwise leave + // every transitive-only install invisible to scan. + if !store_entry && is_legacy_pnpm_store_dir_name(&name_str) { + if entry.file_type.is_some_and(|ft| ft.is_dir()) { + legacy_stores.push(node_modules_path.join(&name_str)); } + continue; + } - let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { - continue; - }; + // Skip hidden files and node_modules + if name_str.starts_with('.') || name_str == "node_modules" { + continue; + } - // Importer trees allow both directories and symlinks (pnpm - // links direct deps); a store entry accepts REAL dirs only - // (see `ScanPolicy::StoreEntry`). - let acceptable = if store_entry { - file_type.is_dir() - } else { - file_type.is_dir() || file_type.is_symlink() - }; - if !acceptable { - continue; - } + let Some(file_type) = entry.file_type else { + continue; + }; + if !Self::acceptable_package_entry(file_type, store_entry) { + continue; + } - let entry_path = node_modules_path.join(&name_str); + children.push((node_modules_path.join(&name_str), name_str, file_type)); + } + let mut events: Vec = children + .into_par_iter() + .map(|(entry_path, name_str, file_type)| { if name_str.starts_with('@') { // Scoped packages - let scoped = Self::scan_scoped_packages(&entry_path, seen, policy).await; - results.extend(scoped); + Self::gather_scoped_packages(&entry_path, &name_str, store_entry) } else { - // Regular package. `identity_seen` marks this exact dir - // as already inventoried by the importer pass — skip - // the redundant package.json read, but still descend - // below: bundled dependencies are real dirs nested - // inside the package itself (pnpm cannot link them - // out), physically present only here. - if identity_seen != Some(name_str.as_str()) { - if let Some(pkg) = Self::check_package(&entry_path, seen).await { - results.push(pkg); - } + Self::gather_package( + entry_path, + store_entry.then_some(name_str), + file_type.is_dir(), + ) + } + }) + .collect::>() + .into_iter() + .flatten() + .collect(); + + if let Some(store_path) = pnpm_store { + let entries = Self::list_pnpm_store_entries_sync(&store_path, true); + events.extend(Self::gather_store_entries(entries)); + } + for store_path in legacy_stores { + let entries = Self::collect_nested_store_entries_sync(&store_path) + .into_iter() + .map(|(name, node_modules)| StoreEntryDir { + name, + node_modules, + listing: None, + }) + .collect(); + events.extend(Self::gather_store_entries(entries)); + } + + events + } + + /// Importer trees allow both directories and symlinks (pnpm links + /// direct deps); a store entry accepts REAL dirs only (see + /// [`Self::gather_node_modules`]). + fn acceptable_package_entry(file_type: FileType, store_entry: bool) -> bool { + if store_entry { + file_type.is_dir() + } else { + file_type.is_dir() || file_type.is_symlink() + } + } + + /// One package dir: its `check_package` candidate event, then — only + /// for a real directory (`recurse`), never a symlink, which would walk + /// into pnpm's content-addressed store or an `npm link` target outside + /// the project — its nested `node_modules`, always an importer-style + /// tree. `entry_key` is set for a store entry's direct children (see + /// [`ScanEvent::Package`]); the dir is still descended when the merge + /// skips its package.json, because bundled dependencies are real dirs + /// nested inside the package itself (pnpm cannot link them out), + /// physically present only there. + fn gather_package(path: PathBuf, entry_key: Option, recurse: bool) -> Vec { + let identity = read_package_json_sync(&path.join("package.json")); + let nested = recurse.then(|| path.join("node_modules")); + let mut events = vec![ScanEvent::Package { + path, + identity, + entry_key, + }]; + if let Some(nested) = nested { + events.extend(Self::gather_node_modules(&nested, None, false)); + } + events + } + + /// Gather a scoped packages directory (`@scope/`). `store_entry` + /// carries the caller's traversal policy; nested `node_modules` below a + /// scoped package are always regular importer-style trees. A store + /// entry's `identity_seen` names the full `@scope/name`, so that is the + /// key recorded for its direct children. + fn gather_scoped_packages( + scope_path: &Path, + scope_name: &str, + store_entry: bool, + ) -> Vec { + let children: Vec<(String, FileType)> = list_dir_sync(scope_path) + .entries + .into_iter() + .filter_map(|entry| { + if entry.name_str.starts_with('.') { + return None; + } + let file_type = entry.file_type?; + Self::acceptable_package_entry(file_type, store_entry) + .then_some((entry.name_str, file_type)) + }) + .collect(); + + children + .into_par_iter() + .map(|(name_str, file_type)| { + Self::gather_package( + scope_path.join(&name_str), + store_entry.then(|| format!("{scope_name}/{name_str}")), + file_type.is_dir(), + ) + }) + .collect::>() + .into_iter() + .flatten() + .collect() + } + + /// Gather each virtual-store entry's `node_modules` (entries come from + /// [`Self::list_pnpm_store_entries_sync`] or + /// [`Self::collect_nested_store_entries_sync`]) under the store-entry + /// policy, in parallel, preserving entry order. + fn gather_store_entries(entries: Vec) -> Vec { + entries + .into_par_iter() + .map(|entry| ScanEvent::StoreEntry { + decoded: decode_pnpm_store_entry_name(&entry.name), + events: Self::gather_node_modules(&entry.node_modules, entry.listing, true), + }) + .collect() + } + + /// Replay gathered [`ScanEvent`]s in order against `seen`, exactly as + /// the sequential walk's `check_package` calls would have: a package + /// is recorded only if its package.json parsed and its PURL is new. + /// + /// A store entry whose name decodes to a name@version already + /// inventoried (every root-linked direct dep — the importer pass wins + /// the `seen` dedup) gets `identity_seen` = that name, decided HERE, + /// against the dedup state at this point of the replay: its matching + /// direct child is skipped (the sequential walk did not even read its + /// package.json), while everything below it is still replayed. + fn merge_scan_events( + events: Vec, + identity_seen: Option<&str>, + seen: &mut HashSet, + packages: &mut Vec, + ) { + for event in events { + match event { + ScanEvent::Package { + path, + identity, + entry_key, + } => { + if identity_seen.is_some() && entry_key.as_deref() == identity_seen { + continue; } - // Recurse into nested node_modules only for real - // directories (not symlinks). Following a symlink here - // would walk into pnpm's content-addressed store (or an - // `npm link` target outside the project). - if file_type.is_dir() { - let nested = Self::scan_node_modules( - &entry_path.join("node_modules"), - seen, - ScanPolicy::Importer, - ) - .await; - results.extend(nested); + let Some((full_name, version)) = identity else { + continue; + }; + let (namespace, name) = parse_package_name(&full_name); + let purl = build_npm_purl(namespace.as_deref(), &name, &version); + if !seen.insert(purl.clone()) { + continue; } + packages.push(CrawledPackage { + name, + version, + namespace, + purl, + path, + }); + } + ScanEvent::StoreEntry { decoded, events } => { + let identity_seen = decoded + .filter(|(full_name, version)| { + let (ns, bare) = parse_package_name(full_name); + seen.contains(&build_npm_purl(ns.as_deref(), &bare, version)) + }) + .map(|(full_name, _version)| full_name); + Self::merge_scan_events(events, identity_seen.as_deref(), seen, packages); } } - - if let Some(store_path) = pnpm_store { - let entries = Self::list_pnpm_store_entries(&store_path).await; - results.extend(Self::scan_store_entries(entries, seen).await); - } - for store_path in legacy_stores { - let mut entries = Vec::new(); - Self::collect_nested_store_entries(&store_path, &mut entries).await; - results.extend(Self::scan_store_entries(entries, seen).await); - } - - results - }) + } } /// Enumerate pnpm virtual-store (`node_modules/.pnpm`) entries, @@ -1029,32 +1298,74 @@ impl NpmCrawler { /// treating it as an empty entry silently hid every transitive-only /// install (apply exited 0 claiming success with nothing written) — /// descend it instead. Shared by the resolver - /// (`collect_nested_node_modules`) and the scan pass - /// (`scan_store_entries` callers) so the store-layout policy lives - /// once. + /// (`collect_nested_node_modules`), the scan pass and the peer-variant + /// finder so the store-layout policy lives once. + /// + /// Entries are probed in parallel and yielded in listing order. With + /// `read_listings` (the scan, which lists every entry's `node_modules` + /// next anyway) the existence probe IS that readdir: a `node_modules` + /// that opens as a directory is one, and its listing rides along in + /// [`StoreEntryDir::listing`]; one that does not open falls back to + /// the `is_dir` stat so an unreadable-but-present dir keeps its + /// flat-entry classification. + fn list_pnpm_store_entries_sync(store_path: &Path, read_listings: bool) -> Vec { + let candidates: Vec = list_dir_sync(store_path) + .entries + .into_iter() + .filter(|entry| { + !(entry.name_str.starts_with('.') || entry.name_str == "node_modules") + && entry.file_type.is_some_and(|ft| ft.is_dir()) + }) + .collect(); + + candidates + .into_par_iter() + .map(|entry| { + let entry_path = store_path.join(&entry.name); + let entry_nm = entry_path.join("node_modules"); + if read_listings { + if let Some((entries, complete)) = read_dir_entries_sync(&entry_nm) { + return vec![StoreEntryDir { + name: entry.name_str, + node_modules: entry_nm, + listing: Some(Listing::from_entries(entries, complete)), + }]; + } + } + if is_dir_sync(&entry_nm) { + vec![StoreEntryDir { + name: entry.name_str, + node_modules: entry_nm, + listing: None, + }] + } else { + Self::collect_nested_store_entries_sync(&entry_path) + .into_iter() + .map(|(name, node_modules)| StoreEntryDir { + name, + node_modules, + listing: None, + }) + .collect() + } + }) + .collect::>() + .into_iter() + .flatten() + .collect() + } + + /// Async `(name, node_modules)` view of + /// [`Self::list_pnpm_store_entries_sync`] for the async callers. async fn list_pnpm_store_entries(store_path: &Path) -> Vec<(String, PathBuf)> { - let mut entries = Vec::new(); - for entry in crate::utils::fs::list_dir_entries(store_path).await { - let name = entry.file_name(); - let name_str = name.to_string_lossy(); - if name_str.starts_with('.') || name_str == "node_modules" { - continue; - } - let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { - continue; - }; - if !file_type.is_dir() { - continue; - } - let entry_path = store_path.join(&name); - let entry_nm = entry_path.join("node_modules"); - if is_dir(&entry_nm).await { - entries.push((name_str.into_owned(), entry_nm)); - } else { - Self::collect_nested_store_entries(&entry_path, &mut entries).await; - } - } - entries + let store_path = store_path.to_path_buf(); + run_blocking(move || { + Self::list_pnpm_store_entries_sync(&store_path, false) + .into_iter() + .map(|entry| (entry.name, entry.node_modules)) + .collect() + }) + .await } /// Descend a *nested* virtual-store host dir, yielding @@ -1076,37 +1387,41 @@ impl NpmCrawler { /// (the pending-name filter, the `identity_seen` dedup) treat nested /// and flat entries identically; a shape that doesn't fit stays an /// undecodable — always-probed — name, the conservative direction. - async fn collect_nested_store_entries(host_path: &Path, entries: &mut Vec<(String, PathBuf)>) { + /// + /// Deliberately sequential: the `NESTED_STORE_MAX_DIRS` budget is + /// spent in breadth-first listing order, which decides exactly which + /// entries survive when it runs out. + fn collect_nested_store_entries_sync(host_path: &Path) -> Vec<(String, PathBuf)> { + let mut entries = Vec::new(); let mut remaining = NESTED_STORE_MAX_DIRS; let mut queue: VecDeque<(PathBuf, String, usize)> = VecDeque::from([(host_path.to_path_buf(), String::new(), 0)]); while let Some((dir, rel, depth)) = queue.pop_front() { - for entry in crate::utils::fs::list_dir_entries(&dir).await { - let name = entry.file_name(); - let name_str = name.to_string_lossy(); + for entry in list_dir_sync(&dir).entries { + let name_str = entry.name_str; // A `node_modules` here belongs to a parent entry (already // yielded), never a name/version coordinate. if name_str == "node_modules" { continue; } - let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { + let Some(file_type) = entry.file_type else { continue; }; if !file_type.is_dir() { continue; } if remaining == 0 { - return; + return entries; } remaining -= 1; - let child = dir.join(&name); + let child = dir.join(&entry.name); let child_rel = if rel.is_empty() { - name_str.into_owned() + name_str } else { format!("{rel}/{name_str}") }; let child_nm = child.join("node_modules"); - if is_dir(&child_nm).await { + if is_dir_sync(&child_nm) { // `//node_modules` — a package home. // Anything deeper belongs to that package's own tree, // which the store-entry scan walks itself. @@ -1125,130 +1440,16 @@ impl NpmCrawler { } } } + entries } - /// Inventory the packages under each virtual-store entry's - /// `node_modules` (entries come from `list_pnpm_store_entries` or - /// `collect_nested_store_entries`). An entry whose name decodes to a - /// name@version the importer pass already inventoried (every - /// root-linked direct dep) skips the redundant package.json re-read - /// via `identity_seen` — the entry is still walked, because - /// bundled/injected dependencies are real dirs that physically live - /// only inside the store entry. - async fn scan_store_entries( - entries: Vec<(String, PathBuf)>, - seen: &mut HashSet, - ) -> Vec { - let mut results = Vec::new(); - - for (entry_name, entry_nm) in entries { - let identity_seen = decode_pnpm_store_entry_name(&entry_name) - .filter(|(full_name, version)| { - let (ns, bare) = parse_package_name(full_name); - seen.contains(&build_npm_purl(ns.as_deref(), &bare, version)) - }) - .map(|(full_name, _version)| full_name); - let found = Self::scan_node_modules( - &entry_nm, - seen, - ScanPolicy::StoreEntry { - identity_seen: identity_seen.as_deref(), - }, - ) - .await; - results.extend(found); - } - - results - } - - /// Scan a scoped packages directory (`@scope/`). `policy` carries the - /// caller's traversal rules (see [`ScanPolicy`]); nested `node_modules` - /// below a scoped package are always regular importer-style trees. - fn scan_scoped_packages<'a>( - scope_path: &'a Path, - seen: &'a mut HashSet, - policy: ScanPolicy<'a>, - ) -> std::pin::Pin> + 'a>> { - Box::pin(async move { - let mut results = Vec::new(); - let (store_entry, identity_seen) = match policy { - ScanPolicy::Importer => (false, None), - ScanPolicy::StoreEntry { identity_seen } => (true, identity_seen), - }; - // `identity_seen` names the full `@scope/name`; this dir is the - // `@scope` half. - let scope_name = scope_path - .file_name() - .map(|s| s.to_string_lossy().into_owned()) - .unwrap_or_default(); - - for entry in crate::utils::fs::list_dir_entries(scope_path).await { - let name = entry.file_name(); - let name_str = name.to_string_lossy().to_string(); - - if name_str.starts_with('.') { - continue; - } - - let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { - continue; - }; - - let acceptable = if store_entry { - file_type.is_dir() - } else { - file_type.is_dir() || file_type.is_symlink() - }; - if !acceptable { - continue; - } - - let pkg_path = scope_path.join(&name_str); - let already_inventoried = - identity_seen.is_some_and(|full| full == format!("{scope_name}/{name_str}")); - if !already_inventoried { - if let Some(pkg) = Self::check_package(&pkg_path, seen).await { - results.push(pkg); - } - } - - // Nested node_modules only for real directories - if file_type.is_dir() { - let nested = Self::scan_node_modules( - &pkg_path.join("node_modules"), - seen, - ScanPolicy::Importer, - ) - .await; - results.extend(nested); - } - } - - results - }) - } - - /// Check a package directory and return `CrawledPackage` if valid. - /// Deduplicates by PURL via the `seen` set. - async fn check_package(pkg_path: &Path, seen: &mut HashSet) -> Option { - let pkg_json_path = pkg_path.join("package.json"); - let (full_name, version) = read_package_json(&pkg_json_path).await?; - let (namespace, name) = parse_package_name(&full_name); - let purl = build_npm_purl(namespace.as_deref(), &name, &version); - - if seen.contains(&purl) { - return None; - } - seen.insert(purl.clone()); - - Some(CrawledPackage { - name, - version, - namespace, - purl, - path: pkg_path.to_path_buf(), - }) + /// Async view of [`Self::collect_nested_store_entries_sync`], appending + /// to `entries`. + async fn collect_nested_store_entries(host_path: &Path, entries: &mut Vec<(String, PathBuf)>) { + let host_path = host_path.to_path_buf(); + entries.extend( + run_blocking(move || Self::collect_nested_store_entries_sync(&host_path)).await, + ); } // ------------------------------------------------------------------ diff --git a/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs b/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs new file mode 100644 index 00000000..d11688bc --- /dev/null +++ b/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs @@ -0,0 +1,1555 @@ +//! The pre-parallel (one tokio `spawn_blocking` hop per filesystem call) +//! npm crawler, kept verbatim as the equivalence oracle for the +//! blocking-pool walkers in the parent module: the randomized fixture +//! tests assert both produce identical `crawl_all` / `find_by_purls` / +//! store-enumeration output. Test-only; never compiled into the binary. + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::path::{Path, PathBuf}; + +use super::{ + build_npm_purl, decode_pnpm_store_entry_name, is_legacy_pnpm_store_dir_name, + is_safe_npm_component, parse_package_name, read_package_json, NpmCrawler, Target, + NESTED_STORE_MAX_DEPTH, NESTED_STORE_MAX_DIRS, SKIP_DIRS, +}; +use crate::crawlers::types::{CrawledPackage, CrawlerOptions}; +use crate::utils::fs::is_dir; + +/// Which kind of `node_modules` directory a scan pass is walking — the one +/// traversal-policy bit that differs between them. +#[derive(Clone, Copy)] +enum ScanPolicy<'a> { + /// An importer's or package's `node_modules`: symlinked entries are + /// recorded (pnpm links direct deps; `npm link` targets) but never + /// traversed into, and a `.pnpm` child is the virtual store, scanned + /// in a deferred pass. + Importer, + /// One pnpm virtual-store entry's `node_modules`: only REAL + /// directories are inventoried — a symlinked entry here is the + /// package's dependency pointing at a sibling `.pnpm` store entry, + /// which is inventoried via that entry; following it would record the + /// same package under a path owned by a different store entry. + /// `identity_seen` optionally carries the entry's own package name + /// (what the store dir name decodes to) when its name@version is + /// already inventoried — the importer pass wins the `seen` dedup for + /// every root-linked direct dep — so that child's package.json is not + /// read a second time; everything below it is still scanned. + StoreEntry { identity_seen: Option<&'a str> }, +} + +pub(super) struct LegacyNpmCrawler; + +impl LegacyNpmCrawler { + /// The old `NpmCrawler::crawl_all`: global roots come from the (shared, + /// unchanged) global-path logic; local roots from the old async walk. + pub(super) async fn crawl_all(options: &CrawlerOptions) -> Vec { + let mut packages = Vec::new(); + let mut seen = HashSet::new(); + + let nm_paths = if options.global || options.global_prefix.is_some() { + NpmCrawler::new() + .get_node_modules_paths(options) + .await + .unwrap_or_default() + } else { + Self::find_local_node_modules_dirs(&options.cwd).await + }; + + for nm_path in &nm_paths { + let found = Self::scan_node_modules(nm_path, &mut seen, ScanPolicy::Importer).await; + packages.extend(found); + } + + packages + } + + pub(super) async fn find_by_purls( + node_modules_path: &Path, + purls: &[String], + ) -> Result>, std::io::Error> { + let mut result: HashMap> = HashMap::new(); + + let mut pending: Vec = Vec::new(); + for purl in purls { + let Some((namespace, name, version)) = NpmCrawler::parse_purl_components(purl) else { + continue; + }; + + // SECURITY: `namespace`/`name` come straight from the (untrusted) + // manifest PURL and are joined onto `node_modules_path` below, + // then patched in place. A real npm scope/name is a single + // path segment, so reject any that could traverse out of the + // tree (`pkg:npm/../../evil@1.0.0`). Fail closed — twin of the + // deno/go/maven coordinate gates. + let ns_safe = namespace + .as_deref() + .map(is_safe_npm_component) + .unwrap_or(true); + if !ns_safe || !is_safe_npm_component(&name) { + continue; + } + + let dir_key = match &namespace { + Some(ns) => format!("{ns}/{name}"), + None => name.clone(), + }; + pending.push(Target { + namespace, + name, + version, + purl: purl.clone(), + dir_key, + }); + } + + // Pass 1 — filtered: `.pnpm` virtual-store entries are enqueued + // only when their dir name decodes to a still-pending target's + // name (a manifest routinely lists packages that simply aren't + // installed here, and probing every entry of a large monorepo + // store for them would add a readdir+stat storm to every + // apply/rollback run). + let pending = + Self::resolve_pending_targets(node_modules_path, pending, &mut result, true).await; + + // Pass 2 — unfiltered fallback, only for targets pass 1 could not + // resolve: a target can physically exist ONLY inside another + // package's store entry (a bundled dependency at + // `.pnpm/host@1.0.0/node_modules/host/node_modules/`), + // whose entry name decodes to the HOST's name — the pass-1 filter + // skips it, leaving an installed, scan-visible package invisible + // to apply (fail-open: apply reported it not installed). Probe + // every store entry for just the leftovers; the common all- + // resolved case never reaches this pass, so its perf is intact. + if !pending.is_empty() { + Self::resolve_pending_targets(node_modules_path, pending, &mut result, false).await; + } + + Ok(result) + } + + /// One breadth-first resolution pass over the tree rooted at + /// `node_modules_path`: the root `node_modules` first (so a root-level + /// install always wins), then — only while targets remain unresolved — + /// each nested `node_modules`. npm nests a conflicting version under + /// the dependent package, so a patched version can exist *only* + /// nested; CLI_CONTRACT ("Deeply nested transitive dependencies are + /// fully supported") promises those are patched identically to direct + /// deps, and `crawl_all` (scan) already discovers them at unbounded + /// depth. + /// + /// EVERY matching physical copy of each target lands in `result` + /// (keyed by the target's verbatim PURL, root-copy-first). Targets are + /// kept live across the whole walk — a duplicate copy can live at any + /// depth — so the traversal continues past the first match rather than + /// stopping. Targets for which NO copy was found anywhere are returned + /// (the pass-2 fallback re-probes them with the unfiltered store walk). + /// `filter_store_entries` selects whether pnpm virtual-store entries are + /// bounded by the still-unmatched-name filter (pass 1) or all probed + /// (the pass-2 fallback) — see `find_by_purls`. + async fn resolve_pending_targets( + node_modules_path: &Path, + mut pending: Vec, + result: &mut HashMap>, + filter_store_entries: bool, + ) -> Vec { + if pending.is_empty() { + return pending; + } + let mut queue: VecDeque = VecDeque::from([node_modules_path.to_path_buf()]); + while let Some(nm_path) = queue.pop_front() { + for target in &pending { + let pkg_path = nm_path.join(&target.dir_key); + let pkg_json_path = pkg_path.join("package.json"); + + match read_package_json(&pkg_json_path).await { + // The on-disk *name* must match too: an alias install + // (`npm i foo@npm:bar@1.0.0`) puts a different package + // in `node_modules/foo`, so matching on version alone + // would misidentify it and patch the wrong package's + // files. + Some((found_name, found_version)) + if found_name == target.dir_key && found_version == target.version => + { + let copies = result.entry(target.purl.clone()).or_default(); + // Record each physical copy once — a path reached + // twice (defensive against overlapping walks) is not + // double-counted. + if !copies.iter().any(|c| c.path == pkg_path) { + copies.push(CrawledPackage { + name: target.name.clone(), + version: found_version, + namespace: target.namespace.clone(), + purl: target.purl.clone(), + path: pkg_path, + }); + } + } + _ => {} + } + } + // Descend importer-tree nested `node_modules` for ALL targets + // (a duplicate copy lives at an unknown depth), but probe the + // pnpm virtual store only for targets NOT YET found anywhere: a + // matched direct dep's store peer-variants are the apply + // engine's fan-out job, and re-probing the store for it would + // add a readdir storm. A target with no importer-tree copy + // (transitive-only) still gets its store entries probed. + let unmatched_names: HashSet<&str> = pending + .iter() + .filter(|t| !result.contains_key(&t.purl)) + .map(|t| t.dir_key.as_str()) + .collect(); + let filter = filter_store_entries.then_some(&unmatched_names); + Self::collect_nested_node_modules(&nm_path, filter, &mut queue).await; + } + // Only the targets with zero copies remain "pending" for pass 2. + pending.retain(|t| !result.contains_key(&t.purl)); + pending + } + + /// Append the `node_modules` dirs living one level below `nm_path` + /// (inside each of its package dirs, scoped or not) to `queue`. + /// Mirrors `scan_node_modules`' traversal policy: hidden entries are + /// skipped and symlinked packages are never traversed — a symlink here + /// points into pnpm's content-addressed store or an `npm link` target + /// outside the project. The one exception is pnpm's `.pnpm` virtual + /// store (see below); `pending_names` — `Some(the still-unresolved + /// targets' full package names)` — bounds which store entries get + /// enqueued, while `None` (the pass-2 fallback of `find_by_purls`) + /// enqueues every store entry. + async fn collect_nested_node_modules( + nm_path: &Path, + pending_names: Option<&HashSet<&str>>, + queue: &mut VecDeque, + ) { + for entry in crate::utils::fs::list_dir_entries(nm_path).await { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + // pnpm's virtual store. Under the isolated linker the store is + // the ONLY physical home of transitive dependencies: the + // importer's node_modules holds symlinks for direct deps only, + // so a transitive-only target (installed at + // `.pnpm//node_modules/`, runtime-loaded) is + // unreachable through the symlink-free walk above — invisible + // to apply despite being importable. Probe REAL store entries' + // `node_modules`; the name+version match in `find_by_purls` + // keeps aliases and multi-version store entries distinct, and + // BFS order guarantees a root-linked install has already been + // probed (and removed from `pending`) before these are + // dequeued, so a package is never resolved twice. + if name_str == ".pnpm" { + let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { + continue; + }; + if !file_type.is_dir() { + continue; + } + let store_path = nm_path.join(&name); + let entries = Self::list_pnpm_store_entries(&store_path).await; + Self::enqueue_pending_store_entries(entries, pending_names, queue); + continue; + } + // pnpm <=3: the virtual store is a hidden `.` dir + // (there is no `.pnpm` at all) with the same + // transitive-only-deps property, so it gets the same probing. + // Must run before the generic hidden-entry skip below, which + // would otherwise swallow it — leaving every transitive-only + // install unpatchable on those layouts. + if is_legacy_pnpm_store_dir_name(&name_str) { + let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { + continue; + }; + if !file_type.is_dir() { + continue; + } + let mut entries = Vec::new(); + Self::collect_nested_store_entries(&nm_path.join(&name), &mut entries).await; + Self::enqueue_pending_store_entries(entries, pending_names, queue); + continue; + } + if name_str.starts_with('.') || name_str == "node_modules" { + continue; + } + let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { + continue; + }; + if !file_type.is_dir() { + continue; + } + let entry_path = nm_path.join(&name); + + if name_str.starts_with('@') { + for scoped in crate::utils::fs::list_dir_entries(&entry_path).await { + let scoped_name = scoped.file_name(); + if scoped_name.to_string_lossy().starts_with('.') { + continue; + } + let Some(scoped_type) = crate::utils::fs::entry_file_type(&scoped).await else { + continue; + }; + if !scoped_type.is_dir() { + continue; + } + let nested = entry_path.join(&scoped_name).join("node_modules"); + if is_dir(&nested).await { + queue.push_back(nested); + } + } + } else { + let nested = entry_path.join("node_modules"); + if is_dir(&nested).await { + queue.push_back(nested); + } + } + } + } + + /// Enqueue virtual-store entries that can still hold a pending target. + /// A manifest routinely lists packages that simply aren't installed + /// here, and probing every entry of a large monorepo store for them + /// would add a readdir+stat storm to every apply/rollback run. The + /// entry name advertises the entry's package, so filter by PENDING + /// NAME only — the version is deliberately NOT matched at this stage + /// (dir-name versions can carry peer/build decorations; the + /// package.json probe stays the authority). An undecodable name + /// (truncated/hash-suffixed dirs, git/URL deps, `_`-bearing names) + /// reveals nothing about what's inside, so it stays probeable. + /// + /// `pending_names = None` disables the filter entirely: the entry name + /// only advertises the entry's OWN package, so a target present solely + /// as a bundled dependency INSIDE another package's entry hides behind + /// a non-matching name — `find_by_purls`' pass-2 fallback probes every + /// entry for exactly those. Both enumerators only yield entries whose + /// `node_modules` exists, so no re-stat here. + fn enqueue_pending_store_entries( + entries: Vec<(String, PathBuf)>, + pending_names: Option<&HashSet<&str>>, + queue: &mut VecDeque, + ) { + for (entry_name, entry_nm) in entries { + if let Some(filter) = pending_names { + if let Some((entry_pkg, _version)) = decode_pnpm_store_entry_name(&entry_name) { + if !filter.contains(entry_pkg.as_str()) { + continue; + } + } + } + queue.push_back(entry_nm); + } + } + + /// Find `node_modules` directories within the project root. + /// Recursively searches for workspace `node_modules` but stays within the + /// project. + async fn find_local_node_modules_dirs(start_path: &Path) -> Vec { + let mut results = Vec::new(); + + // Direct node_modules in start_path + let direct = start_path.join("node_modules"); + if is_dir(&direct).await { + results.push(direct); + } + + // Recursively search for workspace node_modules + Self::find_workspace_node_modules(start_path, &mut results).await; + + results + } + + /// Recursively find `node_modules` in subdirectories (for monorepos / workspaces). + /// Skips symlinks, hidden dirs, and well-known non-workspace dirs. + fn find_workspace_node_modules<'a>( + dir: &'a Path, + results: &'a mut Vec, + ) -> std::pin::Pin + 'a>> { + Box::pin(async move { + for entry in crate::utils::fs::list_dir_entries(dir).await { + let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { + continue; + }; + if !file_type.is_dir() { + continue; + } + + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + + // Skip node_modules, hidden dirs, and well-known build dirs + if name_str == "node_modules" + || name_str.starts_with('.') + || SKIP_DIRS.contains(&name_str.as_ref()) + { + continue; + } + + let full_path = dir.join(&name); + + // Check if this subdirectory has its own node_modules + let sub_nm = full_path.join("node_modules"); + if is_dir(&sub_nm).await { + results.push(sub_nm); + } + + // Recurse + Self::find_workspace_node_modules(&full_path, results).await; + } + }) + } + + // ------------------------------------------------------------------ + // Private helpers – scanning + // ------------------------------------------------------------------ + + /// Scan a `node_modules` directory, returning all valid packages found. + /// Recurses into each package's own nested `node_modules`. The one + /// policy bit distinguishing an importer/package tree from a pnpm + /// virtual-store entry is carried by [`ScanPolicy`]. + fn scan_node_modules<'a>( + node_modules_path: &'a Path, + seen: &'a mut HashSet, + policy: ScanPolicy<'a>, + ) -> std::pin::Pin> + 'a>> { + Box::pin(async move { + let mut results = Vec::new(); + let mut pnpm_store: Option = None; + let mut legacy_stores: Vec = Vec::new(); + let (store_entry, identity_seen) = match policy { + ScanPolicy::Importer => (false, None), + ScanPolicy::StoreEntry { identity_seen } => (true, identity_seen), + }; + + for entry in crate::utils::fs::list_dir_entries(node_modules_path).await { + let name = entry.file_name(); + let name_str = name.to_string_lossy().to_string(); + + // pnpm's virtual store: under the isolated linker it is the + // ONLY physical home of transitive dependencies (the + // importer's node_modules symlinks direct deps only), so + // skipping it as just-another-hidden-dir leaves every + // transitive-only install invisible to scan. Deferred until + // after this loop so root-level entries are inventoried + // first and win the `seen` name@version dedup at their + // importer-root paths. (A store entry's own children never + // include a nested `.pnpm`; under `StoreEntry` policy the + // name falls through to the hidden-entry skip below.) + if !store_entry && name_str == ".pnpm" { + let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { + continue; + }; + if file_type.is_dir() { + pnpm_store = Some(node_modules_path.join(&name_str)); + } + continue; + } + + // pnpm <=3 virtual store (a hidden `.` dir; + // no `.pnpm` exists on those layouts): same + // transitive-only-home property, same deferred scan so + // root-level entries win the `seen` dedup. Must run before + // the hidden-entry skip below, which would otherwise leave + // every transitive-only install invisible to scan. + if !store_entry && is_legacy_pnpm_store_dir_name(&name_str) { + let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { + continue; + }; + if file_type.is_dir() { + legacy_stores.push(node_modules_path.join(&name_str)); + } + continue; + } + + // Skip hidden files and node_modules + if name_str.starts_with('.') || name_str == "node_modules" { + continue; + } + + let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { + continue; + }; + + // Importer trees allow both directories and symlinks (pnpm + // links direct deps); a store entry accepts REAL dirs only + // (see `ScanPolicy::StoreEntry`). + let acceptable = if store_entry { + file_type.is_dir() + } else { + file_type.is_dir() || file_type.is_symlink() + }; + if !acceptable { + continue; + } + + let entry_path = node_modules_path.join(&name_str); + + if name_str.starts_with('@') { + // Scoped packages + let scoped = Self::scan_scoped_packages(&entry_path, seen, policy).await; + results.extend(scoped); + } else { + // Regular package. `identity_seen` marks this exact dir + // as already inventoried by the importer pass — skip + // the redundant package.json read, but still descend + // below: bundled dependencies are real dirs nested + // inside the package itself (pnpm cannot link them + // out), physically present only here. + if identity_seen != Some(name_str.as_str()) { + if let Some(pkg) = Self::check_package(&entry_path, seen).await { + results.push(pkg); + } + } + // Recurse into nested node_modules only for real + // directories (not symlinks). Following a symlink here + // would walk into pnpm's content-addressed store (or an + // `npm link` target outside the project). + if file_type.is_dir() { + let nested = Self::scan_node_modules( + &entry_path.join("node_modules"), + seen, + ScanPolicy::Importer, + ) + .await; + results.extend(nested); + } + } + } + + if let Some(store_path) = pnpm_store { + let entries = Self::list_pnpm_store_entries(&store_path).await; + results.extend(Self::scan_store_entries(entries, seen).await); + } + for store_path in legacy_stores { + let mut entries = Vec::new(); + Self::collect_nested_store_entries(&store_path, &mut entries).await; + results.extend(Self::scan_store_entries(entries, seen).await); + } + + results + }) + } + + /// Enumerate pnpm virtual-store (`node_modules/.pnpm`) entries, + /// yielding `(entry_name, /node_modules)` for every entry whose + /// `node_modules` actually exists. The child literally named + /// `node_modules` is pnpm's internal hoist dir (nothing but symlinks + /// into sibling entries) and hidden children are store metadata — both + /// skipped. A REAL directory child with a `node_modules` of its own is + /// a flat (pnpm 6+) entry; one *without* is the pnpm 4/5 nested layout + /// — the child is a registry-host dir + /// (`.pnpm////node_modules/`), so + /// treating it as an empty entry silently hid every transitive-only + /// install (apply exited 0 claiming success with nothing written) — + /// descend it instead. Shared by the resolver + /// (`collect_nested_node_modules`) and the scan pass + /// (`scan_store_entries` callers) so the store-layout policy lives + /// once. + pub(super) async fn list_pnpm_store_entries(store_path: &Path) -> Vec<(String, PathBuf)> { + let mut entries = Vec::new(); + for entry in crate::utils::fs::list_dir_entries(store_path).await { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str.starts_with('.') || name_str == "node_modules" { + continue; + } + let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { + continue; + }; + if !file_type.is_dir() { + continue; + } + let entry_path = store_path.join(&name); + let entry_nm = entry_path.join("node_modules"); + if is_dir(&entry_nm).await { + entries.push((name_str.into_owned(), entry_nm)); + } else { + Self::collect_nested_store_entries(&entry_path, &mut entries).await; + } + } + entries + } + + /// Descend a *nested* virtual-store host dir, yielding + /// `(name@version, /node_modules)` for each package home + /// found. Covers the two pre-flat layouts (both confirmed against + /// captured real installs): + /// - pnpm 4/5: `.pnpm//…` — called on a `.pnpm` child + /// that has no `node_modules` of its own; + /// - pnpm <=3: `node_modules/./…` — called on the + /// hidden store root directly. + /// + /// Below the host, path components are registry coordinates (`@scope`, + /// name, version), NOT package dirs, so the importer-walk hidden-name + /// skip does not apply here — but symlinks are never traversed (a link + /// inside the store points at a sibling entry or out of tree, and + /// following one could cycle), and both depth and total fan-out are + /// bounded. Each found dir's host-relative path is synthesized into + /// the flat `name@version` entry-name form so downstream consumers + /// (the pending-name filter, the `identity_seen` dedup) treat nested + /// and flat entries identically; a shape that doesn't fit stays an + /// undecodable — always-probed — name, the conservative direction. + pub(super) async fn collect_nested_store_entries( + host_path: &Path, + entries: &mut Vec<(String, PathBuf)>, + ) { + let mut remaining = NESTED_STORE_MAX_DIRS; + let mut queue: VecDeque<(PathBuf, String, usize)> = + VecDeque::from([(host_path.to_path_buf(), String::new(), 0)]); + while let Some((dir, rel, depth)) = queue.pop_front() { + for entry in crate::utils::fs::list_dir_entries(&dir).await { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + // A `node_modules` here belongs to a parent entry (already + // yielded), never a name/version coordinate. + if name_str == "node_modules" { + continue; + } + let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { + continue; + }; + if !file_type.is_dir() { + continue; + } + if remaining == 0 { + return; + } + remaining -= 1; + let child = dir.join(&name); + let child_rel = if rel.is_empty() { + name_str.into_owned() + } else { + format!("{rel}/{name_str}") + }; + let child_nm = child.join("node_modules"); + if is_dir(&child_nm).await { + // `//node_modules` — a package home. + // Anything deeper belongs to that package's own tree, + // which the store-entry scan walks itself. + let entry_name = match child_rel.rsplit_once('/') { + Some((pkg, version)) => format!("{pkg}@{version}"), + // Directly under the host there is no name/version + // split; the raw component stays the entry name + // (undecodable ⇒ probed). + None => child_rel, + }; + entries.push((entry_name, child_nm)); + continue; + } + if depth + 1 < NESTED_STORE_MAX_DEPTH { + queue.push_back((child, child_rel, depth + 1)); + } + } + } + } + + /// Inventory the packages under each virtual-store entry's + /// `node_modules` (entries come from `list_pnpm_store_entries` or + /// `collect_nested_store_entries`). An entry whose name decodes to a + /// name@version the importer pass already inventoried (every + /// root-linked direct dep) skips the redundant package.json re-read + /// via `identity_seen` — the entry is still walked, because + /// bundled/injected dependencies are real dirs that physically live + /// only inside the store entry. + async fn scan_store_entries( + entries: Vec<(String, PathBuf)>, + seen: &mut HashSet, + ) -> Vec { + let mut results = Vec::new(); + + for (entry_name, entry_nm) in entries { + let identity_seen = decode_pnpm_store_entry_name(&entry_name) + .filter(|(full_name, version)| { + let (ns, bare) = parse_package_name(full_name); + seen.contains(&build_npm_purl(ns.as_deref(), &bare, version)) + }) + .map(|(full_name, _version)| full_name); + let found = Self::scan_node_modules( + &entry_nm, + seen, + ScanPolicy::StoreEntry { + identity_seen: identity_seen.as_deref(), + }, + ) + .await; + results.extend(found); + } + + results + } + + /// Scan a scoped packages directory (`@scope/`). `policy` carries the + /// caller's traversal rules (see [`ScanPolicy`]); nested `node_modules` + /// below a scoped package are always regular importer-style trees. + fn scan_scoped_packages<'a>( + scope_path: &'a Path, + seen: &'a mut HashSet, + policy: ScanPolicy<'a>, + ) -> std::pin::Pin> + 'a>> { + Box::pin(async move { + let mut results = Vec::new(); + let (store_entry, identity_seen) = match policy { + ScanPolicy::Importer => (false, None), + ScanPolicy::StoreEntry { identity_seen } => (true, identity_seen), + }; + // `identity_seen` names the full `@scope/name`; this dir is the + // `@scope` half. + let scope_name = scope_path + .file_name() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_default(); + + for entry in crate::utils::fs::list_dir_entries(scope_path).await { + let name = entry.file_name(); + let name_str = name.to_string_lossy().to_string(); + + if name_str.starts_with('.') { + continue; + } + + let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { + continue; + }; + + let acceptable = if store_entry { + file_type.is_dir() + } else { + file_type.is_dir() || file_type.is_symlink() + }; + if !acceptable { + continue; + } + + let pkg_path = scope_path.join(&name_str); + let already_inventoried = + identity_seen.is_some_and(|full| full == format!("{scope_name}/{name_str}")); + if !already_inventoried { + if let Some(pkg) = Self::check_package(&pkg_path, seen).await { + results.push(pkg); + } + } + + // Nested node_modules only for real directories + if file_type.is_dir() { + let nested = Self::scan_node_modules( + &pkg_path.join("node_modules"), + seen, + ScanPolicy::Importer, + ) + .await; + results.extend(nested); + } + } + + results + }) + } + + /// Check a package directory and return `CrawledPackage` if valid. + /// Deduplicates by PURL via the `seen` set. + async fn check_package(pkg_path: &Path, seen: &mut HashSet) -> Option { + let pkg_json_path = pkg_path.join("package.json"); + let (full_name, version) = read_package_json(&pkg_json_path).await?; + let (namespace, name) = parse_package_name(&full_name); + let purl = build_npm_purl(namespace.as_deref(), &name, &version); + + if seen.contains(&purl) { + return None; + } + seen.insert(purl.clone()); + + Some(CrawledPackage { + name, + version, + namespace, + purl, + path: pkg_path.to_path_buf(), + }) + } +} + +/// Equivalence of the blocking-pool walkers against the oracle above, over +/// randomized fixture trees exercising every layout rule the walks encode: +/// flat/nested/legacy pnpm stores, scoped packages, symlinks (live, +/// dangling, into the store), duplicate identities, aliases, broken / BOM'd +/// / FIFO / directory package.json, unreadable and unsearchable dirs, and +/// case variants of `node_modules`. +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeMap; + + type Row = (String, String, String, Option, PathBuf); + + fn rows(pkgs: &[CrawledPackage]) -> Vec { + pkgs.iter() + .map(|p| { + ( + p.purl.clone(), + p.name.clone(), + p.version.clone(), + p.namespace.clone(), + p.path.clone(), + ) + }) + .collect() + } + + fn map_rows(map: &HashMap>) -> BTreeMap> { + map.iter().map(|(k, v)| (k.clone(), rows(v))).collect() + } + + const NAMES: &[&str] = &[ + "foo", + "bar", + "baz", + "dup", + "Foo", + "lodash._x", + "@s/a", + "@s/b", + "@t/c", + ]; + const VERSIONS: &[&str] = &["1.0.0", "1.0.1", "2.0.0"]; + const WS_NAMES: &[&str] = &[ + "packages", "apps", "a", "b", "lib", "dist", "vendor", ".git", "tmp", + ]; + + /// Restores permissions the generator stripped, before the tempdir is + /// removed (declare it AFTER the tempdir so it drops first). + struct PermGuard(Vec); + impl Drop for PermGuard { + fn drop(&mut self) { + #[cfg(unix)] + for p in self.0.iter().rev() { + use std::os::unix::fs::PermissionsExt as _; + let _ = std::fs::set_permissions(p, std::fs::Permissions::from_mode(0o755)); + } + } + } + + struct Gen { + state: u64, + /// Out-of-tree dir for symlink targets that get traversed. + scratch: PathBuf, + /// Real package dirs created so far (symlink targets). + pkg_dirs: Vec, + /// Real `node_modules` dirs created so far (symlink targets). + nm_dirs: Vec, + /// Dirs whose permissions were stripped; applied at the END (a + /// stripped dir must not block the rest of the generation). + lock_plan: Vec<(PathBuf, u32)>, + uniq: usize, + } + + impl Gen { + fn new(seed: u64, scratch: PathBuf) -> Self { + Self { + state: seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1, + scratch, + pkg_dirs: Vec::new(), + nm_dirs: Vec::new(), + lock_plan: Vec::new(), + uniq: 0, + } + } + + fn next(&mut self) -> u64 { + // xorshift64* + let mut x = self.state; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.state = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } + + fn chance(&mut self, pct: usize) -> bool { + self.below(100) < pct + } + + fn pick<'a>(&mut self, items: &'a [&'a str]) -> &'a str { + items[self.below(items.len())] + } + + fn uniq(&mut self) -> usize { + self.uniq += 1; + self.uniq + } + + #[allow(unused_variables)] + fn symlink(target: &Path, link: &Path) { + #[cfg(unix)] + { + if let Some(parent) = link.parent() { + let _ = std::fs::create_dir_all(parent); + } + let _ = std::os::unix::fs::symlink(target, link); + } + } + + #[allow(unused_variables)] + fn plan_lock(&mut self, dir: &Path, mode: u32) { + #[cfg(unix)] + self.lock_plan.push((dir.to_path_buf(), mode)); + } + + fn apply_locks(&mut self, guard: &mut PermGuard) { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + // Deepest first, so a stripped parent never blocks a child. + let mut plan = std::mem::take(&mut self.lock_plan); + plan.sort_by_key(|(p, _)| std::cmp::Reverse(p.components().count())); + for (dir, mode) in plan { + if std::fs::symlink_metadata(&dir).is_ok_and(|m| m.is_dir()) + && std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(mode)) + .is_ok() + { + guard.0.push(dir); + } + } + } + #[cfg(not(unix))] + let _ = guard; + } + + /// Write `dir/package.json` in one of many shapes. + fn package_json(&mut self, dir: &Path, name: &str, version: &str) { + let _ = std::fs::create_dir_all(dir); + let pj = dir.join("package.json"); + if pj.symlink_metadata().is_ok() { + // Never write through an existing one (it may be a FIFO). + return; + } + let valid = format!(r#"{{"name": "{name}", "version": "{version}"}}"#); + match self.below(100) { + 0..=69 => { + let _ = std::fs::write(&pj, valid); + } + 70..=74 => { + let _ = std::fs::write(&pj, format!("\u{feff}{valid}")); + } + 75..=78 => { + let _ = std::fs::write(&pj, "not json"); + } + 79..=82 => { + let _ = std::fs::write(&pj, format!(r#"{{"name": "{name}"}}"#)); + } + 83..=85 => { + let _ = + std::fs::write(&pj, format!(r#"{{"name": "", "version": "{version}"}}"#)); + } + 86..=90 => { + #[cfg(unix)] + { + let c = std::ffi::CString::new(pj.to_str().unwrap()).unwrap(); + // SAFETY: plain libc call on a valid C string. + unsafe { libc::mkfifo(c.as_ptr(), 0o644) }; + } + #[cfg(not(unix))] + let _ = std::fs::write(&pj, valid); + } + 91..=93 => { + let _ = std::fs::create_dir_all(&pj); + } + _ => {} + } + } + + /// A package dir `nm/` (maybe with nested node_modules). + fn package(&mut self, nm: &Path, depth: usize) { + let dir_name = self.pick(NAMES).to_string(); + // Mostly the dir's own name; sometimes an alias install. + let pkg_name = if self.chance(88) { + dir_name.clone() + } else { + self.pick(NAMES).to_string() + }; + let version = self.pick(VERSIONS).to_string(); + let dir = nm.join(&dir_name); + if dir.exists() { + return; + } + self.package_json(&dir, &pkg_name, &version); + self.pkg_dirs.push(dir.clone()); + if depth < 3 && self.chance(30) { + let store_ok = self.chance(10); + self.node_modules(&dir.join("node_modules"), depth + 1, store_ok); + } + if self.chance(4) { + self.plan_lock(&dir, 0o000); + } + } + + fn node_modules(&mut self, nm: &Path, depth: usize, store_ok: bool) { + let _ = std::fs::create_dir_all(nm); + self.nm_dirs.push(nm.to_path_buf()); + let mut store_pkgs: Vec<(String, PathBuf)> = Vec::new(); + if store_ok && self.chance(60) { + store_pkgs = self.pnpm_store(&nm.join(".pnpm"), depth); + } else if self.chance(5) { + let _ = std::fs::write(nm.join(".pnpm"), "not a store"); + } + if store_ok && self.chance(15) { + self.legacy_store(&nm.join(".registry.npmjs.org"), depth); + } + let n = self.below(6); + for _ in 0..n { + match self.below(100) { + 0..=54 => self.package(nm, depth), + 55..=62 => { + // Symlinked entry: live, into the store, or dangling. + let name = self.pick(NAMES); + let link = nm.join(name); + if link.symlink_metadata().is_ok() { + continue; + } + let target = if !store_pkgs.is_empty() && self.chance(50) { + { + let i = self.below(store_pkgs.len()); + store_pkgs[i].1.clone() + } + } else if !self.pkg_dirs.is_empty() && self.chance(80) { + { + let i = self.below(self.pkg_dirs.len()); + self.pkg_dirs[i].clone() + } + } else { + nm.join(format!("dangling-{}", self.uniq())) + }; + Self::symlink(&target, &link); + } + 63..=66 => { + let hidden = nm.join(format!(".cache{}", self.uniq())); + self.package_json(&hidden.join("foo"), "foo", "1.0.0"); + } + 67..=70 => { + let _ = std::fs::write(nm.join(format!("README{}", self.uniq())), "x"); + } + 71..=74 => { + // A symlinked scope dir (to a scope living outside + // the tree, so the followed walk cannot cycle). + let id = self.uniq(); + let target = self.scratch.join(format!("scope{id}")); + let version = self.pick(VERSIONS).to_string(); + self.package_json(&target.join("x"), "@link/x", &version); + let link = nm.join(format!("@link{}", self.uniq())); + Self::symlink(&target, &link); + } + 75..=79 => { + let bin = nm.join(".bin"); + let _ = std::fs::create_dir_all(&bin); + if let Some(target) = self.pkg_dirs.first().cloned() { + let link = bin.join(format!("tool{}", self.uniq())); + Self::symlink(&target, &link); + } + } + 80..=82 => { + // A nested `node_modules` entry inside node_modules. + self.package_json(&nm.join("node_modules").join("foo"), "foo", "1.0.0"); + } + _ => { + let scoped = self.pick(&["@s/a", "@s/b", "@t/c"]).to_string(); + let dir = nm.join(&scoped); + if !dir.exists() { + let version = self.pick(VERSIONS).to_string(); + self.package_json(&dir, &scoped, &version); + self.pkg_dirs.push(dir.clone()); + if depth < 3 && self.chance(25) { + self.node_modules(&dir.join("node_modules"), depth + 1, false); + } + } + } + } + } + // Root-linked direct deps: importer symlinks into the store. + for (name, target) in store_pkgs { + let link = nm.join(&name); + if self.chance(50) && link.symlink_metadata().is_err() { + Self::symlink(&target, &link); + } + } + if self.chance(3) { + self.plan_lock(nm, 0o000); + } + } + + fn store_entry_name(&mut self, name: &str, version: &str) -> String { + let escaped = name.replace('/', "+"); + match self.below(10) { + 0 => format!("{escaped}@{version}(peer@1.0.0)"), + 1 => format!("{escaped}@{version}_peer@1.0.0"), + 2 => format!("{escaped}@github.com+u+r@abc{}", self.uniq()), + 3 => format!("truncated-{}_abcdef", self.uniq()), + _ => format!("{escaped}@{version}"), + } + } + + /// A `.pnpm` virtual store; returns `(name, package dir)` of the + /// flat entries' own packages (importer symlink targets). + fn pnpm_store(&mut self, store: &Path, depth: usize) -> Vec<(String, PathBuf)> { + let _ = std::fs::create_dir_all(store); + let _ = std::fs::write(store.join("lock.yaml"), "x"); + let mut own = Vec::new(); + let n = 1 + self.below(6); + for _ in 0..n { + let name = self.pick(NAMES).to_string(); + let version = self.pick(VERSIONS).to_string(); + match self.below(100) { + 0..=59 => { + let entry = store.join(self.store_entry_name(&name, &version)); + let entry_nm = entry.join("node_modules"); + let pkg_name = if self.chance(90) { + name.clone() + } else { + self.pick(NAMES).to_string() + }; + let pkg = entry_nm.join(&name); + self.package_json(&pkg, &pkg_name, &version); + self.pkg_dirs.push(pkg.clone()); + own.push((name.clone(), pkg.clone())); + // Dependencies: symlinks to sibling entries. + for _ in 0..self.below(3) { + if let Some((dep, target)) = own.first().cloned() { + Self::symlink(&target, &entry_nm.join(format!("{dep}-dep"))); + } + } + // Another real package in the entry (injected dep). + if self.chance(20) { + self.package(&entry_nm, depth + 1); + } + // Bundled deps below the package itself. + if depth < 2 && self.chance(25) { + self.node_modules(&pkg.join("node_modules"), depth + 1, false); + } + match self.below(100) { + 0..=3 => self.plan_lock(&entry_nm, 0o000), + 4..=6 => self.plan_lock(&entry, 0o000), + 7..=9 => self.plan_lock(&entry_nm, 0o300), + _ => {} + } + } + 60..=67 => { + // pnpm 4/5 nested host layout. + let pkg = store + .join("registry.npmjs.org") + .join(&name) + .join(&version) + .join("node_modules") + .join(&name); + self.package_json(&pkg, &name, &version); + self.pkg_dirs.push(pkg); + } + 68..=71 => { + let _ = std::fs::create_dir_all( + store.join(format!("empty{}@1.0.0", self.uniq())), + ); + } + 72..=75 => { + let entry = store.join(self.store_entry_name(&name, &version)); + let _ = std::fs::create_dir_all(&entry); + let _ = std::fs::write(entry.join("node_modules"), "file"); + } + 76..=79 => { + // An entry whose node_modules is a symlink. + if let Some(target) = self.nm_dirs.first().cloned() { + let entry = store.join(self.store_entry_name(&name, &version)); + let _ = std::fs::create_dir_all(&entry); + Self::symlink(&target, &entry.join("node_modules")); + } + } + 80..=83 => { + // The entry itself is a symlink (skipped). + if let Some(target) = self.pkg_dirs.first().cloned() { + let link = store.join(format!("{name}@9.9.{}", self.uniq())); + Self::symlink(&target, &link); + } + } + 84..=89 => { + // pnpm's hoist dir: symlinks only. + let hoist = store.join("node_modules"); + let _ = std::fs::create_dir_all(&hoist); + if let Some((dep, target)) = own.first().cloned() { + Self::symlink(&target, &hoist.join(dep)); + } + } + _ => { + // Nested host with a scoped coordinate. + let pkg = store + .join("registry.npmjs.org") + .join("@s") + .join("a") + .join(&version) + .join("node_modules") + .join("@s") + .join("a"); + self.package_json(&pkg, "@s/a", &version); + } + } + } + own + } + + /// A pnpm <=3 `.registry.npmjs.org` store. + fn legacy_store(&mut self, store: &Path, depth: usize) { + for _ in 0..1 + self.below(4) { + let name = self.pick(NAMES).to_string(); + let version = self.pick(VERSIONS).to_string(); + let home = store.join(&name).join(&version).join("node_modules"); + self.package_json(&home.join(&name), &name, &version); + if depth < 2 && self.chance(20) { + self.package(&home, depth + 1); + } + } + } + + fn workspace(&mut self, dir: &Path, depth: usize) { + let _ = std::fs::create_dir_all(dir); + if self.chance(70) { + self.node_modules(&dir.join("node_modules"), 0, true); + } + if depth >= 3 { + return; + } + for _ in 0..self.below(5) { + let child = dir.join(format!("{}{}", self.pick(WS_NAMES), self.uniq())); + match self.below(100) { + 0..=49 => self.workspace(&child, depth + 1), + 50..=57 => { + // Skipped by name (dist/vendor/.git/tmp as-is). + let skipped = dir.join(self.pick(&["dist", "vendor", ".git", "tmp"])); + self.workspace(&skipped, depth + 1); + } + 58..=63 => { + // A symlinked workspace dir (never walked). + if let Some(target) = self.nm_dirs.first().and_then(|p| p.parent()) { + let target = target.to_path_buf(); + Self::symlink(&target, &child); + } + } + 64..=69 => { + // node_modules itself a symlink. + let _ = std::fs::create_dir_all(&child); + if let Some(target) = self.nm_dirs.first().cloned() { + Self::symlink(&target, &child.join("node_modules")); + } + } + 70..=73 => { + let _ = std::fs::create_dir_all(&child); + let _ = std::fs::write(child.join("node_modules"), "file"); + } + 74..=79 => { + // Case variant (aliases node_modules on APFS/NTFS). + let _ = std::fs::create_dir_all(&child); + let variant = self.pick(&["Node_Modules", "NODE_MODULES"]); + self.node_modules(&child.join(variant), 1, false); + } + 80..=85 => { + self.workspace(&child, depth + 1); + self.plan_lock(&child, 0o000); + } + 86..=91 => { + // Readable but not searchable: lists fine, stats fail. + self.workspace(&child, depth + 1); + self.plan_lock(&child, 0o600); + } + _ => { + let _ = std::fs::create_dir_all(&child); + let _ = std::fs::write(child.join("package.json"), "{}"); + } + } + } + } + } + + /// Every purl worth resolving against a tree: all crawled identities, + /// qualified / percent-encoded spellings, absent versions, and names + /// that only exist inside store entries or aliases. + fn probe_purls(crawled: &[CrawledPackage]) -> Vec { + let mut purls: Vec = crawled.iter().map(|p| p.purl.clone()).collect(); + for name in NAMES { + for version in VERSIONS { + purls.push(format!("pkg:npm/{name}@{version}")); + } + } + purls.push("pkg:npm/%40s/a@1.0.0".to_string()); + purls.push("pkg:npm/foo@1.0.0?vcs_url=git@x".to_string()); + purls.push("pkg:npm/absent@1.0.0".to_string()); + purls.push("pkg:npm/../evil@1.0.0".to_string()); + purls.sort(); + purls.dedup(); + purls + } + + async fn assert_equivalent(root: &Path, label: &str) { + let options = CrawlerOptions { + cwd: root.to_path_buf(), + global: false, + global_prefix: None, + }; + let crawler = NpmCrawler::new(); + + let new_paths = crawler.get_node_modules_paths(&options).await.unwrap(); + let old_paths = LegacyNpmCrawler::find_local_node_modules_dirs(root).await; + assert_eq!(new_paths, old_paths, "{label}: node_modules roots differ"); + + let new_pkgs = crawler.crawl_all(&options).await; + let old_pkgs = LegacyNpmCrawler::crawl_all(&options).await; + assert_eq!( + rows(&new_pkgs), + rows(&old_pkgs), + "{label}: crawl_all differs" + ); + + let purls = probe_purls(&new_pkgs); + for nm in &new_paths { + let new_found = crawler.find_by_purls(nm, &purls).await.unwrap(); + let old_found = LegacyNpmCrawler::find_by_purls(nm, &purls).await.unwrap(); + assert_eq!( + map_rows(&new_found), + map_rows(&old_found), + "{label}: find_by_purls differs under {}", + nm.display() + ); + + let store = nm.join(".pnpm"); + assert_eq!( + NpmCrawler::list_pnpm_store_entries(&store).await, + LegacyNpmCrawler::list_pnpm_store_entries(&store).await, + "{label}: store entries differ under {}", + store.display() + ); + let legacy = nm.join(".registry.npmjs.org"); + let mut new_nested = Vec::new(); + NpmCrawler::collect_nested_store_entries(&legacy, &mut new_nested).await; + let mut old_nested = Vec::new(); + LegacyNpmCrawler::collect_nested_store_entries(&legacy, &mut old_nested).await; + assert_eq!( + new_nested, old_nested, + "{label}: nested store entries differ" + ); + } + } + + #[tokio::test(flavor = "multi_thread")] + async fn randomized_trees_match_the_sequential_oracle() { + let mut nonempty = 0; + for seed in 0..64u64 { + let tmp = tempfile::tempdir().unwrap(); + let mut guard = PermGuard(Vec::new()); + let root = tmp.path().join("proj"); + let mut gen = Gen::new(seed, tmp.path().join("scratch")); + gen.workspace(&root, 0); + gen.apply_locks(&mut guard); + + assert_equivalent(&root, &format!("seed {seed}")).await; + let options = CrawlerOptions { + cwd: root.clone(), + global: false, + global_prefix: None, + }; + if !NpmCrawler::new().crawl_all(&options).await.is_empty() { + nonempty += 1; + } + drop(guard); + } + // The generator must actually produce packages most of the time, + // or the comparison above is vacuous. + assert!(nonempty > 32, "only {nonempty} non-empty trees"); + } + + /// Hand-built tree with one of every tricky shape (so each is covered + /// regardless of what the random generator happens to draw), asserting + /// equivalence and pinning a few load-bearing outcomes. + #[tokio::test] + async fn kitchen_sink_tree_matches_the_sequential_oracle() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + let nm = root.join("node_modules"); + let write = |dir: &Path, name: &str, version: &str| { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write( + dir.join("package.json"), + format!(r#"{{"name": "{name}", "version": "{version}"}}"#), + ) + .unwrap(); + }; + // Importer tree: plain, scoped, nested duplicate, alias. + write(&nm.join("foo"), "foo", "1.0.0"); + write(&nm.join("@s").join("a"), "@s/a", "1.0.0"); + write( + &nm.join("bar").join("node_modules").join("foo"), + "foo", + "1.0.0", + ); + write(&nm.join("bar"), "bar", "1.0.0"); + write(&nm.join("alias"), "baz", "2.0.0"); + // Flat store: a root-linked dep's own entry (identity_seen skip) with + // a bundled dep, a transitive-only entry, peer variants. + let store = nm.join(".pnpm"); + let q = store.join("qux@1.0.0").join("node_modules"); + write(&q.join("qux"), "qux", "1.0.0"); + write( + &q.join("qux").join("node_modules").join("bundled"), + "bundled", + "3.0.0", + ); + write( + &store.join("t@1.0.0").join("node_modules").join("t"), + "t", + "1.0.0", + ); + write( + &store + .join("p@1.0.0(r@17.0.0)") + .join("node_modules") + .join("p"), + "p", + "1.0.0", + ); + write( + &store + .join("p@1.0.0(r@18.0.0)") + .join("node_modules") + .join("p"), + "p", + "1.0.0", + ); + write( + &store + .join("@s+b@1.0.0") + .join("node_modules") + .join("@s") + .join("b"), + "@s/b", + "1.0.0", + ); + // Nested (pnpm 4/5) host and legacy (pnpm <=3) store. + write( + &store + .join("registry.npmjs.org") + .join("n") + .join("1.0.0") + .join("node_modules") + .join("n"), + "n", + "1.0.0", + ); + write( + &nm.join(".registry.npmjs.org") + .join("l") + .join("1.0.0") + .join("node_modules") + .join("l"), + "l", + "1.0.0", + ); + // Broken / BOM'd package.json. + std::fs::create_dir_all(nm.join("broken")).unwrap(); + std::fs::write(nm.join("broken").join("package.json"), "{").unwrap(); + std::fs::create_dir_all(nm.join("bom")).unwrap(); + std::fs::write( + nm.join("bom").join("package.json"), + "\u{feff}{\"name\":\"bom\",\"version\":\"1.0.0\"}", + ) + .unwrap(); + // Workspaces: plain, skipped, case variant, node_modules-as-file. + write( + &root + .join("packages") + .join("w") + .join("node_modules") + .join("w"), + "w", + "1.0.0", + ); + write( + &root.join("dist").join("node_modules").join("d"), + "d", + "1.0.0", + ); + write( + &root.join("cv").join("Node_Modules").join("cv"), + "cv", + "1.0.0", + ); + std::fs::create_dir_all(root.join("nf")).unwrap(); + std::fs::write(root.join("nf").join("node_modules"), "file").unwrap(); + + let tmp_guard; + #[cfg(unix)] + { + use std::os::unix::fs::{symlink, PermissionsExt as _}; + // Root-linked direct dep into the store, a dangling link. + symlink(q.join("qux"), nm.join("qux")).unwrap(); + symlink(root.join("nowhere"), nm.join("dangling")).unwrap(); + // FIFO package.json. + std::fs::create_dir_all(nm.join("fifo")).unwrap(); + let c = std::ffi::CString::new(nm.join("fifo").join("package.json").to_str().unwrap()) + .unwrap(); + // SAFETY: plain libc call on a valid C string. + assert_eq!(unsafe { libc::mkfifo(c.as_ptr(), 0o644) }, 0); + // A workspace whose node_modules is a symlink. + std::fs::create_dir_all(root.join("ln")).unwrap(); + symlink( + root.join("packages").join("w").join("node_modules"), + root.join("ln").join("node_modules"), + ) + .unwrap(); + // Unreadable store entry node_modules; unsearchable workspace. + let locked_nm = store.join("z@1.0.0").join("node_modules"); + write(&locked_nm.join("z"), "z", "1.0.0"); + write( + &root.join("rw").join("node_modules").join("r"), + "r", + "1.0.0", + ); + std::fs::set_permissions(&locked_nm, std::fs::Permissions::from_mode(0o000)).unwrap(); + std::fs::set_permissions(root.join("rw"), std::fs::Permissions::from_mode(0o600)) + .unwrap(); + tmp_guard = PermGuard(vec![locked_nm, root.join("rw")]); + } + #[cfg(not(unix))] + { + tmp_guard = PermGuard(Vec::new()); + } + + assert_equivalent(&root, "kitchen sink").await; + + let options = CrawlerOptions { + cwd: root.clone(), + global: false, + global_prefix: None, + }; + let purls: Vec = NpmCrawler::new() + .crawl_all(&options) + .await + .into_iter() + .map(|p| p.purl) + .collect(); + for expected in [ + "pkg:npm/foo@1.0.0", + "pkg:npm/@s/a@1.0.0", + "pkg:npm/bundled@3.0.0", + "pkg:npm/t@1.0.0", + "pkg:npm/p@1.0.0", + "pkg:npm/@s/b@1.0.0", + "pkg:npm/n@1.0.0", + "pkg:npm/l@1.0.0", + "pkg:npm/bom@1.0.0", + "pkg:npm/w@1.0.0", + ] { + assert!( + purls.iter().any(|p| p == expected), + "missing {expected}: {purls:?}" + ); + } + assert!( + !purls.iter().any(|p| p == "pkg:npm/d@1.0.0"), + "dist/ must be skipped" + ); + drop(tmp_guard); + } +} diff --git a/crates/socket-patch-core/src/utils/fs.rs b/crates/socket-patch-core/src/utils/fs.rs index b2ef02d0..6f0cacd7 100644 --- a/crates/socket-patch-core/src/utils/fs.rs +++ b/crates/socket-patch-core/src/utils/fs.rs @@ -90,6 +90,50 @@ pub(crate) async fn is_dir(path: &Path) -> bool { .unwrap_or(false) } +/// Blocking twin of [`list_dir_entries`] for walkers that run whole on the +/// blocking pool (one hop per crawl instead of one per filesystem call). +/// +/// Same tolerate-and-truncate contract — `None` when the directory cannot +/// be opened, and iteration stops at the first entry error — plus a +/// `complete` flag that is `false` when such an entry error cut the +/// listing short, so a caller that answers "is child X here?" from the +/// listing can tell a proven absence from an unread tail. +pub(crate) fn read_dir_entries_sync(path: &Path) -> Option<(Vec, bool)> { + let entries = std::fs::read_dir(path).ok()?; + let mut out = Vec::new(); + for entry in entries { + match entry { + Ok(entry) => out.push(entry), + Err(_) => return Some((out, false)), + } + } + Some((out, true)) +} + +/// Blocking twin of [`is_dir`]: follows symlinks, and a failed stat means +/// "not a dir". +pub(crate) fn is_dir_sync(path: &Path) -> bool { + std::fs::metadata(path).map(|m| m.is_dir()).unwrap_or(false) +} + +/// Run a blocking closure on tokio's blocking pool and hand back its value. +/// A panic inside `f` is re-raised on the awaiting task (the same outcome +/// as when the closure's body ran inline on that task); cancellation only +/// happens at runtime shutdown, when nothing is left to observe the value. +pub(crate) async fn run_blocking(f: F) -> T +where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static, +{ + match tokio::task::spawn_blocking(f).await { + Ok(value) => value, + Err(err) => match err.try_into_panic() { + Ok(payload) => std::panic::resume_unwind(payload), + Err(err) => panic!("blocking crawl task cancelled: {err}"), + }, + } +} + /// Check whether `path` is a regular file, following symlinks. /// /// Returns `false` if the stat fails (missing path, broken symlink, From c13eae62b202b569bf8f3d8048965ce149f51ea1 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 12:59:27 -0400 Subject: [PATCH 011/237] perf(crawl): find_by_purls lists each node_modules once and probes only listed names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The resolver opened `//package.json` for every pending target in every visited node_modules — targets × dirs failed opens, each its own spawn_blocking hop — then listed the same dir again for the descent. Both passes now run as one blocking-pool task: each dequeued dir is listed once, a target is probed there only when the listing could hold its first path component, the surviving probes run in parallel and fold back in target order, and the same listing drives the descent (whose per-entry stats also run in parallel, appended in listing order). The name filter is a strict superset: it only engages for a complete, all-ASCII listing and matches ASCII-case-insensitively (APFS/NTFS), and components a filesystem can resolve to a differently spelled entry (non-ASCII, `~` 8.3 aliases, trailing dot/space) are always probed. BFS root-first order, every-copy collection, the name+version identity check, the pass-2 fallback and the store-entry name filter are unchanged. `.pnpm` entry names are still filtered after the `node_modules` stat, not before: an entry without one is a nested host whose synthesized children can match, so the stat decides the result. The oracle equivalence suite (now also covering case-variant package and scope dirs) asserts identical find_by_purls maps on every generated root. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/crawlers/npm_crawler.rs | 311 +++++++++++------- .../src/crawlers/npm_crawler/oracle.rs | 7 + 2 files changed, 200 insertions(+), 118 deletions(-) diff --git a/crates/socket-patch-core/src/crawlers/npm_crawler.rs b/crates/socket-patch-core/src/crawlers/npm_crawler.rs index d34c24cf..6946197e 100644 --- a/crates/socket-patch-core/src/crawlers/npm_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/npm_crawler.rs @@ -296,6 +296,52 @@ fn may_alias_ascii_name(name: &OsStr, target: &str) -> bool { } } +/// Which first path components a `nm_path.join(dir_key)` lookup could +/// resolve through, given `nm_path`'s listing — lets the resolver skip the +/// package.json probes that could only fail, instead of opening +/// `//package.json` for every pending target in every visited +/// `node_modules`. +/// +/// A superset by construction: filtering is on only for a complete listing +/// whose names are all plain ASCII, and then matches +/// ASCII-case-insensitively (so APFS/NTFS case-insensitive lookups are +/// covered). Components a filesystem may resolve to a differently spelled +/// entry — non-ASCII (Unicode folding / normalization), `~` (Windows 8.3 +/// short-name aliases) or a trailing `.`/space (stripped by Win32 path +/// normalization) — are always probed. +struct ProbeFilter { + /// `None` = the listing cannot prove absence; probe everything. + lower_names: Option>, +} + +impl ProbeFilter { + fn new(listing: &Listing) -> Self { + let exhaustive = listing.complete + && listing + .entries + .iter() + .all(|e| e.name.to_str().is_some_and(|name| name.is_ascii())); + let lower_names = exhaustive.then(|| { + listing + .entries + .iter() + .map(|e| e.name_str.to_ascii_lowercase()) + .collect() + }); + Self { lower_names } + } + + fn may_resolve(&self, component: &str) -> bool { + let Some(lower_names) = &self.lower_names else { + return true; + }; + if !component.is_ascii() || component.contains('~') || component.ends_with(['.', ' ']) { + return true; + } + lower_names.contains(&component.to_ascii_lowercase()) + } +} + /// What the blocking-pool scan of one `node_modules` tree records, in the /// exact order the sequential walk visits it; /// [`NpmCrawler::merge_scan_events`] then replays the order-dependent @@ -612,8 +658,6 @@ impl NpmCrawler { node_modules_path: &Path, purls: &[String], ) -> Result>, std::io::Error> { - let mut result: HashMap> = HashMap::new(); - let mut pending: Vec = Vec::new(); for purl in purls { let Some((namespace, name, version)) = Self::parse_purl_components(purl) else { @@ -647,29 +691,39 @@ impl NpmCrawler { }); } - // Pass 1 — filtered: `.pnpm` virtual-store entries are enqueued - // only when their dir name decodes to a still-pending target's - // name (a manifest routinely lists packages that simply aren't - // installed here, and probing every entry of a large monorepo - // store for them would add a readdir+stat storm to every - // apply/rollback run). - let pending = - Self::resolve_pending_targets(node_modules_path, pending, &mut result, true).await; - - // Pass 2 — unfiltered fallback, only for targets pass 1 could not - // resolve: a target can physically exist ONLY inside another - // package's store entry (a bundled dependency at - // `.pnpm/host@1.0.0/node_modules/host/node_modules/`), - // whose entry name decodes to the HOST's name — the pass-1 filter - // skips it, leaving an installed, scan-visible package invisible - // to apply (fail-open: apply reported it not installed). Probe - // every store entry for just the leftovers; the common all- - // resolved case never reaches this pass, so its perf is intact. - if !pending.is_empty() { - Self::resolve_pending_targets(node_modules_path, pending, &mut result, false).await; - } + // Both passes run as one blocking-pool task: each visited dir is + // listed once, and that listing both bounds which targets are + // probed there and drives the descent (see + // `resolve_pending_targets`). + let node_modules_path = node_modules_path.to_path_buf(); + Ok(run_blocking(move || { + let mut result: HashMap> = HashMap::new(); + + // Pass 1 — filtered: `.pnpm` virtual-store entries are enqueued + // only when their dir name decodes to a still-pending target's + // name (a manifest routinely lists packages that simply aren't + // installed here, and probing every entry of a large monorepo + // store for them would add a readdir+stat storm to every + // apply/rollback run). + let pending = + Self::resolve_pending_targets(&node_modules_path, pending, &mut result, true); + + // Pass 2 — unfiltered fallback, only for targets pass 1 could not + // resolve: a target can physically exist ONLY inside another + // package's store entry (a bundled dependency at + // `.pnpm/host@1.0.0/node_modules/host/node_modules/`), + // whose entry name decodes to the HOST's name — the pass-1 filter + // skips it, leaving an installed, scan-visible package invisible + // to apply (fail-open: apply reported it not installed). Probe + // every store entry for just the leftovers; the common all- + // resolved case never reaches this pass, so its perf is intact. + if !pending.is_empty() { + Self::resolve_pending_targets(&node_modules_path, pending, &mut result, false); + } - Ok(result) + result + }) + .await) } /// One breadth-first resolution pass over the tree rooted at @@ -691,7 +745,13 @@ impl NpmCrawler { /// `filter_store_entries` selects whether pnpm virtual-store entries are /// bounded by the still-unmatched-name filter (pass 1) or all probed /// (the pass-2 fallback) — see `find_by_purls`. - async fn resolve_pending_targets( + /// + /// Each dequeued dir is listed ONCE: a target is probed there only if + /// the listing could hold its first path component (see + /// [`ProbeFilter`]; a skipped probe could only have failed), the + /// surviving package.json probes run in parallel and are folded back + /// in target order, and the same listing drives the descent. + fn resolve_pending_targets( node_modules_path: &Path, mut pending: Vec, result: &mut HashMap>, @@ -702,11 +762,22 @@ impl NpmCrawler { } let mut queue: VecDeque = VecDeque::from([node_modules_path.to_path_buf()]); while let Some(nm_path) = queue.pop_front() { - for target in &pending { + let listing = list_dir_sync(&nm_path); + let probe_filter = ProbeFilter::new(&listing); + let probes: Vec> = pending + .par_iter() + .map(|target| { + let first_component = target.namespace.as_deref().unwrap_or(&target.name); + if !probe_filter.may_resolve(first_component) { + return None; + } + read_package_json_sync(&nm_path.join(&target.dir_key).join("package.json")) + }) + .collect(); + for (target, probe) in pending.iter().zip(probes) { let pkg_path = nm_path.join(&target.dir_key); - let pkg_json_path = pkg_path.join("package.json"); - match read_package_json(&pkg_json_path).await { + match probe { // The on-disk *name* must match too: an alias install // (`npm i foo@npm:bar@1.0.0`) puts a different package // in `node_modules/foo`, so matching on version alone @@ -745,7 +816,7 @@ impl NpmCrawler { .map(|t| t.dir_key.as_str()) .collect(); let filter = filter_store_entries.then_some(&unmatched_names); - Self::collect_nested_node_modules(&nm_path, filter, &mut queue).await; + Self::collect_nested_node_modules(&nm_path, listing, filter, &mut queue); } // Only the targets with zero copies remain "pending" for pass 2. pending.retain(|t| !result.contains_key(&t.purl)); @@ -753,103 +824,105 @@ impl NpmCrawler { } /// Append the `node_modules` dirs living one level below `nm_path` - /// (inside each of its package dirs, scoped or not) to `queue`. - /// Mirrors `scan_node_modules`' traversal policy: hidden entries are - /// skipped and symlinked packages are never traversed — a symlink here - /// points into pnpm's content-addressed store or an `npm link` target - /// outside the project. The one exception is pnpm's `.pnpm` virtual - /// store (see below); `pending_names` — `Some(the still-unresolved - /// targets' full package names)` — bounds which store entries get - /// enqueued, while `None` (the pass-2 fallback of `find_by_purls`) - /// enqueues every store entry. - async fn collect_nested_node_modules( + /// (inside each of its package dirs, scoped or not) to `queue`, given + /// `nm_path`'s listing. Mirrors the scan's traversal policy: hidden + /// entries are skipped and symlinked packages are never traversed — a + /// symlink here points into pnpm's content-addressed store or an `npm + /// link` target outside the project. The one exception is pnpm's + /// `.pnpm` virtual store (see below); `pending_names` — `Some(the + /// still-unresolved targets' full package names)` — bounds which store + /// entries get enqueued, while `None` (the pass-2 fallback of + /// `find_by_purls`) enqueues every store entry. + /// + /// Entries are examined in parallel; their contributions are appended + /// in listing order, so the breadth-first queue is unchanged. + fn collect_nested_node_modules( nm_path: &Path, + listing: Listing, pending_names: Option<&HashSet<&str>>, queue: &mut VecDeque, ) { - for entry in crate::utils::fs::list_dir_entries(nm_path).await { - let name = entry.file_name(); - let name_str = name.to_string_lossy(); - // pnpm's virtual store. Under the isolated linker the store is - // the ONLY physical home of transitive dependencies: the - // importer's node_modules holds symlinks for direct deps only, - // so a transitive-only target (installed at - // `.pnpm//node_modules/`, runtime-loaded) is - // unreachable through the symlink-free walk above — invisible - // to apply despite being importable. Probe REAL store entries' - // `node_modules`; the name+version match in `find_by_purls` - // keeps aliases and multi-version store entries distinct, and - // BFS order guarantees a root-linked install has already been - // probed (and removed from `pending`) before these are - // dequeued, so a package is never resolved twice. - if name_str == ".pnpm" { - let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { - continue; - }; - if !file_type.is_dir() { - continue; - } - let store_path = nm_path.join(&name); - let entries = Self::list_pnpm_store_entries(&store_path).await; - Self::enqueue_pending_store_entries(entries, pending_names, queue); - continue; - } - // pnpm <=3: the virtual store is a hidden `.` dir - // (there is no `.pnpm` at all) with the same - // transitive-only-deps property, so it gets the same probing. - // Must run before the generic hidden-entry skip below, which - // would otherwise swallow it — leaving every transitive-only - // install unpatchable on those layouts. - if is_legacy_pnpm_store_dir_name(&name_str) { - let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { - continue; - }; - if !file_type.is_dir() { - continue; - } - let mut entries = Vec::new(); - Self::collect_nested_store_entries(&nm_path.join(&name), &mut entries).await; - Self::enqueue_pending_store_entries(entries, pending_names, queue); - continue; - } - if name_str.starts_with('.') || name_str == "node_modules" { - continue; + let found: Vec> = listing + .entries + .into_par_iter() + .map(|entry| Self::nested_node_modules_of(nm_path, entry, pending_names)) + .collect(); + queue.extend(found.into_iter().flatten()); + } + + /// The `node_modules` dirs one listing entry of `nm_path` contributes + /// to the resolver's queue (see [`Self::collect_nested_node_modules`]). + fn nested_node_modules_of( + nm_path: &Path, + entry: ListedEntry, + pending_names: Option<&HashSet<&str>>, + ) -> Vec { + let name_str = entry.name_str.as_str(); + // pnpm's virtual store. Under the isolated linker the store is + // the ONLY physical home of transitive dependencies: the + // importer's node_modules holds symlinks for direct deps only, + // so a transitive-only target (installed at + // `.pnpm//node_modules/`, runtime-loaded) is + // unreachable through the symlink-free walk above — invisible + // to apply despite being importable. Probe REAL store entries' + // `node_modules`; the name+version match in `find_by_purls` + // keeps aliases and multi-version store entries distinct, and + // BFS order guarantees a root-linked install has already been + // probed (and removed from `pending`) before these are + // dequeued, so a package is never resolved twice. + if name_str == ".pnpm" { + if !entry.file_type.is_some_and(|ft| ft.is_dir()) { + return Vec::new(); } - let Some(file_type) = crate::utils::fs::entry_file_type(&entry).await else { - continue; - }; - if !file_type.is_dir() { - continue; + let entries = Self::list_pnpm_store_entries_sync(&nm_path.join(&entry.name), false) + .into_iter() + .map(|e| (e.name, e.node_modules)) + .collect(); + return Self::pending_store_entries(entries, pending_names); + } + // pnpm <=3: the virtual store is a hidden `.` dir + // (there is no `.pnpm` at all) with the same + // transitive-only-deps property, so it gets the same probing. + // Must run before the generic hidden-entry skip below, which + // would otherwise swallow it — leaving every transitive-only + // install unpatchable on those layouts. + if is_legacy_pnpm_store_dir_name(name_str) { + if !entry.file_type.is_some_and(|ft| ft.is_dir()) { + return Vec::new(); } - let entry_path = nm_path.join(&name); + let entries = Self::collect_nested_store_entries_sync(&nm_path.join(&entry.name)); + return Self::pending_store_entries(entries, pending_names); + } + if name_str.starts_with('.') || name_str == "node_modules" { + return Vec::new(); + } + if !entry.file_type.is_some_and(|ft| ft.is_dir()) { + return Vec::new(); + } + let entry_path = nm_path.join(&entry.name); - if name_str.starts_with('@') { - for scoped in crate::utils::fs::list_dir_entries(&entry_path).await { - let scoped_name = scoped.file_name(); - if scoped_name.to_string_lossy().starts_with('.') { - continue; - } - let Some(scoped_type) = crate::utils::fs::entry_file_type(&scoped).await else { - continue; - }; - if !scoped_type.is_dir() { - continue; - } - let nested = entry_path.join(&scoped_name).join("node_modules"); - if is_dir(&nested).await { - queue.push_back(nested); - } - } + if name_str.starts_with('@') { + list_dir_sync(&entry_path) + .entries + .into_iter() + .filter(|scoped| { + !scoped.name_str.starts_with('.') + && scoped.file_type.is_some_and(|ft| ft.is_dir()) + }) + .map(|scoped| entry_path.join(&scoped.name).join("node_modules")) + .filter(|nested| is_dir_sync(nested)) + .collect() + } else { + let nested = entry_path.join("node_modules"); + if is_dir_sync(&nested) { + vec![nested] } else { - let nested = entry_path.join("node_modules"); - if is_dir(&nested).await { - queue.push_back(nested); - } + Vec::new() } } } - /// Enqueue virtual-store entries that can still hold a pending target. + /// The virtual-store entries that can still hold a pending target. /// A manifest routinely lists packages that simply aren't installed /// here, and probing every entry of a large monorepo store for them /// would add a readdir+stat storm to every apply/rollback run. The @@ -866,11 +939,11 @@ impl NpmCrawler { /// a non-matching name — `find_by_purls`' pass-2 fallback probes every /// entry for exactly those. Both enumerators only yield entries whose /// `node_modules` exists, so no re-stat here. - fn enqueue_pending_store_entries( + fn pending_store_entries( entries: Vec<(String, PathBuf)>, pending_names: Option<&HashSet<&str>>, - queue: &mut VecDeque, - ) { + ) -> Vec { + let mut out = Vec::new(); for (entry_name, entry_nm) in entries { if let Some(filter) = pending_names { if let Some((entry_pkg, _version)) = decode_pnpm_store_entry_name(&entry_name) { @@ -879,8 +952,9 @@ impl NpmCrawler { } } } - queue.push_back(entry_nm); + out.push(entry_nm); } + out } // ------------------------------------------------------------------ @@ -1445,6 +1519,7 @@ impl NpmCrawler { /// Async view of [`Self::collect_nested_store_entries_sync`], appending /// to `entries`. + #[cfg(test)] async fn collect_nested_store_entries(host_path: &Path, entries: &mut Vec<(String, PathBuf)>) { let host_path = host_path.to_path_buf(); entries.extend( diff --git a/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs b/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs index d11688bc..36b2b7a9 100644 --- a/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs +++ b/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs @@ -1277,6 +1277,8 @@ mod tests { purls.push("pkg:npm/%40s/a@1.0.0".to_string()); purls.push("pkg:npm/foo@1.0.0?vcs_url=git@x".to_string()); purls.push("pkg:npm/absent@1.0.0".to_string()); + purls.push("pkg:npm/casedir@1.0.0".to_string()); + purls.push("pkg:npm/@cs/x@1.0.0".to_string()); purls.push("pkg:npm/../evil@1.0.0".to_string()); purls.sort(); purls.dedup(); @@ -1446,6 +1448,11 @@ mod tests { "l", "1.0.0", ); + // A dir whose spelling differs from its package's name only by case + // (resolves under the lowercase name on case-insensitive volumes), + // and a scope spelled likewise. + write(&nm.join("CaseDir"), "casedir", "1.0.0"); + write(&nm.join("@Cs").join("x"), "@cs/x", "1.0.0"); // Broken / BOM'd package.json. std::fs::create_dir_all(nm.join("broken")).unwrap(); std::fs::write(nm.join("broken").join("package.json"), "{").unwrap(); From b5dc36dbc51fa6b91ba20311f3c74d7a521cab6d Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 13:05:27 -0400 Subject: [PATCH 012/237] perf(crawl): run the nine ecosystem crawlers concurrently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `crawl_all_ecosystems` awaited each crawler in turn, and the crawlers that block (maven's walkdir walk + POM reads, `gem env`, the python site-packages probe, `composer global config home`) did so inline on the async task. The crawlers are independent — none prints, none mutates shared state — so they are now joined, with every blocking walk or subprocess moved onto the blocking pool, and their results are consumed in the fixed Npm, Pypi, Cargo, Gem, Golang, Maven, Composer, Nuget, Deno order, so packages and counts are exactly the serial run's. The joined futures are heap-allocated from a non-async constructor so the caller's poll frame does not grow by their combined size (Windows main-stack budget). `gem env gemdir` and `gem env gempath` run concurrently but are still two calls consumed gemdir-then-gempath (no single-call merge: platform path separators). A polyglot `--global-prefix` test pins the joined output against the serial sequence. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/ecosystem_dispatch.rs | 137 +++++++++++++++--- .../src/crawlers/composer_crawler.rs | 8 +- .../src/crawlers/maven_crawler.rs | 14 +- .../src/crawlers/python_crawler.rs | 19 ++- .../src/crawlers/ruby_crawler.rs | 26 +++- 5 files changed, 164 insertions(+), 40 deletions(-) diff --git a/crates/socket-patch-cli/src/ecosystem_dispatch.rs b/crates/socket-patch-cli/src/ecosystem_dispatch.rs index da32594c..4768f075 100644 --- a/crates/socket-patch-cli/src/ecosystem_dispatch.rs +++ b/crates/socket-patch-cli/src/ecosystem_dispatch.rs @@ -521,6 +521,16 @@ pub async fn find_manifest_package_copies( find_all_packages_for_rollback(&partitioned, &crawler_options, quiet).await } +/// Box the future `make` returns, constructing it inside this (non-async) +/// frame so the caller's poll frame only ever holds the pointer. +fn boxed<'a, T, F, Fut>(make: F) -> std::pin::Pin + 'a>> +where + F: FnOnce() -> Fut, + Fut: std::future::Future + 'a, +{ + Box::pin(make()) +} + /// Crawl all ecosystems and return all packages, per-ecosystem counts and /// the gem crawl's refused config-sourced `BUNDLE_PATH` /// (`BundleStoreDiscovery::skipped_config_path`, local mode only) — @@ -533,29 +543,42 @@ pub async fn crawl_all_ecosystems( HashMap, Option, ) { + // The nine crawlers are independent (none prints, none mutates shared + // state), so they run concurrently; their blocking walks and + // subprocesses sit on the blocking pool. Results are consumed in the + // fixed order below, so packages and counts are exactly the serial + // run's. Each future is heap-allocated through `boxed` (constructed + // in that helper's frame) so joining nine does not grow the caller's + // poll frame by their combined size. + let (npm, pypi, cargo, (gems, gem_discovery), golang, maven, composer, nuget, deno) = tokio::join!( + boxed(|| NpmCrawler.crawl_all(options)), + boxed(|| PythonCrawler.crawl_all(options)), + boxed(|| CargoCrawler.crawl_all(options)), + boxed(|| RubyCrawler.crawl_all_with_discovery(options)), + boxed(|| GoCrawler.crawl_all(options)), + boxed(|| MavenCrawler.crawl_all(options)), + boxed(|| ComposerCrawler.crawl_all(options)), + boxed(|| NuGetCrawler.crawl_all(options)), + boxed(|| DenoCrawler.crawl_all(options)), + ); + let mut all_packages = Vec::new(); let mut counts: HashMap = HashMap::new(); - - macro_rules! crawl { - ($eco:expr, $crawler:expr) => {{ - let pkgs = $crawler.crawl_all(options).await; - counts.insert($eco, pkgs.len()); - all_packages.extend(pkgs); - }}; + for (eco, pkgs) in [ + (Ecosystem::Npm, npm), + (Ecosystem::Pypi, pypi), + (Ecosystem::Cargo, cargo), + (Ecosystem::Gem, gems), + (Ecosystem::Golang, golang), + (Ecosystem::Maven, maven), + (Ecosystem::Composer, composer), + (Ecosystem::Nuget, nuget), + (Ecosystem::Deno, deno), + ] { + counts.insert(eco, pkgs.len()); + all_packages.extend(pkgs); } - crawl!(Ecosystem::Npm, NpmCrawler); - crawl!(Ecosystem::Pypi, PythonCrawler); - crawl!(Ecosystem::Cargo, CargoCrawler); - let (gems, gem_discovery) = RubyCrawler.crawl_all_with_discovery(options).await; - counts.insert(Ecosystem::Gem, gems.len()); - all_packages.extend(gems); - crawl!(Ecosystem::Golang, GoCrawler); - crawl!(Ecosystem::Maven, MavenCrawler); - crawl!(Ecosystem::Composer, ComposerCrawler); - crawl!(Ecosystem::Nuget, NuGetCrawler); - crawl!(Ecosystem::Deno, DenoCrawler); - let skipped_config_path = gem_discovery.and_then(|d| d.skipped_config_path); (all_packages, counts, skipped_config_path) } @@ -1337,6 +1360,82 @@ mod tests { } } + /// The concurrent crawl must yield exactly the serial run's packages, + /// in the fixed ecosystem order, with the same counts. A + /// `--global-prefix` root is handed to every crawler verbatim, so one + /// polyglot dir exercises several ecosystems at once. + #[tokio::test(flavor = "multi_thread")] + async fn crawl_all_ecosystems_matches_serial_order() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + for (name, version) in [("zeta", "1.0.0"), ("alpha", "2.0.0"), ("mid", "3.0.0")] { + let pkg_dir = root.join(name); + std::fs::create_dir_all(&pkg_dir).unwrap(); + std::fs::write( + pkg_dir.join("package.json"), + format!(r#"{{"name":"{name}","version":"{version}"}}"#), + ) + .unwrap(); + } + for (name, version) in [("requests", "2.31.0"), ("attrs", "23.1.0")] { + let dist = root.join(format!("{name}-{version}.dist-info")); + std::fs::create_dir_all(&dist).unwrap(); + std::fs::write( + dist.join("METADATA"), + format!("Name: {name}\nVersion: {version}\n"), + ) + .unwrap(); + } + for (name, version) in [("serde", "1.0.0"), ("anyhow", "1.0.75")] { + let krate = root.join(format!("{name}-{version}")); + std::fs::create_dir_all(&krate).unwrap(); + std::fs::write( + krate.join("Cargo.toml"), + format!("[package]\nname = \"{name}\"\nversion = \"{version}\"\n"), + ) + .unwrap(); + } + let options = CrawlerOptions { + cwd: root.to_path_buf(), + global: false, + global_prefix: Some(root.to_path_buf()), + }; + + let (packages, counts, _) = crawl_all_ecosystems(&options).await; + + let mut serial: Vec = Vec::new(); + let mut serial_counts: HashMap = HashMap::new(); + macro_rules! serial { + ($eco:expr, $pkgs:expr) => {{ + let pkgs = $pkgs; + serial_counts.insert($eco, pkgs.len()); + serial.extend(pkgs); + }}; + } + serial!(Ecosystem::Npm, NpmCrawler.crawl_all(&options).await); + serial!(Ecosystem::Pypi, PythonCrawler.crawl_all(&options).await); + serial!(Ecosystem::Cargo, CargoCrawler.crawl_all(&options).await); + serial!(Ecosystem::Gem, RubyCrawler.crawl_all(&options).await); + serial!(Ecosystem::Golang, GoCrawler.crawl_all(&options).await); + serial!(Ecosystem::Maven, MavenCrawler.crawl_all(&options).await); + serial!( + Ecosystem::Composer, + ComposerCrawler.crawl_all(&options).await + ); + serial!(Ecosystem::Nuget, NuGetCrawler.crawl_all(&options).await); + serial!(Ecosystem::Deno, DenoCrawler.crawl_all(&options).await); + + let key = |p: &CrawledPackage| (p.purl.clone(), p.path.clone()); + assert_eq!( + packages.iter().map(key).collect::>(), + serial.iter().map(key).collect::>() + ); + assert_eq!(counts, serial_counts); + // Non-vacuous: several ecosystems contributed. + assert!(counts[&Ecosystem::Npm] >= 3, "{counts:?}"); + assert!(counts[&Ecosystem::Pypi] >= 2, "{counts:?}"); + } + /// Deno is the ONE dispatch branch no other test drives end-to-end /// (lcov: every other ecosystem's `scan_ecosystem!` invocation has /// executed, deno's never has). Stage the JSR cache layout diff --git a/crates/socket-patch-core/src/crawlers/composer_crawler.rs b/crates/socket-patch-core/src/crawlers/composer_crawler.rs index 4253176b..80998d9e 100644 --- a/crates/socket-patch-core/src/crawlers/composer_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/composer_crawler.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use super::types::{CrawledPackage, CrawlerOptions}; use crate::patch::path_safety; -use crate::utils::fs::{is_dir, is_file, normalize_lexically}; +use crate::utils::fs::{is_dir, is_file, normalize_lexically, run_blocking}; use crate::utils::process::{CommandRunner, SystemCommandRunner}; /// PHP/Composer ecosystem crawler for discovering packages in Composer @@ -243,8 +243,10 @@ async fn get_composer_home() -> Option { } } - // Try `composer global config home` - if let Some(stdout) = SystemCommandRunner.run("composer", &["global", "config", "home"]) { + // Try `composer global config home` (a subprocess: on the blocking pool) + let stdout = + run_blocking(|| SystemCommandRunner.run("composer", &["global", "config", "home"])).await; + if let Some(stdout) = stdout { if let Some(path) = parse_composer_home_output(&stdout) { if is_dir(&path).await { return Some(path); diff --git a/crates/socket-patch-core/src/crawlers/maven_crawler.rs b/crates/socket-patch-core/src/crawlers/maven_crawler.rs index d77c00df..918a2eb2 100644 --- a/crates/socket-patch-core/src/crawlers/maven_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/maven_crawler.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use super::types::{CrawledPackage, CrawlerOptions}; use crate::patch::path_safety; -use crate::utils::fs::is_dir; +use crate::utils::fs::{is_dir, run_blocking}; // --------------------------------------------------------------------------- // POM XML minimal parser @@ -397,8 +397,16 @@ impl MavenCrawler { let repo_paths = self.get_maven_repo_paths(options).await.unwrap_or_default(); - for repo_path in &repo_paths { - let found = self.scan_maven_repo(repo_path, &mut seen); + for repo_path in repo_paths { + // The walkdir walk and POM reads are blocking: run each repo + // on the blocking pool so concurrently crawled ecosystems keep + // making progress (the dedup set rides along and comes back). + let (found, returned_seen) = run_blocking(move || { + let found = MavenCrawler.scan_maven_repo(&repo_path, &mut seen); + (found, seen) + }) + .await; + seen = returned_seen; packages.extend(found); } diff --git a/crates/socket-patch-core/src/crawlers/python_crawler.rs b/crates/socket-patch-core/src/crawlers/python_crawler.rs index bce50479..9755e6f6 100644 --- a/crates/socket-patch-core/src/crawlers/python_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/python_crawler.rs @@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use super::types::{CrawledPackage, CrawlerOptions}; -use crate::utils::fs::read_regular_to_string; +use crate::utils::fs::{read_regular_to_string, run_blocking}; use crate::utils::process::{CommandRunner, SystemCommandRunner}; // --------------------------------------------------------------------------- @@ -991,19 +991,22 @@ pub async fn get_global_python_site_packages() -> Vec { } } - // 1. Ask Python for site-packages - if let Some(python_cmd) = find_python_command() { + // 1. Ask Python for site-packages (subprocesses: on the blocking pool) + let site_output = run_blocking(|| { + let python_cmd = find_python_command()?; let runner = SystemCommandRunner; - if let Some(stdout) = runner.run( + runner.run( python_cmd, &[ "-c", "import site; print('\\n'.join(site.getsitepackages())); print(site.getusersitepackages())", ], - ) { - for p in parse_python_site_packages_output(&stdout) { - add_path(p, &mut seen, &mut results); - } + ) + }) + .await; + if let Some(stdout) = site_output { + for p in parse_python_site_packages_output(&stdout) { + add_path(p, &mut seen, &mut results); } } diff --git a/crates/socket-patch-core/src/crawlers/ruby_crawler.rs b/crates/socket-patch-core/src/crawlers/ruby_crawler.rs index 1a027d7a..9f92dc52 100644 --- a/crates/socket-patch-core/src/crawlers/ruby_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/ruby_crawler.rs @@ -4,7 +4,9 @@ use std::path::{Path, PathBuf}; use super::types::{CrawledPackage, CrawlerOptions}; use crate::patch::path_safety; -use crate::utils::fs::{entry_is_dir, home_dir, is_dir, list_dir_entries, normalize_lexically}; +use crate::utils::fs::{ + entry_is_dir, home_dir, is_dir, list_dir_entries, normalize_lexically, run_blocking, +}; use crate::utils::process::{CommandRunner, SystemCommandRunner}; /// Ruby/RubyGems ecosystem crawler for discovering gems in Bundler vendor @@ -230,11 +232,17 @@ impl RubyCrawler { /// `gempath` (`GEM_PATH`) entry. Non-existent homes and duplicates are /// dropped, so the result is the deduped set of installed-gem roots in /// RubyGems' own precedence order. + /// + /// The two `gem env` subprocesses run concurrently (each is one + /// ruby boot); their answers are consumed in the fixed order above. async fn gem_env_gems_dirs() -> Vec { let mut paths = Vec::new(); let mut seen = HashSet::new(); - if let Some(gemdir) = Self::run_gem_env("gemdir").await { + let (gemdir, gempath) = + tokio::join!(Self::run_gem_env("gemdir"), Self::run_gem_env("gempath")); + + if let Some(gemdir) = gemdir { let gems_path = PathBuf::from(gemdir).join("gems"); if is_dir(&gems_path).await && seen.insert(gems_path.clone()) { paths.push(gems_path); @@ -246,7 +254,7 @@ impl RubyCrawler { // `:` shreds Windows drive-letter paths (`C:\Ruby\...;D:\...`) into // `["C", "\Ruby\...;D", "\..."]`, so defer to `split_paths`, which // honors the platform separator — same as the Go crawler's GOPATH. - if let Some(gempath) = Self::run_gem_env("gempath").await { + if let Some(gempath) = gempath { for gems_path in gem_homes_to_gems_dirs(&gempath) { if is_dir(&gems_path).await && seen.insert(gems_path.clone()) { paths.push(gems_path); @@ -513,10 +521,14 @@ impl RubyCrawler { paths } - /// Run `gem env ` and return the trimmed stdout. - async fn run_gem_env(key: &str) -> Option { - let stdout = SystemCommandRunner.run("gem", &["env", key]); - parse_gem_env_output(stdout.as_deref().unwrap_or("")) + /// Run `gem env ` (on the blocking pool — it waits on a + /// subprocess) and return the trimmed stdout. + async fn run_gem_env(key: &'static str) -> Option { + run_blocking(move || { + let stdout = SystemCommandRunner.run("gem", &["env", key]); + parse_gem_env_output(stdout.as_deref().unwrap_or("")) + }) + .await } /// Scan a gem directory and return all valid gem packages found. From c0982891788014315476275f6ee1b7116c52d480 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 13:22:18 -0400 Subject: [PATCH 013/237] perf(crawl): resolve find_by_purls one BFS level at a time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit fanned each visited dir's probes and descent stats out to the rayon pool separately, one injection per dir: on a deep pnpm tree the per-dir handoff latency outweighed the parallelism, and `apply --dry-run` on a large monorepo ran slower than the async walk. A visit's reads depend only on the dir and the fixed target list, never on what earlier dirs resolved, so the walk now proceeds level by level (exactly the FIFO queue's order: everything a dir enqueues lands behind the rest of its level). Each level's visits — listing, filtered probes, nested-dir discovery with the virtual store's entries returned whole — are gathered in one parallel pass, then the order-dependent part (folding matches into the result, the unmatched-name store filter, next-level order) is replayed sequentially in queue order. Output is unchanged; the oracle equivalence suite still covers it. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/crawlers/npm_crawler.rs | 217 +++++++++++------- 1 file changed, 130 insertions(+), 87 deletions(-) diff --git a/crates/socket-patch-core/src/crawlers/npm_crawler.rs b/crates/socket-patch-core/src/crawlers/npm_crawler.rs index 6946197e..73e34cc2 100644 --- a/crates/socket-patch-core/src/crawlers/npm_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/npm_crawler.rs @@ -342,6 +342,22 @@ impl ProbeFilter { } } +/// The read-only result of one `find_by_purls` resolver visit (see +/// [`NpmCrawler::visit_resolver_dir`]). +struct ResolverVisit { + nm_path: PathBuf, + probes: Vec>, + nested: Vec, +} + +/// One contribution to the resolver's next BFS level: a nested importer +/// `node_modules`, or a virtual store's entries, still to be narrowed by +/// the pending-name filter when the visit is replayed. +enum NestedNodeModules { + Dir(PathBuf), + StoreEntries(Vec<(String, PathBuf)>), +} + /// What the blocking-pool scan of one `node_modules` tree records, in the /// exact order the sequential walk visits it; /// [`NpmCrawler::merge_scan_events`] then replays the order-dependent @@ -746,11 +762,18 @@ impl NpmCrawler { /// bounded by the still-unmatched-name filter (pass 1) or all probed /// (the pass-2 fallback) — see `find_by_purls`. /// - /// Each dequeued dir is listed ONCE: a target is probed there only if + /// Each visited dir is listed ONCE: a target is probed there only if /// the listing could hold its first path component (see - /// [`ProbeFilter`]; a skipped probe could only have failed), the - /// surviving package.json probes run in parallel and are folded back - /// in target order, and the same listing drives the descent. + /// [`ProbeFilter`]; a skipped probe could only have failed), and the + /// same listing drives the descent. + /// + /// The walk runs level by level — exactly the FIFO queue's order, since + /// everything a dir enqueues lands behind the rest of its level. What a + /// visit READS depends only on the dir and the (fixed) target list, not + /// on what earlier dirs resolved, so each level's visits are gathered + /// in parallel ([`Self::visit_resolver_dir`]); the order-dependent part + /// — folding matches into `result` and the unmatched-name store filter + /// — is then replayed sequentially in queue order. fn resolve_pending_targets( node_modules_path: &Path, mut pending: Vec, @@ -760,103 +783,122 @@ impl NpmCrawler { if pending.is_empty() { return pending; } - let mut queue: VecDeque = VecDeque::from([node_modules_path.to_path_buf()]); - while let Some(nm_path) = queue.pop_front() { - let listing = list_dir_sync(&nm_path); - let probe_filter = ProbeFilter::new(&listing); - let probes: Vec> = pending - .par_iter() - .map(|target| { - let first_component = target.namespace.as_deref().unwrap_or(&target.name); - if !probe_filter.may_resolve(first_component) { - return None; - } - read_package_json_sync(&nm_path.join(&target.dir_key).join("package.json")) - }) + let mut level: Vec = vec![node_modules_path.to_path_buf()]; + while !level.is_empty() { + let visits: Vec = level + .into_par_iter() + .map(|nm_path| Self::visit_resolver_dir(nm_path, &pending)) .collect(); - for (target, probe) in pending.iter().zip(probes) { - let pkg_path = nm_path.join(&target.dir_key); - - match probe { - // The on-disk *name* must match too: an alias install - // (`npm i foo@npm:bar@1.0.0`) puts a different package - // in `node_modules/foo`, so matching on version alone - // would misidentify it and patch the wrong package's - // files. - Some((found_name, found_version)) - if found_name == target.dir_key && found_version == target.version => - { - let copies = result.entry(target.purl.clone()).or_default(); - // Record each physical copy once — a path reached - // twice (defensive against overlapping walks) is not - // double-counted. - if !copies.iter().any(|c| c.path == pkg_path) { - copies.push(CrawledPackage { - name: target.name.clone(), - version: found_version, - namespace: target.namespace.clone(), - purl: target.purl.clone(), - path: pkg_path, - }); + let mut next_level: Vec = Vec::new(); + for visit in visits { + let nm_path = visit.nm_path; + for (target, probe) in pending.iter().zip(visit.probes) { + let pkg_path = nm_path.join(&target.dir_key); + + match probe { + // The on-disk *name* must match too: an alias install + // (`npm i foo@npm:bar@1.0.0`) puts a different package + // in `node_modules/foo`, so matching on version alone + // would misidentify it and patch the wrong package's + // files. + Some((found_name, found_version)) + if found_name == target.dir_key && found_version == target.version => + { + let copies = result.entry(target.purl.clone()).or_default(); + // Record each physical copy once — a path reached + // twice (defensive against overlapping walks) is not + // double-counted. + if !copies.iter().any(|c| c.path == pkg_path) { + copies.push(CrawledPackage { + name: target.name.clone(), + version: found_version, + namespace: target.namespace.clone(), + purl: target.purl.clone(), + path: pkg_path, + }); + } + } + _ => {} + } + } + // Descend importer-tree nested `node_modules` for ALL targets + // (a duplicate copy lives at an unknown depth), but probe the + // pnpm virtual store only for targets NOT YET found anywhere: a + // matched direct dep's store peer-variants are the apply + // engine's fan-out job, and re-probing the store for it would + // add a readdir storm. A target with no importer-tree copy + // (transitive-only) still gets its store entries probed. + let unmatched_names: HashSet<&str> = pending + .iter() + .filter(|t| !result.contains_key(&t.purl)) + .map(|t| t.dir_key.as_str()) + .collect(); + let filter = filter_store_entries.then_some(&unmatched_names); + for nested in visit.nested { + match nested { + NestedNodeModules::Dir(dir) => next_level.push(dir), + NestedNodeModules::StoreEntries(entries) => { + next_level.extend(Self::pending_store_entries(entries, filter)) } } - _ => {} } } - // Descend importer-tree nested `node_modules` for ALL targets - // (a duplicate copy lives at an unknown depth), but probe the - // pnpm virtual store only for targets NOT YET found anywhere: a - // matched direct dep's store peer-variants are the apply - // engine's fan-out job, and re-probing the store for it would - // add a readdir storm. A target with no importer-tree copy - // (transitive-only) still gets its store entries probed. - let unmatched_names: HashSet<&str> = pending - .iter() - .filter(|t| !result.contains_key(&t.purl)) - .map(|t| t.dir_key.as_str()) - .collect(); - let filter = filter_store_entries.then_some(&unmatched_names); - Self::collect_nested_node_modules(&nm_path, listing, filter, &mut queue); + level = next_level; } // Only the targets with zero copies remain "pending" for pass 2. pending.retain(|t| !result.contains_key(&t.purl)); pending } - /// Append the `node_modules` dirs living one level below `nm_path` - /// (inside each of its package dirs, scoped or not) to `queue`, given - /// `nm_path`'s listing. Mirrors the scan's traversal policy: hidden - /// entries are skipped and symlinked packages are never traversed — a - /// symlink here points into pnpm's content-addressed store or an `npm - /// link` target outside the project. The one exception is pnpm's - /// `.pnpm` virtual store (see below); `pending_names` — `Some(the - /// still-unresolved targets' full package names)` — bounds which store - /// entries get enqueued, while `None` (the pass-2 fallback of - /// `find_by_purls`) enqueues every store entry. + /// The read-only half of one resolver visit to `nm_path`: its listing, + /// each target's package.json probe (in target order; `None` for a + /// probe the listing proves would fail), and the nested `node_modules` + /// the dir contributes, in listing order. + fn visit_resolver_dir(nm_path: PathBuf, pending: &[Target]) -> ResolverVisit { + let listing = list_dir_sync(&nm_path); + let probe_filter = ProbeFilter::new(&listing); + let probes = pending + .iter() + .map(|target| { + let first_component = target.namespace.as_deref().unwrap_or(&target.name); + if !probe_filter.may_resolve(first_component) { + return None; + } + read_package_json_sync(&nm_path.join(&target.dir_key).join("package.json")) + }) + .collect(); + let nested = Self::collect_nested_node_modules(&nm_path, listing); + ResolverVisit { + nm_path, + probes, + nested, + } + } + + /// The `node_modules` dirs living one level below `nm_path` (inside each + /// of its package dirs, scoped or not), given `nm_path`'s listing. + /// Mirrors the scan's traversal policy: hidden entries are skipped and + /// symlinked packages are never traversed — a symlink here points into + /// pnpm's content-addressed store or an `npm link` target outside the + /// project. The one exception is pnpm's virtual store (see below), + /// whose entries are returned whole: which of them get enqueued is + /// decided by the caller's pending-name filter + /// ([`Self::pending_store_entries`]) at replay time. /// - /// Entries are examined in parallel; their contributions are appended - /// in listing order, so the breadth-first queue is unchanged. - fn collect_nested_node_modules( - nm_path: &Path, - listing: Listing, - pending_names: Option<&HashSet<&str>>, - queue: &mut VecDeque, - ) { - let found: Vec> = listing + /// Entries are examined in parallel; their contributions keep listing + /// order. + fn collect_nested_node_modules(nm_path: &Path, listing: Listing) -> Vec { + let found: Vec> = listing .entries .into_par_iter() - .map(|entry| Self::nested_node_modules_of(nm_path, entry, pending_names)) + .map(|entry| Self::nested_node_modules_of(nm_path, entry)) .collect(); - queue.extend(found.into_iter().flatten()); + found.into_iter().flatten().collect() } - /// The `node_modules` dirs one listing entry of `nm_path` contributes - /// to the resolver's queue (see [`Self::collect_nested_node_modules`]). - fn nested_node_modules_of( - nm_path: &Path, - entry: ListedEntry, - pending_names: Option<&HashSet<&str>>, - ) -> Vec { + /// What one listing entry of `nm_path` contributes to the resolver's + /// next level (see [`Self::collect_nested_node_modules`]). + fn nested_node_modules_of(nm_path: &Path, entry: ListedEntry) -> Vec { let name_str = entry.name_str.as_str(); // pnpm's virtual store. Under the isolated linker the store is // the ONLY physical home of transitive dependencies: the @@ -878,7 +920,7 @@ impl NpmCrawler { .into_iter() .map(|e| (e.name, e.node_modules)) .collect(); - return Self::pending_store_entries(entries, pending_names); + return vec![NestedNodeModules::StoreEntries(entries)]; } // pnpm <=3: the virtual store is a hidden `.` dir // (there is no `.pnpm` at all) with the same @@ -891,7 +933,7 @@ impl NpmCrawler { return Vec::new(); } let entries = Self::collect_nested_store_entries_sync(&nm_path.join(&entry.name)); - return Self::pending_store_entries(entries, pending_names); + return vec![NestedNodeModules::StoreEntries(entries)]; } if name_str.starts_with('.') || name_str == "node_modules" { return Vec::new(); @@ -911,11 +953,12 @@ impl NpmCrawler { }) .map(|scoped| entry_path.join(&scoped.name).join("node_modules")) .filter(|nested| is_dir_sync(nested)) + .map(NestedNodeModules::Dir) .collect() } else { let nested = entry_path.join("node_modules"); if is_dir_sync(&nested) { - vec![nested] + vec![NestedNodeModules::Dir(nested)] } else { Vec::new() } From b89f1cd0667e151c79c5472edc64bcda48c4602f Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 13:36:07 -0400 Subject: [PATCH 014/237] test(crawl): keep the oracle fixture generator warning-free on non-Unix targets Symlinks, FIFOs and permission stripping are generated on Unix only, so the fields that record them are never read elsewhere. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs b/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs index 36b2b7a9..42eda9c5 100644 --- a/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs +++ b/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs @@ -813,7 +813,9 @@ mod tests { ]; /// Restores permissions the generator stripped, before the tempdir is - /// removed (declare it AFTER the tempdir so it drops first). + /// removed (declare it AFTER the tempdir so it drops first). Only Unix + /// strips permissions. + #[cfg_attr(not(unix), allow(dead_code))] struct PermGuard(Vec); impl Drop for PermGuard { fn drop(&mut self) { @@ -825,6 +827,8 @@ mod tests { } } + // Symlinks, FIFOs and permission stripping are generated on Unix only. + #[cfg_attr(not(unix), allow(dead_code))] struct Gen { state: u64, /// Out-of-tree dir for symlink targets that get traversed. From fe7f385ef5a2251c4def9d3430989d6768d22cf7 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 14:30:29 -0400 Subject: [PATCH 015/237] style(crawl): keep the test-only oracle module out of the npm crawler's use block Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/socket-patch-core/src/crawlers/npm_crawler.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/socket-patch-core/src/crawlers/npm_crawler.rs b/crates/socket-patch-core/src/crawlers/npm_crawler.rs index 73e34cc2..5ec33db7 100644 --- a/crates/socket-patch-core/src/crawlers/npm_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/npm_crawler.rs @@ -9,10 +9,10 @@ use serde::Deserialize; use super::types::{CrawledPackage, CrawlerOptions}; use crate::patch::path_safety; use crate::utils::fs::{is_dir, is_dir_sync, read_dir_entries_sync, run_blocking}; +use crate::utils::purl::{percent_decode_purl_component, strip_purl_qualifiers}; #[cfg(test)] mod oracle; -use crate::utils::purl::{percent_decode_purl_component, strip_purl_qualifiers}; /// Directories to skip when searching for workspace node_modules. const SKIP_DIRS: &[&str] = &[ From 26814250b679de656e06cf73ac314c214ed6551e Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 14:31:14 -0400 Subject: [PATCH 016/237] fix(crawl): walk workspace roots level by level instead of recursing The parallel roots walk recursed once per directory level on rayon and blocking-pool threads (2 MiB stacks), where the old async walk recursed through boxed futures on the 8 MiB main thread. A deep enough directory chain (reachable under Linux's 4096-byte PATH_MAX, and deeper on Windows long paths) aborted the scan with a stack overflow the old walk survived. Read the tree one level at a time, each level's dirs in parallel, record each dir's child range, then emit with an explicit stack in the same depth-first order. Stack use no longer grows with depth; a new test runs a 400-deep chain on 256 KiB walk threads (the recursive walk overflowed there). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/crawlers/npm_crawler.rs | 129 ++++++++++++++---- 1 file changed, 99 insertions(+), 30 deletions(-) diff --git a/crates/socket-patch-core/src/crawlers/npm_crawler.rs b/crates/socket-patch-core/src/crawlers/npm_crawler.rs index 5ec33db7..6d778ada 100644 --- a/crates/socket-patch-core/src/crawlers/npm_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/npm_crawler.rs @@ -1100,49 +1100,83 @@ impl NpmCrawler { results } - /// Recursively find `node_modules` in subdirectories (for monorepos / - /// workspaces), given `dir`'s own listing. Skips symlinks, hidden dirs, - /// and well-known non-workspace dirs. + /// Find `node_modules` in subdirectories (for monorepos / workspaces), + /// at any depth, given `dir`'s own listing. Skips symlinks, hidden + /// dirs, and well-known non-workspace dirs. /// - /// Subdirectories are walked in parallel; the per-child results are - /// concatenated in listing order, each child contributing its own - /// `node_modules` first and then its subtree's — the sequential - /// depth-first order. A child whose listing (which the walk needs - /// anyway) proves it has no `node_modules` skips the stat; see - /// [`has_node_modules_dir`]. + /// The result is in the sequential depth-first order: children in + /// listing order, each contributing its own `node_modules` first and + /// then its subtree's. The tree is read one level at a time, each + /// level's dirs in parallel, and the order is reassembled from the + /// recorded child ranges afterwards — no recursion, so an arbitrarily + /// deep directory chain cannot exhaust a thread's stack. A child whose + /// listing (which the walk needs anyway) proves it has no + /// `node_modules` skips the stat; see [`has_node_modules_dir`]. fn find_workspace_node_modules(dir: &Path, listing: Listing) -> Vec { - let children: Vec = listing + /// One walked dir: its `node_modules` (if any) and the indices of + /// its walked children in `nodes`. + struct WalkedDir { + node_modules: Option, + children: std::ops::Range, + } + + // Dirs are numbered in visit order, level by level, so each dir's + // children occupy a contiguous range of the next level. + let mut nodes: Vec = Vec::new(); + let mut level = Self::workspace_children(dir, listing); + let roots = 0..level.len(); + while !level.is_empty() { + let visits: Vec<(Option, Vec)> = level + .into_par_iter() + .map(|full_path| { + let listing = list_dir_sync(&full_path); + // Check if this subdirectory has its own node_modules + let node_modules = has_node_modules_dir(&full_path, &listing) + .then(|| full_path.join("node_modules")); + (node_modules, Self::workspace_children(&full_path, listing)) + }) + .collect(); + let next_base = nodes.len() + visits.len(); + let mut next_level = Vec::new(); + for (node_modules, children) in visits { + let start = next_base + next_level.len(); + next_level.extend(children); + nodes.push(WalkedDir { + node_modules, + children: start..next_base + next_level.len(), + }); + } + level = next_level; + } + + // Pre-order emission with an explicit stack. + let mut results = Vec::new(); + let mut stack: Vec = roots.rev().collect(); + while let Some(index) = stack.pop() { + let node = &mut nodes[index]; + results.extend(node.node_modules.take()); + stack.extend(node.children.clone().rev()); + } + results + } + + /// The subdirectories of `dir` the workspace walk descends into, in + /// listing order: real dirs only (symlinks are never followed), minus + /// `node_modules`, hidden dirs and well-known build dirs. + fn workspace_children(dir: &Path, listing: Listing) -> Vec { + listing .entries .into_iter() .filter(|entry| { - // Skip non-dirs (symlinks included), node_modules, hidden - // dirs, and well-known build dirs entry.file_type.is_some_and(|ft| ft.is_dir()) && !(entry.name_str == "node_modules" || entry.name_str.starts_with('.') || SKIP_DIRS.contains(&entry.name_str.as_str())) }) .map(|entry| dir.join(&entry.name)) - .collect(); - - children - .into_par_iter() - .map(|full_path| { - let listing = list_dir_sync(&full_path); - let mut found = Vec::new(); - // Check if this subdirectory has its own node_modules - if has_node_modules_dir(&full_path, &listing) { - found.push(full_path.join("node_modules")); - } - // Recurse - found.extend(Self::find_workspace_node_modules(&full_path, listing)); - found - }) - .collect::>() - .into_iter() - .flatten() .collect() } + // ------------------------------------------------------------------ // Private helpers – scanning // ------------------------------------------------------------------ @@ -2624,4 +2658,39 @@ mod tests { "fnm layout must be discovered under $HOME; got {paths:?}" ); } + + /// The workspace roots walk is iterative: a directory chain far deeper + /// than a small stack can recurse through completes on walk threads + /// with only 256 KiB of stack (the recursive walk overflowed there), + /// and still yields depth-first order — each dir's `node_modules` + /// before anything below it. + #[test] + fn test_workspace_walk_deep_chain_small_stack() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_path_buf(); + // Stay well inside macOS's 1024-byte PATH_MAX. + let depth = 400.min(900usize.saturating_sub(root.as_os_str().len()) / 2); + assert!(depth >= 200, "temp dir path too long: {}", root.display()); + + let mut expected = vec![root.join("node_modules")]; + let mut chain = root.clone(); + for level in 1..=depth { + chain.push("a"); + if level == depth / 2 || level == depth { + expected.push(chain.join("node_modules")); + } + } + std::fs::create_dir_all(&chain).unwrap(); + for nm in &expected { + std::fs::create_dir_all(nm).unwrap(); + } + + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(2) + .stack_size(256 * 1024) + .build() + .unwrap(); + let found = pool.install(|| NpmCrawler::find_local_node_modules_dirs(&root)); + assert_eq!(found, expected); + } } From 58f594269823544022941534746667ce6ed15288 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 14:38:03 -0400 Subject: [PATCH 017/237] fix(crawl): run npm walks on a main-sized stack within the descriptor budget Two properties of the old sequential async walk did not survive the move to parallel sync walks on rayon's global pool: - Stack: the recursive node_modules gather ran on 2 MiB worker threads instead of the 8 MiB main thread. The npm walks now run on a dedicated walk pool whose threads get the main thread's 8 MiB. - Descriptors: every walker treats a failed read_dir/open, EMFILE included, as an absent dir, and the old crawl held one descriptor at a time with the nine crawlers run back to back. With one walk thread per CPU plus concurrent crawlers, depscan lost packages silently below `ulimit -n 24` (5349 of 5520 at 20) where the old crawl was intact down to 14. Under a soft RLIMIT_NOFILE below 128 the walk pool now gets one thread and the crawlers run one at a time (the old descriptor profile); above it the pool is capped at half of what is left after a 64-descriptor reserve. depscan now matches the baseline byte-for-byte at every limit from 16 to 256. New tests: pool sizing, a 4 MiB frame fitting on a walk thread, and an e2e scan under `ulimit -n 16` that must match the ample-limit JSON (the one-thread-per-CPU pool lost most of that tree there). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/ecosystem_dispatch.rs | 44 +++-- .../tests/crawl_fd_limit_e2e.rs | 152 +++++++++++++++ crates/socket-patch-core/src/crawlers/mod.rs | 1 + .../src/crawlers/npm_crawler.rs | 22 +-- .../src/crawlers/walk_pool.rs | 180 ++++++++++++++++++ 5 files changed, 376 insertions(+), 23 deletions(-) create mode 100644 crates/socket-patch-cli/tests/crawl_fd_limit_e2e.rs create mode 100644 crates/socket-patch-core/src/crawlers/walk_pool.rs diff --git a/crates/socket-patch-cli/src/ecosystem_dispatch.rs b/crates/socket-patch-cli/src/ecosystem_dispatch.rs index 4768f075..8acf4af7 100644 --- a/crates/socket-patch-cli/src/ecosystem_dispatch.rs +++ b/crates/socket-patch-cli/src/ecosystem_dispatch.rs @@ -7,6 +7,7 @@ use std::path::PathBuf; use crate::args::GlobalArgs; +use socket_patch_core::crawlers::walk_pool; use socket_patch_core::crawlers::CargoCrawler; use socket_patch_core::crawlers::ComposerCrawler; use socket_patch_core::crawlers::DenoCrawler; @@ -549,18 +550,37 @@ pub async fn crawl_all_ecosystems( // fixed order below, so packages and counts are exactly the serial // run's. Each future is heap-allocated through `boxed` (constructed // in that helper's frame) so joining nine does not grow the caller's - // poll frame by their combined size. - let (npm, pypi, cargo, (gems, gem_discovery), golang, maven, composer, nuget, deno) = tokio::join!( - boxed(|| NpmCrawler.crawl_all(options)), - boxed(|| PythonCrawler.crawl_all(options)), - boxed(|| CargoCrawler.crawl_all(options)), - boxed(|| RubyCrawler.crawl_all_with_discovery(options)), - boxed(|| GoCrawler.crawl_all(options)), - boxed(|| MavenCrawler.crawl_all(options)), - boxed(|| ComposerCrawler.crawl_all(options)), - boxed(|| NuGetCrawler.crawl_all(options)), - boxed(|| DenoCrawler.crawl_all(options)), - ); + // poll frame by their combined size. Under a tight descriptor limit + // they run one at a time instead, keeping the serial run's descriptor + // profile (see `walk_pool`): a crawler treats a failed open as an + // absent dir, so extra concurrent descriptors could silently drop + // packages there. + let (npm, pypi, cargo, (gems, gem_discovery), golang, maven, composer, nuget, deno) = + if walk_pool::fd_limit_is_tight() { + ( + boxed(|| NpmCrawler.crawl_all(options)).await, + boxed(|| PythonCrawler.crawl_all(options)).await, + boxed(|| CargoCrawler.crawl_all(options)).await, + boxed(|| RubyCrawler.crawl_all_with_discovery(options)).await, + boxed(|| GoCrawler.crawl_all(options)).await, + boxed(|| MavenCrawler.crawl_all(options)).await, + boxed(|| ComposerCrawler.crawl_all(options)).await, + boxed(|| NuGetCrawler.crawl_all(options)).await, + boxed(|| DenoCrawler.crawl_all(options)).await, + ) + } else { + tokio::join!( + boxed(|| NpmCrawler.crawl_all(options)), + boxed(|| PythonCrawler.crawl_all(options)), + boxed(|| CargoCrawler.crawl_all(options)), + boxed(|| RubyCrawler.crawl_all_with_discovery(options)), + boxed(|| GoCrawler.crawl_all(options)), + boxed(|| MavenCrawler.crawl_all(options)), + boxed(|| ComposerCrawler.crawl_all(options)), + boxed(|| NuGetCrawler.crawl_all(options)), + boxed(|| DenoCrawler.crawl_all(options)), + ) + }; let mut all_packages = Vec::new(); let mut counts: HashMap = HashMap::new(); diff --git a/crates/socket-patch-cli/tests/crawl_fd_limit_e2e.rs b/crates/socket-patch-cli/tests/crawl_fd_limit_e2e.rs new file mode 100644 index 00000000..6b9f529b --- /dev/null +++ b/crates/socket-patch-cli/tests/crawl_fd_limit_e2e.rs @@ -0,0 +1,152 @@ +//! The crawl under a tight `RLIMIT_NOFILE` must inventory exactly what it +//! does with ample descriptors. +//! +//! Every crawler treats a failed `read_dir`/open — `EMFILE` included — as +//! an absent directory, so a crawl that holds more descriptors at once +//! than the old sequential walk (parallel walk threads, crawlers running +//! concurrently) would silently drop packages under a limit the old walk +//! handled. Below the walk pool's tight-limit threshold the crawl keeps +//! the sequential descriptor profile; this suite pins that by scanning the +//! same tree under `ulimit -n 16` and under the inherited limit and +//! requiring byte-identical JSON. (The sequential walk scans this tree +//! fully at 14; with one walk thread per CPU it lost most of it at 16.) +#![cfg(unix)] + +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +use serde_json::Value; + +fn binary() -> PathBuf { + env!("CARGO_BIN_EXE_socket-patch").into() +} + +fn write_package(dir: &Path, name: &str, version: &str) { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write( + dir.join("package.json"), + format!(r#"{{"name":"{name}","version":"{version}"}}"#), + ) + .unwrap(); +} + +/// A tree wide enough that parallel walk threads each hold a descriptor +/// at once: a root `node_modules` with nested and scoped packages, a pnpm +/// virtual store, and workspace packages with their own `node_modules`. +/// Returns the number of distinct packages it holds. +fn build_tree(root: &Path) -> usize { + std::fs::write( + root.join("package.json"), + r#"{"name":"root","version":"0.0.0"}"#, + ) + .unwrap(); + let mut count = 0; + let nm = root.join("node_modules"); + for i in 0..60 { + let pkg = nm.join(format!("pkg{i}")); + write_package(&pkg, &format!("pkg{i}"), "1.0.0"); + count += 1; + for j in 0..3 { + let name = format!("nested{i}-{j}"); + write_package(&pkg.join("node_modules").join(&name), &name, "2.0.0"); + count += 1; + } + } + for i in 0..15 { + let name = format!("scoped{i}"); + write_package( + &nm.join("@scope").join(&name), + &format!("@scope/{name}"), + "3.0.0", + ); + count += 1; + } + for i in 0..30 { + let name = format!("stored{i}"); + write_package( + &nm.join(".pnpm") + .join(format!("{name}@4.0.0")) + .join("node_modules") + .join(&name), + &name, + "4.0.0", + ); + count += 1; + } + for w in 0..10 { + let ws_nm = root + .join("packages") + .join(format!("ws{w}")) + .join("node_modules"); + for i in 0..10 { + let name = format!("ws{w}-dep{i}"); + write_package(&ws_nm.join(&name), &name, "5.0.0"); + count += 1; + } + } + count +} + +/// `scan --json` against an unreachable API (the crawl still runs and the +/// JSON still reports what it found), optionally under `ulimit -n`. +fn scan(root: &Path, nofile: Option) -> Output { + let mut script = String::new(); + if let Some(limit) = nofile { + script.push_str(&format!("ulimit -n {limit} || exit 99; ")); + } + script.push_str(r#"exec "$0" "$@""#); + let mut cmd = Command::new("/bin/sh"); + cmd.arg("-c") + .arg(script) + .arg(binary()) + .args([ + "scan", + "--json", + "--no-telemetry", + "--api-url", + "http://127.0.0.1:1", + "--api-token", + "x", + "--org", + "test-org", + ]) + .current_dir(root); + for (key, _) in std::env::vars_os() { + let name = key.to_string_lossy(); + if name.starts_with("SOCKET_") && !name.contains("TELEMETRY") && name != "SOCKET_NO_CONFIG" + { + cmd.env_remove(&key); + } + } + cmd.output().unwrap() +} + +#[test] +fn tight_descriptor_limit_scans_the_same_packages() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + let expected = build_tree(root); + + let ample = scan(root, None); + let tight = scan(root, Some(16)); + assert_ne!(tight.status.code(), Some(99), "ulimit -n 16 was refused"); + + let ample_json: Value = serde_json::from_slice(&le.stdout).unwrap_or_else(|e| { + panic!( + "ample-limit scan printed no JSON ({e}); stderr:\n{}", + String::from_utf8_lossy(&le.stderr) + ) + }); + assert_eq!( + ample_json["scannedPackages"].as_u64(), + Some(expected as u64), + "{ample_json}" + ); + assert_eq!( + String::from_utf8_lossy(&tight.stdout), + String::from_utf8_lossy(&le.stdout), + "tight-limit stderr:\n{}", + String::from_utf8_lossy(&tight.stderr) + ); + assert_eq!(tight.status.code(), ample.status.code()); +} diff --git a/crates/socket-patch-core/src/crawlers/mod.rs b/crates/socket-patch-core/src/crawlers/mod.rs index f95b4b51..baeade24 100644 --- a/crates/socket-patch-core/src/crawlers/mod.rs +++ b/crates/socket-patch-core/src/crawlers/mod.rs @@ -10,6 +10,7 @@ pub mod pkg_managers; pub mod python_crawler; pub mod ruby_crawler; pub mod types; +pub mod walk_pool; pub use cargo_crawler::CargoCrawler; pub use composer_crawler::ComposerCrawler; diff --git a/crates/socket-patch-core/src/crawlers/npm_crawler.rs b/crates/socket-patch-core/src/crawlers/npm_crawler.rs index 6d778ada..255e76e8 100644 --- a/crates/socket-patch-core/src/crawlers/npm_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/npm_crawler.rs @@ -7,8 +7,9 @@ use rayon::prelude::*; use serde::Deserialize; use super::types::{CrawledPackage, CrawlerOptions}; +use super::walk_pool::run_walk; use crate::patch::path_safety; -use crate::utils::fs::{is_dir, is_dir_sync, read_dir_entries_sync, run_blocking}; +use crate::utils::fs::{is_dir, is_dir_sync, read_dir_entries_sync}; use crate::utils::purl::{percent_decode_purl_component, strip_purl_qualifiers}; #[cfg(test)] @@ -618,13 +619,14 @@ impl NpmCrawler { options: &CrawlerOptions, ) -> Result, std::io::Error> { let options = options.clone(); - Ok(run_blocking(move || Self::node_modules_paths_sync(&options)).await) + Ok(run_walk(move || Self::node_modules_paths_sync(&options)).await) } /// Crawl all discovered `node_modules` and return every package found. /// - /// The whole walk runs as ONE blocking-pool task (instead of one - /// `spawn_blocking` round trip per readdir/stat/read): directory I/O is + /// The whole walk runs as ONE task on the walk pool (instead of one + /// `spawn_blocking` round trip per readdir/stat/read; see + /// [`super::walk_pool`]): directory I/O is /// gathered in parallel into per-root [`ScanEvent`] trees that record /// the sequential visit order, then [`Self::merge_scan_events`] replays /// them single-threaded so the order-dependent `seen` dedup (and the @@ -632,7 +634,7 @@ impl NpmCrawler { /// sequential walk would have — same packages, same paths, same order. pub async fn crawl_all(&self, options: &CrawlerOptions) -> Vec { let options = options.clone(); - run_blocking(move || Self::crawl_all_sync(&options)).await + run_walk(move || Self::crawl_all_sync(&options)).await } fn crawl_all_sync(options: &CrawlerOptions) -> Vec { @@ -707,12 +709,12 @@ impl NpmCrawler { }); } - // Both passes run as one blocking-pool task: each visited dir is + // Both passes run as one walk-pool task: each visited dir is // listed once, and that listing both bounds which targets are // probed there and drives the descent (see // `resolve_pending_targets`). let node_modules_path = node_modules_path.to_path_buf(); - Ok(run_blocking(move || { + Ok(run_walk(move || { let mut result: HashMap> = HashMap::new(); // Pass 1 — filtered: `.pnpm` virtual-store entries are enqueued @@ -1510,7 +1512,7 @@ impl NpmCrawler { /// [`Self::list_pnpm_store_entries_sync`] for the async callers. async fn list_pnpm_store_entries(store_path: &Path) -> Vec<(String, PathBuf)> { let store_path = store_path.to_path_buf(); - run_blocking(move || { + run_walk(move || { Self::list_pnpm_store_entries_sync(&store_path, false) .into_iter() .map(|entry| (entry.name, entry.node_modules)) @@ -1599,9 +1601,7 @@ impl NpmCrawler { #[cfg(test)] async fn collect_nested_store_entries(host_path: &Path, entries: &mut Vec<(String, PathBuf)>) { let host_path = host_path.to_path_buf(); - entries.extend( - run_blocking(move || Self::collect_nested_store_entries_sync(&host_path)).await, - ); + entries.extend(run_walk(move || Self::collect_nested_store_entries_sync(&host_path)).await); } // ------------------------------------------------------------------ diff --git a/crates/socket-patch-core/src/crawlers/walk_pool.rs b/crates/socket-patch-core/src/crawlers/walk_pool.rs new file mode 100644 index 00000000..fdfb031b --- /dev/null +++ b/crates/socket-patch-core/src/crawlers/walk_pool.rs @@ -0,0 +1,180 @@ +//! The dedicated thread pool the npm crawler's parallel walks run on, and +//! the file-descriptor budget that sizes it. +//! +//! Two properties of the old one-`spawn_blocking`-per-call async walk must +//! survive the move to parallel sync walks: +//! +//! - **Stack depth.** The async walk recursed through `Box::pin` futures +//! polled on the `#[tokio::main]` thread (8 MiB on Unix, and on Windows +//! via the `/STACK` link flag in `.cargo/config.toml`). Rayon's and +//! tokio's worker threads default to 2 MiB, so the recursive gather runs +//! on pool threads built with the same 8 MiB ([`WALK_STACK_SIZE`]). +//! - **Descriptor headroom.** The sequential walk held at most one +//! directory stream (or package.json) open at a time, and the nine +//! ecosystem crawlers ran one after another. Every walker treats a failed +//! `read_dir`/open — `EMFILE` included — as "absent", so a process that +//! needs more descriptors than the old one would silently drop packages +//! under a tight `RLIMIT_NOFILE` the old one handled. Under such a limit +//! ([`fd_limit_is_tight`]) the pool gets ONE thread and the crawlers run +//! serially (the old descriptor profile); otherwise the thread count is +//! capped so the extra descriptors stay well inside the limit. + +use std::sync::OnceLock; + +use crate::utils::fs::run_blocking; + +/// Stack size of each walk thread: the main thread's (see module docs). +const WALK_STACK_SIZE: usize = 8 * 1024 * 1024; + +/// A soft `RLIMIT_NOFILE` below this runs the crawl with the sequential +/// walk's descriptor profile: one walk thread, crawlers one at a time. +/// (macOS's default soft limit is 256, Linux's 1024.) +const TIGHT_NOFILE_LIMIT: u64 = 128; + +/// Descriptors left to the rest of the process (stdio, the runtime, the +/// other crawlers' walks and subprocess pipes) when sizing the pool above +/// the tight limit; each walk thread holds at most one descriptor, and +/// the pool takes at most half of what remains. +const RESERVED_FDS: u64 = 64; + +/// The process's soft `RLIMIT_NOFILE`, read once. `None` when unlimited +/// or unknown (and on Windows, whose handle table has no comparable +/// per-process cap). +fn soft_nofile_limit() -> Option { + static LIMIT: OnceLock> = OnceLock::new(); + *LIMIT.get_or_init(read_soft_nofile_limit) +} + +#[cfg(unix)] +fn read_soft_nofile_limit() -> Option { + let mut limit = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + // SAFETY: getrlimit only writes the struct we pass it. + if unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) } != 0 { + return None; + } + if limit.rlim_cur == libc::RLIM_INFINITY { + return None; + } + #[allow(clippy::unnecessary_cast)] // rlim_t is u32 on some targets + Some(limit.rlim_cur as u64) +} + +#[cfg(not(unix))] +fn read_soft_nofile_limit() -> Option { + None +} + +/// Whether the descriptor limit is too tight for concurrent crawling (see +/// the module docs). The ecosystem dispatch then runs the crawlers one at +/// a time, and the walk pool has a single thread. +pub fn fd_limit_is_tight() -> bool { + is_tight(soft_nofile_limit()) +} + +fn is_tight(soft_limit: Option) -> bool { + soft_limit.is_some_and(|limit| limit < TIGHT_NOFILE_LIMIT) +} + +/// Walk threads for `cpus` logical CPUs under `soft_limit`. +fn walk_threads(cpus: usize, soft_limit: Option) -> usize { + if is_tight(soft_limit) { + return 1; + } + let cpus = cpus.max(1); + match soft_limit { + Some(limit) => { + let budget = + usize::try_from(limit.saturating_sub(RESERVED_FDS) / 2).unwrap_or(usize::MAX); + cpus.min(budget).max(1) + } + None => cpus, + } +} + +/// Logical CPUs, honoring `RAYON_NUM_THREADS` like rayon's global pool. +fn default_cpus() -> usize { + std::env::var("RAYON_NUM_THREADS") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|&n| n > 0) + .or_else(|| std::thread::available_parallelism().ok().map(|n| n.get())) + .unwrap_or(1) +} + +/// The walk pool, built on first use. `None` if its threads could not be +/// spawned; the walk then runs on the calling thread (rayon falls back to +/// its global pool for the parallel parts). +fn walk_pool() -> Option<&'static rayon::ThreadPool> { + static POOL: OnceLock> = OnceLock::new(); + POOL.get_or_init(|| { + rayon::ThreadPoolBuilder::new() + .num_threads(walk_threads(default_cpus(), soft_nofile_limit())) + .stack_size(WALK_STACK_SIZE) + .thread_name(|i| format!("socket-patch-walk-{i}")) + .build() + .ok() + }) + .as_ref() +} + +/// Run a blocking walk on the walk pool, from a blocking-pool thread so +/// the async runtime is never stalled, and hand back its value. A panic +/// inside `f` is re-raised on the awaiting task (see [`run_blocking`]). +pub(crate) async fn run_walk(f: F) -> T +where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static, +{ + run_blocking(move || match walk_pool() { + Some(pool) => pool.install(f), + None => f(), + }) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tight_limit_means_one_thread() { + assert!(is_tight(Some(20))); + assert!(is_tight(Some(TIGHT_NOFILE_LIMIT - 1))); + assert!(!is_tight(Some(TIGHT_NOFILE_LIMIT))); + assert!(!is_tight(None)); + for limit in [0, 1, 20, 64, TIGHT_NOFILE_LIMIT - 1] { + assert_eq!(walk_threads(64, Some(limit)), 1, "limit {limit}"); + } + } + + #[test] + fn thread_count_stays_inside_the_descriptor_budget() { + assert_eq!(walk_threads(14, Some(256)), 14); + assert_eq!(walk_threads(14, Some(1024)), 14); + assert_eq!(walk_threads(14, None), 14); + assert_eq!(walk_threads(0, None), 1); + assert_eq!(walk_threads(256, Some(TIGHT_NOFILE_LIMIT)), 32); + assert_eq!(walk_threads(256, Some(1024)), 256); + assert_eq!(walk_threads(1024, Some(1024)), 480); + for limit in [TIGHT_NOFILE_LIMIT, 200, 256, 1024, 4096] { + let threads = walk_threads(1024, Some(limit)) as u64; + assert!(threads >= 1 && threads + RESERVED_FDS <= limit, "{limit}"); + } + } + + /// The walk runs on a pool thread with the main thread's stack, not + /// the 2 MiB default of rayon/tokio workers: a 4 MiB stack frame fits. + #[tokio::test] + async fn walk_runs_on_a_main_sized_stack() { + #[inline(never)] + fn big_frame() -> u8 { + let mut buf = [0u8; 4 << 20]; + std::hint::black_box(&mut buf); + buf[(4 << 20) - 1] + } + assert_eq!(run_walk(big_frame).await, 0); + } +} From 8de0602f6754f68e6aff2006306bb6374969bf6b Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 12:44:56 -0400 Subject: [PATCH 018/237] perf(redirect): parse each pnpm lock once and splice in one pass The hosted pnpm rewriter re-parsed every lock (entries, the early shrinkwrap sniff, the residual gate) and rebuilt the whole lock string once per dep: O(deps x lock) work that cost ~430 ms of critical-path CPU on depscan's 2 MB lock with 74 redirected deps. Each lock is now parsed and key-indexed once; a dep's instances are found by binary search, the residual gate judges each instance on its post-splice body, and committed splices are applied in one pass at the end. A later dep that hits an already-spliced entry (a duplicate name@version override) folds the pending splices in and re-indexes first, so it re-reads the rewritten text exactly as before, and the vendored-marker scan runs over the post-splice text the same way. Output bytes, the FileEdit list (order and original fragments), warnings and refusals are unchanged: the previous implementation is kept as a test oracle and compared on a depscan-sized synthetic lock set, on 300 randomized mixes of every lock flavor, and on duplicate-override and peer-suffixed multi-instance cases. depscan wet run: pnpm-lock.yaml and redirect-state.json byte-identical. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/patch/redirect/mod.rs | 305 +++++-- .../patch/redirect/pnpm_equivalence_tests.rs | 783 ++++++++++++++++++ 2 files changed, 1018 insertions(+), 70 deletions(-) create mode 100644 crates/socket-patch-core/src/patch/redirect/pnpm_equivalence_tests.rs diff --git a/crates/socket-patch-core/src/patch/redirect/mod.rs b/crates/socket-patch-core/src/patch/redirect/mod.rs index ebcd2fab..1c97b612 100644 --- a/crates/socket-patch-core/src/patch/redirect/mod.rs +++ b/crates/socket-patch-core/src/patch/redirect/mod.rs @@ -36,6 +36,8 @@ mod pipenv; // pub(crate): manifest-less VEX discovery (`vex::discover::npm`) reads // hosted pnpm locks with the SAME grammar this rewriter writes them in. pub(crate) mod pnpm; +#[cfg(test)] +mod pnpm_equivalence_tests; mod poetry; mod replay; mod requirements; @@ -2633,6 +2635,7 @@ fn plan_cargo_config( /// Audit every matching package instance after planning edits. A malformed /// resolution or unsupported suffix refuses this dependency across all locks; /// snapshots and other versions do not participate in resolution. +#[cfg(test)] fn pnpm_unrewritten_instances( content: &str, fname: &str, @@ -2643,13 +2646,166 @@ fn pnpm_unrewritten_instances( .into_iter() .filter_map(|entry| { pnpm::suffix(entry.key, fname, version)?; - let rewritten = - pnpm::resolution(&entry).is_some_and(|r| r.tarball() == Some(artifact_url)); - (!rewritten).then(|| entry.key.to_string()) + (!pnpm_resolves_to(&entry, artifact_url)).then(|| entry.key.to_string()) }) .collect() } +/// Whether `entry` resolves to exactly `artifact_url` — the per-instance +/// residual-gate predicate. +fn pnpm_resolves_to(entry: &pnpm::Entry<'_>, artifact_url: &str) -> bool { + pnpm::resolution(entry).is_some_and(|r| r.tarball() == Some(artifact_url)) +} + +/// One pnpm lock under rewrite. `text` is the lock as of the last +/// materialization; `pending` holds the resolution splices committed since, +/// in `text`'s byte coordinates, and `spliced` the entries they touch. +/// +/// The logical (post-splice) lock is `text` with `pending` applied. Parsing +/// once and indexing is sound because a resolution splice never changes the +/// entry structure: the replaced range and its replacement are only +/// resolution-field material (6-space-indented `k: v` child lines of a block +/// resolution, or the `{…}` flow value after ` resolution:`), and no raw +/// newline can enter a value (`Resolution::rewrite` JSON-quotes whitespace). +/// So every column-0 line (the shrinkwrap-version sniff) and every entry +/// boundary line survives unchanged, and an entry no pending splice touched +/// has byte-identical key and body. An entry that WAS touched is re-read +/// only after materializing, so a later dep with the same name@version (a +/// duplicate override) sees the rewritten text exactly as before. +struct PnpmLockState<'f> { + path: &'f String, + text: Cow<'f, str>, + early_shrinkwrap: bool, + /// (key span, body span) per `packages:` entry, in file order. + entries: Vec<(std::ops::Range, std::ops::Range)>, + /// Entry indices sorted by normalized (unquoted, `/`-stripped) key. + sorted: Vec, + pending: Vec<(std::ops::Range, String)>, + spliced: std::collections::HashSet, + changed: bool, +} + +impl<'f> PnpmLockState<'f> { + fn new(path: &'f String, text: &'f str) -> Self { + let mut state = PnpmLockState { + path, + text: Cow::Borrowed(text), + early_shrinkwrap: pnpm::unsupported_early_shrinkwrap(text), + entries: Vec::new(), + sorted: Vec::new(), + pending: Vec::new(), + spliced: Default::default(), + changed: false, + }; + state.reindex(); + state + } + + fn reindex(&mut self) { + let text: &str = &self.text; + let base = text.as_ptr() as usize; + self.entries = pnpm::entries(text) + .iter() + .map(|e| { + let key_start = e.key.as_ptr() as usize - base; + ( + key_start..key_start + e.key.len(), + e.offset..e.offset + e.body.len(), + ) + }) + .collect(); + let mut sorted: Vec = (0..self.entries.len()).collect(); + sorted.sort_by(|&a, &b| self.norm_key(a).cmp(self.norm_key(b)).then(a.cmp(&b))); + self.sorted = sorted; + } + + fn entry(&self, i: usize) -> pnpm::Entry<'_> { + let (key, body) = &self.entries[i]; + pnpm::Entry { + key: &self.text[key.clone()], + body: &self.text[body.clone()], + offset: body.start, + } + } + + /// The key as [`pnpm::suffix`] compares it. + fn norm_key(&self, i: usize) -> &str { + let key = pnpm::unquote(&self.text[self.entries[i].0.clone()]); + key.strip_prefix('/').unwrap_or(key) + } + + /// Entries whose key names `fname@version` (any suffix), in file order — + /// the same set a full [`pnpm::suffix`] scan of the logical lock yields. + fn hits(&mut self, fname: &str, version: &str) -> Vec { + let hits = self.lookup(fname, version); + if hits.iter().any(|i| self.spliced.contains(i)) { + self.materialize(); + return self.lookup(fname, version); + } + hits + } + + fn lookup(&self, fname: &str, version: &str) -> Vec { + let mut out = Vec::new(); + for sep in ['@', '/'] { + let prefix = format!("{fname}{sep}{version}"); + let start = self + .sorted + .partition_point(|&i| self.norm_key(i) < prefix.as_str()); + out.extend( + self.sorted[start..] + .iter() + .take_while(|&&i| self.norm_key(i).starts_with(prefix.as_str())) + .copied(), + ); + } + out.sort_unstable(); + out.dedup(); + out.retain(|&i| pnpm::suffix(self.entry(i).key, fname, version).is_some()); + out + } + + /// Fold `pending` into `text` and re-parse. + fn materialize(&mut self) { + if self.pending.is_empty() { + return; + } + #[cfg(debug_assertions)] + let keys_before: Vec = (0..self.entries.len()) + .map(|i| self.entry(i).key.to_string()) + .collect(); + let mut pending = std::mem::take(&mut self.pending); + pending.sort_by_key(|(range, _)| range.start); + let mut out = String::with_capacity(self.text.len()); + let mut cursor = 0usize; + for (range, replacement) in pending { + out.push_str(&self.text[cursor..range.start]); + out.push_str(&replacement); + cursor = range.end; + } + out.push_str(&self.text[cursor..]); + self.text = Cow::Owned(out); + self.spliced.clear(); + self.reindex(); + #[cfg(debug_assertions)] + debug_assert_eq!( + keys_before, + (0..self.entries.len()) + .map(|i| self.entry(i).key.to_string()) + .collect::>(), + "a resolution splice changed the pnpm entry structure" + ); + } + + fn into_rewritten(mut self) -> Option<(&'f String, String)> { + if !self.changed { + return None; + } + self.materialize(); + Some((self.path, self.text.into_owned())) + } +} + fn rewrite_pnpm_lock( files: &BTreeMap, overrides: &[DepOverride], @@ -2672,21 +2828,23 @@ fn rewrite_pnpm_lock( if npm.is_empty() || lock_keys.is_empty() { return; } - let mut contents: Vec<(&String, String, bool)> = lock_keys + // Each lock is parsed and indexed ONCE; splices accumulate per lock and + // are applied in one pass at the end (see `PnpmLockState`). + let mut locks: Vec = lock_keys .iter() - .map(|k| (*k, files[*k].clone(), false)) + .map(|k| PnpmLockState::new(k, &files[*k])) .collect(); for dep in &npm { let fname = full_name(dep); - let unsafe_locks: Vec<_> = contents + let hits: Vec> = locks + .iter_mut() + .map(|lock| lock.hits(&fname, &dep.version)) + .collect(); + let unsafe_locks: Vec<_> = locks .iter() - .filter(|(_, content, _)| { - pnpm::unsupported_early_shrinkwrap(content) - && pnpm::entries(content) - .iter() - .any(|e| pnpm::suffix(e.key, &fname, &dep.version).is_some()) - }) - .map(|(path, _, _)| path.as_str()) + .zip(&hits) + .filter(|(lock, hits)| lock.early_shrinkwrap && !hits.is_empty()) + .map(|(lock, _)| lock.path.as_str()) .collect(); if !unsafe_locks.is_empty() { result.refused_pnpm_uuids.insert(dep.patch_uuid.clone()); @@ -2710,75 +2868,77 @@ fn rewrite_pnpm_lock( // residual gate below proves no instance of this dep escaped the // splice grammar in ANY lock — committing lock-by-lock as we go // would ship exactly the partial rewrite the gate exists to refuse. - let mut planned: Vec<(usize, String, Vec)> = Vec::new(); + type Splice = (usize, std::ops::Range, String); + let mut planned: Vec<(usize, Vec, Vec)> = Vec::new(); let mut residuals: Vec<(&str, Vec)> = Vec::new(); - for (idx, (lock_key, content, _)) in contents.iter().enumerate() { - // (byte range to replace, replacement text) per instance, plus - // one FileEdit per instance keyed by the canonical instance key — - // per-instance edits keep the revert ledger lossless when several - // instances of one dep live in the same lock. - let mut splices: Vec<(std::ops::Range, String)> = Vec::new(); + for (idx, (lock, hits)) in locks.iter().zip(&hits).enumerate() { + // (entry, byte range to replace, replacement text) per instance, + // plus one FileEdit per instance keyed by the canonical instance + // key — per-instance edits keep the revert ledger lossless when + // several instances of one dep live in the same lock. + let mut splices: Vec = Vec::new(); let mut instance_edits: Vec = Vec::new(); - for entry in pnpm::entries(content) { - let Some(suffix) = pnpm::suffix(entry.key, &fname, &dep.version) else { - continue; + // Residual gate, judged per instance on its POST-splice body: + // any instance of this exact name@version still resolving + // somewhere other than the hosted artifact — in a spelling the + // splice grammar cannot parse (e.g. an unbalanced peer suffix) — + // makes this a partial rewrite. Shipping it would confirm and + // VEX-attest the dep while dependents through the unmatched + // instance keep installing the unpatched upstream tarball, so + // the dep is refused instead. + let mut leftover: Vec = Vec::new(); + for &i in hits { + let entry = lock.entry(i); + let suffix = pnpm::suffix(entry.key, &fname, &dep.version) + .expect("hits only holds entries naming this dep"); + let resolution = if pnpm::supported_suffix(suffix) { + pnpm::resolution(&entry) + } else { + None }; - if !pnpm::supported_suffix(suffix) { - continue; - } - let Some(resolution) = pnpm::resolution(&entry) else { + let Some(resolution) = resolution else { + if !pnpm_resolves_to(&entry, &dep.artifact_url) { + leftover.push(entry.key.to_string()); + } continue; }; matched_any = true; - let original = &content[resolution.range.clone()]; + let original = &lock.text[resolution.range.clone()]; let rebuilt = resolution.rewrite(&sha512, &dep.artifact_url); + let rel = + resolution.range.start - entry.offset..resolution.range.end - entry.offset; + let body = format!( + "{}{rebuilt}{}", + &entry.body[..rel.start], + &entry.body[rel.end..] + ); + let after = pnpm::Entry { + key: entry.key, + body: &body, + offset: 0, + }; + if !pnpm_resolves_to(&after, &dep.artifact_url) { + leftover.push(entry.key.to_string()); + } if rebuilt == original { continue; } - splices.push((resolution.range, rebuilt.clone())); instance_edits.push(FileEdit { - path: (*lock_key).clone(), + path: lock.path.clone(), kind: "redirect_pnpm_resolution".into(), action: "rewritten".into(), key: Some(format!("{fname}@{}{suffix}", dep.version)), original: Some(Value::String(original.to_string())), - new: Some(Value::String(rebuilt)), + new: Some(Value::String(rebuilt.clone())), }); + splices.push((i, resolution.range, rebuilt)); } - // Splice by byte range (package blocks are disjoint and ordered) — a string replace could hit the wrong - // instance when two entries share identical surrounding bytes. - let candidate: Option = if splices.is_empty() { - None - } else { - let mut out = String::with_capacity(content.len()); - let mut cursor = 0usize; - for (range, replacement) in splices { - out.push_str(&content[cursor..range.start]); - out.push_str(&replacement); - cursor = range.end; - } - out.push_str(&content[cursor..]); - Some(out) - }; - // Residual gate, run over the POST-splice text: any instance of - // this exact name@version still resolving somewhere other than - // the hosted artifact — in a spelling the splice grammar cannot - // parse (e.g. an unbalanced peer suffix) — makes this a partial - // rewrite. Shipping it would confirm and VEX-attest the dep while - // dependents through the unmatched instance keep installing the - // unpatched upstream tarball, so the dep is refused instead. - let leftover = pnpm_unrewritten_instances( - candidate.as_deref().unwrap_or(content), - &fname, - &dep.version, - &dep.artifact_url, - ); if !leftover.is_empty() { - residuals.push(((*lock_key).as_str(), leftover)); + residuals.push((lock.path.as_str(), leftover)); continue; } - if let Some(out) = candidate { - planned.push((idx, out, instance_edits)); + if !splices.is_empty() { + planned.push((idx, splices, instance_edits)); } } // ANY residual anywhere refuses the dep across the WHOLE lock set — @@ -2804,10 +2964,13 @@ fn rewrite_pnpm_lock( } continue; } - for (idx, out, mut instance_edits) in planned { - let (_, content, changed) = &mut contents[idx]; - *content = out; - *changed = true; + for (idx, splices, mut instance_edits) in planned { + let lock = &mut locks[idx]; + for (i, range, replacement) in splices { + lock.spliced.insert(i); + lock.pending.push((range, replacement)); + } + lock.changed = true; result.edits.append(&mut instance_edits); } // The entry-not-found warning fires only when the dep matched in NO @@ -2822,8 +2985,10 @@ fn rewrite_pnpm_lock( if !matched_any { let v9_vendored_key = format!("{fname}@file:"); let override_key = format!("{fname}@{}", dep.version); - let vendored = contents.iter().any(|(_, content, _)| { - content.lines().any(|line| { + // Scanned over the post-splice text, so fold pending splices in. + let vendored = locks.iter_mut().any(|lock| { + lock.materialize(); + lock.text.lines().any(|line| { let t = line.trim_start(); let t = t.strip_prefix('\'').unwrap_or(t); // v9 packages/snapshots key (leading `/` in v6 spelling). @@ -2870,8 +3035,8 @@ fn rewrite_pnpm_lock( } } } - for (key, content, changed) in contents { - if changed { + for lock in locks { + if let Some((key, content)) = lock.into_rewritten() { result.files.insert(key.clone(), content); } } diff --git a/crates/socket-patch-core/src/patch/redirect/pnpm_equivalence_tests.rs b/crates/socket-patch-core/src/patch/redirect/pnpm_equivalence_tests.rs new file mode 100644 index 00000000..c972ad30 --- /dev/null +++ b/crates/socket-patch-core/src/patch/redirect/pnpm_equivalence_tests.rs @@ -0,0 +1,783 @@ +//! Equivalence oracle for the indexed pnpm hosted rewriter: the previous +//! implementation (re-parse every lock per dep, splice per dep) is kept here +//! verbatim as `rewrite_pnpm_lock_oracle`, and the production +//! `rewrite_pnpm_lock` must produce the identical `RewriteResult` — output +//! bytes, the FileEdit list (order and `original` fragments), warnings and +//! refusals — on depscan-sized synthetic locks and on randomized mixes of +//! every lock flavor the grammar handles. + +use super::*; + +type Snapshot = ( + BTreeMap, + Vec, + Vec<(String, String)>, + std::collections::BTreeSet, +); + +fn snapshot(r: &RewriteResult) -> Snapshot { + ( + r.files.clone(), + r.edits.clone(), + r.warnings + .iter() + .map(|w| (w.code.clone(), w.detail.clone())) + .collect(), + r.refused_pnpm_uuids.clone(), + ) +} + +/// Run both implementations and assert they agree; returns the result. +fn assert_equivalent(files: &BTreeMap, overrides: &[DepOverride]) -> RewriteResult { + let mut want = RewriteResult::default(); + rewrite_pnpm_lock_oracle(files, overrides, &mut want); + let mut got = RewriteResult::default(); + rewrite_pnpm_lock(files, overrides, &mut got); + let (want_s, got_s) = (snapshot(&want), snapshot(&got)); + for (path, want_text) in &want_s.0 { + let got_text = got_s.0.get(path); + assert!( + got_text == Some(want_text), + "rewritten bytes differ for {path} (first diff at byte {:?})", + got_text.map(|g| g + .bytes() + .zip(want_text.bytes()) + .position(|(a, b)| a != b) + .unwrap_or(g.len().min(want_text.len()))) + ); + } + assert_eq!( + got_s.0.keys().collect::>(), + want_s.0.keys().collect::>(), + "rewritten file set" + ); + assert_eq!(got_s.1.len(), want_s.1.len(), "edit count"); + for (i, (g, w)) in got_s.1.iter().zip(&want_s.1).enumerate() { + assert_eq!(g, w, "edit #{i}"); + } + assert_eq!(got_s.2, want_s.2, "warnings (code, detail) in order"); + assert_eq!(got_s.3, want_s.3, "refused pnpm uuids"); + got +} + +/// Deterministic xorshift64* — no `rand` dev-dependency. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } + fn chance(&mut self, percent: u64) -> bool { + self.next() % 100 < percent + } +} + +#[derive(Clone, Copy, PartialEq)] +enum Flavor { + /// lockfileVersion 9: bare `name@ver` packages keys, flow resolutions, + /// peer contexts only in `snapshots:`. + V9, + /// lockfileVersion 6: `/name@ver(peer@x)` packages keys. + V6, + /// lockfileVersion 5.4: `/name/ver_peer@x` keys, flow resolutions. + V54, + /// lockfileVersion 5.1: `/name/ver` keys, BLOCK resolutions. + V51, + /// Early pnpm 1: shrinkwrapVersion 3 with no minor — refused. + EarlyShrinkwrap, +} + +#[derive(Clone)] +struct Pkg { + name: String, + version: String, +} + +fn pkg_name(i: usize) -> String { + match i % 5 { + 0 => format!("@scope{}/pkg-{i}", i % 7), + _ => format!("pkg-{i}"), + } +} + +fn sri(tag: &str) -> String { + format!("sha512-{tag}==") +} + +/// Render one lock. `extra` lines are appended verbatim to `packages:` (for +/// hand-placed residual / vendored / already-hosted instances). +fn render_lock( + flavor: Flavor, + pkgs: &[Pkg], + rng: &mut Rng, + extra: &[String], + crlf: bool, +) -> String { + let mut out = String::new(); + match flavor { + Flavor::V9 => { + out.push_str("lockfileVersion: '9.0'\n\nsettings:\n autoInstallPeers: true\n\n") + } + Flavor::V6 => out.push_str("lockfileVersion: '6.0'\n\n"), + Flavor::V54 => out.push_str("lockfileVersion: 5.4\n\n"), + Flavor::V51 => out.push_str("lockfileVersion: 5.1\n\n"), + Flavor::EarlyShrinkwrap => out.push_str("shrinkwrapVersion: 3\n\n"), + } + out.push_str("importers:\n .:\n dependencies:\n"); + for p in pkgs.iter().take(20) { + let q = if p.name.starts_with('@') { "'" } else { "" }; + out.push_str(&format!( + " {q}{}{q}:\n specifier: ^{}\n version: {}\n", + p.name, p.version, p.version + )); + } + out.push_str("\npackages:\n\n"); + let key = |p: &Pkg, suffix: &str| -> String { + let raw = match flavor { + Flavor::V9 => format!("{}@{}{suffix}", p.name, p.version), + Flavor::V6 => format!("/{}@{}{suffix}", p.name, p.version), + Flavor::V54 | Flavor::V51 | Flavor::EarlyShrinkwrap => { + format!("/{}/{}{suffix}", p.name, p.version) + } + }; + if raw.starts_with('@') || (raw.contains('(') && rng_quote(&raw)) { + format!("'{raw}'") + } else { + raw + } + }; + for (i, p) in pkgs.iter().enumerate() { + // Peer-suffixed multi-instance keys where the flavor encodes them. + let suffixes: Vec = match flavor { + Flavor::V6 if rng.chance(15) => { + let mut s = vec![String::new(), "(react@18.2.0)".to_string()]; + if rng.chance(50) { + s.push("(react@18.2.0(scheduler@0.23.2))(typescript@5.4.5)".into()); + } + s + } + Flavor::V54 if rng.chance(15) => { + vec![ + "_react@18.2.0".into(), + "_react@17.0.2+typescript@5.4.5".into(), + ] + } + _ => vec![String::new()], + }; + for suffix in suffixes { + out.push_str(&format!(" {}:\n", key(p, &suffix))); + let integrity = sri(&format!("UP{i}")); + let tarball = rng.chance(5); + match flavor { + Flavor::V51 | Flavor::EarlyShrinkwrap => { + out.push_str(&format!(" resolution:\n integrity: {integrity}\n")); + if tarball { + out.push_str(&format!( + " tarball: https://registry.npmjs.org/{}/-/x-{}.tgz\n", + p.name, p.version + )); + } + } + _ => { + if tarball { + out.push_str(&format!( + " resolution: {{integrity: {integrity}, tarball: https://registry.npmjs.org/{}/-/x-{}.tgz}}\n", + p.name, p.version + )); + } else { + out.push_str(&format!(" resolution: {{integrity: {integrity}}}\n")); + } + } + } + if rng.chance(30) { + out.push_str(" engines: {node: '>=12'}\n"); + } + if rng.chance(30) { + out.push_str(" dependencies:\n dep-a: 1.0.0\n dep-b: 2.0.0\n"); + } + if matches!(flavor, Flavor::V6 | Flavor::V54 | Flavor::V51) { + out.push_str(" dev: false\n"); + } + out.push('\n'); + } + } + for line in extra { + out.push_str(line); + out.push('\n'); + } + if flavor == Flavor::V9 { + out.push_str("snapshots:\n\n"); + for p in pkgs { + let q = if p.name.starts_with('@') { "'" } else { "" }; + out.push_str(&format!(" {q}{}@{}{q}: {{}}\n\n", p.name, p.version)); + if rng.chance(10) { + out.push_str(&format!( + " {}@{}(react@18.2.0):\n dependencies:\n react: 18.2.0\n\n", + p.name, p.version + )); + } + } + } + if crlf { + out = out.replace('\n', "\r\n"); + } + out +} + +/// pnpm quotes keys carrying flow delimiters inconsistently across +/// releases; exercise both spellings deterministically. +fn rng_quote(raw: &str) -> bool { + raw.len().is_multiple_of(2) +} + +fn dep(p: &Pkg, uuid: usize, url_tag: &str, sha512: Option<&str>) -> DepOverride { + let (namespace, name) = match p.name.split_once('/') { + Some((ns, name)) if p.name.starts_with('@') => (Some(ns.to_string()), name.to_string()), + _ => (None, p.name.clone()), + }; + DepOverride { + ecosystem: "npm".into(), + name, + namespace, + version: p.version.clone(), + token: String::new(), + patch_uuid: format!("00000000-0000-4000-8000-{uuid:012}"), + artifact_url: format!( + "https://patch.socket.dev/{url_tag}/{}-{}.tgz", + p.name, p.version + ), + berry_zip_url: None, + registry_override: None, + integrity: Integrity { + sha512: sha512.map(str::to_string), + ..Default::default() + }, + } +} + +fn packages(n: usize, rng: &mut Rng) -> Vec { + (0..n) + .map(|i| Pkg { + name: pkg_name(i), + version: format!("{}.{}.{}", rng.below(20), rng.below(20), rng.below(20)), + }) + .collect() +} + +/// A depscan-sized lock set (~5.5k instances across a root v9 lock plus a +/// nested v6 lock and a nested block-resolution v5.1 lock) with ~110 overrides +/// covering every outcome: rewritten single and peer-suffixed multi-instance +/// deps, duplicate overrides of one name@version (same and different URL), +/// already-hosted, residual (unbalanced peer suffix), vendored, not-found and +/// missing-sha512 deps, plus a non-npm override the rewriter must ignore. +#[test] +fn indexed_pnpm_rewrite_matches_oracle_on_depscan_sized_lock_set() { + let mut rng = Rng(0x5eed_cafe_f00d_0001); + let root = packages(4000, &mut rng); + let nested = packages(1200, &mut rng); + let legacy = packages(300, &mut rng); + let already = &root[7]; + let residual = &nested[11]; + let vendored = Pkg { + name: "vendored-pkg".into(), + version: "1.0.0".into(), + }; + let root_extra = vec![ + format!( + " vendored-pkg@file:.socket/vendor/npm/vendored-pkg-1.0.0.tgz:\n resolution: {{tarball: file:.socket/vendor/npm/vendored-pkg-1.0.0.tgz}}\n" + ), + // A second copy of an entry that already points at the hosted + // artifact (a re-run): rebuilt == original, no edit. + format!( + " {}@{}(zzz@1.0.0):\n resolution: {{integrity: sha512-ALREADY==, tarball: https://patch.socket.dev/a/{}-{}.tgz}}\n", + already.name, already.version, already.name, already.version + ), + ]; + let nested_extra = vec![format!( + " /{}@{}(react@18.2.0:\n resolution: {{integrity: sha512-UNBALANCED==}}\n", + residual.name, residual.version + )]; + let mut files = BTreeMap::new(); + files.insert( + "pnpm-lock.yaml".to_string(), + render_lock(Flavor::V9, &root, &mut rng, &root_extra, false), + ); + files.insert( + "packages/app/pnpm-lock.yaml".to_string(), + render_lock(Flavor::V6, &nested, &mut rng, &nested_extra, false), + ); + files.insert( + "legacy/shrinkwrap.yaml".to_string(), + render_lock(Flavor::V51, &legacy, &mut rng, &[], true), + ); + // Not a lock: never read. + files.insert("package.json".to_string(), "{}\n".to_string()); + + let mut overrides = Vec::new(); + let mut uuid = 0; + for i in 0..90 { + let p = match i % 3 { + 0 => &root[rng.below(root.len())], + 1 => &nested[rng.below(nested.len())], + _ => &legacy[rng.below(legacy.len())], + }; + uuid += 1; + overrides.push(dep(p, uuid, "a", Some(&sri(&format!("P{i}"))))); + } + // Duplicate overrides of one name@version: different URL (the second + // must see the first's rewritten text) and identical URL (no-op). + for (i, p) in [&root[100], &nested[200], &legacy[30]] + .into_iter() + .enumerate() + { + uuid += 1; + overrides.push(dep(p, uuid, "a", Some(&sri(&format!("D{i}"))))); + uuid += 1; + overrides.push(dep(p, uuid, "b", Some(&sri(&format!("E{i}"))))); + uuid += 1; + overrides.push(dep(p, uuid, "b", Some(&sri(&format!("E{i}"))))); + } + uuid += 1; + overrides.push(dep(already, uuid, "a", Some(&sri("A")))); + uuid += 1; + overrides.push(dep(residual, uuid, "a", Some(&sri("R")))); + uuid += 1; + overrides.push(dep(&vendored, uuid, "a", Some(&sri("V")))); + for i in 0..5 { + uuid += 1; + let absent = Pkg { + name: format!("absent-{i}"), + version: "9.9.9".into(), + }; + overrides.push(dep(&absent, uuid, "a", Some(&sri("N")))); + } + uuid += 1; + overrides.push(dep(&root[50], uuid, "a", None)); + uuid += 1; + let mut pypi = dep(&root[60], uuid, "a", Some(&sri("Y"))); + pypi.ecosystem = "pypi".into(); + overrides.push(pypi); + // Interleave so duplicate / residual / not-found deps land between + // ordinary rewrites (pending splices exist when they are processed). + let mut shuffled = Vec::with_capacity(overrides.len()); + while !overrides.is_empty() { + let i = rng.below(overrides.len()); + shuffled.push(overrides.remove(i)); + } + + let r = assert_equivalent(&files, &shuffled); + // The fixture must actually exercise the interesting paths. + assert!(r.edits.len() > 90, "edits: {}", r.edits.len()); + assert_eq!(r.files.len(), 3, "{:?}", r.files.keys()); + let codes: Vec<&str> = r.warnings.iter().map(|w| w.code.as_str()).collect(); + for code in [ + "redirect_pnpm_unsupported_lock_key", + "redirect_pnpm_entry_vendored", + "redirect_pnpm_entry_not_found", + "redirect_pnpm_missing_sha512", + ] { + assert!(codes.contains(&code), "missing {code}: {codes:?}"); + } +} + +/// Two overrides of the SAME name@version with different artifacts: the +/// second re-reads the first's rewritten resolution (its edit's `original` +/// is the first's `new`), exactly as the per-dep re-parse did. +#[test] +fn duplicate_override_sees_the_prior_rewrite() { + let p = Pkg { + name: "left-pad".into(), + version: "1.3.0".into(), + }; + let lock = "lockfileVersion: '6.0'\n\npackages:\n\n /left-pad@1.3.0:\n resolution: {integrity: sha512-UP==}\n dev: false\n\n /left-pad@1.3.0(react@18.2.0):\n resolution: {integrity: sha512-UP==}\n dev: false\n"; + let files = BTreeMap::from([("pnpm-lock.yaml".to_string(), lock.to_string())]); + let first = dep(&p, 1, "a", Some("sha512-FIRST==")); + let second = dep(&p, 2, "b", Some("sha512-SECOND==")); + let r = assert_equivalent(&files, &[first.clone(), second.clone()]); + assert_eq!(r.edits.len(), 4, "{:#?}", r.edits); + for i in 0..2 { + assert_eq!(r.edits[i + 2].original, r.edits[i].new, "edit {i}"); + } + let out = &r.files["pnpm-lock.yaml"]; + assert_eq!( + out.matches(second.artifact_url.as_str()).count(), + 2, + "{out}" + ); + assert!(!out.contains(first.artifact_url.as_str()), "{out}"); + // The same pair with identical artifacts: the second is a no-op. + let r = assert_equivalent(&files, &[second.clone(), second]); + assert_eq!(r.edits.len(), 2, "{:#?}", r.edits); +} + +/// Every peer-suffixed instance gets its own edit, in file order, keyed by +/// the canonical instance key — across a quoted scoped v6 key, nested peer +/// contexts and a v5 `_` suffix in a second lock. +#[test] +fn peer_suffixed_instances_rewrite_in_file_order() { + let lock_v6 = "lockfileVersion: '6.0'\n\npackages:\n\n '/@s/p@1.0.0(react@18.2.0(scheduler@0.23.2))':\n resolution: {integrity: sha512-UP==}\n\n /@s/p@1.0.0:\n resolution: {integrity: sha512-UP==}\n\n /@s/p@1.0.0(react@17.0.2):\n resolution: {integrity: sha512-UP==}\n\n /@s/p@1.0.01:\n resolution: {integrity: sha512-OTHER==}\n"; + let lock_v5 = "lockfileVersion: 5.4\n\npackages:\n\n /@s/p/1.0.0_react@18.2.0:\n resolution: {integrity: sha512-UP==}\n"; + let files = BTreeMap::from([ + ("a/pnpm-lock.yaml".to_string(), lock_v6.to_string()), + ("b/pnpm-lock.yaml".to_string(), lock_v5.to_string()), + ]); + let p = Pkg { + name: "@s/p".into(), + version: "1.0.0".into(), + }; + let r = assert_equivalent(&files, &[dep(&p, 1, "a", Some("sha512-P=="))]); + let keys: Vec<(&str, &str)> = r + .edits + .iter() + .map(|e| (e.path.as_str(), e.key.as_deref().unwrap_or(""))) + .collect(); + assert_eq!( + keys, + vec![ + ( + "a/pnpm-lock.yaml", + "@s/p@1.0.0(react@18.2.0(scheduler@0.23.2))" + ), + ("a/pnpm-lock.yaml", "@s/p@1.0.0"), + ("a/pnpm-lock.yaml", "@s/p@1.0.0(react@17.0.2)"), + ("b/pnpm-lock.yaml", "@s/p@1.0.0_react@18.2.0"), + ] + ); + assert!(r.files["a/pnpm-lock.yaml"].contains("sha512-OTHER==")); +} + +/// Randomized small lock sets over every flavor (including the refused +/// early shrinkwrap and CRLF), with overrides drawn so hits, duplicates, +/// residuals and misses all interleave. +#[test] +fn indexed_pnpm_rewrite_matches_oracle_on_random_lock_sets() { + let flavors = [ + Flavor::V9, + Flavor::V6, + Flavor::V54, + Flavor::V51, + Flavor::EarlyShrinkwrap, + ]; + for seed in 1..=300u64 { + let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); + let lock_count = 1 + rng.below(3); + let mut files = BTreeMap::new(); + let mut all: Vec = Vec::new(); + for l in 0..lock_count { + let flavor = if rng.chance(8) { + Flavor::EarlyShrinkwrap + } else { + flavors[rng.below(4)] + }; + // Overlapping name pools across locks, so one dep spans locks. + let mut pkgs = packages(4 + rng.below(12), &mut rng); + for p in pkgs.iter_mut() { + if rng.chance(40) { + p.version = "1.0.0".into(); + } + } + let mut extra = Vec::new(); + if rng.chance(20) { + let p = &pkgs[rng.below(pkgs.len())]; + extra.push(match flavor { + Flavor::V9 => format!( + " {}@{}(x@1:\n resolution: {{integrity: sha512-BAD==}}\n", + p.name, p.version + ), + Flavor::V6 => format!( + " /{}@{}(x@1:\n resolution: {{integrity: sha512-BAD==}}\n", + p.name, p.version + ), + _ => format!( + " /{}/{}_x@1)(:\n resolution: {{integrity: sha512-BAD==}}\n", + p.name, p.version + ), + }); + } + if rng.chance(20) { + let p = &pkgs[rng.below(pkgs.len())]; + extra.push(format!( + " {}@{}:\n resolution: {{integrity: sha512-X==, tarball: https://patch.socket.dev/a/{}-{}.tgz}}\n", + p.name, p.version, p.name, p.version + )); + } + if rng.chance(10) { + let p = &pkgs[rng.below(pkgs.len())]; + // A malformed resolution (nested value) — refused, residual. + extra.push(format!( + " /{}@{}(y@2.0.0):\n resolution: {{integrity: {{nested: 1}}}}\n", + p.name, p.version + )); + } + let crlf = rng.chance(15); + let path = match l { + 0 => "pnpm-lock.yaml".to_string(), + 1 => "packages/x/pnpm-lock.yaml".to_string(), + _ => "common/config/rush/shrinkwrap.yaml".to_string(), + }; + files.insert(path, render_lock(flavor, &pkgs, &mut rng, &extra, crlf)); + all.extend(pkgs); + } + let mut overrides = Vec::new(); + for i in 0..(1 + rng.below(10)) { + let p = if rng.chance(15) { + Pkg { + name: format!("missing-{i}"), + version: "1.0.0".into(), + } + } else { + all[rng.below(all.len())].clone() + }; + let tag = ["a", "b"][rng.below(2)]; + let sha = if rng.chance(5) { + None + } else { + Some(sri(&format!("S{}", rng.below(3)))) + }; + overrides.push(dep(&p, i, tag, sha.as_deref())); + if rng.chance(15) { + // Immediate duplicate of the same name@version. + let tag = ["a", "b"][rng.below(2)]; + overrides.push(dep(&p, i + 100, tag, Some(&sri("DUP")))); + } + } + assert_equivalent(&files, &overrides); + } +} + +// ── oracle: the pre-index implementation, verbatim ────────────────────────── + +fn rewrite_pnpm_lock_oracle( + files: &BTreeMap, + overrides: &[DepOverride], + result: &mut RewriteResult, +) { + let npm: Vec<&DepOverride> = overrides.iter().filter(|o| o.ecosystem == "npm").collect(); + // A pnpm lock lives at the project root or at any nested path (e.g. Rush + // repos keep them under `common/config/rush/`); every such files-map key + // is rewritten under the same grammar. Deterministic order: BTreeMap + // iterates keys sorted, so goldens are stable across every lock in the set. + let lock_keys: Vec<&String> = files + .keys() + .filter(|k| { + matches!( + k.rsplit('/').next(), + Some("pnpm-lock.yaml" | "shrinkwrap.yaml") + ) + }) + .collect(); + if npm.is_empty() || lock_keys.is_empty() { + return; + } + let mut contents: Vec<(&String, String, bool)> = lock_keys + .iter() + .map(|k| (*k, files[*k].clone(), false)) + .collect(); + for dep in &npm { + let fname = full_name(dep); + let unsafe_locks: Vec<_> = contents + .iter() + .filter(|(_, content, _)| { + pnpm::unsupported_early_shrinkwrap(content) + && pnpm::entries(content) + .iter() + .any(|e| pnpm::suffix(e.key, &fname, &dep.version).is_some()) + }) + .map(|(path, _, _)| path.as_str()) + .collect(); + if !unsafe_locks.is_empty() { + result.refused_pnpm_uuids.insert(dep.patch_uuid.clone()); + result.warnings.push(RewriteWarning { + code: "redirect_pnpm_legacy_lockfile_unsupported".into(), + detail: format!("{} uses early pnpm 1 shrinkwrapVersion 3 without a supported minor version. Those installers discard hosted tarball URLs; {fname}@{} was left unchanged in every lock. Upgrade to a tested pnpm release (1.43.1 or newer) and regenerate the lock, or use `scan --mode agent` for installed-file patching.", unsafe_locks.join(", "), dep.version), + }); + continue; + } + let Some(sha512) = dep.integrity.sha512.clone() else { + result.warnings.push(RewriteWarning { + code: "redirect_pnpm_missing_sha512".into(), + detail: format!("{fname}@{} has no sha512 integrity", dep.version), + }); + continue; + }; + // Every peer instance must be redirected, including nested peer + // contexts and the block resolutions emitted by pnpm 1–5. + let mut matched_any = false; + // Per-lock rewrites are PLANNED first and committed only after the + // residual gate below proves no instance of this dep escaped the + // splice grammar in ANY lock — committing lock-by-lock as we go + // would ship exactly the partial rewrite the gate exists to refuse. + let mut planned: Vec<(usize, String, Vec)> = Vec::new(); + let mut residuals: Vec<(&str, Vec)> = Vec::new(); + for (idx, (lock_key, content, _)) in contents.iter().enumerate() { + // (byte range to replace, replacement text) per instance, plus + // one FileEdit per instance keyed by the canonical instance key — + // per-instance edits keep the revert ledger lossless when several + // instances of one dep live in the same lock. + let mut splices: Vec<(std::ops::Range, String)> = Vec::new(); + let mut instance_edits: Vec = Vec::new(); + for entry in pnpm::entries(content) { + let Some(suffix) = pnpm::suffix(entry.key, &fname, &dep.version) else { + continue; + }; + if !pnpm::supported_suffix(suffix) { + continue; + } + let Some(resolution) = pnpm::resolution(&entry) else { + continue; + }; + matched_any = true; + let original = &content[resolution.range.clone()]; + let rebuilt = resolution.rewrite(&sha512, &dep.artifact_url); + if rebuilt == original { + continue; + } + splices.push((resolution.range, rebuilt.clone())); + instance_edits.push(FileEdit { + path: (*lock_key).clone(), + kind: "redirect_pnpm_resolution".into(), + action: "rewritten".into(), + key: Some(format!("{fname}@{}{suffix}", dep.version)), + original: Some(Value::String(original.to_string())), + new: Some(Value::String(rebuilt)), + }); + } + // Splice by byte range (package blocks are disjoint and ordered) — a string replace could hit the wrong + // instance when two entries share identical surrounding bytes. + let candidate: Option = if splices.is_empty() { + None + } else { + let mut out = String::with_capacity(content.len()); + let mut cursor = 0usize; + for (range, replacement) in splices { + out.push_str(&content[cursor..range.start]); + out.push_str(&replacement); + cursor = range.end; + } + out.push_str(&content[cursor..]); + Some(out) + }; + // Residual gate, run over the POST-splice text: any instance of + // this exact name@version still resolving somewhere other than + // the hosted artifact — in a spelling the splice grammar cannot + // parse (e.g. an unbalanced peer suffix) — makes this a partial + // rewrite. Shipping it would confirm and VEX-attest the dep while + // dependents through the unmatched instance keep installing the + // unpatched upstream tarball, so the dep is refused instead. + let leftover = pnpm_unrewritten_instances( + candidate.as_deref().unwrap_or(content), + &fname, + &dep.version, + &dep.artifact_url, + ); + if !leftover.is_empty() { + residuals.push(((*lock_key).as_str(), leftover)); + continue; + } + if let Some(out) = candidate { + planned.push((idx, out, instance_edits)); + } + } + // ANY residual anywhere refuses the dep across the WHOLE lock set — + // nothing rewritten, nothing recorded, nothing confirmed (the same + // fail-closed contract the pre-splice v5/v6 refusal had): a rewrite + // committed in one lock while another still resolves the dep + // upstream would confirm the dep set-wide. + if !residuals.is_empty() { + result.refused_pnpm_uuids.insert(dep.patch_uuid.clone()); + for (lock_key, keys) in &residuals { + result.warnings.push(RewriteWarning { + code: "redirect_pnpm_unsupported_lock_key".into(), + detail: format!( + "{fname}@{} still resolves through pnpm lock key(s) whose \ + resolution the redirect grammar cannot repoint: {} in \ + {lock_key}; the dep is left unredirected in EVERY lock \ + (nothing rewritten, nothing confirmed) — regenerate the \ + lock with a current pnpm (lockfileVersion 9) and re-run", + dep.version, + keys.join(", ") + ), + }); + } + continue; + } + for (idx, out, mut instance_edits) in planned { + let (_, content, changed) = &mut contents[idx]; + *content = out; + *changed = true; + result.edits.append(&mut instance_edits); + } + // The entry-not-found warning fires only when the dep matched in NO + // pnpm lock across the whole set, not once per lock. A VENDORED dep + // is named as such: `socket-patch vendor` removes the registry + // resolution this grammar looks for (v9 respells the packages key + // `@file:.socket/vendor/…`; v5/v6 rekey it to a bare `file:` + // key but keep the `@: file:…` overrides line), so + // the generic not-locked wording would send users on a wild-goose + // `pnpm install` when the real path is a mode switch. Fail-closed + // either way: nothing is rewritten for the dep. + if !matched_any { + let v9_vendored_key = format!("{fname}@file:"); + let override_key = format!("{fname}@{}", dep.version); + let vendored = contents.iter().any(|(_, content, _)| { + content.lines().any(|line| { + let t = line.trim_start(); + let t = t.strip_prefix('\'').unwrap_or(t); + // v9 packages/snapshots key (leading `/` in v6 spelling). + // The vendor backend always writes the RELATIVE + // `file:.socket/vendor/…` spelling here, so anchoring on + // it keeps a user's own `file:` dep of the same name + // from being misreported as vendored. + let key = t.strip_prefix('/').unwrap_or(t); + if key + .strip_prefix(&v9_vendored_key) + .is_some_and(|rest| rest.starts_with(".socket/vendor/")) + { + return true; + } + // overrides / root-dep line: `@: file:…` + // (pnpm <=8 absolutizes the value, so only the + // `.socket/vendor/` tail is stable enough to match). + t.strip_prefix(&override_key) + .map(|rest| rest.strip_prefix('\'').unwrap_or(rest)) + .and_then(|rest| rest.strip_prefix(':')) + .is_some_and(|rest| { + rest.contains("file:") && rest.contains(".socket/vendor/") + }) + }) + }); + if vendored { + result.warnings.push(RewriteWarning { + code: "redirect_pnpm_entry_vendored".into(), + detail: format!( + "{fname}@{} has no registry resolution because it is \ + VENDORED (the lock resolves it to a \ + file:.socket/vendor/… tarball); the hosted redirect \ + does not apply — run `socket-patch vendor --revert` to \ + restore the registry resolution, then re-run `scan \ + --mode hosted`", + dep.version + ), + }); + } else { + result.warnings.push(RewriteWarning { + code: "redirect_pnpm_entry_not_found".into(), + detail: format!("no resolution for {fname}@{}", dep.version), + }); + } + } + } + for (key, content, changed) in contents { + if changed { + result.files.insert(key.clone(), content); + } + } +} From 1ce8ed4f7ad6888e02df46f64e2469da3435cbe0 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 12:54:30 -0400 Subject: [PATCH 019/237] perf(hosted): probe Python locks once and fetch wheel metadata concurrently Deciding which pypi deps need hosted wheel metadata ran a full `rewrite_python_lock` (parse, a second parse for the source-scope check on script locks, mutate, serialize) per dep per lock, only to test the result for `Some`. The rewrite's refusal and not-applicable checks now live in one planning step that `rewrite_python_lock` and a new `PythonLockProbe` share: the probe parses each lock once and answers exactly `matches!(rewrite_python_lock(..), Ok(Some(_)))` per dep, and the rewrite no longer re-parses the lock for the scope check. The qualifying wheels are then downloaded through an ordered `buffered(8)` stream and folded in dep order, so `python_metadata`, the withheld artifacts and the `python_metadata_unavailable` skips are unchanged. The stream is inlined here (futures-util added with the same workspace spec as the scan-concurrency branch); it moves onto the shared ordered-concurrency helper once that lands. New tests: a probe/rewrite equivalence sweep over every lock shape and outcome, and a hosted scan whose slow first failure must still be reported before a fast second one. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commands/scan/hosted.rs | 134 +++++---- .../tests/hosted_wheel_metadata_order.rs | 269 +++++++++++++++++ .../src/utils/python_lock.rs | 273 ++++++++++++++++-- 3 files changed, 597 insertions(+), 79 deletions(-) create mode 100644 crates/socket-patch-cli/tests/hosted_wheel_metadata_order.rs diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index def69ed8..3c72fea9 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -1779,66 +1779,90 @@ pub(crate) async fn run_redirect_selected( // it rides the same atomic-write / ledger-first machinery as the locks. let mut python_metadata = std::collections::BTreeMap::new(); let mut unavailable_python_artifacts = std::collections::BTreeSet::new(); - for dep in candidates - .iter() - .map(|c| &c.dep) - .filter(|dep| dep.ecosystem == "pypi") { - let Some(sha256) = dep.integrity.sha256.as_deref() else { - continue; - }; - if !dep - .artifact_url - .split(['?', '#']) - .next() - .is_some_and(|path| path.ends_with(".whl")) + use socket_patch_core::utils::python_lock::{ArtifactSource, PythonLockProbe}; + // Each native Python lock is parsed once, on the first dep that + // needs the probe, rather than rewritten per dep just to learn + // whether it would be. + let mut probes: Option> = None; + let mut wheel_deps: Vec<(&DepOverride, &str)> = Vec::new(); + for dep in candidates + .iter() + .map(|c| &c.dep) + .filter(|dep| dep.ecosystem == "pypi") { - continue; + let Some(sha256) = dep.integrity.sha256.as_deref() else { + continue; + }; + if !dep + .artifact_url + .split(['?', '#']) + .next() + .is_some_and(|path| path.ends_with(".whl")) + { + continue; + } + let native_target = probes + .get_or_insert_with(|| { + files + .iter() + .filter(|(path, _)| { + *path == "uv.lock" + || socket_patch_core::utils::python_lock::is_script_lock_name(path) + }) + .map(|(_, text)| PythonLockProbe::new(text)) + .collect() + }) + .iter() + .any(|probe| { + probe.rewrites( + &dep.name, + &dep.version, + ArtifactSource::Url(&dep.artifact_url), + ) + }); + if native_target { + wheel_deps.push((dep, sha256)); + } } - let native_target = files - .iter() - .filter(|(path, _)| { - *path == "uv.lock" - || socket_patch_core::utils::python_lock::is_script_lock_name(path) - }) - .any(|(_, text)| { - socket_patch_core::utils::python_lock::rewrite_python_lock( - text, - &dep.name, - &dep.version, - socket_patch_core::utils::python_lock::ArtifactSource::Url(&dep.artifact_url), + // The wheels are fetched concurrently but folded in dep order, so + // `python_metadata`, `unavailable_python_artifacts` and `skipped` + // come out exactly as the serial loop's did. + // TODO(perf): switch to `utils::concurrent::ordered_concurrent` once + // it lands (added in parallel on the scan-concurrency branch). + const WHEEL_METADATA_CONCURRENCY: usize = 8; + use futures_util::StreamExt as _; + let mut fetches = std::pin::pin!(futures_util::stream::iter(wheel_deps.iter()) + .map(|&(dep, sha256)| { + socket_patch_core::vendor::pypi::fetch_hosted_wheel_metadata( + api_client, + &dep.artifact_url, sha256, ) - .ok() - .flatten() - .is_some() - }); - if !native_target { - continue; - } - status.set(format!( - "Fetching hosted wheel metadata for {}...", - dep.name - )); - match socket_patch_core::vendor::pypi::fetch_hosted_wheel_metadata( - api_client, - &dep.artifact_url, - sha256, - ) - .await - { - Ok(Some(metadata)) => { - python_metadata.insert(dep.artifact_url.clone(), metadata); - } - Ok(None) => {} - Err(detail) => { - unavailable_python_artifacts.insert(dep.artifact_url.clone()); - skipped.push(serde_json::json!({ - "purl": format!("pkg:pypi/{}@{}", dep.name, dep.version), - "uuid": dep.patch_uuid, - "reason": "python_metadata_unavailable", - "detail": detail.replace(&dep.artifact_url, ""), - })); + }) + .buffered(WHEEL_METADATA_CONCURRENCY)); + for &(dep, _) in &wheel_deps { + status.set(format!( + "Fetching hosted wheel metadata for {}...", + dep.name + )); + let Some(fetched) = fetches.next().await else { + break; + }; + match fetched { + Ok(Some(metadata)) => { + python_metadata.insert(dep.artifact_url.clone(), metadata); + } + Ok(None) => {} + Err(detail) => { + unavailable_python_artifacts.insert(dep.artifact_url.clone()); + skipped.push(serde_json::json!({ + "purl": format!("pkg:pypi/{}@{}", dep.name, dep.version), + "uuid": dep.patch_uuid, + "reason": "python_metadata_unavailable", + "detail": detail.replace(&dep.artifact_url, ""), + })); + } } } } diff --git a/crates/socket-patch-cli/tests/hosted_wheel_metadata_order.rs b/crates/socket-patch-cli/tests/hosted_wheel_metadata_order.rs new file mode 100644 index 00000000..58ff1982 --- /dev/null +++ b/crates/socket-patch-cli/tests/hosted_wheel_metadata_order.rs @@ -0,0 +1,269 @@ +//! `scan --mode hosted` over a uv project with SEVERAL pypi wheel patches: +//! the hosted wheel metadata each native uv rewrite embeds is fetched +//! concurrently, but every outcome must fold in dep order — the +//! `python_metadata_unavailable` skips come out in the same order the old +//! one-at-a-time loop produced, whatever order the downloads finish in. +//! +//! The first failing wheel is served SLOWLY and the second fails at once, +//! so a fold in completion order would swap them. +//! +//! Runs the built binary as a subprocess (`common::run_with_env`) against a +//! wiremock patch API. Unix-only: the fabricated `.venv` uses the POSIX +//! site-packages layout. + +#![cfg(unix)] + +use std::path::Path; +use std::time::Duration; + +use serde_json::{json, Value}; +use wiremock::matchers::{method, path, path_regex}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +#[path = "common/mod.rs"] +mod common; + +const ORG: &str = "test-org"; + +/// (name, version, patch uuid) — purl order is name order. +const PKGS: [(&str, &str, &str); 4] = [ + ("aaa-pkg", "1.0.0", "11111111-1111-4111-8111-111111111111"), + ("bbb-pkg", "2.0.0", "22222222-2222-4222-8222-222222222222"), + ("ccc-pkg", "3.0.0", "33333333-3333-4333-8333-333333333333"), + ("ddd-pkg", "4.0.0", "44444444-4444-4444-8444-444444444444"), +]; + +fn purl(name: &str, version: &str) -> String { + format!("pkg:pypi/{name}@{version}") +} + +fn wheel_file(name: &str, version: &str) -> String { + format!("{}-{version}-py3-none-any.whl", name.replace('-', "_")) +} + +/// A minimal but valid wheel: `-.dist-info/METADATA` with the +/// three required core-metadata headers. Returns the bytes and their sha256. +fn build_wheel(name: &str, version: &str) -> (Vec, String) { + use std::io::Write as _; + let mut buf = std::io::Cursor::new(Vec::new()); + { + let mut writer = zip::ZipWriter::new(&mut buf); + let opts = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Stored); + let dist = name.replace('-', "_"); + writer + .start_file(format!("{dist}-{version}.dist-info/METADATA"), opts) + .unwrap(); + writer + .write_all( + format!("Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n\n").as_bytes(), + ) + .unwrap(); + writer.finish().unwrap(); + } + let bytes = buf.into_inner(); + let sha = common::sha256_hex(&bytes); + (bytes, sha) +} + +fn write_uv_project(root: &Path) { + std::fs::create_dir_all(root).unwrap(); + let deps: Vec = PKGS + .iter() + .map(|(name, version, _)| format!("\"{name}=={version}\"")) + .collect(); + std::fs::write( + root.join("pyproject.toml"), + format!( + "[project]\nname = \"socket-uv-order-fixture\"\nversion = \"0.1.0\"\nrequires-python = \">=3.9\"\ndependencies = [{}]\n", + deps.join(", ") + ), + ) + .unwrap(); + let mut lock = String::from( + "version = 1\nrevision = 3\nrequires-python = \">=3.9\"\n\n[[package]]\nname = \"socket-uv-order-fixture\"\nversion = \"0.1.0\"\nsource = { virtual = \".\" }\ndependencies = [\n", + ); + for (name, _, _) in PKGS { + lock.push_str(&format!(" {{ name = \"{name}\" }},\n")); + } + lock.push_str("]\n\n[package.metadata]\nrequires-dist = ["); + lock.push_str( + &PKGS + .iter() + .map(|(name, version, _)| { + format!("{{ name = \"{name}\", specifier = \"=={version}\" }}") + }) + .collect::>() + .join(", "), + ); + lock.push_str("]\n"); + for (name, version, _) in PKGS { + let file = wheel_file(name, version); + lock.push_str(&format!( + "\n[[package]]\nname = \"{name}\"\nversion = \"{version}\"\nsource = {{ registry = \"https://pypi.org/simple\" }}\nwheels = [\n {{ url = \"https://files.pythonhosted.org/packages/xx/{file}\", hash = \"sha256:{}\" }},\n]\n", + "0".repeat(64) + )); + } + std::fs::write(root.join("uv.lock"), lock).unwrap(); + let site = root + .join(".venv") + .join("lib") + .join("python3.11") + .join("site-packages"); + for (name, version, _) in PKGS { + let dist = site.join(format!("{}-{version}.dist-info", name.replace('-', "_"))); + std::fs::create_dir_all(&dist).unwrap(); + std::fs::write( + dist.join("METADATA"), + format!("Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n"), + ) + .unwrap(); + } +} + +/// Discovery, per-package search and the reference grants for all of PKGS; +/// wheels: `aaa` and `ccc` serve valid bytes (reversed delays), `bbb` is a +/// SLOW 404 and `ddd` serves bytes that do not match the granted sha256. +async fn mock_api(server: &MockServer) { + let patch = |name: &str, version: &str, uuid: &str| { + json!({ + "uuid": uuid, "purl": purl(name, version), "tier": "free", + "cveIds": [], "ghsaIds": [], "severity": "high", + "title": format!("{name} fixture") + }) + }; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/batch"))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "packages": PKGS.iter().map(|(name, version, uuid)| json!({ + "purl": purl(name, version), + "patches": [patch(name, version, uuid)], + })).collect::>(), + "canAccessPaidPatches": false, + }))) + .mount(server) + .await; + for (name, version, uuid) in PKGS { + Mock::given(method("GET")) + .and(path_regex(format!( + "^/v0/orgs/{ORG}/patches/by-package/.*{name}.*$" + ))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "patches": [{ + "uuid": uuid, "purl": purl(name, version), + "publishedAt": "2024-01-01T00:00:00Z", + "description": "x", "license": "MIT", "tier": "free", + "vulnerabilities": {} + }], + "canAccessPaidPatches": false, + }))) + .mount(server) + .await; + } + let mut results = serde_json::Map::new(); + for (i, (name, version, uuid)) in PKGS.into_iter().enumerate() { + let file = wheel_file(name, version); + let url = format!("{}/wheels/{file}", server.uri()); + let (bytes, sha256) = build_wheel(name, version); + let delay = Duration::from_millis([600, 900, 0, 0][i]); + let response = match name { + "bbb-pkg" => ResponseTemplate::new(404), + "ddd-pkg" => { + ResponseTemplate::new(200).set_body_bytes(b"not the granted wheel".to_vec()) + } + _ => ResponseTemplate::new(200).set_body_bytes(bytes), + }; + Mock::given(method("GET")) + .and(path(format!("/wheels/{file}"))) + .respond_with(response.set_delay(delay)) + .mount(server) + .await; + results.insert( + uuid.to_string(), + json!({ + "status": "granted", + "url": url, + "purl": purl(name, version), + "artifacts": [{ "kind": "tarball", "url": url, "integrity": { "sha256": sha256 } }], + "registryOverride": null + }), + ); + } + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/package"))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "results": results }))) + .mount(server) + .await; +} + +#[tokio::test] +async fn wheel_metadata_failures_fold_in_dep_order() { + let server = MockServer::start().await; + mock_api(&server).await; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + write_uv_project(&root); + let lock_before = std::fs::read(root.join("uv.lock")).unwrap(); + + let cwd = root.to_str().unwrap().to_string(); + let api = server.uri(); + let (code, stdout, stderr) = common::run_with_env( + &root, + &[ + "scan", + "--mode", + "hosted", + "--dry-run", + "--json", + "--cwd", + &cwd, + "--api-url", + &api, + "--org", + ORG, + "--api-token", + "fake", + ], + &[], + ); + let doc: Value = serde_json::from_str(&stdout) + .unwrap_or_else(|e| panic!("JSON envelope ({e}):\n{stdout}\n{stderr}")); + assert_eq!(code, 0, "{doc:#}\n{stderr}"); + + let skipped: Vec<(String, String)> = doc["redirect"]["skipped"] + .as_array() + .unwrap_or_else(|| panic!("redirect.skipped: {doc:#}")) + .iter() + .filter(|s| s["reason"] == "python_metadata_unavailable") + .map(|s| { + ( + s["purl"].as_str().unwrap().to_string(), + s["uuid"].as_str().unwrap().to_string(), + ) + }) + .collect(); + assert_eq!( + skipped, + vec![ + (purl("bbb-pkg", "2.0.0"), PKGS[1].2.to_string()), + (purl("ddd-pkg", "4.0.0"), PKGS[3].2.to_string()), + ], + "metadata failures must be reported in dep order: {doc:#}" + ); + for s in doc["redirect"]["skipped"].as_array().unwrap() { + if s["reason"] == "python_metadata_unavailable" { + let detail = s["detail"].as_str().unwrap(); + assert!( + !detail.contains(&server.uri()), + "the hosted URL is redacted from the detail: {detail}" + ); + } + } + // The two good wheels still redirect; the refused two stay upstream. + assert_eq!(doc["redirect"]["redirected"], 2, "{doc:#}"); + assert_eq!( + std::fs::read(root.join("uv.lock")).unwrap(), + lock_before, + "--dry-run writes nothing" + ); +} diff --git a/crates/socket-patch-core/src/utils/python_lock.rs b/crates/socket-patch-core/src/utils/python_lock.rs index 04887529..bf69d235 100644 --- a/crates/socket-patch-core/src/utils/python_lock.rs +++ b/crates/socket-patch-core/src/utils/python_lock.rs @@ -451,6 +451,10 @@ pub fn check_python_lock_source_scope(text: &str, name: &str, version: &str) -> let document: DocumentMut = text .parse() .map_err(|error| format!("invalid Python lock: {error}"))?; + source_scope(&document, name, version) +} + +fn source_scope(document: &DocumentMut, name: &str, version: &str) -> Result<(), String> { let name = canonicalize_pypi_name(name); for collection in ["package", "distribution"] { if let Some(packages) = document.get(collection).and_then(Item::as_array_of_tables) { @@ -622,22 +626,33 @@ pub fn complete_python_lock_metadata( Ok(preserve_line_endings(text, document.to_string())) } -pub fn rewrite_python_lock( +/// Everything [`rewrite_python_lock`] decides before it mutates the +/// document: every refusal (`Err`) and every not-applicable (`None`) is +/// settled here, so a plan that exists always rewrites. +struct PythonLockPlan { + pep751: bool, + legacy: bool, + collection: &'static str, + index: usize, + name: String, + original_source: Option, + legacy_strings: bool, + legacy_artifact_tables: bool, +} + +fn plan_python_lock_rewrite( + document: &DocumentMut, text: &str, name: &str, version: &str, artifact: ArtifactSource<'_>, - sha256: &str, -) -> Result, String> { - let mut document: DocumentMut = text - .parse() - .map_err(|error| format!("invalid Python lock: {error}"))?; +) -> Result, String> { if document .get("manifest") .and_then(Item::as_table_like) .is_some_and(|manifest| manifest.contains_key("requirements")) { - check_python_lock_source_scope(text, name, version)?; + source_scope(document, name, version)?; } let pep751 = document.get("lock-version").is_some(); let legacy = document.get("distribution").is_some(); @@ -656,10 +671,7 @@ pub fn rewrite_python_lock( "package" }; let name = canonicalize_pypi_name(name); - let Some(packages) = document - .get_mut(collection) - .and_then(Item::as_array_of_tables_mut) - else { + let Some(packages) = document.get(collection).and_then(Item::as_array_of_tables) else { return Ok(None); }; let matches: Vec = packages @@ -672,12 +684,10 @@ pub fn rewrite_python_lock( "multiple lock entries for {name}@{version}; source selection is ambiguous" )); } - let Some(index) = matches.first() else { + let Some(&index) = matches.first() else { return Ok(None); }; - let package = packages - .get_mut(*index) - .expect("matching package index exists"); + let package = packages.get(index).expect("matching package index exists"); let original_source = package.get("source").cloned(); // uv 0.2.18 through 0.2.34 kept the `[[distribution]]` table name but had // already moved to inline-table sources (`source = { registry = … }`, @@ -731,8 +741,7 @@ pub fn rewrite_python_lock( .rsplit('/') .next() .unwrap_or(&location); - let wheel = filename.ends_with(".whl"); - if !wheel + if !filename.ends_with(".whl") && !filename.ends_with(".tar.gz") && !filename.ends_with(".zip") && !filename.ends_with(".tar.bz2") @@ -740,6 +749,91 @@ pub fn rewrite_python_lock( { return Err("patch artifact is not a Python distribution archive".to_string()); } + if !pep751 && legacy && matches!(artifact, ArtifactSource::Path(_)) { + // Both `[[distribution]]` shapes record ABSOLUTE paths/file URLs + // for local artifacts (uv 0.2.34 writes `source = { path = "/abs/…" }` + // and `wheels = [{ url = "file:///abs/…" }]`), so a committed + // relative wheel cannot be expressed portably before 0.2.35. + return Err("uv `[[distribution]]` lockfiles (uv < 0.2.35, experimental `uv lock`) cannot carry a portable local wheel: `--locked` rejects relative paths and `uv lock`/`uv sync` rewrite them to absolute ones; upgrade to uv >=0.2.35 for native vendoring, or use a requirements.txt installation".to_string()); + } + Ok(Some(PythonLockPlan { + pep751, + legacy, + collection, + index, + name, + original_source, + legacy_strings, + legacy_artifact_tables, + })) +} + +/// A Python lock parsed ONCE for repeated "would [`rewrite_python_lock`] +/// rewrite this dep?" checks — the hosted wheel-metadata gate asks it for +/// every pypi dep without re-parsing and re-serializing the lock each time. +pub struct PythonLockProbe<'t> { + text: &'t str, + /// `None` when the lock does not parse (every dep then probes false, + /// as `rewrite_python_lock`'s `Err` would). + document: Option, +} + +impl<'t> PythonLockProbe<'t> { + pub fn new(text: &'t str) -> Self { + Self { + text, + document: text.parse().ok(), + } + } + + /// Exactly `matches!(rewrite_python_lock(text, …), Ok(Some(_)))`. + pub fn rewrites(&self, name: &str, version: &str, artifact: ArtifactSource<'_>) -> bool { + self.document.as_ref().is_some_and(|document| { + matches!( + plan_python_lock_rewrite(document, self.text, name, version, artifact), + Ok(Some(_)) + ) + }) + } +} + +pub fn rewrite_python_lock( + text: &str, + name: &str, + version: &str, + artifact: ArtifactSource<'_>, + sha256: &str, +) -> Result, String> { + let mut document: DocumentMut = text + .parse() + .map_err(|error| format!("invalid Python lock: {error}"))?; + let Some(PythonLockPlan { + pep751, + legacy, + collection, + index, + name, + original_source, + legacy_strings, + legacy_artifact_tables, + }) = plan_python_lock_rewrite(&document, text, name, version, artifact)? + else { + return Ok(None); + }; + let package = document + .get_mut(collection) + .and_then(Item::as_array_of_tables_mut) + .and_then(|packages| packages.get_mut(index)) + .expect("planned package index exists"); + let location = artifact.location(); + let filename = location + .split(['?', '#']) + .next() + .unwrap_or(&location) + .rsplit('/') + .next() + .unwrap_or(&location); + let wheel = filename.ends_with(".whl"); for key in ["sdist", "wheel", "wheels", "archive"] { package.remove(key); } @@ -754,13 +848,7 @@ pub fn rewrite_python_lock( ])), ); } else { - let source = if legacy && matches!(artifact, ArtifactSource::Path(_)) { - // Both `[[distribution]]` shapes record ABSOLUTE paths/file URLs - // for local artifacts (uv 0.2.34 writes `source = { path = "/abs/…" }` - // and `wheels = [{ url = "file:///abs/…" }]`), so a committed - // relative wheel cannot be expressed portably before 0.2.35. - return Err("uv `[[distribution]]` lockfiles (uv < 0.2.35, experimental `uv lock`) cannot carry a portable local wheel: `--locked` rejects relative paths and `uv lock`/`uv sync` rewrite them to absolute ones; upgrade to uv >=0.2.35 for native vendoring, or use a requirements.txt installation".to_string()); - } else if legacy_strings { + let source = if legacy_strings { Item::Value(Value::from(format!("direct+{location}"))) } else { Item::Value(inline(&[(artifact.key(), Value::from(location.clone()))])) @@ -1611,3 +1699,140 @@ wheels = [ ); } } + +/// `PythonLockProbe::rewrites` must agree with `rewrite_python_lock` +/// returning `Ok(Some(_))` for every lock shape and every refusal / +/// not-applicable path — the hosted wheel-metadata gate relies on it. +#[cfg(test)] +mod probe_equivalence_tests { + use super::{rewrite_python_lock, ArtifactSource, PythonLockProbe}; + + const SHA256: &str = "ccc9a9e0b18a5efc7038c504cfc580e47d2e02e5390f2e29cad833cbccb956b6"; + const NATIVE: &str = r#"version = 1 +revision = 3 + +[[package]] +name = "project" +version = "1" +source = { virtual = "." } +dependencies = [ + { name = "urllib3", version = "1.26.18", source = { registry = "https://pypi.org/simple" } }, +] + +[[package]] +name = "urllib3" +version = "1.26.18" +source = { registry = "https://pypi.org/simple" } +wheels = [{ url = "https://pypi.org/urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:old" }] + +[[package]] +name = "Zope.Interface" +version = "6.0" +source = { git = "https://example.test/zope" } + +[[package]] +name = "six" +version = "1.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://pypi.org/six-1.16.0.tar.gz", hash = "sha256:old" } +"#; + const LEGACY_STRINGS: &str = r#"version = 1 + +[[distribution]] +name = "urllib3" +version = "1.26.18" +source = "registry+https://pypi.org/simple" + +[[distribution.wheel]] +url = "https://pypi.org/urllib3-1.26.18-py2.py3-none-any.whl" +hash = "sha256:old" + +[[distribution]] +name = "six" +version = "1.16.0" +source = "git+https://example.test/six" +"#; + const PEP751: &str = r#"lock-version = "1.0" + +[[packages]] +name = "urllib3" +version = "1.26.18" +index = "https://pypi.org/simple" +wheels = [{ url = "https://pypi.org/urllib3-1.26.18-py2.py3-none-any.whl", hashes = { sha256 = "old" } }] + +[[packages]] +name = "six" +version = "1.16.0" +vcs = { type = "git", url = "https://example.test/six", commit-id = "abc" } +"#; + + #[test] + fn probe_matches_rewrite_outcome_for_every_shape() { + let script = NATIVE.replacen( + "[[package]]", + "[manifest]\nrequirements = [{ name = \"urllib3\", specifier = \"==1.26.18\" }]\n\n[[package]]", + 1, + ); + let multi_version = format!( + "{script}\n[[package]]\nname = \"urllib3\"\nversion = \"2.0.0\"\nsource = {{ registry = \"https://pypi.org/simple\" }}\n" + ); + let duplicate = NATIVE.replace( + "name = \"six\"\nversion = \"1.16.0\"", + "name = \"urllib3\"\nversion = \"1.26.18\"", + ); + let locks = [ + NATIVE.to_string(), + NATIVE.replace("\n", "\r\n"), + script, + multi_version, + duplicate, + LEGACY_STRINGS.to_string(), + PEP751.to_string(), + "version = 2\n".to_string(), + "lock-version = '2.0'\n".to_string(), + "version = 1\n".to_string(), + "this is [not toml".to_string(), + ]; + let wheel = "https://patch.socket.dev/pkg/urllib3-1.26.18-py2.py3-none-any.whl"; + let sdist = "https://patch.socket.dev/pkg/six-1.16.0.tar.gz?token=x#frag"; + let artifacts = [ + ArtifactSource::Url(wheel), + ArtifactSource::Url(sdist), + ArtifactSource::Url("https://patch.socket.dev/pkg/urllib3.exe"), + ArtifactSource::Path(".socket/vendor/pypi/id/urllib3-1.26.18-py2.py3-none-any.whl"), + ]; + let targets = [ + ("urllib3", "1.26.18"), + ("URLLIB3", "1.26.18"), + ("urllib3", "2.0.0"), + ("urllib3", "9.9.9"), + ("six", "1.16.0"), + ("zope-interface", "6.0"), + ("absent", "1.0"), + ]; + let mut outcomes = std::collections::BTreeSet::new(); + for lock in &locks { + let probe = PythonLockProbe::new(lock); + for (name, version) in targets { + for artifact in artifacts { + let rewrite = rewrite_python_lock(lock, name, version, artifact, SHA256); + outcomes.insert(match &rewrite { + Ok(Some(_)) => "some", + Ok(None) => "none", + Err(_) => "err", + }); + assert_eq!( + probe.rewrites(name, version, artifact), + matches!(rewrite, Ok(Some(_))), + "{name}@{version} via {artifact:?} over:\n{lock}\nrewrite: {rewrite:?}" + ); + } + } + } + assert_eq!( + outcomes.len(), + 3, + "every outcome is exercised: {outcomes:?}" + ); + } +} From ba64f3e9efa8267082baa22bada38ae7222e4f67 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 13:10:47 -0400 Subject: [PATCH 020/237] perf(redirect): derive npm and yarn-classic entry identities once per lock The npm package-lock rewriter re-derived every `packages` entry's identity (the `node_modules/` key split plus the `name`/`version` lookups) for every dep, and the classic yarn.lock rewriter re-split every block's key patterns for every dep: O(deps x entries) work that dominated both rewriters' CPU. Each identity is now computed once per lock. npm entries keep theirs by map position (a rewrite only touches `resolved`/`integrity`, never a key, `name` or `version`); a yarn block's key and sole real package are recomputed whenever this run rewrites that block, so later deps still see its current text. Output bytes, FileEdits and warnings are unchanged: both previous implementations are kept as test oracles and compared on 400 randomized locks each (aliases, links, bundled copies, workspaces, v1/v2 dependency trees, alias-only and fork-substitution yarn keys, CRLF and mixed line endings, duplicate overrides). Rewrite-phase CPU on the lockfile-only benches: npm-socket 103 -> 61 ms, yarn-strapi 49 -> 32 ms (whole-process medians). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../redirect/lock_index_equivalence_tests.rs | 622 ++++++++++++++++++ .../src/patch/redirect/mod.rs | 105 ++- 2 files changed, 692 insertions(+), 35 deletions(-) create mode 100644 crates/socket-patch-core/src/patch/redirect/lock_index_equivalence_tests.rs diff --git a/crates/socket-patch-core/src/patch/redirect/lock_index_equivalence_tests.rs b/crates/socket-patch-core/src/patch/redirect/lock_index_equivalence_tests.rs new file mode 100644 index 00000000..d89fcac7 --- /dev/null +++ b/crates/socket-patch-core/src/patch/redirect/lock_index_equivalence_tests.rs @@ -0,0 +1,622 @@ +//! Equivalence oracles for the npm package-lock and classic yarn.lock hosted +//! rewriters, which now derive each entry's identity once per lock instead +//! of once per entry per dep. The previous implementations are kept here +//! verbatim and the production rewriters must produce the identical output +//! bytes, FileEdit list, warnings and refusals on randomized locks. + +use super::*; + +type Snapshot = ( + BTreeMap, + Vec, + Vec<(String, String)>, +); + +fn snapshot(r: &RewriteResult) -> Snapshot { + ( + r.files.clone(), + r.edits.clone(), + r.warnings + .iter() + .map(|w| (w.code.clone(), w.detail.clone())) + .collect(), + ) +} + +fn assert_same(want: &RewriteResult, got: &RewriteResult, what: &str) { + let (want, got) = (snapshot(want), snapshot(got)); + assert_eq!(got.0, want.0, "{what}: rewritten bytes"); + assert_eq!(got.1.len(), want.1.len(), "{what}: edit count"); + for (i, (g, w)) in got.1.iter().zip(&want.1).enumerate() { + assert_eq!(g, w, "{what}: edit #{i}"); + } + assert_eq!(got.2, want.2, "{what}: warnings (code, detail) in order"); +} + +/// Deterministic xorshift64* — no `rand` dev-dependency. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } + fn chance(&mut self, percent: u64) -> bool { + self.next() % 100 < percent + } +} + +fn name(i: usize) -> String { + match i % 4 { + 0 => format!("@scope/pkg-{i}"), + _ => format!("pkg-{i}"), + } +} + +fn version(rng: &mut Rng) -> String { + ["1.0.0", "1.2.3", "2.0.0", "0.1.0-beta.1"][rng.below(4)].to_string() +} + +fn dep(name: &str, version: &str, uuid: usize, rng: &mut Rng) -> DepOverride { + let (namespace, bare) = match name.split_once('/') { + Some((ns, bare)) if name.starts_with('@') => (Some(ns.to_string()), bare.to_string()), + _ => (None, name.to_string()), + }; + let tag = ["a", "b"][rng.below(2)]; + DepOverride { + ecosystem: "npm".into(), + name: bare, + namespace, + version: version.to_string(), + token: String::new(), + patch_uuid: format!("00000000-0000-4000-8000-{uuid:012}"), + artifact_url: format!("https://patch.socket.dev/{tag}/{name}-{version}.tgz"), + berry_zip_url: None, + registry_override: None, + integrity: Integrity { + sha512: (!rng.chance(5)).then(|| format!("sha512-P{}==", rng.below(3))), + sha1: rng.chance(30).then(|| "0123456789abcdef".to_string()), + ..Default::default() + }, + } +} + +/// Overrides drawn from `pool` (plus misses), with immediate duplicates of +/// one name@version so a later dep re-reads an already-rewritten entry. +fn overrides(pool: &[(String, String)], rng: &mut Rng) -> Vec { + let mut out = Vec::new(); + for i in 0..(1 + rng.below(12)) { + let (n, v) = if rng.chance(10) || pool.is_empty() { + (format!("absent-{i}"), "1.0.0".to_string()) + } else { + pool[rng.below(pool.len())].clone() + }; + out.push(dep(&n, &v, i, rng)); + if rng.chance(20) { + out.push(dep(&n, &v, i + 100, rng)); + } + } + out +} + +fn npm_lock(rng: &mut Rng, pool: &mut Vec<(String, String)>) -> String { + let lock_version = 1 + rng.below(3) as u64; + let mut packages = serde_json::Map::new(); + packages.insert("".into(), json!({ "name": "root", "version": "0.0.0" })); + packages.insert( + "packages/ws".into(), + json!({ "name": "pkg-1", "version": "1.0.0" }), + ); + for i in 0..(5 + rng.below(40)) { + let n = name(rng.below(30)); + let v = version(rng); + pool.push((n.clone(), v.clone())); + let key = match rng.below(4) { + 0 => format!("node_modules/{}/node_modules/{n}", name(rng.below(30))), + 1 => format!("packages/ws/node_modules/{n}"), + _ => format!("node_modules/{n}"), + }; + let mut entry = serde_json::Map::new(); + if rng.chance(10) { + // An alias install: keyed by the alias, `name` is the real one. + entry.insert("name".into(), json!(name(rng.below(30)))); + } + if !rng.chance(5) { + entry.insert("version".into(), json!(v)); + } + if rng.chance(80) { + entry.insert( + "resolved".into(), + json!(format!("https://registry.npmjs.org/{n}/-/x-{v}.tgz")), + ); + entry.insert("integrity".into(), json!(format!("sha512-UP{i}=="))); + } + if rng.chance(5) { + entry.insert("link".into(), json!(true)); + } + if rng.chance(5) { + entry.insert("inBundle".into(), json!(true)); + } + let value = if rng.chance(3) { + json!("not an object") + } else { + Value::Object(entry) + }; + packages.insert(key, value); + } + let mut lock = json!({ + "name": "root", + "version": "0.0.0", + "lockfileVersion": lock_version, + "requires": true, + }); + if lock_version >= 2 { + lock["packages"] = Value::Object(packages); + } + if lock_version <= 2 { + let mut deps = serde_json::Map::new(); + for _ in 0..(2 + rng.below(10)) { + let n = name(rng.below(30)); + let v = version(rng); + pool.push((n.clone(), v.clone())); + let mut entry = json!({ + "version": v, + "resolved": format!("https://registry.npmjs.org/{n}/-/x-{v}.tgz"), + "integrity": "sha512-UPV2==", + }); + if rng.chance(10) { + entry["bundled"] = json!(true); + } + if rng.chance(20) { + entry["dependencies"] = json!({ + n.clone(): { "version": v, "resolved": "https://registry.npmjs.org/x.tgz" } + }); + } + deps.insert(n, entry); + } + lock["dependencies"] = Value::Object(deps); + } + let mut text = serialize_json(&lock); + if rng.chance(3) { + text.truncate(text.len() / 2); + } + text +} + +#[test] +fn indexed_npm_lock_rewrite_matches_oracle() { + let mut edits = 0; + let mut codes = std::collections::BTreeSet::new(); + for seed in 1..=400u64 { + let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); + let mut pool = Vec::new(); + let text = npm_lock(&mut rng, &mut pool); + let deps = overrides(&pool, &mut rng); + let refs: Vec<&DepOverride> = deps.iter().collect(); + for lockfile in ["package-lock.json", "npm-shrinkwrap.json"] { + let mut want = RewriteResult::default(); + rewrite_one_npm_lock_oracle(&text, lockfile, &refs, &mut want); + let mut got = RewriteResult::default(); + rewrite_one_npm_lock(&text, lockfile, &refs, &mut got); + assert_same(&want, &got, &format!("seed {seed} {lockfile}")); + edits += got.edits.len(); + codes.extend(got.warnings.iter().map(|w| w.code.clone())); + } + } + // The generator must actually reach every path. + assert!(edits > 400, "edits: {edits}"); + for code in [ + "redirect_npm_missing_sha512", + "redirect_npm_link_entry_skipped", + "redirect_npm_bundled_instance_skipped", + "redirect_npm_entry_not_found", + "redirect_npm_legacy_client", + "redirect_npm_lock_unparseable", + ] { + assert!(codes.contains(code), "missing {code}: {codes:?}"); + } +} + +fn yarn_block(rng: &mut Rng, pool: &mut Vec<(String, String)>, i: usize) -> String { + let n = name(rng.below(20)); + let v = version(rng); + pool.push((n.clone(), v.clone())); + let q = |p: String| { + if p.starts_with('@') || p.contains(':') { + format!("\"{p}\"") + } else { + p + } + }; + let mut patterns = vec![q(format!("{n}@^{v}"))]; + match rng.below(6) { + 0 => patterns.push(q(format!("{n}@~{v}"))), + // An alias descriptor consuming the same package. + 1 => patterns.push(q(format!("alias-{i}@npm:{n}@^{v}"))), + // Only reachable through an alias. + 2 => patterns = vec![q(format!("alias-{i}@npm:{n}@^{v}"))], + // Fork substitution: the name is ours, the package is not. + 3 => patterns = vec![q(format!("{n}@npm:fork-{i}@^{v}"))], + // Mixed real packages: never ours. + 4 => patterns.push(q(format!("{}@^1.0.0", name(rng.below(20))))), + _ => {} + } + let mut block = String::new(); + if rng.chance(5) { + block.push_str("# a comment\n"); + } + block.push_str(&format!("{}:\n version \"{v}\"\n", patterns.join(", "))); + if rng.chance(90) { + block.push_str(&format!( + " resolved \"https://registry.yarnpkg.com/{n}/-/x-{v}.tgz#abc{i}\"\n" + )); + } + if rng.chance(70) { + block.push_str(&format!(" integrity sha512-UP{i}==\n")); + } + if rng.chance(30) { + block.push_str(" dependencies:\n dep-a \"^1.0.0\"\n"); + } + block +} + +#[test] +fn indexed_yarn_classic_rewrite_matches_oracle() { + let mut edits = 0; + let mut codes = std::collections::BTreeSet::new(); + for seed in 1..=400u64 { + let mut rng = Rng(seed.wrapping_mul(0xD1B5_4A32_D192_ED03) | 1); + let mut pool = Vec::new(); + let mut text = String::from( + "# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.\n# yarn lockfile v1\n\n", + ); + let count = 3 + rng.below(40); + for i in 0..count { + text.push('\n'); + text.push_str(&yarn_block(&mut rng, &mut pool, i)); + } + match rng.below(20) { + 0 => text = text.replace('\n', "\r\n"), + 1 => text = text.replacen('\n', "\r", 1), + _ => {} + } + let files = BTreeMap::from([("yarn.lock".to_string(), text)]); + let deps = overrides(&pool, &mut rng); + let mut want = RewriteResult::default(); + rewrite_yarn_classic_oracle(&files, &deps, &mut want); + let mut got = RewriteResult::default(); + rewrite_yarn_classic(&files, &deps, &mut got); + assert_same(&want, &got, &format!("seed {seed}")); + edits += got.edits.len(); + codes.extend(got.warnings.iter().map(|w| w.code.clone())); + } + assert!(edits > 400, "edits: {edits}"); + for code in [ + "redirect_yarn_classic_missing_sha512", + "redirect_yarn_classic_alias_skipped", + "redirect_yarn_classic_entry_not_found", + "redirect_yarn_classic_unsupported_line_endings", + ] { + assert!(codes.contains(code), "missing {code}: {codes:?}"); + } +} + +// ── oracles: the pre-index implementations, verbatim ──────────────────────── + +fn rewrite_one_npm_lock_oracle( + content: &str, + lockfile: &str, + npm: &[&DepOverride], + result: &mut RewriteResult, +) { + let Ok(mut lock) = serde_json::from_str::(content) else { + // A corrupt lockfile is strictly worse than a missing one (which + // warns in the caller) — never skip the whole npm redirect silently. + result.warnings.push(RewriteWarning { + code: "redirect_npm_lock_unparseable".into(), + detail: format!("{lockfile} is not valid JSON; npm redirect skipped"), + }); + return; + }; + let mut changed = false; + for dep in npm { + let fname = full_name(dep); + let Some(sha512) = dep.integrity.sha512.clone() else { + result.warnings.push(RewriteWarning { + code: "redirect_npm_missing_sha512".into(), + detail: format!("{fname}@{} has no sha512 integrity", dep.version), + }); + continue; + }; + let mut matched_any = false; + if let Some(packages) = lock.get_mut("packages").and_then(Value::as_object_mut) { + for (key, entry) in packages.iter_mut() { + // Only `node_modules/` keys are installable dependencies: + // "" is the project root and other bare keys are workspace + // members — SOURCE dirs a resolved/integrity insert would + // corrupt. + let Some((_, key_name)) = key.rsplit_once("node_modules/") else { + continue; + }; + // The package a lock entry stands for: the explicit `name` + // field when present (npm writes it for aliases — `npm i + // alias@npm:real` keys the entry by the ALIAS), else the + // key's trailing path. Mirrors `vendor::npm_lock`'s + // `entry_name`, so an alias install of the patched package + // redirects and an entry that merely SHARES the key name + // (`npm i @npm:other`) is never hijacked. + let entry_nm = entry + .get("name") + .and_then(Value::as_str) + .unwrap_or(key_name); + let matches_ver = + entry.get("version").and_then(Value::as_str) == Some(dep.version.as_str()); + if entry_nm != fname || !matches_ver { + continue; + } + if entry.get("link").and_then(Value::as_bool) == Some(true) { + matched_any = true; + result.warnings.push(RewriteWarning { + code: "redirect_npm_link_entry_skipped".into(), + detail: format!( + "lock entry `{key}` is a link (npm workspaces/file: dir); skipped" + ), + }); + continue; + } + // npm reify extracts a bundled copy from its PARENT's tarball + // and ignores the entry's resolved/integrity, so a rewrite + // here would put the hosted URL in the lockfile (confirming + // and VEX-attesting the patch) while the unpatched bundled + // bytes keep installing. Mirrors the vendored backend's + // `vendor_bundled_instance_skipped` refusal. + if entry.get("inBundle").and_then(Value::as_bool) == Some(true) { + matched_any = true; + result.warnings.push(RewriteWarning { + code: "redirect_npm_bundled_instance_skipped".into(), + detail: format!( + "lock entry `{key}` is bundled inside its parent's tarball and \ + CANNOT be redirected — that copy stays UNPATCHED; vendor or \ + update the bundling parent to cover it" + ), + }); + continue; + } + matched_any = true; + if let Some(edit) = rewrite_npm_entry( + entry, + dep, + &sha512, + lockfile, + "redirect_npm_lock_entry", + key, + ) { + result.edits.push(edit); + changed = true; + } + } + } + // v2 legacy `dependencies` tree (keyed by name), recursive. + if let Some(deps) = lock.get_mut("dependencies").and_then(Value::as_object_mut) { + changed = rewrite_npm_v2_deps( + deps, + &fname, + dep, + &sha512, + lockfile, + result, + &mut matched_any, + ) || changed; + } + // Parity with the pnpm/berry/uv rewriters: a granted dep the + // lockfile cannot pin must be SAID, not silently dropped from the + // redirected count. + if !matched_any { + result.warnings.push(RewriteWarning { + code: "redirect_npm_entry_not_found".into(), + detail: format!("no {lockfile} entry for {fname}@{}", dep.version), + }); + } + } + if changed { + // npm <= 6 (the only writer of lockfileVersion 1) installs a registry + // dependency from the CONFIGURED registry and ignores the entry's + // `resolved` — verified against real npm 6.14.18, while npm 7 / 11 + // fetch the rewritten url from the same v1 lock. Under npm 6 the + // redirected lock therefore fails EINTEGRITY against the patched + // sha512 pin (fail-closed: the unpatched bytes never install). Say + // so instead of letting an npm 6 CI discover it. + if lock.get("lockfileVersion").and_then(Value::as_u64) == Some(1) { + result.warnings.push(RewriteWarning { + code: "redirect_npm_legacy_client".into(), + detail: format!( + "{lockfile} is lockfileVersion 1 (written by npm <= 6). npm <= 6 installs \ + registry dependencies from the configured registry and ignores the \ + redirected `resolved` url, so its installs fail EINTEGRITY against the \ + patched sha512 pin (the unpatched bytes are never installed); install \ + with npm >= 7, which fetches the hosted patch (and upgrades the lock)" + ), + }); + } + result.files.insert(lockfile.into(), serialize_json(&lock)); + } +} + +fn rewrite_yarn_classic_oracle( + files: &BTreeMap, + overrides: &[DepOverride], + result: &mut RewriteResult, +) { + use crate::vendor::yarn_classic_lock::{pattern_real_name, split_key_patterns, split_pattern}; + + let npm: Vec<&DepOverride> = overrides.iter().filter(|o| o.ecosystem == "npm").collect(); + if npm.is_empty() || !files.contains_key("yarn.lock") { + return; + } + let raw = &files["yarn.lock"]; + if is_berry_lock(raw) { + return; // yarn-berry — not classic + } + // CRLF locks (core.autocrlf Windows checkouts — yarn v1 parses them fine) + // are processed LF-normalized and re-expanded on output, so untouched + // lines round-trip byte-identically. Without this, `split("\n\n")` never + // splits a CRLF file: the whole lock becomes ONE block and the + // leftmost-match replaces below would rewrite the FIRST entry in the + // file, not the target's. Bare `\r`s outside a CRLF pair make the + // round-trip lossy, so such a lock is refused untouched. + let crlf = raw.contains('\r'); + let normalized: String; + let content: &str = if crlf { + normalized = raw.replace("\r\n", "\n"); + if normalized.contains('\r') { + result.warnings.push(RewriteWarning { + code: "redirect_yarn_classic_unsupported_line_endings".into(), + detail: "yarn.lock contains bare carriage returns (mixed line endings); \ + leaving it untouched" + .into(), + }); + return; + } + &normalized + } else { + raw + }; + let mut blocks: Vec = content.split("\n\n").map(String::from).collect(); + let resolved_re = + Regex::new(r#"\n {2}resolved "[^"]*""#).expect("static resolved-line regex is valid"); + let integrity_re = + Regex::new(r"\n {2}integrity [^\n]*").expect("static integrity-line regex is valid"); + let mut changed = false; + for dep in &npm { + let fname = full_name(dep); + let Some(sha512) = dep.integrity.sha512.clone() else { + result.warnings.push(RewriteWarning { + code: "redirect_yarn_classic_missing_sha512".into(), + detail: format!("{fname}@{} has no sha512 integrity", dep.version), + }); + continue; + }; + let version_re = + Regex::new(&(String::from(r#"\n {2}version ""#) + ®ex::escape(&dep.version) + "\"")) + .expect("version regex from the escaped version is valid"); + let mut matched_any = false; + let mut alias_skipped = false; + for block in blocks.iter_mut() { + // The block's key line names its consumers; resolve every + // comma-joined pattern to the REAL package it stands for + // (`alias@npm:target@range` → target). A key like + // `@npm:@…` — yarn v1's fork-substitution + // idiom — resolves to , so it is NOT ours to touch: + // matching on the alias name alone would hijack the fork. + let Some(key_line) = block + .lines() + .find(|l| !l.is_empty() && !l.starts_with([' ', '\t', '#'])) + else { + continue; + }; + let Some(key) = key_line.strip_suffix(':') else { + continue; + }; + let patterns = split_key_patterns(key); + if patterns.is_empty() + || !patterns + .iter() + .all(|p| pattern_real_name(p) == Some(fname.as_str())) + { + continue; + } + if !version_re.is_match(block) { + continue; + } + // A block reached only through `alias@npm:@range` + // descriptors is left byte-identical (mirroring the berry + // rewriter), but never silently: that copy keeps installing the + // unpatched artifact. + if !patterns + .iter() + .any(|p| split_pattern(p).is_some_and(|(n, _)| n == fname)) + { + alias_skipped = true; + result.warnings.push(RewriteWarning { + code: "redirect_yarn_classic_alias_skipped".into(), + detail: format!( + "lock entry `{key}` consumes {fname}@{} only through npm: alias \ + descriptors; the hosted redirect does not rewrite alias entries, \ + so this copy stays unpatched", + dep.version + ), + }); + continue; + } + matched_any = true; + let frag = dep + .integrity + .sha1 + .as_ref() + .map(|s| format!("#{s}")) + .unwrap_or_default(); + let mut rewritten = resolved_re + .replace( + block, + format!("\n resolved \"{}{frag}\"", dep.artifact_url).as_str(), + ) + .to_string(); + if integrity_re.is_match(&rewritten) { + rewritten = integrity_re + .replace(&rewritten, format!("\n integrity {sha512}").as_str()) + .to_string(); + } else { + rewritten = resolved_re + .replace( + &rewritten, + // $0 re-inserts the matched resolved line, then add integrity. + format!( + "\n resolved \"{}{frag}\"\n integrity {sha512}", + dep.artifact_url + ) + .as_str(), + ) + .to_string(); + } + if rewritten != *block { + // Ledger originals record the on-disk byte form, so a future + // revert of a CRLF lock can match what the file really held. + let (edit_original, edit_new) = if crlf { + (block.replace('\n', "\r\n"), rewritten.replace('\n', "\r\n")) + } else { + (block.clone(), rewritten.clone()) + }; + result.edits.push(FileEdit { + path: "yarn.lock".into(), + kind: "redirect_yarn_classic_entry".into(), + action: "rewritten".into(), + key: Some(format!("{fname}@{}", dep.version)), + original: Some(Value::String(edit_original)), + new: Some(Value::String(edit_new)), + }); + *block = rewritten; + changed = true; + } + } + if !matched_any && !alias_skipped { + result.warnings.push(RewriteWarning { + code: "redirect_yarn_classic_entry_not_found".into(), + detail: format!("no yarn.lock entry resolving {fname}@{}", dep.version), + }); + } + } + if changed { + let mut out = blocks.join("\n\n"); + if crlf { + out = out.replace('\n', "\r\n"); + } + result.files.insert("yarn.lock".into(), out); + } +} diff --git a/crates/socket-patch-core/src/patch/redirect/mod.rs b/crates/socket-patch-core/src/patch/redirect/mod.rs index 1c97b612..6435e630 100644 --- a/crates/socket-patch-core/src/patch/redirect/mod.rs +++ b/crates/socket-patch-core/src/patch/redirect/mod.rs @@ -30,6 +30,8 @@ use crate::vendor::yarn_berry_lock::yarnrc_compression_level; mod bun_binary; pub use bun_binary::{preflight_bun_binary, rewrite_bun_binary}; pub mod golang_local; +#[cfg(test)] +mod lock_index_equivalence_tests; pub mod npmrc; mod pdm; mod pipenv; @@ -510,6 +512,40 @@ fn rewrite_one_npm_lock( }); return; }; + // The (package, version) each `packages` entry stands for, by map + // position, computed once: the per-dep scan below compares against it + // instead of re-deriving it for every entry for every dep. Sound + // because a rewrite only ever touches an entry's `resolved`/`integrity` + // (never a key, `name` or `version`), so positions and identities hold. + let package_ids: Vec)>> = lock + .get("packages") + .and_then(Value::as_object) + .map(|packages| { + packages + .iter() + .map(|(key, entry)| { + // Only `node_modules/` keys are installable dependencies: + // "" is the project root and other bare keys are workspace + // members — SOURCE dirs a resolved/integrity insert would + // corrupt. + let (_, key_name) = key.rsplit_once("node_modules/")?; + // The package a lock entry stands for: the explicit `name` + // field when present (npm writes it for aliases — `npm i + // alias@npm:real` keys the entry by the ALIAS), else the + // key's trailing path. Mirrors `vendor::npm_lock`'s + // `entry_name`, so an alias install of the patched package + // redirects and an entry that merely SHARES the key name + // (`npm i @npm:other`) is never hijacked. + let entry_nm = entry + .get("name") + .and_then(Value::as_str) + .unwrap_or(key_name); + let version = entry.get("version").and_then(Value::as_str); + Some((entry_nm.to_string(), version.map(str::to_string))) + }) + .collect() + }) + .unwrap_or_default(); let mut changed = false; for dep in npm { let fname = full_name(dep); @@ -522,28 +558,11 @@ fn rewrite_one_npm_lock( }; let mut matched_any = false; if let Some(packages) = lock.get_mut("packages").and_then(Value::as_object_mut) { - for (key, entry) in packages.iter_mut() { - // Only `node_modules/` keys are installable dependencies: - // "" is the project root and other bare keys are workspace - // members — SOURCE dirs a resolved/integrity insert would - // corrupt. - let Some((_, key_name)) = key.rsplit_once("node_modules/") else { + for ((key, entry), id) in packages.iter_mut().zip(&package_ids) { + let Some((entry_nm, version)) = id else { continue; }; - // The package a lock entry stands for: the explicit `name` - // field when present (npm writes it for aliases — `npm i - // alias@npm:real` keys the entry by the ALIAS), else the - // key's trailing path. Mirrors `vendor::npm_lock`'s - // `entry_name`, so an alias install of the patched package - // redirects and an entry that merely SHARES the key name - // (`npm i @npm:other`) is never hijacked. - let entry_nm = entry - .get("name") - .and_then(Value::as_str) - .unwrap_or(key_name); - let matches_ver = - entry.get("version").and_then(Value::as_str) == Some(dep.version.as_str()); - if entry_nm != fname || !matches_ver { + if *entry_nm != fname || version.as_deref() != Some(dep.version.as_str()) { continue; } if entry.get("link").and_then(Value::as_bool) == Some(true) { @@ -3048,7 +3067,7 @@ fn rewrite_yarn_classic( overrides: &[DepOverride], result: &mut RewriteResult, ) { - use crate::vendor::yarn_classic_lock::{pattern_real_name, split_key_patterns, split_pattern}; + use crate::vendor::yarn_classic_lock::{split_key_patterns, split_pattern}; let npm: Vec<&DepOverride> = overrides.iter().filter(|o| o.ecosystem == "npm").collect(); if npm.is_empty() || !files.contains_key("yarn.lock") { @@ -3087,6 +3106,11 @@ fn rewrite_yarn_classic( Regex::new(r#"\n {2}resolved "[^"]*""#).expect("static resolved-line regex is valid"); let integrity_re = Regex::new(r"\n {2}integrity [^\n]*").expect("static integrity-line regex is valid"); + // Each block's key and the one real package all its patterns stand for + // (see `yarn_classic_block_head`), computed once per block and redone + // only for a block this run rewrites — not re-split per block per dep. + let mut heads: Vec)>> = + blocks.iter().map(|b| yarn_classic_block_head(b)).collect(); let mut changed = false; for dep in &npm { let fname = full_name(dep); @@ -3102,33 +3126,23 @@ fn rewrite_yarn_classic( .expect("version regex from the escaped version is valid"); let mut matched_any = false; let mut alias_skipped = false; - for block in blocks.iter_mut() { + for (i, block) in blocks.iter_mut().enumerate() { // The block's key line names its consumers; resolve every // comma-joined pattern to the REAL package it stands for // (`alias@npm:target@range` → target). A key like // `@npm:@…` — yarn v1's fork-substitution // idiom — resolves to , so it is NOT ours to touch: // matching on the alias name alone would hijack the fork. - let Some(key_line) = block - .lines() - .find(|l| !l.is_empty() && !l.starts_with([' ', '\t', '#'])) - else { + let Some((key, real_name)) = &heads[i] else { continue; }; - let Some(key) = key_line.strip_suffix(':') else { - continue; - }; - let patterns = split_key_patterns(key); - if patterns.is_empty() - || !patterns - .iter() - .all(|p| pattern_real_name(p) == Some(fname.as_str())) - { + if real_name.as_deref() != Some(fname.as_str()) { continue; } if !version_re.is_match(block) { continue; } + let patterns = split_key_patterns(key); // A block reached only through `alias@npm:@range` // descriptors is left byte-identical (mirroring the berry // rewriter), but never silently: that copy keeps installing the @@ -3196,6 +3210,7 @@ fn rewrite_yarn_classic( new: Some(Value::String(edit_new)), }); *block = rewritten; + heads[i] = yarn_classic_block_head(block); changed = true; } } @@ -3215,6 +3230,26 @@ fn rewrite_yarn_classic( } } +/// A classic yarn.lock block's key (its first non-indented, non-comment +/// line, minus the trailing `:`) and the real package EVERY comma-joined +/// pattern of that key resolves to — `None` when the key has no pattern, +/// one does not parse, or they name different packages. `None` overall +/// when the block has no key line. +fn yarn_classic_block_head(block: &str) -> Option<(String, Option)> { + use crate::vendor::yarn_classic_lock::{pattern_real_name, split_key_patterns}; + let key_line = block + .lines() + .find(|l| !l.is_empty() && !l.starts_with([' ', '\t', '#']))?; + let key = key_line.strip_suffix(':')?; + let patterns = split_key_patterns(key); + let mut names = patterns.iter().map(|p| pattern_real_name(p)); + let real_name = match names.next() { + Some(Some(first)) => names.all(|n| n == Some(first)).then(|| first.to_string()), + _ => None, + }; + Some((key.to_string(), real_name)) +} + // ── yarn.lock (berry / v2+) ────────────────────────────────────────────────── // Berry derives its fetch URL from the descriptor's `npm:` resolution and // verifies the CONVERTED CACHE ZIP against the lock's `checksum:` (a From f1b0ca14ff89ff66911e1cf8ff86cf2b4d7729dd Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 14:10:05 -0400 Subject: [PATCH 021/237] test(redirect): pin the pnpm residual boundaries on the production gate The indexed rewriter judges residuals inline, so the boundary test over `pnpm_unrewritten_instances` now covers only the test-only reference. Feed the same boundary locks through `rewrite_registry_redirect`: hosted, longer-version, scoped and snapshot keys never count, v6 nested-paren and v5 `_` instances are repointed, and only the unparseable instance is named in the refusal. The helper's doc comment now says what it is. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/patch/redirect/mod.rs | 116 +++++++++++++++++- 1 file changed, 113 insertions(+), 3 deletions(-) diff --git a/crates/socket-patch-core/src/patch/redirect/mod.rs b/crates/socket-patch-core/src/patch/redirect/mod.rs index 6435e630..b836db3f 100644 --- a/crates/socket-patch-core/src/patch/redirect/mod.rs +++ b/crates/socket-patch-core/src/patch/redirect/mod.rs @@ -2651,9 +2651,11 @@ fn plan_cargo_config( // ── pnpm-lock.yaml ─────────────────────────────────────────────────────────── -/// Audit every matching package instance after planning edits. A malformed -/// resolution or unsupported suffix refuses this dependency across all locks; -/// snapshots and other versions do not participate in resolution. +/// Test-only reference for the residual gate: every instance of this exact +/// name@version in `content` that does not resolve to `artifact_url`. +/// Production judges the same predicate inline, per instance, on each +/// indexed hit's post-splice body in `rewrite_pnpm_lock`; snapshots and +/// other versions do not participate in resolution. #[cfg(test)] fn pnpm_unrewritten_instances( content: &str, @@ -12869,6 +12871,114 @@ packages: ); } + /// The same boundaries, judged by the PRODUCTION inline residual gate + /// (`rewrite_pnpm_lock` over indexed hits), not the reference probe: an + /// instance already on the hosted artifact, a longer version sharing + /// the prefix, a different quoted scoped package and resolution-less + /// `snapshots:` keys never count as residuals, v6 nested-paren and v5 + /// `_` instances are repointed rather than refused, and the one + /// instance whose suffix the grammar cannot parse is the only key the + /// refusal names. + #[test] + fn pnpm_residual_gate_respects_version_and_section_boundaries() { + let url = "http://patch.test/left-pad-1.3.0.tgz"; + let overrides = vec![npm_override("left-pad", "1.3.0", url, "sha512-PATCHED==")]; + let residual_warnings = |r: &RewriteResult| -> Vec { + r.warnings + .iter() + .filter(|w| w.code == "redirect_pnpm_unsupported_lock_key") + .map(|w| w.detail.clone()) + .collect() + }; + let boundaries = format!( + "lockfileVersion: '9.0' + +packages: + left-pad@1.3.0: + resolution: {{integrity: sha512-PATCHED==, tarball: {url}}} + left-pad@1.3.01: + resolution: {{integrity: sha512-OTHERVERSION==}} + '@scope/left-pad@1.3.0': + resolution: {{integrity: sha512-OTHERPACKAGE==}} + +snapshots: + left-pad@1.3.0(react@18.2.0): + dependencies: + react: 18.2.0 +" + ); + let files = BTreeMap::from([("pnpm-lock.yaml".to_string(), boundaries.clone())]); + let r = rewrite_registry_redirect(&files, &overrides); + assert!( + residual_warnings(&r).is_empty() && r.refused_pnpm_uuids.is_empty(), + "rewritten instances, other versions/packages, and resolution-less \ + snapshots keys must not count: {:?}", + r.warnings + ); + let out = r.files.get("pnpm-lock.yaml").unwrap_or(&boundaries); + assert!( + out.contains("sha512-OTHERVERSION==") && out.contains("sha512-OTHERPACKAGE=="), + "{out}" + ); + + for lock in [ + "lockfileVersion: '6.0' + +packages: + + /left-pad@1.3.0(react@18.2.0(scheduler@0.23.2)): + resolution: {integrity: sha512-UPSTREAM==} + dev: false +", + "lockfileVersion: 5.4 + +packages: + + /left-pad/1.3.0_react@18.2.0: + resolution: {integrity: sha512-UPSTREAM==} + dev: false +", + ] { + let files = BTreeMap::from([("pnpm-lock.yaml".to_string(), lock.to_string())]); + let r = rewrite_registry_redirect(&files, &overrides); + assert!( + residual_warnings(&r).is_empty() && r.refused_pnpm_uuids.is_empty(), + "a spliceable suffixed instance is repointed, not refused: {:?}", + r.warnings + ); + assert!( + r.files["pnpm-lock.yaml"].contains(url) && r.edits.len() == 1, + "{:?}", + r.edits + ); + } + + let with_unparseable = boundaries.replace( + "\nsnapshots:", + " left-pad@1.3.0(react@18.2.0: + resolution: {integrity: sha512-UPSTREAM==} + +snapshots:", + ); + assert_ne!(with_unparseable, boundaries); + let files = BTreeMap::from([("pnpm-lock.yaml".to_string(), with_unparseable)]); + let r = rewrite_registry_redirect(&files, &overrides); + assert!(r.files.is_empty() && r.edits.is_empty(), "{:?}", r.edits); + assert_eq!( + r.refused_pnpm_uuids.len(), + 1, + "the dep is refused: {:?}", + r.warnings + ); + let details = residual_warnings(&r); + assert_eq!(details.len(), 1, "{details:?}"); + assert!( + details[0].contains("cannot repoint: left-pad@1.3.0(react@18.2.0 in pnpm-lock.yaml;"), + "only the unparseable instance is named: {}", + details[0] + ); + } + /// A dist block with no `url` has nothing to redirect: pinning a shasum /// onto it would claim a redirect that cannot happen. #[test] From ed0c78515f9b806ad8feaf87a504df1bd8708791 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 14:10:05 -0400 Subject: [PATCH 022/237] perf(hosted): cap wheel-metadata fetches at 4 and test that they overlap Each in-flight wheel download buffers the whole wheel under its own body timeout and retry budget, so memory and link sharing scale with the limit; 4 keeps the overlapped round trips while halving that. The comment records what concurrency changes that output cannot see (status line names the awaited dep, debug lines interleave, Retry-After pauses one fetch). The order test now also records request arrivals and fails if `bbb` is not requested before `aaa`'s delayed response is due, so a regression to serial fetching is caught. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commands/scan/hosted.rs | 15 ++++- .../tests/hosted_wheel_metadata_order.rs | 61 +++++++++++++++++-- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 3c72fea9..1bd61f7e 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -1828,9 +1828,20 @@ pub(crate) async fn run_redirect_selected( // The wheels are fetched concurrently but folded in dep order, so // `python_metadata`, `unavailable_python_artifacts` and `skipped` // come out exactly as the serial loop's did. + // + // Kept small on purpose: the gain is overlapped round trips, while + // each in-flight download buffers a whole wheel (up to + // MAX_VENDOR_PACKAGE_BYTES) under its own body timeout and retry + // budget, so peak memory and link sharing scale with this limit and + // a 429 `Retry-After` pauses only the download that received it. The + // status line names the dep whose result is being awaited (the + // baseline's messages, in the baseline's order), and the client's + // opt-in debug lines (GET / attempt-failed) interleave across the + // in-flight downloads. // TODO(perf): switch to `utils::concurrent::ordered_concurrent` once - // it lands (added in parallel on the scan-concurrency branch). - const WHEEL_METADATA_CONCURRENCY: usize = 8; + // it lands (added in parallel on the scan-concurrency branch), and + // let a `Retry-After` pause the whole fan-out rather than one fetch. + const WHEEL_METADATA_CONCURRENCY: usize = 4; use futures_util::StreamExt as _; let mut fetches = std::pin::pin!(futures_util::stream::iter(wheel_deps.iter()) .map(|&(dep, sha256)| { diff --git a/crates/socket-patch-cli/tests/hosted_wheel_metadata_order.rs b/crates/socket-patch-cli/tests/hosted_wheel_metadata_order.rs index 58ff1982..f87b8da9 100644 --- a/crates/socket-patch-cli/tests/hosted_wheel_metadata_order.rs +++ b/crates/socket-patch-cli/tests/hosted_wheel_metadata_order.rs @@ -5,7 +5,9 @@ //! one-at-a-time loop produced, whatever order the downloads finish in. //! //! The first failing wheel is served SLOWLY and the second fails at once, -//! so a fold in completion order would swap them. +//! so a fold in completion order would swap them. Request arrivals are +//! recorded too, so a regression to one-at-a-time fetching (which keeps the +//! order but loses the overlap) fails as well. //! //! Runs the built binary as a subprocess (`common::run_with_env`) against a //! wiremock patch API. Unix-only: the fabricated `.venv` uses the POSIX @@ -14,11 +16,12 @@ #![cfg(unix)] use std::path::Path; -use std::time::Duration; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; use serde_json::{json, Value}; use wiremock::matchers::{method, path, path_regex}; -use wiremock::{Mock, MockServer, ResponseTemplate}; +use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate}; #[path = "common/mod.rs"] mod common; @@ -121,10 +124,32 @@ fn write_uv_project(root: &Path) { } } +/// When each wheel request ARRIVED at the mock (before its response delay). +type Arrivals = Arc>>; + +/// Serves `template` and records the request's arrival under `name`. +struct RecordArrival { + name: &'static str, + template: ResponseTemplate, + arrivals: Arrivals, +} + +impl Respond for RecordArrival { + fn respond(&self, _: &Request) -> ResponseTemplate { + self.arrivals + .lock() + .unwrap() + .push((self.name, Instant::now())); + self.template.clone() + } +} + /// Discovery, per-package search and the reference grants for all of PKGS; /// wheels: `aaa` and `ccc` serve valid bytes (reversed delays), `bbb` is a /// SLOW 404 and `ddd` serves bytes that do not match the granted sha256. -async fn mock_api(server: &MockServer) { +/// Returns the wheel-request arrival log. +async fn mock_api(server: &MockServer) -> Arrivals { + let arrivals: Arrivals = Arc::default(); let patch = |name: &str, version: &str, uuid: &str| { json!({ "uuid": uuid, "purl": purl(name, version), "tier": "free", @@ -175,7 +200,11 @@ async fn mock_api(server: &MockServer) { }; Mock::given(method("GET")) .and(path(format!("/wheels/{file}"))) - .respond_with(response.set_delay(delay)) + .respond_with(RecordArrival { + name, + template: response.set_delay(delay), + arrivals: arrivals.clone(), + }) .mount(server) .await; results.insert( @@ -194,12 +223,13 @@ async fn mock_api(server: &MockServer) { .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "results": results }))) .mount(server) .await; + arrivals } #[tokio::test] async fn wheel_metadata_failures_fold_in_dep_order() { let server = MockServer::start().await; - mock_api(&server).await; + let arrivals = mock_api(&server).await; let tmp = tempfile::tempdir().unwrap(); let root = tmp.path().join("proj"); write_uv_project(&root); @@ -266,4 +296,23 @@ async fn wheel_metadata_failures_fold_in_dep_order() { lock_before, "--dry-run writes nothing" ); + + // The fetches overlap: `bbb` is requested while `aaa`'s 600 ms response + // is still pending. A one-at-a-time loop cannot request `bbb` until + // `aaa` has been answered. (No wall-clock budget: arrival order only.) + let arrivals = arrivals.lock().unwrap().clone(); + let first = |name: &str| { + arrivals + .iter() + .find(|(n, _)| *n == name) + .map(|(_, at)| *at) + .unwrap_or_else(|| panic!("no request for {name}: {arrivals:?}")) + }; + let (aaa, bbb) = (first("aaa-pkg"), first("bbb-pkg")); + assert!( + bbb < aaa + Duration::from_millis(600), + "bbb must be requested before aaa's delayed response is due \ + (arrived {:?} after aaa): {arrivals:?}", + bbb.saturating_duration_since(aaa) + ); } From 029e179c2ad9b64b8d3ae748f968c7adbb983a39 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 12:36:49 -0400 Subject: [PATCH 023/237] perf(scripts): add a record/replay network benchmark harness scan time is dominated by API round trips, so live timings are noisy and unrepeatable. scripts/perf/replay.py stands in for api.socket.dev, patch.socket.dev and the public proxy: `record` forwards and stores every response, `replay` serves only from the store with a fixed or recorded per-request latency (plus optional per-connection latency) and reports request counts per endpoint, max in-flight, connections and network span. Batch POSTs replay per purl, so a build that changes chunking or order still gets identical answers. The listener skips HTTPServer's getfqdn(), which stalls ~35 s under the macOS sandbox. scripts/perf/bench.sh drives it: `record`, `replay`, and `ab`, which runs BASE and NEW interleaved against one store and fails unless every run's stdout sha256 and exit code match the first BASE run. Stores hold real API responses (possibly paid-patch data), so bench.sh refuses a store path inside the repository. Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/perf/README.md | 74 +++++++ scripts/perf/bench.sh | 153 ++++++++++++++ scripts/perf/replay.py | 448 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 675 insertions(+) create mode 100644 scripts/perf/README.md create mode 100755 scripts/perf/bench.sh create mode 100755 scripts/perf/replay.py diff --git a/scripts/perf/README.md b/scripts/perf/README.md new file mode 100644 index 00000000..91796365 --- /dev/null +++ b/scripts/perf/README.md @@ -0,0 +1,74 @@ +# Network benchmark harness + +Most `scan` time is spent waiting on API round trips. Live timings are noisy +and can't be repeated, so this harness records the API traffic of one real run +and then replays it locally. That makes timing deterministic and gives a +byte-level oracle for "same output" when comparing two builds. + +- `replay.py` is a plain-HTTP stand-in for `api.socket.dev`, + `patch.socket.dev` and the public proxy `patches-api.socket.dev`. It has + two modes: + - `record` forwards each request and stores the response. + - `replay` serves responses from the store only. It can add a fixed + per-request latency (`--latency-ms`) or replay the latency measured while + recording (`--latency recorded`). It can also add a per-connection + latency (`--conn-latency-ms`). + + Batch POSTs are replayed per purl, so a build that changes chunking, batch + size or purl order still gets the same answers. Each run's stats report + request counts per endpoint, `max_inflight`, connections, `network_span_s`, + average parallelism, `misses` and `batch_unknown_purls`. +- `bench.sh` starts `replay.py`, points the CLI at it through + `SOCKET_API_URL`, `SOCKET_PROXY_URL` and `SOCKET_PATCH_SERVER_URL`, and + prints one stats line per run. Run `bench.sh` with no arguments to see all + of its settings. + +A store holds real API responses, which can include paid-patch data. Keep +stores outside the repository: `bench.sh` refuses to use a store path inside +the repository. For the same reason, never commit recorded responses or +third-party lockfiles. + +## Usage + +```sh +S=/some/dir/outside/the/repo +P=~/Projects/some-project + +# 1. Record one real run. This uses your normal token/org config; replay must +# use the same auth state, because it decides which routes the CLI calls. +CWD=$P BIN=./target/release/socket-patch \ + scripts/perf/bench.sh record $S/store -- scan --json --mode hosted --dry-run --no-telemetry + +# 2. Replay three times with 100 ms of simulated latency per request (no network). +CWD=$P scripts/perf/bench.sh replay $S/store 100 3 -- scan --json --mode hosted --dry-run --no-telemetry + +# 3. A/B test: runs BASE, NEW, BASE, NEW, ... (3 pairs here) against the same store. +CWD=$P BASE=$S/socket-patch-base NEW=./target/release/socket-patch PORT=18200 \ + scripts/perf/bench.sh ab $S/store 100 3 -- scan --json --mode hosted --dry-run --no-telemetry +``` + +Use a different `PORT` for each concurrent bench (it takes `PORT` through +`PORT+2`). Latency `0` isolates local work such as the crawl and rewrites. +`recorded` gives realistic totals. `FILL=1` forwards and records requests the +store doesn't have yet, for when a change adds endpoints. + +## Reading the results + +`ab` passes only if every run's stdout sha256 and exit code match the first +BASE run. Otherwise it exits 1 and lists the runs that differ as +`vs_base=DIFFERS`. It also prints the median wall time for each binary and +whether stderr was byte-identical across runs. Per-run stdout, stderr and +stats JSON, plus `ab-summary.tsv`, go to `$OUT` (default: `STORE/runs`). +Check every run for all of the following: + +- `misses=0` and `unknown_purls=0`. Anything else means the run asked for + something that was never recorded. Record again, or use `FILL=1`. +- `by_kind` totals match between BASE and NEW, unless the change is meant to + alter request counts. +- A concurrency change shows up as a higher `max_inflight` and a lower + `net_span`. + +On a loaded machine, compare only interleaved runs, which is what `ab` does, +and look at medians. A wet (non-`--dry-run`) run changes the project, so set +`PRE_RUN` to a command that restores a scratch copy of the project before each +invocation. Never point a wet run at a checkout you care about. diff --git a/scripts/perf/bench.sh b/scripts/perf/bench.sh new file mode 100755 index 00000000..f2601631 --- /dev/null +++ b/scripts/perf/bench.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# Deterministic socket-patch network benchmark driver (see scripts/perf/README.md). +# +# bench.sh record STORE -- +# bench.sh replay STORE [LATENCY_MS|recorded] [RUNS] -- +# bench.sh ab STORE [LATENCY_MS|recorded] [PAIRS] -- +# +# record runs BIN once against the real services through replay.py, filling +# STORE with every response. +# replay runs BIN RUNS times against STORE only (no network). +# ab runs BASE and NEW interleaved (BASE, NEW, BASE, NEW, ...) PAIRS times +# against STORE and fails unless every run's exit code and stdout +# sha256 equal the first BASE run's. +# +# Env knobs: +# CWD=/path/to/project (required; passed as --cwd) +# BIN=/path/to/binary (record/replay; default: /target/release/socket-patch) +# BASE=... NEW=... (ab; the two binaries to compare) +# PORT=18080 (api.socket.dev stand-in; PORT+1 = patch.socket.dev, +# PORT+2 = patches-api.socket.dev public proxy) +# CONN_MS=0 (replay/ab: extra latency per new TCP connection) +# FILL=1 (replay/ab: forward + record misses instead of 599) +# PRE_RUN='cmd' (shell command run before every CLI invocation, +# e.g. restoring a wet-run copy of the project) +# OUT=dir (per-run stdout/stderr/stats; default STORE/runs) +# +# The CLI is pointed at the stand-ins with SOCKET_API_URL, SOCKET_PROXY_URL and +# SOCKET_PATCH_SERVER_URL; the API token / org come from the usual config, and +# must match between record and replay (they choose the routes the CLI takes). +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../.." && pwd)" + +usage() { awk 'NR > 1 && /^#/ { sub(/^# ?/, ""); print; next } NR > 1 { exit }' "$0" >&2; exit 2; } +[[ $# -ge 2 ]] || usage +MODE="$1"; STORE="$2"; shift 2 +case "$MODE" in record|replay|ab) ;; *) usage ;; esac +LAT=0; RUNS=1 +if [[ "$MODE" != record ]]; then + if [[ $# -gt 0 && "$1" != -- ]]; then LAT="$1"; shift; fi + if [[ $# -gt 0 && "$1" != -- ]]; then RUNS="$1"; shift; fi +fi +if [[ $# -gt 0 && "$1" == -- ]]; then shift; fi +[[ $# -gt 0 ]] || usage + +: "${CWD:?set CWD to the project directory to scan}" +if [[ "$MODE" == ab ]]; then + : "${BASE:?set BASE to the baseline binary}"; : "${NEW:?set NEW to the candidate binary}" + BINS=() + for _ in $(seq "$RUNS"); do BINS+=("$BASE" "$NEW"); done +else + BIN="${BIN:-$REPO/target/release/socket-patch}" + BINS=() + for _ in $(seq "$RUNS"); do BINS+=("$BIN"); done +fi +for b in "${BINS[@]}"; do [[ -x "$b" ]] || { echo "not an executable: $b" >&2; exit 2; }; done + +# Recorded responses may carry paid-patch data: never let a store land in git. +realpath_() { python3 -c 'import os, sys; print(os.path.realpath(sys.argv[1]))' "$1"; } +STORE="$(realpath_ "$STORE")" +case "$STORE/" in + "$(realpath_ "$REPO")/"*) echo "refusing a STORE inside the repository: $STORE" >&2; exit 2 ;; +esac +mkdir -p "$STORE" + +PORT="${PORT:-18080}"; PATCH_PORT="$((PORT + 1))"; PROXY_PORT="$((PORT + 2))" +OUT="${OUT:-$STORE/runs}"; mkdir -p "$OUT" + +lat_args=(--latency-ms 0); lat_tag="${LAT}ms" +if [[ "$MODE" != record ]]; then + if [[ "$LAT" == recorded ]]; then lat_args=(--latency recorded); lat_tag=recorded; else lat_args=(--latency-ms "$LAT"); fi +fi +proxy_mode=replay; [[ "$MODE" == record ]] && proxy_mode=record +fill_args=(); [[ -n "${FILL:-}" ]] && fill_args=(--fill) + +python3 "$HERE/replay.py" "$proxy_mode" --store "$STORE" "${lat_args[@]}" \ + --conn-latency-ms "${CONN_MS:-0}" \ + --route "$PORT=https://api.socket.dev" \ + --route "$PATCH_PORT=https://patch.socket.dev" \ + --route "$PROXY_PORT=https://patches-api.socket.dev" \ + ${fill_args[@]+"${fill_args[@]}"} --stats-file "$OUT/last-proxy-stats.json" 2>"$OUT/proxy.log" & +PROXY=$! +trap 'kill $PROXY 2>/dev/null; wait $PROXY 2>/dev/null || true' EXIT +up=0 +for _ in $(seq 200); do + kill -0 $PROXY 2>/dev/null || { echo "replay.py died:"; cat "$OUT/proxy.log"; exit 1; } >&2 + grep -q "127.0.0.1:$PROXY_PORT" "$OUT/proxy.log" 2>/dev/null && { up=1; break; } + sleep 0.05 +done +[[ $up == 1 ]] || { echo "replay.py did not start (see $OUT/proxy.log)" >&2; exit 1; } + +now() { perl -MTime::HiRes=time -e 'printf "%.3f", time'; } +sha() { shasum -a 256 "$1" | cut -c1-64; } + +ref_sha=; ref_rc=; mismatches=0; i=0 +summary="$OUT/$MODE-summary.tsv" +printf 'tag\tbin\trc\twall_s\tstdout_sha256\tstderr_sha256\n' >"$summary" +for b in "${BINS[@]}"; do + i=$((i + 1)) + label=run; [[ "$MODE" == ab ]] && { [[ $((i % 2)) == 1 ]] && label=base || label=new; } + tag="$MODE-$lat_tag-$i-$label" + [[ -n "${PRE_RUN:-}" ]] && bash -c "$PRE_RUN" + curl -sf -X POST "http://127.0.0.1:$PORT/__reset" >/dev/null + start=$(now) + set +e + SOCKET_API_URL="http://127.0.0.1:$PORT" \ + SOCKET_PROXY_URL="http://127.0.0.1:$PROXY_PORT" \ + SOCKET_PATCH_SERVER_URL="http://127.0.0.1:$PATCH_PORT" \ + SOCKET_NO_UPDATE_CHECK=1 SOCKET_TELEMETRY_DISABLED=1 \ + "$b" "$@" --cwd "$CWD" >"$OUT/$tag.stdout" 2>"$OUT/$tag.stderr" + rc=$? + set -e + end=$(now) + curl -sf "http://127.0.0.1:$PORT/__stats" >"$OUT/$tag.stats.json" + out_sha=$(sha "$OUT/$tag.stdout"); err_sha=$(sha "$OUT/$tag.stderr") + printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$tag" "$b" "$rc" \ + "$(python3 -c 'import sys; print("%.3f" % (float(sys.argv[2]) - float(sys.argv[1])))' "$start" "$end")" \ + "$out_sha" "$err_sha" >>"$summary" + same= + if [[ "$MODE" == ab ]]; then + if [[ -z "$ref_sha" ]]; then ref_sha=$out_sha; ref_rc=$rc; same=ref + elif [[ "$out_sha" == "$ref_sha" && "$rc" == "$ref_rc" ]]; then same=same + else same=DIFFERS; mismatches=$((mismatches + 1)); fi + fi + python3 - "$OUT/$tag.stats.json" "$start" "$end" "$rc" "$tag" "$out_sha" "$same" <<'EOF' +import json, sys +s = json.load(open(sys.argv[1])); wall = float(sys.argv[3]) - float(sys.argv[2]) +same = f" vs_base={sys.argv[7]}" if sys.argv[7] else "" +print(f"{sys.argv[5]:<24} rc={sys.argv[4]} wall={wall:6.2f}s stdout={sys.argv[6][:12]}{same} " + f"requests={s['requests']:4d} max_inflight={s['max_inflight']:2d} conns={s['connections']:3d} " + f"misses={s['misses']} unknown_purls={s['batch_unknown_purls']} net_span={s['network_span_s']}s " + f"first_req={s['first_request_s']}s par={s['avg_parallelism']} by_kind={s['by_kind']}") +EOF +done + +if [[ "$MODE" == ab ]]; then + python3 - "$summary" <<'EOF' +import csv, statistics, sys +rows = list(csv.DictReader(open(sys.argv[1]), delimiter="\t")) +for label in ("base", "new"): + walls = [float(r["wall_s"]) for r in rows if r["tag"].endswith("-" + label)] + print(f"{label:<4} median wall {statistics.median(walls):6.2f}s runs={len(walls)} " + f"min={min(walls):.2f}s max={max(walls):.2f}s") +errs = {r["stderr_sha256"] for r in rows} +print("stderr: identical across runs" if len(errs) == 1 else + f"stderr: {len(errs)} distinct outputs across runs (diff the .stderr files under the OUT dir)") +EOF + if [[ $mismatches -gt 0 ]]; then + echo "FAIL: $mismatches run(s) differ from the first BASE run (stdout sha256 or exit code)" >&2 + exit 1 + fi + echo "OK: every run's stdout sha256 and exit code match the first BASE run" +fi diff --git a/scripts/perf/replay.py b/scripts/perf/replay.py new file mode 100755 index 00000000..71ea5fce --- /dev/null +++ b/scripts/perf/replay.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python3 +"""Record/replay HTTP stand-in for deterministic socket-patch benchmarks. + +Each `--route PORT=UPSTREAM` opens a plain-HTTP listener on 127.0.0.1:PORT +that stands in for UPSTREAM (e.g. https://api.socket.dev). Point the CLI at +it with SOCKET_API_URL / SOCKET_PROXY_URL / SOCKET_PATCH_SERVER_URL (see +scripts/perf/bench.sh, which does this for you). + +Modes + record forward every request upstream, store the response, serve it. + replay serve from the store only; a miss is a 599 (or forwarded and + recorded when --fill is given). + +Batch endpoints (POST .../patches/batch, POST /patch/batch) are replayed +SEMANTICALLY: recorded responses are split into per-purl entries and any +requested purl set is re-assembled, so a CLI that changes batch size, +chunking or purl order still replays deterministically. Purls never seen +while recording are counted in stats (`batch_unknown_purls`) and answered +as "no patches". + +Latency: `--latency-ms N` sleeps N ms before each response (simulated RTT, +per request; concurrent requests sleep in parallel like a real network); +`--latency recorded` replays each response's measured upstream time. +`--conn-latency-ms N` adds N ms once per new TCP connection (TLS+TCP +handshake stand-in), exposing missing connection reuse. + +Stats (requests, max in-flight, connections, per-endpoint counts, bytes, +timeline) are served at GET /__stats, reset with POST /__reset, and written +to --stats-file on SIGINT/SIGTERM. + +The store holds real API responses (possibly paid-patch data): keep it +outside the repository. +""" +import argparse +import hashlib +import http.client +import http.server +import json +import os +import re +import signal +import socketserver +import sys +import threading +import time +import urllib.parse + +HOP = { + "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", + "te", "trailers", "transfer-encoding", "upgrade", "host", "content-length", + "accept-encoding", +} +BATCH_RE = re.compile(r"(/v0/orgs/[^/]+/patches/batch|/patch/batch)$") +# Response headers that vary per request and would only add noise to a store. +VOLATILE = {"set-cookie", "date", "cf-ray", "server"} + + +def canon_body(body: bytes) -> str: + """Order-insensitive key for a request body: JSON objects are key-sorted + and lists of strings are sorted, so reordered purls hit the same entry.""" + if not body: + return "" + try: + v = json.loads(body) + except Exception: + return hashlib.sha256(body).hexdigest() + + def norm(x): + if isinstance(x, dict): + return {k: norm(x[k]) for k in sorted(x)} + if isinstance(x, list): + xs = [norm(i) for i in x] + if all(isinstance(i, str) for i in xs): + return sorted(xs) + return xs + return x + + return hashlib.sha256(json.dumps(norm(v), sort_keys=True).encode()).hexdigest() + + +def endpoint_kind(method, path): + """Coarse endpoint label for the per-kind request counts.""" + p = urllib.parse.urlsplit(path).path + if BATCH_RE.search(p): + return f"{method} batch" + m = re.search(r"/patch(?:es)?/(by-package|by-cve|by-ghsa|view|diff|blob|package)/", p) + if m: + return f"{method} {m.group(1)}" + if p.endswith("/patches/package") or p.endswith("/patch/package"): + return f"{method} package-vendor" + if "/organizations" in p: + return f"{method} organizations" + return f"{method} {'/'.join(p.split('/')[:4])}" + + +class Store: + """entries.jsonl (one line per recorded exchange, last write wins), + bodies/ (content-addressed response bodies) and + batch_index.json (purl -> batch package entry, or null for "no patches").""" + + def __init__(self, root): + self.root = root + os.makedirs(os.path.join(root, "bodies"), exist_ok=True) + self.lock = threading.Lock() + self.entries = {} # key -> entry + self.batch = {} # purl -> package entry (with patches) or None + self.paid = False + self._load() + + def _load(self): + p = os.path.join(self.root, "entries.jsonl") + if os.path.exists(p): + with open(p) as f: + for line in f: + if line.strip(): + e = json.loads(line) + self.entries[e["key"]] = e + b = os.path.join(self.root, "batch_index.json") + if os.path.exists(b): + with open(b) as f: + d = json.load(f) + self.batch = d["purls"] + self.paid = d["canAccessPaidPatches"] + + def body(self, e): + with open(os.path.join(self.root, "bodies", e["body"]), "rb") as f: + return f.read() + + def put(self, key, method, path, status, headers, body, upstream_ms): + sha = hashlib.sha256(body).hexdigest() + bp = os.path.join(self.root, "bodies", sha) + if not os.path.exists(bp): + with open(bp, "wb") as f: + f.write(body) + e = {"key": key, "method": method, "path": path, "status": status, + "headers": headers, "body": sha, "upstream_ms": upstream_ms} + with self.lock: + self.entries[key] = e + with open(os.path.join(self.root, "entries.jsonl"), "a") as f: + f.write(json.dumps(e) + "\n") + return e + + def put_batch(self, req_body, resp_body): + try: + purls = [c["purl"] for c in json.loads(req_body)["components"]] + resp = json.loads(resp_body) + except Exception: + return + with self.lock: + for p in purls: + self.batch.setdefault(p, None) + for pkg in resp.get("packages", []): + self.batch[pkg["purl"]] = pkg + self.paid = self.paid or bool(resp.get("canAccessPaidPatches")) + with open(os.path.join(self.root, "batch_index.json"), "w") as f: + json.dump({"purls": self.batch, "canAccessPaidPatches": self.paid}, f) + + def synth_batch(self, req_body): + """Re-assemble a batch response for any purl set, in request order. + Returns (body, number of purls never seen while recording).""" + purls = [c["purl"] for c in json.loads(req_body)["components"]] + pkgs, unknown = [], 0 + for p in purls: + if p not in self.batch: + unknown += 1 + elif self.batch[p] is not None: + pkgs.append(self.batch[p]) + body = json.dumps({"packages": pkgs, "canAccessPaidPatches": self.paid}).encode() + return body, unknown + + +class Stats: + def __init__(self): + self.lock = threading.Lock() + self.reset() + + def reset(self): + with self.lock: + self.t0 = time.time() + self.requests = 0 + self.inflight = 0 + self.max_inflight = 0 + self.connections = 0 + self.max_open_connections = 0 + self.open_connections = 0 + self.by_kind = {} + self.by_status = {} + self.bytes_out = 0 + self.bytes_in = 0 + self.misses = 0 + self.batch_unknown_purls = 0 + self.timeline = [] # [start_s, end_s, kind, status] + + def snapshot(self): + with self.lock: + tl = self.timeline + span = (max(e[1] for e in tl) - min(e[0] for e in tl)) if tl else 0.0 + busy = sum(e[1] - e[0] for e in tl) + return { + "requests": self.requests, "max_inflight": self.max_inflight, + "connections": self.connections, + "max_open_connections": self.max_open_connections, + "by_kind": dict(self.by_kind), "by_status": dict(self.by_status), + "bytes_out": self.bytes_out, "bytes_in": self.bytes_in, + "misses": self.misses, + "batch_unknown_purls": self.batch_unknown_purls, + "first_request_s": round(min(e[0] for e in tl), 3) if tl else None, + "last_response_s": round(max(e[1] for e in tl), 3) if tl else None, + "network_span_s": round(span, 3), + "summed_request_s": round(busy, 3), + "avg_parallelism": round(busy / span, 2) if span else 0.0, + "timeline": [[round(a, 4), round(b, 4), k, s] for a, b, k, s in tl], + } + + +class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + server_version = "replay/1" + + def log_message(self, fmt, *args): + if self.server.cfg.verbose: + sys.stderr.write("[%d] %s\n" % (self.server.server_port, fmt % args)) + + def setup(self): + super().setup() + st = self.server.stats + with st.lock: + st.connections += 1 + st.open_connections += 1 + st.max_open_connections = max(st.max_open_connections, st.open_connections) + if self.server.cfg.conn_latency_ms: + time.sleep(self.server.cfg.conn_latency_ms / 1000.0) + + def finish(self): + try: + super().finish() + finally: + # One handler thread per client connection: its upstream + # keep-alive connections end with it. + for conn in self.server.tls.__dict__.pop("conns", {}).values(): + conn.close() + st = self.server.stats + with st.lock: + st.open_connections -= 1 + + def _read_body(self): + te = self.headers.get("Transfer-Encoding", "") + if "chunked" in te.lower(): + out = b"" + while True: + n = int(self.rfile.readline().strip().split(b";")[0], 16) + if n == 0: + self.rfile.readline() + return out + out += self.rfile.read(n) + self.rfile.readline() + n = int(self.headers.get("Content-Length") or 0) + return self.rfile.read(n) if n else b"" + + def _send(self, status, headers, body): + self.send_response(status) + for k, v in headers.items(): + if k.lower() not in HOP: + self.send_header(k, v) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + + def _control(self): + st = self.server.stats + if self.path.startswith("/__stats"): + body = json.dumps(st.snapshot(), indent=1).encode() + self._send(200, {"Content-Type": "application/json"}, body) + elif self.path.startswith("/__reset"): + st.reset() + self._send(200, {}, b"ok") + else: + self._send(404, {}, b"") + + def _upstream(self, method, body): + """Forward to the upstream over a per-thread keep-alive connection, + retrying once on a dropped connection.""" + cfg = self.server.cfg + up = urllib.parse.urlsplit(self.server.upstream) + conns = self.server.tls.__dict__.setdefault("conns", {}) + conn = conns.get(up.netloc) + for attempt in range(2): + if conn is None: + cls = http.client.HTTPSConnection if up.scheme == "https" else http.client.HTTPConnection + conn = cls(up.netloc, timeout=cfg.upstream_timeout) + conns[up.netloc] = conn + hdrs = {k: v for k, v in self.headers.items() if k.lower() not in HOP} + hdrs["Accept-Encoding"] = "identity" + try: + t = time.time() + conn.request(method, up.path.rstrip("/") + self.path, body=body or None, headers=hdrs) + r = conn.getresponse() + data = r.read() + ms = (time.time() - t) * 1000 + rh = {k: v for k, v in r.getheaders() + if k.lower() not in HOP and k.lower() not in VOLATILE} + return r.status, rh, data, ms + except (http.client.HTTPException, OSError): + conn.close() + conn = None + conns.pop(up.netloc, None) + if attempt: + raise + + def _forward_and_store(self, key, method, body): + """Record one exchange. An unreachable upstream is answered with a + 502 and NOT stored, so a flaky record run never poisons the store.""" + try: + status, headers, data, upstream_ms = self._upstream(method, body) + except (http.client.HTTPException, OSError) as e: + sys.stderr.write(f"UPSTREAM ERROR {method} {self.path}: {e}\n") + return 502, {"Content-Type": "text/plain"}, f"replay upstream error: {e}".encode(), 0.0, False + self.server.store.put(key, method, self.path, status, headers, data, upstream_ms) + return status, headers, data, upstream_ms, True + + def _handle(self): + if self.path.startswith("/__"): + return self._control() + cfg, st, store = self.server.cfg, self.server.stats, self.server.store + method = self.command + body = self._read_body() if method in ("POST", "PUT", "PATCH") else b"" + kind = endpoint_kind(method, self.path) + with st.lock: + st.requests += 1 + st.inflight += 1 + st.max_inflight = max(st.max_inflight, st.inflight) + st.bytes_in += len(body) + t0 = time.time() + status = 599 + try: + key = f"{self.server.upstream} {method} {self.path} {canon_body(body)}" + is_batch = method == "POST" and BATCH_RE.search(urllib.parse.urlsplit(self.path).path) + upstream_ms = 0.0 + if cfg.mode == "record": + status, headers, data, upstream_ms, stored = self._forward_and_store(key, method, body) + if stored and is_batch and status == 200: + store.put_batch(body, data) + elif is_batch and store.batch: + data, unknown = store.synth_batch(body) + status, headers = 200, {"Content-Type": "application/json"} + with st.lock: + st.batch_unknown_purls += unknown + e = store.entries.get(key) + upstream_ms = e["upstream_ms"] if e else cfg.default_recorded_ms + else: + e = store.entries.get(key) + if e is None: + with st.lock: + st.misses += 1 + if cfg.verbose: + sys.stderr.write(f"MISS {key}\n") + if e is None and cfg.fill: + status, headers, data, upstream_ms, _ = self._forward_and_store(key, method, body) + elif e is None: + status, headers, data = 599, {"Content-Type": "text/plain"}, b"replay miss" + else: + status, headers, data, upstream_ms = e["status"], e["headers"], store.body(e), e["upstream_ms"] + if cfg.mode == "replay": + delay = upstream_ms / 1000.0 if cfg.latency == "recorded" else cfg.latency_ms / 1000.0 + if delay > 0: + time.sleep(delay) + self._send(status, headers, data) + with st.lock: + st.bytes_out += len(data) + finally: + t1 = time.time() + with st.lock: + st.inflight -= 1 + st.by_kind[kind] = st.by_kind.get(kind, 0) + 1 + st.by_status[str(status)] = st.by_status.get(str(status), 0) + 1 + st.timeline.append([t0 - st.t0, t1 - st.t0, kind, status]) + + do_GET = do_POST = do_PUT = do_HEAD = do_DELETE = do_PATCH = _handle + + +class Server(socketserver.ThreadingMixIn, http.server.HTTPServer): + daemon_threads = True + allow_reuse_address = True + request_queue_size = 256 + + def server_bind(self): + # Skip HTTPServer.server_bind's socket.getfqdn(): under the macOS + # sandbox the reverse lookup stalls ~35 s before the port is usable. + socketserver.TCPServer.server_bind(self) + self.server_name = "127.0.0.1" + self.server_port = self.server_address[1] + + +def make_server(port, upstream, cfg, store, stats): + """Bind one route (port 0 picks a free port) and serve it on a daemon + thread. Every route shares one store and one stats object.""" + s = Server(("127.0.0.1", int(port)), Handler) + s.cfg, s.store, s.stats = cfg, store, stats + s.upstream, s.tls = upstream.rstrip("/"), threading.local() + threading.Thread(target=s.serve_forever, daemon=True).start() + return s + + +def parse_args(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("mode", choices=["record", "replay"]) + ap.add_argument("--store", required=True) + ap.add_argument("--route", action="append", required=True, help="PORT=UPSTREAM_URL") + ap.add_argument("--latency-ms", type=float, default=0.0) + ap.add_argument("--latency", choices=["fixed", "recorded"], default="fixed") + ap.add_argument("--default-recorded-ms", type=float, default=100.0, + help="recorded latency for a synthesized batch with no exact entry") + ap.add_argument("--conn-latency-ms", type=float, default=0.0) + ap.add_argument("--fill", action="store_true", help="replay: forward+record misses") + ap.add_argument("--stats-file") + ap.add_argument("--upstream-timeout", type=float, default=120.0) + ap.add_argument("--verbose", action="store_true") + return ap.parse_args(argv) + + +def main(): + cfg = parse_args() + store, stats = Store(cfg.store), Stats() + for r in cfg.route: + port, upstream = r.split("=", 1) + s = make_server(port, upstream, cfg, store, stats) + # bench.sh waits for this line before starting the CLI. + sys.stderr.write(f"replay.py {cfg.mode}: http://127.0.0.1:{s.server_port} -> {upstream}\n") + sys.stderr.flush() + + done = threading.Event() + + def stop(*_): + done.set() + + signal.signal(signal.SIGINT, stop) + signal.signal(signal.SIGTERM, stop) + done.wait() + if cfg.stats_file: + with open(cfg.stats_file, "w") as f: + json.dump(stats.snapshot(), f, indent=1) + sys.stderr.flush() + os._exit(0) + + +if __name__ == "__main__": + main() From a82079ac0d13aa52c6eedf6ba97184bef9c1489f Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 12:36:50 -0400 Subject: [PATCH 024/237] test(scripts): cover the perf record/replay harness offline Syntax-checks replay.py (py_compile) and bench.sh (bash -n), then drives the harness against a local upstream stub: record-then-replay with batch re-assembly across chunks and orders, miss/unknown-purl accounting, --fill, a 502 (never stored) for an unreachable upstream, per-request latency with max in-flight, the getfqdn-free bind, the in-repo store refusal, and an end-to-end `bench.sh ab` pass and sha-mismatch failure with fake CLI binaries. Picked up by the existing `unittest discover -s scripts/tests` CI step. Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/tests/test_perf_harness.py | 316 +++++++++++++++++++++++++++++ 1 file changed, 316 insertions(+) create mode 100644 scripts/tests/test_perf_harness.py diff --git a/scripts/tests/test_perf_harness.py b/scripts/tests/test_perf_harness.py new file mode 100644 index 00000000..23182b2f --- /dev/null +++ b/scripts/tests/test_perf_harness.py @@ -0,0 +1,316 @@ +"""Offline coverage for the scripts/perf record/replay benchmark harness.""" + +import concurrent.futures +import http.client +import http.server +import importlib.util +import io +import json +import os +from pathlib import Path +import py_compile +import shutil +import socket +import subprocess +import tempfile +import textwrap +import threading +import time +import unittest +from unittest.mock import patch + + +PERF = Path(__file__).resolve().parents[1] / 'perf' +spec = importlib.util.spec_from_file_location('perf_replay', PERF / 'replay.py') +replay = importlib.util.module_from_spec(spec) +spec.loader.exec_module(replay) + +BATCH = '/v0/orgs/acme/patches/batch' +KNOWN = { + 'pkg:npm/a@1.0.0': {'purl': 'pkg:npm/a@1.0.0', 'patches': [{'uuid': 'u-a'}]}, + 'pkg:npm/c@3.0.0': {'purl': 'pkg:npm/c@3.0.0', 'patches': [{'uuid': 'u-c'}]}, +} + + +class Upstream(http.server.BaseHTTPRequestHandler): + """Stand-in for api.socket.dev: a batch endpoint and one detail GET.""" + protocol_version = 'HTTP/1.1' + hits = [] + + def log_message(self, *args): + pass + + def _reply(self, status, obj): + body = json.dumps(obj).encode() + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_POST(self): + req = json.loads(self.rfile.read(int(self.headers['Content-Length']))) + Upstream.hits.append(self.path) + purls = [c['purl'] for c in req['components']] + self._reply(200, {'packages': [KNOWN[p] for p in purls if p in KNOWN], + 'canAccessPaidPatches': True}) + + def do_GET(self): + Upstream.hits.append(self.path) + self._reply(200, {'path': self.path}) + + +def cfg(mode, **kw): + argv = [mode, '--store', 'unused', '--route', '0=http://unused'] + c = replay.parse_args(argv) + for k, v in kw.items(): + setattr(c, k, v) + return c + + +def request(port, method, path, obj=None): + conn = http.client.HTTPConnection('127.0.0.1', port, timeout=30) + body = json.dumps(obj).encode() if obj is not None else None + conn.request(method, path, body=body, headers={'Content-Type': 'application/json'}) + r = conn.getresponse() + data = r.read() + conn.close() + return r.status, data + + +def batch_body(*purls): + return {'components': [{'purl': p} for p in purls]} + + +def free_port_run(n): + """First port of n consecutive free localhost ports.""" + for _ in range(50): + with socket.socket() as s: + s.bind(('127.0.0.1', 0)) + base = s.getsockname()[1] + if base + n > 65535: + continue + socks = [] + try: + for p in range(base, base + n): + s = socket.socket() + socks.append(s) + s.bind(('127.0.0.1', p)) + return base + except OSError: + continue + finally: + for s in socks: + s.close() + raise RuntimeError('no run of free ports') + + +class SyntaxTests(unittest.TestCase): + def test_replay_compiles(self): + with tempfile.TemporaryDirectory() as temp: + py_compile.compile(str(PERF / 'replay.py'), cfile=os.path.join(temp, 'r.pyc'), doraise=True) + + @unittest.skipUnless(shutil.which('bash'), 'bash not installed') + def test_bench_sh_parses(self): + subprocess.run(['bash', '-n', str(PERF / 'bench.sh')], check=True) + + +class KeyTests(unittest.TestCase): + def test_body_key_ignores_key_and_purl_order(self): + a = json.dumps({'components': ['pkg:npm/a@1', 'pkg:npm/b@2'], 'x': 1}).encode() + b = json.dumps({'x': 1, 'components': ['pkg:npm/b@2', 'pkg:npm/a@1']}).encode() + self.assertEqual(replay.canon_body(a), replay.canon_body(b)) + self.assertNotEqual(replay.canon_body(a), replay.canon_body(b'not json')) + self.assertEqual(replay.canon_body(b''), '') + + def test_endpoint_kinds(self): + self.assertEqual(replay.endpoint_kind('POST', BATCH), 'POST batch') + self.assertEqual(replay.endpoint_kind('POST', '/patch/batch'), 'POST batch') + self.assertEqual(replay.endpoint_kind('GET', '/v0/orgs/acme/patches/by-package/pkg%3Anpm%2Fa'), + 'GET by-package') + self.assertEqual(replay.endpoint_kind('GET', '/v0/orgs/acme/patches/view/u-a?x=1'), 'GET view') + self.assertEqual(replay.endpoint_kind('POST', '/v0/orgs/acme/patches/package'), 'POST package-vendor') + self.assertEqual(replay.endpoint_kind('GET', '/v0/organizations'), 'GET organizations') + + +class ServerTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.store_dir = os.path.join(self.temp.name, 'store') + self.servers = [] + Upstream.hits = [] + + def tearDown(self): + for s in self.servers: + s.shutdown() + s.server_close() + self.temp.cleanup() + + def serve(self, port, upstream, c, store, stats): + s = replay.make_server(port, upstream, c, store, stats) + self.servers.append(s) + return s + + def upstream(self): + s = replay.Server(('127.0.0.1', 0), Upstream) + threading.Thread(target=s.serve_forever, daemon=True).start() + self.servers.append(s) + return f'http://127.0.0.1:{s.server_port}' + + def test_bind_skips_reverse_lookup(self): + # HTTPServer.server_bind calls socket.getfqdn(), which stalls ~35 s + # under the macOS sandbox; the harness must never reach it. + with patch('socket.getfqdn', side_effect=AssertionError('getfqdn called')): + s = replay.Server(('127.0.0.1', 0), replay.Handler) + self.assertEqual(s.server_name, '127.0.0.1') + self.assertEqual(s.server_port, s.socket.getsockname()[1]) + s.server_close() + + def test_record_then_replay_reassembles_batches_and_counts_misses(self): + up = self.upstream() + rec = self.serve(0, up, cfg('record'), replay.Store(self.store_dir), replay.Stats()) + status, data = request(rec.server_port, 'POST', BATCH, + batch_body('pkg:npm/a@1.0.0', 'pkg:npm/b@2.0.0')) + self.assertEqual(status, 200) + self.assertEqual(request(rec.server_port, 'POST', BATCH, batch_body('pkg:npm/c@3.0.0'))[0], 200) + self.assertEqual(request(rec.server_port, 'GET', '/v0/orgs/acme/patches/by-package/x')[0], 200) + self.assertEqual(len(Upstream.hits), 3) + + # A fresh Store reloads everything from disk; replay never goes upstream. + stats = replay.Stats() + rep = self.serve(0, up, cfg('replay'), replay.Store(self.store_dir), stats) + # One chunk spanning both recorded batches, in a new order, plus a purl + # never seen while recording. + status, data = request(rep.server_port, 'POST', BATCH, + batch_body('pkg:npm/c@3.0.0', 'pkg:npm/z@9.9.9', 'pkg:npm/b@2.0.0', + 'pkg:npm/a@1.0.0')) + self.assertEqual(status, 200) + got = json.loads(data) + self.assertEqual([p['purl'] for p in got['packages']], ['pkg:npm/c@3.0.0', 'pkg:npm/a@1.0.0']) + self.assertTrue(got['canAccessPaidPatches']) + status, data = request(rep.server_port, 'GET', '/v0/orgs/acme/patches/by-package/x') + self.assertEqual((status, json.loads(data)), (200, {'path': '/v0/orgs/acme/patches/by-package/x'})) + self.assertEqual(request(rep.server_port, 'GET', '/v0/orgs/acme/patches/view/nope')[0], 599) + self.assertEqual(len(Upstream.hits), 3) + + snap = stats.snapshot() + self.assertEqual(snap['requests'], 3) + self.assertEqual(snap['misses'], 1) + self.assertEqual(snap['batch_unknown_purls'], 1) + self.assertEqual(snap['by_kind'], {'POST batch': 1, 'GET by-package': 1, 'GET view': 1}) + self.assertEqual(snap['by_status'], {'200': 2, '599': 1}) + + def test_fill_forwards_and_records_a_miss(self): + up = self.upstream() + stats = replay.Stats() + rep = self.serve(0, up, cfg('replay', fill=True), replay.Store(self.store_dir), stats) + self.assertEqual(request(rep.server_port, 'GET', '/v0/orgs/acme/patches/view/u-a')[0], 200) + self.assertEqual(request(rep.server_port, 'GET', '/v0/orgs/acme/patches/view/u-a')[0], 200) + self.assertEqual(len(Upstream.hits), 1) + self.assertEqual(stats.snapshot()['misses'], 1) + + def test_unreachable_upstream_is_502_and_not_stored(self): + with socket.socket() as s: + s.bind(('127.0.0.1', 0)) + dead = s.getsockname()[1] + store = replay.Store(self.store_dir) + rec = self.serve(0, f'http://127.0.0.1:{dead}', cfg('record', upstream_timeout=5), + store, replay.Stats()) + with patch('sys.stderr', new_callable=io.StringIO) as err: + status, _ = request(rec.server_port, 'GET', '/v0/orgs/acme/patches/view/u-a') + self.assertEqual(status, 502) + self.assertIn('UPSTREAM ERROR GET /v0/orgs/acme/patches/view/u-a', err.getvalue()) + self.assertEqual(store.entries, {}) + self.assertEqual(replay.Store(self.store_dir).entries, {}) + + def test_latency_is_per_request_and_max_inflight_is_counted(self): + up = self.upstream() + rec = self.serve(0, up, cfg('record'), replay.Store(self.store_dir), replay.Stats()) + request(rec.server_port, 'GET', '/v0/orgs/acme/patches/view/u-a') + stats = replay.Stats() + rep = self.serve(0, up, cfg('replay', latency_ms=1000.0), replay.Store(self.store_dir), stats) + t = time.time() + with concurrent.futures.ThreadPoolExecutor(4) as pool: + codes = list(pool.map(lambda _: request(rep.server_port, 'GET', + '/v0/orgs/acme/patches/view/u-a')[0], range(4))) + self.assertEqual(codes, [200] * 4) + self.assertGreaterEqual(time.time() - t, 1.0) + snap = stats.snapshot() + self.assertEqual(snap['max_inflight'], 4) + self.assertEqual(snap['connections'], 4) + self.assertGreater(snap['avg_parallelism'], 1.0) + stats.reset() + self.assertEqual(stats.snapshot()['requests'], 0) + + +@unittest.skipUnless(all(shutil.which(t) for t in ('bash', 'curl', 'perl', 'shasum')), + 'bench.sh needs bash, curl, perl and shasum') +class BenchScriptTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + + def tearDown(self): + self.temp.cleanup() + + def fake_cli(self, name, suffix=''): + """A stand-in binary: one batch POST through SOCKET_API_URL, echoed.""" + p = self.root / name + p.write_text(textwrap.dedent(f'''\ + #!/usr/bin/env bash + curl -sf -X POST "$SOCKET_API_URL{BATCH}" \\ + -H 'Content-Type: application/json' \\ + -d '{{"components":[{{"purl":"pkg:npm/a@1.0.0"}}]}}' + echo "{suffix}" + echo "args: $*" >&2 + ''')) + p.chmod(0o755) + return str(p) + + def bench(self, *args, **env): + full = dict(os.environ, CWD=str(self.root), OUT=str(self.root / 'out'), **env) + return subprocess.run(['bash', str(PERF / 'bench.sh'), *args], env=full, + capture_output=True, text=True, timeout=300) + + def test_refuses_a_store_inside_the_repository(self): + inside = PERF.parents[1] / 'target' / 'perf-store-must-not-exist' + r = self.bench('replay', str(inside), '--', 'scan', BIN=shutil.which('true')) + try: + self.assertEqual(r.returncode, 2, r.stderr) + self.assertIn('refusing a STORE inside the repository', r.stderr) + self.assertFalse(inside.exists(), 'a refused STORE must not be created') + finally: + shutil.rmtree(inside, ignore_errors=True) + + def test_ab_checks_stdout_sha_against_the_first_base_run(self): + # Seed the store with the batch the fake CLI sends (unit tests never + # reach the real services). + store = replay.Store(str(self.root / 'store')) + req = json.dumps(batch_body('pkg:npm/a@1.0.0')).encode() + resp = json.dumps({'packages': [KNOWN['pkg:npm/a@1.0.0']], 'canAccessPaidPatches': False}).encode() + store.put_batch(req, resp) + port = str(free_port_run(3)) + same = self.fake_cli('same') + r = self.bench('ab', str(self.root / 'store'), '5', '2', '--', 'scan', '--json', + BASE=same, NEW=same, PORT=port, + PRE_RUN=f'echo reset >> {self.root / "pre-run.log"}') + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual((self.root / 'pre-run.log').read_text(), 'reset\n' * 4) + self.assertIn('OK: every run', r.stdout) + self.assertEqual(r.stdout.count('vs_base=same'), 3) + self.assertIn('stderr: identical across runs', r.stdout) + self.assertIn('max_inflight= 1', r.stdout) + out = (self.root / 'out' / 'ab-5ms-1-base.stdout').read_text() + self.assertIn('pkg:npm/a@1.0.0', out) + self.assertEqual((self.root / 'out' / 'ab-5ms-1-base.stderr').read_text(), + f'args: scan --json --cwd {self.root}\n') + + r = self.bench('ab', str(self.root / 'store'), '0', '1', '--', 'scan', + BASE=same, NEW=self.fake_cli('changed', suffix='extra'), PORT=port) + self.assertEqual(r.returncode, 1, r.stdout + r.stderr) + self.assertIn('vs_base=DIFFERS', r.stdout) + self.assertIn('FAIL: 1 run(s) differ', r.stderr) + + +if __name__ == '__main__': + unittest.main() From 62dc8c570e8b2bd31fc88bb61483c11898a28507 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 14:56:17 -0400 Subject: [PATCH 025/237] refactor(hosted): fetch wheel metadata through the shared ordered_concurrent helper The inline stream::iter().buffered() from the wheel-metadata fan-out predates utils::concurrent landing; route it through ordered_concurrent with the same limit (4) and the same in-order fold. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../socket-patch-cli/src/commands/scan/hosted.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 1bd61f7e..2a7e5c41 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -1838,20 +1838,20 @@ pub(crate) async fn run_redirect_selected( // baseline's messages, in the baseline's order), and the client's // opt-in debug lines (GET / attempt-failed) interleave across the // in-flight downloads. - // TODO(perf): switch to `utils::concurrent::ordered_concurrent` once - // it lands (added in parallel on the scan-concurrency branch), and - // let a `Retry-After` pause the whole fan-out rather than one fetch. + // TODO(perf): let a `Retry-After` pause the whole fan-out rather + // than one fetch. const WHEEL_METADATA_CONCURRENCY: usize = 4; - use futures_util::StreamExt as _; - let mut fetches = std::pin::pin!(futures_util::stream::iter(wheel_deps.iter()) - .map(|&(dep, sha256)| { + let mut fetches = std::pin::pin!(ordered_concurrent( + wheel_deps.iter(), + WHEEL_METADATA_CONCURRENCY, + |&(dep, sha256)| { socket_patch_core::vendor::pypi::fetch_hosted_wheel_metadata( api_client, &dep.artifact_url, sha256, ) - }) - .buffered(WHEEL_METADATA_CONCURRENCY)); + }, + )); for &(dep, _) in &wheel_deps { status.set(format!( "Fetching hosted wheel metadata for {}...", From ef34d3662d69cc79ab4072785fb8e72ac909792b Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 17:01:37 -0400 Subject: [PATCH 026/237] fix(scan): deliver background telemetry before the first stderr write too 0ef24902 flushed the scan event before the first stdout write after it fires, but stderr raises SIGPIPE just as well (main restores SIG_DFL). Two stderr writers could run in that window with the send still in flight: the lenient redirect-ledger load's "Warning: " (non-hosted JSON and human paths, before discover_selected or the human flush) and, on the report-only JSON arm, the GC and VEX build ahead of the envelope. The inline send it replaced was always delivered first. The ledger load is inlined at its scan call site so the send is flushed right before its warning (only when it warns, so the overlap with the detail fetches is kept), and the JSON arm flushes before the GC/VEX step instead of just before the envelope. The --apply arm's warnings already follow discover_selected's flush. Test: telemetry_e2e runs a scan over a malformed redirect ledger with stderr closed and requires the event delivered (red before: SIGPIPE, 0 events). It uses a well-shaped token so the token-shape warning does not kill the child before the event fires. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../socket-patch-cli/src/commands/scan/mod.rs | 25 +++- .../socket-patch-cli/tests/telemetry_e2e.rs | 129 +++++++++++++++++- 2 files changed, 147 insertions(+), 7 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index ef6ff472..65318165 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -2114,11 +2114,20 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { Err(corrupt) => (None, Some(corrupt.to_string())), } } else { - ( - crate::commands::load_redirect_state_lenient(&args.common.cwd, args.common.silent) - .await, - None, - ) + // `load_redirect_state_lenient`, with the scan event's send flushed + // before its warning: that line can be this run's first write since + // the event fired, and a closed stderr's SIGPIPE must find the event + // delivered, as the inline send it replaced was. + match socket_patch_core::patch::redirect::load_redirect_state(&args.common.cwd).await { + Ok(state) => (state, None), + Err(corrupt) => { + if !args.common.silent { + telemetry.flush().await; + eprintln!("Warning: {corrupt}"); + } + (None, None) + } + } }; let update_manifest = merge_ledger_records_for_updates( existing_manifest.as_ref(), @@ -2384,6 +2393,11 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { .await; } + // The GC and the VEX build below can write to stderr; the report- + // only arm has not flushed the scan event yet (the `--apply` arm + // did, in `discover_selected`). + telemetry.flush().await; + // --- GC (post-apply, or standalone --prune GC-sweep) ------------- if prune { result["gc"] = gc_json( @@ -2405,7 +2419,6 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { &mut result, ) .await; - telemetry.flush().await; print_json(&result); return final_code; } diff --git a/crates/socket-patch-cli/tests/telemetry_e2e.rs b/crates/socket-patch-cli/tests/telemetry_e2e.rs index 6f7df9e1..cbfdb3af 100644 --- a/crates/socket-patch-cli/tests/telemetry_e2e.rs +++ b/crates/socket-patch-cli/tests/telemetry_e2e.rs @@ -63,6 +63,30 @@ fn build_cmd( subcommand: &str, extra_args: &[&str], extra_env: &[(&str, &str)], +) -> Command { + build_cmd_with_token( + "fake-token-for-test", + cwd, + api_url, + subcommand, + extra_args, + extra_env, + ) +} + +/// A `sktsec_<44 chars>_api`-shaped token: the client's token-shape check +/// stays quiet, so a run's stderr carries only what the command under test +/// writes (the default fake token draws a warning before anything runs). +const WELL_SHAPED_TOKEN: &str = "sktsec_00000000000000000000000000000000000000000000_api"; + +/// [`build_cmd`] with an explicit `--api-token`. +fn build_cmd_with_token( + api_token: &str, + cwd: &Path, + api_url: &str, + subcommand: &str, + extra_args: &[&str], + extra_env: &[(&str, &str)], ) -> Command { let mut args = vec![ subcommand, @@ -70,7 +94,7 @@ fn build_cmd( "--api-url", api_url, "--api-token", - "fake-token-for-test", + api_token, "--org", ORG_SLUG, ]; @@ -944,3 +968,106 @@ async fn scan_delivers_telemetry_before_writing_to_a_closed_stdout() { ); } } + +/// The stderr twin of the closed-stdout test above: a malformed hosted +/// redirect ledger makes a non-hosted scan warn on stderr (the lenient +/// read-only consult) right after the scan event fires, BEFORE any stdout +/// write — so with stderr closed that warning is the run's first +/// SIGPIPE-raising write, and the background send must be flushed ahead of +/// it, as the inline send it replaced always was. The plain envelope does +/// no network work between the event and the warning, so without the flush +/// the delayed send deterministically loses the race; the vendored arm is +/// covered too (its warning also precedes `discover_selected`'s flush). +#[tokio::test] +async fn scan_delivers_telemetry_before_writing_to_a_closed_stderr() { + const TELEMETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(800); + + let cases: [(&str, &[&str]); 2] = [ + ("plain envelope", &[]), + ("vendored", &["--mode", "vendored", "--dry-run"]), + ]; + for (label, extra_args) in cases { + let mock = MockServer::start().await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/patches/batch"))) + .respond_with(ResponseTemplate::new(200).set_body_json( + serde_json::json!({ "packages": [], "canAccessPaidPatches": false }), + )) + .mount(&mock) + .await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/telemetry"))) + .respond_with(ResponseTemplate::new(201).set_delay(TELEMETRY_DELAY)) + .mount(&mock) + .await; + + let tmp = tempfile::tempdir().expect("tempdir"); + write_root_package_json(tmp.path()); + write_npm_package(tmp.path(), "minimist", "1.2.2"); + let ledger_dir = tmp.path().join(".socket").join("vendor"); + std::fs::create_dir_all(&ledger_dir).expect("mkdir .socket/vendor"); + std::fs::write(ledger_dir.join("redirect-state.json"), "{ not json") + .expect("write malformed redirect ledger"); + + // Sanity: with stderr open the run does warn about the ledger, so + // the closed-stderr run below really has a write to die on. + let out = build_cmd_with_token( + WELL_SHAPED_TOKEN, + tmp.path(), + &mock.uri(), + "scan", + extra_args, + &[], + ) + .output() + .expect("run socket-patch"); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.starts_with("Warning: ") && stderr.contains("redirect-state.json"), + "{label}: the malformed redirect ledger's warning must be the \ + run's first stderr write; got: {stderr}" + ); + mock.reset().await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/patches/batch"))) + .respond_with(ResponseTemplate::new(200).set_body_json( + serde_json::json!({ "packages": [], "canAccessPaidPatches": false }), + )) + .mount(&mock) + .await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/telemetry"))) + .respond_with(ResponseTemplate::new(201).set_delay(TELEMETRY_DELAY)) + .mount(&mock) + .await; + + let mut child = build_cmd_with_token( + WELL_SHAPED_TOKEN, + tmp.path(), + &mock.uri(), + "scan", + extra_args, + &[], + ) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn socket-patch"); + // Close the read end before the child can write anything: its + // first stderr write now raises SIGPIPE. + drop(child.stderr.take()); + let status = child.wait().expect("wait socket-patch"); + + assert_eq!( + telemetry_post_count(&mock, Some("patch_scanned")).await, + 1, + "{label}: the patch_scanned event must be delivered before stderr \ + is written (exit status {status:?})" + ); + assert_eq!( + telemetry_post_count(&mock, None).await, + 1, + "{label}: no other telemetry event" + ); + } +} From 5623fbfaeae288f9f39f5e1f2a50c46c4a9df96b Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 17:02:05 -0400 Subject: [PATCH 027/237] docs(telemetry): say why the inline scan trackers stay public scan now sends patch_scanned / patch_scan_failed through the spawn_* variants, which leaves the inline trackers without an in-tree caller. They stay: socket-patch-core is published to crates.io, removing a pub fn is a breaking change there, and every other event keeps its inline tracker. The doc comments now say so, so a later cleanup does not read them as leftovers. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/socket-patch-core/src/telemetry.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/socket-patch-core/src/telemetry.rs b/crates/socket-patch-core/src/telemetry.rs index 156c9106..c21217a1 100644 --- a/crates/socket-patch-core/src/telemetry.rs +++ b/crates/socket-patch-core/src/telemetry.rs @@ -597,6 +597,10 @@ fn patch_scanned_metadata( /// dashboard needs — grouping them into a struct would force callers /// to build a config object for a single fire-and-forget call, which /// is worse ergonomics for a tracker. +/// +/// The CLI's `scan` sends this event through [`spawn_patch_scanned`]; the +/// inline tracker stays as this published crate's public API, alongside +/// the inline tracker every other event has. #[allow(clippy::too_many_arguments)] pub async fn track_patch_scanned( packages_scanned: usize, @@ -665,7 +669,9 @@ fn patch_scan_failed_metadata(fallback_to_proxy: bool) -> serde_json::Value { serde_json::json!({ "fallback_to_proxy": fallback_to_proxy }) } -/// Track a failed `scan`. +/// Track a failed `scan`. The CLI sends it through +/// [`spawn_patch_scan_failed`]; kept as public API like +/// [`track_patch_scanned`]. pub async fn track_patch_scan_failed( error: impl std::fmt::Display, fallback_to_proxy: bool, From 42675ecb165e3ee8eafc16cebb2f8a9ad372f2e0 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 17:03:29 -0400 Subject: [PATCH 028/237] test(scan): pin that the mid-run fallback discards in-flight answers batch_fallback_mid_run_replays_from_the_failing_chunk checked the folded uuids and the proxied tail, but not that chunks 4-5 were ever sent to the authenticated API. A serial loop (or a window of 1) would never request them and still pass. Assert all 6 authenticated requests: chunk 0 alone, then the whole 1..6 window in flight, so the discard path really runs. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../socket-patch-cli/tests/scan_ordered_concurrency_e2e.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/socket-patch-cli/tests/scan_ordered_concurrency_e2e.rs b/crates/socket-patch-cli/tests/scan_ordered_concurrency_e2e.rs index 00502519..bfd54cb2 100644 --- a/crates/socket-patch-cli/tests/scan_ordered_concurrency_e2e.rs +++ b/crates/socket-patch-cli/tests/scan_ordered_concurrency_e2e.rs @@ -363,6 +363,12 @@ async fn batch_fallback_mid_run_replays_from_the_failing_chunk() { let mut tail: Vec = order[3..].iter().map(|&i| purl(NAMES[i])).collect(); tail.sort(); assert_eq!(proxied, tail); + + // The authenticated API saw every chunk: chunk 0 alone, then the + // whole 1..6 window in flight at once. So the answers for chunks 4-5 + // really existed (they arrived before chunk 3's 401) and were + // dropped — the uuids above prove it — rather than never requested. + assert_eq!(batch_requests(&auth, &auth_batch_route()).await.len(), 6); } /// Mixed 500s in chunks 2 and 4 with reversed latencies: the per-batch From 90616df272b0373e27426b0ae7a8977a9fb00b36 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 17:11:12 -0400 Subject: [PATCH 029/237] perf(api): share one in-flight cap across concurrent proxy batch calls On the public proxy scan runs up to PROXY_API_CONCURRENCY (4) batch windows at once. Each window's search_patches_batch degrades to the legacy per-package GETs (10 at a time) when /patch/batch rejects the chunk: a 400 from one exotic purl such as pkg:jsr, or an old proxy with no batch route. So a polyglot project on the proxy could put 4 x 10 by-package GETs in flight where the serial loop peaked at 10. That path swallows per-purl errors as "no patches", so extra load that saturates the proxy could change which packages come back. The client now holds a semaphore of PROXY_BATCH_PATH_CONCURRENCY (10) slots, shared by clones. Every proxy /patch/batch POST and every legacy per-package GET takes a slot, so all concurrent batch calls on one client stay within the old peak. A single call never waits: its groups of 10 fit the cap exactly as before. The authenticated API is untouched. Test: four concurrent batch calls of 10 purls each, all rejected with 400, keep at most 10 by-package GETs in flight and still reach 10 (red without the slots: 40). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/socket-patch-core/src/api/client.rs | 142 ++++++++++++++++++++- 1 file changed, 137 insertions(+), 5 deletions(-) diff --git a/crates/socket-patch-core/src/api/client.rs b/crates/socket-patch-core/src/api/client.rs index 4f3bd71c..dff30425 100644 --- a/crates/socket-patch-core/src/api/client.rs +++ b/crates/socket-patch-core/src/api/client.rs @@ -118,8 +118,20 @@ pub struct ApiClient { /// retryable failure (transport / 429 / 5xx after every retry) — the /// run-level circuit breaker. Shared by clones: one CLI run, one count. vendor_outage: Arc, + /// In-flight slots for the public proxy's batch path — the + /// `/patch/batch` POSTs and the legacy per-package GETs they degrade + /// to. Shared by clones, so concurrent [`Self::search_patches_batch`] + /// calls on one client (scan's batch windows) stay within + /// [`PROXY_BATCH_PATH_CONCURRENCY`] requests in total: the peak the + /// serial batch loop reached, never that peak times the window. + proxy_batch_slots: Arc, } +/// Most requests the public proxy's batch path keeps in flight per client: +/// the legacy per-package fallback's fan-out (one call runs its PURLs in +/// groups of this size), and the cap all concurrent calls share. +const PROXY_BATCH_PATH_CONCURRENCY: usize = 10; + /// Retry policy for the vendoring service's package-reference POST and /// archive GET: `attempts` tries in total, exponential delays from `base` /// with ±25% jitter, each capped at `max_delay` (a `Retry-After` in seconds @@ -271,9 +283,18 @@ impl ApiClient { org_slug: options.org_slug, vendor_retry: VendorRetryPolicy::default(), vendor_outage: Arc::new(AtomicU32::new(0)), + proxy_batch_slots: Arc::new(tokio::sync::Semaphore::new(PROXY_BATCH_PATH_CONCURRENCY)), } } + /// Wait for a [`Self::proxy_batch_slots`] slot; held until dropped. + async fn proxy_batch_slot(&self) -> tokio::sync::OwnedSemaphorePermit { + Arc::clone(&self.proxy_batch_slots) + .acquire_owned() + .await + .expect("proxy_batch_slots is never closed") + } + /// Override the vendoring-service retry policy (tests; a policy of /// [`VendorRetryPolicy::none`] makes a single attempt). pub fn with_vendor_retry(mut self, policy: VendorRetryPolicy) -> Self { @@ -558,6 +579,8 @@ impl ApiClient { let body = BatchSearchBody::new(purls); + // Held until this call returns, response body read. + let _slot = self.proxy_batch_slot().await; let resp = self .client .post(&url) @@ -614,18 +637,18 @@ impl ApiClient { /// proxy gained `POST /patch/batch`, this is the legacy path for /// deployments that predate it. /// - /// Processes PURLs in batches of `CONCURRENCY_LIMIT` to avoid - /// overwhelming the server while remaining efficient. + /// Processes PURLs in batches of `PROXY_BATCH_PATH_CONCURRENCY` to + /// avoid overwhelming the server while remaining efficient; each GET + /// also takes a [`Self::proxy_batch_slots`] slot, so concurrent calls + /// on one client share that cap instead of multiplying it. async fn search_patches_batch_via_individual_queries( &self, purls: &[String], ) -> Result { - const CONCURRENCY_LIMIT: usize = 10; - // Collect all (purl, response) pairs let mut all_results: Vec<(String, Option)> = Vec::new(); - for chunk in purls.chunks(CONCURRENCY_LIMIT) { + for chunk in purls.chunks(PROXY_BATCH_PATH_CONCURRENCY) { // Use tokio::JoinSet for concurrent execution within each chunk let mut join_set = tokio::task::JoinSet::new(); @@ -633,7 +656,9 @@ impl ApiClient { let purl = purl.clone(); let client = self.clone(); join_set.spawn(async move { + let slot = client.proxy_batch_slot().await; let resp = client.search_patches_by_package(&purl).await; + drop(slot); match resp { Ok(r) => (purl, Some(r)), Err(e) => { @@ -4698,3 +4723,110 @@ mod authenticated_batch_tests { assert!(result.can_access_paid_patches); } } + +#[cfg(test)] +mod proxy_batch_path_cap_tests { + //! Concurrent `search_patches_batch` calls on one public-proxy client + //! (scan's batch windows) must share the batch path's in-flight cap, + //! not multiply it: when every chunk degrades to the legacy per-package + //! GETs, the proxy sees at most `PROXY_BATCH_PATH_CONCURRENCY` of them + //! at once — what the serial batch loop peaked at. + use super::*; + use std::sync::Mutex; + use std::time::Instant; + use wiremock::matchers::{method, path, path_regex}; + use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate}; + + /// Every by-package answer takes this long, so the requests in flight + /// at an arrival are exactly those that arrived less than this before. + const GET_DELAY: Duration = Duration::from_millis(300); + + /// Records each by-package GET's arrival time, then answers it (empty, + /// after [`GET_DELAY`]). + struct Arrivals(Arc>>); + + impl Respond for Arrivals { + fn respond(&self, _: &Request) -> ResponseTemplate { + self.0.lock().unwrap().push(Instant::now()); + ResponseTemplate::new(200) + .set_body_json(serde_json::json!({ + "patches": [], + "canAccessPaidPatches": false, + })) + .set_delay(GET_DELAY) + } + } + + /// Most GETs whose arrivals fall inside one window shorter than + /// [`GET_DELAY`] — a lower bound on the peak in flight that a capped + /// run cannot exceed (a slot frees only when its answer, `GET_DELAY` + /// after its arrival, is back). + fn peak_in_flight(arrivals: &[Instant]) -> usize { + let mut sorted = arrivals.to_vec(); + sorted.sort(); + let window = GET_DELAY.mul_f32(0.8); + (0..sorted.len()) + .map(|i| { + sorted[i..] + .iter() + .take_while(|t| t.duration_since(sorted[i]) < window) + .count() + }) + .max() + .unwrap_or(0) + } + + #[tokio::test] + async fn concurrent_batches_share_the_legacy_fallback_cap() { + let server = MockServer::start().await; + // A validation 400 for every chunk: each degrades to per-package + // GETs (one exotic PURL per chunk is enough in the wild). + Mock::given(method("POST")) + .and(path("/patch/batch")) + .respond_with(ResponseTemplate::new(400).set_body_string("bad purl")) + .mount(&server) + .await; + let arrivals = Arc::new(Mutex::new(Vec::new())); + Mock::given(method("GET")) + .and(path_regex("^/patch/by-package/")) + .respond_with(Arrivals(Arc::clone(&arrivals))) + .mount(&server) + .await; + + let client = ApiClient::new(ApiClientOptions { + api_url: server.uri(), + api_token: None, + use_public_proxy: true, + org_slug: None, + }); + // Four windows of 10 PURLs, as scan's proxy batch windows run them. + let chunks: Vec> = (0..4) + .map(|c| { + (0..PROXY_BATCH_PATH_CONCURRENCY) + .map(|i| format!("pkg:npm/cap-{c}-{i}@1.0.0")) + .collect() + }) + .collect(); + let results = futures_util::future::join_all( + chunks + .iter() + .map(|chunk| client.search_patches_batch(chunk)), + ) + .await; + for result in results { + let response = result.expect("the per-package path swallows nothing here"); + assert!(response.packages.is_empty()); + } + + let arrivals = arrivals.lock().unwrap(); + assert_eq!(arrivals.len(), 40, "one GET per PURL"); + let peak = peak_in_flight(&arrivals); + assert!( + peak <= PROXY_BATCH_PATH_CONCURRENCY, + "{peak} by-package GETs in flight at once; the cap is \ + {PROXY_BATCH_PATH_CONCURRENCY}" + ); + // And the cap is reached, not undershot: the calls still overlap. + assert_eq!(peak, PROXY_BATCH_PATH_CONCURRENCY); + } +} From e183bc5797f5db3cb67ddc2ed7d475e87c92ee29 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 17:49:27 -0400 Subject: [PATCH 030/237] fix(crawl): never fall back to rayon's global pool when walk threads can't spawn When the walk pool could not be built (the OS refusing threads: a tight RLIMIT_NPROC or cgroup pids.max, or a huge RAYON_NUM_THREADS), run_walk ran the walk on the calling thread and the first parallel iterator then tried to build rayon's global pool, which needs the same refused threads and panics (exit 101) where the sequential walk succeeded. - Retry the pool build with half the threads on each failure, down to 1. - Route every parallel gather through walk_pool::par_map, which maps sequentially (in order) on a thread outside any rayon pool, so the no-pool fallback never reaches the global pool. - RAYON_NUM_THREADS can lower the walk thread count but no longer raise it past available_parallelism. Tests: halving build, par_map's sequential/ordered contract, run_walk's no-pool path, and the randomized oracle comparison with the pool off. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/crawlers/npm_crawler.rs | 169 ++++++++--------- .../src/crawlers/npm_crawler/oracle.rs | 17 ++ .../src/crawlers/walk_pool.rs | 170 ++++++++++++++++-- 3 files changed, 248 insertions(+), 108 deletions(-) diff --git a/crates/socket-patch-core/src/crawlers/npm_crawler.rs b/crates/socket-patch-core/src/crawlers/npm_crawler.rs index 255e76e8..508c6dc5 100644 --- a/crates/socket-patch-core/src/crawlers/npm_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/npm_crawler.rs @@ -3,11 +3,10 @@ use std::ffi::{OsStr, OsString}; use std::fs::FileType; use std::path::{Path, PathBuf}; -use rayon::prelude::*; use serde::Deserialize; use super::types::{CrawledPackage, CrawlerOptions}; -use super::walk_pool::run_walk; +use super::walk_pool::{par_map, run_walk}; use crate::patch::path_safety; use crate::utils::fs::{is_dir, is_dir_sync, read_dir_entries_sync}; use crate::utils::purl::{percent_decode_purl_component, strip_purl_qualifiers}; @@ -639,10 +638,9 @@ impl NpmCrawler { fn crawl_all_sync(options: &CrawlerOptions) -> Vec { let nm_paths = Self::node_modules_paths_sync(options); - let gathered: Vec> = nm_paths - .par_iter() - .map(|nm_path| Self::gather_node_modules(nm_path, None, false)) - .collect(); + let gathered: Vec> = par_map(&nm_paths, |nm_path| { + Self::gather_node_modules(nm_path, None, false) + }); let mut packages = Vec::new(); let mut seen = HashSet::new(); @@ -787,10 +785,8 @@ impl NpmCrawler { } let mut level: Vec = vec![node_modules_path.to_path_buf()]; while !level.is_empty() { - let visits: Vec = level - .into_par_iter() - .map(|nm_path| Self::visit_resolver_dir(nm_path, &pending)) - .collect(); + let visits: Vec = + par_map(level, |nm_path| Self::visit_resolver_dir(nm_path, &pending)); let mut next_level: Vec = Vec::new(); for visit in visits { let nm_path = visit.nm_path; @@ -890,11 +886,9 @@ impl NpmCrawler { /// Entries are examined in parallel; their contributions keep listing /// order. fn collect_nested_node_modules(nm_path: &Path, listing: Listing) -> Vec { - let found: Vec> = listing - .entries - .into_par_iter() - .map(|entry| Self::nested_node_modules_of(nm_path, entry)) - .collect(); + let found: Vec> = par_map(listing.entries, |entry| { + Self::nested_node_modules_of(nm_path, entry) + }); found.into_iter().flatten().collect() } @@ -1128,16 +1122,13 @@ impl NpmCrawler { let mut level = Self::workspace_children(dir, listing); let roots = 0..level.len(); while !level.is_empty() { - let visits: Vec<(Option, Vec)> = level - .into_par_iter() - .map(|full_path| { - let listing = list_dir_sync(&full_path); - // Check if this subdirectory has its own node_modules - let node_modules = has_node_modules_dir(&full_path, &listing) - .then(|| full_path.join("node_modules")); - (node_modules, Self::workspace_children(&full_path, listing)) - }) - .collect(); + let visits: Vec<(Option, Vec)> = par_map(level, |full_path| { + let listing = list_dir_sync(&full_path); + // Check if this subdirectory has its own node_modules + let node_modules = has_node_modules_dir(&full_path, &listing) + .then(|| full_path.join("node_modules")); + (node_modules, Self::workspace_children(&full_path, listing)) + }); let next_base = nodes.len() + visits.len(); let mut next_level = Vec::new(); for (node_modules, children) in visits { @@ -1260,24 +1251,21 @@ impl NpmCrawler { children.push((node_modules_path.join(&name_str), name_str, file_type)); } - let mut events: Vec = children - .into_par_iter() - .map(|(entry_path, name_str, file_type)| { - if name_str.starts_with('@') { - // Scoped packages - Self::gather_scoped_packages(&entry_path, &name_str, store_entry) - } else { - Self::gather_package( - entry_path, - store_entry.then_some(name_str), - file_type.is_dir(), - ) - } - }) - .collect::>() - .into_iter() - .flatten() - .collect(); + let mut events: Vec = par_map(children, |(entry_path, name_str, file_type)| { + if name_str.starts_with('@') { + // Scoped packages + Self::gather_scoped_packages(&entry_path, &name_str, store_entry) + } else { + Self::gather_package( + entry_path, + store_entry.then_some(name_str), + file_type.is_dir(), + ) + } + }) + .into_iter() + .flatten() + .collect(); if let Some(store_path) = pnpm_store { let entries = Self::list_pnpm_store_entries_sync(&store_path, true); @@ -1355,19 +1343,16 @@ impl NpmCrawler { }) .collect(); - children - .into_par_iter() - .map(|(name_str, file_type)| { - Self::gather_package( - scope_path.join(&name_str), - store_entry.then(|| format!("{scope_name}/{name_str}")), - file_type.is_dir(), - ) - }) - .collect::>() - .into_iter() - .flatten() - .collect() + par_map(children, |(name_str, file_type)| { + Self::gather_package( + scope_path.join(&name_str), + store_entry.then(|| format!("{scope_name}/{name_str}")), + file_type.is_dir(), + ) + }) + .into_iter() + .flatten() + .collect() } /// Gather each virtual-store entry's `node_modules` (entries come from @@ -1375,13 +1360,10 @@ impl NpmCrawler { /// [`Self::collect_nested_store_entries_sync`]) under the store-entry /// policy, in parallel, preserving entry order. fn gather_store_entries(entries: Vec) -> Vec { - entries - .into_par_iter() - .map(|entry| ScanEvent::StoreEntry { - decoded: decode_pnpm_store_entry_name(&entry.name), - events: Self::gather_node_modules(&entry.node_modules, entry.listing, true), - }) - .collect() + par_map(entries, |entry| ScanEvent::StoreEntry { + decoded: decode_pnpm_store_entry_name(&entry.name), + events: Self::gather_node_modules(&entry.node_modules, entry.listing, true), + }) } /// Replay gathered [`ScanEvent`]s in order against `seen`, exactly as @@ -1471,41 +1453,38 @@ impl NpmCrawler { }) .collect(); - candidates - .into_par_iter() - .map(|entry| { - let entry_path = store_path.join(&entry.name); - let entry_nm = entry_path.join("node_modules"); - if read_listings { - if let Some((entries, complete)) = read_dir_entries_sync(&entry_nm) { - return vec![StoreEntryDir { - name: entry.name_str, - node_modules: entry_nm, - listing: Some(Listing::from_entries(entries, complete)), - }]; - } - } - if is_dir_sync(&entry_nm) { - vec![StoreEntryDir { + par_map(candidates, |entry| { + let entry_path = store_path.join(&entry.name); + let entry_nm = entry_path.join("node_modules"); + if read_listings { + if let Some((entries, complete)) = read_dir_entries_sync(&entry_nm) { + return vec![StoreEntryDir { name: entry.name_str, node_modules: entry_nm, - listing: None, - }] - } else { - Self::collect_nested_store_entries_sync(&entry_path) - .into_iter() - .map(|(name, node_modules)| StoreEntryDir { - name, - node_modules, - listing: None, - }) - .collect() + listing: Some(Listing::from_entries(entries, complete)), + }]; } - }) - .collect::>() - .into_iter() - .flatten() - .collect() + } + if is_dir_sync(&entry_nm) { + vec![StoreEntryDir { + name: entry.name_str, + node_modules: entry_nm, + listing: None, + }] + } else { + Self::collect_nested_store_entries_sync(&entry_path) + .into_iter() + .map(|(name, node_modules)| StoreEntryDir { + name, + node_modules, + listing: None, + }) + .collect() + } + }) + .into_iter() + .flatten() + .collect() } /// Async `(name, node_modules)` view of diff --git a/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs b/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs index 42eda9c5..f92d62f4 100644 --- a/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs +++ b/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs @@ -1366,6 +1366,23 @@ mod tests { assert!(nonempty > 32, "only {nonempty} non-empty trees"); } + /// With no walk pool (the OS refused every walk thread) the walk runs + /// sequentially on the calling thread and still matches the oracle. + #[tokio::test] + async fn walk_without_a_pool_matches_the_sequential_oracle() { + let _off = crate::crawlers::walk_pool::test_hooks::DisablePool::new(); + for seed in 0..16u64 { + let tmp = tempfile::tempdir().unwrap(); + let mut guard = PermGuard(Vec::new()); + let root = tmp.path().join("proj"); + let mut gen = Gen::new(seed, tmp.path().join("scratch")); + gen.workspace(&root, 0); + gen.apply_locks(&mut guard); + assert_equivalent(&root, &format!("no pool, seed {seed}")).await; + drop(guard); + } + } + /// Hand-built tree with one of every tricky shape (so each is covered /// regardless of what the random generator happens to draw), asserting /// equivalence and pinning a few load-bearing outcomes. diff --git a/crates/socket-patch-core/src/crawlers/walk_pool.rs b/crates/socket-patch-core/src/crawlers/walk_pool.rs index fdfb031b..884d1854 100644 --- a/crates/socket-patch-core/src/crawlers/walk_pool.rs +++ b/crates/socket-patch-core/src/crawlers/walk_pool.rs @@ -21,6 +21,8 @@ use std::sync::OnceLock; +use rayon::iter::{IntoParallelIterator, ParallelIterator}; + use crate::utils::fs::run_blocking; /// Stack size of each walk thread: the main thread's (see module docs). @@ -94,32 +96,71 @@ fn walk_threads(cpus: usize, soft_limit: Option) -> usize { } } -/// Logical CPUs, honoring `RAYON_NUM_THREADS` like rayon's global pool. +/// Logical CPUs. `RAYON_NUM_THREADS` can lower the count (like rayon's +/// global pool) but never raise it past the machine's parallelism. fn default_cpus() -> usize { + let cpus = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1); std::env::var("RAYON_NUM_THREADS") .ok() .and_then(|v| v.parse::().ok()) .filter(|&n| n > 0) - .or_else(|| std::thread::available_parallelism().ok().map(|n| n.get())) - .unwrap_or(1) + .map_or(cpus, |n| n.min(cpus)) } -/// The walk pool, built on first use. `None` if its threads could not be -/// spawned; the walk then runs on the calling thread (rayon falls back to -/// its global pool for the parallel parts). +/// The walk pool, built on first use. `None` if not even one walk thread +/// could be spawned; the walk then runs on the calling thread (see +/// [`par_map`]). fn walk_pool() -> Option<&'static rayon::ThreadPool> { static POOL: OnceLock> = OnceLock::new(); POOL.get_or_init(|| { - rayon::ThreadPoolBuilder::new() - .num_threads(walk_threads(default_cpus(), soft_nofile_limit())) - .stack_size(WALK_STACK_SIZE) - .thread_name(|i| format!("socket-patch-walk-{i}")) - .build() - .ok() + build_with_fallback( + walk_threads(default_cpus(), soft_nofile_limit()), + |threads| { + rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .stack_size(WALK_STACK_SIZE) + .thread_name(|i| format!("socket-patch-walk-{i}")) + .build() + }, + ) }) .as_ref() } +/// Build a pool of `threads` threads, halving the count on every failure +/// (the OS refusing threads: `RLIMIT_NPROC`, a cgroup `pids.max`, memory +/// for the stacks) down to one. `None` once even one thread fails. +fn build_with_fallback(threads: usize, build: impl Fn(usize) -> Result) -> Option

{ + let mut threads = threads.max(1); + loop { + match build(threads) { + Ok(pool) => return Some(pool), + Err(_) if threads > 1 => threads /= 2, + Err(_) => return None, + } + } +} + +/// `items.map(f)` in order: in parallel on the walk pool's threads, and +/// sequentially on any thread outside a rayon pool — the calling thread +/// [`run_walk`] falls back to when no walk thread could be spawned. A +/// parallel iterator there would instead build rayon's global pool, which +/// needs the very threads the OS just refused, and panic when it cannot. +pub(crate) fn par_map(items: I, f: F) -> Vec +where + I: IntoParallelIterator + IntoIterator::Item>, + F: Fn(::Item) -> T + Sync + Send, + T: Send, +{ + if rayon::current_thread_index().is_some() { + items.into_par_iter().map(f).collect() + } else { + items.into_iter().map(f).collect() + } +} + /// Run a blocking walk on the walk pool, from a blocking-pool thread so /// the async runtime is never stalled, and hand back its value. A panic /// inside `f` is re-raised on the awaiting task (see [`run_blocking`]). @@ -128,13 +169,55 @@ where F: FnOnce() -> T + Send + 'static, T: Send + 'static, { - run_blocking(move || match walk_pool() { + let pool_disabled = test_hooks::pool_disabled(); + run_blocking(move || match walk_pool().filter(|_| !pool_disabled) { Some(pool) => pool.install(f), None => f(), }) .await } +/// Lets a test drive [`run_walk`]'s no-pool fallback (walk on the calling +/// thread) without actually exhausting the OS's threads. The switch is +/// per calling thread (read before the hop to the blocking pool), so it +/// never leaks into concurrently running tests. +pub(crate) mod test_hooks { + #[cfg(test)] + thread_local! { + static POOL_DISABLED: std::cell::Cell = const { std::cell::Cell::new(false) }; + } + + #[cfg(test)] + pub(crate) fn pool_disabled() -> bool { + POOL_DISABLED.with(|d| d.get()) + } + + #[cfg(not(test))] + pub(crate) fn pool_disabled() -> bool { + false + } + + /// Runs walks started from this thread without the walk pool until + /// the guard drops. + #[cfg(test)] + pub(crate) struct DisablePool(()); + + #[cfg(test)] + impl DisablePool { + pub(crate) fn new() -> Self { + POOL_DISABLED.with(|d| d.set(true)); + Self(()) + } + } + + #[cfg(test)] + impl Drop for DisablePool { + fn drop(&mut self) { + POOL_DISABLED.with(|d| d.set(false)); + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -165,6 +248,67 @@ mod tests { } } + #[test] + fn pool_build_halves_the_thread_count_until_it_succeeds() { + let tried = std::sync::Mutex::new(Vec::new()); + let built = build_with_fallback(14, |n| { + tried.lock().unwrap().push(n); + if n <= 3 { + Ok(n) + } else { + Err(()) + } + }); + assert_eq!(built, Some(3)); + assert_eq!(*tried.lock().unwrap(), [14, 7, 3]); + + tried.lock().unwrap().clear(); + let none = build_with_fallback(5, |n| { + tried.lock().unwrap().push(n); + Err::(()) + }); + assert_eq!(none, None); + assert_eq!(*tried.lock().unwrap(), [5, 2, 1]); + assert_eq!(build_with_fallback(0, Ok::), Some(1)); + } + + /// Outside a rayon pool (the no-walk-pool fallback) `par_map` runs + /// every item on the calling thread and never touches rayon's global + /// pool; inside one it fans out. Both keep input order. + #[test] + fn par_map_is_sequential_outside_a_pool_and_ordered_everywhere() { + let out = std::thread::spawn(|| { + assert!(rayon::current_thread_index().is_none()); + let caller = std::thread::current().id(); + par_map((0..1000).collect::>(), move |i| { + assert_eq!(std::thread::current().id(), caller); + i * 2 + }) + }) + .join() + .unwrap(); + assert_eq!(out, (0..1000).map(|i| i * 2).collect::>()); + + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .unwrap(); + let items: Vec = (0..1000).collect(); + let out = pool.install(|| par_map(&items, |&i| i + 1)); + assert_eq!(out, (1..=1000).collect::>()); + } + + /// With no walk pool the walk runs off-pool (so `par_map` stays + /// sequential); otherwise on a walk-pool thread. + #[tokio::test] + async fn walk_without_a_pool_runs_outside_rayon() { + { + let _off = test_hooks::DisablePool::new(); + assert_eq!(run_walk(rayon::current_thread_index).await, None); + } + assert!(run_walk(rayon::current_thread_index).await.is_some()); + } + /// The walk runs on a pool thread with the main thread's stack, not /// the 2 MiB default of rayon/tokio workers: a 4 MiB stack frame fits. #[tokio::test] From b563c2c13159eccbe2af3abadc4ebefce9414652 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 17:49:27 -0400 Subject: [PATCH 031/237] test(crawl): pin the store-entry identity_seen skip against the oracle A store entry whose own child's package.json disagrees with the entry's name@version, for a root-installed package, must have that child skipped by name as the sequential walk did; nothing failed when the merge dropped the skip. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/crawlers/npm_crawler/oracle.rs | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs b/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs index f92d62f4..9af2873d 100644 --- a/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs +++ b/crates/socket-patch-core/src/crawlers/npm_crawler/oracle.rs @@ -1366,6 +1366,85 @@ mod tests { assert!(nonempty > 32, "only {nonempty} non-empty trees"); } + /// A store entry whose own child's package.json disagrees with the + /// entry's name@version, for a root-installed package: the sequential + /// walk skips that child by name (`identity_seen`) without reading it, + /// so the foreign identity must never surface. + #[tokio::test] + async fn store_entry_child_with_a_foreign_identity_is_skipped_like_the_oracle() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + let nm = root.join("node_modules"); + let write = |dir: &Path, name: &str, version: &str| { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write( + dir.join("package.json"), + format!(r#"{{"name": "{name}", "version": "{version}"}}"#), + ) + .unwrap(); + }; + write(&nm.join("qux"), "qux", "1.0.0"); + write(&nm.join("@s").join("q"), "@s/q", "1.0.0"); + let store = nm.join(".pnpm"); + let q = store.join("qux@1.0.0").join("node_modules"); + write(&q.join("qux"), "qux", "9.9.9"); + // Below the skipped child is still walked. + write( + &q.join("qux").join("node_modules").join("inner"), + "inner", + "1.0.0", + ); + // A sibling of the skipped child is not skipped. + write(&q.join("sib"), "sib", "1.0.0"); + write( + &store + .join("@s+q@1.0.0") + .join("node_modules") + .join("@s") + .join("q"), + "@s/q", + "9.9.9", + ); + // Not root-installed: its child is read, foreign identity and all. + write( + &store.join("free@1.0.0").join("node_modules").join("free"), + "free", + "7.7.7", + ); + + assert_equivalent(&root, "foreign identity").await; + + let options = CrawlerOptions { + cwd: root.clone(), + global: false, + global_prefix: None, + }; + let purls: Vec = NpmCrawler::new() + .crawl_all(&options) + .await + .into_iter() + .map(|p| p.purl) + .collect(); + for present in [ + "pkg:npm/qux@1.0.0", + "pkg:npm/@s/q@1.0.0", + "pkg:npm/inner@1.0.0", + "pkg:npm/sib@1.0.0", + "pkg:npm/free@7.7.7", + ] { + assert!( + purls.iter().any(|p| p == present), + "missing {present}: {purls:?}" + ); + } + for absent in ["pkg:npm/qux@9.9.9", "pkg:npm/@s/q@9.9.9"] { + assert!( + !purls.iter().any(|p| p == absent), + "{absent} must be skipped: {purls:?}" + ); + } + } + /// With no walk pool (the OS refused every walk thread) the walk runs /// sequentially on the calling thread and still matches the oracle. #[tokio::test] From c85bbd7170eba13a127d4f25e355d6bda6d4fc3b Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 17:49:27 -0400 Subject: [PATCH 032/237] test(crawl): pin the resolver probe filter's always-probe components Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/crawlers/npm_crawler.rs | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/crates/socket-patch-core/src/crawlers/npm_crawler.rs b/crates/socket-patch-core/src/crawlers/npm_crawler.rs index 508c6dc5..bddeaaf4 100644 --- a/crates/socket-patch-core/src/crawlers/npm_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/npm_crawler.rs @@ -1779,6 +1779,54 @@ fn is_safe_npm_component(component: &str) -> bool { mod tests { use super::*; + fn listing_of(names: &[&str], complete: bool) -> Listing { + Listing { + entries: names + .iter() + .map(|name| ListedEntry { + name: OsString::from(name), + name_str: name.to_string(), + file_type: None, + }) + .collect(), + complete, + } + } + + /// A complete all-ASCII listing proves plain-ASCII absence (matched + /// ASCII-case-insensitively), but components a filesystem may resolve + /// to a differently spelled entry — `~` (8.3 short names), a trailing + /// `.`/space (Win32 stripping), non-ASCII (Unicode folding) — are + /// probed even when absent. + #[test] + fn probe_filter_only_skips_provably_absent_plain_ascii_names() { + let filter = ProbeFilter::new(&listing_of(&["foo", "Bar", "@scope"], true)); + assert!(filter.may_resolve("foo")); + assert!(filter.may_resolve("FOO")); + assert!(filter.may_resolve("bar")); + assert!(filter.may_resolve("@Scope")); + assert!(!filter.may_resolve("absent")); + assert!(!filter.may_resolve("fo")); + for alias in [ + "FOO~1", + "absent~2", + "absent.", + "absent ", + "foo.", + "caf\u{e9}", + "\u{212a}elvin", + ] { + assert!(filter.may_resolve(alias), "{alias:?} must be probed"); + } + + // An incomplete listing, or one holding a non-ASCII name, proves + // nothing: everything is probed. + let partial = ProbeFilter::new(&listing_of(&["foo"], false)); + assert!(partial.may_resolve("absent")); + let unicode = ProbeFilter::new(&listing_of(&["foo", "caf\u{e9}"], true)); + assert!(unicode.may_resolve("absent")); + } + #[test] fn test_parse_package_name_scoped() { let (ns, name) = parse_package_name("@types/node"); From b41efed0a610e12b3cb15c43fa946dc88202e3ae Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 17:49:28 -0400 Subject: [PATCH 033/237] test(cli): pin the one-at-a-time crawler dispatch under a tight fd limit The npm-only tree could not tell the serial dispatch from the concurrent one: the walk budget alone gives it one walk thread at ulimit -n 16. Add a tree where python, bundler and composer crawlers also find packages, and scan it at 12 on macOS (the serial dispatch is complete down to 11; running the crawlers concurrently drops 40-80 packages at 12). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/crawl_fd_limit_e2e.rs | 123 +++++++++++++++++- 1 file changed, 120 insertions(+), 3 deletions(-) diff --git a/crates/socket-patch-cli/tests/crawl_fd_limit_e2e.rs b/crates/socket-patch-cli/tests/crawl_fd_limit_e2e.rs index 6b9f529b..6b6379f9 100644 --- a/crates/socket-patch-cli/tests/crawl_fd_limit_e2e.rs +++ b/crates/socket-patch-cli/tests/crawl_fd_limit_e2e.rs @@ -7,9 +7,11 @@ //! concurrently) would silently drop packages under a limit the old walk //! handled. Below the walk pool's tight-limit threshold the crawl keeps //! the sequential descriptor profile; this suite pins that by scanning the -//! same tree under `ulimit -n 16` and under the inherited limit and -//! requiring byte-identical JSON. (The sequential walk scans this tree -//! fully at 14; with one walk thread per CPU it lost most of it at 16.) +//! same tree under a tight `ulimit -n` and under the inherited limit and +//! requiring byte-identical JSON. The npm-only tree pins the single walk +//! thread (the sequential walk scans it fully at 14; with one walk thread +//! per CPU it lost most of it at 16); the multi-ecosystem tree pins the +//! crawlers running one at a time. #![cfg(unix)] use std::path::{Path, PathBuf}; @@ -87,6 +89,58 @@ fn build_tree(root: &Path) -> usize { count } +/// [`build_tree`] plus installs for three more ecosystems — a Python +/// virtualenv, a Bundler `vendor/bundle` and a Composer `vendor/` — so the +/// crawlers that run alongside npm hold descriptors of their own: run +/// concurrently instead of one at a time, they need more than a tight +/// limit leaves. Returns the number of distinct packages it holds. +fn build_multi_ecosystem_tree(root: &Path) -> usize { + let mut count = build_tree(root); + let site = root + .join(".venv") + .join("lib") + .join("python3.11") + .join("site-packages"); + for i in 0..40 { + let dist = site.join(format!("pydist{i}-1.0.{i}.dist-info")); + std::fs::create_dir_all(&dist).unwrap(); + std::fs::write( + dist.join("METADATA"), + format!("Metadata-Version: 2.1\nName: pydist{i}\nVersion: 1.0.{i}\n\n"), + ) + .unwrap(); + count += 1; + } + let gems = root + .join("vendor") + .join("bundle") + .join("ruby") + .join("3.2.0"); + for i in 0..40 { + let gem = gems.join("gems").join(format!("rgem{i}-2.0.{i}")); + std::fs::create_dir_all(gem.join("lib")).unwrap(); + std::fs::write(gem.join("lib").join(format!("rgem{i}.rb")), "").unwrap(); + count += 1; + } + std::fs::create_dir_all(gems.join("specifications")).unwrap(); + let composer = root.join("vendor").join("composer"); + std::fs::create_dir_all(&composer).unwrap(); + let mut installed = Vec::new(); + for i in 0..20 { + let name = format!("acme/lib{i}"); + std::fs::create_dir_all(root.join("vendor").join(&name)).unwrap(); + installed.push(serde_json::json!({"name": name, "version": format!("3.0.{i}")})); + count += 1; + } + std::fs::write(root.join("composer.json"), "{}").unwrap(); + std::fs::write( + composer.join("installed.json"), + serde_json::json!({ "packages": installed }).to_string(), + ) + .unwrap(); + count +} + /// `scan --json` against an unreachable API (the crawl still runs and the /// JSON still reports what it found), optionally under `ulimit -n`. fn scan(root: &Path, nofile: Option) -> Output { @@ -118,6 +172,16 @@ fn scan(root: &Path, nofile: Option) -> Output { cmd.env_remove(&key); } } + // Keep the Python and Bundler discovery on the fixture's own installs. + for key in [ + "VIRTUAL_ENV", + "BUNDLE_PATH", + "BUNDLE_APP_CONFIG", + "GEM_HOME", + "GEM_PATH", + ] { + cmd.env_remove(key); + } cmd.output().unwrap() } @@ -150,3 +214,56 @@ fn tight_descriptor_limit_scans_the_same_packages() { ); assert_eq!(tight.status.code(), ample.status.code()); } + +/// The same, with every crawler finding packages: the tight-limit run +/// must crawl the ecosystems one at a time, as the sequential dispatch +/// did, or the concurrently running crawlers' descriptors crowd each other +/// out and packages go missing. At 16 that is headroom; at 12 (checked on +/// macOS, where it was measured: the sequential dispatch scans this tree +/// fully down to 11, while running the crawlers concurrently loses 40-80 +/// of its packages at 12) it is what pins the one-at-a-time dispatch. +#[test] +fn tight_descriptor_limit_scans_every_ecosystem_the_same() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + let expected = build_multi_ecosystem_tree(root); + + let ample = scan(root, None); + let ample_json: Value = serde_json::from_slice(&le.stdout).unwrap_or_else(|e| { + panic!( + "ample-limit scan printed no JSON ({e}); stderr:\n{}", + String::from_utf8_lossy(&le.stderr) + ) + }); + assert_eq!( + ample_json["scannedPackages"].as_u64(), + Some(expected as u64), + "{ample_json}" + ); + + let limits: &[u32] = if cfg!(target_os = "macos") { + // The concurrent crawlers' overlap is timing-dependent: repeat. + &[16, 12, 12, 12] + } else { + &[16] + }; + for &limit in limits { + let tight = scan(root, Some(limit)); + assert_ne!( + tight.status.code(), + Some(99), + "ulimit -n {limit} was refused" + ); + assert_eq!( + String::from_utf8_lossy(&tight.stdout), + String::from_utf8_lossy(&le.stdout), + "ulimit -n {limit} stderr:\n{}", + String::from_utf8_lossy(&tight.stderr) + ); + assert_eq!( + tight.status.code(), + ample.status.code(), + "ulimit -n {limit}" + ); + } +} From 4e9d3ec3c82ae8999812c6f632fa3ae1236b47d5 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 16:48:45 -0400 Subject: [PATCH 034/237] perf(hosted): settle only first attempts concurrently, retry wheels serially The concurrent wheel-metadata fetch let every in-flight download run its own retry loop. Against a host that serves one download at a time and 429s the rest with a shared Retry-After, the retries woke together, collided again and drained their budgets, so a redirect the serial loop makes was dropped as python_metadata_unavailable; the opt-in debug lines also interleaved across downloads. Now only first attempts run concurrently, with their debug lines held back and printed at the dep's fold. The first attempt the client would retry stops the fan-out: in-flight attempts are awaited (so the host is idle, as the serial loop finds it) and that dep plus every later one without a settled attempt is fetched one at a time with the full retry budget and Retry-After pacing. Outcomes, JSON and the debug stream match the serial loop's. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commands/scan/hosted.rs | 82 +++++--- .../tests/hosted_wheel_metadata_order.rs | 189 ++++++++++++++++-- crates/socket-patch-core/src/api/client.rs | 138 ++++++++++++- crates/socket-patch-core/src/vendor/pypi.rs | 54 +++++ 4 files changed, 406 insertions(+), 57 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 2a7e5c41..b480d0a9 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -1825,40 +1825,66 @@ pub(crate) async fn run_redirect_selected( wheel_deps.push((dep, sha256)); } } - // The wheels are fetched concurrently but folded in dep order, so - // `python_metadata`, `unavailable_python_artifacts` and `skipped` - // come out exactly as the serial loop's did. + // The wheels' FIRST attempts run concurrently and are folded in dep + // order, so `python_metadata`, `unavailable_python_artifacts` and + // `skipped` come out exactly as the serial loop's did; each attempt's + // opt-in debug lines are held back and printed at its fold, so they + // keep the serial order too. // - // Kept small on purpose: the gain is overlapped round trips, while - // each in-flight download buffers a whole wheel (up to - // MAX_VENDOR_PACKAGE_BYTES) under its own body timeout and retry - // budget, so peak memory and link sharing scale with this limit and - // a 429 `Retry-After` pauses only the download that received it. The - // status line names the dep whose result is being awaited (the - // baseline's messages, in the baseline's order), and the client's - // opt-in debug lines (GET / attempt-failed) interleave across the - // in-flight downloads. - // TODO(perf): let a `Retry-After` pause the whole fan-out rather - // than one fetch. + // An attempt the client would RETRY (a 429 / 5xx / transport + // failure) is never settled concurrently. At the first one no + // further attempt is started, the ones already in flight are awaited + // (so the host is idle again, as the serial loop would find it), and + // that dep plus every later one without a settled attempt are + // fetched one at a time with the full retry budget and `Retry-After` + // pacing, exactly as the serial loop fetched them. A rate-limited or + // struggling host therefore sees the serial loop's behavior from + // there on, and no outcome can be worse than the serial loop's. + // + // Kept small on purpose: each in-flight download buffers a whole + // wheel (up to MAX_VENDOR_PACKAGE_BYTES) under its own body timeout. const WHEEL_METADATA_CONCURRENCY: usize = 4; - let mut fetches = std::pin::pin!(ordered_concurrent( - wheel_deps.iter(), - WHEEL_METADATA_CONCURRENCY, - |&(dep, sha256)| { - socket_patch_core::vendor::pypi::fetch_hosted_wheel_metadata( - api_client, - &dep.artifact_url, - sha256, - ) - }, - )); - for &(dep, _) in &wheel_deps { + use futures_util::StreamExt as _; + use socket_patch_core::vendor::pypi::{ + fetch_hosted_wheel_metadata, try_fetch_hosted_wheel_metadata_once, + HostedWheelMetadataAttempt, + }; + // Lazily built: a dep's attempt starts only once it is pulled here. + let mut unstarted = wheel_deps.iter().map(|&(dep, sha256)| { + try_fetch_hosted_wheel_metadata_once(api_client, &dep.artifact_url, sha256) + }); + let mut in_flight: futures_util::stream::FuturesOrdered<_> = unstarted + .by_ref() + .take(WHEEL_METADATA_CONCURRENCY) + .collect(); + // Once serial: the attempts that were in flight when a dep needed a + // retry, in dep order (the deps after them were never started). + let mut drained: Option> = None; + for &(dep, sha256) in &wheel_deps { status.set(format!( "Fetching hosted wheel metadata for {}...", dep.name )); - let Some(fetched) = fetches.next().await else { - break; + let settled = match drained.as_mut() { + Some(drained) => drained.pop_front(), + None => match in_flight.next().await { + Some(HostedWheelMetadataAttempt::Retry) | None => { + let mut rest = std::collections::VecDeque::new(); + while let Some(later) = in_flight.next().await { + rest.push_back(later); + } + drained = Some(rest); + None + } + Some(settled) => { + in_flight.extend(unstarted.next()); + Some(settled) + } + }, + }; + let fetched = match settled.and_then(|settled| settled.into_result()) { + Some(fetched) => fetched, + None => fetch_hosted_wheel_metadata(api_client, &dep.artifact_url, sha256).await, }; match fetched { Ok(Some(metadata)) => { diff --git a/crates/socket-patch-cli/tests/hosted_wheel_metadata_order.rs b/crates/socket-patch-cli/tests/hosted_wheel_metadata_order.rs index f87b8da9..a52b0d70 100644 --- a/crates/socket-patch-cli/tests/hosted_wheel_metadata_order.rs +++ b/crates/socket-patch-cli/tests/hosted_wheel_metadata_order.rs @@ -144,12 +144,60 @@ impl Respond for RecordArrival { } } -/// Discovery, per-package search and the reference grants for all of PKGS; -/// wheels: `aaa` and `ccc` serve valid bytes (reversed delays), `bbb` is a -/// SLOW 404 and `ddd` serves bytes that do not match the granted sha256. -/// Returns the wheel-request arrival log. +/// A serve host that handles ONE download at a time: a wheel request that +/// arrives while another is still being answered gets `429 Retry-After: 1` +/// (a per-token rate limiter / CDN cap). Records every arrival and status. +struct OneAtATime { + name: &'static str, + body: Vec, + busy_until: Arc>, + log: Arc>>, +} + +impl Respond for OneAtATime { + fn respond(&self, _: &Request) -> ResponseTemplate { + let now = Instant::now(); + let mut busy_until = self.busy_until.lock().unwrap(); + let (status, template) = if now < *busy_until { + ( + 429, + ResponseTemplate::new(429).insert_header("Retry-After", "1"), + ) + } else { + let delay = Duration::from_millis(200); + *busy_until = now + delay; + ( + 200, + ResponseTemplate::new(200) + .set_body_bytes(self.body.clone()) + .set_delay(delay), + ) + }; + self.log.lock().unwrap().push((self.name, status)); + template + } +} + +/// Which wheel host [`mock_api`] mounts. +enum WheelHost { + /// `aaa` and `ccc` serve valid bytes (reversed delays), `bbb` is a SLOW + /// 404 and `ddd` serves bytes that do not match the granted sha256. + Mixed(Arrivals), + /// Every wheel is valid, behind a [`OneAtATime`] host. + OneAtATime(Arc>>), +} + +/// Discovery, per-package search and the reference grants for all of PKGS, +/// with the wheels served per [`WheelHost::Mixed`]. Returns the +/// wheel-request arrival log. async fn mock_api(server: &MockServer) -> Arrivals { let arrivals: Arrivals = Arc::default(); + mock_api_with(server, WheelHost::Mixed(arrivals.clone())).await; + arrivals +} + +async fn mock_api_with(server: &MockServer, host: WheelHost) { + let busy_until = Arc::new(Mutex::new(Instant::now())); let patch = |name: &str, version: &str, uuid: &str| { json!({ "uuid": uuid, "purl": purl(name, version), "tier": "free", @@ -190,23 +238,38 @@ async fn mock_api(server: &MockServer) -> Arrivals { let file = wheel_file(name, version); let url = format!("{}/wheels/{file}", server.uri()); let (bytes, sha256) = build_wheel(name, version); - let delay = Duration::from_millis([600, 900, 0, 0][i]); - let response = match name { - "bbb-pkg" => ResponseTemplate::new(404), - "ddd-pkg" => { - ResponseTemplate::new(200).set_body_bytes(b"not the granted wheel".to_vec()) + let wheel = Mock::given(method("GET")).and(path(format!("/wheels/{file}"))); + match &host { + WheelHost::Mixed(arrivals) => { + let delay = Duration::from_millis([600, 900, 0, 0][i]); + let response = match name { + "bbb-pkg" => ResponseTemplate::new(404), + "ddd-pkg" => { + ResponseTemplate::new(200).set_body_bytes(b"not the granted wheel".to_vec()) + } + _ => ResponseTemplate::new(200).set_body_bytes(bytes), + }; + wheel + .respond_with(RecordArrival { + name, + template: response.set_delay(delay), + arrivals: arrivals.clone(), + }) + .mount(server) + .await; } - _ => ResponseTemplate::new(200).set_body_bytes(bytes), - }; - Mock::given(method("GET")) - .and(path(format!("/wheels/{file}"))) - .respond_with(RecordArrival { - name, - template: response.set_delay(delay), - arrivals: arrivals.clone(), - }) - .mount(server) - .await; + WheelHost::OneAtATime(log) => { + wheel + .respond_with(OneAtATime { + name, + body: bytes, + busy_until: busy_until.clone(), + log: log.clone(), + }) + .mount(server) + .await; + } + } results.insert( uuid.to_string(), json!({ @@ -223,7 +286,6 @@ async fn mock_api(server: &MockServer) -> Arrivals { .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "results": results }))) .mount(server) .await; - arrivals } #[tokio::test] @@ -316,3 +378,88 @@ async fn wheel_metadata_failures_fold_in_dep_order() { bbb.saturating_duration_since(aaa) ); } + +/// A host that serves one download at a time and 429s the rest must end up +/// with the one-at-a-time loop's outcome: every wheel's metadata fetched, +/// nothing skipped. Concurrent attempts that share a `Retry-After` wake up +/// together and collide again, so letting each one retry on its own drains +/// the budgets and drops a redirect the serial loop makes; a retryable +/// failure must hand the rest of the fan-out back to the serial loop. The +/// opt-in debug stream must also match the serial loop's: one wheel GET per +/// dep, in dep order, and no failed attempt (the serial loop never collides). +#[tokio::test] +async fn rate_limited_wheel_host_matches_the_serial_outcome() { + let server = MockServer::start().await; + let log: Arc>> = Arc::default(); + mock_api_with(&server, WheelHost::OneAtATime(log.clone())).await; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + write_uv_project(&root); + + let cwd = root.to_str().unwrap().to_string(); + let api = server.uri(); + let (code, stdout, stderr) = common::run_with_env( + &root, + &[ + "scan", + "--mode", + "hosted", + "--dry-run", + "--json", + "--cwd", + &cwd, + "--api-url", + &api, + "--org", + ORG, + "--api-token", + "fake", + ], + &[("SOCKET_DEBUG", "1")], + ); + let doc: Value = serde_json::from_str(&stdout) + .unwrap_or_else(|e| panic!("JSON envelope ({e}):\n{stdout}\n{stderr}")); + assert_eq!(code, 0, "{doc:#}\n{stderr}"); + let log = log.lock().unwrap().clone(); + assert_eq!( + doc["redirect"]["redirected"], + PKGS.len(), + "every wheel redirects, as in the serial loop: {doc:#}\nwheel requests: {log:?}" + ); + let metadata_skips: Vec<&Value> = doc["redirect"]["skipped"] + .as_array() + .map(|skipped| { + skipped + .iter() + .filter(|s| s["reason"] == "python_metadata_unavailable") + .collect() + }) + .unwrap_or_default(); + assert!( + metadata_skips.is_empty(), + "no wheel may be skipped: {metadata_skips:?}\nwheel requests: {log:?}" + ); + + let wheel_debug: Vec<&str> = stderr + .lines() + .filter(|line| line.starts_with("[socket-patch debug]") && line.contains("/wheels/")) + .collect(); + let expected: Vec = PKGS + .iter() + .map(|(name, version, _)| { + format!( + "[socket-patch debug] GET vendor package {}/wheels/{}", + server.uri(), + wheel_file(name, version) + ) + }) + .collect(); + assert_eq!( + wheel_debug, expected, + "debug lines must follow the serial loop's order:\n{stderr}" + ); + assert!( + !stderr.contains("vendor package download attempt"), + "the serial loop never collides, so no attempt fails:\n{stderr}" + ); +} diff --git a/crates/socket-patch-core/src/api/client.rs b/crates/socket-patch-core/src/api/client.rs index dff30425..2c628ea5 100644 --- a/crates/socket-patch-core/src/api/client.rs +++ b/crates/socket-patch-core/src/api/client.rs @@ -77,7 +77,45 @@ fn status_error(head: &str, status: StatusCode, text: &str) -> String { /// Log debug messages when debug mode is enabled. fn debug_log(message: &str) { if is_debug_enabled() { - eprintln!("[socket-patch debug] {}", message); + let line = format!("[socket-patch debug] {}", message); + let mut line = Some(line); + let deferred = DEFERRED_DEBUG.try_with(|lines| { + if let Some(line) = line.take() { + lines.borrow_mut().push(line); + } + }); + if deferred.is_err() { + if let Some(line) = line { + eprintln!("{line}"); + } + } + } +} + +tokio::task_local! { + /// Set around a speculative (concurrent) request so its debug lines are + /// held back and the caller can print them, or drop them, in the order a + /// one-at-a-time loop would have produced. + static DEFERRED_DEBUG: std::cell::RefCell>; +} + +/// Run `fut` with its [`debug_log`] lines captured instead of printed; +/// returns its output and the captured lines, in emission order. +pub(crate) async fn with_deferred_debug( + fut: impl std::future::Future, +) -> (T, Vec) { + DEFERRED_DEBUG + .scope(std::cell::RefCell::new(Vec::new()), async move { + let out = fut.await; + (out, DEFERRED_DEBUG.with(|lines| lines.take())) + }) + .await +} + +/// Print lines captured by [`with_deferred_debug`] (already prefixed). +pub(crate) fn flush_deferred_debug(lines: Vec) { + for line in lines { + eprintln!("{line}"); } } @@ -1283,19 +1321,37 @@ impl ApiClient { /// integrity. A 404/410/408 surfaces as an error (a secondary the /// reference promised should be present). pub(crate) async fn download_artifact(&self, url: &str) -> Result, ApiError> { - match self.download_vendor_archive(url).await { - ServeDownload::Ok(bytes) => Ok(bytes), - ServeDownload::NotFound => Err(ApiError::Other(format!("artifact not found: {url}"))), - ServeDownload::Pending => { - Err(ApiError::Other(format!("artifact still building: {url}"))) - } - ServeDownload::Failed(e) => Err(e), + artifact_download_result(self.download_vendor_archive(url).await, url) + } + + /// The first attempt of [`Self::download_artifact`] alone: `Some` with + /// exactly the result `download_artifact` would return when that attempt + /// settles it (success, or a failure it would not retry), `None` when it + /// would retry. A caller that gets `None` runs `download_artifact` from + /// scratch, so it keeps the full retry budget and `Retry-After` pacing. + pub(crate) async fn download_artifact_first_attempt( + &self, + url: &str, + ) -> Option, ApiError>> { + match self.download_vendor_archive_once(url).await { + (ServeDownload::Failed(_), Some(_)) if self.vendor_retry.attempts.max(1) > 1 => None, + (outcome, _) => Some(artifact_download_result(outcome, url)), } } } // ── Free functions ──────────────────────────────────────────────────── +/// [`ApiClient::download_artifact`]'s mapping of a serve outcome. +fn artifact_download_result(outcome: ServeDownload, url: &str) -> Result, ApiError> { + match outcome { + ServeDownload::Ok(bytes) => Ok(bytes), + ServeDownload::NotFound => Err(ApiError::Other(format!("artifact not found: {url}"))), + ServeDownload::Pending => Err(ApiError::Other(format!("artifact still building: {url}"))), + ServeDownload::Failed(e) => Err(e), + } +} + /// Cap on a single prebuilt-archive download (defensive bound against a /// runaway / hostile serve response). Generous enough for any real package. const MAX_VENDOR_PACKAGE_BYTES: u64 = 256 * 1024 * 1024; @@ -4108,6 +4164,72 @@ mod vendor_retry_tests { .with_vendor_retry(policy) } + /// `download_artifact_first_attempt` settles exactly what + /// `download_artifact` would return on its first attempt, and declines + /// (`None`, one request, no backoff) wherever `download_artifact` would + /// retry — unless the policy has no retry to give, where it settles. + #[tokio::test] + async fn first_attempt_settles_or_defers_like_download_artifact() { + let server = MockServer::start().await; + for (route, status) in [("/ok", 200), ("/gone", 404), ("/busy", 429), ("/down", 503)] { + Mock::given(method("GET")) + .and(path(route)) + .respond_with(ResponseTemplate::new(status).set_body_bytes(BYTES.to_vec())) + .mount(&server) + .await; + } + let url = |route: &str| format!("{}{route}", server.uri()); + let retrying = client(&server.uri(), fast()); + + let ok = retrying.download_artifact_first_attempt(&url("/ok")).await; + assert_eq!(ok.expect("200 settles").expect("200 is Ok"), BYTES); + let gone = retrying + .download_artifact_first_attempt(&url("/gone")) + .await + .expect("404 is terminal, so it settles") + .expect_err("404 is an error"); + let full = retrying + .download_artifact(&url("/gone")) + .await + .expect_err("404 is an error"); + assert_eq!(gone.to_string(), full.to_string()); + for route in ["/busy", "/down"] { + assert!( + retrying + .download_artifact_first_attempt(&url(route)) + .await + .is_none(), + "{route} is retried by download_artifact, so it defers" + ); + } + let gets = |route: &'static str| { + let server = &server; + async move { + server + .received_requests() + .await + .unwrap_or_default() + .iter() + .filter(|r| r.url.path() == route) + .count() + } + }; + assert_eq!(gets("/busy").await, 1, "a deferral makes one request"); + assert_eq!(gets("/down").await, 1, "a deferral makes one request"); + + let single = client(&server.uri(), VendorRetryPolicy::none()); + let settled = single + .download_artifact_first_attempt(&url("/down")) + .await + .expect("with no retry budget the first attempt is final") + .expect_err("503 is an error"); + let full = single + .download_artifact(&url("/down")) + .await + .expect_err("503 is an error"); + assert_eq!(settled.to_string(), full.to_string()); + } + fn granted(server: &MockServer, uuid: &str) -> ResponseTemplate { let url = format!("{}{SERVE}", server.uri()); let sri = format!( diff --git a/crates/socket-patch-core/src/vendor/pypi.rs b/crates/socket-patch-core/src/vendor/pypi.rs index 48a1ba98..ab8d539d 100644 --- a/crates/socket-patch-core/src/vendor/pypi.rs +++ b/crates/socket-patch-core/src/vendor/pypi.rs @@ -136,6 +136,60 @@ pub async fn fetch_hosted_wheel_metadata( decode_hosted_wheel_metadata(&bytes, sha256) } +/// One speculative [`fetch_hosted_wheel_metadata`], for running several +/// concurrently while keeping the one-at-a-time loop's outcomes. +pub enum HostedWheelMetadataAttempt { + /// Settled on the first attempt: [`Self::into_result`] gives exactly + /// what `fetch_hosted_wheel_metadata` would have returned. + Settled { + result: Result, String>, + debug: Vec, + }, + /// The first attempt failed in a way `fetch_hosted_wheel_metadata` + /// retries; the caller must run that instead (with a fresh budget). + Retry, +} + +impl HostedWheelMetadataAttempt { + /// The settled result, printing the attempt's held-back debug lines — + /// call it where the one-at-a-time loop would have fetched this wheel. + pub fn into_result(self) -> Option, String>> { + match self { + Self::Settled { result, debug } => { + crate::api::client::flush_deferred_debug(debug); + Some(result) + } + Self::Retry => None, + } + } +} + +/// [`fetch_hosted_wheel_metadata`]'s first attempt only, with its debug +/// lines held back (see [`HostedWheelMetadataAttempt`]). +pub async fn try_fetch_hosted_wheel_metadata_once( + client: &ApiClient, + url: &str, + sha256: &str, +) -> HostedWheelMetadataAttempt { + if let Err(error) = validate_hosted_wheel_sha256(sha256) { + return HostedWheelMetadataAttempt::Settled { + result: Err(error), + debug: Vec::new(), + }; + } + let (attempt, debug) = + crate::api::client::with_deferred_debug(client.download_artifact_first_attempt(url)).await; + match attempt { + None => HostedWheelMetadataAttempt::Retry, + Some(downloaded) => HostedWheelMetadataAttempt::Settled { + result: downloaded + .map_err(|error| format!("cannot fetch hosted wheel metadata: {error}")) + .and_then(|bytes| decode_hosted_wheel_metadata(&bytes, sha256)), + debug, + }, + } +} + const SETUP_ALTERNATIVE: &str = "use the `socket-patch setup` .pth install hook instead, which patches installed \ site-packages without lockfile edits"; From 25a579510aa2cd0d1aed697e42bf7692bfc5d0a7 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 16:48:46 -0400 Subject: [PATCH 035/237] test(redirect): require the random pnpm sweep to reach every outcome The randomized oracle sweep now asserts it produced edits, refusals, a duplicate that re-reads the prior rewrite and each residual warning code, so a generator change cannot silently narrow it to the plain rewrite path. The vendored-marker scan materializes pending splices in its own loop instead of inside the any() predicate. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/patch/redirect/mod.rs | 4 ++- .../patch/redirect/pnpm_equivalence_tests.rs | 36 ++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/crates/socket-patch-core/src/patch/redirect/mod.rs b/crates/socket-patch-core/src/patch/redirect/mod.rs index b836db3f..0034a519 100644 --- a/crates/socket-patch-core/src/patch/redirect/mod.rs +++ b/crates/socket-patch-core/src/patch/redirect/mod.rs @@ -3007,8 +3007,10 @@ fn rewrite_pnpm_lock( let v9_vendored_key = format!("{fname}@file:"); let override_key = format!("{fname}@{}", dep.version); // Scanned over the post-splice text, so fold pending splices in. - let vendored = locks.iter_mut().any(|lock| { + for lock in locks.iter_mut() { lock.materialize(); + } + let vendored = locks.iter().any(|lock| { lock.text.lines().any(|line| { let t = line.trim_start(); let t = t.strip_prefix('\'').unwrap_or(t); diff --git a/crates/socket-patch-core/src/patch/redirect/pnpm_equivalence_tests.rs b/crates/socket-patch-core/src/patch/redirect/pnpm_equivalence_tests.rs index c972ad30..855e6d56 100644 --- a/crates/socket-patch-core/src/patch/redirect/pnpm_equivalence_tests.rs +++ b/crates/socket-patch-core/src/patch/redirect/pnpm_equivalence_tests.rs @@ -466,6 +466,10 @@ fn indexed_pnpm_rewrite_matches_oracle_on_random_lock_sets() { Flavor::V51, Flavor::EarlyShrinkwrap, ]; + // Which outcomes the sweep actually reached, so a generator change that + // stops producing the refusal / residual / duplicate shapes fails here + // instead of quietly testing only the plain rewrite path. + let mut outcomes: std::collections::BTreeSet = std::collections::BTreeSet::new(); for seed in 1..=300u64 { let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); let lock_count = 1 + rng.below(3); @@ -549,7 +553,37 @@ fn indexed_pnpm_rewrite_matches_oracle_on_random_lock_sets() { overrides.push(dep(&p, i + 100, tag, Some(&sri("DUP")))); } } - assert_equivalent(&files, &overrides); + let r = assert_equivalent(&files, &overrides); + if !r.edits.is_empty() { + outcomes.insert("edit".into()); + } + if !r.refused_pnpm_uuids.is_empty() { + outcomes.insert("refused".into()); + } + outcomes.extend(r.warnings.iter().map(|w| w.code.clone())); + if r.edits.iter().enumerate().any(|(i, e)| { + r.edits[..i] + .iter() + .any(|prior| prior.path == e.path && prior.key == e.key && prior.new == e.original) + }) { + outcomes.insert("duplicate_sees_prior".into()); + } + } + // (`redirect_pnpm_entry_vendored` is not generated here; the + // depscan-sized sweep above requires it.) + for want in [ + "edit", + "refused", + "duplicate_sees_prior", + "redirect_pnpm_entry_not_found", + "redirect_pnpm_legacy_lockfile_unsupported", + "redirect_pnpm_missing_sha512", + "redirect_pnpm_unsupported_lock_key", + ] { + assert!( + outcomes.contains(want), + "sweep never reached {want}: {outcomes:?}" + ); } } From 9c5d3e7264f9a63ab483bb03c305b110f4798c49 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 16:05:25 -0400 Subject: [PATCH 036/237] perf(scripts): let bench.sh take explicit ports and refuse busy ones PATCH_PORT and PROXY_PORT can now be set independently of PORT (they still default to PORT+1 / PORT+2), and bench.sh exits 2 before starting replay.py when any of its three ports is already listening, instead of colliding with (or tempting someone to kill) another bench's stand-in. Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/perf/README.md | 3 ++- scripts/perf/bench.sh | 14 +++++++++--- scripts/tests/test_perf_harness.py | 34 ++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/scripts/perf/README.md b/scripts/perf/README.md index 91796365..af571dc9 100644 --- a/scripts/perf/README.md +++ b/scripts/perf/README.md @@ -48,7 +48,8 @@ CWD=$P BASE=$S/socket-patch-base NEW=./target/release/socket-patch PORT=18200 \ ``` Use a different `PORT` for each concurrent bench (it takes `PORT` through -`PORT+2`). Latency `0` isolates local work such as the crawl and rewrites. +`PORT+2`, or set `PATCH_PORT` / `PROXY_PORT` explicitly). `bench.sh` refuses +to start if any of them is already listening, rather than killing it. Latency `0` isolates local work such as the crawl and rewrites. `recorded` gives realistic totals. `FILL=1` forwards and records requests the store doesn't have yet, for when a change adds endpoints. diff --git a/scripts/perf/bench.sh b/scripts/perf/bench.sh index f2601631..fab37504 100755 --- a/scripts/perf/bench.sh +++ b/scripts/perf/bench.sh @@ -16,8 +16,9 @@ # CWD=/path/to/project (required; passed as --cwd) # BIN=/path/to/binary (record/replay; default: /target/release/socket-patch) # BASE=... NEW=... (ab; the two binaries to compare) -# PORT=18080 (api.socket.dev stand-in; PORT+1 = patch.socket.dev, -# PORT+2 = patches-api.socket.dev public proxy) +# PORT=18080 (api.socket.dev stand-in) +# PATCH_PORT=PORT+1 (patch.socket.dev stand-in) +# PROXY_PORT=PORT+2 (patches-api.socket.dev public proxy stand-in) # CONN_MS=0 (replay/ab: extra latency per new TCP connection) # FILL=1 (replay/ab: forward + record misses instead of 599) # PRE_RUN='cmd' (shell command run before every CLI invocation, @@ -63,7 +64,14 @@ case "$STORE/" in esac mkdir -p "$STORE" -PORT="${PORT:-18080}"; PATCH_PORT="$((PORT + 1))"; PROXY_PORT="$((PORT + 2))" +PORT="${PORT:-18080}"; PATCH_PORT="${PATCH_PORT:-$((PORT + 1))}"; PROXY_PORT="${PROXY_PORT:-$((PORT + 2))}" +# Refuse (never kill) a port someone else is listening on: it is usually +# another bench, and killing it would corrupt that run. +for p in "$PORT" "$PATCH_PORT" "$PROXY_PORT"; do + if lsof -nP -iTCP:"$p" -sTCP:LISTEN >/dev/null 2>&1; then + echo "port $p is already in use (another bench?): pick another PORT; do not kill it" >&2; exit 2 + fi +done OUT="${OUT:-$STORE/runs}"; mkdir -p "$OUT" lat_args=(--latency-ms 0); lat_tag="${LAT}ms" diff --git a/scripts/tests/test_perf_harness.py b/scripts/tests/test_perf_harness.py index 23182b2f..91d126a4 100644 --- a/scripts/tests/test_perf_harness.py +++ b/scripts/tests/test_perf_harness.py @@ -282,6 +282,40 @@ def test_refuses_a_store_inside_the_repository(self): finally: shutil.rmtree(inside, ignore_errors=True) + def test_refuses_a_busy_port_without_killing_its_listener(self): + port = free_port_run(3) + with socket.socket() as busy: + busy.bind(('127.0.0.1', port + 2)) + busy.listen() + r = self.bench('replay', str(self.root / 'store'), '--', 'scan', + BIN=shutil.which('true'), PORT=str(port)) + self.assertEqual(r.returncode, 2, r.stdout + r.stderr) + self.assertIn(f'port {port + 2} is already in use', r.stderr) + # The other bench's listener is still there and still accepting. + with socket.create_connection(('127.0.0.1', port + 2), timeout=5): + pass + self.assertFalse((self.root / 'out' / 'proxy.log').exists(), + 'replay.py must not start when a port is refused') + + def test_patch_and_proxy_ports_can_be_set_explicitly(self): + # Three distinct, typically non-consecutive ports: hold all three + # sockets open while the OS picks them. + socks = [socket.socket() for _ in range(3)] + try: + for s in socks: + s.bind(('127.0.0.1', 0)) + port, patch_port, proxy_port = (s.getsockname()[1] for s in socks) + finally: + for s in socks: + s.close() + r = self.bench('replay', str(self.root / 'store'), '0', '1', '--', 'scan', + BIN=shutil.which('true'), PORT=str(port), + PATCH_PORT=str(patch_port), PROXY_PORT=str(proxy_port)) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + log = (self.root / 'out' / 'proxy.log').read_text() + for p in (port, patch_port, proxy_port): + self.assertIn(f'127.0.0.1:{p}', log) + def test_ab_checks_stdout_sha_against_the_first_base_run(self): # Seed the store with the batch the fake CLI sends (unit tests never # reach the real services). From 167f684c8c2f65a8ecd3f60600d0653b64dd6d39 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 18:29:03 -0400 Subject: [PATCH 037/237] fix(scan): request one at a time under a tight descriptor limit Every in-flight request holds its own socket, and the serial loops never held more than one. Under a soft RLIMIT_NOFILE the crawl already treats as tight (walk_pool::fd_limit_is_tight), the concurrent batch window opened enough sockets to fail with EMFILE where the serial loop got the server's own answer. crawl_fd_limit_e2e caught this at ulimit -n 12 once perf/wp1 and perf/wp2 met. api_concurrency and the wheel-metadata fan-out now drop to 1 under that limit, restoring the serial descriptor profile. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commands/scan/hosted.rs | 11 ++++++-- .../socket-patch-core/src/utils/concurrent.rs | 25 ++++++++++++++++++- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index b480d0a9..c53ceadf 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -1843,7 +1843,14 @@ pub(crate) async fn run_redirect_selected( // // Kept small on purpose: each in-flight download buffers a whole // wheel (up to MAX_VENDOR_PACKAGE_BYTES) under its own body timeout. - const WHEEL_METADATA_CONCURRENCY: usize = 4; + // Under a tight descriptor limit it is 1, like the API loops (see + // `api_concurrency`): the serial loop held one socket at a time. + let wheel_metadata_concurrency = + if socket_patch_core::crawlers::walk_pool::fd_limit_is_tight() { + 1 + } else { + 4 + }; use futures_util::StreamExt as _; use socket_patch_core::vendor::pypi::{ fetch_hosted_wheel_metadata, try_fetch_hosted_wheel_metadata_once, @@ -1855,7 +1862,7 @@ pub(crate) async fn run_redirect_selected( }); let mut in_flight: futures_util::stream::FuturesOrdered<_> = unstarted .by_ref() - .take(WHEEL_METADATA_CONCURRENCY) + .take(wheel_metadata_concurrency) .collect(); // Once serial: the attempts that were in flight when a dep needed a // retry, in dep order (the deps after them were never started). diff --git a/crates/socket-patch-core/src/utils/concurrent.rs b/crates/socket-patch-core/src/utils/concurrent.rs index 64c80e82..43cbdbef 100644 --- a/crates/socket-patch-core/src/utils/concurrent.rs +++ b/crates/socket-patch-core/src/utils/concurrent.rs @@ -28,8 +28,23 @@ pub const PROXY_API_CONCURRENCY: usize = 4; /// The in-flight cap for a client on the public proxy (`true`) or the /// authenticated API (`false`). +/// +/// Under a tight `RLIMIT_NOFILE` +/// ([`crate::crawlers::walk_pool::fd_limit_is_tight`]) the cap is 1: each +/// in-flight request holds its own socket, and the serial loop never held +/// more than one, so extra connections could fail with `EMFILE` where the +/// serial loop's single connection succeeded (or failed differently). pub fn api_concurrency(use_public_proxy: bool) -> usize { - if use_public_proxy { + api_concurrency_under( + use_public_proxy, + crate::crawlers::walk_pool::fd_limit_is_tight(), + ) +} + +fn api_concurrency_under(use_public_proxy: bool, fd_limit_is_tight: bool) -> usize { + if fd_limit_is_tight { + 1 + } else if use_public_proxy { PROXY_API_CONCURRENCY } else { API_CONCURRENCY @@ -58,6 +73,14 @@ where #[cfg(test)] mod tests { + #[test] + fn a_tight_descriptor_limit_runs_requests_one_at_a_time() { + assert_eq!(api_concurrency_under(false, false), API_CONCURRENCY); + assert_eq!(api_concurrency_under(true, false), PROXY_API_CONCURRENCY); + assert_eq!(api_concurrency_under(false, true), 1); + assert_eq!(api_concurrency_under(true, true), 1); + } + use super::*; use std::cell::Cell; use std::time::Duration; From 59d5e25a653abc8624fa2b1cd26e96da98780008 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 19:00:48 -0400 Subject: [PATCH 038/237] perf(get): fetch patch views concurrently ahead of the download loop The shared download loop (agent and vendored engines), the release- variant narrowing, the vendor stager's content top-up and scan's baseline pre-verification each awaited one patch-view GET at a time. Each now plans the exact views its loop fetches (same refusal, ledger and held-view checks, over inputs the loop never mutates), runs them through the ordered concurrency helper and takes the next result where it used to await the request. Results fold in selection order, and each request's --debug lines are held back and printed at its old turn (new core HeldBack/hold_back_debug), so stdout, stderr and the JSON records match the serial loop. vex's record fetch swaps its chunk-at-a-time JoinSet for a sliding window of the same size, consumed in input order. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commands/fetch_stage.rs | 20 +- crates/socket-patch-cli/src/commands/get.rs | 282 ++++++++++++++++-- .../src/commands/scan/discovery.rs | 69 +++-- .../src/commands/vex_sources.rs | 44 +-- crates/socket-patch-core/src/api/client.rs | 26 ++ 5 files changed, 380 insertions(+), 61 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/fetch_stage.rs b/crates/socket-patch-cli/src/commands/fetch_stage.rs index 3179bab5..dccba610 100644 --- a/crates/socket-patch-cli/src/commands/fetch_stage.rs +++ b/crates/socket-patch-cli/src/commands/fetch_stage.rs @@ -10,13 +10,15 @@ use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; +use futures_util::StreamExt; use socket_patch_core::api::blob_fetcher::{ fetch_missing_blobs, fetch_missing_sources, get_missing_archives, get_missing_blobs, DownloadMode, FetchMissingBlobsResult, }; -use socket_patch_core::api::client::{get_api_client_with_overrides, ApiClient}; +use socket_patch_core::api::client::{get_api_client_with_overrides, hold_back_debug, ApiClient}; use socket_patch_core::manifest::schema::{PatchManifest, PatchRecord}; use socket_patch_core::patch::apply::{is_valid_blob_hash, PatchSources}; +use socket_patch_core::utils::concurrent::{api_concurrency, ordered_concurrent}; use tempfile::TempDir; use super::get::base64_decode; @@ -565,7 +567,16 @@ pub(crate) async fn stage_vendor_sources_in_memory( } }; let mut failed: Vec<&str> = Vec::new(); - for (i, (purl, uuid)) in to_fetch.iter().enumerate() { + // The views are fetched concurrently (at most `api_concurrency` in + // flight) but consumed in `to_fetch` order, each request's `--debug` + // lines released at its turn, so `mem`, `failed` and every error + // line fold exactly as the serial loop's did. + let mut views = std::pin::pin!(ordered_concurrent( + to_fetch.iter(), + api_concurrency(client.uses_public_proxy()), + |(_, uuid)| hold_back_debug(client.fetch_patch(uuid)), + )); + for (i, (purl, _)) in to_fetch.iter().enumerate() { if to_fetch.len() > 1 { status.set(format!( "{} ({}/{})", @@ -574,7 +585,10 @@ pub(crate) async fn stage_vendor_sources_in_memory( to_fetch.len() )); } - match client.fetch_patch(uuid).await { + let Some(view) = views.next().await else { + break; + }; + match view.release() { Ok(Some(patch)) => { let mut complete = true; for (file, info) in &patch.files { diff --git a/crates/socket-patch-cli/src/commands/get.rs b/crates/socket-patch-cli/src/commands/get.rs index 2e32ee27..8c182df9 100644 --- a/crates/socket-patch-cli/src/commands/get.rs +++ b/crates/socket-patch-cli/src/commands/get.rs @@ -1,8 +1,9 @@ use clap::Args; +use futures_util::StreamExt; use regex::Regex; use socket_patch_core::api::client::{ - build_proxy_fallback_client, get_api_client_with_overrides, is_fallback_candidate, ApiClient, - ApiError, + build_proxy_fallback_client, get_api_client_with_overrides, hold_back_debug, + is_fallback_candidate, ApiClient, ApiError, }; use socket_patch_core::api::ranking::{cmp_search_results, severity_order}; use socket_patch_core::api::types::{ @@ -17,6 +18,7 @@ use socket_patch_core::manifest::schema::{ use socket_patch_core::patch::apply::{is_valid_blob_hash, select_installed_variants}; use socket_patch_core::patch::apply_lock::{LockError, LockGuard}; use socket_patch_core::telemetry::{track_patch_fetch_failed, track_patch_fetched}; +use socket_patch_core::utils::concurrent::{api_concurrency, ordered_concurrent}; use socket_patch_core::utils::purl::{ canonical_purl, is_purl, normalize_purl, strip_purl_qualifiers, }; @@ -1359,6 +1361,22 @@ async fn filter_to_installed_releases( let partitioned = partition_purls(&all_qualified, None); let paths = find_packages_for_rollback(&partitioned, crawler_options, true).await; + // Every installed base's variant views, fetched concurrently (at most + // `api_concurrency` in flight) in the order the loop below consumes + // them: bases in `multi` order, skipping the uninstalled ones, each + // base's variants in order. Nothing here prints between fetches, and + // each request's `--debug` lines are released at its old turn. + let installed_variants: Vec = multi + .iter() + .filter(|(_, variants)| variants.iter().any(|s| paths.contains_key(&s.purl))) + .flat_map(|(_, variants)| variants.iter().map(|s| s.uuid.clone())) + .collect(); + let mut variant_views = std::pin::pin!(ordered_concurrent( + installed_variants, + api_concurrency(api_client.uses_public_proxy()), + |uuid| async move { hold_back_debug(api_client.fetch_patch(&uuid)).await }, + )); + for (base, variants) in multi { // Any variant's resolved path works — they all map to the same // installed package directory. @@ -1379,7 +1397,12 @@ async fn filter_to_installed_releases( // kept for the download loop — it is the same GET it would issue. let mut candidates: Vec<(String, HashMap)> = Vec::new(); for s in &variants { - match api_client.fetch_patch(&s.uuid).await { + let view = match variant_views.next().await { + Some(view) => view.release(), + // Unreachable: the plan holds one view per variant here. + None => api_client.fetch_patch(&s.uuid).await, + }; + match view { Ok(Some(patch)) => { candidates.push((s.purl.clone(), files_with_both_hashes(&patch))); views.insert(s.uuid.clone(), patch); @@ -1774,6 +1797,22 @@ impl FetchBatch { } } +/// The record a detached ledger entry already carries for `purl` at +/// exactly `uuid` — the ledger store's idempotency skip (no view fetch). +/// Always `None` for the manifest store. +fn detached_ledger_record<'a>( + store: RecordStore<'a>, + purl: &str, + uuid: &str, +) -> Option<&'a PatchRecord> { + let RecordStore::Ledger(entries) = store else { + return None; + }; + lookup_entry(entries, purl) + .filter(|e| e.detached && e.uuid == uuid) + .and_then(|e| e.record.as_ref()) +} + /// The fetch loop both download engines share: installed-release /// narrowing, the caller's Bun refusal, the per-store skip decision, the /// view fetch (served from `prefetched` when the narrowing or the caller @@ -1825,6 +1864,31 @@ async fn fetch_selected_patches( warnings, }; + // The view GETs the loop below makes — every patch past the refusal + // and the ledger skip whose view is not already held in `prefetched` + // (the same three checks, in the loop's order, over inputs the loop + // never mutates) — run concurrently ahead of it, at most + // `api_concurrency` in flight, and come back in selection order. The + // loop takes the next one exactly where it used to await the request, + // and each request's `--debug` lines print there too, so stdout, the + // per-patch stderr lines and the JSON records fold exactly as the + // serial loop's did. + let mut held: std::collections::HashSet<&str> = prefetched.keys().map(String::as_str).collect(); + let to_fetch: Vec<&str> = selected + .iter() + .filter(|sr| { + bun_refusal.filter(|r| r.applies_to(&sr.purl)).is_none() + && detached_ledger_record(store, &sr.purl, &sr.uuid).is_none() + && !held.remove(sr.uuid.as_str()) + }) + .map(|sr| sr.uuid.as_str()) + .collect(); + let mut views = std::pin::pin!(ordered_concurrent( + to_fetch, + api_concurrency(api_client.uses_public_proxy()), + |uuid| async move { (uuid, hold_back_debug(api_client.fetch_patch(uuid)).await) }, + )); + for search_result in &selected { let (purl, uuid) = (search_result.purl.as_str(), search_result.uuid.as_str()); @@ -1850,30 +1914,35 @@ async fn fetch_selected_patches( // Idempotency (ledger store): a detached entry already at this uuid // carries its own record — no view fetch needed. - if let RecordStore::Ledger(entries) = store { - if let Some(record) = lookup_entry(entries, purl) - .filter(|e| e.detached && e.uuid == uuid) - .and_then(|e| e.record.clone()) - { - if !quiet { - eprintln!("{}", format_record_skip(purl, "already vendored")); - } - batch.patches_json.push(serde_json::json!({ - "purl": purl, - "uuid": uuid, - "action": "skipped", - })); - batch.reused.push((purl.to_string(), record)); - batch.skipped += 1; - continue; + if let Some(record) = detached_ledger_record(store, purl, uuid).cloned() { + if !quiet { + eprintln!("{}", format_record_skip(purl, "already vendored")); } + batch.patches_json.push(serde_json::json!({ + "purl": purl, + "uuid": uuid, + "action": "skipped", + })); + batch.reused.push((purl.to_string(), record)); + batch.skipped += 1; + continue; } // The view: from memory when the narrowing (or the uuid path's own - // identifier fetch) already fetched it, else the network. + // identifier fetch) already fetched it, else the network — the next + // of the concurrent GETs above, which were planned for exactly + // these turns. let view = match prefetched.remove(uuid) { Some(patch) => Ok(Some(patch)), - None => api_client.fetch_patch(uuid).await, + None => match views.next().await { + Some((planned, view)) if planned == uuid => view.release(), + // Unreachable (the plan mirrors this loop's checks); a + // live fetch keeps the outcome right regardless. + _ => { + debug_assert!(false, "view prefetch plan out of step at {uuid}"); + api_client.fetch_patch(uuid).await + } + }, }; let patch = match view { Ok(Some(patch)) => patch, @@ -6824,6 +6893,177 @@ mod tests { ); } + /// The download loop's view GETs run concurrently but fold in selection + /// order: with later views answering FIRST (reversed latencies) and a + /// mix of 200 / 404 / 500 / held-in-memory / ledger-reused patches, + /// every per-patch record keeps its selection slot and its serial + /// action + error text, and only the views the serial loop fetched are + /// requested (the held and reused ones never are). + #[tokio::test] + #[serial_test::serial] + async fn download_patch_records_concurrent_views_fold_in_selection_order() { + use wiremock::matchers::{method, path as wm_path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let _env = EnvVarGuard::scrub(&["SOCKET_PROXY_URL", "SOCKET_PATCH_PROXY_URL"]); + let server = MockServer::start().await; + let uuid = |c: char| { + format!("{0}{0}{0}{0}{0}{0}{0}{0}-{0}{0}{0}{0}-4{0}{0}{0}-8{0}{0}{0}-{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}", c) + }; + let purl = |n: &str| format!("pkg:npm/covgap-order-{n}@1.0.0"); + let view = |u: &str, p: &str| { + serde_json::json!({ + "uuid": u, "purl": p, + "publishedAt": "2024-01-01T00:00:00Z", + "files": { "package/index.js": { + "beforeHash": "0".repeat(64), "afterHash": "1".repeat(64), + }}, + "vulnerabilities": {}, "description": "d", "license": "MIT", "tier": "free", + }) + }; + // Selection order a..f; the slowest answers belong to the earliest. + let (a, b, c, d, e, f) = ( + uuid('a'), + uuid('b'), + uuid('c'), + uuid('d'), + uuid('e'), + uuid('f'), + ); + let mount = |u: &str, resp: ResponseTemplate| { + Mock::given(method("GET")) + .and(wm_path(format!("/v0/orgs/test-org/patches/view/{u}"))) + .respond_with(resp) + .expect(1) + }; + mount( + &a, + ResponseTemplate::new(200) + .set_body_json(view(&a, &purl("a"))) + .set_delay(Duration::from_millis(600)), + ) + .mount(&server) + .await; + mount( + &b, + ResponseTemplate::new(404).set_delay(Duration::from_millis(400)), + ) + .mount(&server) + .await; + mount( + &c, + ResponseTemplate::new(500) + .set_body_string("boom") + .set_delay(Duration::from_millis(200)), + ) + .mount(&server) + .await; + // `d` is held in memory and `e` is reused from the ledger: never + // requested. + for u in [&d, &e] { + Mock::given(method("GET")) + .and(wm_path(format!("/v0/orgs/test-org/patches/view/{u}"))) + .respond_with(ResponseTemplate::new(500)) + .expect(0) + .mount(&server) + .await; + } + mount( + &f, + ResponseTemplate::new(200).set_body_json(view(&f, &purl("f"))), + ) + .mount(&server) + .await; + + let tmp = tempfile::tempdir().unwrap(); + let vendor = tmp.path().join(".socket/vendor"); + std::fs::create_dir_all(&vendor).unwrap(); + std::fs::write( + vendor.join("state.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "version": 1, + "entries": { purl("e"): { + "ecosystem": "npm", + "basePurl": purl("e"), + "uuid": e, + "detached": true, + "record": { + "uuid": e, + "exportedAt": "2024-01-01T00:00:00Z", + "files": { "package/index.js": { + "beforeHash": "0".repeat(64), "afterHash": "1".repeat(64), + }}, + "vulnerabilities": {}, + "description": "d", "license": "MIT", "tier": "free", + }, + "artifact": { + "path": format!(".socket/vendor/npm/{e}/covgap-order-e-1.0.0.tgz"), + }, + "wiring": [] + }} + })) + .unwrap(), + ) + .unwrap(); + + let mut held: PatchResponse = serde_json::from_value(view(&d, &purl("d"))).unwrap(); + held.uuid = d.clone(); + let selected: Vec = [ + (&a, "a"), + (&b, "b"), + (&c, "c"), + (&d, "d"), + (&e, "e"), + (&f, "f"), + ] + .iter() + .map(|(u, n)| mk_patch(u, &purl(n), "free", "2024-01-01")) + .collect(); + let client = test_client(&server.uri()).await; + let (_code, json, records, _blobs) = download_patch_records_with( + &selected, + &detached_params(tmp.path()), + &client, + HashMap::from([(d.clone(), held)]), + ) + .await; + + let rows: Vec<(String, String, String)> = json["patches"] + .as_array() + .unwrap() + .iter() + .map(|p| { + ( + p["purl"].as_str().unwrap_or_default().to_string(), + p["action"].as_str().unwrap_or_default().to_string(), + p["error"].as_str().unwrap_or_default().to_string(), + ) + }) + .collect(); + let row = + |n: &str, action: &str, error: &str| (purl(n), action.to_string(), error.to_string()); + assert_eq!( + rows, + vec![ + row("a", "downloaded", ""), + row("b", "failed", "could not fetch details"), + row("c", "failed", "API request failed with status 500: boom"), + row("d", "downloaded", ""), + row("e", "skipped", ""), + row("f", "downloaded", ""), + ], + "json={json}" + ); + assert_eq!(json["downloaded"], 3, "json={json}"); + assert_eq!(json["failed"], 2, "json={json}"); + assert_eq!(json["skipped"], 1, "json={json}"); + let mut got: Vec<&String> = records.keys().collect(); + got.sort(); + assert_eq!(got, vec![&purl("a"), &purl("d"), &purl("e"), &purl("f")]); + // `.expect` counts are verified on drop. + drop(server); + } + /// The env guard must RESTORE a variable that was set before the scrub — /// the suite depends on it not leaking scrubbed state across tests. #[test] diff --git a/crates/socket-patch-cli/src/commands/scan/discovery.rs b/crates/socket-patch-cli/src/commands/scan/discovery.rs index c4fb8fde..c297fd8f 100644 --- a/crates/socket-patch-cli/src/commands/scan/discovery.rs +++ b/crates/socket-patch-cli/src/commands/scan/discovery.rs @@ -2,11 +2,14 @@ //! supplements, update detection against the existing manifest, vendor //! baseline pre-verification, and the table's vuln-ID / severity helpers. +use futures_util::StreamExt; +use socket_patch_core::api::client::hold_back_debug; use socket_patch_core::api::ranking::cmp_batch_infos; use socket_patch_core::api::types::{ BatchPackagePatches, BatchPatchInfo, PatchResponse, PatchSearchResult, }; use socket_patch_core::manifest::schema::{PatchManifest, PatchRecord}; +use socket_patch_core::utils::concurrent::{api_concurrency, ordered_concurrent}; use socket_patch_core::utils::purl::{normalize_purl, strip_purl_qualifiers}; use socket_patch_core::vendor::lock_inventory::LockfileEntry; use socket_patch_core::vendor::VendorState; @@ -283,29 +286,58 @@ pub(super) async fn preverify_vendor_baselines( let mut mismatched: HashSet = HashSet::new(); let mut views: HashMap = HashMap::new(); - for (i, patch) in selected.iter().enumerate() { + // Per patch, what the loop below compares: `None` to skip it, else the + // installed copy plus the ledger's embedded record (`None` = fetch the + // view). Local and read-only, so it is computed up front. + let plan: Vec< + Option<( + &socket_patch_core::crawlers::types::CrawledPackage, + Option<&PatchRecord>, + )>, + > = selected + .iter() + .map(|patch| { + // API purls come percent-encoded, crawler purls literal — + // purl_eq bridges the two spellings. + let base = strip_purl_qualifiers(&patch.purl); + // Lockfile-only packages have no installed bytes to compare + // — the vendor engine fetches them pristine (nothing to + // annotate). + if lockfile_only_contains(lockfile_only, base) { + return None; + } + let pkg = crawled.iter().find(|c| purl_eq(&c.purl, base))?; + // The same predicate as the download phase's ledger + // idempotency skip: its no-fetch set and this one must be + // the same set. + let embedded = vendor + .and_then(|entries| lookup_entry(entries, &patch.purl)) + .filter(|e| e.detached && e.uuid == patch.uuid) + .and_then(|e| e.record.as_ref()); + Some((pkg, embedded)) + }) + .collect(); + // The views the loop needs, fetched concurrently (at most + // `api_concurrency` in flight) and consumed in `selected` order, each + // request's `--debug` lines released at its turn. + let mut details = std::pin::pin!(ordered_concurrent( + selected + .iter() + .zip(&plan) + .filter(|(_, step)| matches!(step, Some((_, None)))) + .map(|(patch, _)| patch.uuid.as_str()), + api_concurrency(api_client.uses_public_proxy()), + |uuid| hold_back_debug(api_client.fetch_patch(uuid)), + )); + for (i, (patch, step)) in selected.iter().zip(&plan).enumerate() { status.set(format!( "Checking installed files against patch baselines... ({}/{})", i + 1, selected.len() )); - // API purls come percent-encoded, crawler purls literal — purl_eq - // bridges the two spellings. - let base = strip_purl_qualifiers(&patch.purl); - // Lockfile-only packages have no installed bytes to compare — the - // vendor engine fetches them pristine (nothing to annotate). - if lockfile_only_contains(lockfile_only, base) { - continue; - } - let Some(pkg) = crawled.iter().find(|c| purl_eq(&c.purl, base)) else { + let Some((pkg, embedded)) = *step else { continue; }; - // The same predicate as the download phase's ledger idempotency - // skip: its no-fetch set and this one must be the same set. - let embedded = vendor - .and_then(|entries| lookup_entry(entries, &patch.purl)) - .filter(|e| e.detached && e.uuid == patch.uuid) - .and_then(|e| e.record.as_ref()); let files: Vec<(String, PatchFileInfo)> = match embedded { Some(record) => record .files @@ -313,7 +345,10 @@ pub(super) async fn preverify_vendor_baselines( .map(|(file, info)| (file.clone(), info.clone())) .collect(), None => { - let Ok(Some(detail)) = api_client.fetch_patch(&patch.uuid).await else { + let Some(detail) = details.next().await else { + continue; + }; + let Ok(Some(detail)) = detail.release() else { continue; }; let files = detail diff --git a/crates/socket-patch-cli/src/commands/vex_sources.rs b/crates/socket-patch-cli/src/commands/vex_sources.rs index 77b69e3c..e009e0ea 100644 --- a/crates/socket-patch-cli/src/commands/vex_sources.rs +++ b/crates/socket-patch-cli/src/commands/vex_sources.rs @@ -69,11 +69,14 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use futures_util::StreamExt; + use socket_patch_core::api::client::{ build_proxy_fallback_client, get_api_client_with_overrides, is_fallback_candidate, }; use socket_patch_core::manifest::schema::{PatchManifest, PatchRecord}; use socket_patch_core::patch::redirect::RedirectState; +use socket_patch_core::utils::concurrent::ordered_concurrent; use socket_patch_core::utils::purl::strip_purl_qualifiers; use socket_patch_core::vendor::state::{lookup_entry_kv, VendorArtifact, VendorEntry, VendorState}; use socket_patch_core::vex::discover::{ @@ -902,28 +905,29 @@ async fn fetch_records( loop { let mut auth_refused: Vec = Vec::new(); let mut auth_error: Option = None; - for chunk in pending.chunks(FETCH_CONCURRENCY) { - status.set(format!( - "Fetching {}... ({done}/{total})", - if total == 1 { - "the patch record" - } else { - "patch records" - } + // A sliding window of at most FETCH_CONCURRENCY views in flight + // (it used to wait for each whole chunk of that size to drain + // before starting the next), consumed in `pending` order. + { + let client = &client; + let mut views = std::pin::pin!(ordered_concurrent( + pending.iter(), + FETCH_CONCURRENCY, + |uuid| async move { (uuid, client.fetch_patch(uuid).await) }, )); - let mut set = tokio::task::JoinSet::new(); - for uuid in chunk { - let client = client.clone(); - let uuid = uuid.clone(); - set.spawn(async move { - let result = client.fetch_patch(&uuid).await; - (uuid, result) - }); - } - while let Some(joined) = set.join_next().await { - let Ok((uuid, result)) = joined else { - continue; + loop { + status.set(format!( + "Fetching {}... ({done}/{total})", + if total == 1 { + "the patch record" + } else { + "patch records" + } + )); + let Some((uuid, result)) = views.next().await else { + break; }; + let uuid = uuid.clone(); done += 1; match result { Ok(Some(view)) => { diff --git a/crates/socket-patch-core/src/api/client.rs b/crates/socket-patch-core/src/api/client.rs index 2c628ea5..a731a710 100644 --- a/crates/socket-patch-core/src/api/client.rs +++ b/crates/socket-patch-core/src/api/client.rs @@ -119,6 +119,32 @@ pub(crate) fn flush_deferred_debug(lines: Vec) { } } +/// A request's output fetched ahead of the loop that consumes it, with its +/// `--debug` lines held back until [`Self::release`] — call that where the +/// one-at-a-time loop would have issued the request, so the debug stream +/// keeps the serial interleaving with the loop's own stderr lines. Dropping +/// it unreleased discards the lines (the serial loop never made the call). +#[derive(Debug)] +pub struct HeldBack { + value: T, + debug: Vec, +} + +impl HeldBack { + /// The output, printing the held-back debug lines first. + pub fn release(self) -> T { + flush_deferred_debug(self.debug); + self.value + } +} + +/// Run `fut` (a request made ahead of its turn) with its debug lines held +/// back; see [`HeldBack`]. +pub async fn hold_back_debug(fut: impl std::future::Future) -> HeldBack { + let (value, debug) = with_deferred_debug(fut).await; + HeldBack { value, debug } +} + /// Options for constructing an [`ApiClient`]. #[derive(Debug, Clone)] pub struct ApiClientOptions { From ca9c2f6216eb9f6d41b7eea0c906200f9f97b58f Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 19:08:08 -0400 Subject: [PATCH 039/237] perf(vendor): reuse scan's npm crawl instead of walking node_modules twice more A vendored scan crawled the tree, then its vendor engine walked it again twice: find_packages_for_rollback rediscovered every node_modules root, and a single alias-installed ("missing") npm purl triggered a full NpmCrawler::crawl_all in npm_paths_by_identity. scan now keeps the npm half of its crawl (the crawler's packages and the roots it walked, via new NpmCrawler::crawl_all_with_roots) and hands it to vendor_records_reusing: the roots replace the targeted lookup's root discovery (each root is still searched by find_by_purls, so copy choice and order are unchanged) and the packages replace the identity crawl. The snapshot is only used for the exact options it was taken with, and only when nothing can have changed the tree since: the JSON arm, and the interactive arm when its prompt answers without waiting on a person. The vendor command, repair and get keep crawling. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../socket-patch-cli/src/commands/scan/mod.rs | 16 +- .../src/commands/scan/vendor_flow.rs | 52 ++- .../socket-patch-cli/src/commands/vendor.rs | 36 ++- .../src/ecosystem_dispatch.rs | 301 ++++++++++++++++-- .../src/crawlers/npm_crawler.rs | 16 +- 5 files changed, 375 insertions(+), 46 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index 65318165..03fd4d82 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -30,7 +30,7 @@ use std::path::Path; use crate::args::{apply_env_toggles, GlobalArgs}; use crate::commands::vex::{generate_vex_from_manifest_path, VexEmbedArgs}; -use crate::ecosystem_dispatch::crawl_all_ecosystems; +use crate::ecosystem_dispatch::crawl_all_ecosystems_with_npm; use crate::ui::{self, plural, print_json, StatusLine}; use super::get::{download_and_apply_patches_with, select_patches, DownloadParams, DownloadRun}; @@ -1577,9 +1577,11 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { let mut status = StatusLine::stderr(args.common.json, args.common.silent); status.set(format!("Scanning {scan_target}...")); - // Crawl packages - let (mut all_crawled, mut eco_counts, skipped_bundle_config_path) = - crawl_all_ecosystems(&crawler_options).await; + // Crawl packages. The npm half is kept for the vendored path: its + // engine resolves the same untouched tree and reuses this crawl + // instead of walking `node_modules` again. + let (mut all_crawled, mut eco_counts, skipped_bundle_config_path, npm_crawl) = + crawl_all_ecosystems_with_npm(&crawler_options).await; // Lockfile supplement: dependencies the project's lockfile resolves // that have NO installed copy (fresh clone, partial install). They join @@ -2389,6 +2391,7 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { telemetry_token.as_deref(), telemetry_org.as_deref(), telemetry, + Some(&npm_crawl), ) .await; } @@ -2918,6 +2921,10 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { HashMap::new() }; + // Whether the prompt below waits on a person: the tree may change while + // it does, so the vendor step then crawls afresh instead of reusing the + // pre-prompt crawl (`--yes` / `--json` / non-terminal answer at once). + let prompt_waits = !(args.common.yes || args.common.json) && std::io::stdin().is_terminal(); if !ui::confirm(&render::confirm_prompt(plan), true, &args.common) { if !silent { println!(); @@ -2957,6 +2964,7 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { prune, telemetry_token.as_deref(), telemetry_org.as_deref(), + (!prompt_waits).then_some(&npm_crawl), ) .await } else { diff --git a/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs b/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs index 42050908..355282c3 100644 --- a/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs +++ b/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs @@ -35,8 +35,9 @@ use crate::commands::fetch_stage::{stage_vendor_sources_in_memory, MemStageOutco use crate::commands::get::{download_patch_records_with, DetachedDownload, DownloadParams}; use crate::commands::lock_cli::lock_failure; use crate::commands::vendor::{ - note_classic_migration_risk, track_outcomes_for_vendor, vendor_records, + note_classic_migration_risk, track_outcomes_for_vendor, vendor_records_reusing, }; +use crate::ecosystem_dispatch::NpmCrawlSnapshot; use crate::json_envelope::{Command as EnvelopeCommand, Envelope}; use crate::ui::{plural, print_json}; @@ -178,6 +179,9 @@ async fn run_scan_vendor_step( // all (the step is a silent no-op then). `get --mode vendored` wants // it; scan's interactive arm prints its own closing line instead. report_empty: bool, + // The npm half of scan's crawl, for the engine to reuse instead of + // walking the untouched tree again (see `vendor_records_reusing`). + prior: Option<&NpmCrawlSnapshot>, ) -> VendorStepResult { let mut env = Envelope::new(EnvelopeCommand::Vendor); env.dry_run = common.dry_run; @@ -215,6 +219,7 @@ async fn run_scan_vendor_step( client, use_public_proxy, &mut env, + prior, ) .await { @@ -242,6 +247,7 @@ async fn run_scan_vendor_step( /// embeds its record). The caller holds the apply lock. `Err` is the /// `no_local_source` fold (staging could not obtain the patch content — /// offline, or the view fetch failed). +#[allow(clippy::too_many_arguments)] async fn stage_and_vendor( common: &GlobalArgs, socket_dir: &Path, @@ -250,6 +256,7 @@ async fn stage_and_vendor( client: ApiClient, use_public_proxy: bool, env: &mut Envelope, + prior: Option<&NpmCrawlSnapshot>, ) -> Result { // Loaded ONCE under the lock: the staging harvest reads it here, then // the engine takes it over for its persists. An unreadable ledger @@ -289,6 +296,7 @@ async fn stage_and_vendor( Some(&service), ledger, env, + prior, ) .await) } @@ -460,6 +468,8 @@ async fn run_vendor_json_path( // Scan's pending telemetry, flushed by `discover_selected` before // anything below writes to stdout. telemetry: &mut PendingTelemetry, + // The npm half of scan's crawl, for the vendor engine to reuse. + prior: Option<&NpmCrawlSnapshot>, ) -> i32 { // Same discovery as `--apply`. Vendored purls are NOT filtered here — // re-vendoring a stale uuid is the point of the flag (same-uuid re-runs @@ -523,12 +533,13 @@ async fn run_vendor_json_path( // 2) The vendor engine, under the same lock as apply/vendor (a no-op // that creates nothing when there is nothing to vendor). - let vendor_code = match boxed_scan_vendor_step( + let vendor_code = match boxed_scan_vendor_step_reusing( &args.common, records, blobs, api_client.clone(), use_public_proxy, + prior, ) .await { @@ -619,6 +630,9 @@ async fn run_vendor_interactive_path( prune: bool, telemetry_token: Option<&str>, telemetry_org: Option<&str>, + // The npm half of scan's crawl, for the vendor engine to reuse — + // `None` when the tree may have changed since (an answered prompt). + prior: Option<&NpmCrawlSnapshot>, ) -> i32 { // The download phase is quiet about its own header in vendored mode // (only the manifest-mode download prints it), so this arm does. @@ -647,6 +661,7 @@ async fn run_vendor_interactive_path( blobs, api_client.clone(), use_public_proxy, + prior, ) .await { @@ -823,6 +838,7 @@ pub(super) fn boxed_vendor_json_path<'a>( telemetry_token: Option<&'a str>, telemetry_org: Option<&'a str>, telemetry: &'a mut PendingTelemetry, + prior: Option<&'a NpmCrawlSnapshot>, ) -> std::pin::Pin + 'a>> { Box::pin(run_vendor_json_path( args, @@ -839,6 +855,7 @@ pub(super) fn boxed_vendor_json_path<'a>( telemetry_token, telemetry_org, telemetry, + prior, )) } @@ -859,6 +876,7 @@ pub(super) fn boxed_vendor_interactive_path<'a>( prune: bool, telemetry_token: Option<&'a str>, telemetry_org: Option<&'a str>, + prior: Option<&'a NpmCrawlSnapshot>, ) -> std::pin::Pin + 'a>> { Box::pin(run_vendor_interactive_path( args, @@ -874,6 +892,7 @@ pub(super) fn boxed_vendor_interactive_path<'a>( prune, telemetry_token, telemetry_org, + prior, )) } @@ -898,6 +917,28 @@ pub(crate) fn boxed_scan_vendor_step<'a>( client, use_public_proxy, true, + None, + )) +} + +/// [`boxed_scan_vendor_step`] handing the engine scan's npm crawl to reuse +/// (see `vendor_records_reusing`). +fn boxed_scan_vendor_step_reusing<'a>( + common: &'a GlobalArgs, + records: HashMap, + seed: HashMap>, + client: ApiClient, + use_public_proxy: bool, + prior: Option<&'a NpmCrawlSnapshot>, +) -> std::pin::Pin + 'a>> { + Box::pin(run_scan_vendor_step( + common, + records, + seed, + client, + use_public_proxy, + true, + prior, )) } @@ -909,6 +950,7 @@ fn boxed_scan_vendor_step_quiet_empty<'a>( seed: HashMap>, client: ApiClient, use_public_proxy: bool, + prior: Option<&'a NpmCrawlSnapshot>, ) -> std::pin::Pin + 'a>> { Box::pin(run_scan_vendor_step( common, @@ -917,6 +959,7 @@ fn boxed_scan_vendor_step_quiet_empty<'a>( client, use_public_proxy, false, + prior, )) } @@ -944,14 +987,15 @@ fn boxed_vendor_records<'a>( service: Option<&'a socket_patch_core::vendor::VendorServiceConfig>, ledger: std::io::Result, env: &'a mut Envelope, + prior: Option<&'a NpmCrawlSnapshot>, ) -> std::pin::Pin + 'a>> { // `scan --vendor` threads the SAME service config the `vendor` command // builds (honoring `--vendor-source`), so both entry points vendor the // same bytes by default. See `run_scan_vendor_step`. Always detached: // vendored mode is manifest-free. The ledger is the one the harvest // just read, handed over so the engine does not reload it. - Box::pin(vendor_records( - common, records, sources, /*detached=*/ true, false, env, service, ledger, + Box::pin(vendor_records_reusing( + common, records, sources, /*detached=*/ true, false, env, service, ledger, prior, )) } diff --git a/crates/socket-patch-cli/src/commands/vendor.rs b/crates/socket-patch-cli/src/commands/vendor.rs index 04cd4227..34f75908 100644 --- a/crates/socket-patch-cli/src/commands/vendor.rs +++ b/crates/socket-patch-cli/src/commands/vendor.rs @@ -47,7 +47,8 @@ use crate::commands::vex::{ generate_vex_from_manifest_path, generate_vex_without_manifest, ManifestlessVex, VexEmbedArgs, }; use crate::ecosystem_dispatch::{ - find_packages_for_rollback, npm_paths_by_identity, partition_purls, + find_packages_for_rollback_reusing, npm_paths_by_identity, npm_paths_by_identity_in, + partition_purls, NpmCrawlSnapshot, }; use crate::json_envelope::{ Command, Envelope, EnvelopeError, PatchAction, PatchEvent, RunWarning, Status, VexSummary, @@ -1159,6 +1160,30 @@ pub(crate) async fn vendor_records( // command and `scan --vendor` pass `Some(_)`, honoring `--vendor-source`. service: Option<&VendorServiceConfig>, ledger: std::io::Result, +) -> bool { + vendor_records_reusing( + common, records, sources, detached, force, env, service, ledger, None, + ) + .await +} + +/// [`vendor_records`], resolving npm packages from `prior` — the npm half +/// of a crawl this process made earlier with the same options, over a tree +/// nothing has touched since (`scan`'s own crawl) — instead of walking +/// `node_modules` again: its roots feed the targeted lookup and its +/// packages the alias identity fallback. `None` (or a snapshot taken with +/// other options) crawls as before. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn vendor_records_reusing( + common: &GlobalArgs, + records: &HashMap, + sources: &PatchSources<'_>, + detached: bool, + force: bool, + env: &mut Envelope, + service: Option<&VendorServiceConfig>, + ledger: std::io::Result, + prior: Option<&NpmCrawlSnapshot>, ) -> bool { let mut has_errors = false; // Lockfile flavors the backends wired THIS run (from the returned ledger @@ -1240,10 +1265,11 @@ pub(crate) async fn vendor_records( // registry download, and (for gem) a HashMap-order platform coin-flip. // The rollback variant fans each base path back out to every qualified // manifest purl (same invariant as `find_manifest_package_paths`). - let mut all_packages = find_packages_for_rollback( + let mut all_packages = find_packages_for_rollback_reusing( &vendorable_partition, &crawler_options, common.silent || common.json, + prior, ) .await; @@ -1257,7 +1283,11 @@ pub(crate) async fn vendor_records( .flatten() .filter(|p| !all_packages.contains_key(*p)) .collect(); - for (purl, paths) in npm_paths_by_identity(&crawler_options, &missing_npm).await { + let by_identity = match prior.and_then(|p| p.packages_for(&crawler_options)) { + Some(installed) => npm_paths_by_identity_in(installed, &missing_npm), + None => npm_paths_by_identity(&crawler_options, &missing_npm).await, + }; + for (purl, paths) in by_identity { all_packages.insert(purl, paths[0].clone()); } diff --git a/crates/socket-patch-cli/src/ecosystem_dispatch.rs b/crates/socket-patch-cli/src/ecosystem_dispatch.rs index 8acf4af7..5923e37e 100644 --- a/crates/socket-patch-cli/src/ecosystem_dispatch.rs +++ b/crates/socket-patch-cli/src/ecosystem_dispatch.rs @@ -244,11 +244,16 @@ fn passthrough_purls(purls: &[String]) -> Vec { /// inserts the crawler-returned PURL with first-wins semantics. It is /// applied to the release-variant ecosystems (PyPI / RubyGems / Maven), /// which are also queried with deduped base PURLs. +/// +/// `npm_roots`, when given, are the `node_modules` roots an earlier crawl of +/// the same options and (untouched) tree walked — used instead of walking +/// the tree for them again ([`NpmRootsCrawler`]). async fn dispatch_find( partitioned: &HashMap>, options: &CrawlerOptions, silent: bool, variant_merge: MergeFn, + npm_roots: Option<&[PathBuf]>, ) -> HashMap> { let mut out: HashMap> = HashMap::new(); @@ -258,7 +263,7 @@ async fn dispatch_find( eco = Ecosystem::Npm, options = options, silent = silent, - crawler = NpmCrawler, + crawler = NpmRootsCrawler { roots: npm_roots }, get_paths = get_node_modules_paths, using_label = "global npm packages", err_label = "npm packages", @@ -419,7 +424,7 @@ pub async fn find_all_packages_for_purls( // `merge_qualified`'s `push_path`. Single-copy ecosystems keep true // first-wins via their own `merge_first_wins` wiring in // `dispatch_find`. - dispatch_find(partitioned, options, silent, merge_variant_copies).await + dispatch_find(partitioned, options, silent, merge_variant_copies, None).await } /// Multi-copy variant of `find_packages_for_rollback` (qualified-aware @@ -429,7 +434,7 @@ pub async fn find_all_packages_for_rollback( options: &CrawlerOptions, silent: bool, ) -> HashMap> { - dispatch_find(partitioned, options, silent, merge_qualified).await + dispatch_find(partitioned, options, silent, merge_qualified, None).await } /// Qualified-aware PURL resolution for rollback, vendor, repair and @@ -446,7 +451,80 @@ pub async fn find_packages_for_rollback( options: &CrawlerOptions, silent: bool, ) -> HashMap { - collapse_to_first(find_all_packages_for_rollback(partitioned, options, silent).await) + find_packages_for_rollback_reusing(partitioned, options, silent, None).await +} + +/// [`find_packages_for_rollback`], taking the npm `node_modules` roots from +/// `prior` (a crawl of the same options earlier in this process, over a +/// tree nothing has touched since) instead of walking the tree for them +/// again. Only the root discovery is reused: each root is still searched +/// by `find_by_purls`, so copy choice and order are unchanged. A snapshot +/// taken with other options is ignored. +pub async fn find_packages_for_rollback_reusing( + partitioned: &HashMap>, + options: &CrawlerOptions, + silent: bool, + prior: Option<&NpmCrawlSnapshot>, +) -> HashMap { + let npm_roots = prior + .filter(|p| p.taken_with(options)) + .map(|p| p.roots.as_slice()); + collapse_to_first(dispatch_find(partitioned, options, silent, merge_qualified, npm_roots).await) +} + +/// The npm half of one [`crawl_all_ecosystems_with_npm`] run: the packages +/// the npm crawler found (its whole output, in crawl order) and the +/// `node_modules` roots it walked, with the options they were taken with. +/// Handed from `scan`'s crawl to its vendor step so the vendor engine does +/// not walk the same untouched tree again ([`npm_paths_by_identity_in`], +/// [`find_packages_for_rollback_reusing`]). +#[derive(Debug, Clone)] +pub struct NpmCrawlSnapshot { + cwd: PathBuf, + global: bool, + global_prefix: Option, + roots: Vec, + packages: Vec, +} + +impl NpmCrawlSnapshot { + /// Whether this snapshot was crawled with exactly `options`. + fn taken_with(&self, options: &CrawlerOptions) -> bool { + self.cwd == options.cwd + && self.global == options.global + && self.global_prefix == options.global_prefix + } + + /// The crawled npm packages, when crawled with exactly `options`. + pub(crate) fn packages_for(&self, options: &CrawlerOptions) -> Option<&[CrawledPackage]> { + self.taken_with(options).then_some(self.packages.as_slice()) + } +} + +/// [`NpmCrawler`] for [`dispatch_find`], answering its root discovery from +/// an earlier crawl's roots when it has them. +struct NpmRootsCrawler<'a> { + roots: Option<&'a [PathBuf]>, +} + +impl NpmRootsCrawler<'_> { + async fn get_node_modules_paths( + &self, + options: &CrawlerOptions, + ) -> Result, std::io::Error> { + match self.roots { + Some(roots) => Ok(roots.to_vec()), + None => NpmCrawler.get_node_modules_paths(options).await, + } + } + + async fn find_by_purls( + &self, + node_modules_path: &std::path::Path, + purls: &[String], + ) -> Result>, std::io::Error> { + NpmCrawler.find_by_purls(node_modules_path, purls).await + } } /// The installed copy of each npm purl in `purls`, found by its @@ -464,11 +542,21 @@ pub(crate) async fn npm_paths_by_identity( options: &CrawlerOptions, purls: &[&String], ) -> HashMap> { - let mut out = HashMap::new(); if purls.is_empty() { - return out; + return HashMap::new(); } let installed = NpmCrawler::new().crawl_all(options).await; + npm_paths_by_identity_in(&installed, purls) +} + +/// [`npm_paths_by_identity`] over an npm crawl already in hand (the whole +/// output of `NpmCrawler::crawl_all` for the same options, over a tree +/// nothing has touched since) instead of crawling again. +pub(crate) fn npm_paths_by_identity_in( + installed: &[CrawledPackage], + purls: &[&String], +) -> HashMap> { + let mut out = HashMap::new(); for purl in purls { let want = canonical_purl(purl); let paths: Vec = installed @@ -543,6 +631,43 @@ pub async fn crawl_all_ecosystems( Vec, HashMap, Option, +) { + let (packages, counts, skipped_config_path, _) = crawl_every_ecosystem(options).await; + (packages, counts, skipped_config_path) +} + +/// [`crawl_all_ecosystems`], also handing back the npm half of the crawl as +/// an [`NpmCrawlSnapshot`] (its packages are the leading `counts[Npm]` +/// entries of the package list). +pub async fn crawl_all_ecosystems_with_npm( + options: &CrawlerOptions, +) -> ( + Vec, + HashMap, + Option, + NpmCrawlSnapshot, +) { + let (packages, counts, skipped_config_path, npm_roots) = crawl_every_ecosystem(options).await; + let npm_count = counts.get(&Ecosystem::Npm).copied().unwrap_or(0); + let snapshot = NpmCrawlSnapshot { + cwd: options.cwd.clone(), + global: options.global, + global_prefix: options.global_prefix.clone(), + roots: npm_roots, + packages: packages[..npm_count].to_vec(), + }; + (packages, counts, skipped_config_path, snapshot) +} + +/// The crawl behind both entry points above; the fourth element is the npm +/// crawler's `node_modules` roots. +async fn crawl_every_ecosystem( + options: &CrawlerOptions, +) -> ( + Vec, + HashMap, + Option, + Vec, ) { // The nine crawlers are independent (none prints, none mutates shared // state), so they run concurrently; their blocking walks and @@ -555,32 +680,41 @@ pub async fn crawl_all_ecosystems( // profile (see `walk_pool`): a crawler treats a failed open as an // absent dir, so extra concurrent descriptors could silently drop // packages there. - let (npm, pypi, cargo, (gems, gem_discovery), golang, maven, composer, nuget, deno) = - if walk_pool::fd_limit_is_tight() { - ( - boxed(|| NpmCrawler.crawl_all(options)).await, - boxed(|| PythonCrawler.crawl_all(options)).await, - boxed(|| CargoCrawler.crawl_all(options)).await, - boxed(|| RubyCrawler.crawl_all_with_discovery(options)).await, - boxed(|| GoCrawler.crawl_all(options)).await, - boxed(|| MavenCrawler.crawl_all(options)).await, - boxed(|| ComposerCrawler.crawl_all(options)).await, - boxed(|| NuGetCrawler.crawl_all(options)).await, - boxed(|| DenoCrawler.crawl_all(options)).await, - ) - } else { - tokio::join!( - boxed(|| NpmCrawler.crawl_all(options)), - boxed(|| PythonCrawler.crawl_all(options)), - boxed(|| CargoCrawler.crawl_all(options)), - boxed(|| RubyCrawler.crawl_all_with_discovery(options)), - boxed(|| GoCrawler.crawl_all(options)), - boxed(|| MavenCrawler.crawl_all(options)), - boxed(|| ComposerCrawler.crawl_all(options)), - boxed(|| NuGetCrawler.crawl_all(options)), - boxed(|| DenoCrawler.crawl_all(options)), - ) - }; + let ( + (npm, npm_roots), + pypi, + cargo, + (gems, gem_discovery), + golang, + maven, + composer, + nuget, + deno, + ) = if walk_pool::fd_limit_is_tight() { + ( + boxed(|| NpmCrawler.crawl_all_with_roots(options)).await, + boxed(|| PythonCrawler.crawl_all(options)).await, + boxed(|| CargoCrawler.crawl_all(options)).await, + boxed(|| RubyCrawler.crawl_all_with_discovery(options)).await, + boxed(|| GoCrawler.crawl_all(options)).await, + boxed(|| MavenCrawler.crawl_all(options)).await, + boxed(|| ComposerCrawler.crawl_all(options)).await, + boxed(|| NuGetCrawler.crawl_all(options)).await, + boxed(|| DenoCrawler.crawl_all(options)).await, + ) + } else { + tokio::join!( + boxed(|| NpmCrawler.crawl_all_with_roots(options)), + boxed(|| PythonCrawler.crawl_all(options)), + boxed(|| CargoCrawler.crawl_all(options)), + boxed(|| RubyCrawler.crawl_all_with_discovery(options)), + boxed(|| GoCrawler.crawl_all(options)), + boxed(|| MavenCrawler.crawl_all(options)), + boxed(|| ComposerCrawler.crawl_all(options)), + boxed(|| NuGetCrawler.crawl_all(options)), + boxed(|| DenoCrawler.crawl_all(options)), + ) + }; let mut all_packages = Vec::new(); let mut counts: HashMap = HashMap::new(); @@ -600,7 +734,7 @@ pub async fn crawl_all_ecosystems( } let skipped_config_path = gem_discovery.and_then(|d| d.skipped_config_path); - (all_packages, counts, skipped_config_path) + (all_packages, counts, skipped_config_path, npm_roots) } #[cfg(test)] @@ -1380,6 +1514,107 @@ mod tests { } } + /// The vendor engine's reuse of scan's npm crawl is an oracle-equal + /// substitute: over one tree (a hoisted dep, a nested duplicate, an + /// alias install, a workspace member's own `node_modules`), the + /// snapshot's roots and packages equal what the engine's own discovery + /// and identity crawl find, and both lookups built on it answer + /// exactly as the crawling ones do. A snapshot taken with other + /// options is never used. + #[tokio::test] + async fn npm_crawl_snapshot_matches_the_crawls_it_replaces() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + let write = |dir: &std::path::Path, name: &str, version: &str| { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write( + dir.join("package.json"), + format!(r#"{{"name":"{name}","version":"{version}"}}"#), + ) + .unwrap(); + }; + std::fs::write( + root.join("package.json"), + r#"{"name":"root","version":"1.0.0","workspaces":["packages/*"]}"#, + ) + .unwrap(); + write(&root.join("node_modules/foo"), "foo", "1.0.0"); + write(&root.join("node_modules/bar"), "bar", "2.0.0"); + write( + &root.join("node_modules/bar/node_modules/foo"), + "foo", + "0.9.0", + ); + write(&root.join("node_modules/@s/qux"), "@s/qux", "4.0.0"); + // `"lp": "npm:left-pad@1.3.0"` installs under the alias key. + write(&root.join("node_modules/lp"), "left-pad", "1.3.0"); + write(&root.join("packages/app"), "app", "0.1.0"); + write(&root.join("packages/app/node_modules/baz"), "baz", "3.0.0"); + write(&root.join("packages/app/node_modules/foo"), "foo", "0.9.0"); + let options = local_options(root.to_path_buf()); + + let (packages, counts, _, snapshot) = crawl_all_ecosystems_with_npm(&options).await; + let npm_count = counts[&Ecosystem::Npm]; + let pairs = |pkgs: &[CrawledPackage]| -> Vec<(String, PathBuf)> { + pkgs.iter() + .map(|p| (p.purl.clone(), p.path.clone())) + .collect() + }; + assert_eq!( + snapshot.roots, + NpmCrawler.get_node_modules_paths(&options).await.unwrap() + ); + assert_eq!( + pairs(snapshot.packages_for(&options).unwrap()), + pairs(&NpmCrawler.crawl_all(&options).await) + ); + assert_eq!(pairs(&snapshot.packages), pairs(&packages[..npm_count])); + assert!(snapshot.roots.len() >= 2, "roots={:?}", snapshot.roots); + + let purls: Vec = [ + "pkg:npm/foo@1.0.0", + "pkg:npm/foo@0.9.0", + "pkg:npm/bar@2.0.0", + "pkg:npm/baz@3.0.0", + "pkg:npm/%40s/qux@4.0.0", + "pkg:npm/left-pad@1.3.0", + "pkg:npm/absent@9.9.9", + ] + .map(String::from) + .to_vec(); + let partitioned = partition_purls(&purls, None); + let crawled = find_packages_for_rollback(&partitioned, &options, true).await; + let reused = + find_packages_for_rollback_reusing(&partitioned, &options, true, Some(&snapshot)).await; + assert_eq!(reused, crawled); + assert!(crawled.contains_key("pkg:npm/baz@3.0.0"), "{crawled:?}"); + + let missing: Vec<&String> = purls.iter().filter(|p| !crawled.contains_key(*p)).collect(); + assert!( + missing.iter().any(|p| p.contains("left-pad")), + "{missing:?}" + ); + let by_crawl = npm_paths_by_identity(&options, &missing).await; + let by_snapshot = + npm_paths_by_identity_in(snapshot.packages_for(&options).unwrap(), &missing); + assert_eq!(by_snapshot, by_crawl); + assert_eq!( + by_crawl.get("pkg:npm/left-pad@1.3.0"), + Some(&vec![root.join("node_modules/lp")]) + ); + + let elsewhere = local_options(root.join("packages/app")); + assert!(snapshot.packages_for(&elsewhere).is_none()); + let app_purls = vec!["pkg:npm/foo@1.0.0".to_string()]; + let app_partitioned = partition_purls(&app_purls, None); + assert_eq!( + find_packages_for_rollback_reusing(&app_partitioned, &elsewhere, true, Some(&snapshot)) + .await, + find_packages_for_rollback(&app_partitioned, &elsewhere, true).await, + "a snapshot of another root must not answer for this one" + ); + } + /// The concurrent crawl must yield exactly the serial run's packages, /// in the fixed ecosystem order, with the same counts. A /// `--global-prefix` root is handed to every crawler verbatim, so one diff --git a/crates/socket-patch-core/src/crawlers/npm_crawler.rs b/crates/socket-patch-core/src/crawlers/npm_crawler.rs index bddeaaf4..58201b92 100644 --- a/crates/socket-patch-core/src/crawlers/npm_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/npm_crawler.rs @@ -632,11 +632,23 @@ impl NpmCrawler { /// store entries' `identity_seen` decisions) see exactly the state the /// sequential walk would have — same packages, same paths, same order. pub async fn crawl_all(&self, options: &CrawlerOptions) -> Vec { + self.crawl_all_with_roots(options).await.0 + } + + /// [`Self::crawl_all`], also handing back the `node_modules` roots it + /// walked — exactly what [`Self::get_node_modules_paths`] returns for + /// the same options and tree — so a caller that resolves purls against + /// the same untouched tree later in the process can skip rediscovering + /// them. + pub async fn crawl_all_with_roots( + &self, + options: &CrawlerOptions, + ) -> (Vec, Vec) { let options = options.clone(); run_walk(move || Self::crawl_all_sync(&options)).await } - fn crawl_all_sync(options: &CrawlerOptions) -> Vec { + fn crawl_all_sync(options: &CrawlerOptions) -> (Vec, Vec) { let nm_paths = Self::node_modules_paths_sync(options); let gathered: Vec> = par_map(&nm_paths, |nm_path| { Self::gather_node_modules(nm_path, None, false) @@ -647,7 +659,7 @@ impl NpmCrawler { for events in gathered { Self::merge_scan_events(events, None, &mut seen, &mut packages); } - packages + (packages, nm_paths) } /// Find specific packages by PURL inside a single `node_modules` tree. From ed3d4237325465988b2cd0906672ae35d9e5f7c3 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 24 Sep 2026 19:18:40 -0400 Subject: [PATCH 040/237] perf(vendor): prefetch service downloads ahead of the serial wiring loop Each vendored npm package made its two patch-service round trips (the package-reference POST and the archive GET) inside the serial dispatch loop, back to back with its lockfile and ledger writes. vendor_records now attaches a download plan to the run's client: the npm records the loop is expected to download (in loop order, past the Bun refusal and the takeover gate, and without a committed artifact the ledger anchors at the record's uuid, which a re-run reuses). A background task fetches the plan at most api_concurrency ahead, and fetch_vendor_package takes a planned uuid's outcome instead of making the requests. Single-uuid POSTs are kept. The plan is advisory; every decision stays at the loop's own call, in order: the circuit breaker is checked and its count updated there exactly as before (the prefetch never touches it, and a call it skips discards the prefetched bytes), skipped or unplanned calls fetch live, and each prefetched request's --debug lines print at its call. The task starts only at the first planned call and stops speculating after the breaker threshold of its own consecutive availability failures; the plan detaches (aborting the task) when vendor_records returns. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../socket-patch-cli/src/commands/vendor.rs | 30 ++ crates/socket-patch-core/src/api/client.rs | 76 ++- crates/socket-patch-core/src/api/mod.rs | 1 + .../src/api/vendor_prefetch.rs | 485 ++++++++++++++++++ crates/socket-patch-core/src/vendor/mod.rs | 27 + .../src/vendor/registry_fetch.rs | 54 +- .../src/vendor/toml_surgery.rs | 3 +- 7 files changed, 656 insertions(+), 20 deletions(-) create mode 100644 crates/socket-patch-core/src/api/vendor_prefetch.rs diff --git a/crates/socket-patch-cli/src/commands/vendor.rs b/crates/socket-patch-cli/src/commands/vendor.rs index 34f75908..74b0c9f4 100644 --- a/crates/socket-patch-cli/src/commands/vendor.rs +++ b/crates/socket-patch-cli/src/commands/vendor.rs @@ -1511,6 +1511,36 @@ pub(crate) async fn vendor_records_reusing( // line prints on a clean line. let mut status = StatusLine::stderr(common.json, common.silent); let total = all_packages.len(); + // Service downloads, fetched ahead of this serial loop (the wiring and + // every write stay here, in order). The plan is the npm records the + // loop is expected to download: in loop order, past the Bun refusal + // and the takeover gate below, with no committed artifact the ledger + // anchors at the record's uuid (those re-runs reuse it and never ask + // the service). It is advisory — the breaker and every outcome are + // still decided at the loop's own call (see `VendorPrefetch`). + let _service_prefetch = service.filter(|_| !common.dry_run).and_then(|cfg| { + let planned: Vec = all_packages + .iter() + .filter(|(purl, _)| Ecosystem::from_purl(purl) == Some(Ecosystem::Npm)) + .filter(|(purl, _)| bun_refusal.as_ref().is_none_or(|r| !r.applies_to(purl))) + .filter(|(purl, _)| { + redirect_ledger_corrupt.is_none() + && redirect_ledger.as_ref().is_none_or(|l| { + !l.records + .keys() + .any(|k| canonical_purl(k) == canonical_purl(purl)) + }) + }) + .filter_map(|(purl, _)| records.get(purl)) + .filter(|record| { + !state.entries.values().any(|e| { + e.ecosystem == "npm" && e.uuid == record.uuid && !e.artifact.sha256.is_empty() + }) + }) + .map(|record| record.uuid.clone()) + .collect(); + cfg.prefetch_archives(planned) + }); for (index, (purl, pkg_path)) in all_packages.iter().enumerate() { let is_variant_eco = Ecosystem::from_purl(purl).is_some_and(|e| e.supports_release_variants()); diff --git a/crates/socket-patch-core/src/api/client.rs b/crates/socket-patch-core/src/api/client.rs index a731a710..991f1c58 100644 --- a/crates/socket-patch-core/src/api/client.rs +++ b/crates/socket-patch-core/src/api/client.rs @@ -13,6 +13,8 @@ use serde::Serialize; use crate::api::ranking::severity_order as get_severity_order; use crate::api::ranking::{cmp_batch_infos, cmp_search_results}; use crate::api::types::*; +use crate::api::vendor_prefetch::VendorPrefetch; +pub use crate::api::vendor_prefetch::VendorPrefetchGuard; use crate::constants::USER_AGENT as USER_AGENT_VALUE; use crate::utils::env_compat::{is_debug_enabled, is_offline_env, proxy_url_from_env}; use crate::utils::notice::{notice_once, Notice}; @@ -131,6 +133,15 @@ pub struct HeldBack { } impl HeldBack { + pub(crate) fn new(value: T, debug: Vec) -> Self { + Self { value, debug } + } + + /// The output, without releasing the debug lines. + pub(crate) fn peek(&self) -> &T { + &self.value + } + /// The output, printing the held-back debug lines first. pub fn release(self) -> T { flush_deferred_debug(self.debug); @@ -189,6 +200,11 @@ pub struct ApiClient { /// [`PROXY_BATCH_PATH_CONCURRENCY`] requests in total: the peak the /// serial batch loop reached, never that peak times the window. proxy_batch_slots: Arc, + /// The vendor loop's download plan, while one is attached + /// ([`Self::prefetch_vendor_packages`]): [`Self::fetch_vendor_package`] + /// takes a planned uuid's outcome from it instead of requesting it. + /// Shared by clones, like the breaker it defers to. + vendor_prefetch: Arc>>>, } /// Most requests the public proxy's batch path keeps in flight per client: @@ -257,7 +273,7 @@ impl VendorRetryPolicy { /// Consecutive retryable vendor-service failures after which the rest of the /// run skips the service without any I/O (`auto` then builds locally, /// `service` fails closed — the existing miss policy). -const VENDOR_BREAKER_THRESHOLD: u32 = 2; +pub(crate) const VENDOR_BREAKER_THRESHOLD: u32 = 2; /// A jitter sample in `[0, 1)` from std's randomly keyed hasher (no RNG /// dependency; the quality needed here is "not synchronized"). @@ -348,6 +364,7 @@ impl ApiClient { vendor_retry: VendorRetryPolicy::default(), vendor_outage: Arc::new(AtomicU32::new(0)), proxy_batch_slots: Arc::new(tokio::sync::Semaphore::new(PROXY_BATCH_PATH_CONCURRENCY)), + vendor_prefetch: Arc::new(std::sync::Mutex::new(None)), } } @@ -366,6 +383,12 @@ impl ApiClient { self } + /// The run-level breaker's consecutive-failure count (tests). + #[cfg(test)] + pub(crate) fn vendor_outage_count(&self) -> u32 { + self.vendor_outage.load(Ordering::Relaxed) + } + /// Returns the API token, if set. pub fn api_token(&self) -> Option<&String> { self.api_token.as_ref() @@ -938,9 +961,28 @@ impl ApiClient { this run" ))); } - let (outcome, retryable_failure) = self - .fetch_vendor_package_once(uuid, free_only, vendor_url, patch_server_url) - .await; + // A download the attached plan already fetched stands in for the + // live requests; everything around it (the breaker check above, the + // counter update below) runs here, in call order, as before. + let plan = self + .vendor_prefetch + .lock() + .ok() + .and_then(|slot| slot.clone()); + let prefetched = match plan { + Some(plan) => { + plan.take(self, uuid, free_only, vendor_url, patch_server_url) + .await + } + None => None, + }; + let (outcome, retryable_failure) = match prefetched { + Some(fetched) => fetched.release(), + None => { + self.fetch_vendor_package_once(uuid, free_only, vendor_url, patch_server_url) + .await + } + }; match &outcome { VendorServiceOutcome::Failed(_) if retryable_failure => { self.vendor_outage.fetch_add(1, Ordering::Relaxed); @@ -953,9 +995,33 @@ impl ApiClient { outcome } + /// Attach a download plan: `uuids` are the packages the vendor loop is + /// expected to download from the service, in loop order, with these + /// request parameters. Until the guard drops, the loop's + /// [`Self::fetch_vendor_package`] call for a planned uuid takes an + /// outcome fetched ahead of it (at most `window` in flight) — see + /// [`super::vendor_prefetch`] for why nothing observable changes. The + /// plan replaces any plan already attached. + pub fn prefetch_vendor_packages( + &self, + uuids: Vec, + free_only: bool, + vendor_url: Option<&str>, + patch_server_url: Option<&str>, + window: usize, + ) -> VendorPrefetchGuard { + let plan = VendorPrefetch::new(uuids, free_only, vendor_url, patch_server_url, window); + if let Ok(mut slot) = self.vendor_prefetch.lock() { + *slot = Some(Arc::new(plan)); + } + VendorPrefetchGuard { + slot: Arc::clone(&self.vendor_prefetch), + } + } + /// [`Self::fetch_vendor_package`] without the breaker: the outcome, and /// whether a `Failed` one was a retryable (availability) failure. - async fn fetch_vendor_package_once( + pub(crate) async fn fetch_vendor_package_once( &self, uuid: &str, free_only: bool, diff --git a/crates/socket-patch-core/src/api/mod.rs b/crates/socket-patch-core/src/api/mod.rs index ab918e7a..f59e8518 100644 --- a/crates/socket-patch-core/src/api/mod.rs +++ b/crates/socket-patch-core/src/api/mod.rs @@ -3,3 +3,4 @@ pub mod client; pub mod date; pub mod ranking; pub mod types; +pub(crate) mod vendor_prefetch; diff --git a/crates/socket-patch-core/src/api/vendor_prefetch.rs b/crates/socket-patch-core/src/api/vendor_prefetch.rs new file mode 100644 index 00000000..86187322 --- /dev/null +++ b/crates/socket-patch-core/src/api/vendor_prefetch.rs @@ -0,0 +1,485 @@ +//! Vendor-service downloads fetched ahead of the serial vendor loop. +//! +//! The vendor engine wires one package at a time (lockfile and ledger +//! writes stay serial, in sorted order), and each package's service path +//! makes two round trips — the package-reference POST and the archive GET +//! ([`ApiClient::fetch_vendor_package`]). A run that downloads many +//! prebuilt archives paid those round trips back to back. A +//! [`VendorPrefetch`] plan names the uuids the loop is expected to +//! download, in loop order; a background task fetches them ahead of the +//! loop, at most `window` in flight, and the loop's own call for a planned +//! uuid takes the fetched outcome instead of making the requests. +//! +//! Nothing observable may change, so the plan is advisory and every +//! decision stays at consumption time, in loop order: +//! +//! * The run-level circuit breaker is evaluated exactly as before — the +//! prefetch never touches its counter. A call the breaker skips never +//! consults the plan (its fetched outcome, if any, is discarded along +//! with its debug lines), and a consumed outcome updates the counter as +//! the live request would have. Outage messages therefore match the +//! serial loop package for package. +//! * A call for a uuid the plan does not hold (or holds only behind the +//! point already consumed) makes the live requests. Planned uuids the +//! loop never asks for (a flavor refused the package first) are dropped +//! when a later one is taken. +//! * Each prefetched request's `--debug` lines are held back and printed +//! when the loop takes its outcome, where the serial request would have +//! printed them. +//! +//! The task only starts at the loop's first planned call (a run whose +//! packages all refuse before the service makes no speculative request), +//! and it stops speculating after [`super::client::VENDOR_BREAKER_THRESHOLD`] +//! consecutive availability failures of its own. Dropping the +//! [`VendorPrefetchGuard`] detaches the plan and aborts the task. + +use std::sync::Arc; + +use futures_util::StreamExt; + +use super::client::{ + with_deferred_debug, ApiClient, HeldBack, VendorServiceOutcome, VENDOR_BREAKER_THRESHOLD, +}; +use crate::utils::concurrent::ordered_concurrent; + +/// One fetched outcome: `(outcome, retryable failure)` as +/// `fetch_vendor_package_once` returned it, debug lines held back. +type Fetched = HeldBack<(VendorServiceOutcome, bool)>; + +/// A planned run of vendor-service downloads; see the module docs. +#[derive(Debug)] +pub(crate) struct VendorPrefetch { + /// The request parameters every planned call must match. + free_only: bool, + vendor_url: Option, + patch_server_url: Option, + /// Planned uuids, in the order the vendor loop consumes them. + planned: Vec, + /// Most downloads in flight (and queued unconsumed) at once. + window: usize, + state: tokio::sync::Mutex, +} + +#[derive(Debug, Default)] +struct PrefetchState { + /// First plan position not yet consumed or passed over. + cursor: usize, + /// Outcomes from the task, tagged with their plan position, in order. + /// `None` until the loop's first planned call starts the task. + rx: Option>, + task: Option>, +} + +impl Drop for PrefetchState { + fn drop(&mut self) { + if let Some(task) = &self.task { + task.abort(); + } + } +} + +impl VendorPrefetch { + pub(crate) fn new( + planned: Vec, + free_only: bool, + vendor_url: Option<&str>, + patch_server_url: Option<&str>, + window: usize, + ) -> Self { + Self { + free_only, + vendor_url: vendor_url.map(str::to_string), + patch_server_url: patch_server_url.map(str::to_string), + planned, + window: window.max(1), + state: tokio::sync::Mutex::new(PrefetchState::default()), + } + } + + /// The prefetched outcome of the loop's call for `uuid`, or `None` to + /// make the live requests. Plan positions before `uuid`'s are passed + /// over (their outcomes discarded unreleased). + pub(crate) async fn take( + &self, + client: &ApiClient, + uuid: &str, + free_only: bool, + vendor_url: Option<&str>, + patch_server_url: Option<&str>, + ) -> Option { + if free_only != self.free_only + || vendor_url != self.vendor_url.as_deref() + || patch_server_url != self.patch_server_url.as_deref() + { + return None; + } + let mut state = self.state.lock().await; + let position = state.cursor + + self.planned[state.cursor..] + .iter() + .position(|planned| planned == uuid)?; + state.cursor = position + 1; + if state.rx.is_none() { + self.start(&mut state, client, position); + } + let rx = state.rx.as_mut()?; + loop { + match rx.recv().await { + Some((index, fetched)) if index == position => return Some(fetched), + Some((index, _)) if index < position => continue, + // Past the position (never sent out of order) or the task + // stopped: fetch live from here on. + _ => { + state.rx = None; + state.cursor = self.planned.len(); + return None; + } + } + } + } + + /// Spawn the task fetching `planned[from..]` in order. + fn start(&self, state: &mut PrefetchState, client: &ApiClient, from: usize) { + let (tx, rx) = tokio::sync::mpsc::channel(self.window); + let client = client.clone(); + let planned: Vec = self.planned[from..].to_vec(); + let (free_only, window) = (self.free_only, self.window); + let vendor_url = self.vendor_url.clone(); + let patch_server_url = self.patch_server_url.clone(); + state.task = Some(tokio::spawn(async move { + let (client, vendor_url, patch_server_url) = + (&client, vendor_url.as_deref(), patch_server_url.as_deref()); + let mut fetched = std::pin::pin!(ordered_concurrent( + planned.into_iter().enumerate(), + window, + move |(offset, uuid): (usize, String)| async move { + let (outcome, debug) = with_deferred_debug(client.fetch_vendor_package_once( + &uuid, + free_only, + vendor_url, + patch_server_url, + )) + .await; + (from + offset, HeldBack::new(outcome, debug)) + }, + )); + let mut consecutive_failures = 0; + while let Some((index, held)) = fetched.next().await { + match held.peek() { + (VendorServiceOutcome::Failed(_), true) => consecutive_failures += 1, + (VendorServiceOutcome::Failed(_), false) => {} + _ => consecutive_failures = 0, + } + if tx.send((index, held)).await.is_err() { + return; + } + // The breaker would skip the service from here on unless a + // package in between succeeds; stop speculating (later + // calls fetch live, through the breaker). + if consecutive_failures >= VENDOR_BREAKER_THRESHOLD { + return; + } + } + })); + state.rx = Some(rx); + } +} + +/// Keeps a [`VendorPrefetch`] plan attached to its client; dropping it +/// detaches the plan and aborts any downloads still in flight. +#[must_use = "the plan is detached when the guard drops"] +#[derive(Debug)] +pub struct VendorPrefetchGuard { + pub(crate) slot: Arc>>>, +} + +impl Drop for VendorPrefetchGuard { + fn drop(&mut self) { + if let Ok(mut slot) = self.slot.lock() { + slot.take(); + } + } +} + +#[cfg(test)] +mod tests { + //! Oracle tests: every call sequence yields, call for call, exactly the + //! outcomes (and final breaker count) of the same sequence with no plan + //! attached — the serial loop. + use super::*; + use crate::api::client::{ApiClientOptions, VendorRetryPolicy}; + use base64::Engine as _; + use sha2::{Digest as _, Sha512}; + use std::time::Duration; + use wiremock::matchers::{body_partial_json, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + const POST_PATH: &str = "/v0/orgs/acme/patches/package"; + + /// How the service answers one uuid. + #[derive(Clone, Copy)] + enum Script { + /// Granted, with this delay on the POST. + Granted(u64), + /// 503 on every POST attempt (a retryable availability failure). + Down, + /// 403 (a non-retryable failure: says nothing about availability). + Forbidden, + Pending, + NotFound, + } + + fn uuid(i: usize) -> String { + format!("{i:08x}-0000-4000-8000-{i:012x}") + } + + fn client(uri: &str) -> ApiClient { + ApiClient::new(ApiClientOptions { + api_url: uri.to_string(), + api_token: Some("sktsec_placeholder_value_for_tests_api".into()), + use_public_proxy: false, + org_slug: Some("acme".into()), + }) + .with_vendor_retry(VendorRetryPolicy { + attempts: 3, + base: Duration::from_millis(1), + max_delay: Duration::from_millis(5), + ..VendorRetryPolicy::default() + }) + } + + async fn serve(scripts: &[Script]) -> MockServer { + let server = MockServer::start().await; + for (i, script) in scripts.iter().enumerate() { + let u = uuid(i); + let serve_path = format!("/serve/{u}.tgz"); + let bytes = u.as_bytes().to_vec(); + let post = |resp: ResponseTemplate| { + Mock::given(method("POST")) + .and(path(POST_PATH)) + .and(body_partial_json( + serde_json::json!({ "uuids": [u.clone()] }), + )) + .respond_with(resp) + }; + let status_body = |status: &str| { + ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "results": { u.clone(): { "status": status, "url": null, "artifacts": [] } } + })) + }; + let resp = match *script { + Script::Granted(delay) => { + let url = format!("{}{serve_path}", server.uri()); + let sri = format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(Sha512::digest(&bytes)) + ); + ResponseTemplate::new(200) + .set_body_json(serde_json::json!({ + "results": { u.clone(): { "status": "granted", "url": url, + "artifacts": [{ "kind": "tarball", "url": url, + "integrity": { "sha512": sri } }] } } + })) + .set_delay(Duration::from_millis(delay)) + } + Script::Down => ResponseTemplate::new(503), + Script::Forbidden => ResponseTemplate::new(403), + Script::Pending => status_body("pending_build"), + Script::NotFound => status_body("not_found"), + }; + post(resp).mount(&server).await; + Mock::given(method("GET")) + .and(path(serve_path)) + .respond_with(ResponseTemplate::new(200).set_body_bytes(bytes)) + .mount(&server) + .await; + } + server + } + + fn summary(outcome: &VendorServiceOutcome) -> String { + match outcome { + VendorServiceOutcome::Ready(pkg) => format!( + "ready {} {} {}", + String::from_utf8_lossy(&pkg.tarball), + pkg.integrity_sri, + pkg.source_url + ), + VendorServiceOutcome::Pending => "pending".to_string(), + VendorServiceOutcome::Unavailable(reason) => format!("unavailable {reason}"), + VendorServiceOutcome::Failed(e) => format!("failed {e}"), + } + } + + /// Run `calls` (indices into the scripted uuids) one at a time, with + /// `plan` attached when given; the per-call outcomes and final count. + async fn run( + server: &MockServer, + plan: Option<&[usize]>, + calls: &[usize], + ) -> (Vec, u32) { + let c = client(&server.uri()); + let _guard = plan.map(|plan| { + c.prefetch_vendor_packages( + plan.iter().map(|&i| uuid(i)).collect(), + false, + None, + None, + 4, + ) + }); + let mut out = Vec::new(); + for &i in calls { + out.push(summary( + &c.fetch_vendor_package(&uuid(i), false, None, None).await, + )); + } + (out, c.vendor_outage_count()) + } + + async fn assert_matches_serial(scripts: &[Script], plan: &[usize], calls: &[usize]) { + let server = serve(scripts).await; + let serial = run(&server, None, calls).await; + let planned = run(&server, Some(plan), calls).await; + assert_eq!(planned, serial); + } + + /// Later packages answer first (reversed latencies); every outcome + /// still lands on its own call. + #[tokio::test] + async fn outcomes_land_on_their_own_calls_under_reversed_latencies() { + let scripts: Vec