diff --git a/.gitattributes b/.gitattributes index e7f9e267b..917dd2c79 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,3 +10,9 @@ crates/socket-patch-core/tests/fixtures/pdm-native/*.lock -text # refuse CRLF by design (vendor_lockfile_crlf_unsupported), and the tests # derive their CRLF variants from the LF bytes themselves. crates/socket-patch-core/tests/fixtures/pnpm-hosted/** -text + +# The legacy vendor ledgers are what the base binary wrote, byte for byte: +# vendor_ledger_schema_e2e replays them through `vendor --revert` and +# compares the result byte for byte, so a CRLF checkout would change both +# the replayed wiring files and the expected revert. +crates/socket-patch-cli/tests/fixtures/legacy-ledgers/** -text diff --git a/CHANGELOG.md b/CHANGELOG.md index 21874c7e3..f76d43b73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,13 +22,57 @@ into the new version's section — see docs/releasing.md. > manifest-free, moves vendored cargo wiring from `.cargo/config*` into > `Cargo.toml`, tags vendored cargo copies' versions with `+socket.` > (visible to the patched crate as `CARGO_PKG_VERSION`), turns a plain -> non-TTY `scan` report-only, and makes `vex` -> refuse to attest stale ledger records and corrupt vendor ledgers — all +> non-TTY `scan` report-only, makes `vex` +> refuse to attest stale ledger records and corrupt vendor ledgers, and +> retries a throttled patch API (new error text, added waiting, a throttled +> package failing its legacy-proxy batch) — all > MAJOR per CLI_CONTRACT.md's semver policy — so it ships as the next major > release (v5.0). ### Changed (BREAKING) +- **Vendored runs refuse lock-text failures before downloading them.** + `scan --mode vendored` and `get --mode vendored` evaluate the vendor + backends' pure lock-text gates — pnpm, yarn classic and yarn berry + (coordinates; the lock / manifest reads and their line-ending, version, + `cacheKey` and `.yarnrc.yml` gates; override and `resolutions` + conflicts; the lock entry present and rewritable) and cargo's + `locked_version_mismatch` when it is the crate's first refusal — before + fetching patch views and pristine sources, so a package that will be + refused costs no network. This applies only to a package the vendor loop + would hand to its backend — one installed on disk, or one the lockfile + resolves to a verifiable registry source (the pristine fetch would + happen); a package absent from the lock and not installed keeps its + `skipped` / `package_not_installed` vendor event and its download + record, exactly as before. Such a package is now reported in the download + phase: `download.patches[]` records it as `action: "failed"` with the + backend's exact `errorCode` and `error`, `download.downloaded` drops and + `download.failed` rises by the number of such packages, and the vendor + envelope no longer carries their `failed` events (`vendor.summary.failed` + drops by the same number) nor, for lockfile-only packages, their + `vendor_fetched_missing` events. Exit code and the top-level `status` + are unchanged; the nested `vendor.status` becomes `success` when those + refusals were the vendor step's only failures (and when every selected + package is refused this way, the human arm prints `Nothing was + vendored: N patches failed (see above).`). + (The interactive human `scan --vendor` arm still fetches the views its + pre-prompt baseline check verifies.) Purls the hosted redirect ledger + claims keep the vendor loop's refusal. Because no view is fetched, the + lock-text refusal now takes precedence over every outcome that came + from the view: a package that would also have hit a paid-access 403, a + failed view fetch or the no-applicable-files guardrail reports the lock + refusal instead. + The manifest-driven `vendor` command keeps its `failed` events but no + longer fetches the pristine source of a lockfile-only package it refuses + this way (no `vendor_fetched_missing` event, no registry request; with an + unreachable registry the gate's code replaces `vendor_fetch_failed`) — + again only when the lockfile resolves it to a verifiable source; one the + lock does not resolve keeps its `package_not_installed` skip. + On the polyglot monorepo fixture: 80 of 560 packages (74 + `vendor_lock_entry_not_found`, 4 `vendor_override_conflict`, 2 + `vendor_lock_entry_unsupported`) move to the download phase, saving 80 + view requests and 3 registry tarballs per run. + - **Vendored cargo copies carry a tagged version: `+socket.`.** The vendored copy's own `Cargo.toml` `[package] version` is rewritten to the patch-tagged version (`1.0.4+socket.`; a version that already @@ -261,8 +305,51 @@ into the new version's section — see docs/releasing.md. (`wiring_conflict`), and when the lockfile wires a package to patch U, a manifest or ledger record for it under another uuid is superseded. +- **A throttled patch API is retried with a bounded backoff.** An HTTP + 429 or 503 from the patch API made the affected batch (or patch-list + query) fail on the first answer — likelier now that up to 32 requests are + in flight. Now every patch-API JSON call (batch search, per-package patch + lists, patch views and VEX record fetches, hosted package references) + retries a 429 / 503 up to 3 times: it waits as long as `Retry-After` asks + (delta-seconds or HTTP-date; one over 30 s is not waited out — the answer + is final at once — and one under the jittered first step, such as `0` or a + past date, waits that step), otherwise 0.5 s, 1 s, 2 s (steps capped at + 8 s, jittered into their upper half). All retries in a run must end + within a 60 s wall-clock window opened by the run's first retry, so a + throttled run adds at most about a minute however many requests it makes + (requests waiting in parallel each keep their retries). + `SOCKET_API_MAX_RETRIES=` (0-10) changes the count; `0` restores the + old single attempt. Nothing else is retried: 401/403 still trigger the + proxy fallback at once, and the public proxy's permanent `503 "Patch API + is not configured"` is never retried on any path (the batch still + degrades to per-package lookups at once; a per-package lookup answering + it is still skipped). Output folds in request order exactly as an + unthrottled run's. What breaks: a throttled run now takes longer before + it fails (up to ~60 s of added waiting); the error text changes — it + names why retrying stopped (`Rate limit exceeded (HTTP 429, gave up after + 3 retries). Please try again later.`, `API request failed with status + 503: (gave up after 3 retries)`, `(Retry-After 120 s exceeds the + 30 s retry cap)`, `(the run's 60 s retry window has closed)`); and on + the token-less legacy per-package proxy path (a proxy without `POST + /patch/batch`) a package still throttled after its retries now fails its + whole batch query — every package in that batch goes unchecked and the + batch is reported as failed (warning, or the all-failed error when it was + the only batch) — instead of that one package being skipped silently. + ### Added +- **`--json` reports a failed patch-API query as a warning.** Under + `--json`, a batch query that failed (after the bounded retry above) while + others succeeded vanished from the envelope without a trace, exit 0, and + the agent / hosted / vendored flows' failed per-package patch-list + queries did the same; the human run already warned on stderr. Each is now + a run-level `warnings[]` entry carrying the human line's text: + `{code: "api_batch_failed", detail: "API batch of failed: + "}` (in batch order) and `{code: "patch_details_failed", detail: + "could not fetch details for : "}`. Additive: `status` and + the exit code are unchanged while some query succeeded, and the + all-failed error envelope and exit 1 still apply when none did. + - **`redirect_yarn_berry_mixed_line_endings` and `vendor_yarn_berry_mixed_line_endings`.** A `yarn.lock` (or, vendored, a root `package.json`) that mixes CRLF and LF line endings — or holds a bare @@ -624,9 +711,46 @@ into the new version's section — see docs/releasing.md. `hosted_revert_unsupported` before the manifest mutation), and remove's default GC extends from blobs-only to blobs + diff + package archives (parity with rollback/repair/`scan --prune`). +- **`SOCKET_API_CONCURRENCY` paces `scan`'s patch-API requests.** `scan` + now keeps several patch-API requests in flight (8 authenticated, 4 on the + public proxy) instead of one at a time. Set this variable — clamped to + `1`-`32`, and on the public proxy only downward — when something in front + of the API caps in-flight requests per client (a self-hosted `--api-url`, + a corporate reverse proxy, a WAF, a CDN) and a scan starts losing + requests to it. `SOCKET_API_CONCURRENCY=1` restores one request at a + time. Unset, empty or non-numeric values keep the defaults. Results, + warnings and their order never depend on the setting. + + Two request-count consequences an operator may see before they read the + code, neither of which changes any output: + + - A vendored run fetches prebuilt archives ahead of the wiring loop. + The plan it fetches is exact — it is gated by the same pre-flight each + vendor backend runs before it would ask the service (an unsupported or + absent lockfile entry, an override conflict, a workspace gate), so a + package the run does not end up vendoring is never asked for: the + `POST /v0/orgs//patches/package` download grants, which can start + a server-side archive build and count against quota, are exactly the + one-at-a-time loop's (71 on a fresh depscan run, where an earlier + draft of the look-ahead issued 74). What changes is only their timing: + up to four are in flight at once. `SOCKET_API_CONCURRENCY=1` turns the + look-ahead off entirely. + - A token revoked *mid-run* now costs the authenticated batch endpoint + the requests already in flight — up to the in-flight cap instead of + one — before the run downgrades to the public proxy. Their answers are + discarded and the connections are dropped mid-response, so the + endpoint's access log shows them; the downgrade warning, the patches + and the exit code are the same as before. ### Fixed +- **`vex`'s API-fallback note no longer depends on which refusal landed + first.** When the patch API refuses several patch records, the + `api_auth_fallback` note quoted whichever refusal happened to answer + first — a race, so two runs of the same project could report different + text (`Unauthorized` or `Forbidden`) and retry the refused records in a + different order. Both now follow the order the records are listed in, + like every other note. - **Hosted Go redirects no longer claim patches that did not land.** `scan`/`get --mode hosted` counted a Go module as redirected (recorded it in the redirect ledger, so `vex` attested it) whenever any project @@ -1196,6 +1320,217 @@ into the new version's section — see docs/releasing.md. ### Changed +- **The npm crawl skips tagged cache directories.** The walk that finds + workspace `node_modules` trees no longer descends into a directory that + carries a [Cache Directory Tagging](https://bford.info/cachedir/) + `CACHEDIR.TAG` beginning with the standard signature (every cargo + `target/` does), using the directory listing it already reads. One + semantic change: a `node_modules` inside such a directory, or anywhere + below it, is no longer crawled, so its packages are no longer scanned, + patched or attested. Every command that looks for installed npm copies + walks the same trees, so the change reaches past `scan`: `scan --prune` / + `--sync` treat a package installed only under a tagged directory as not + installed and garbage-collect its manifest entry and blobs (unless a + lockfile still resolves it), and `apply`, `rollback`, `remove`, `repair`, + `vendor` and `vex` no longer find copies there — so `remove` leaves such a + copy's patched files in place. The scan root itself is always crawled, and a + `CACHEDIR.TAG` without the signature (or that is a directory or a + symlink) prunes nothing. On a Rust-plus-JS monorepo this skipped 57% of + the walked directories. See docs/ecosystems.md. + +- **`scan` sends up to 32 patch-API requests at once on the authenticated + API, up from 8.** Each step sizes its window from the requests it has to + make: a quarter of them, between 8 and 32 — the batch queries, the + per-package patch lists, the hosted and vendored record views, discovery's + baseline views and `get`'s views. A step with 32 or fewer requests still + runs 8 at once; one with 128 or more runs 32. The fixed-size windows + follow the new cap up to their own ceilings: `vex` / `scan --vex` record + fetches now run up to 10 at once (was 8), wheel metadata stays at 4 and the + vendored archive prefetch at 4. The public proxy stays at 4. + `SOCKET_API_CONCURRENCY=` still overrides the adaptive cap (1-32; on + the proxy it can only lower it); the fixed windows keep their own ceilings + on top of it. Output is unchanged — every window folds its + answers in request order — but a large monorepo's hosted scan at 100 ms of + latency drops from ~20 s to ~9 s. + +- **`scan` queries the authenticated API 500 packages per batch, up from + 100.** Unset, `--batch-size` / `SOCKET_BATCH_SIZE` now follows the + endpoint: 500 purls per `POST /v0/orgs/{org}/patches/batch` (the server's + own per-request maximum) and 100 per `POST {proxy}/patch/batch` on the + public proxy, as before. A given size still applies as-is on either + endpoint. A batch whose JSON body would pass 256 KiB (the public proxy's + body cap) is now split, deterministically, into consecutive smaller + batches; at the default sizes that takes purls averaging over ~500 bytes. + A run downgraded to the proxy mid-run keeps its chunks, so it can send + the proxy batches of up to 500 purls (within the proxy's 256 KiB cap and + its upstream's 500-purl limit). Output is unchanged; the request count + and shape change — a large monorepo sends 30 batch requests instead of + 147 (depscan: 12 instead of 56), and `api_batch_failed` warnings number + the larger batches (`API batch 2 of 12 failed: …`). + +- **The crawl's directory walks run on 4 threads by default.** The walk + pool behind the `node_modules` walk and the Maven repository walk (and its + POM parse) used one thread per logical CPU (up to 16), but the walk is + bound by the kernel's directory cache: on a 14-core Mac and on Linux + ext4, 4 threads walked a large monorepo's `node_modules` as fast as or + faster than one per CPU, with a quarter of the system time (see + `walk_pool.rs` for the measurements; the Maven walk shares the pool and was + not measured separately). The default is now 4, or the performance-core + count when that is lower (`hw.perflevel0.logicalcpu` on Apple silicon). + `SOCKET_WALK_THREADS=` overrides it, clamped to 1-16 and to the CPU + count. What the crawl finds, and its order, are unchanged. + +- **Maven discovery takes coordinates from the `~/.m2` path.** A POM at its + canonical `///-.pom` + location is no longer opened: its groupId / artifactId / version come from + the directory names, which is where Maven itself writes every POM, so the + crawl skips reading and parsing tens of thousands of files. The path only + spells the right group when the scan root is the repository root, so each + top-level group directory (`org/`, `com/`, ...) is confirmed first: the + first canonical POM under it whose contents parse must agree with its + path, and a directory whose first such POM disagrees — every one of them + when `--global-prefix` / `SOCKET_GLOBAL_PREFIX` / `MAVEN_REPO_LOCAL` points + one level above or inside the repository — is read content-first exactly + as before. Other `.pom` files (timestamped SNAPSHOT POMs, hand-placed + extras, a dotted group directory) are parsed as before. One semantic + change: under a confirmed directory, a POM at a canonical path whose + contents disagree with its directory (hand-placed, or a legacy upstream + POM with mismatched coordinates) now reports the directory's coordinates + instead of the ones in the file. +- **`scan --ecosystems` crawls only the named ecosystems.** Without + `--prune`/`--sync`, a `scan -e npm` no longer walks `~/.m2`, the cargo + registry, the Go module cache and the rest only to filter their packages + away. What the run counts, queries and shows is unchanged + (`scannedPackages` already counted only the selected ecosystems), with + one exception: `lockfileOnlyPackages` and the human "not yet installed" + note now count only the selected ecosystems' lockfile-only entries + instead of every ecosystem's. A GC run (`--prune`/`--sync`) still crawls + every ecosystem — the prune needs the full installed set — and reports + exactly what it did before. + +- **An already-vendored project re-runs `vendor` without the network.** A + vendorable purl with no installed copy (the fresh-clone case) used to have + its pristine artifact downloaded and verified before the backend was even + asked, although the backend's in-sync check answers from the committed + artifact alone. That download is now deferred to the backend branch that + actually reads the pristine tree, whenever the vendor ledger already + covers the purl (its entry records the record's patch uuid and the + committed artifact is on disk — a file artifact such as a wheel or + tarball only while it still hashes to the ledger's `sha256`; `--force` + keeps the eager fetch), and for every lockfile-only cargo crate the + registry could fetch and verify (a crates.io `Cargo.lock` entry with a + checksum, or the pre-vendor resolution the ledger recovers) while the + patch service is enabled (the cargo backend reads the pristine source + only once `cargo_service_copy` falls back to the local build). A git, + path or custom-registry crate is never deferred: it keeps the eager + ladder's `vendor_fetch_unverifiable` + `package_not_installed` refusal + and is not vendored from the service's crates.io build, and a committed + file artifact that no longer matches its pin keeps the eager ladder's + outcome too. Visible effects: an idempotent re-run + makes no registry requests and no longer reports `vendor_fetched_missing` + for fetches it never needed; with no network (or under `--offline`) the + re-run of an already-vendored pypi, cargo, go or lockfile-only gem + project now SUCCEEDS (`already_vendored`, exit 0) instead of failing + `vendor_fetch_failed` / `package_not_installed`; a cargo crate the service + serves is never downloaded from the registry. When a deferred fetch does + happen (a drifted committed copy being rebuilt locally, a service miss), + its `vendor_fetched_missing` warning is recorded just ahead of that + package's own event instead of in the up-front fetch pass, and a failed, + unverifiable or `--offline`-refused deferred fetch reports exactly the + eager ladder's outcome for the purl (`vendor_fetch_failed` / + `vendor_fetch_unverifiable` + `package_not_installed` / + `package_not_installed`), in loop order. `vendor --vendor-source build` + (or no service config) now refuses a not-installed gem that the lock can + verify and no ledger entry covers with `gem_spec_missing` BEFORE + downloading it: a local build can never vendor a downloaded `.gem` (no + eval-able stub gemspec), so the download was pure waste. The refusal + keeps the backend's detail text and drops the `vendor_fetched_missing` + warning that used to precede it; since it now comes first, a gem the + backend would have refused for another reason after the download (an + uneditable Gemfile declaration, a Gemfile.lock it cannot edit) also + reports `gem_spec_missing`. `--dry-run` is unchanged: it still fetches + the gem and previews it (`vendor_fetched_missing` + `verified`). + +- **The vendor ledger stores a whole-file snapshot's new text as an edit.** + Several backends record an entire file as a wiring record's `original` / + `new` (maven's `pom.xml`, nuget's config, `pylock*.toml`, PEP 723 scripts + and hatch's project files), so a ledger held two near-identical copies of + that file per vendored package. In `.socket/vendor/state.json` such a + record's `new` text of 1 KiB or more is now written as a line-level edit + of the same record's `original`: `{"snapshot": "", + "ops": [[start, len] | "inserted text", …]}` (a `[start, len]` pair copies + that byte range of the `original`, a string inserts itself), and the + ledger's `"version"` is `2`. The `original` stays the plain string it + always was, every lockfile fragment record (poetry, composer, npm, …) is + untouched, and a ledger without such a record keeps its version-1 bytes. + Every command reads both versions: version-1 ledgers load and revert + exactly as before, and a version-2 edit is rebuilt and checked against its + hash (an edit that does not reproduce its text, has no `original` to + apply to, or any other `{"snapshot": …}` value, is + `vendor_state_unreadable`). Revert, repair, `vex`, rollback and the + re-vendor carry-forward see the same full texts as before. Each record is + self-contained, so an older socket-patch that re-saves the ledger (it + keeps `original` / `new` verbatim) loses nothing; reading a version-2 + record it leaves that fragment alone with its drift warning. No consumer + outside socket-patch reads `state.json` wiring. + +- **A vendored run commits its lockfile and ledger edits once, not per + package.** `vendor`, `scan --mode vendored` and `get --mode vendored` used + to rewrite every touched lockfile / `package.json` / `pnpm-workspace.yaml` + / config and the whole `.socket/vendor/state.json` after EACH package. The + run now captures those edits in memory (every backend still reads its own + and its siblings' earlier edits) and writes the final state once, after + the loop, through a roll-forward journal + (`.socket/vendor/.commit-journal.json`, removed when the commit + completes). The packages that succeeded are committed even when others + failed, so a completed run leaves exactly the files per-package commits + left. Crash semantics move from per-package to per-run: a crash before + the commit leaves the project's lockfiles and ledgers as they were before + the run (the artifacts it wrote are orphans the next run re-vendors over); + a crash during the commit is finished by the next command that takes the + apply lock, before it reads any of those files. When a file the journal + covers was edited since, that file is never written over and the journal + is set aside (`.commit-journal.set-aside-.json`, which keeps every + file's pre-commit bytes; a stderr warning names `repair`): if the edited + files still carry the commit's own lines the rest of the commit is + finished around them (so the ledger records the wiring on disk), if none + of them does the files the crash had already replaced are put back to + their pre-commit bytes, and otherwise nothing is applied. A journal that + would write through a symbolic link, or outside the lockfiles and + ledgers, is set aside unapplied. A replay that fails on I/O keeps the + journal and fails the command's lock acquire (`lock_io`) rather than + letting it work over a half-committed project. A re-vendor under a newer + patch uuid now removes the replaced uuid's artifact dir after the commit, + so its `vendor_stale_artifact_removed` event comes after the run's + per-package events instead of right after the package's own; a golang + takeover likewise deletes the `.socket/go-patches/` copy only after the + commit that repoints `go.mod` away from it. A commit that cannot be + written fails the run with the new top-level error `vendor_commit_failed` + (exit 1), leaving the pre-run lockfiles and ledger in place — or, when + putting back the files already replaced failed too, keeping the journal + (the message says so) for the next locked command to finish the commit. + `repair`, `vendor --revert` and `rollback` keep their per-entry saves. + +- **Vendored artifacts are no longer fsynced one by one.** The files a + vendored run produces under `.socket/vendor///` — patched copy + trees, the `.tgz` / `.whl` / `.nupkg` / `.jar` + `.pom` artifacts and their + `.sha1` sidecars, and the marker — are still written atomically (stage + + rename) but without their own `fsync`/`F_FULLFSYNC`. One durability + barrier syncs every such file and, once per directory, their directories + (with a single `F_FULLFSYNC` per device on macOS) before the next durable + commit point — a lockfile, `go.mod`/`go.sum`, `pom.xml`, `nuget.config`, + `package.json`, `pnpm-workspace.yaml`, the vendor ledger or the redirect + ledger — is written, so nothing durable ever names an artifact that could + still be lost. An artifact rebuilt in place that no commit point follows + (a drifted committed artifact healed with the lockfiles and ledger + unchanged) is synced by the same barrier at the end of the vendored run's + commit and when the command releases the apply lock, so no command + returns with an unsynced artifact the committed state names; a failed + barrier keeps its files pending for the next one. A crash can at worst + lose an artifact nothing durable names yet, which the next run rebuilds + (see `socket_patch_core::utils::durability` for the full argument). The + in-place `apply` of an installed tree keeps its per-file durable writes. + - **Release publishing decomposed into per-registry workflows.** The crates.io, npm, PyPI, and RubyGems legs of the `Release` workflow now live in their own workflows diff --git a/Cargo.lock b/Cargo.lock index 9dc768837..63d885591 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1602,6 +1602,16 @@ dependencies = [ "cfg-if", "cpufeatures 0.2.17", "digest", + "sha2-asm", +] + +[[package]] +name = "sha2-asm" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b845214d6175804686b2bd482bcffe96651bb2d1200742b712003504a2dac1ab" +dependencies = [ + "cc", ] [[package]] @@ -1664,6 +1674,7 @@ dependencies = [ "dialoguer", "flate2", "fs2", + "futures-util", "glob", "hex", "libc", @@ -1690,13 +1701,16 @@ dependencies = [ name = "socket-patch-core" version = "4.0.0" dependencies = [ + "aho-corasick", "base64", "flate2", "fs2", + "futures-util", "hex", "libc", "once_cell", "qbsdiff", + "rayon", "regex", "reqwest", "same-file", diff --git a/Cargo.toml b/Cargo.toml index c1e954830..ce052f346 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ sha1 = "=0.10.6" hex = "=0.4.3" reqwest = { version = "=0.12.28", features = ["rustls-tls", "json"], default-features = false } tokio = { version = "=1.50.0", features = ["full"] } +futures-util = { version = "=0.3.32", default-features = false, features = ["std"] } thiserror = "=2.0.18" walkdir = "=2.5.0" uuid = { version = "=1.21.0", features = ["v4"] } @@ -28,10 +29,12 @@ console = "=0.15.11" dialoguer = "=0.11.0" tempfile = "=3.26.0" regex = "=1.12.3" +aho-corasick = "=1.1.4" glob = "=0.3.4" toml_edit = "=0.25.12" once_cell = "=1.21.3" qbsdiff = "=1.4.4" +rayon = "=1.12.0" tar = "=0.4.46" flate2 = "=1.1.9" zip = { version = "=8.6.0", default-features = false, features = ["deflate"] } diff --git a/README.md b/README.md index 21e18aa83..5499c01e0 100644 --- a/README.md +++ b/README.md @@ -531,6 +531,40 @@ warning — it never breaks a command or pollutes `--json` output. `socket-patch cloned repo must never be able to redirect where patches come from or spend your token. (Full rationale: [docs/design/configuration.md](docs/design/configuration.md).) +One more env-only knob tunes *pacing* rather than routing. `scan` queries the patch API +with several requests in flight: against the authenticated endpoint, a quarter of the +requests a step has to make, between 8 and 32 (so a step with 128 or more requests runs +32 at once, one with 32 or fewer runs 8); against the public proxy, which shares one +server-side limit across anonymous callers, 4. The patch-record fetches behind `vex` and +`scan --vex` run up to 10 at once (4 on the proxy). +`SOCKET_API_CONCURRENCY=` overrides that, clamped to `1`-`32`; on the public proxy it +can only lower it. Set it when an endpoint in front of the API caps in-flight requests +per client — a self-hosted `--api-url`, a corporate reverse proxy, a WAF or a CDN — and +a scan starts reporting fewer patches than it should because some requests are being +rejected. `SOCKET_API_CONCURRENCY=1` sends one request at a time, the slowest and most +conservative setting. An unset, empty or non-numeric value leaves the defaults in place. + +A throttled patch API is retried, within bounds. An HTTP `429` or `503` answer to any +patch-API query (batch search, patch lists, patch views, VEX record fetches, hosted +package references) is retried up to 3 times, waiting as long as the server's +`Retry-After` asks (seconds or an HTTP date; a request asked to wait more than 30 s gives +up at once) or, without one, 0.5 s, 1 s, 2 s with jitter. All retries in one run must +finish within 60 s of the run's first retry (wall-clock: requests waiting in parallel +don't add up), so a heavily throttled run gives up instead of hanging. `SOCKET_API_MAX_RETRIES=` changes +the per-request count (`0`-`10`; `0` turns retries off). Other errors are never retried. +A query still throttled after its retries is reported, never dropped: a failed batch +prints `Warning: API batch of failed: …` (under `--json`, a top-level +`warnings[]` entry with code `api_batch_failed`), a failed patch-list lookup prints +`Warning: could not fetch details for : …` (`--json`: `patch_details_failed`), and +if every query fails the scan exits 1 with an error, as before. + +The crawl has a pacing knob too. Its directory walks (`node_modules`, and the Maven +repository with its POM parse) run on a small pool of threads: 4 by default (fewer on a machine with fewer performance cores), because the walk is bound +by the kernel's directory cache and more threads only add system time. +`SOCKET_WALK_THREADS=` overrides that, clamped to `1`-`16` and to the machine's CPU +count; an unset, empty or non-numeric value leaves the default in place. A soft open-file +limit below 128 still runs the walk on one thread, whatever the knob says. + The sections below list only each command's **command-specific** flags. ### `scan` @@ -566,7 +600,7 @@ socket-patch scan [options] | `--mode ` | — | Selects one of the three [patch modes](#three-patch-modes), summarized above. Combining `--mode` with a legacy boolean flag of a *different* mode is an error (exit 2); the same mode spelled both ways is accepted. | | `--prune` | — | Garbage-collect after the scan: remove manifest entries for packages no longer present in the crawl (installed trees + lockfiles — a wiped `node_modules` alone doesn't prune lockfile-listed entries) and delete orphan blob/diff/package-archive files. Off by default. [Vendored](#vendor) packages are exempt from the crawl-based prune (an absent installed copy is their normal state), but a vendored entry whose dependency has left the lockfile is reverted (and any manifest entry it still had dropped). Orthogonal to `--mode` — combines with any mode. | | `--detached` | — | Hidden compatibility no-op. Vendored mode is manifest-free by default: the vendor ledger (`.socket/vendor/state.json`) embeds the patch records and `.socket/manifest.json` is never written, so this former opt-in changes nothing. Still an error without `--mode vendored`. | -| `--batch-size ` | `SOCKET_BATCH_SIZE` | Packages per API request (default: `100`). | +| `--batch-size ` | `SOCKET_BATCH_SIZE` | Packages per API request (default: `500` on the authenticated API, `100` on the public proxy). A request whose body would exceed 256 KiB is split into smaller ones. | | `--all-releases` | `SOCKET_ALL_RELEASES` | Store patches for every release/distribution variant, not just the installed one — PyPI wheel/sdist, RubyGems platform, Maven classifier. Makes the manifest portable across environments (e.g. cross-platform CI caches). | | `--vex ` | `SOCKET_VEX` | On a successful scan, also write an OpenVEX 0.2.0 document to this path. See [Inline VEX generation](#inline-vex-on-apply--scan--vendor). | | `--vex-product`, `--vex-no-verify`, `--vex-doc-id`, `--vex-compact` | `SOCKET_VEX_*` | Passthrough to the embedded VEX builder; mirror the standalone [`vex`](#vex) knobs. Inert unless `--vex` is set. | diff --git a/crates/socket-patch-cli/CLI_CONTRACT.md b/crates/socket-patch-cli/CLI_CONTRACT.md index f6991b705..594ec5f01 100644 --- a/crates/socket-patch-cli/CLI_CONTRACT.md +++ b/crates/socket-patch-cli/CLI_CONTRACT.md @@ -80,7 +80,7 @@ Beyond the globals above, each subcommand defines a small set of local arguments | `scan` | `--redirect` | — | Hosted mode's legacy boolean spelling (**hidden from `--help`** and **deprecated** — `--mode hosted` is the documented spelling; this alias is scheduled for removal in v4): rewrite lockfiles / registry configs so ONLY the patched dependencies resolve to Socket's hosted patch server; no artifact bytes land in the repo. Conflicts with `--apply`/`--sync`/`--vendor` | | `scan` | `--apply` / `--prune` / `--sync` | — | Mode selectors (sync = apply + prune); `--apply` == `--mode agent` | | `scan` | `--vendor` / `--detached` | — | Vendor every patched dependency instead of applying in place (`--vendor` == `--mode vendored`; conflicts with `--apply`/`--sync`, combines with `--prune`). Vendored mode is manifest-free (v5.0): the vendor ledger embeds the patch records and `.socket/manifest.json` is never written. `--detached` — the former opt-in for exactly that — is **hidden** and retained for compatibility as a no-op; it is still a usage error (exit 2) without vendored mode in either spelling | -| `scan` | `--batch-size` | `SOCKET_BATCH_SIZE` | API batch chunk size (default `100`) | +| `scan` | `--batch-size` | `SOCKET_BATCH_SIZE` | API batch chunk size. Unset (v5.0): `500` on the authenticated API (the server's per-request maximum), `100` on the public proxy; a given value applies on either endpoint (`0` is floored to `1`). A chunk whose request body would exceed 256 KiB (the public proxy's body cap) is split into consecutive smaller chunks, deterministically (greedy, in crawl order). A mid-run downgrade to the proxy keeps the chunks already formed | | `get`, `scan` | `--all-releases` | `SOCKET_ALL_RELEASES` | Download patches for every release/distribution variant of a matched package — PyPI wheel/sdist (`artifact_id`), RubyGems (`platform`), Maven (`classifier`) — not just the one(s) matching the locally-installed distribution. On `scan` this makes the stored manifest portable across environments (e.g. cross-platform CI caches). On `get` (v3.6) it ALSO disables the coarse installed-**version** narrowing of CVE/GHSA fan-outs (see "get --mode and installed narrowing"): every found version's patch is fetched, installed or not | | `get` | positional `identifier`; `--id` / `--cve` / `--ghsa` / `--package` (`-p`); `--save-only` (alias `--no-apply`); `--one-off` (hidden from `--help`: always fails "not yet implemented"); `--mode ` | `SOCKET_SAVE_ONLY`, `SOCKET_ONE_OFF` | Patch lookup + consumption mode (v3.6). `--mode` reuses scan's value enum (same hidden value aliases `host`/`redirect`/`vendor`; deliberately no env binding, matching scan). Default `agent` = today's save+apply flow, unchanged. `--save-only` conflicts with `--mode hosted\|vendored` — rejected with **exit 1** via get's established self-enforced-conflict style (unlike scan's exit-2 mode conflicts; see the exit-code table) | | `remove` | positional `identifier`; `--skip-rollback`; `--preserve-state` (v5.0) | `SOCKET_SKIP_ROLLBACK`, `SOCKET_PRESERVE_STATE` | Manifest entry removal. `--preserve-state` is the single-patch twin of `rollback --preserve-state`: restore the tree and unwind the identifier's vendored/hosted wiring, but keep the manifest entry, the vendored artifact + ledger entry, and skip all GC. Combining it with `--skip-rollback` is a self-enforced usage error (exit 2): one flag keeps the tree and drops the state, the other restores the tree and keeps the state — together they select the do-nothing quadrant ("the combination would be a no-op: nothing would change"). The conflict fires whether either flag is spelled on the command line or sourced from its env var | @@ -111,12 +111,20 @@ For a **9.0 root lock**, the CLI ensures `pnpm-workspace.yaml` carries `trustLoc `scan` queries the patch API in `--batch-size` chunks. Authenticated runs POST `/v0/orgs/{slug}/patches/batch`; token-less runs POST `{proxy}/patch/batch` on the public proxy and degrade to per-package `GET /patch/by-package/:purl` requests in two cases: the deployed proxy predates the batch endpoint (legacy proxies answer the POST with their `400 "Unsupported endpoint"` catch-all), or the all-or-nothing batch validation rejects the chunk (e.g. a crawled PURL type the server doesn't recognize, such as `pkg:jsr/…` — the per-package path tolerates those individually, preserving the pre-batch scan semantics). Rate limits and over-capacity 503s surface instead of silently degrading. +**Throttling: bounded retry, then a reported failure.** Every patch-API JSON call (the batch query, the per-package patch lists, patch views and VEX record fetches, hosted package references) retries an HTTP `429` or `503` answer up to 3 times (`SOCKET_API_MAX_RETRIES=`, `0`-`10`; `0` = no retry). The wait honors `Retry-After` (delta-seconds or HTTP-date); a `Retry-After` over 30 s is not waited out — the answer is final at once — and one under the jittered first backoff step (`0`, a past date) waits that step instead. Without one it backs off 0.5 s / 1 s / 2 s (each step up to 8 s, with jitter in its upper half). All retries in one run share a 60 s wall-clock window that opens with the run's first retry: a retry whose wait would end after it closes is refused and the answer is final. Parallel requests wait in parallel, so each still gets its retries while the run adds at most about 60 s. Nothing else is retried (401/403 still drive the proxy fallback on the first answer; the public proxy's permanent `503 "Patch API is not configured"` is never retried on any path — the batch query still degrades to the per-package path at once, and a per-package lookup or patch view answering it is the same non-throttle failure it always was, so the legacy per-package path still skips that package), and a retried answer folds exactly where the first attempt's would have, so output is identical to an unthrottled run's. A request still throttled after that is a failure in the channel its siblings use: a failed batch is the human `Warning: API batch of failed: ` line and, under `--json`, a run-level `warnings[]` entry `{code: "api_batch_failed", detail: "API batch of failed: "}` (additive; `status` stays `success`, exit 0 — the other batches' packages are reported); a failed per-package patch-list query in the agent / hosted / vendored flows is the human `Warning: could not fetch details for : ` line and, under `--json`, `{code: "patch_details_failed", detail: "could not fetch details for : "}`. When every batch (or every patch-list query) fails, the existing all-failed error envelope and exit 1 apply. The error names the exhausted retry: `Rate limit exceeded (HTTP 429, gave up after 3 retries). Please try again later.` / `API request failed with status 503: (gave up after 3 retries)` (or `(Retry-After s exceeds the 30 s retry cap)` / `(the run's 60 s retry window has closed)`); with retries off it is the pre-retry text. On the token-less legacy per-package proxy path (a proxy without `POST /patch/batch`), a package still throttled (429 / over-capacity 503) after its retries fails its whole batch query, so every package in that batch goes unchecked and is reported through the batch-failure channel above (an unresolvable PURL, or a "not configured" 503, is still skipped individually). Before this, a throttled batch vanished from a `--json` envelope without a trace. Pinned by `tests/scan_api_retry_e2e.rs` and the core crate's `tests/api_retry_e2e.rs`. + **Lockfile supplement (v3.4)**: `scan` discovery is no longer limited to installed trees. The project's lockfiles (`package-lock.json`/`npm-shrinkwrap.json`, `pnpm-lock.yaml` v9, `yarn.lock` classic + berry, `bun.lock`, `Cargo.lock`, `go.sum`, `composer.lock`, `Gemfile.lock`, `uv.lock`/`poetry.lock`/pinned `requirements.txt`) are inventoried and dependencies with NO installed copy join discovery — counts, the API lookup, the table (flagged ` [NOT INSTALLED]`, plus a stderr note), and the prune "scanned" set (a wiped node_modules no longer prunes lockfile-listed entries). JSON gains a top-level `lockfileOnlyPackages` count and an additive `notInstalled: true` on matching `packages[]` entries. `--apply` partitions lockfile-only patches out BEFORE download (calm `skipped`/`package_not_installed` records — never an error exit, never a manifest write); `--vendor` passes them through to the vendor engine's auto-fetch. Vendored-ledger entries likewise stay discoverable on a fresh clone (the committed artifact is the dependency). Global scans (`--global`) get no supplement. **Rush monorepos** (no root lockfile, `rush.json` present): the npm-lock inventory falls back to the Rush source-of-truth locks — `common/config/rush/pnpm-lock.yaml` plus every `common/config/subspaces/*/pnpm-lock.yaml` (`read_dir`-sorted, repo-relative paths preserved) — so a Rush repo's dependencies still join discovery. **Plug'n'Play layouts are an explicit refusal, not an empty inventory**: a `.pnp.*` loader means the npm packages are structurally unreachable in EVERY mode (under yarn PnP the installed-tree crawl is empty too — no `node_modules/`), so `scan` surfaces an additive top-level `warnings[]` array (`{code, detail}` objects, omitted when empty) carrying `yarn_pnp_unsupported` (same code as apply's refusal; remedy `yarn patch `) or `pnpm_pnp_unsupported` (pnpm's `node-linker=pnp` twin; pnpm remedies), plus a stderr `Warning (): …` line on the human path. Exit code and `status` are deliberately unchanged (exit 0 / `success` — the same posture as hosted refusals, which exit 0 with `redirected: 0`); the warning is the machine-readable signal that nothing was checked. Pinned by `tests/e2e_safety_yarn_pnp.rs`. -**Vendor auto-fetch (v3.4)**: `vendor`/`scan --vendor` no longer fail on lockfile-resolved packages with no installed copy. Already-vendored purls stage from their committed artifact (sha256-verified against the vendor ledger; offline-safe). Otherwise the pristine artifact is fetched per the lockfile resolution and verified against the lock's recorded integrity FAIL-CLOSED before any write: npm SRI (or yarn classic's sha1 fragment), yarn berry's cache-zip checksum (rebuilt from the fetched tarball; cacheKey 10c0 only), Cargo.lock sha256 over the .crate, go.sum `h1:` dirhash over the module zip, composer `dist.shasum` (sha1), Gemfile.lock `CHECKSUMS` sha256, uv.lock wheel sha256 (pure `py3-none-any` wheels only). Entries the lock cannot verify are NEVER fetched (`vendor_fetch_unverifiable` warning + the calm `package_not_installed` skip). Registry bases honor `SOCKET_NPM_REGISTRY`, `SOCKET_CRATES_REGISTRY`, `SOCKET_GOPROXY` (else `GOPROXY`, `GONOPROXY` and `GOPRIVATE` the way go reads them — see the env table); npm/yarn/composer/gem/uv lock-recorded URLs are used verbatim. `--offline` refuses the fetch with the calm skip (the detail names the lockfile resolution). The fetch stages into a private tempdir — the project tree is never touched. +**Vendor auto-fetch (v3.4)**: `vendor`/`scan --vendor` no longer fail on lockfile-resolved packages with no installed copy. Already-vendored purls stage from their committed artifact (sha256-verified against the vendor ledger; offline-safe). Otherwise the pristine artifact is fetched per the lockfile resolution and verified against the lock's recorded integrity FAIL-CLOSED before any write: npm SRI (or yarn classic's sha1 fragment), yarn berry's cache-zip checksum (rebuilt from the fetched tarball; cacheKey 10c0 only), Cargo.lock sha256 over the .crate, go.sum `h1:` dirhash over the module zip, composer `dist.shasum` (sha1), Gemfile.lock `CHECKSUMS` sha256, uv.lock wheel sha256 (pure `py3-none-any` wheels only). Entries the lock cannot verify are NEVER fetched (`vendor_fetch_unverifiable` warning + the calm `package_not_installed` skip). Registry bases honor `SOCKET_NPM_REGISTRY`, `SOCKET_CRATES_REGISTRY`, `SOCKET_GOPROXY` (else `GOPROXY`, `GONOPROXY` and `GOPRIVATE` the way go reads them — see the env table); npm/yarn/composer/gem/uv lock-recorded URLs are used verbatim. `--offline` refuses the fetch with the calm skip (the detail names the lockfile resolution). The fetch stages into a private tempdir — the project tree is never touched. **Deferred fetch (v5.0):** a purl the vendor ledger already covers (its entry records the record's patch uuid and the committed artifact is on disk — a file artifact only while it hashes to the ledger's `sha256`; not under `--force`), and a lockfile-only cargo crate the registry could fetch and verify (a crates.io `Cargo.lock` entry with a checksum, or the pre-vendor resolution the ledger recovers) while the patch service is enabled, are NOT downloaded up front: the fetch runs only if the backend reaches a branch that reads the pristine tree (a drifted committed copy rebuilt locally, a service miss). An in-sync re-run therefore makes no registry request, reports no `vendor_fetched_missing`, and succeeds with no network or under `--offline`. A deferred fetch that does run records its `vendor_fetched_missing` just ahead of the package's own event; one that fails, is unverifiable, or is refused by `--offline` reports the same events the up-front fetch would have. A git, path or custom-registry cargo crate is never deferred, so it keeps `vendor_fetch_unverifiable` + `package_not_installed` and is never vendored from the service's crates.io build. **Gem, local build only** (`--vendor-source build`, or no service config): a not-installed gem the lock can verify (bundler >= 2.6 `CHECKSUMS`) and no ledger entry covers is refused `gem_spec_missing` (`failed`, the backend's own detail) BEFORE any download — a downloaded `.gem` carries no eval-able stub gemspec, so a local build can never vendor it; no `vendor_fetched_missing` precedes it, and a refusal the backend would have reached first on the fetched copy reports as `gem_spec_missing` too. Not under `--dry-run`, which still fetches and previews the gem (`vendor_fetched_missing` + `verified`). + +**Vendored write durability (v5.0)**: every write is atomic (stage + rename), but only the durable commit points — lockfiles, `go.mod`/`go.sum`, `pom.xml`, `nuget.config`, `package.json`, `pnpm-workspace.yaml`, `.cargo/config.toml`, the Python/Ruby manifests, `.socket/vendor/state.json` and `redirect-state.json` — are fsynced on write. The content-verified artifacts under `.socket/vendor///` (patched copies, packed/rebuilt archives and sidecars, markers) are written without an fsync and made durable by one barrier (file + directory fsync, one `F_FULLFSYNC` per device on macOS) ahead of the next commit point — and, for an artifact rebuilt in place that no commit point follows, at the end of the vendored run's commit and when the command releases the apply lock — so a crash can only lose an artifact that no durable commit point names yet, which the next run rebuilds. + +**Vendored group commit (v5.0)**: `vendor`, `scan --mode vendored` and `get --mode vendored` capture every lockfile / manifest / config edit and every ledger save of the run in memory (reads inside the run see them) and commit them ONCE after the per-package loop — including the packages that succeeded in a run where others failed, so a completed run leaves the same files per-package commits would. Captured: every file under the project root outside `.socket/`, plus `.socket/vendor/state.json` and `.socket/vendor/redirect-state.json`; artifacts are written directly (see the durability note). A multi-file commit goes through a roll-forward journal, `.socket/vendor/.commit-journal.json` (the new bytes of every changed file, plus the bytes each replaces and their sha256; deleted once the commit completes). **Crash semantics**: before the journal is durable, nothing is committed — the lockfiles and ledgers are the pre-run ones and the run's artifacts are unreferenced orphans; after it, the next command that takes the apply lock replays the journal before reading anything (files already at their new bytes are left alone), so a locked command never observes a half-committed run. A journal that matches neither side of some file (edited by hand since the crash) is renamed to `.socket/vendor/.commit-journal.set-aside-.json` (keeping every file's pre-commit bytes) and stderr says what was done (`Warning: an interrupted vendored run's commit could not be finished as written: …`): the edited files are never written over; when they all still carry the commit's own lines the rest of the commit is finished around them, when none of them does the files the crash had already replaced are put back to their pre-commit bytes, and otherwise nothing is applied. A journal that is unreadable, names a path outside the lockfiles and ledgers, or would write through a symbolic link is set aside with nothing applied. A replay that fails on I/O keeps the journal and fails the lock acquire (`lock_io`, naming the journal). Read-only commands that take no lock (`vex`, `list`) may observe the interrupted state until then. A re-vendor under a newer uuid removes the replaced uuid's dir only after the commit (its `vendor_stale_artifact_removed` event follows the run's per-package events), and a golang takeover removes the `.socket/go-patches/` copy only after the commit that repoints `go.mod`. A commit write failure is the top-level error `vendor_commit_failed` (exit 1; the pre-run lockfiles and ledger stay — unless putting back the files already replaced failed too, in which case the journal is kept and the next locked command finishes the commit). `repair`, `vendor --revert` and `rollback` still save per entry. `scan --sync` is sugar for `--apply --prune` — the canonical single-flag bot invocation. `scan --json --sync --yes` discovers, applies, and reconciles state in one pass. +**`scan --ecosystems` scopes the crawl (v5.0)**: without `--prune`/`--sync`, a `scan` given `--ecosystems`/`-e` runs only the named ecosystems' crawlers — everything the run counts, queries and shows (`scannedPackages`, the batch query, `packages[]`, the table, `updates[]`, `wiringLive`, the `gem_bundle_config_path_ignored` warning) was already narrowed to them, so the skipped crawls could only be filtered away. The one visible difference: `lockfileOnlyPackages` (and the human "not yet installed" note) counts only the selected ecosystems' lockfile-only entries — previously it also counted other ecosystems' uninstalled lockfile entries, which a skipped crawl can no longer vouch for. A GC run (`--prune`, or `--sync`, which implies it — in every mode, hosted included) still crawls every ecosystem, because the prune judges each manifest entry against the FULL installed set (see `scan --prune` above); its output, `lockfileOnlyPackages` included, is unchanged. Without `--ecosystems` nothing changes. Pinned by `tests/scan_ecosystems_scope_e2e.rs`. + **Path-scoped scans (`scan [PATHS]...`, v5.0)**: optional variadic positional path globs scope DISCOVERY at the **purl level** — a package is in scope iff ANY of its crawled installed copies sits under a matching path, and a selected package is then handled with ALL its copies (scoping selects which packages are considered, never which copies). Glob semantics (shared with `rollback`'s path targets, `src/path_scope.rs`): Unix-shell globs with `require_literal_separator` — `*`/`?` never cross a `/`, `**` spans directories; a pattern matching any **ancestor** directory of the copy path also matches, so a bare `scan packages/foo` scopes the whole subtree without `/**`; relative patterns match against the copy path relativized to `--cwd`, absolute patterns against the absolute path (the ONLY way to reach paths outside the project tree, e.g. `--global` stores — a relative pattern never matches outside `--cwd`); leading `./` and trailing `/` are normalized away, matching is purely textual (no filesystem access or symlink resolution), case-sensitive except on Windows (whose filesystems are not); an unparseable or empty pattern is a usage error (exit 2). **The prune universe is never narrowed**: the path filter is applied strictly AFTER the `scanned_purls` capture (and after `--ecosystems`), so `scan PATHS --prune` prunes exactly what an unscoped `scan --prune` would — a scoped scan can never treat an out-of-scope package as uninstalled (the same fail-safe as the `--ecosystems` filter). Lockfile-only and vendor-ledger supplement records have no installed path and are EXCLUDED from a path-scoped scan, surfaced as one run-level `path_scope_excluded_supplements` warning carrying the count. A scope matching nothing is a normal empty scan — exit 0, zero packages, **no GC** (the zero-package early return fires before any GC). `PATHS` with `--mode hosted` or `--mode vendored` is a usage error (exit 2, `resolve_mode_flags`: "path targeting … applies to agent-mode and read-only scans" — their lockfile rewiring is whole-project by construction); `PATHS` with `--apply`/`--sync`/`--prune`/`--global` is fine. Every scan JSON shape (success, zero-package, and error alike) gains an additive always-present `paths` key echoing the patterns verbatim (empty array when unscoped). One-sentence duality rule: **a target that selects nothing is an error on `rollback` (exit 1) and an empty scan on `scan` (exit 0)**. `scan --vendor` swaps the in-place apply for the vendor pipeline: discover → download the selected patch records **into memory** (no manifest write) → vendor every selected dependency via the same engine as the `vendor` command (under the same lock). Vendored mode is **manifest-free (v5.0)**: `.socket/manifest.json` is never written or read by a vendored run; each ledger entry carries `detached: true` plus an embedded copy of the patch record (`record`) as its verification source, and the run's footprint is `.socket/vendor/**` only. The vendor step's scope is what discovery selected — the former "whole manifest is vendored" re-vendor on an empty discovery is retired (`repair` verifies and rebuilds committed vendored state; `scan --prune` reconciles ledger entries whose dependency left the lockfile). A package the ledger holds at an older patch uuid is still **re-vendored automatically** when discovery selects the newer patch (its old uuid dir is removed — `vendor_stale_artifact_removed`); same-uuid re-runs reuse the embedded record, skip the patch-view fetch, and are `already_vendored` skips. **Legacy manifest-mode entries**: when a vendored run vendors a purl that also has a `.socket/manifest.json` record (a project vendored by a pre-5.0 binary, or by standalone `vendor` from an agent-mode manifest), that manifest record is dropped in the same run — the ledger becomes the owner (migration write); an emptied manifest is left as `{"patches": {}}`, never deleted. The migration is reported through the run-level `warnings[]` (stderr in human mode), never as a run error: `vendor_manifest_record_migrated` (`N manifest records moved to the vendor ledger (vendored mode is manifest-free): `) or `vendor_manifest_migration_failed` (the manifest or the ledger could not be read or rewritten; the legacy records were left in place) — so a corrupt `.socket/manifest.json` no longer fails a vendored run (standalone `vendor`, the one manifest-driven writer, still fails closed on it). With `--prune`, GC runs **after** the vendor step (the step never reads the manifest, and running the sweep last lets it reclaim what the run itself orphaned — a migrated legacy record's blobs, a superseded uuid dir). JSON output gains a `download` sub-object — the detached download envelope `{found, downloaded, skipped, failed, detached: true, patches: [{purl, uuid, action: "downloaded" | "skipped" | "failed", …}], warnings?}` (no `applied` field — nothing is applied in place; `detached: true` is pinned and always present; a `downloaded` record whose purl the ledger already holds at another uuid carries the additive `oldUuid` — the re-vendor the vendor step then performs — and its human `[fetch]` line reads ` (replacing )`) — and a `vendor` sub-object (a full vendor Envelope). Patch blobs are held in memory (see "Patch sources stay in memory" under the vendor contract). `--dry-run` previews per-patch `would_vendor` | `would_revendor` (+`oldUuid`) | `already_vendored` — plus, additive, `would_refuse` (+`errorCode`, `error`) for npm purls the wet run's Bun preflight (see the `get --mode vendored` bullet below) would refuse — without network downloads or disk writes; the preview never flips status or exit (the human path — `scan` and `get` alike, through one shared printer — prints `[would-refuse] (): ` lines behind the `--silent` gate). Interactive mode prompts "Download and vendor N patches?" (singular for one). @@ -133,13 +141,15 @@ The rewriter reads a fixed set of candidate files from the project root: the npm **Mode ledgers (contract surfaces).** Each committable mode persists its state at a stable repo-relative path; external tools (and the depscan backend's GitHub-app PR flows) read and write these files, so path + schema are part of the contract: -* `.socket/vendor/state.json` — the **vendored**-mode ledger (see "Ownership, state, and reversal" below): wiring edits with verbatim pre-vendor originals, artifact fingerprints, and the embedded patch `record` — for every entry written by `scan`/`get --mode vendored` beside `detached: true` (the record is that entry's only source), and for standalone `vendor` fed by an agent-mode manifest as a fallback copy without `detached` (the manifest record stays authoritative while the manifest covers the entry, by ledger key or base purl; `vex`, `list` and `setup --check` fall back to the embedded copy when it does not, `repair` only with no manifest at all). Entries written before 5.0 by standalone `vendor` carry no `record`; readers tolerate its absence. +* `.socket/vendor/state.json` — the **vendored**-mode ledger (see "Ownership, state, and reversal" below): wiring edits with verbatim pre-vendor originals, artifact fingerprints, and the embedded patch `record` — for every entry written by `scan`/`get --mode vendored` beside `detached: true` (the record is that entry's only source), and for standalone `vendor` fed by an agent-mode manifest as a fallback copy without `detached` (the manifest record stays authoritative while the manifest covers the entry, by ledger key or base purl; `vex`, `list` and `setup --check` fall back to the embedded copy when it does not, `repair` only with no manifest at all). Entries written before 5.0 by standalone `vendor` carry no `record`; readers tolerate its absence. **Schema version 2 (v5.0)**: the `new` of a whole-file wiring record (kinds `maven_pom_repository`, `nuget_config_source`, `python_lock_document`, `python_script_metadata`, `hatch_document`) of 1 KiB or more, when its `original` is a string, is stored as an edit of that same record's `original`: `{"snapshot": "", "ops": [[start, len] | "inserted text", …]}` (the text is the ops concatenated in order: a `[start, len]` byte range copied from the `original`, a string inserted as is), and the ledger's `version` is `2`; the `original` stays a plain string, no other record kind is touched, and a ledger without such a record keeps the version-1 bytes. Both versions are read; a version-2 edit is rebuilt and checked against its hash (a mismatch, a missing `original`, an out-of-range copy, or any other `{"snapshot": …}` value is `vendor_state_unreadable`), so every consumer sees the same full texts as with an inline version-1 ledger. Records are self-contained, so an older socket-patch re-saving a version-2 ledger (it keeps `original` / `new` verbatim and drops unknown fields) loses nothing. * `.socket/vendor/redirect-state.json` — the **hosted**-mode ledger (`RedirectState` in `socket-patch-core/src/patch/redirect/state.rs`): `{ version, mode: "hosted", edits[], records{} }`. `edits` are recorded `FileEdit`s (append-only across re-runs — merge, never clobber: the pre-redirect originals a future revert needs live here; v5.0: a byte-identical re-save is skipped, which still satisfies the rule); `records` maps PURL → the full manifest `PatchRecord`, one of `vex`'s record sources for redirected patches with no manifest entry (a record attests only while a lockfile still wires its hosted patch — see "Manifest-less VEX" below). The `mode` string is opaque to the loader (pre-rename ledgers carrying `"redirect"` still load; a hosted re-run normalizes them to `"hosted"`). Written identically by this CLI and by the depscan backend's hosted PR flow (`github-patch-pr-hosted.ts`). **get --mode and installed narrowing (v3.6).** `get --mode hosted|vendored` consumes the resolved patch(es) through the SAME engines as `scan --mode hosted|vendored`, so for the same selected (purl, uuid) set the on-disk result is identical by construction — this is the per-advisory selector hosted/vendored previously lacked (the old workaround, `get --save-only` then `vendor`, still works but is superseded). **Agent mode (v5.0 lock + residue rules)**: the download phase runs under `<.socket>/apply.lock` and hands the guard to the nested apply, so download → manifest write → apply is one lock window (the nested apply never re-acquires and inherits every caller flag — `--lock-timeout` and `--verbose` included); a failed acquire is `{status: "error", errorCode: "lock_held" | "lock_io", error}` on get's legacy envelope, exit 1, before any fetch (a read-only `.socket/` fails here, naming the lock path). `.socket/` and `.socket/blobs/` are created only when a record is actually persisted — an all-skipped or all-failed run leaves no `.socket/` on a fresh project — and a same-uuid `get ` re-run rewrites neither the manifest nor the blobs. Semantics: * **Hosted** (`get GHSA-… --mode hosted`): resolves the advisory, then hands the selected (purl, uuid) pairs to scan's hosted engine — reference grants, cross-mode takeover pre-revert, lockfile rewrite, `redirect-state.json` ledger (merge-never-clobber), gem stale-install probe, warnings, confirmation rules (cargo via `confirmed_cargo_uuids`, golang via `confirmed_golang_uuids` only) all identical to `scan --mode hosted`, and (v5.0) under the same `apply.lock` acquisition — taken around the first wet write, never on `--dry-run` or when nothing would be written; a failed acquire folds as top-level `errorCode: "lock_held" | "lock_io"` + string `error` (exit 1), and `--dry-run` under a held lock still exits 0. **No manifest write, no blobs** — the ledger is the persistence. JSON: get's legacy envelope gains the same nested `redirect` sub-object as scan's (`{mode:"hosted", redirected, rewrittenFiles, skipped, warnings, dryRun}`); the top-level shape is `{status, found, patches:[], warnings?}` — `downloaded`/`applied` are absent (nothing is downloaded into `.socket/`). Exit codes follow scan's hosted semantics: skipped grants and rewriter warnings never flip the exit; infra errors (reference fetch, corrupt/unwritable ledger, file writes) exit 1. Human prompt: `Redirect N packages to the hosted patch server?` (singular for one; get keeps its confirm gate, `--yes`/`--json`/non-TTY auto-accept as usual; as of v5.0 human `scan --mode hosted` prompts too — see the hosted section above). * **Vendored** (`get GHSA-… --mode vendored`): the download phase is scan's vendored posture — **manifest-free (v5.0)**: the selected records are fetched into memory (`download_patch_records`; blobs held in memory; nothing under `.socket/` is written; the nested apply never runs), then scan's vendor step runs under the apply lock over exactly the selected records, like `scan --mode vendored` (no whole-manifest scope and no `[note]` about other records — that blast radius is retired with the manifest; a legacy manifest record for a vendored purl is migrated out of `.socket/manifest.json` the same way scan does it). JSON: get's envelope takes the detached download envelope's shape — `{status, found, downloaded, skipped, failed, detached: true, patches: [{purl, uuid, action: "downloaded" | "skipped" | "failed", …}], warnings?}` (`applied` is absent; `detached: true` is pinned; a `downloaded` record for a purl the vendor ledger holds at another uuid carries the additive `oldUuid`, derived from the ledger — the human `[fetch]` line reads ` (replacing )`) — and gains the nested `vendor` Envelope exactly like scan's `result["vendor"]`; a vendor-step error folds the partial envelope + `{status:"error", error:{code,message}}` in (a pre-failure takeover reconcile may have already mutated the ledger — its events must reach the consumer). Exit: download failures or vendor `has_errors` → `partial_failure`/1. Human prompt: `Download and vendor N patches?`; `--dry-run` prints `[dry-run] Would download and vendor N patches. No changes made.` on both identifier paths (uuid and search). Telemetry mirrors scan's vendored arms (`track_outcomes_for_vendor` / `track_patch_vendor_failed`). **Bun vendored preflight (additive)** — shared by `get --mode vendored` on both its paths and `scan --mode vendored`: before ANY patch download, and only when the selection holds a `pkg:npm/` purl, the download phase reads `bun.lock`/`bun.lockb` once (`preflight_vendor`) and, when the vendor backend would refuse the project — a malformed, unreadable or unsupported `bun.lockb` → `vendor_bun_lockb_invalid`; an unreadable `bun.lock` → `vendor_lockfile_missing`; a `lockfileVersion` other than 0/1/2 or a non-canonical `packages` grammar → `vendor_lockfile_version_unsupported`; `workspace:` packages in a lock below version 2 → `vendor_bun_workspace_unsupported` — every `pkg:npm/` result becomes `{action:"failed", errorCode:, error:}` with NO fetch (the patch view is never requested) and no patch record; other ecosystems' results are untouched. **Search path** (`get --mode vendored`) and `scan --mode vendored`: the records ride `patches[]` / `download.patches[]` with `downloaded: 0`, the download phase writes nothing under `.socket/` (v5.0 — a pre-existing `.socket/manifest.json`, including a record seeded for another purl, is left byte-untouched; previously the run re-serialized the manifest), the vendor step still runs over the remaining records (no event for the refused purl), exit `partial_failure`/1. **uuid path** (`get --mode vendored`): the uuid lookup is the only fetch; the run exits 1 BEFORE the vendor step with exactly `{status:"error", found:1, downloaded:0, skipped:0, failed:1, error:{code, message}, patches:[{purl, uuid, action:"failed", errorCode, error}]}` (the `error` OBJECT is the vendored-mode error shape of the vendor-step fold-in above) and writes nothing — no `.socket/` on a fresh project; human mode prints `Error (): ` on stderr. **Already-vendored exemption**: a purl is exempt from the workspace refusal only when every instance of its `name@version` in `bun.lock` is already a `.socket/vendor/npm/…` local tuple (any uuid; the digest-less 2-tuple counts) — the engine's own criterion — so in-sync re-runs, `repair`, and a superseding patch uuid on a project vendored before it grew a workspace member all flow to the engine (re-pinning an already-local tuple adds no workspace-relative exposure); a wiped ledger alone is not a refusal (the engine path decides). UUID equality in the ledger alone never exempts a purl: `rollback --preserve-state` retains its record after unwiring. Dry-run refusal takes priority over `already_vendored`. **Unreadable vendor ledger**: a `.socket/vendor/state.json` the preflight cannot read or parse is itself the refusal — `vendor_state_unreadable` with the io/parse detail, fail-closed (nothing is exempt) — on the uuid path, the search / `scan` path and the `--dry-run` preview alike; never a Bun lock code. **`--silent`** is "errors only" and never mutes the refusal: the code-tagged `[error] (): ` (per-patch paths) / `Error (): …` (uuid path) line stays on stderr with an empty stdout. **`--dry-run`** previews the refusal as the additive `would_refuse` action (see `--dry-run` below). Agent-mode `get --save-only` is NOT preflighted (record-only intent has no consumption precondition). Pinned by `tests/in_process_vendor_bun.rs` (exact uuid-path envelope, seeded-manifest survival, `--silent`, `--dry-run`) and `tests/scan_vendor_e2e.rs`. + +**Lock-text refusals before the download (v5.0)** — shared by `get --mode vendored` on both its paths and `scan --mode vendored`, after the Bun preflight above and the ledger's `already vendored` skip: a `pkg:npm/` result in a **pnpm, yarn classic or yarn berry** project, or a `pkg:cargo/` result, that its vendor backend refuses on the project's lock and manifest text alone is refused BEFORE its patch view is fetched — the pnpm / classic / berry gates the backend runs before it reads the package (coordinates, the lock and manifest reads and their line-ending / version / `cacheKey` / `.yarnrc.yml` gates, override and `resolutions` conflicts, the lock entry present and rewritable) and cargo's `locked_version_mismatch` (only when it is the crate's FIRST refusal; an in-tree `cargo vendor` copy still refuses in the loop as `already_vendored_in_tree`). **Scope:** only a package the vendor loop would hand to its backend is refused early — one installed on disk (the loop's own qualified-aware resolver plus the npm identity lookup), or one the lockfile inventory resolves to a verifiable registry source (a lock entry with an integrity, or the ledger-recovered pre-vendor resolution — exactly the entry the pristine fetch would use). A package absent from the lock and not installed never reached its backend and is untouched: its view is fetched, it downloads, and the vendor loop skips it `skipped` / `package_not_installed` as in v4.x (so cargo's `locked_version_mismatch` is refused early only for a crate installed at the unlocked version). The result becomes `{action:"failed", errorCode:, error:}` in `download.patches[]` / `patches[]` with the backend's exact code and detail, no view and no pristine fetch, no patch record, and therefore no vendor event: compared with v4.x, `download.downloaded` drops and `download.failed` rises by the number of such packages, `vendor.summary.failed` and `vendor.events` lose their `failed` events, and a lockfile-only package among them loses its `vendor_fetched_missing` event (it is never fetched). Exit code and top-level `status` are unchanged (`partial_failure`/1); the nested `vendor.status` becomes `success` when those refusals were the vendor step's only failures (observed on the depscan fixture: 3 refusals, `partialFailure` → `success`), and when every selected package is refused this way the human `scan --vendor` arm prints `Nothing was vendored: N patches failed (see above).`. **Precedence:** the lock-text refusal is decided before the view, so it wins over every view-derived outcome — a package that would also have been a paid-access 403 (`[PAID]`/no access), a failed view fetch, or a no-applicable-files skip reports the lock refusal instead (the Bun refusal and the ledger's `already vendored` skip still come first). The human `[error] (): ` line is printed during the download instead of the vendor step's failure line (the interactive human `scan --vendor` arm's pre-prompt baseline check still fetches the views it verifies; only the download, the pristine fetch and the vendor step skip the package there). A purl the hosted redirect ledger claims keeps the loop's refusal (its takeover revert rewrites the lock the gates read), as does every purl when that ledger is malformed; other flavors (package-lock, pnpm-legacy, bun) and ecosystems are untouched, and `--dry-run` is unchanged. `vendor` (manifest-driven, no view fetch) keeps its per-package `failed` events but no longer fetches the pristine source of a lockfile-only package it refuses this way — the source is deferred to the backend, which refuses before reading it (no `vendor_fetched_missing` event and no registry request; a refused package whose registry is unreachable reports the gate's code instead of `vendor_fetch_failed`); only a package the lock resolves to a verifiable source is deferred, and one it does not resolve keeps its `package_not_installed` skip. Pinned by `tests/scan_vendor_e2e.rs` (`exact_download_plan`: scan and exact-purl get, pnpm and cargo scope), `tests/e2e_yarn_legacy_cachekey_refusal_build.rs` and `tests/vendor_rerun_no_network_e2e.rs`. * **Installed-version narrowing** (all modes, `get`'s search path): a CVE/GHSA fan-out returns one patch record per patched VERSION; get keeps only versions present here and emits calm `skipped` records (`errorCode: "package_not_installed"`) for the rest — never an error exit. Presence = installed on disk (qualified-aware resolver) ∪ already tracked in the manifest (record maintenance keeps working on hosts without an installed copy); hosted/vendored modes additionally count lockfile-resolved deps and vendor-ledger purls (mirroring scan's discovery supplements, including their `--global` gate). **Exempt** (no narrowing): UUID identifiers, exact-versioned PURL identifiers (explicit intent), `--save-only` runs (record-only has no installation precondition — the fresh-clone record→vendor flow keeps working), `--all-releases`, and the package-name path (already installed-derived). When EVERY found patch is filtered out, get exits 0 with the additive status **`not_installed`** (`{status:"not_installed", found:N, downloaded:0, applied:0, patches:[], warnings?}`) — never `no_match`, which remains pinned to the fuzzy package-name path. PnP layouts are surfaced, not misreported: yarn-PnP npm results skip with `errorCode: "yarn_pnp_unsupported"` in every mode; pnpm-PnP skips carry `pnpm_pnp_unsupported` in agent/vendored modes; hosted mode — the refusal's own remedy — keeps ONLY the versions the raw `pnpm-lock.yaml` text actually resolves (boundary-anchored probe over the v5/v6/v9 key spellings, so a large fan-out never requests grants for every version ever patched), labels a JUDGED miss `package_not_installed` exactly like a non-PnP project (the layout blocked nothing — the lock was read and the version isn't resolved), and reserves the layout code for an unreadable lock (no judgment possible). When EVERY narrowed-out result is a PnP refusal, the human terminal names the layout instead of claiming "not installed" and never advises `--all-releases` (which cannot make PnP patchable); the JSON status stays `not_installed` — consumers dispatch on the per-record `errorCode`. Hosted mode also runs the per-release VARIANT filter (`filter_to_installed_releases`) on its search path before requesting grants — agent/vendored runs get it inside the download engines — with the same keep-all-plus-warning fallbacks (surfaced as `(release_narrowing)`-prefixed strings in `warnings[]`). An ecosystem this binary has no crawler for is likewise never judged: its results are KEPT (absence from a crawl that never looked carries no information — the same fail-safe as scan's prune GC). The human `Found N patches:` listing shows only the patches whose package version survived the narrowing (the narrowing is judged over every result, so an installed package's paid fix a free user cannot download still lists as `[PAID] (no access)`, while skip records and counts cover only accessible patches), sorted by PURL in natural version order (`4.17.2` before `4.17.10`); the narrowed-out ones are summarized on stderr in one line per reason (`Skipped N patches for M package versions not installed here (use --all-releases to include them).`), and `--verbose` adds one `[skip] ()` line per skipped version after that summary, in natural version order. When the candidates hold more patches than were selected and the pick was made without a menu (a paid user's auto-pick, `--yes`, a non-TTY run), a `Selected:` block names the patch (purl, tier, short uuid, advisories) that will be installed before the prompt. Machine output (the prompt count, the JSON envelope) uses the kept set, unchanged. The finer per-release variant narrowing (`filter_to_installed_releases`) is unchanged and still runs inside the download engines (and before an agent-mode `--dry-run` preview, so the preview names only the variants a wet run would fetch). * **Deliberate divergences from scan** (documented, not drift): get keeps its `selection_required` JSON posture for free multi-patch PURLs (scan auto-picks); get has no `--vex` (an ambient `SOCKET_VEX` is ignored by get's modes), no `--detached` (moot — `get --mode vendored` is manifest-free by construction), no `--prune`; get does not run scan's pre-confirm vendor baseline annotation; and an all-narrowed-out run exits `not_installed` without entering the vendor step (heal-after-wipe re-vendoring stays `scan --mode vendored`'s job). Agent-mode `get` honors `--dry-run` too (v5.x; it used to download, save and apply anyway): the search and uuid paths classify each selected patch against the manifest (read-only; an unreadable manifest fails closed like the wet run) and stop before the prompt, the download, any `.socket/` write and the apply — human `[would-add]` / `[would-update] … (replacing )` / `[skip] … (already in manifest)` lines then `[dry-run] Would download and apply N patches. No changes made.`; JSON `{status:"success", dryRun:true, found, downloaded:0, skipped, applied:0, patches:[{purl, uuid, action:"would_add"|"would_update"(+oldUuid)|"skipped"}, ], warnings?}`, exit 0. @@ -1032,7 +1042,7 @@ Empty string means unset at every layer: exported-but-empty flag-bound vars are | `SOCKET_TELEMETRY_DISABLED` | `--no-telemetry` | `false` | **Renamed in v3.0** (was `SOCKET_PATCH_TELEMETRY_DISABLED`). | | `SOCKET_FORCE` | `apply --force` / `-f`, `--update --force` | `false` | Local to `apply` and `--update`. | | `SOCKET_PATCH_VERSION` | `--update ` | (latest) | Local to `--update`; the same pin `install.sh` and the gem launcher honor. Not one of the deprecated legacy `SOCKET_PATCH_*` trio. | -| `SOCKET_BATCH_SIZE` | `scan --batch-size` | `100` | Local to `scan`. | +| `SOCKET_BATCH_SIZE` | `scan --batch-size` | `500` authenticated / `100` proxy | Local to `scan`. | | `SOCKET_SAVE_ONLY` | `get --save-only` | `false` | Local to `get`. | | `SOCKET_ONE_OFF` | `get --one-off` / `rollback --one-off` | `false` | Local to `get`/`rollback`. Both are **not yet implemented**: the flag parses (boolishly, empty-tolerant) and the command fails up front with a "not yet implemented" error, before any network or disk activity (on `rollback`, with no identifier-shaped target it instead fails "requires an identifier", equally up front). | | `SOCKET_ALL_RELEASES` | `get --all-releases` / `scan --all-releases` | `false` | Local to `get`/`scan`. Download patches for every release/distribution variant, not just the installed one. | @@ -1212,6 +1222,7 @@ Every `--json` invocation emits a single JSON object that follows the **unified | `hosted_state_not_preservable` | rollback `warnings[]` | rollback `--preserve-state` (v5.0): hosted redirects were unwound and their ledger records dropped anyway — hosted has no preservable local state; re-run `scan --mode hosted` to re-wire. (`remove --preserve-state` prints the same note on stderr.) | | `out_of_scope_copies_restored` | rollback `warnings[]` | path-scoped rollback (v5.0): a selected patch had installed copies outside the given patterns; ALL copies were restored (patches are per-package). Informational — never flips the exit. | | `path_scope_excluded_supplements` | scan `warnings[]` | path-scoped scan (v5.0): lockfile-only / vendor-ledger supplement packages have no installed path and were excluded from the scoped scan; the detail carries the count. | +| `vendor_commit_failed` | top-level error (`vendor`, and the nested vendor envelope of `scan` / `get --mode vendored`) | v5.0 group commit: the run's lockfile / manifest / ledger edits could not be written (the detail names the I/O error). Exit 1; the project's lockfiles and `.socket/vendor/state.json` are left as they were before the run (a partially-applied commit is put back), and the per-package events describe the uncommitted outcome. When putting a partially-applied commit back fails too, the journal is kept instead and the detail says the next socket-patch command in the project finishes the commit. | | `vendor_state_unreadable` / `redirect_state_unreadable` | rollback `warnings[]`; remove top-level error | corrupt-ledger containment (v5.0). Rollback: an unreadable vendor ledger skips the vendored leg + manifest cleanup + GC; an unreadable redirect ledger skips the hosted leg (quarantine/restore remedy in the detail); either drives `partial_failure` exit 1 while the agent leg still restores files. Remove: `vendor_state_unreadable` is a hard top-level error before any mutation (an unreadable redirect ledger only warns — the identifier may match other stores). Also the Bun vendored preflight's refusal code: `get` / `scan --mode vendored`, `vendor`'s pre-takeover check and the `--dry-run` `would_refuse` preview report an unreadable `.socket/vendor/state.json` as itself (`errorCode` in `patches[]` / `download.patches[]`, or `get `'s top-level `error.code`), fail-closed — nothing is exempt — instead of a Bun lock code. | | `manifest_write_failed` | rollback `warnings[]` | rollback (v5.0): the post-rollback manifest update could not be written; no entries were removed (`manifest.removedEntries: []`) and the run exits `partial_failure` 1. | | `redirect_pnpm_trust_scaffold_modified` | rollback/remove `warnings[]` | hosted replay (v5.0): the redirect-created `pnpm-workspace.yaml` scaffold was modified since; the file was kept and only the `trustLockfile: true` line removed. | @@ -1253,7 +1264,7 @@ Every `--json` invocation emits a single JSON object that follows the **unified | `vendor_override_conflict` | `failed` | vendor (pnpm/yarn-berry): a user-authored override/resolution for the package already exists. | | `vendor_integrity_unverified` | `skipped` (warning) | vendor (pipenv): the lockfile format does not hash-check file entries; the committed wheel bytes are the protection. | | `vendor_content_mismatch_overwritten` | `skipped` (warning) | vendor: a staged file matched NEITHER beforeHash nor afterHash (patch built against different bytes, or local edits); the stage was overwritten with the verified patched content and the vendor succeeded. | -| `vendor_fetched_missing` | `skipped` (warning) | vendor: the package was not installed; its pristine artifact was fetched per the lockfile resolution (or staged from the committed vendor artifact), integrity-verified, and vendored — the project tree was not touched. For `poetry.lock` (which records hashes but no URLs) the pure-Python wheel's sha256 selects the file through PyPI's JSON API (`SOCKET_PYPI_JSON_API` overrides the endpoint); Poetry 0.12's bare `[metadata.hashes]` names no wheel, so those locks still need an installed copy (`vendor_fetch_unverifiable`). | +| `vendor_fetched_missing` | `skipped` (warning) | vendor: the package was not installed; its pristine artifact was fetched per the lockfile resolution (or staged from the committed vendor artifact), integrity-verified, and vendored — the project tree was not touched. Not emitted when no fetch happened: an in-sync re-run of a ledger-covered purl, or a cargo crate the patch service served (see Vendor auto-fetch § Deferred fetch). For `poetry.lock` (which records hashes but no URLs) the pure-Python wheel's sha256 selects the file through PyPI's JSON API (`SOCKET_PYPI_JSON_API` overrides the endpoint); Poetry 0.12's bare `[metadata.hashes]` names no wheel, so those locks still need an installed copy (`vendor_fetch_unverifiable`). | | `vendor_fetch_failed` | `failed` | vendor: the lockfile-resolved fetch was attempted and failed (HTTP error, size cap, integrity mismatch, or a PRESENT-but-corrupt committed artifact — pointed at `socket-patch repair`). A MISSING committed artifact no longer lands here: it falls through to the ledger-recovered registry fetch. Suppresses the duplicate `package_not_installed` skip. | | `vendor_fetch_unverifiable` | `skipped` (warning) | vendor: the lockfile records no usable integrity for the missing package; nothing was fetched (fail-closed) and the `package_not_installed` skip follows. | | `vendor_artifact_missing` | `skipped` (warning) / `failed` | vendor: the committed artifact is gone — the registry resolution is recovered from the ledger and the artifact rebuilt (warning); repair `--offline` with no local source surfaces it as the per-entry failure instead. | diff --git a/crates/socket-patch-cli/Cargo.toml b/crates/socket-patch-cli/Cargo.toml index 00046ebe9..9c8dc1e9d 100644 --- a/crates/socket-patch-cli/Cargo.toml +++ b/crates/socket-patch-cli/Cargo.toml @@ -22,6 +22,7 @@ clap = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } +futures-util = { workspace = true } console = { workspace = true } dialoguer = { workspace = true } uuid = { workspace = true } @@ -62,6 +63,10 @@ docker-e2e = [] setup-e2e = [] [dev-dependencies] +# vendor_crash_safety_e2e / vendor_group_commit_e2e crash the binary through +# its failpoints; with this, `cargo test --release` (the test-release job) +# builds them in too. Dev-only: resolver 2 keeps it out of normal builds. +socket-patch-core = { workspace = true, features = ["failpoints"] } sha2 = { workspace = true } # docker_e2e_vendor_maven's host oracle recomputes the maven2 .jar.sha1 sidecar. sha1 = { workspace = true } diff --git a/crates/socket-patch-cli/src/commands/fetch_stage.rs b/crates/socket-patch-cli/src/commands/fetch_stage.rs index 3179bab5e..9ebe55e24 100644 --- a/crates/socket-patch-cli/src/commands/fetch_stage.rs +++ b/crates/socket-patch-cli/src/commands/fetch_stage.rs @@ -10,13 +10,15 @@ use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; +use futures_util::StreamExt; use socket_patch_core::api::blob_fetcher::{ fetch_missing_blobs, fetch_missing_sources, get_missing_archives, get_missing_blobs, DownloadMode, FetchMissingBlobsResult, }; -use socket_patch_core::api::client::{get_api_client_with_overrides, ApiClient}; +use socket_patch_core::api::client::{get_api_client_with_overrides, hold_back_debug, ApiClient}; use socket_patch_core::manifest::schema::{PatchManifest, PatchRecord}; use socket_patch_core::patch::apply::{is_valid_blob_hash, PatchSources}; +use socket_patch_core::utils::concurrent::{api_concurrency_for, ordered_concurrent}; use tempfile::TempDir; use super::get::base64_decode; @@ -565,6 +567,15 @@ pub(crate) async fn stage_vendor_sources_in_memory( } }; let mut failed: Vec<&str> = Vec::new(); + // The views are fetched concurrently (at most `api_concurrency` in + // flight) but consumed in `to_fetch` order, each request's `--debug` + // lines released at its turn, so `mem`, `failed` and every error + // line fold exactly as the serial loop's did. + let mut views = std::pin::pin!(ordered_concurrent( + to_fetch.iter(), + api_concurrency_for(client.uses_public_proxy(), to_fetch.len()), + |(_, uuid)| async move { (*uuid, hold_back_debug(client.fetch_patch(uuid)).await) }, + )); for (i, (purl, uuid)) in to_fetch.iter().enumerate() { if to_fetch.len() > 1 { status.set(format!( @@ -574,7 +585,19 @@ pub(crate) async fn stage_vendor_sources_in_memory( to_fetch.len() )); } - match client.fetch_patch(uuid).await { + let view = match views.next().await { + Some((planned, view)) if planned == *uuid => view.release(), + // Unreachable: the plan IS this list. Falling back to the + // live request keeps the staging COMPLETE if the two ever + // fall out of step — running dry here would otherwise + // return `Ready` with blobs missing and nothing in + // `failed`. + _ => { + debug_assert!(false, "view prefetch plan out of step with the fetch list"); + client.fetch_patch(uuid).await + } + }; + match view { Ok(Some(patch)) => { let mut complete = true; for (file, info) in &patch.files { diff --git a/crates/socket-patch-cli/src/commands/get.rs b/crates/socket-patch-cli/src/commands/get.rs index 2e32ee27e..f9fea8ca1 100644 --- a/crates/socket-patch-cli/src/commands/get.rs +++ b/crates/socket-patch-cli/src/commands/get.rs @@ -1,8 +1,9 @@ use clap::Args; +use futures_util::StreamExt; use regex::Regex; use socket_patch_core::api::client::{ - build_proxy_fallback_client, get_api_client_with_overrides, is_fallback_candidate, ApiClient, - ApiError, + build_proxy_fallback_client, get_api_client_with_overrides, hold_back_debug, + is_fallback_candidate, ApiClient, ApiError, }; use socket_patch_core::api::ranking::{cmp_search_results, severity_order}; use socket_patch_core::api::types::{ @@ -17,6 +18,7 @@ use socket_patch_core::manifest::schema::{ use socket_patch_core::patch::apply::{is_valid_blob_hash, select_installed_variants}; use socket_patch_core::patch::apply_lock::{LockError, LockGuard}; use socket_patch_core::telemetry::{track_patch_fetch_failed, track_patch_fetched}; +use socket_patch_core::utils::concurrent::{api_concurrency_for, ordered_concurrent}; use socket_patch_core::utils::purl::{ canonical_purl, is_purl, normalize_purl, strip_purl_qualifiers, }; @@ -1359,6 +1361,26 @@ async fn filter_to_installed_releases( let partitioned = partition_purls(&all_qualified, None); let paths = find_packages_for_rollback(&partitioned, crawler_options, true).await; + // Every installed base's variant views, fetched concurrently (at most + // `api_concurrency` in flight) in the order the loop below consumes + // them: bases in `multi` order, skipping the uninstalled ones, each + // base's variants in order. Nothing here prints between fetches, and + // each request's `--debug` lines are released at its old turn. + let installed_variants: Vec = multi + .iter() + .filter(|(_, variants)| variants.iter().any(|s| paths.contains_key(&s.purl))) + .flat_map(|(_, variants)| variants.iter().map(|s| s.uuid.clone())) + .collect(); + let window_len = installed_variants.len(); + let mut variant_views = std::pin::pin!(ordered_concurrent( + installed_variants, + api_concurrency_for(api_client.uses_public_proxy(), window_len), + |uuid| async move { + let view = hold_back_debug(api_client.fetch_patch(&uuid)).await; + (uuid, view) + }, + )); + for (base, variants) in multi { // Any variant's resolved path works — they all map to the same // installed package directory. @@ -1379,7 +1401,22 @@ async fn filter_to_installed_releases( // kept for the download loop — it is the same GET it would issue. let mut candidates: Vec<(String, HashMap)> = Vec::new(); for s in &variants { - match api_client.fetch_patch(&s.uuid).await { + let view = match variant_views.next().await { + Some((planned, view)) if planned == s.uuid => view.release(), + // Unreachable: the plan holds one view per variant of + // every installed base. Checking matters — a plan out of + // step would hash-match this variant against ANOTHER + // release's files and store that response under this + // uuid for the download engine. + _ => { + debug_assert!( + false, + "variant view prefetch plan out of step with the variants" + ); + api_client.fetch_patch(&s.uuid).await + } + }; + match view { Ok(Some(patch)) => { candidates.push((s.purl.clone(), files_with_both_hashes(&patch))); views.insert(s.uuid.clone(), patch); @@ -1774,12 +1811,91 @@ impl FetchBatch { } } +/// Selected purls the vendor backend will refuse on the project's lock +/// text alone, with the backend's `(code, detail)`. +type LockRefusals = HashMap; + +/// The lock-text refusals of the vendored download phase (see +/// [`socket_patch_core::vendor::lock_text_refusals`]: the pnpm / yarn +/// classic / yarn berry gates and cargo's locked-version gate), over the +/// patches the phase would otherwise fetch a view for — past the Bun +/// refusal and the ledger's idempotency skip, which take precedence in the +/// fetch loop. A purl the hosted redirect ledger claims is left to the +/// vendor loop: its takeover reverts the hosted lock edits first, and the +/// revert rewrites the very text the gates read. A redirect ledger that +/// cannot be read leaves every purl to the loop. +/// +/// Only a package the vendor loop would hand to its backend is refused +/// here (see [`crate::commands::vendor::lock_refusals_reaching_backend`]): +/// one installed on disk, or one the lockfile resolves to a verifiable +/// registry source. A package with neither — absent from the lock and not +/// installed — never reaches its backend: the loop reports it `skipped` / +/// `package_not_installed`, and so it still does. `prior` is scan's npm +/// crawl, when the caller has it (the installed-copy lookup reuses it). +async fn lock_text_refusals_for( + params: &DownloadParams, + selected: &[PatchSearchResult], + ledger: &VendorState, + bun_refusal: Option<&BunVendorRefusal>, + prior: Option<&crate::ecosystem_dispatch::NpmCrawlSnapshot>, +) -> LockRefusals { + let cwd = params.cwd.as_path(); + let claimed: Vec = + match socket_patch_core::patch::redirect::load_redirect_state(cwd).await { + Ok(Some(state)) => state.records.keys().map(|k| canonical_purl(k)).collect(), + Ok(None) => Vec::new(), + Err(_) => return HashMap::new(), + }; + let candidates: Vec<(&str, &str)> = selected + .iter() + .filter(|sr| bun_refusal.filter(|r| r.applies_to(&sr.purl)).is_none()) + .filter(|sr| { + detached_ledger_record(RecordStore::Ledger(&ledger.entries), &sr.purl, &sr.uuid) + .is_none() + }) + .filter(|sr| !claimed.contains(&canonical_purl(&sr.purl))) + .map(|sr| (sr.purl.as_str(), sr.uuid.as_str())) + .collect(); + let refused = socket_patch_core::vendor::lock_text_refusals(cwd, &candidates).await; + let options = CrawlerOptions { + cwd: params.cwd.clone(), + global: params.global, + global_prefix: params.global_prefix.clone(), + }; + crate::commands::vendor::lock_refusals_reaching_backend( + cwd, + refused, + &ledger.entries, + |purls| async move { + crate::commands::vendor::installed_purls(&options, &purls, prior).await + }, + ) + .await +} + +/// The record a detached ledger entry already carries for `purl` at +/// exactly `uuid` — the ledger store's idempotency skip (no view fetch). +/// Always `None` for the manifest store. +fn detached_ledger_record<'a>( + store: RecordStore<'a>, + purl: &str, + uuid: &str, +) -> Option<&'a PatchRecord> { + let RecordStore::Ledger(entries) = store else { + return None; + }; + lookup_entry(entries, purl) + .filter(|e| e.detached && e.uuid == uuid) + .and_then(|e| e.record.as_ref()) +} + /// The fetch loop both download engines share: installed-release /// narrowing, the caller's Bun refusal, the per-store skip decision, the /// view fetch (served from `prefetched` when the narrowing or the caller /// already holds the view), the no-applicable-files guardrail, optional /// blob persistence, and every per-patch failure record. Every pinned /// stderr line and JSON action lives here once. +#[allow(clippy::too_many_arguments)] async fn fetch_selected_patches( selected: &[PatchSearchResult], params: &DownloadParams, @@ -1787,6 +1903,7 @@ async fn fetch_selected_patches( store: RecordStore<'_>, blobs_dir: Option<&Path>, bun_refusal: Option<&BunVendorRefusal>, + lock_refusals: &LockRefusals, mut prefetched: HashMap, ) -> FetchBatch { let quiet = params.quiet(); @@ -1825,6 +1942,33 @@ async fn fetch_selected_patches( warnings, }; + // The view GETs the loop below makes — every patch past the refusal + // and the ledger skip whose view is not already held in `prefetched` + // (the same three checks, in the loop's order, over inputs the loop + // never mutates) — run concurrently ahead of it, at most + // `api_concurrency` in flight, and come back in selection order. The + // loop takes the next one exactly where it used to await the request, + // and each request's `--debug` lines print there too, so stdout, the + // per-patch stderr lines and the JSON records fold exactly as the + // serial loop's did. + let mut held: std::collections::HashSet<&str> = prefetched.keys().map(String::as_str).collect(); + let to_fetch: Vec<&str> = selected + .iter() + .filter(|sr| { + bun_refusal.filter(|r| r.applies_to(&sr.purl)).is_none() + && detached_ledger_record(store, &sr.purl, &sr.uuid).is_none() + && !lock_refusals.contains_key(&sr.purl) + && !held.remove(sr.uuid.as_str()) + }) + .map(|sr| sr.uuid.as_str()) + .collect(); + let window_len = to_fetch.len(); + let mut views = std::pin::pin!(ordered_concurrent( + to_fetch, + api_concurrency_for(api_client.uses_public_proxy(), window_len), + |uuid| async move { (uuid, hold_back_debug(api_client.fetch_patch(uuid)).await) }, + )); + for search_result in &selected { let (purl, uuid) = (search_result.purl.as_str(), search_result.uuid.as_str()); @@ -1850,30 +1994,54 @@ async fn fetch_selected_patches( // Idempotency (ledger store): a detached entry already at this uuid // carries its own record — no view fetch needed. - if let RecordStore::Ledger(entries) = store { - if let Some(record) = lookup_entry(entries, purl) - .filter(|e| e.detached && e.uuid == uuid) - .and_then(|e| e.record.clone()) - { - if !quiet { - eprintln!("{}", format_record_skip(purl, "already vendored")); - } - batch.patches_json.push(serde_json::json!({ - "purl": purl, - "uuid": uuid, - "action": "skipped", - })); - batch.reused.push((purl.to_string(), record)); - batch.skipped += 1; - continue; + if let Some(record) = detached_ledger_record(store, purl, uuid).cloned() { + if !quiet { + eprintln!("{}", format_record_skip(purl, "already vendored")); } + batch.patches_json.push(serde_json::json!({ + "purl": purl, + "uuid": uuid, + "action": "skipped", + })); + batch.reused.push((purl.to_string(), record)); + batch.skipped += 1; + continue; + } + + // Lock-text refusal (see `lock_text_refusals_for`): the vendor + // backend refuses this package on the project's lock alone, so its + // view is never fetched (nor, downstream, its pristine source) — + // reported with the backend's code and words, as the Bun refusal is. + if let Some((code, detail)) = lock_refusals.get(purl) { + batch.fail( + params.json, + Some(format!("[error] {purl} ({code}): {detail}")), + purl, + uuid, + detail, + Some(code), + ); + continue; } // The view: from memory when the narrowing (or the uuid path's own - // identifier fetch) already fetched it, else the network. + // identifier fetch) already fetched it, else the network — the next + // of the concurrent GETs above, which were planned for exactly + // these turns. let view = match prefetched.remove(uuid) { Some(patch) => Ok(Some(patch)), - None => api_client.fetch_patch(uuid).await, + None => match views.next().await { + Some((planned, view)) if planned == uuid => view.release(), + // Unreachable (the plan mirrors this loop's checks); a + // live fetch keeps the outcome right regardless. + _ => { + debug_assert!( + false, + "view prefetch plan out of step with the download loop" + ); + api_client.fetch_patch(uuid).await + } + }, }; let patch = match view { Ok(Some(patch)) => patch, @@ -2045,6 +2213,18 @@ pub(crate) async fn download_patch_records_with( params: &DownloadParams, api_client: &ApiClient, prefetched: HashMap, +) -> DetachedDownload { + download_patch_records_reusing(selected, params, api_client, prefetched, None).await +} + +/// [`download_patch_records_with`], handing `prior` (scan's npm crawl of +/// the untouched tree) to the lock-text refusals' installed-copy lookup. +pub(crate) async fn download_patch_records_reusing( + selected: &[PatchSearchResult], + params: &DownloadParams, + api_client: &ApiClient, + prefetched: HashMap, + prior: Option<&crate::ecosystem_dispatch::NpmCrawlSnapshot>, ) -> DetachedDownload { // The ledger load outcome is handed to the preflight AS a result: an // unreadable ledger must surface as `vendor_state_unreadable` from the @@ -2073,6 +2253,7 @@ pub(crate) async fn download_patch_records_with( prefetched, vendor_state, bun_refusal.as_ref(), + prior, ) .await } @@ -2089,8 +2270,11 @@ async fn download_patch_records_preflighted( prefetched: HashMap, vendor_state: std::io::Result, bun_refusal: Option<&BunVendorRefusal>, + prior: Option<&crate::ecosystem_dispatch::NpmCrawlSnapshot>, ) -> DetachedDownload { let vendor_state = vendor_state.unwrap_or_default(); + let lock_refusals = + lock_text_refusals_for(params, selected, &vendor_state, bun_refusal, prior).await; let blobs_dir = params.socket_dir().join("blobs"); let batch = fetch_selected_patches( @@ -2100,6 +2284,7 @@ async fn download_patch_records_preflighted( RecordStore::Ledger(&vendor_state.entries), params.persist_blobs.then_some(blobs_dir.as_path()), bun_refusal, + &lock_refusals, prefetched, ) .await; @@ -2318,6 +2503,7 @@ pub async fn download_and_apply_patches_with( RecordStore::Manifest(&manifest), params.persist_blobs.then_some(blobs_dir.as_path()), None, + &HashMap::new(), HashMap::new(), ) .await; @@ -3483,6 +3669,7 @@ async fn run_get_hosted( api_client, &pairs, scan_result, + None, ) .await } @@ -3618,6 +3805,7 @@ async fn run_get_vendored( prefetched_views, vendor_state, bun_refusal.as_ref(), + None, )) .await } else { @@ -6824,6 +7012,313 @@ mod tests { ); } + /// The download loop's view GETs run concurrently but fold in selection + /// order: with later views answering FIRST (reversed latencies) and a + /// mix of 200 / 404 / 500 / held-in-memory / ledger-reused patches, + /// every per-patch record keeps its selection slot and its serial + /// action + error text, and only the views the serial loop fetched are + /// requested (the held and reused ones never are). + #[tokio::test] + #[serial_test::serial] + async fn download_patch_records_concurrent_views_fold_in_selection_order() { + use wiremock::matchers::{method, path as wm_path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let _env = EnvVarGuard::scrub(&["SOCKET_PROXY_URL", "SOCKET_PATCH_PROXY_URL"]); + let server = MockServer::start().await; + let uuid = |c: char| { + format!("{0}{0}{0}{0}{0}{0}{0}{0}-{0}{0}{0}{0}-4{0}{0}{0}-8{0}{0}{0}-{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}", c) + }; + let purl = |n: &str| format!("pkg:npm/covgap-order-{n}@1.0.0"); + let view = |u: &str, p: &str| { + serde_json::json!({ + "uuid": u, "purl": p, + "publishedAt": "2024-01-01T00:00:00Z", + "files": { "package/index.js": { + "beforeHash": "0".repeat(64), "afterHash": "1".repeat(64), + }}, + "vulnerabilities": {}, "description": "d", "license": "MIT", "tier": "free", + }) + }; + // Selection order a..f; the slowest answers belong to the earliest. + let (a, b, c, d, e, f) = ( + uuid('a'), + uuid('b'), + uuid('c'), + uuid('d'), + uuid('e'), + uuid('f'), + ); + let mount = |u: &str, resp: ResponseTemplate| { + Mock::given(method("GET")) + .and(wm_path(format!("/v0/orgs/test-org/patches/view/{u}"))) + .respond_with(resp) + .expect(1) + }; + mount( + &a, + ResponseTemplate::new(200) + .set_body_json(view(&a, &purl("a"))) + .set_delay(Duration::from_millis(600)), + ) + .mount(&server) + .await; + mount( + &b, + ResponseTemplate::new(404).set_delay(Duration::from_millis(400)), + ) + .mount(&server) + .await; + mount( + &c, + ResponseTemplate::new(500) + .set_body_string("boom") + .set_delay(Duration::from_millis(200)), + ) + .mount(&server) + .await; + // `d` is held in memory and `e` is reused from the ledger: never + // requested. + for u in [&d, &e] { + Mock::given(method("GET")) + .and(wm_path(format!("/v0/orgs/test-org/patches/view/{u}"))) + .respond_with(ResponseTemplate::new(500)) + .expect(0) + .mount(&server) + .await; + } + mount( + &f, + ResponseTemplate::new(200).set_body_json(view(&f, &purl("f"))), + ) + .mount(&server) + .await; + + let tmp = tempfile::tempdir().unwrap(); + let vendor = tmp.path().join(".socket/vendor"); + std::fs::create_dir_all(&vendor).unwrap(); + std::fs::write( + vendor.join("state.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "version": 1, + "entries": { purl("e"): { + "ecosystem": "npm", + "basePurl": purl("e"), + "uuid": e, + "detached": true, + "record": { + "uuid": e, + "exportedAt": "2024-01-01T00:00:00Z", + "files": { "package/index.js": { + "beforeHash": "0".repeat(64), "afterHash": "1".repeat(64), + }}, + "vulnerabilities": {}, + "description": "d", "license": "MIT", "tier": "free", + }, + "artifact": { + "path": format!(".socket/vendor/npm/{e}/covgap-order-e-1.0.0.tgz"), + }, + "wiring": [] + }} + })) + .unwrap(), + ) + .unwrap(); + + let mut held: PatchResponse = serde_json::from_value(view(&d, &purl("d"))).unwrap(); + held.uuid = d.clone(); + let selected: Vec = [ + (&a, "a"), + (&b, "b"), + (&c, "c"), + (&d, "d"), + (&e, "e"), + (&f, "f"), + ] + .iter() + .map(|(u, n)| mk_patch(u, &purl(n), "free", "2024-01-01")) + .collect(); + let client = test_client(&server.uri()).await; + let (_code, json, records, _blobs) = download_patch_records_with( + &selected, + &detached_params(tmp.path()), + &client, + HashMap::from([(d.clone(), held)]), + ) + .await; + + let rows: Vec<(String, String, String)> = json["patches"] + .as_array() + .unwrap() + .iter() + .map(|p| { + ( + p["purl"].as_str().unwrap_or_default().to_string(), + p["action"].as_str().unwrap_or_default().to_string(), + p["error"].as_str().unwrap_or_default().to_string(), + ) + }) + .collect(); + let row = + |n: &str, action: &str, error: &str| (purl(n), action.to_string(), error.to_string()); + assert_eq!( + rows, + vec![ + row("a", "downloaded", ""), + row("b", "failed", "could not fetch details"), + row("c", "failed", "API request failed with status 500: boom"), + row("d", "downloaded", ""), + row("e", "skipped", ""), + row("f", "downloaded", ""), + ], + "json={json}" + ); + assert_eq!(json["downloaded"], 3, "json={json}"); + assert_eq!(json["failed"], 2, "json={json}"); + assert_eq!(json["skipped"], 1, "json={json}"); + let mut got: Vec<&String> = records.keys().collect(); + got.sort(); + assert_eq!(got, vec![&purl("a"), &purl("d"), &purl("e"), &purl("f")]); + // `.expect` counts are verified on drop. + drop(server); + } + + /// Release-variant narrowing fetches every installed base's variant + /// views concurrently, so each must come back to the variant that + /// planned it. Two installed multi-variant bases with REVERSED + /// latencies (the first base's views answer last) plus an uninstalled + /// one: each base keeps the variant whose file hashes match its own + /// installed bytes, every cached view is its own variant's, and the + /// uninstalled base's views are never requested (its variants are not + /// in the plan, and a plan that drifted to include them would trip + /// their `expect(0)`). + #[tokio::test] + #[serial_test::serial] + async fn release_narrowing_pairs_each_concurrent_view_with_its_own_variant() { + use socket_patch_core::hash::git_sha256::compute_git_sha256_from_bytes; + use wiremock::matchers::{method, path as wm_path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let _env = EnvVarGuard::scrub(&["SOCKET_PROXY_URL", "SOCKET_PATCH_PROXY_URL"]); + let site = tempfile::tempdir().unwrap(); + // Two installed pypi distributions, each with its own bytes. + let installed = |name: &str, body: &[u8]| { + let dist = site.path().join(format!("{name}-1.0.0.dist-info")); + std::fs::create_dir_all(&dist).unwrap(); + std::fs::write(dist.join("METADATA"), format!("Name: {name}\nVersion: 1.0.0\n")) + .unwrap(); + std::fs::write(site.path().join(format!("{name}.py")), body).unwrap(); + compute_git_sha256_from_bytes(body) + }; + let alpha_hash = installed("alpha", b"alpha installed\n"); + let beta_hash = installed("beta", b"beta installed\n"); + + let server = MockServer::start().await; + let uuid = |n: &str| format!("{n:-<8}-0000-4000-8000-000000000000").replace(' ', "-"); + // `(uuid, file, hash, delay)`: the WHEEL variant of each base names + // the installed file at its real hash (so it is the one kept); the + // SDIST variant names a file that base does not have. + let mount = |u: String, file: String, hash: String, delay: u64| { + let server = &server; + async move { + Mock::given(method("GET")) + .and(wm_path(format!("/v0/orgs/test-org/patches/view/{u}"))) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(serde_json::json!({ + "uuid": u, + "purl": "pkg:pypi/ignored@1.0.0", + "publishedAt": "2024-01-01T00:00:00Z", + "files": { file: { + "beforeHash": hash, + "afterHash": "1".repeat(64), + }}, + "vulnerabilities": {}, "description": "d", + "license": "MIT", "tier": "free", + })) + .set_delay(Duration::from_millis(delay)), + ) + .expect(1) + .mount(server) + .await; + } + }; + // Alpha answers LAST, beta first. + mount(uuid("aw"), "alpha.py".into(), alpha_hash, 300).await; + mount(uuid("as"), "alpha_sdist.py".into(), "0".repeat(64), 300).await; + mount(uuid("bw"), "beta.py".into(), beta_hash, 0).await; + mount(uuid("bs"), "beta_sdist.py".into(), "0".repeat(64), 0).await; + for n in ["gw", "gs"] { + Mock::given(method("GET")) + .and(wm_path(format!("/v0/orgs/test-org/patches/view/{}", uuid(n)))) + .respond_with(ResponseTemplate::new(500)) + .expect(0) + .mount(&server) + .await; + } + + let variant = |n: &str, base: &str, artifact: &str| { + mk_patch( + &uuid(n), + &format!("pkg:pypi/{base}@1.0.0?artifact_id={artifact}"), + "free", + "2024-01-01", + ) + }; + let selected = vec![ + variant("aw", "alpha", "wheel"), + variant("as", "alpha", "sdist"), + // `ghost` is not installed: both its variants are kept, with a + // warning, and neither view is fetched. + variant("gw", "ghost", "wheel"), + variant("gs", "ghost", "sdist"), + variant("bw", "beta", "wheel"), + variant("bs", "beta", "sdist"), + ]; + let options = CrawlerOptions { + cwd: site.path().to_path_buf(), + global: false, + global_prefix: Some(site.path().to_path_buf()), + }; + let (kept, warnings, views) = filter_to_installed_releases( + &selected, + /*all_releases=*/ false, + &options, + /*quiet=*/ true, + &test_client(&server.uri()).await, + ) + .await; + + let mut kept_purls: Vec<&str> = kept.iter().map(|s| s.purl.as_str()).collect(); + kept_purls.sort(); + assert_eq!( + kept_purls, + vec![ + "pkg:pypi/alpha@1.0.0?artifact_id=wheel", + "pkg:pypi/beta@1.0.0?artifact_id=wheel", + "pkg:pypi/ghost@1.0.0?artifact_id=sdist", + "pkg:pypi/ghost@1.0.0?artifact_id=wheel", + ], + "warnings={warnings:?}" + ); + let mut cached: Vec<(String, String)> = views + .iter() + .map(|(u, v)| (u.clone(), v.uuid.clone())) + .collect(); + cached.sort(); + // Narrowed-out variants' views are dropped, so only the two kept + // wheels ride on — each under its own uuid. A view that landed on + // the wrong variant would both keep the wrong variant above and + // pair a uuid with another variant's response here. + assert_eq!( + cached, + vec![(uuid("aw"), uuid("aw")), (uuid("bw"), uuid("bw"))], + "each cached view must be its own variant's" + ); + // `.expect` counts are verified on drop. + drop(server); + } + /// The env guard must RESTORE a variable that was set before the scrub — /// the suite depends on it not leaking scrubbed state across tests. #[test] diff --git a/crates/socket-patch-cli/src/commands/repair_vendor.rs b/crates/socket-patch-cli/src/commands/repair_vendor.rs index c283c8d32..e4dc5ec55 100644 --- a/crates/socket-patch-cli/src/commands/repair_vendor.rs +++ b/crates/socket-patch-cli/src/commands/repair_vendor.rs @@ -1333,9 +1333,32 @@ pub(crate) async fn repair_vendored_artifacts_with_references( _ => None, }; match pristine { + // Repair always rebuilds locally, so the pristine tree is read + // either way: materialise it right here, where an extraction + // failure is still the fetch failure it was before the write + // moved off the fetch. PristineFetch::Fetched(fetched) => { - all_packages.insert(c.purl.clone(), fetched.dir().to_path_buf()); - holders.push(fetched); + match fetched.dir().await.map(std::path::Path::to_path_buf) { + Ok(dir) => { + all_packages.insert(c.purl.clone(), dir); + holders.push(fetched); + } + Err(detail) => { + if c.soft { + soft_restore_without_fingerprint( + env, + common, + &c.purl, + &c.entry.artifact.path, + &format!("the pristine fetch failed ({detail})"), + ); + rebuilt += 1; + } else { + fail(env, common.json, &c.purl, "vendor_fetch_failed", detail); + } + unrebuildable.insert(c.purl.clone()); + } + } } PristineFetch::NoSource | PristineFetch::Unverifiable(_) => { // Last rung (npm): the REWIRED lockfile still records the @@ -1353,10 +1376,23 @@ pub(crate) async fn repair_vendored_artifacts_with_references( .await { Ok(fetched) => { - all_packages - .insert(c.purl.clone(), fetched.dir().to_path_buf()); - holders.push(fetched); - must_verify.insert(c.purl.clone(), wired); + match fetched.dir().await.map(std::path::Path::to_path_buf) { + Ok(dir) => { + all_packages.insert(c.purl.clone(), dir); + holders.push(fetched); + must_verify.insert(c.purl.clone(), wired); + } + Err(d) => { + fail( + env, + common.json, + &c.purl, + "vendor_fetch_failed", + d, + ); + unrebuildable.insert(c.purl.clone()); + } + } continue; } Err(registry_fetch::FetchError::Failed(d)) @@ -1427,6 +1463,7 @@ pub(crate) async fn repair_vendored_artifacts_with_references( // ── Rebuild via the normal backends ────────────────────────────────── let vendored_at = now_rfc3339(); let pipenv_version = tokio::sync::OnceCell::new(); + let installed_sites = socket_patch_core::vendor::pypi::InstalledSiteListings::default(); for c in candidates { if unrebuildable.contains(&c.purl) { continue; @@ -1499,7 +1536,7 @@ pub(crate) async fn repair_vendored_artifacts_with_references( }; let outcome = dispatch_vendor_one( &c.purl, - &pkg_path, + pkg_path.as_path().into(), &common.cwd, &c.record, &sources, @@ -1509,6 +1546,7 @@ pub(crate) async fn repair_vendored_artifacts_with_references( // Repair rebuilds locally from the recorded patch — no service. None, &pipenv_version, + &installed_sites, ) .await; match outcome { diff --git a/crates/socket-patch-cli/src/commands/scan/discovery.rs b/crates/socket-patch-cli/src/commands/scan/discovery.rs index c4fb8fde1..90b1f1765 100644 --- a/crates/socket-patch-cli/src/commands/scan/discovery.rs +++ b/crates/socket-patch-cli/src/commands/scan/discovery.rs @@ -2,11 +2,14 @@ //! supplements, update detection against the existing manifest, vendor //! baseline pre-verification, and the table's vuln-ID / severity helpers. +use futures_util::StreamExt; +use socket_patch_core::api::client::hold_back_debug; use socket_patch_core::api::ranking::cmp_batch_infos; use socket_patch_core::api::types::{ BatchPackagePatches, BatchPatchInfo, PatchResponse, PatchSearchResult, }; use socket_patch_core::manifest::schema::{PatchManifest, PatchRecord}; +use socket_patch_core::utils::concurrent::{api_concurrency_for, ordered_concurrent}; use socket_patch_core::utils::purl::{normalize_purl, strip_purl_qualifiers}; use socket_patch_core::vendor::lock_inventory::LockfileEntry; use socket_patch_core::vendor::VendorState; @@ -86,9 +89,15 @@ pub(crate) fn unsupported_layout_warnings( /// path (hash verify → NotFound, apply → partitioned skip, vendor → /// auto-fetch). Global scans target the machine's global tree, not this /// project's lockfile, so they get no supplement. +/// +/// `only` is the crawl's ecosystem scope (`None`: every ecosystem was +/// crawled): an entry of an ecosystem the crawl skipped is never counted +/// lockfile-only, since there is no crawl to tell whether it is installed. +/// `entries` still holds the full inventory. pub(super) async fn lockfile_supplement( common: &GlobalArgs, crawled: &[socket_patch_core::crawlers::types::CrawledPackage], + only: Option<&[String]>, ) -> LockfileSupplement { use socket_patch_core::vendor::lock_inventory; @@ -102,8 +111,14 @@ pub(super) async fn lockfile_supplement( return out; } let crawled_purls: HashSet<&str> = crawled.iter().map(|p| p.purl.as_str()).collect(); + let in_scope = |purl: &str| { + only.is_none_or(|list| { + socket_patch_core::crawlers::Ecosystem::from_purl(purl) + .is_some_and(|eco| list.iter().any(|name| name == eco.cli_name())) + }) + }; for entry in &entries { - if crawled_purls.contains(entry.purl.as_str()) { + if crawled_purls.contains(entry.purl.as_str()) || !in_scope(&entry.purl) { continue; } let Some(pkg) = crawled_from_purl(&entry.purl, &common.cwd) else { @@ -283,29 +298,61 @@ pub(super) async fn preverify_vendor_baselines( let mut mismatched: HashSet = HashSet::new(); let mut views: HashMap = HashMap::new(); - for (i, patch) in selected.iter().enumerate() { + // Per patch, what the loop below compares: `None` to skip it, else the + // installed copy plus the ledger's embedded record (`None` = fetch the + // view). Local and read-only, so it is computed up front. + let plan: Vec< + Option<( + &socket_patch_core::crawlers::types::CrawledPackage, + Option<&PatchRecord>, + )>, + > = selected + .iter() + .map(|patch| { + // API purls come percent-encoded, crawler purls literal — + // purl_eq bridges the two spellings. + let base = strip_purl_qualifiers(&patch.purl); + // Lockfile-only packages have no installed bytes to compare + // — the vendor engine fetches them pristine (nothing to + // annotate). + if lockfile_only_contains(lockfile_only, base) { + return None; + } + let pkg = crawled.iter().find(|c| purl_eq(&c.purl, base))?; + // The same predicate as the download phase's ledger + // idempotency skip: its no-fetch set and this one must be + // the same set. + let embedded = vendor + .and_then(|entries| lookup_entry(entries, &patch.purl)) + .filter(|e| e.detached && e.uuid == patch.uuid) + .and_then(|e| e.record.as_ref()); + Some((pkg, embedded)) + }) + .collect(); + // The views the loop needs, fetched concurrently (at most + // `api_concurrency` in flight) and consumed in `selected` order, each + // request's `--debug` lines released at its turn. + let to_fetch: Vec<&str> = selected + .iter() + .zip(&plan) + .filter(|(_, step)| matches!(step, Some((_, None)))) + .map(|(patch, _)| patch.uuid.as_str()) + .collect(); + let window_len = to_fetch.len(); + let mut details = std::pin::pin!(ordered_concurrent( + to_fetch, + api_concurrency_for(api_client.uses_public_proxy(), window_len), + |uuid| async move { (uuid, hold_back_debug(api_client.fetch_patch(uuid)).await) }, + )); + for (i, (patch, step)) in selected.iter().zip(&plan).enumerate() { status.set(format!( "Checking installed files against patch baselines... ({}/{})", i + 1, selected.len() )); - // API purls come percent-encoded, crawler purls literal — purl_eq - // bridges the two spellings. - let base = strip_purl_qualifiers(&patch.purl); - // Lockfile-only packages have no installed bytes to compare — the - // vendor engine fetches them pristine (nothing to annotate). - if lockfile_only_contains(lockfile_only, base) { - continue; - } - let Some(pkg) = crawled.iter().find(|c| purl_eq(&c.purl, base)) else { + let Some((pkg, embedded)) = *step else { continue; }; - // The same predicate as the download phase's ledger idempotency - // skip: its no-fetch set and this one must be the same set. - let embedded = vendor - .and_then(|entries| lookup_entry(entries, &patch.purl)) - .filter(|e| e.detached && e.uuid == patch.uuid) - .and_then(|e| e.record.as_ref()); let files: Vec<(String, PatchFileInfo)> = match embedded { Some(record) => record .files @@ -313,7 +360,22 @@ pub(super) async fn preverify_vendor_baselines( .map(|(file, info)| (file.clone(), info.clone())) .collect(), None => { - let Ok(Some(detail)) = api_client.fetch_patch(&patch.uuid).await else { + // The plan names exactly the patches this arm reaches, so + // the next view is this patch's. Checking says so out + // loud: a plan out of step would otherwise annotate this + // package against ANOTHER patch's hashes and store that + // response under this uuid for the download phase. + let detail = match details.next().await { + Some((planned, detail)) if planned == patch.uuid => detail.release(), + _ => { + debug_assert!( + false, + "baseline view prefetch plan out of step with the patches" + ); + api_client.fetch_patch(&patch.uuid).await + } + }; + let Ok(Some(detail)) = detail else { continue; }; let files = detail @@ -1934,6 +1996,150 @@ mod tests { } } + /// The baseline views are fetched concurrently, so every one must land + /// on the patch that planned it. Over a `selected` list mixing all four + /// plan outcomes — lockfile-only, uncrawled, an embedded ledger record + /// and two fetched views — with the EARLIER view answering LAST, each + /// package is compared against its own patch's files (a view that + /// slipped by one would name a file that package does not have, and + /// annotate nothing), the `views` map pairs each uuid with its own + /// response, and only the two planned views are ever requested. + #[tokio::test] + async fn preverify_pairs_each_concurrent_view_with_its_own_patch() { + use socket_patch_core::manifest::schema::{PatchFileInfo, PatchRecord}; + use socket_patch_core::vendor::state::VendorArtifact; + use socket_patch_core::vendor::VendorEntry; + use wiremock::matchers::{method, path as wm_path}; + + let mock = wiremock::MockServer::start().await; + // Each patch names a file only ITS OWN package has installed, so a + // view taken by the wrong patch verifies a path that is not there. + let view = |uuid: &str, file: &str, delay_ms: u64| { + wiremock::Mock::given(method("GET")) + .and(wm_path(format!("/patch/view/{uuid}"))) + .respond_with( + wiremock::ResponseTemplate::new(200) + .set_body_json(serde_json::json!({ + "uuid": uuid, + "purl": "pkg:npm/ignored@1.0.0", + "publishedAt": "2026-01-01T00:00:00Z", + "files": { file: { + "beforeHash": "0".repeat(64), + "afterHash": "1".repeat(64), + }}, + "vulnerabilities": {}, + "description": "", + "license": "MIT", + "tier": "free", + })) + .set_delay(std::time::Duration::from_millis(delay_ms)), + ) + .expect(1) + }; + view("u-alpha", "alpha.js", 300).mount(&mock).await; + view("u-beta", "beta.js", 0).mount(&mock).await; + + let tmp = tempfile::tempdir().unwrap(); + let installed = |name: &str, file: &str| { + let dir = tmp.path().join("node_modules").join(name); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join(file), b"installed bytes\n").unwrap(); + dir + }; + let crawled = vec![ + crawled_pkg( + "lockonly", + "pkg:npm/lockonly@1.0.0", + std::path::PathBuf::from("/nonexistent"), + ), + crawled_pkg("alpha", "pkg:npm/alpha@1.0.0", installed("alpha", "alpha.js")), + crawled_pkg( + "embedded", + "pkg:npm/embedded@1.0.0", + installed("embedded", "embedded.js"), + ), + // `beta` is installed but WITHOUT the file its own patch names, + // so only a view that slipped onto it could annotate it. + crawled_pkg("beta", "pkg:npm/beta@1.0.0", installed("beta", "other.js")), + ]; + let selected = vec![ + search_result("u-lockonly", "pkg:npm/lockonly@1.0.0"), + search_result("u-alpha", "pkg:npm/alpha@1.0.0"), + search_result("u-embedded", "pkg:npm/embedded@1.0.0"), + search_result("u-ghost", "pkg:npm/ghost@1.0.0"), + search_result("u-beta", "pkg:npm/beta@1.0.0"), + ]; + let ledger = HashMap::from([( + "pkg:npm/embedded@1.0.0".to_string(), + VendorEntry { + ecosystem: "npm".into(), + base_purl: "pkg:npm/embedded@1.0.0".into(), + uuid: "u-embedded".into(), + artifact: VendorArtifact { + path: ".socket/vendor/npm/u-embedded/embedded-1.0.0.tgz".into(), + sha256: String::new(), + size: None, + platform_locked: None, + file_inventory: None, + }, + wiring: Vec::new(), + lock: None, + took_over_go_patches: false, + detached: true, + record: Some(PatchRecord { + uuid: "u-embedded".into(), + exported_at: "2026-01-01T00:00:00Z".into(), + files: HashMap::from([( + "embedded.js".to_string(), + PatchFileInfo { + before_hash: "0".repeat(64), + after_hash: "1".repeat(64), + }, + )]), + vulnerabilities: HashMap::new(), + description: String::new(), + license: "MIT".into(), + tier: "free".into(), + }), + flavor: None, + uv: None, + pnpm: None, + poetry: None, + pdm: None, + pipenv: None, + }, + )]); + + let (mismatched, views) = preverify_vendor_baselines( + &api_client_for(&mock.uri()), + &selected, + &crawled, + &std::iter::once("pkg:npm/lockonly@1.0.0".to_string()).collect(), + Some(&ledger), + &mut crate::ui::StatusLine::new(Vec::new(), false, false, 80), + ) + .await; + + let mut flagged: Vec<&String> = mismatched.iter().collect(); + flagged.sort(); + assert_eq!(flagged, vec!["u-alpha", "u-embedded"], "{mismatched:?}"); + let mut cached: Vec<(&String, &String)> = + views.iter().map(|(uuid, v)| (uuid, &v.uuid)).collect(); + cached.sort(); + assert_eq!( + cached, + vec![ + (&"u-alpha".to_string(), &"u-alpha".to_string()), + (&"u-beta".to_string(), &"u-beta".to_string()), + ], + "each cached view must be its own patch's" + ); + // `.expect(1)` on both views is verified on drop: the skipped and + // embedded patches never reached the network. + assert_eq!(mock.received_requests().await.unwrap().len(), 2); + drop(mock); + } + #[test] fn collect_vuln_ids_dedups_across_patches() { // The same CVE appears on two patches of one package; it must be diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 2c218c539..799a89c35 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -6,9 +6,14 @@ use std::path::Path; use std::time::Duration; +use futures_util::StreamExt; +use socket_patch_core::api::client::hold_back_debug; use socket_patch_core::api::types::BatchPackagePatches; use socket_patch_core::patch::apply_lock::LockGuard; use socket_patch_core::patch::redirect::DepOverride; +use socket_patch_core::utils::concurrent::{ + api_concurrency, api_concurrency_for, ordered_concurrent, +}; use socket_patch_core::utils::purl::purl_parts; use crate::commands::vex::generate_vex_from_manifest_path; @@ -84,6 +89,25 @@ const REDIRECT_CANDIDATE_FILES: &[&str] = &[ // decision here so the omission reads as deliberate, not forgotten. ]; +/// Most hosted wheel-metadata downloads in flight at once, below the patch +/// API's own in-flight cap: each one buffers a whole wheel (up to +/// `MAX_VENDOR_PACKAGE_BYTES`) where the serial loop held one, so the +/// window is bounded by what it costs as well as by what it saves. +const WHEEL_METADATA_CONCURRENCY: usize = 4; + +/// The in-flight cap for the hosted wheel-metadata window. +/// +/// These GETs go to the patch server, so they are paced by the same knob as +/// every other patch-API window ([`api_concurrency`], and with it +/// `SOCKET_API_CONCURRENCY`) — an operator who caps in-flight requests per +/// client must be able to cap this one too, or a `uv.lock` project's wheels +/// land in `skipped` as `python_metadata_unavailable`. `api_concurrency` +/// already returns 1 under a tight descriptor limit, which is what the +/// serial loop's one-socket-at-a-time profile needs. +fn wheel_metadata_concurrency(use_public_proxy: bool) -> usize { + api_concurrency(use_public_proxy).min(WHEEL_METADATA_CONCURRENCY) +} + /// `scheme://[user[:pass]@]host[:port]/…` → `host[:port]`, NEVER userinfo. /// For user-facing messages that name where a lockfile now points — the /// hosted artifact host follows `--api-url`, so hardcoding `patch.socket.dev` @@ -224,11 +248,27 @@ fn pnpm_lock_carries_hosted_redirect( lock_text: &str, overrides: &[socket_patch_core::patch::redirect::DepOverride], ) -> bool { - overrides.iter().filter(|o| o.ecosystem == "npm").any(|o| { - let encoded = socket_patch_core::utils::uri::encode_uri_component(&o.artifact_url); - socket_patch_core::patch::redirect::artifact_url_present(lock_text, &o.artifact_url) - || lock_text.contains(encoded.as_str()) - }) + let groups: Vec> = overrides + .iter() + .filter(|o| o.ecosystem == "npm") + .map(|o| npm_lock_url_needles(&o.artifact_url)) + .collect(); + socket_patch_core::patch::redirect::presence::groups_present(&[lock_text], &groups) + .into_iter() + .any(|present| present) +} + +/// The spellings of an npm artifact URL a pnpm lock may carry — raw or +/// `\/`-escaped ([`artifact_url_spellings`](socket_patch_core::patch::redirect::artifact_url_spellings), +/// the `artifact_url_present` pair) plus the percent-encoded form — searched +/// in one multi-needle pass ([`groups_present`](socket_patch_core::patch::redirect::presence::groups_present)). +fn npm_lock_url_needles(artifact_url: &str) -> Vec { + let mut needles: Vec = + socket_patch_core::patch::redirect::artifact_url_spellings(artifact_url).into(); + needles.push(socket_patch_core::utils::uri::encode_uri_component( + artifact_url, + )); + needles } /// The HEAL-ON-RERUN gate: when this run spliced no root pnpm-lock.yaml @@ -749,22 +789,18 @@ fn gem_stale_cache_warning(purl: &str, cache_path: &Path) -> serde_json::Value { /// already-patched install must not produce a delete prescription. /// (`current_hash` is `Some` only when the bytes were really hashed, which /// also excludes the absent-new-file `Ready`.) +/// +/// The probes take this from the same one-pass +/// [`socket_patch_core::vex::verify::judge_installed_record`] that decides +/// PATCHED (`stale_evidence`); this view of it is what the unit tests pin. +#[cfg(test)] async fn installed_stale_positive_evidence( package_dir: &Path, record: &socket_patch_core::manifest::schema::PatchRecord, ) -> bool { - use socket_patch_core::patch::apply::{verify_file_patch, VerifyStatus}; - for (file_name, info) in &record.files { - let result = verify_file_patch(package_dir, file_name, info).await; - if matches!( - result.status, - VerifyStatus::Ready | VerifyStatus::HashMismatch - ) && result.current_hash.is_some() - { - return true; - } - } - false + socket_patch_core::vex::verify::judge_installed_record(package_dir, record) + .await + .stale_evidence } /// Post-rewrite stale-materialization probe for gem redirects — the guard @@ -785,12 +821,14 @@ async fn installed_stale_positive_evidence( /// itself). Record availability is part of the candidate filter, and the /// probe returns before any crawler work (or `gem env` subprocess spawn) /// when no judgment is possible. -/// * PATCHED means [`verify_patch_record`] `Ok` — the one shared oracle. +/// * PATCHED means [`verify_patch_record`] `Ok` — the one shared oracle +/// (decided, with STALE, by one pass of +/// [`socket_patch_core::vex::verify::judge_installed_record`]). /// Judgments are grouped BY INSTALLED DIR: platform-variant purls of one /// gem resolve to the same dir, and if ANY variant's record proves the /// dir patched, the dir is patched — never warned. -/// * STALE requires [`installed_stale_positive_evidence`] — never inferred from -/// missing/unreadable files. +/// * STALE requires positive evidence (the `installed_stale_positive_evidence` +/// rule) — never inferred from missing/unreadable files. /// * A committed `vendor/cache/.gem` whose sha256 differs from the /// patched artifact's is stale too (bundler installs from it first, fresh /// checkouts included): folded into a project-local install warning's @@ -815,7 +853,7 @@ async fn gem_stale_install_warnings( use socket_patch_core::crawlers::RubyCrawler; use socket_patch_core::manifest::schema::PatchRecord; use socket_patch_core::vendor::file_sha256_hex; - use socket_patch_core::vex::verify::verify_patch_record; + use socket_patch_core::vex::verify::judge_installed_record; let mut out = StaleInstallOutcome::default(); let find_record = @@ -849,16 +887,22 @@ async fn gem_stale_install_warnings( global_prefix, }; let gem_paths = crawler.get_gem_paths(&options).await.unwrap_or_default(); + // Every candidate's installed dir in every gem home, one blocking pass + // (and at most one listing) per home — the per-candidate lookups the + // loop below consumes, in the same (candidate, home) order. + let stripped: Vec = candidates + .iter() + .map(|(purl, _)| socket_patch_core::utils::purl::strip_purl_qualifiers(purl).to_string()) + .collect(); + let mut found_per_home = Vec::with_capacity(gem_paths.len()); + for gems_dir in &gem_paths { + found_per_home.push(crawler.find_each_by_purl(gems_dir, &stripped).await); + } let mut dir_state: std::collections::BTreeMap = std::collections::BTreeMap::new(); - for (purl, record) in &candidates { - let stripped = socket_patch_core::utils::purl::strip_purl_qualifiers(purl).to_string(); - for gems_dir in &gem_paths { - let found = crawler - .find_by_purls(gems_dir, std::slice::from_ref(&stripped)) - .await - .unwrap_or_default(); - let Some(pkg) = found.get(&stripped) else { + for (index, (purl, record)) in candidates.iter().enumerate() { + for found in &found_per_home { + let Some(pkg) = &found[index] else { continue; }; // A dir whose leaf isn't clean UTF-8 cannot be a real crawler @@ -875,10 +919,10 @@ async fn gem_stale_install_warnings( patched: false, positive: false, }); - if verify_patch_record(&pkg.path, record).await.is_ok() { + let judged = judge_installed_record(&pkg.path, record).await; + if judged.patched { entry.patched = true; - } else if !entry.positive && installed_stale_positive_evidence(&pkg.path, record).await - { + } else if !entry.positive && judged.stale_evidence { entry.positive = true; entry.purl = (*purl).to_string(); } @@ -966,6 +1010,31 @@ async fn gem_stale_install_warnings( out } +/// Whether the hosted flow's Pipenv probe is CERTAIN to run: the candidates +/// that no wheel-metadata failure can drop (none shares a fetched wheel's +/// artifact URL) already target an entry of Pipfile.lock, so +/// `pipenv_lock_targets` over the post-fetch overrides — a superset of +/// them — is true whatever the fetch returns. +fn pipenv_probe_certain<'a>( + files: &std::collections::BTreeMap, + candidates: impl Iterator, + fetched_wheel_urls: impl Iterator, +) -> bool { + // `pipenv_lock_targets` answers false on its first line when there is + // no Pipfile.lock, and EVERY hosted redirect run reaches this — an + // npm-only one with hundreds of candidates included. Ask that question + // before building the list to ask it with. + if !files.contains_key("Pipfile.lock") { + return false; + } + let droppable: std::collections::BTreeSet<&str> = fetched_wheel_urls.collect(); + let kept: Vec = candidates + .filter(|dep| !droppable.contains(dep.artifact_url.as_str())) + .cloned() + .collect(); + socket_patch_core::patch::redirect::pipenv_lock_targets(files, &kept) +} + /// The `(name, version)` key the gem artifact-sha map uses — derived from /// the purl so overrides (which carry no purl) and confirmed purls meet on /// neutral ground. @@ -989,6 +1058,13 @@ pub(super) async fn run_redirect( // it so the hosted `--json` envelope stays schema-consistent with every // other scan; `.take()` at each terminal (error or success) folds it in. mut scan_result: Option, + // Scan's pending telemetry, flushed by `discover_selected` before + // anything below writes to stdout. + telemetry: &mut socket_patch_core::telemetry::PendingTelemetry, + // Scan's npm crawl (`Some` only when an embedded `--vex` will run), + // handed to the VEX step so it does not walk the tree for the npm + // roots again. + npm_prior: Option<&crate::ecosystem_dispatch::NpmCrawlSnapshot>, ) -> i32 { // Same discovery/selection as `--apply`/`--vendor`. let selected = match discover_selected( @@ -998,6 +1074,8 @@ pub(super) async fn run_redirect( &args.common, false, false, + telemetry, + scan_result.as_mut(), ) .await { @@ -1033,10 +1111,75 @@ pub(super) async fn run_redirect( api_client, &pairs, scan_result, + npm_prior, ) .await } +/// How the confirmation probe in [`run_redirect_selected`] settles one +/// candidate: a non-substring rule (a transactional rewriter's own report, a +/// refusal) decides it outright, otherwise it is confirmed iff any of its +/// needles occurs in a final text. +enum ProbeStep { + Decided(bool), + Needles(Vec), +} + +/// The substrings whose presence in a final text confirms `dep`'s redirect — +/// the override's own targets: artifact URL; per-dependency registry index +/// URL; fail-closed maven's globally-unique `-socket.` suffixed version +/// (never the `.pom` URL). +/// +/// - The artifact URL in the rewriters' own spellings +/// ([`artifact_url_spellings`](socket_patch_core::patch::redirect::artifact_url_spellings), +/// raw or the `\/`-escaped slashes an old composer.lock spells them with), +/// so a writer's spelling can never be one this probe misses. It was: the +/// composer rewriter emitted `\/`-escaped urls this probe never looked for, +/// so a fully successful composer redirect reported `redirected: 0`, fetched +/// no patch record into the ledger, and left the patch unattestable by +/// `vex`. +/// - The percent-encoded URL: the berry rewriter writes it into the lock's +/// `::__archiveUrl=` binding, so the raw form is absent. +/// - The registry index URL and the maven suffixed version, when present. +fn candidate_presence_needles( + dep: &socket_patch_core::patch::redirect::DepOverride, +) -> Vec { + let artifact_url = dep.artifact_url.as_str(); + let registry = dep.registry_override.as_ref(); + let mut needles: Vec = + socket_patch_core::patch::redirect::artifact_url_spellings(artifact_url).into(); + needles.push(socket_patch_core::utils::uri::encode_uri_component( + artifact_url, + )); + if let Some(o) = registry { + needles.push(o.index_url.clone()); + if let Some(sv) = o.identifiers.maven_suffixed_version.as_deref() { + needles.push(sv.to_string()); + } + } + needles +} + +/// The per-candidate probe [`candidate_presence_needles`] + +/// `groups_present` replaced, kept as the equivalence oracle. +#[cfg(test)] +fn candidate_present_oracle( + final_texts: &[&String], + dep: &socket_patch_core::patch::redirect::DepOverride, +) -> bool { + let artifact_url = dep.artifact_url.as_str(); + let registry = dep.registry_override.as_ref(); + let index_url = registry.map(|o| o.index_url.as_str()); + let suffixed_version = registry.and_then(|o| o.identifiers.maven_suffixed_version.as_deref()); + let encoded = socket_patch_core::utils::uri::encode_uri_component(artifact_url); + final_texts.iter().any(|text| { + socket_patch_core::patch::redirect::artifact_url_present(text, artifact_url) + || text.contains(encoded.as_str()) + || index_url.is_some_and(|iu| text.contains(iu)) + || suffixed_version.is_some_and(|sv| text.contains(sv)) + }) +} + /// The hosted-redirect engine over an ALREADY-SELECTED `(purl, uuid)` set: /// reference grants → DepOverride build → apply lock (wet runs with a grant) /// → ledger load → vendored→hosted takeover pre-revert (symlink-checked @@ -1066,6 +1209,7 @@ pub(crate) async fn run_redirect_selected( api_client: &socket_patch_core::api::client::ApiClient, selected: &[(String, String)], mut scan_result: Option, + npm_prior: Option<&crate::ecosystem_dispatch::NpmCrawlSnapshot>, ) -> i32 { use socket_patch_core::manifest::schema::PatchRecord; use socket_patch_core::patch::redirect::{ @@ -1773,66 +1917,147 @@ pub(crate) async fn run_redirect_selected( // it rides the same atomic-write / ledger-first machinery as the locks. let mut python_metadata = std::collections::BTreeMap::new(); let mut unavailable_python_artifacts = std::collections::BTreeSet::new(); - for dep in candidates - .iter() - .map(|c| &c.dep) - .filter(|dep| dep.ecosystem == "pypi") + // `pipenv --version` (see `pipenv_major` below), started before the + // wheel metadata fetch when that probe is certain to be needed. + let mut pipenv_probe: Option>> = None; { - let Some(sha256) = dep.integrity.sha256.as_deref() else { - continue; - }; - if !dep - .artifact_url - .split(['?', '#']) - .next() - .is_some_and(|path| path.ends_with(".whl")) - { - continue; - } - let native_target = files + use socket_patch_core::utils::python_lock::{ArtifactSource, PythonLockProbe}; + // Each native Python lock is parsed once, on the first dep that + // needs the probe, rather than rewritten per dep just to learn + // whether it would be. + let mut probes: Option> = None; + let mut wheel_deps: Vec<(&DepOverride, &str)> = Vec::new(); + for dep in candidates .iter() - .filter(|(path, _)| { - *path == "uv.lock" - || socket_patch_core::utils::python_lock::is_script_lock_name(path) - }) - .any(|(_, text)| { - socket_patch_core::utils::python_lock::rewrite_python_lock( - text, - &dep.name, - &dep.version, - socket_patch_core::utils::python_lock::ArtifactSource::Url(&dep.artifact_url), - sha256, - ) - .ok() - .flatten() - .is_some() - }); - if !native_target { - continue; - } - status.set(format!( - "Fetching hosted wheel metadata for {}...", - dep.name - )); - match socket_patch_core::vendor::pypi::fetch_hosted_wheel_metadata( - api_client, - &dep.artifact_url, - sha256, - ) - .await + .map(|c| &c.dep) + .filter(|dep| dep.ecosystem == "pypi") { - Ok(Some(metadata)) => { - python_metadata.insert(dep.artifact_url.clone(), metadata); + let Some(sha256) = dep.integrity.sha256.as_deref() else { + continue; + }; + if !dep + .artifact_url + .split(['?', '#']) + .next() + .is_some_and(|path| path.ends_with(".whl")) + { + continue; } - Ok(None) => {} - Err(detail) => { - unavailable_python_artifacts.insert(dep.artifact_url.clone()); - skipped.push(serde_json::json!({ - "purl": format!("pkg:pypi/{}@{}", dep.name, dep.version), - "uuid": dep.patch_uuid, - "reason": "python_metadata_unavailable", - "detail": detail.replace(&dep.artifact_url, ""), - })); + let native_target = probes + .get_or_insert_with(|| { + files + .iter() + .filter(|(path, _)| { + *path == "uv.lock" + || socket_patch_core::utils::python_lock::is_script_lock_name(path) + }) + .map(|(_, text)| PythonLockProbe::new(text)) + .collect() + }) + .iter() + .any(|probe| { + probe.rewrites( + &dep.name, + &dep.version, + ArtifactSource::Url(&dep.artifact_url), + ) + }); + if native_target { + wheel_deps.push((dep, sha256)); + } + } + // The only candidates the metadata fetch can still drop are those + // sharing a fetched wheel's artifact URL. If the rest already + // target an entry of Pipfile.lock, the Pipenv probe below is certain + // to run: start it now so it overlaps the fetch. Otherwise it runs + // (or not) exactly where it always did. + if pipenv_probe_certain( + &files, + candidates.iter().map(|c| &c.dep), + wheel_deps.iter().map(|(dep, _)| dep.artifact_url.as_str()), + ) { + let root = common.cwd.clone(); + pipenv_probe = Some(tokio::spawn(async move { + socket_patch_core::utils::pipenv::installed_major(&root).await + })); + } + // The wheels' FIRST attempts run concurrently and are folded in dep + // order, so `python_metadata`, `unavailable_python_artifacts` and + // `skipped` come out exactly as the serial loop's did; each attempt's + // opt-in debug lines are held back and printed at its fold, so they + // keep the serial order too. + // + // An attempt the client would RETRY (a 429 / 5xx / transport + // failure) is never settled concurrently. At the first one no + // further attempt is started, the ones already in flight are awaited + // (so the host is idle again, as the serial loop would find it), and + // that dep plus every later one are finished one at a time. A + // deferred attempt is RESUMED, not restarted: `Retry-After` is + // waited out and only the budget it left is spent, so each wheel + // costs the host exactly the requests the serial loop's would have. + // What stays different is only their overlap: a host that answers a + // burst differently than it answers the same requests one at a time + // (a sliding-window limiter, a bot challenge) can still hand back a + // status the serial loop would not have seen. The first such answer + // is what closes the window. + // + // Kept small on purpose, and paced by `SOCKET_API_CONCURRENCY` like + // every other patch-API window — see `wheel_metadata_concurrency`. + let wheel_metadata_concurrency = wheel_metadata_concurrency(api_client.uses_public_proxy()); + use futures_util::StreamExt as _; + use socket_patch_core::vendor::pypi::{ + finish_hosted_wheel_metadata, try_fetch_hosted_wheel_metadata_once, + }; + // Lazily built: a dep's attempt starts only once it is pulled here. + let mut unstarted = wheel_deps.iter().map(|&(dep, sha256)| { + try_fetch_hosted_wheel_metadata_once(api_client, &dep.artifact_url, sha256) + }); + let mut in_flight: futures_util::stream::FuturesOrdered<_> = unstarted + .by_ref() + .take(wheel_metadata_concurrency) + .collect(); + // Once serial: the attempts that were in flight when a dep needed a + // retry, in dep order (the deps after them were never started). + let mut drained: Option> = None; + for &(dep, sha256) in &wheel_deps { + status.set(format!( + "Fetching hosted wheel metadata for {}...", + dep.name + )); + let attempt = match drained.as_mut() { + Some(drained) => drained.pop_front(), + None => match in_flight.next().await { + Some(attempt) if !attempt.needs_retry() => { + in_flight.extend(unstarted.next()); + Some(attempt) + } + // A retryable failure (or nothing left in flight): let + // the host go idle, then finish one at a time from here. + struggling => { + let mut rest = std::collections::VecDeque::new(); + while let Some(later) = in_flight.next().await { + rest.push_back(later); + } + drained = Some(rest); + struggling + } + }, + }; + match finish_hosted_wheel_metadata(api_client, &dep.artifact_url, sha256, attempt).await + { + Ok(Some(metadata)) => { + python_metadata.insert(dep.artifact_url.clone(), metadata); + } + Ok(None) => {} + Err(detail) => { + unavailable_python_artifacts.insert(dep.artifact_url.clone()); + skipped.push(serde_json::json!({ + "purl": format!("pkg:pypi/{}@{}", dep.name, dep.version), + "uuid": dep.patch_uuid, + "reason": "python_metadata_unavailable", + "detail": detail.replace(&dep.artifact_url, ""), + })); + } } } } @@ -1849,10 +2074,19 @@ pub(crate) async fn run_redirect_selected( // run must neither spawn Pipenv nor warn about its absence. let targets_pipenv_lock = socket_patch_core::patch::redirect::pipenv_lock_targets(&files, &overrides); - let pipenv_major = if targets_pipenv_lock { - socket_patch_core::utils::pipenv::installed_major(&common.cwd).await - } else { - None + let pipenv_major = match (targets_pipenv_lock, pipenv_probe) { + (true, Some(probe)) => match probe.await { + Ok(major) => major, + Err(err) => std::panic::resume_unwind(err.into_panic()), + }, + (true, None) => socket_patch_core::utils::pipenv::installed_major(&common.cwd).await, + // Unreachable (the early start implies the target), but never leave + // a probe running: aborting drops it, which reaps the child. + (false, Some(probe)) => { + probe.abort(); + None + } + (false, None) => None, }; let binary_content = if binary_bun && overrides.iter().any(|o| o.ecosystem == "npm") { Some( @@ -1876,12 +2110,23 @@ pub(crate) async fn run_redirect_selected( .filter(|o| !(binary_content.as_ref().is_some_and(Result::is_err) && o.ecosystem == "npm")) .cloned() .collect(); - let mut rewrite = rewrite_registry_redirect_with_pipenv_version( - &files, - &rewrite_overrides, - &python_metadata, - pipenv_major, - ); + // Pure CPU over every lock text (the independent rewriter groups run + // concurrently inside), so it runs on the blocking pool rather than on a + // runtime worker; `files` comes back for the confirmation probe below. + let (files, mut rewrite) = tokio::task::spawn_blocking(move || { + let rewrite = rewrite_registry_redirect_with_pipenv_version( + &files, + &rewrite_overrides, + &python_metadata, + pipenv_major, + ); + (files, rewrite) + }) + .await + .unwrap_or_else(|e| match e.try_into_panic() { + Ok(payload) => std::panic::resume_unwind(payload), + Err(e) => panic!("hosted rewrite task failed: {e}"), + }); if let Some(content) = binary_content { rewrite .warnings @@ -2040,20 +2285,20 @@ pub(crate) async fn run_redirect_selected( // `\/`-escaped via artifact_url_present, plus the percent-encoded // spelling) so a writer's spelling can never be one this filter // misses. - let mut hosts: Vec<&str> = overrides + let npm_overrides: Vec<_> = overrides.iter().filter(|o| o.ecosystem == "npm").collect(); + let groups: Vec> = npm_overrides .iter() - .filter(|o| o.ecosystem == "npm") - .filter(|o| { - let encoded = - socket_patch_core::utils::uri::encode_uri_component(&o.artifact_url); - pnpm_lock_texts.iter().any(|text| { - socket_patch_core::patch::redirect::artifact_url_present( - text, - &o.artifact_url, - ) || text.contains(encoded.as_str()) - }) - }) - .filter_map(|o| url_host(&o.artifact_url)) + .map(|o| npm_lock_url_needles(&o.artifact_url)) + .collect(); + let present = socket_patch_core::patch::redirect::presence::groups_present( + &pnpm_lock_texts, + &groups, + ); + let mut hosts: Vec<&str> = npm_overrides + .iter() + .zip(present) + .filter(|(_, present)| *present) + .filter_map(|(o, _)| url_host(&o.artifact_url)) // Dry-run takeover purls land in the root lock on the wet run. .chain(takeover_pnpm_urls.iter().filter_map(|url| url_host(url))) .collect(); @@ -2204,24 +2449,26 @@ pub(crate) async fn run_redirect_selected( None; { let npm_hosts: Vec<&str> = { - let mut hosts: Vec<&str> = overrides + let npm_lock_texts: Vec<&String> = NPM_LOCKS .iter() - .filter(|o| o.ecosystem == "npm") - .filter(|o| { - NPM_LOCKS.iter().any(|lock| { - rewrite - .files - .get(*lock) - .or_else(|| files.get(*lock)) - .is_some_and(|text| { - socket_patch_core::patch::redirect::artifact_url_present( - text, - &o.artifact_url, - ) - }) - }) + .filter_map(|lock| rewrite.files.get(*lock).or_else(|| files.get(*lock))) + .collect(); + let npm_overrides: Vec<_> = overrides.iter().filter(|o| o.ecosystem == "npm").collect(); + let groups: Vec<[String; 2]> = npm_overrides + .iter() + .map(|o| { + socket_patch_core::patch::redirect::artifact_url_spellings(&o.artifact_url) }) - .filter_map(|o| url_host(&o.artifact_url)) + .collect(); + let present = socket_patch_core::patch::redirect::presence::groups_present( + &npm_lock_texts, + &groups, + ); + let mut hosts: Vec<&str> = npm_overrides + .iter() + .zip(present) + .filter(|(_, present)| *present) + .filter_map(|(o, _)| url_host(&o.artifact_url)) // A dry-run vendored→hosted takeover: the wet run reverts // the vendored wiring in a root npm lock and splices the // hosted URL there, so preview the `.npmrc` write too. @@ -2345,16 +2592,22 @@ pub(crate) async fn run_redirect_selected( .map(|(_, content)| content), ) .collect(); - let confirmed: Vec<(String, String)> = candidates + // Every non-substring rule decides a candidate outright; the rest are + // confirmed by substring presence of their needles in the final texts. + // All needle groups are answered in ONE multi-needle pass per text + // (`groups_present`), which is the per-candidate `any()` exactly — + // presence does not depend on search order, and `confirmed` keeps + // candidate order. See `confirm_candidates_oracle` for the old form. + let steps: Vec = candidates .iter() - .filter(|c| { + .map(|c| { let purl = c.purl.as_str(); let uuid = c.dep.patch_uuid.as_str(); if binary_bun && purl.starts_with("pkg:npm/") { - return rewrite.confirmed_bun_binary_uuids.contains(uuid); + return ProbeStep::Decided(rewrite.confirmed_bun_binary_uuids.contains(uuid)); } if rewrite.refused_pipenv_uuids.contains(uuid) { - return false; + return ProbeStep::Decided(false); } // pdm is transactional like cargo: a refused uuid is never // confirmed, and when `pdm.lock` is the PyPI install driver @@ -2367,28 +2620,32 @@ pub(crate) async fn run_redirect_selected( // backend, which registers every pypi uuid as hatch-owned while // the lock's presence keeps hatch from confirming any of them. if rewrite.refused_pdm_uuids.contains(uuid) { - return false; + return ProbeStep::Decided(false); } if purl.starts_with("pkg:pypi/") && socket_patch_core::patch::redirect::pdm_drives(&files) { - return rewrite.confirmed_pdm_uuids.contains(uuid); + return ProbeStep::Decided(rewrite.confirmed_pdm_uuids.contains(uuid)); } if rewrite.python_lock_uuids.contains(uuid) { - return rewrite.confirmed_python_lock_uuids.contains(uuid) - && !rewrite.refused_python_lock_uuids.contains(uuid); + return ProbeStep::Decided( + rewrite.confirmed_python_lock_uuids.contains(uuid) + && !rewrite.refused_python_lock_uuids.contains(uuid), + ); } if rewrite.hatch_uuids.contains(uuid) { - return rewrite.confirmed_hatch_uuids.contains(uuid); + return ProbeStep::Decided(rewrite.confirmed_hatch_uuids.contains(uuid)); } // A Pipfile.lock rewrite confirms its own uuids (the sibling // requirements.txt rewriter may have had nothing to do). if purl.starts_with("pkg:pypi/") { - return rewrite.confirmed_pipenv_uuids.contains(uuid) - || rewrite.confirmed_requirements_uuids.contains(uuid); + return ProbeStep::Decided( + rewrite.confirmed_pipenv_uuids.contains(uuid) + || rewrite.confirmed_requirements_uuids.contains(uuid), + ); } if rewrite.refused_pnpm_uuids.contains(uuid) { - return false; + return ProbeStep::Decided(false); } // Cargo is transactional: the rewriter reports exactly which // patch uuids FULLY landed (manifest pin + lock + registry @@ -2397,40 +2654,37 @@ pub(crate) async fn run_redirect_selected( // pinning nothing, so a config-block-only rewrite would be // attested with zero enforcement in any build. if purl.starts_with("pkg:cargo/") { - return rewrite.confirmed_cargo_uuids.contains(uuid); + return ProbeStep::Decided(rewrite.confirmed_cargo_uuids.contains(uuid)); } // Golang likewise: the goproxy `indexUrl` is the bare // patch-server origin (present in any other hosted lock), and // the socket module's go.sum lines outlive a removed replace. if purl.starts_with("pkg:golang/") { - return rewrite.confirmed_golang_uuids.contains(uuid); + return ProbeStep::Decided(rewrite.confirmed_golang_uuids.contains(uuid)); } - // The override's own targets: artifact URL; per-dependency - // registry index URL; fail-closed maven's globally-unique - // `-socket.` suffixed version (never the `.pom` URL). - let artifact_url = c.dep.artifact_url.as_str(); - let registry = c.dep.registry_override.as_ref(); - let index_url = registry.map(|o| o.index_url.as_str()); - let suffixed_version = - registry.and_then(|o| o.identifiers.maven_suffixed_version.as_deref()); - let encoded = socket_patch_core::utils::uri::encode_uri_component(artifact_url); - final_texts.iter().any(|text| { - // The rewriters' own predicate — raw, or the `\/`-escaped - // slashes an old composer.lock spells them with — so a - // writer's spelling can never be one this probe misses. It - // was: the composer rewriter emitted `\/`-escaped urls this - // probe never looked for, so a fully successful composer - // redirect reported `redirected: 0`, fetched no patch record - // into the ledger, and left the patch unattestable by `vex`. - socket_patch_core::patch::redirect::artifact_url_present(text, artifact_url) - // The berry rewriter writes the URL percent-encoded into the - // lock's `::__archiveUrl=` binding, so the raw form is absent. - || text.contains(encoded.as_str()) - || index_url.is_some_and(|iu| text.contains(iu)) - || suffixed_version.is_some_and(|sv| text.contains(sv)) - }) + ProbeStep::Needles(candidate_presence_needles(&c.dep)) + }) + .collect(); + let groups: Vec<&[String]> = steps + .iter() + .filter_map(|step| match step { + ProbeStep::Needles(needles) => Some(needles.as_slice()), + ProbeStep::Decided(_) => None, + }) + .collect(); + let mut present = + socket_patch_core::patch::redirect::presence::groups_present(&final_texts, &groups) + .into_iter(); + let confirmed: Vec<(String, String)> = candidates + .iter() + .zip(&steps) + .filter(|(_, step)| match step { + ProbeStep::Decided(keep) => *keep, + ProbeStep::Needles(_) => present + .next() + .expect("one presence answer per needle group"), }) - .map(|c| (c.purl.clone(), c.dep.patch_uuid.clone())) + .map(|(c, _)| (c.purl.clone(), c.dep.patch_uuid.clone())) .collect(); // Dry-run mode-takeover previews were withheld from the rewriters (their // lock fragments still carry the vendored wiring the wet run reverts @@ -2476,12 +2730,34 @@ pub(crate) async fn run_redirect_selected( if !common.dry_run { let total = confirmed.len(); - for (i, (purl, uuid)) in confirmed.iter().enumerate() { + // The views are fetched concurrently but consumed in `confirmed` + // order, so `records` (newest wins) and `record_warnings` fold + // exactly as the serial loop's did. Each response is reduced to + // its record inside the window: a view carries every file's + // `blobContent`, so buffering whole responses would hold the cap's + // worth of patch payloads in memory at once, where the loop only + // ever needed the hashes. `record_from_patch_response` is pure, so + // folding it early changes nothing downstream. Each fetch's + // `--debug` lines are held back and printed at its fold, where the + // serial loop would have made the request. + let mut views = std::pin::pin!(ordered_concurrent( + confirmed.iter(), + api_concurrency_for(api_client.uses_public_proxy(), confirmed.len()), + |(_, uuid)| { + hold_back_debug(async move { + api_client.fetch_patch(uuid).await.map(|resp| { + resp.map(|resp| crate::commands::get::record_from_patch_response(&resp)) + }) + }) + }, + )); + for (i, (purl, _)) in confirmed.iter().enumerate() { status.set(format!("Fetching patch records... ({}/{total})", i + 1)); - match api_client.fetch_patch(uuid).await { - Ok(Some(resp)) => { - let (rec_purl, record) = - crate::commands::get::record_from_patch_response(&resp); + let Some(view) = views.next().await else { + break; + }; + match view.release() { + Ok(Some((rec_purl, record))) => { records.insert(rec_purl, record); } Ok(None) | Err(_) => { @@ -2763,6 +3039,13 @@ pub(crate) async fn run_redirect_selected( let mut vex_code = 0; if vex.vex.is_some() && !common.dry_run { let mut params = vex.to_build_params(); + // Hosted mode wrote only lockfiles and config files since scan's + // crawl, never a directory the npm root walk descends into, so its + // roots and packages still describe the tree (the snapshot checks + // it was taken with these crawler options). The interactive scan + // hands none in when its confirm prompt waited on a person — the + // tree may have changed while it did. + params.npm_prior = npm_prior.cloned(); // Stale-flagged purls are EXCLUDED from assume_applied: the same-run // envelope carries a redirect_gem_stale_install warning proving the // installed materialization unpatched, so attesting that purl from @@ -3306,6 +3589,7 @@ pub(crate) fn boxed_run_redirect_selected<'a>( api_client: &'a socket_patch_core::api::client::ApiClient, selected: &'a [(String, String)], scan_result: Option, + npm_prior: Option<&'a crate::ecosystem_dispatch::NpmCrawlSnapshot>, ) -> std::pin::Pin + 'a>> { Box::pin(run_redirect_selected( common, @@ -3314,6 +3598,7 @@ pub(crate) fn boxed_run_redirect_selected<'a>( api_client, selected, scan_result, + npm_prior, )) } @@ -3337,8 +3622,50 @@ mod tests { pnpm_lock_may_need_store_flag, pnpm_trust_rerun_reminder, sentence_case, split_sentences, wrap_tokens, wrap_words, TAKEOVER_INFO_CODES, }; + use super::{wheel_metadata_concurrency, WHEEL_METADATA_CONCURRENCY}; use socket_patch_core::constants::npm_family; use socket_patch_core::patch::redirect::DepOverride; + use socket_patch_core::utils::concurrent::API_CONCURRENCY_ENV; + + /// The wheel window is a patch-API window, so the documented escape + /// hatch has to reach it: an operator behind something that caps + /// in-flight requests per client sets `SOCKET_API_CONCURRENCY=1` and + /// gets one artifact GET at a time here too — otherwise the capping + /// endpoint rejects the extras and those deps land in `skipped` as + /// `python_metadata_unavailable`. Serial: `SOCKET_*` is process-global. + #[test] + #[serial_test::serial] + fn socket_api_concurrency_paces_the_wheel_metadata_window() { + let orig = std::env::var(API_CONCURRENCY_ENV).ok(); + std::env::remove_var(API_CONCURRENCY_ENV); + // The window's own ceiling still binds: the authenticated cap is 32, + // but a whole wheel per in-flight request is what sizes this one. + assert_eq!( + wheel_metadata_concurrency(false), + WHEEL_METADATA_CONCURRENCY + ); + assert_eq!(wheel_metadata_concurrency(true), WHEEL_METADATA_CONCURRENCY); + + std::env::set_var(API_CONCURRENCY_ENV, "1"); + assert_eq!(wheel_metadata_concurrency(false), 1); + assert_eq!(wheel_metadata_concurrency(true), 1); + + // A value between 1 and the ceiling lowers the window to it. + std::env::set_var(API_CONCURRENCY_ENV, "2"); + assert_eq!(wheel_metadata_concurrency(false), 2); + + // Raising the API cap never raises this one past its own ceiling. + std::env::set_var(API_CONCURRENCY_ENV, "32"); + assert_eq!( + wheel_metadata_concurrency(false), + WHEEL_METADATA_CONCURRENCY + ); + + match orig { + Some(v) => std::env::set_var(API_CONCURRENCY_ENV, v), + None => std::env::remove_var(API_CONCURRENCY_ENV), + } + } /// Lock-head version sniff against the byte-real heads the 2026-08-18 /// matrix captured from pnpm 7/8/9-12: quoted `'9.0'` and `'6.0'`, @@ -3590,6 +3917,101 @@ mod tests { assert!(detail.contains("pnpm clean --lockfile"), "{detail}"); } + /// The early Pipenv probe start is exact: whenever it fires, the + /// post-fetch gate is true for EVERY outcome of the wheel metadata + /// fetch (any subset of the fetched wheels' URLs dropped). + #[test] + fn pipenv_probe_certain_implies_the_post_fetch_gate() { + use super::pipenv_probe_certain; + use socket_patch_core::patch::redirect::pipenv_lock_targets; + + let lock = serde_json::json!({ + "_meta": {"pipfile-spec": 6, "hash": {"sha256": "x"}}, + "default": {"urllib3": {"version": "==1.26.18"}, "six": {"version": "==1.16.0"}}, + "develop": {}, + }); + let files = std::collections::BTreeMap::from([( + "Pipfile.lock".to_string(), + serde_json::to_string_pretty(&lock).unwrap(), + )]); + let names = ["urllib3", "Six", "requests", "idna"]; + let urls = ["u0", "u1", "u2", "u3"]; + let (mut fired, mut held) = (0, 0); + for seed in 0..4096u64 { + // Deterministic spread over names, ecosystems, URLs and the + // fetched-URL set. + let mut x = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1; + let mut next = |n: u64| { + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + (x.wrapping_mul(0x2545_F491_4F6C_DD1D) % n) as usize + }; + let candidates: Vec = (0..next(4)) + .map(|_| { + let mut dep = npm_override(urls[next(4)]); + dep.name = names[next(4)].to_string(); + if next(4) != 0 { + dep.ecosystem = "pypi".to_string(); + } + dep + }) + .collect(); + let fetched: Vec<&str> = urls.iter().copied().filter(|_| next(2) == 0).collect(); + if !pipenv_probe_certain(&files, candidates.iter(), fetched.iter().copied()) { + held += 1; + continue; + } + fired += 1; + for mask in 0..(1u32 << fetched.len()) { + let dropped: Vec<&str> = fetched + .iter() + .enumerate() + .filter(|(i, _)| mask & (1 << i) != 0) + .map(|(_, url)| *url) + .collect(); + let kept: Vec = candidates + .iter() + .filter(|dep| !dropped.contains(&dep.artifact_url.as_str())) + .cloned() + .collect(); + assert!( + pipenv_lock_targets(&files, &kept), + "seed {seed} mask {mask}" + ); + } + } + assert!(fired > 100 && held > 100, "{fired}/{held}"); + } + + /// Without a Pipfile.lock the gate is false whatever the candidates + /// are, exactly as `pipenv_lock_targets` answers it — so the early + /// return skips only the list the question would have been asked with. + #[test] + fn pipenv_probe_certain_is_false_without_a_pipfile_lock() { + use super::pipenv_probe_certain; + use socket_patch_core::patch::redirect::pipenv_lock_targets; + + let files = + std::collections::BTreeMap::from([("package-lock.json".to_string(), "{}".to_string())]); + let mut pypi = npm_override("u1"); + pypi.ecosystem = "pypi".to_string(); + pypi.name = "urllib3".to_string(); + let candidates = [npm_override("u0"), pypi]; + + assert!(!pipenv_probe_certain( + &files, + candidates.iter(), + std::iter::empty() + )); + assert!(!pipenv_lock_targets(&files, &candidates)); + assert!(!pipenv_probe_certain( + &std::collections::BTreeMap::new(), + candidates.iter(), + std::iter::empty() + )); + } + fn npm_override(artifact_url: &str) -> DepOverride { DepOverride { ecosystem: "npm".to_string(), @@ -4968,3 +5390,171 @@ mod tests { } } } + +/// H1 equivalence: the one-pass multi-needle confirmation probe answers +/// exactly what the per-candidate `any()` it replaced answered. +#[cfg(test)] +mod probe_equivalence_tests { + use super::{candidate_presence_needles, candidate_present_oracle, npm_lock_url_needles}; + use socket_patch_core::patch::redirect::presence::groups_present; + use socket_patch_core::patch::redirect::{ + artifact_url_present, artifact_url_spellings, rewrite_registry_redirect, DepOverride, + }; + use socket_patch_core::utils::uri::encode_uri_component; + use std::collections::BTreeMap; + use std::path::{Path, PathBuf}; + + fn golden_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../socket-patch-core/tests/fixtures/redirect") + } + + fn cases(dir: &Path, out: &mut Vec) { + if dir.join("input").is_dir() && dir.join("overrides.json").is_file() { + out.push(dir.to_path_buf()); + return; + } + for entry in std::fs::read_dir(dir).unwrap() { + let p = entry.unwrap().path(); + if p.is_dir() { + cases(&p, out); + } + } + } + + fn read_tree(base: &Path) -> BTreeMap { + fn walk(base: &Path, dir: &Path, out: &mut BTreeMap) { + for entry in std::fs::read_dir(dir).unwrap() { + let p = entry.unwrap().path(); + if p.is_dir() { + walk(base, &p, out); + } else if let Ok(text) = std::fs::read_to_string(&p) { + let rel = p.strip_prefix(base).unwrap().to_string_lossy(); + out.insert(rel.replace('\\', "/"), text); + } + } + } + let mut out = BTreeMap::new(); + if base.is_dir() { + walk(base, base, &mut out); + } + out + } + + /// Every golden fixture (composer `\/`, berry percent-encoded, maven + /// suffixed version, go module path, cargo index url, …): each case's + /// final texts — input overlaid with this CLI's own rewrite, as the probe + /// sees them — plus its authored `expected/` files, probed with EVERY + /// fixture's overrides so hits and misses are both well exercised. + #[test] + fn multi_needle_probe_matches_per_candidate_any_on_golden_fixtures() { + let mut dirs = Vec::new(); + cases(&golden_root(), &mut dirs); + dirs.sort(); + assert!(dirs.len() > 50, "golden fixtures not found: {}", dirs.len()); + + let mut all_overrides: Vec = Vec::new(); + let mut text_sets: Vec> = Vec::new(); + for case in &dirs { + let overrides: Vec = match serde_json::from_str( + &std::fs::read_to_string(case.join("overrides.json")).unwrap(), + ) { + Ok(o) => o, + Err(_) => continue, + }; + let input = read_tree(&case.join("input")); + let rewrite = rewrite_registry_redirect(&input, &overrides); + let finals: Vec = input + .iter() + .map(|(name, text)| rewrite.files.get(name).unwrap_or(text).clone()) + .chain( + rewrite + .files + .iter() + .filter(|(name, _)| !input.contains_key(*name)) + .map(|(_, t)| t.clone()), + ) + .collect(); + text_sets.push(finals); + text_sets.push(read_tree(&case.join("expected")).into_values().collect()); + text_sets.push(input.into_values().collect()); + all_overrides.extend(overrides); + } + text_sets.push(Vec::new()); + // Texts that carry ONE needle kind and nothing else — no golden + // fixture has the maven suffixed version or the registry index URL + // without the artifact URL beside it, so a needle dropped from + // `candidate_presence_needles` would otherwise go unnoticed. + let mut lone_suffixed: Vec = Vec::new(); + for o in all_overrides + .iter() + .filter_map(|d| d.registry_override.as_ref()) + { + text_sets.push(vec![format!("{}\n", o.index_url)]); + if let Some(sv) = o.identifiers.maven_suffixed_version.as_deref() { + lone_suffixed.push(text_sets.len()); + text_sets.push(vec![format!("{sv}\n")]); + } + } + assert!( + !lone_suffixed.is_empty(), + "no maven suffixed-version override" + ); + + let groups: Vec> = all_overrides + .iter() + .map(candidate_presence_needles) + .collect(); + let (mut hits, mut misses) = (0usize, 0usize); + for (set, texts) in text_sets.iter().enumerate() { + let refs: Vec<&String> = texts.iter().collect(); + let fast = groups_present(&refs, &groups); + if lone_suffixed.contains(&set) { + assert!( + fast.iter().any(|hit| *hit), + "a lone suffixed version confirms its maven override" + ); + } + for (dep, got) in all_overrides.iter().zip(&fast) { + let want = candidate_present_oracle(&refs, dep); + assert_eq!( + *got, want, + "{}/{} / {}", + dep.ecosystem, dep.name, dep.artifact_url + ); + if want { + hits += 1; + } else { + misses += 1; + } + } + } + assert!(hits > 100 && misses > 100, "hits={hits} misses={misses}"); + + // The pnpm / npm host filters and the heal probe: the npm lock + // spellings, and the bare `artifact_url_present` pair. + let npm: Vec<&DepOverride> = all_overrides.iter().collect(); + let lock_groups: Vec> = npm + .iter() + .map(|o| npm_lock_url_needles(&o.artifact_url)) + .collect(); + let pair_groups: Vec<[String; 2]> = npm + .iter() + .map(|o| artifact_url_spellings(&o.artifact_url)) + .collect(); + for texts in &text_sets { + let lock_fast = groups_present(texts, &lock_groups); + let pair_fast = groups_present(texts, &pair_groups); + for (i, o) in npm.iter().enumerate() { + let encoded = encode_uri_component(&o.artifact_url); + let pair = texts + .iter() + .any(|t| artifact_url_present(t, &o.artifact_url)); + let lock = texts.iter().any(|t| { + artifact_url_present(t, &o.artifact_url) || t.contains(encoded.as_str()) + }); + assert_eq!(pair_fast[i], pair, "{}", o.artifact_url); + assert_eq!(lock_fast[i], lock, "{}", o.artifact_url); + } + } + } +} diff --git a/crates/socket-patch-cli/src/commands/scan/hosted/python.rs b/crates/socket-patch-cli/src/commands/scan/hosted/python.rs index d48fab66b..d24956c7b 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted/python.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted/python.rs @@ -5,9 +5,9 @@ use std::collections::{BTreeMap, BTreeSet}; use socket_patch_core::crawlers::{types::CrawlerOptions, PythonCrawler}; use socket_patch_core::manifest::schema::PatchRecord; use socket_patch_core::utils::purl::strip_purl_qualifiers; -use socket_patch_core::vex::verify::verify_patch_record; +use socket_patch_core::vex::verify::judge_installed_record; -use super::{installed_stale_positive_evidence, StaleInstallOutcome}; +use super::StaleInstallOutcome; /// A lock rewrite cannot prove a warm virtualenv has installed the wheel. /// Use the same discovery as apply (including Poetry's out-of-tree venvs), @@ -69,26 +69,33 @@ pub(super) async fn stale_install_warnings( // package patched; a healthy *different* package cannot. let mut judgments = BTreeMap::new(); let mut pipenv_purls: BTreeSet = BTreeSet::new(); - for (purl, record) in candidates { + // Every candidate's installed copy in every site, from one listing per + // site — the per-candidate lookups the loop below consumes, in the same + // (candidate, site) order. + let bases: Vec = candidates + .iter() + .map(|(purl, _)| strip_purl_qualifiers(purl).to_string()) + .collect(); + let mut found_per_site = Vec::with_capacity(paths.len()); + for site in &paths { + found_per_site.push(crawler.find_each_by_purl(site, &bases).await); + } + for (index, (purl, record)) in candidates.into_iter().enumerate() { if pipenv_uuids.contains(&record.uuid) { pipenv_purls.insert(purl.clone()); } - let base = strip_purl_qualifiers(purl).to_string(); - for site in &paths { - let found = crawler - .find_by_purls(site, std::slice::from_ref(&base)) - .await - .unwrap_or_default(); - let Some(pkg) = found.get(&base) else { + for found in &found_per_site { + let Some(pkg) = &found[index] else { continue; }; let judgment: &mut Judgment = judgments .entry((pkg.path.clone(), pkg.name.clone(), pkg.version.clone())) .or_default(); judgment.purls.insert(purl.clone()); - if verify_patch_record(&pkg.path, record).await.is_ok() { + let judged = judge_installed_record(&pkg.path, record).await; + if judged.patched { judgment.patched = true; - } else if installed_stale_positive_evidence(&pkg.path, record).await { + } else if judged.stale_evidence { judgment.stale = true; } } diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index 0e727c382..6ab75db78 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -7,15 +7,20 @@ //! small helpers shared across the submodules. use clap::Args; +use futures_util::StreamExt; use socket_patch_core::api::client::{ - build_proxy_fallback_client, get_api_client_with_overrides, is_fallback_candidate, ApiClient, + build_proxy_fallback_client, get_api_client_with_overrides, hold_back_debug, + is_fallback_candidate, ApiClient, ApiError, }; -use socket_patch_core::api::types::{BatchPackagePatches, PatchSearchResult}; +use socket_patch_core::api::types::{BatchPackagePatches, BatchSearchResponse, PatchSearchResult}; use socket_patch_core::crawlers::ruby_crawler::config_path_ignored_warning; use socket_patch_core::crawlers::{CrawlerOptions, Ecosystem}; use socket_patch_core::manifest::operations::read_manifest; use socket_patch_core::manifest::schema::PatchManifest; -use socket_patch_core::telemetry::{track_patch_scan_failed, track_patch_scanned}; +use socket_patch_core::telemetry::{ + spawn_patch_scan_failed, spawn_patch_scanned, PendingTelemetry, +}; +use socket_patch_core::utils::concurrent::{api_concurrency_for, ordered_concurrent}; use socket_patch_core::utils::purl::{normalize_purl, purl_name_version, strip_purl_qualifiers}; use socket_patch_core::vendor::VendorState; use socket_patch_core::vex::discover::{LedgerLiveness, WiringMode}; @@ -25,7 +30,7 @@ use std::path::Path; use crate::args::{apply_env_toggles, GlobalArgs}; use crate::commands::vex::{generate_vex_from_manifest_path, VexEmbedArgs}; -use crate::ecosystem_dispatch::crawl_all_ecosystems; +use crate::ecosystem_dispatch::{crawl_ecosystems, crawl_ecosystems_with_npm}; use crate::ui::{self, plural, print_json, StatusLine}; use super::get::{download_and_apply_patches_with, select_patches, DownloadParams, DownloadRun}; @@ -57,7 +62,78 @@ use self::vendor_flow::{ partition_skipped_selected, }; -const DEFAULT_BATCH_SIZE: usize = 100; +/// Packages per batch request on the authenticated API when `--batch-size` +/// is not given: the server's own per-request maximum +/// (`MAX_PURLS_PER_BATCH` on `POST /v0/orgs/{org}/patches/batch`). +const DEFAULT_BATCH_SIZE: usize = 500; + +/// Packages per batch request on the public proxy when `--batch-size` is +/// not given. The proxy is shared and unauthenticated, so it keeps the +/// historical size. +const DEFAULT_PROXY_BATCH_SIZE: usize = 100; + +/// Upper bound on one batch request's JSON body. A chunk whose purls would +/// serialize past it is split, deterministically, into consecutive smaller +/// chunks. The value is the public proxy's own body cap +/// (`MAX_PATCH_PROXY_BODY_BYTES`, answered with a 413 past it) — the +/// tightest limit any batch route has (the authenticated API accepts +/// 16 MiB). A batch the fallback re-sends to the proxy therefore always +/// fits, whichever endpoint the chunks were sized for. +const BATCH_BODY_BYTE_CAP: usize = 256 * 1024; + +/// The chunk size in effect: `--batch-size` / `SOCKET_BATCH_SIZE` when +/// given (on either endpoint), else [`DEFAULT_BATCH_SIZE`] on the +/// authenticated API and [`DEFAULT_PROXY_BATCH_SIZE`] on the public proxy. +/// Floored at 1: `--batch-size 0` is otherwise unvalidated and would make +/// the chunking below panic, so it degrades to one-package batches. +fn effective_batch_size(requested: Option, use_public_proxy: bool) -> usize { + requested + .unwrap_or(if use_public_proxy { + DEFAULT_PROXY_BATCH_SIZE + } else { + DEFAULT_BATCH_SIZE + }) + .max(1) +} + +/// Serialized length of one `{"purl":…}` component of the batch body, +/// exactly as `serde_json` writes it (quotes and escapes included). +fn batch_component_bytes(purl: &str) -> usize { + // `{"purl":` + the JSON string + `}`. + 8 + serde_json::to_string(purl).map_or(purl.len() + 2, |s| s.len()) + 1 +} + +/// Split `purls` into consecutive batch chunks of at most `batch_size` +/// purls whose request body (`{"components":[{"purl":…},…]}`) stays within +/// `max_body_bytes`. A chunk closes at `batch_size` purls or when the next +/// purl would push its body past the cap, so the boundaries depend only on +/// the purls, their order and the two limits. A single purl too long for +/// the cap on its own still goes, alone (the server judges it); nothing is +/// ever dropped or reordered. With a cap no chunk reaches, this is exactly +/// `purls.chunks(batch_size)`. +fn batch_chunks(purls: &[String], batch_size: usize, max_body_bytes: usize) -> Vec<&[String]> { + // `{"components":[` + `]}`. + const ENVELOPE: usize = 15 + 2; + let batch_size = batch_size.max(1); + let mut chunks = Vec::with_capacity(purls.len().div_ceil(batch_size)); + let mut start = 0usize; + let mut body = ENVELOPE; + for (i, purl) in purls.iter().enumerate() { + let count = i - start; + let item = batch_component_bytes(purl) + usize::from(count > 0); + if count > 0 && (count == batch_size || body + item > max_body_bytes) { + chunks.push(&purls[start..i]); + start = i; + body = ENVELOPE + batch_component_bytes(purl); + } else { + body += item; + } + } + if start < purls.len() { + chunks.push(&purls[start..]); + } + chunks +} /// The three patch-application modes `scan` can drive, selectable via /// `--mode` (the documented spelling). Each variant is equivalent to one @@ -211,9 +287,11 @@ pub struct ScanArgs { #[command(flatten)] pub common: GlobalArgs, - /// Number of packages to query per API request. - #[arg(long = "batch-size", env = "SOCKET_BATCH_SIZE", default_value_t = DEFAULT_BATCH_SIZE)] - pub batch_size: usize, + /// Number of packages to query per API request [default: 500 on the + /// authenticated API, 100 on the public proxy]. A batch whose request + /// body would exceed 256 KiB is split into smaller consecutive batches + #[arg(long = "batch-size", env = "SOCKET_BATCH_SIZE")] + pub batch_size: Option, /// Deprecated spelling of `--mode agent`. With `--json`, download and /// apply the selected patches without prompting (without a mode, @@ -441,7 +519,10 @@ async fn embed_vex_human( /// human-only output knobs of [`fetch_patch_details`] (the JSON callers pass /// `false, false`; the hosted human arm passes the same values as the agent /// human arm, so the two print the same progress counter and per-package -/// warnings). +/// warnings). `json_warnings` is the JSON callers' envelope: a partial +/// failure (some queries failed, not all) adds one +/// [`PATCH_DETAILS_FAILED`] run-level warning per failed package to it. +#[allow(clippy::too_many_arguments)] async fn discover_selected( api_client: &socket_patch_core::api::client::ApiClient, packages: &[BatchPackagePatches], @@ -449,9 +530,15 @@ async fn discover_selected( common: &GlobalArgs, show_progress: bool, warn: bool, + telemetry: &mut PendingTelemetry, + json_warnings: Option<&mut serde_json::Value>, ) -> Result, (i32, String)> { let (all_search_results, failures) = fetch_patch_details(api_client, packages, show_progress, warn).await; + // The scan event's send overlapped the detail fetches; every caller's + // next output (the error line below, a `--json` envelope, a prompt) + // must find it delivered. + telemetry.flush().await; let error_count = failures.len(); if error_count > 0 && error_count == packages.len() { let err = failures @@ -462,6 +549,18 @@ async fn discover_selected( eprintln!("Error: {message}"); return Err((1, message)); } + // Some queries failed, some succeeded: a `--json` run has no stderr + // warning (`warn` is human-only), so each failed package becomes a + // run-level `warnings[]` entry — never a silent drop from the envelope. + if let Some(result) = json_warnings { + for (purl, e) in &failures { + push_scan_json_warning( + result, + PATCH_DETAILS_FAILED, + &format!("could not fetch details for {purl}: {e}"), + ); + } + } if all_search_results.is_empty() { return Ok(Vec::new()); } @@ -530,13 +629,31 @@ async fn fetch_patch_details( // `show_progress` off reads as `--json` to the status line: never // drawn. On, it is live only on a terminal; it never prints a result. let mut status = StatusLine::stderr(!show_progress, false); - for (i, pkg) in packages.iter().enumerate() { + // The queries run concurrently but come back in `packages` order, so + // `results` and `failures` fold exactly as the serial loop's did. The + // counter names the next result awaited, and each query's `--debug` + // lines are held back and printed at its fold, where the serial loop + // would have made the request. + let mut responses = std::pin::pin!(ordered_concurrent( + packages, + api_concurrency_for(api_client.uses_public_proxy(), packages.len()), + |pkg| async move { + ( + pkg, + hold_back_debug(api_client.search_patches_by_package(&pkg.purl)).await, + ) + }, + )); + for i in 0..packages.len() { status.set(format!( "Fetching patch details... ({}/{})", i + 1, packages.len() )); - match api_client.search_patches_by_package(&pkg.purl).await { + let Some((pkg, response)) = responses.next().await else { + break; + }; + match response.release() { Ok(response) => results.extend(response.patches), Err(e) => failures.push((pkg.purl.clone(), e.to_string())), } @@ -634,6 +751,21 @@ fn partition_agent_selection( /// shape (`json`/`silent`) and `save_only` differ per flow; vendored mode /// never persists blobs (its records stay in memory and the vendor step /// consumes the staged sources). +/// The ecosystems a scan crawls (`None`: every one). `--ecosystems` +/// narrows everything the run counts, queries and shows to the named +/// ecosystems, so without a GC the other crawlers' output would only be +/// filtered away: they are not run at all. A GC run (`--prune` / `--sync`) +/// still crawls everything — the GC judges every manifest entry against the +/// FULL installed set (see `scanned_purls` in `run_scan`), and a skipped +/// ecosystem would read as uninstalled. +fn crawl_scope(prune: bool, ecosystems: Option<&[String]>) -> Option<&[String]> { + if prune { + None + } else { + ecosystems + } +} + fn download_params(args: &ScanArgs, save_only: bool, json: bool, silent: bool) -> DownloadParams { DownloadParams { cwd: args.common.cwd.clone(), @@ -1226,6 +1358,21 @@ pub(super) const HOSTED_WIRING_RETAINED: &str = "hosted_wiring_retained"; /// package(s) did NOT convert to agent mode. pub(super) const VENDORED_OWNERSHIP_RETAINED: &str = "vendored_ownership_retained"; +/// Run-level `--json` warning: one API batch query failed (after the +/// client's bounded 429 / 503 retry) while others succeeded, so the +/// packages in that batch were not checked for patches. The detail is the +/// human `Warning: API batch of failed: ` line without +/// its prefix. (Every batch failing is the all-batches-failed error +/// envelope instead.) +pub(super) const API_BATCH_FAILED: &str = "api_batch_failed"; + +/// Run-level `--json` warning: one package's patch-detail query failed +/// (after the client's bounded 429 / 503 retry) while others succeeded, so +/// its patch was left out of the selection. The detail is the human +/// `Warning: could not fetch details for : ` line without its +/// prefix. (Every query failing is the discovery error envelope instead.) +pub(super) const PATCH_DETAILS_FAILED: &str = "patch_details_failed"; + /// The scanned purls whose HOSTED redirect wiring is still live: the /// redirect ledger records the purl AND lockfile discovery proves the /// current lockfile still routes it to that hosted patch — core @@ -1434,7 +1581,19 @@ fn print_zero_error_envelope(err: &str, paths: &[String]) { print_json(&result); } -pub async fn run(mut args: ScanArgs) -> i32 { +pub async fn run(args: ScanArgs) -> i32 { + // Scan's telemetry sends run off the critical path: each is spawned + // where its event fires and flushed before the first stdout write that + // follows it (so a closed pipe's SIGPIPE, or a Ctrl-C at a prompt, still + // finds it delivered, as with an inline send). The flush here is the + // backstop that keeps every event ahead of the process exit. + let mut telemetry = PendingTelemetry::new(); + let code = Box::pin(run_scan(args, &mut telemetry)).await; + telemetry.flush().await; + code +} + +async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { apply_env_toggles(&args.common); // Fold the legacy mode booleans into `args.mode` before anything reads @@ -1499,13 +1658,6 @@ pub async fn run(mut args: ScanArgs) -> i32 { eprintln!("Warning ({REDIRECT_PRUNE_IGNORED}): {REDIRECT_PRUNE_IGNORED_DETAIL}"); } - // A zero batch size would panic the API-query loop below: both - // `all_purls.len().div_ceil(batch_size)` and `all_purls.chunks(batch_size)` - // abort the process on a divisor/chunk-size of 0. `--batch-size 0` - // (or `SOCKET_BATCH_SIZE=0`) is otherwise unvalidated, so clamp to a - // floor of 1 — degrade to one-package batches rather than crash. - let batch_size = args.batch_size.max(1); - // Resolved up-front (rather than at the GC site) because the embedded // `--vex` side-effect reads the manifest at several terminal returns, // including the early "no packages" exit before the GC block. @@ -1515,6 +1667,12 @@ pub async fn run(mut args: ScanArgs) -> i32 { let overrides = args.common.api_client_overrides(); let (mut api_client, mut use_public_proxy) = get_api_client_with_overrides(overrides.clone()).await; + // Sized for the endpoint the run starts on (see `effective_batch_size`; + // a zero `--batch-size` is floored to 1 there rather than crash the + // chunking below). A mid-run downgrade to the proxy keeps these chunk + // boundaries: the failed chunk is retried as-is, and every chunk is + // within the proxy's body cap by construction (`BATCH_BODY_BYTE_CAP`). + let batch_size = effective_batch_size(args.batch_size, use_public_proxy); let telemetry_token = api_client.api_token().cloned(); let telemetry_org = api_client.org_slug().cloned(); // Tracks whether scan was downgraded from the authenticated @@ -1544,15 +1702,31 @@ pub async fn run(mut args: ScanArgs) -> i32 { let mut status = StatusLine::stderr(args.common.json, args.common.silent); status.set(format!("Scanning {scan_target}...")); - // Crawl packages - let (mut all_crawled, mut eco_counts, skipped_bundle_config_path) = - crawl_all_ecosystems(&crawler_options).await; + // Which ecosystems to crawl (see `crawl_scope`). + let crawl_scope = crawl_scope(prune, args.common.ecosystems.as_deref()); + + // Crawl packages. Vendored mode keeps the npm half: its engine + // resolves the same untouched tree and reuses this crawl instead of + // walking `node_modules` again (no snapshot when npm was not crawled). + // Hosted mode keeps it only for an embedded `--vex` (skipped under + // `--dry-run`), whose installed-copy lookup reuses the npm roots. No + // other run reads the snapshot, so no other run pays for copying it. + let keep_npm = vendor || (hosted && args.vex.vex.is_some() && !args.common.dry_run); + let (mut all_crawled, mut eco_counts, skipped_bundle_config_path, npm_crawl) = if keep_npm { + crawl_ecosystems_with_npm(&crawler_options, crawl_scope).await + } else { + let (packages, counts, skipped) = crawl_ecosystems(&crawler_options, crawl_scope).await; + (packages, counts, skipped, None) + }; // Lockfile supplement: dependencies the project's lockfile resolves // that have NO installed copy (fresh clone, partial install). They join // discovery — counts, API lookup, table, the prune "scanned" set — and // are flagged "not yet installed" everywhere a user could act on them. - let lockfile_only = lockfile_supplement(&args.common, &all_crawled).await; + // Scoped to the crawled ecosystems: a skipped ecosystem's lockfile + // entries have no crawl to be measured against, and `--ecosystems` + // filters them out of this run anyway. + let lockfile_only = lockfile_supplement(&args.common, &all_crawled, crawl_scope).await; // Discovery diagnoses unsupported installation layouts and malformed // binary Bun locks. Preserve these on empty scans too: an unreadable // graph is not evidence that a fresh checkout has no dependencies. @@ -1619,7 +1793,8 @@ pub async fn run(mut args: ScanArgs) -> i32 { // and delete it (plus its blobs) — silent cross-ecosystem data loss. // Lockfile-only purls are deliberately included: a dependency the // lockfile still resolves must not be pruned just because node_modules - // is wiped or partially installed. + // is wiped or partially installed. (This is why a GC run never scopes + // the crawl — see `crawl_scope`; a scoped run reads this set nowhere.) let scanned_purls: HashSet = all_crawled.iter().map(|p| p.purl.clone()).collect(); // Vendor-ledger purl keys (from the single load above), shared by the @@ -1705,7 +1880,8 @@ pub async fn run(mut args: ScanArgs) -> i32 { } } // Telemetry: empty-scan still counts as a successful scan. - track_patch_scanned( + spawn_patch_scanned( + telemetry, 0, 0, 0, @@ -1718,8 +1894,9 @@ pub async fn run(mut args: ScanArgs) -> i32 { false, telemetry_token.as_deref(), telemetry_org.as_deref(), - ) - .await; + ); + // The result prints right away: nothing to overlap the send with. + telemetry.flush().await; if args.common.json { // When the crawler finds nothing, GC is intentionally skipped // — pruning every manifest entry on the assumption that the @@ -1850,67 +2027,124 @@ pub async fn run(mut args: ScanArgs) -> i32 { // Query API in batches let mut all_packages_with_patches: Vec = Vec::new(); let mut can_access_paid_patches = false; - let total_batches = all_purls.len().div_ceil(batch_size); + let chunks: Vec<&[String]> = batch_chunks(&all_purls, batch_size, BATCH_BODY_BYTE_CAP); + let total_batches = chunks.len(); let mut batch_error_count = 0usize; let mut last_batch_error: Option = None; - - for (batch_idx, chunk) in all_purls.chunks(batch_size).enumerate() { - status.set(format!( - "Querying API for patches... (batch {}/{total_batches})", - batch_idx + 1 - )); - - let mut result = api_client.search_patches_batch(chunk).await; - - // Fallback: a 401/403 against the authenticated endpoint can - // mean a stale/revoked token. Retry against the public proxy - // (free patches only) once, then continue the rest of the - // loop with the downgraded client. Only triggers on the - // first authenticated batch; subsequent iterations are - // already on the proxy. - if !use_public_proxy { - if let Err(ref e) = result { - if is_fallback_candidate(e) { - // Errors-only under --silent; --json keeps it on stderr - // (the envelope has no slot for a mid-run downgrade). - if !args.common.silent { - status.println(format!( - "Warning: authenticated API returned {e}; \ - falling back to public patch API proxy (free patches only)." - )); - } - api_client = build_proxy_fallback_client(&overrides); - use_public_proxy = true; - fallback_to_proxy = true; - result = api_client.search_patches_batch(chunk).await; + // `--json` twin of the per-batch stderr warnings: `(batch, error)` in + // chunk order, surfaced as run-level `warnings[]` when some batch + // succeeded (all failing is the error envelope below). + let mut failed_batches: Vec<(usize, String)> = Vec::new(); + + // Fold one batch outcome, in chunk order. Every caller below consumes + // outcomes strictly by chunk index, so the per-batch warnings, + // `batch_error_count` and `last_batch_error` come out exactly as the + // serial loop produced them. + let mut fold = |batch_idx: usize, + result: Result, + status: &mut StatusLine<_>| match result { + Ok(response) => { + if response.can_access_paid_patches { + can_access_paid_patches = true; + } + for pkg in response.packages { + if !pkg.patches.is_empty() { + all_packages_with_patches.push(pkg); } } } + Err(e) => { + batch_error_count += 1; + last_batch_error = Some(e.to_string()); + failed_batches.push((batch_idx + 1, e.to_string())); + // Not fatal by itself: the scan goes on with the other + // batches. A one-batch scan says it once, below. + if !args.common.json && !args.common.silent && total_batches > 1 { + status.println(render::batch_failed_warning( + batch_idx + 1, + total_batches, + &e.to_string(), + )); + } + } + }; - match result { - Ok(response) => { - if response.can_access_paid_patches { - can_access_paid_patches = true; - } - for pkg in response.packages { - if !pkg.patches.is_empty() { - all_packages_with_patches.push(pkg); + // The batches run concurrently (at most `api_concurrency` in flight) + // but are CONSUMED in chunk order, one window at a time: + // + // - On the authenticated client the first chunk goes alone, so a stale + // token costs the authenticated API one request before the + // downgrade, as it always did. Already on the proxy there is no + // downgrade left to cap (the fallback arm below is authenticated- + // only), so a token-less run opens the full window at chunk 0. + // - Fallback: a 401/403 against the authenticated endpoint can mean a + // stale/revoked token. At the first consumed chunk `k` whose error is + // a fallback candidate (any index, not just the first), the window is + // dropped — in-flight requests for chunks past `k` are cancelled and + // any responses already received for them are discarded, never + // folded — then chunk `k` is retried against the public proxy (free + // patches only) and the rest continues on the downgraded client. + // That is exactly the serial loop's sequence; on the proxy no further + // fallback applies. A token revoked mid-run does cost the auth + // endpoint the requests the window had already dispatched past `k` + // (up to the in-flight cap, instead of one). Their answers are + // discarded, and so are their `--debug` lines: each chunk's are held + // back until it is folded, so a chunk the window drops announces + // nothing the serial loop would not have announced. + let mut next = 0usize; + 'windows: while next < total_batches { + let end = if next == 0 && !use_public_proxy { + 1 + } else { + total_batches + }; + let mut fallback_error = None; + { + let client = &api_client; + let mut results = std::pin::pin!(ordered_concurrent( + &chunks[next..end], + api_concurrency_for(use_public_proxy, end - next), + |chunk| hold_back_debug(client.search_patches_batch(chunk)), + )); + while next < end { + status.set(format!( + "Querying API for patches... (batch {}/{total_batches})", + next + 1 + )); + // `ordered_concurrent` yields exactly one item per chunk, + // so the window never runs dry early. Should a future + // variant make it, stop the scan here: re-entering the + // outer loop with `next` unchanged would rebuild the very + // same window and re-POST every chunk in it, forever. + let Some(result) = results.next().await else { + debug_assert!(false, "batch window yields one result per chunk"); + break 'windows; + }; + match result.release() { + Err(e) if !use_public_proxy && is_fallback_candidate(&e) => { + fallback_error = Some(e); + break; } + result => fold(next, result, &mut status), } + next += 1; } - Err(e) => { - batch_error_count += 1; - last_batch_error = Some(e.to_string()); - // Not fatal by itself: the scan goes on with the other - // batches. A one-batch scan says it once, below. - if !args.common.json && !args.common.silent && total_batches > 1 { - status.println(render::batch_failed_warning( - batch_idx + 1, - total_batches, - &e.to_string(), - )); - } + } + if let Some(e) = fallback_error { + // Errors-only under --silent; --json keeps it on stderr + // (the envelope has no slot for a mid-run downgrade). + if !args.common.silent { + status.println(format!( + "Warning: authenticated API returned {e}; \ + falling back to public patch API proxy (free patches only)." + )); } + api_client = build_proxy_fallback_client(&overrides); + use_public_proxy = true; + fallback_to_proxy = true; + let result = api_client.search_patches_batch(chunks[next]).await; + fold(next, result, &mut status); + next += 1; } } @@ -1927,13 +2161,15 @@ pub async fn run(mut args: ScanArgs) -> i32 { if total_batches > 0 && batch_error_count == total_batches { status.finish(); let err = last_batch_error.unwrap_or_else(|| "all batches failed".to_string()); - track_patch_scan_failed( + spawn_patch_scan_failed( + telemetry, &err, fallback_to_proxy, telemetry_token.as_deref(), telemetry_org.as_deref(), - ) - .await; + ); + // The failure prints right away: nothing to overlap the send with. + telemetry.flush().await; // A scan in which *every* batch failed produced no trustworthy // patch data. Surfacing `status: "success"` / exit 0 here would be @@ -1997,7 +2233,8 @@ pub async fn run(mut args: ScanArgs) -> i32 { // per-tier counts. `fallback_to_proxy` is `true` iff the batch // loop downgraded from the authenticated endpoint to the public // proxy after a 401/403. - track_patch_scanned( + spawn_patch_scanned( + telemetry, package_count, free_patches, paid_patches, @@ -2010,8 +2247,7 @@ pub async fn run(mut args: ScanArgs) -> i32 { fallback_to_proxy, telemetry_token.as_deref(), telemetry_org.as_deref(), - ) - .await; + ); // Read existing manifest once for update detection. Used by both the // JSON-mode emission (always includes an `updates` array) and the @@ -2042,11 +2278,20 @@ pub async fn run(mut args: ScanArgs) -> i32 { Err(corrupt) => (None, Some(corrupt.to_string())), } } else { - ( - crate::commands::load_redirect_state_lenient(&args.common.cwd, args.common.silent) - .await, - None, - ) + // `load_redirect_state_lenient`, with the scan event's send flushed + // before its warning: that line can be this run's first write since + // the event fired, and a closed stderr's SIGPIPE must find the event + // delivered, as the inline send it replaced was. + match socket_patch_core::patch::redirect::load_redirect_state(&args.common.cwd).await { + Ok(state) => (state, None), + Err(corrupt) => { + if !args.common.silent { + telemetry.flush().await; + eprintln!("Warning: {corrupt}"); + } + (None, None) + } + } }; let update_manifest = merge_ledger_records_for_updates( existing_manifest.as_ref(), @@ -2088,6 +2333,14 @@ pub async fn run(mut args: ScanArgs) -> i32 { if !layout_refusals.is_empty() { result["warnings"] = layout_refusal_json(&layout_refusals); } + // A batch that failed while others succeeded left its packages + // unchecked; the human run warns on stderr, the envelope carries + // the same line per batch (additive, status and exit unchanged). + for (batch, err) in &failed_batches { + let line = render::batch_failed_warning(*batch, total_batches, err); + let detail = line.strip_prefix("Warning: ").unwrap_or(&line); + push_scan_json_warning(&mut result, API_BATCH_FAILED, detail); + } // Flag lockfile-only packages so JSON consumers can tell "patch // available but not installed" from the installed case. Additive // field; absent means installed. Matching bridges the API's @@ -2119,6 +2372,8 @@ pub async fn run(mut args: ScanArgs) -> i32 { &all_packages_with_patches, can_access_paid_patches, Some(result), + telemetry, + npm_crawl.as_ref(), ) .await; } @@ -2163,6 +2418,8 @@ pub async fn run(mut args: ScanArgs) -> i32 { &args.common, false, false, + telemetry, + Some(&mut result), ) .await { @@ -2305,10 +2562,17 @@ pub async fn run(mut args: ScanArgs) -> i32 { prune, telemetry_token.as_deref(), telemetry_org.as_deref(), + telemetry, + npm_crawl.as_ref(), ) .await; } + // The GC and the VEX build below can write to stderr; the report- + // only arm has not flushed the scan event yet (the `--apply` arm + // did, in `discover_selected`). + telemetry.flush().await; + // --- GC (post-apply, or standalone --prune GC-sweep) ------------- if prune { result["gc"] = gc_json( @@ -2334,6 +2598,9 @@ pub async fn run(mut args: ScanArgs) -> i32 { return final_code; } + // Every human exit below prints first; the scan event goes out before. + telemetry.flush().await; + let use_color = ui::stdout_color(); let verbose = args.common.verbose; let silent = args.common.silent; @@ -2549,6 +2816,8 @@ pub async fn run(mut args: ScanArgs) -> i32 { &args.common, human, !silent, + telemetry, + None, ) .await { @@ -2563,7 +2832,12 @@ pub async fn run(mut args: ScanArgs) -> i32 { // so only a wet run with work confirms. `--mode hosted` is explicit // intent, so a non-TTY run auto-proceeds like every other mode — // only the mode-less scan below is report-only. - if !selected.is_empty() && !args.common.dry_run { + let prompts = !selected.is_empty() && !args.common.dry_run; + // Whether that prompt waits on a person: the tree may change while it + // does, so the embedded VEX then walks node_modules afresh instead of + // reusing the pre-prompt crawl (as the vendor path below does). + let prompt_waits = prompts && ui::confirm_waits(&args.common); + if prompts { let prompt = render::hosted_confirm_prompt(selected.len()); // The prompt (or the non-TTY note) opens its own paragraph // under the table's Summary, on the prompt's stream. @@ -2592,6 +2866,7 @@ pub async fn run(mut args: ScanArgs) -> i32 { &api_client, &pairs, None, + npm_crawl.as_ref().filter(|_| !prompt_waits), ) .await; } @@ -2825,6 +3100,10 @@ pub async fn run(mut args: ScanArgs) -> i32 { HashMap::new() }; + // Whether the prompt below waits on a person: the tree may change while + // it does, so the vendor step then crawls afresh instead of reusing the + // pre-prompt crawl (`--yes` / `--json` / non-terminal answer at once). + let prompt_waits = ui::confirm_waits(&args.common); if !ui::confirm(&render::confirm_prompt(plan), true, &args.common) { if !silent { println!(); @@ -2864,6 +3143,7 @@ pub async fn run(mut args: ScanArgs) -> i32 { prune, telemetry_token.as_deref(), telemetry_org.as_deref(), + npm_crawl.as_ref().filter(|_| !prompt_waits), ) .await } else { @@ -2951,6 +3231,136 @@ mod tests { m } + /// The request body `search_patches_batch` sends for `chunk`, as + /// `serde_json` writes it (the client's `BatchSearchBody`). + fn batch_body(chunk: &[String]) -> String { + let components: Vec = chunk + .iter() + .map(|p| serde_json::json!({ "purl": p })) + .collect(); + serde_json::json!({ "components": components }).to_string() + } + + fn purls(n: usize, len: usize) -> Vec { + (0..n) + .map(|i| { + let head = format!("pkg:npm/p{i}-"); + let pad = len.saturating_sub(head.len() + 6); + format!("{head}{}@1.0.0", "x".repeat(pad)) + }) + .collect() + } + + /// Unset, the batch size follows the endpoint: the server's 500-purl + /// maximum on the authenticated API, 100 on the public proxy. A given + /// size wins on both, and 0 is floored to 1. + #[test] + fn batch_size_defaults_per_endpoint_and_honors_an_explicit_value() { + assert_eq!(effective_batch_size(None, false), 500); + assert_eq!(effective_batch_size(None, true), 100); + assert_eq!(effective_batch_size(Some(7), false), 7); + assert_eq!(effective_batch_size(Some(7), true), 7); + assert_eq!(effective_batch_size(Some(0), false), 1); + assert_eq!(effective_batch_size(Some(0), true), 1); + } + + /// The byte arithmetic matches `serde_json`'s output exactly, escapes + /// and non-ASCII included, so the cap is judged on the real body. + #[test] + fn batch_component_bytes_match_the_serialized_body() { + for purl in [ + "pkg:npm/left-pad@1.3.0", + "pkg:npm/%40scope/name@1.0.0", + "pkg:pypi/we\"ird@1.0?x=\\y", + "pkg:cargo/caf\u{e9}@0.1.0", + "pkg:npm/ctl\u{1}@1.0.0", + ] { + let one = vec![purl.to_string()]; + assert_eq!( + 17 + batch_component_bytes(purl), + batch_body(&one).len(), + "{purl}" + ); + } + let many = purls(9, 40); + let sum: usize = many.iter().map(|p| batch_component_bytes(p)).sum(); + assert_eq!(17 + sum + (many.len() - 1), batch_body(&many).len()); + } + + /// With a cap no chunk reaches, the chunks are exactly + /// `purls.chunks(batch_size)`: same boundaries, same order. + #[test] + fn batch_chunks_without_cap_pressure_match_plain_chunking() { + for n in [0usize, 1, 99, 100, 101, 499, 500, 501, 1000, 1234] { + let list = purls(n, 30); + for size in [1usize, 3, 100, 500] { + let want: Vec<&[String]> = list.chunks(size).collect(); + assert_eq!( + batch_chunks(&list, size, BATCH_BODY_BYTE_CAP), + want, + "n={n} size={size}" + ); + } + } + } + + /// An oversize chunk is split greedily at the byte cap: every body fits, + /// every chunk is maximal (its successor's first purl would not have + /// fitted), nothing is dropped or reordered, and the split is the same + /// on every call. + #[test] + fn batch_chunks_split_an_oversize_chunk_at_the_byte_cap() { + let list = purls(1400, 220); + let chunks = batch_chunks(&list, 5000, BATCH_BODY_BYTE_CAP); + assert!(chunks.len() > 1, "1400 x 220-byte purls exceed 256 KiB"); + for (i, chunk) in chunks.iter().enumerate() { + assert!(batch_body(chunk).len() <= BATCH_BODY_BYTE_CAP, "chunk {i}"); + if let Some(next) = chunks.get(i + 1) { + let mut grown = chunk.to_vec(); + grown.push(next[0].clone()); + assert!(batch_body(&grown).len() > BATCH_BODY_BYTE_CAP, "chunk {i}"); + } + } + assert_eq!(chunks.concat(), list); + assert_eq!(chunks, batch_chunks(&list, 5000, BATCH_BODY_BYTE_CAP)); + + // The count limit still applies inside the byte limit, and a body + // exactly at the cap is kept whole. + let small = batch_chunks(&list, 500, BATCH_BODY_BYTE_CAP); + assert!(small.iter().all(|c| c.len() <= 500)); + let exact = batch_body(&list[..10]).len(); + assert_eq!(batch_chunks(&list[..11], 500, exact)[0].len(), 10); + } + + /// A purl too long for the cap on its own still goes, alone, between + /// its neighbours' chunks. + #[test] + fn batch_chunks_send_an_oversize_purl_alone() { + let mut list = purls(4, 30); + list.insert(2, format!("pkg:npm/{}@1.0.0", "y".repeat(400))); + let chunks = batch_chunks(&list, 100, 200); + assert_eq!(chunks.concat(), list); + let alone: Vec<&[String]> = chunks + .iter() + .copied() + .filter(|c| c.contains(&list[2])) + .collect(); + assert_eq!(alone, vec![&list[2..3]]); + assert!(chunks.iter().all(|c| !c.is_empty())); + } + + /// MVN-4: only a non-GC `--ecosystems` run narrows the crawl. A GC run + /// crawls every ecosystem whatever `--ecosystems` says (its prune reads + /// the full installed set), and no `--ecosystems` crawls every one. + #[test] + fn crawl_scope_narrows_only_a_non_gc_filtered_run() { + let npm = vec!["npm".to_string()]; + assert_eq!(crawl_scope(false, Some(&npm)), Some(&npm[..])); + assert_eq!(crawl_scope(true, Some(&npm)), None); + assert_eq!(crawl_scope(false, None), None); + assert_eq!(crawl_scope(true, None), None); + } + // ---- cross-mode ledger takeover (hosted ⇄ vendored) -------------------- // Switching a project's patch mode rewires the lockfile to the new mode // but leaves the OLD mode's ledger on disk asserting stale wiring. These diff --git a/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs b/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs index 662bc2ee1..8f640b0b7 100644 --- a/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs +++ b/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs @@ -5,7 +5,7 @@ //! out of `run`'s poll frame (Windows 1 MiB main-thread stack). //! //! Vendored mode is manifest-free: the download phase fetches the patch -//! records in memory ([`download_patch_records_with`]), the vendor engine +//! records in memory ([`download_patch_records_reusing`]), the vendor engine //! embeds each record in its ledger entry (`detached: true`), and //! `.socket/manifest.json` is never written — a project vendored by an //! older, manifest-mode CLI is migrated on its next vendored run (see @@ -21,7 +21,7 @@ use socket_patch_core::api::client::ApiClient; use socket_patch_core::api::types::{BatchPackagePatches, PatchResponse, PatchSearchResult}; use socket_patch_core::manifest::operations::{read_manifest, write_manifest}; use socket_patch_core::manifest::schema::{PatchManifest, PatchRecord}; -use socket_patch_core::telemetry::track_patch_vendor_failed; +use socket_patch_core::telemetry::{track_patch_vendor_failed, PendingTelemetry}; use socket_patch_core::utils::purl::strip_purl_qualifiers; use socket_patch_core::vendor::{load_state, lookup_entry, save_state, VendorState}; use std::collections::{HashMap, HashSet}; @@ -32,11 +32,12 @@ use std::time::Duration; use crate::args::GlobalArgs; use crate::commands::bun_preflight::bun_vendor_preflight_with_ledger; use crate::commands::fetch_stage::{stage_vendor_sources_in_memory, MemStageOutcome}; -use crate::commands::get::{download_patch_records_with, DetachedDownload, DownloadParams}; +use crate::commands::get::{download_patch_records_reusing, DetachedDownload, DownloadParams}; use crate::commands::lock_cli::lock_failure; use crate::commands::vendor::{ - note_classic_migration_risk, track_outcomes_for_vendor, vendor_records, + note_classic_migration_risk, track_outcomes_for_vendor, vendor_records_reusing, }; +use crate::ecosystem_dispatch::NpmCrawlSnapshot; use crate::json_envelope::{Command as EnvelopeCommand, Envelope}; use crate::ui::{plural, print_json}; @@ -145,7 +146,7 @@ pub(crate) fn print_dry_run_refusals(preview: &serde_json::Value) { /// The vendor step shared by `scan --vendor`'s JSON and interactive arms /// (and, through [`boxed_scan_vendor_step`], `get --mode vendored`): /// acquire the apply lock, stage the in-memory `records` (from -/// [`download_patch_records_with`], whose blob `seed` spares the stager a +/// [`download_patch_records_reusing`], whose blob `seed` spares the stager a /// second view fetch), drive [`vendor_records`] detached — every ledger /// entry embeds its record; `.socket/manifest.json` is never a record /// source — over the run's `client`, then migrate any legacy manifest @@ -178,6 +179,9 @@ async fn run_scan_vendor_step( // all (the step is a silent no-op then). `get --mode vendored` wants // it; scan's interactive arm prints its own closing line instead. report_empty: bool, + // The npm half of scan's crawl, for the engine to reuse instead of + // walking the untouched tree again (see `vendor_records_reusing`). + prior: Option<&NpmCrawlSnapshot>, ) -> VendorStepResult { let mut env = Envelope::new(EnvelopeCommand::Vendor); env.dry_run = common.dry_run; @@ -215,6 +219,7 @@ async fn run_scan_vendor_step( client, use_public_proxy, &mut env, + prior, ) .await { @@ -242,6 +247,7 @@ async fn run_scan_vendor_step( /// embeds its record). The caller holds the apply lock. `Err` is the /// `no_local_source` fold (staging could not obtain the patch content — /// offline, or the view fetch failed). +#[allow(clippy::too_many_arguments)] async fn stage_and_vendor( common: &GlobalArgs, socket_dir: &Path, @@ -250,6 +256,7 @@ async fn stage_and_vendor( client: ApiClient, use_public_proxy: bool, env: &mut Envelope, + prior: Option<&NpmCrawlSnapshot>, ) -> Result { // Loaded ONCE under the lock: the staging harvest reads it here, then // the engine takes it over for its persists. An unreadable ledger @@ -289,6 +296,7 @@ async fn stage_and_vendor( Some(&service), ledger, env, + prior, ) .await) } @@ -457,6 +465,11 @@ async fn run_vendor_json_path( prune: bool, telemetry_token: Option<&str>, telemetry_org: Option<&str>, + // Scan's pending telemetry, flushed by `discover_selected` before + // anything below writes to stdout. + telemetry: &mut PendingTelemetry, + // The npm half of scan's crawl, for the vendor engine to reuse. + prior: Option<&NpmCrawlSnapshot>, ) -> i32 { // Same discovery as `--apply`. Vendored purls are NOT filtered here — // re-vendoring a stale uuid is the point of the flag (same-uuid re-runs @@ -468,6 +481,8 @@ async fn run_vendor_json_path( &args.common, false, false, + telemetry, + Some(&mut *result), ) .await { @@ -513,18 +528,19 @@ async fn run_vendor_json_path( args, /*save_only=*/ true, /*json=*/ true, /*silent=*/ true, ); let (dl_code, dl_json, records, blobs) = - boxed_download_patch_records(&selected, ¶ms, api_client, HashMap::new()).await; + boxed_download_patch_records(&selected, ¶ms, api_client, HashMap::new(), prior).await; let mut has_errors = dl_code != 0; result["download"] = dl_json; // 2) The vendor engine, under the same lock as apply/vendor (a no-op // that creates nothing when there is nothing to vendor). - let vendor_code = match boxed_scan_vendor_step( + let vendor_code = match boxed_scan_vendor_step_reusing( &args.common, records, blobs, api_client.clone(), use_public_proxy, + prior, ) .await { @@ -615,6 +631,9 @@ async fn run_vendor_interactive_path( prune: bool, telemetry_token: Option<&str>, telemetry_org: Option<&str>, + // The npm half of scan's crawl, for the vendor engine to reuse — + // `None` when the tree may have changed since (an answered prompt). + prior: Option<&NpmCrawlSnapshot>, ) -> i32 { // The download phase is quiet about its own header in vendored mode // (only the manifest-mode download prints it), so this arm does. @@ -630,7 +649,7 @@ async fn run_vendor_interactive_path( ); } let (dl_code, dl_json, records, blobs) = - boxed_download_patch_records(selected, params, api_client, prefetched).await; + boxed_download_patch_records(selected, params, api_client, prefetched, prior).await; let mut has_errors = dl_code != 0; // Patches the download phase could not get (it reported each one). let download_failed = dl_json["failed"].as_u64().unwrap_or(0); @@ -643,6 +662,7 @@ async fn run_vendor_interactive_path( blobs, api_client.clone(), use_public_proxy, + prior, ) .await { @@ -818,6 +838,8 @@ pub(super) fn boxed_vendor_json_path<'a>( prune: bool, telemetry_token: Option<&'a str>, telemetry_org: Option<&'a str>, + telemetry: &'a mut PendingTelemetry, + prior: Option<&'a NpmCrawlSnapshot>, ) -> std::pin::Pin + 'a>> { Box::pin(run_vendor_json_path( args, @@ -833,6 +855,8 @@ pub(super) fn boxed_vendor_json_path<'a>( prune, telemetry_token, telemetry_org, + telemetry, + prior, )) } @@ -853,6 +877,7 @@ pub(super) fn boxed_vendor_interactive_path<'a>( prune: bool, telemetry_token: Option<&'a str>, telemetry_org: Option<&'a str>, + prior: Option<&'a NpmCrawlSnapshot>, ) -> std::pin::Pin + 'a>> { Box::pin(run_vendor_interactive_path( args, @@ -868,6 +893,7 @@ pub(super) fn boxed_vendor_interactive_path<'a>( prune, telemetry_token, telemetry_org, + prior, )) } @@ -892,6 +918,28 @@ pub(crate) fn boxed_scan_vendor_step<'a>( client, use_public_proxy, true, + None, + )) +} + +/// [`boxed_scan_vendor_step`] handing the engine scan's npm crawl to reuse +/// (see `vendor_records_reusing`). +fn boxed_scan_vendor_step_reusing<'a>( + common: &'a GlobalArgs, + records: HashMap, + seed: HashMap>, + client: ApiClient, + use_public_proxy: bool, + prior: Option<&'a NpmCrawlSnapshot>, +) -> std::pin::Pin + 'a>> { + Box::pin(run_scan_vendor_step( + common, + records, + seed, + client, + use_public_proxy, + true, + prior, )) } @@ -903,6 +951,7 @@ fn boxed_scan_vendor_step_quiet_empty<'a>( seed: HashMap>, client: ApiClient, use_public_proxy: bool, + prior: Option<&'a NpmCrawlSnapshot>, ) -> std::pin::Pin + 'a>> { Box::pin(run_scan_vendor_step( common, @@ -911,6 +960,7 @@ fn boxed_scan_vendor_step_quiet_empty<'a>( client, use_public_proxy, false, + prior, )) } @@ -922,9 +972,10 @@ fn boxed_download_patch_records<'a>( params: &'a DownloadParams, api_client: &'a ApiClient, prefetched: HashMap, + prior: Option<&'a NpmCrawlSnapshot>, ) -> std::pin::Pin + 'a>> { - Box::pin(download_patch_records_with( - selected, params, api_client, prefetched, + Box::pin(download_patch_records_reusing( + selected, params, api_client, prefetched, prior, )) } @@ -938,14 +989,15 @@ fn boxed_vendor_records<'a>( service: Option<&'a socket_patch_core::vendor::VendorServiceConfig>, ledger: std::io::Result, env: &'a mut Envelope, + prior: Option<&'a NpmCrawlSnapshot>, ) -> std::pin::Pin + 'a>> { // `scan --vendor` threads the SAME service config the `vendor` command // builds (honoring `--vendor-source`), so both entry points vendor the // same bytes by default. See `run_scan_vendor_step`. Always detached: // vendored mode is manifest-free. The ledger is the one the harvest // just read, handed over so the engine does not reload it. - Box::pin(vendor_records( - common, records, sources, /*detached=*/ true, false, env, service, ledger, + Box::pin(vendor_records_reusing( + common, records, sources, /*detached=*/ true, false, env, service, ledger, prior, )) } diff --git a/crates/socket-patch-cli/src/commands/vendor.rs b/crates/socket-patch-cli/src/commands/vendor.rs index 04cd42273..194f4ff28 100644 --- a/crates/socket-patch-cli/src/commands/vendor.rs +++ b/crates/socket-patch-cli/src/commands/vendor.rs @@ -18,6 +18,7 @@ //! CLI_CONTRACT.md "Ownership, state, and reversal". use clap::Args; +use futures_util::StreamExt; use socket_patch_core::api::client::get_api_client_with_overrides; use socket_patch_core::constants::SOCKET_DIR; use socket_patch_core::crawlers::{CrawlerOptions, Ecosystem}; @@ -25,16 +26,19 @@ use socket_patch_core::manifest::operations::{read_manifest, write_manifest}; use socket_patch_core::manifest::schema::{PatchManifest, PatchRecord}; use socket_patch_core::patch::apply::{verify_file_patch, PatchSources}; use socket_patch_core::telemetry::{track_patch_vendor_failed, track_patch_vendored}; +use socket_patch_core::utils::concurrent::{ordered_concurrent, registry_concurrency}; +use socket_patch_core::utils::group_commit::GroupCommit; use socket_patch_core::utils::purl::{canonical_purl, normalize_purl, strip_purl_qualifiers}; use socket_patch_core::utils::socket_dir::remove_tree_and_prune; use socket_patch_core::vendor::{ self, ecosystem_dir_for_purl, load_state, lock_inventory, lookup_entry, registry_fetch, - save_state, RevertOpts, RevertOutcome, VendorEntry, VendorOutcome, VendorServiceConfig, - VendorState, VendorWarning, + save_state, save_state_shared, DeferredMiss, DeferredPackage, PackageSource, RevertOpts, + RevertOutcome, VendorEntry, VendorOutcome, VendorServiceConfig, VendorState, VendorWarning, }; use socket_patch_core::vex::time::now_rfc3339; use std::collections::{HashMap, HashSet}; use std::path::Path; +use std::sync::Arc; use std::time::Duration; use crate::args::{apply_env_toggles, GlobalArgs}; @@ -47,7 +51,8 @@ use crate::commands::vex::{ generate_vex_from_manifest_path, generate_vex_without_manifest, ManifestlessVex, VexEmbedArgs, }; use crate::ecosystem_dispatch::{ - find_packages_for_rollback, npm_paths_by_identity, partition_purls, + find_packages_for_rollback_reusing, npm_paths_by_identity, npm_paths_by_identity_in, + partition_purls, NpmCrawlSnapshot, }; use crate::json_envelope::{ Command, Envelope, EnvelopeError, PatchAction, PatchEvent, RunWarning, Status, VexSummary, @@ -97,11 +102,13 @@ fn refusal_is_benign(code: &str) -> bool { /// Dispatch one purl to its ecosystem backend. `pkg_path` is the crawler's /// installed location (site-packages root for pypi, the package dir -/// otherwise). Returns `None` for purls with no vendor backend in this build. +/// otherwise), or a fetched artifact the backend materialises only if it +/// reaches a branch that reads it. Returns `None` for purls with no vendor +/// backend in this build. #[allow(clippy::too_many_arguments)] pub(crate) async fn dispatch_vendor_one( purl: &str, - pkg_path: &Path, + pkg_path: PackageSource<'_>, project_root: &Path, record: &PatchRecord, sources: &PatchSources<'_>, @@ -114,6 +121,7 @@ pub(crate) async fn dispatch_vendor_one( // rebuilds locally from the recorded patch. service: Option<&VendorServiceConfig>, pipenv_version: &tokio::sync::OnceCell>, + installed_sites: &vendor::pypi::InstalledSiteListings, ) -> Option { let eco = ecosystem_dir_for_purl(purl)?; @@ -160,6 +168,29 @@ pub(crate) async fn dispatch_vendor_one( .await }; } + // Maven and NuGet have no registry-fetch rung — `fetch_and_stage` serves + // no fetcher for either and `stage_local_artifact` is npm-only — so their + // source is always the crawler's own directory. + macro_rules! vend_installed { + ($backend:path) => {{ + debug_assert!( + matches!(pkg_path, PackageSource::Installed(_)), + "{eco} has no fetch rung; a pending source would need materialising" + ); + $backend( + purl, + pkg_path.path(), + project_root, + record, + sources, + vendored_at, + dry_run, + force, + service, + ) + .await + }}; + } Some(match eco { // The flavor router probes the project's lockfile (package-lock / // yarn / pnpm / bun) and dispatches or refuses per flavor. @@ -176,6 +207,7 @@ pub(crate) async fn dispatch_vendor_one( force, service, pipenv_version, + installed_sites, ) .await } @@ -183,8 +215,8 @@ pub(crate) async fn dispatch_vendor_one( "cargo" => vend!(vendor::cargo::vendor_cargo_crate), "golang" => vend!(vendor::golang::vendor_go_module), "composer" => vend!(vendor::composer_lock::vendor_composer), - "nuget" => vend!(vendor::nuget_feed::vendor_nuget), - "maven" => vend!(vendor::maven_repo::vendor_maven), + "nuget" => vend_installed!(vendor::nuget_feed::vendor_nuget), + "maven" => vend_installed!(vendor::maven_repo::vendor_maven), _ => return None, }) } @@ -989,11 +1021,44 @@ pub(crate) async fn persist_vendor_entry( env: &mut Envelope, state: &mut VendorState, candidate: &str, - mut entry: VendorEntry, + entry: VendorEntry, detached: bool, record: &PatchRecord, ) -> bool { - let mut has_errors = false; + let mut shared = std::sync::Arc::new(std::mem::take(state)); + let (has_errors, stale) = + record_vendor_entry(common, env, &mut shared, candidate, entry, detached, record).await; + // A group commit keeps its own reference to the ledger it captured, so + // this copies — once per save, as the by-value save always did. + *state = std::sync::Arc::try_unwrap(shared).unwrap_or_else(|held| (*held).clone()); + if let Some(stale) = stale { + sweep_stale_artifact(common, env, state, stale).await; + } + has_errors +} + +/// The entry a re-vendor under a newer patch uuid replaced, whose uuid dir +/// is an orphan once the new wiring and ledger are committed. +pub(crate) struct StaleArtifact { + candidate: String, + prev: VendorEntry, +} + +/// [`persist_vendor_entry`]'s bookkeeping half: everything but the sweep of +/// the replaced uuid's dir, which is handed back so a group-committed run +/// can hold it until its commit (deleting it earlier would leave the +/// committed, pre-run wiring pointing at a dir that is gone if the run +/// never commits). +#[allow(clippy::too_many_arguments)] +async fn record_vendor_entry( + common: &GlobalArgs, + env: &mut Envelope, + state: &mut std::sync::Arc, + candidate: &str, + mut entry: VendorEntry, + detached: bool, + record: &PatchRecord, +) -> (bool, Option) { let candidate = candidate.to_string(); entry.detached = detached; // EVERY entry embeds its patch record, not only detached (vendored-mode) @@ -1020,60 +1085,73 @@ pub(crate) async fn persist_vendor_entry( vendor::carry_forward_wiring(prev, &mut entry); } let new_uuid = entry.uuid.clone(); - state.entries.insert(candidate.clone(), entry); - // Persist per-package so a crash mid-run leaves a - // ledger that matches what's already wired. - if let Err(e) = save_state(&common.cwd, state).await { - has_errors = true; + // Persist per-package so a crash mid-run leaves a ledger that matches + // what's already wired (under a group commit this lands in the run's + // captured state, committed with the wiring it describes — the group + // holds this very ledger, so the insert is made in place rather than + // on a per-package copy of the whole ledger). + let key = candidate.clone(); + if let Err(e) = save_state_shared(&common.cwd, state, move |s| { + s.entries.insert(key, entry); + }) + .await + { env.record( PatchEvent::new(PatchAction::Failed, candidate.clone()) .with_error("vendor_state_write_failed", e.to_string()), ); - } else if let Some(prev) = prev.filter(|p| p.uuid != new_uuid) { - // Re-vendor under a newer patch uuid: the old - // uuid's dir is an orphan now — the wiring and - // ledger both point at the new uuid — unless - // another entry still shares it (the same - // `(eco, uuid)` ownership test as `--revert`'s - // orphan sweep). Only the live entry would - // otherwise reclaim it, and that never happens. - let still_referenced = state - .entries - .values() - .any(|e| e.ecosystem == prev.ecosystem && e.uuid == prev.uuid); - let stale_rel = vendor::path::vendor_uuid_dir_rel(&prev.ecosystem, &prev.uuid); - if let Some(rel) = stale_rel.filter(|_| !still_referenced) { - if let Err(detail) = vendor::bun_lock::cleanup_binary_workspace_artifacts( - &common.cwd, - &prev, - common.dry_run, - ) + return (true, None); + } + let stale = prev + .filter(|p| p.uuid != new_uuid) + .map(|prev| StaleArtifact { candidate, prev }); + (false, stale) +} + +/// Re-vendor under a newer patch uuid: the old uuid's dir is an orphan now — +/// the wiring and ledger both point at the new uuid — unless another entry +/// still shares it (the same `(eco, uuid)` ownership test as `--revert`'s +/// orphan sweep). Only the live entry would otherwise reclaim it, and that +/// never happens. +async fn sweep_stale_artifact( + common: &GlobalArgs, + env: &mut Envelope, + state: &VendorState, + stale: StaleArtifact, +) { + let StaleArtifact { candidate, prev } = stale; + let still_referenced = state + .entries + .values() + .any(|e| e.ecosystem == prev.ecosystem && e.uuid == prev.uuid); + let stale_rel = vendor::path::vendor_uuid_dir_rel(&prev.ecosystem, &prev.uuid); + let Some(rel) = stale_rel.filter(|_| !still_referenced) else { + return; + }; + if let Err(detail) = + vendor::bun_lock::cleanup_binary_workspace_artifacts(&common.cwd, &prev, common.dry_run) .await - { - record_warning( - env, - &candidate, - &VendorWarning::new("vendor_stale_artifact_kept", detail), - common, - ); - return has_errors; - } - if !common.dry_run { - // Prunes the emptied `/` level too (a uuid change - // within one ecosystem never empties it, but a re-vendor - // that moved ecosystems would otherwise leave a husk). - let _ = remove_tree_and_prune(&common.cwd.join(rel), &common.cwd.join(SOCKET_DIR)) - .await; - } - env.record( - PatchEvent::new(PatchAction::Removed, candidate.clone()).with_reason( - "vendor_stale_artifact_removed", - "previous patch uuid's vendored artifact removed", - ), - ); - } + { + record_warning( + env, + &candidate, + &VendorWarning::new("vendor_stale_artifact_kept", detail), + common, + ); + return; } - has_errors + if !common.dry_run { + // Prunes the emptied `/` level too (a uuid change + // within one ecosystem never empties it, but a re-vendor + // that moved ecosystems would otherwise leave a husk). + let _ = remove_tree_and_prune(&common.cwd.join(rel), &common.cwd.join(SOCKET_DIR)).await; + } + env.record( + PatchEvent::new(PatchAction::Removed, candidate).with_reason( + "vendor_stale_artifact_removed", + "previous patch uuid's vendored artifact removed", + ), + ); } /// One registry-fetch attempt through the pristine-source ladder's network @@ -1130,6 +1208,481 @@ pub(crate) async fn fetch_pristine_package( } } +/// Whether [`fetch_pristine_package`] would pick a VERIFIABLE registry +/// resolution for this purl — the same entry choice, made without the +/// download: the lock's own entry when it carries an integrity, else the +/// pre-vendor resolution the ledger recovers. A cargo crate from a git, +/// path or custom-registry source has neither, so its fetch refuses +/// `vendor_fetch_unverifiable` and the purl is not vendored; deferring that +/// fetch behind the patch service would instead vendor the crates.io patch +/// over it. A purl with neither is also one the ladder has no source for +/// at all (`package_not_installed` when nothing is installed), which is +/// why the early lock-text refusals ([`lock_refusals_reaching_backend`]) +/// stay off it. +pub(crate) async fn pristine_fetch_is_verifiable( + project_root: &Path, + inventory: &[lock_inventory::LockfileEntry], + purl: &str, + ledger_entry: Option<&VendorEntry>, +) -> bool { + let verifiable = + |e: &lock_inventory::LockfileEntry| e.integrity != lock_inventory::LockIntegrity::None; + if lock_inventory::lookup(inventory, purl).is_some_and(verifiable) { + return true; + } + match ledger_entry { + Some(le) => lock_inventory::recover_lock_entry(project_root, le) + .await + .is_ok_and(|e| verifiable(&e)), + None => false, + } +} + +/// The purls among `purls` with an installed copy, found exactly as the +/// vendor loop finds them: the qualified-aware resolver +/// ([`find_packages_for_rollback_reusing`]), then the npm `package.json` +/// identity lookup for an npm purl it missed (an alias install). `prior` +/// is the loop's own reusable npm crawl, when the caller has it. +pub(crate) async fn installed_purls( + options: &CrawlerOptions, + purls: &[String], + prior: Option<&NpmCrawlSnapshot>, +) -> HashSet { + if purls.is_empty() { + return HashSet::new(); + } + let partition = partition_purls(purls, None); + let mut installed: HashSet = + find_packages_for_rollback_reusing(&partition, options, true, prior) + .await + .into_keys() + .collect(); + let missing_npm: Vec<&String> = partition + .get(&Ecosystem::Npm) + .into_iter() + .flatten() + .filter(|p| !installed.contains(*p)) + .collect(); + let by_identity = match prior.and_then(|p| p.packages_for(options)) { + Some(crawled) => npm_paths_by_identity_in(crawled, &missing_npm), + None => npm_paths_by_identity(options, &missing_npm).await, + }; + installed.extend(by_identity.into_keys()); + installed +} + +/// Narrows `refused` (the lock-text refusals of +/// [`vendor::lock_text_refusals`]) to the packages the vendor loop would +/// actually hand to their backend — and so see refused, in these very +/// words — once it has a source for them: an installed copy (`installed` +/// answers, for the purls with no verifiable registry resolution), or a +/// verifiable registry resolution the pristine-source ladder fetches +/// ([`pristine_fetch_is_verifiable`]). A package with neither never +/// reaches its backend: the loop reports it `package_not_installed` (a +/// calm skip), with no pristine fetch, so it is left to the loop and keeps +/// that outcome. The check reads only local files. +pub(crate) async fn lock_refusals_reaching_backend( + cwd: &Path, + mut refused: HashMap, + ledger: &HashMap, + installed: F, +) -> HashMap +where + F: FnOnce(Vec) -> Fut, + Fut: std::future::Future>, +{ + if refused.is_empty() { + return refused; + } + let inventory = lock_inventory::inventory_project(cwd).await; + let mut unresolved: Vec = Vec::new(); + for purl in refused.keys() { + if !pristine_fetch_is_verifiable(cwd, &inventory, purl, lookup_entry(ledger, purl)).await { + unresolved.push(purl.clone()); + } + } + if unresolved.is_empty() { + return refused; + } + unresolved.sort(); + let installed = installed(unresolved.clone()).await; + for purl in unresolved { + if !installed.contains(&purl) { + refused.remove(&purl); + } + } + refused +} + +/// One purl's pristine source while the vendor loop is being assembled. +/// +/// A fetched artifact is held by index into the run's `fetched_holders` +/// rather than by path: the tree is not on disk yet (see +/// [`registry_fetch::FetchedPackage`]), and only a backend branch that +/// actually reads it makes it so. +enum StagedSource { + /// The crawler's installed location. + Installed(std::path::PathBuf), + /// `fetched_holders[i]`. + Fetched(usize), + /// `deferred_holders[i]`: not downloaded unless a backend reads it. + Deferred(usize), +} + +impl StagedSource { + fn as_source<'a>( + &'a self, + holders: &'a [registry_fetch::FetchedPackage], + deferred: &'a [DeferredPackage], + ) -> PackageSource<'a> { + match self { + Self::Installed(dir) => PackageSource::Installed(dir), + Self::Fetched(at) => PackageSource::Pending(&holders[*at]), + Self::Deferred(at) => PackageSource::Deferred(&deferred[*at]), + } + } +} + +/// Where a vendorable purl with no installed copy stands after the local +/// rungs of the pristine-source ladder (see [`missing_local_rung`]). +enum MissingRung { + /// Staged from its own committed artifact (sha256-verified). + Staged(registry_fetch::FetchedPackage), + /// Its committed artifact is present but corrupt (the detail). + StageFailed(String), + /// `--offline`: no registry rung. + Offline, + /// Left for the registry fetch ([`fetch_pristine_package`]). + Fetch, + /// The registry fetch, run only if the backend reads the pristine tree + /// ([`deferred_pristine_package`]). + Deferred, + /// A gem a local build cannot vendor from a download: refused before + /// the fetch. + GemBuildRefused, +} + +impl MissingRung { + /// Whether this purl still needs [`fetch_pristine_package`] — the one + /// predicate behind both the fetch plan and the lazy lock inventory, + /// so they cannot name different purls. + fn needs_registry(&self) -> bool { + matches!(self, MissingRung::Fetch) + } +} + +/// The local rungs for one missing purl, deciding without emitting +/// anything: an already-vendored npm purl with no installed copy (fresh +/// clone) stages from its own committed artifact, sha256-verified against +/// the ledger — offline-safe, no registry traffic — and `--offline` stops +/// before the registry. Also returns the committed artifact's path when it +/// is missing (the caller's `vendor_artifact_missing` warning; the purl +/// then falls through to the registry ladder, which recovers the +/// pre-vendor resolution from the ledger and rebuilds). +async fn missing_local_rung( + common: &GlobalArgs, + ledger_entry: Option<&VendorEntry>, +) -> (Option, MissingRung) { + let mut artifact_missing = None; + if let Some(entry) = + ledger_entry.filter(|e| e.ecosystem == "npm" && e.artifact.path.ends_with(".tgz")) + { + let tgz = common.cwd.join(&entry.artifact.path); + if tokio::fs::metadata(&tgz).await.is_err() { + artifact_missing = Some(entry.artifact.path.clone()); + } else { + match registry_fetch::stage_local_artifact(&tgz, &entry.artifact.sha256).await { + Ok(staged) => return (None, MissingRung::Staged(staged)), + Err(registry_fetch::FetchError::Failed(detail)) => { + return (None, MissingRung::StageFailed(detail)) + } + // No recorded hash (legacy ledger) — fall through to the + // lockfile/registry path. + Err(registry_fetch::FetchError::Unverifiable(_)) => {} + } + } + } + let rung = if common.offline { + MissingRung::Offline + } else { + MissingRung::Fetch + }; + (artifact_missing, rung) +} + +/// Whether the ledger already covers `record` for `entry`'s purl: the entry +/// records this very patch uuid and its committed artifact is on disk — a +/// FILE artifact (wheel, tarball) hashing to the ledger's `sha256`. +/// Read-only, no network. The backend's in-sync hot path answers such a +/// purl from the committed artifact without reading the pristine tree, +/// which is what lets its download be deferred. Some in-sync checks look +/// only for the artifact's presence (pypi's), so a file artifact that no +/// longer hashes to its ledger pin is not covered: it keeps the eager +/// ladder, and a run that cannot reach the registry says so as it always +/// did. A copy DIR's integrity stays the backend's own question — a +/// drifted one is rebuilt, and a rebuild that needs the pristine tree +/// fetches it then. +async fn ledger_covers(cwd: &Path, entry: Option<&VendorEntry>, record: &PatchRecord) -> bool { + match entry { + Some(entry) if entry.uuid == record.uuid => entry.committed_artifact_intact(cwd).await, + _ => false, + } +} + +/// The pristine source for a purl whose download is deferred: the same +/// [`fetch_pristine_package`] ladder, run on the first backend call that +/// reads the tree. `--offline` never reaches the registry, so there the +/// deferred fetch reports the offline stop instead. The outcome's `code` +/// tells [`deferred_miss`] which eager-fetch report to reproduce. +fn deferred_pristine_package( + common: &GlobalArgs, + inventory: &Arc>>, + client: ®istry_fetch::RegistryClient, + purl: &str, + ledger_entry: Option<&VendorEntry>, +) -> DeferredPackage { + let cwd = common.cwd.clone(); + let offline = common.offline; + let inventory = Arc::clone(inventory); + let client = client.clone(); + let owned_purl = purl.to_string(); + let ledger_entry = ledger_entry.cloned(); + DeferredPackage::new( + ®istry_fetch::staged_leaf_for_purl(purl), + Box::new(move || { + Box::pin(async move { + if offline { + return Err(DeferredMiss { + code: "offline", + detail: "--offline prevents fetching the pristine artifact from \ + the registry" + .to_string(), + }); + } + let inv = inventory + .get_or_init(|| lock_inventory::inventory_project(&cwd)) + .await; + match fetch_pristine_package(&cwd, inv, &client, &owned_purl, ledger_entry.as_ref()) + .await + { + PristineFetch::Fetched(fetched) => Ok(fetched), + PristineFetch::NoSource => Err(DeferredMiss { + code: "no_source", + detail: "no installed package found on disk".to_string(), + }), + PristineFetch::Unverifiable(detail) => Err(DeferredMiss { + code: "unverifiable", + detail, + }), + PristineFetch::Failed(detail) => Err(DeferredMiss { + code: "failed", + detail, + }), + } + }) + }), + ) +} + +/// The `vendor_fetched_missing` advisory for a pristine artifact fetched +/// because the package is not installed. +fn record_fetched_missing(env: &mut Envelope, common: &GlobalArgs, purl: &str, url: &str) { + record_warning( + env, + purl, + &VendorWarning::new( + "vendor_fetched_missing", + format!( + "{} is not installed; fetched the pristine artifact from {url} (integrity \ + verified) and vendored from that copy — the project tree was not touched", + normalize_purl(purl) + ), + ), + common, + ); +} + +/// Report a deferred fetch that produced no package exactly as the eager +/// fetch would have reported it for `purl`: a failed download is the same +/// `vendor_fetch_failed` failure (and suppresses the later +/// `package_not_installed` skip for the whole variant group), an +/// unverifiable lock entry the same `vendor_fetch_unverifiable` warning; +/// no source, and the `--offline` stop, say nothing here and leave the +/// candidates to the unmatched pass's `package_not_installed` skip. The +/// caller drops the backend's own outcome — the backend only failed +/// because the tree it asked for never arrived — and un-matches +/// `candidates`. +fn deferred_miss( + env: &mut Envelope, + common: &GlobalArgs, + purl: &str, + miss: &DeferredMiss, + candidates: &[String], + fetch_failed: &mut HashSet, +) { + match miss.code { + "unverifiable" => record_warning( + env, + purl, + &VendorWarning::new("vendor_fetch_unverifiable", miss.detail.clone()), + common, + ), + "no_source" | "offline" => {} + _ => { + fetch_failed.insert(purl.to_string()); + fetch_failed.extend(candidates.iter().cloned()); + env.record( + PatchEvent::new(PatchAction::Failed, purl.to_string()) + .with_error("vendor_fetch_failed", miss.detail.clone()), + ); + report_vendor_failure(common, purl, &format!("fetch failed: {}", miss.detail)); + } + } +} + +/// The patch-service downloads the vendor loop will make, in the loop's +/// order: `all_packages` walked as the loop walks it — release-variant +/// bases fanned out once to their manifest variants, each variant through +/// the same installed-variant probe — past the Bun refusal and the hosted +/// takeover gate, then through the record's backend gate (see +/// [`vendor::service_preflight`], and npm's one-read +/// [`vendor::npm_flavor::preflight_packages`] with the committed-artifact +/// reuse the npm backends answer from the ledger). Only reads: the probe +/// extracts a fetched artifact the loop's own probe would, and a variant +/// whose probe needs a download not made yet is left out — unplanned, the +/// loop simply fetches it live. Every doubt resolves to "not planned", +/// never to a grant the loop does not ask for. +#[allow(clippy::too_many_arguments)] +async fn plan_service_downloads( + cwd: &Path, + force: bool, + all_packages: &[(String, StagedSource)], + (fetched_holders, deferred_holders): (&[registry_fetch::FetchedPackage], &[DeferredPackage]), + variant_groups: &HashMap>, + records: &HashMap, + ledger: &VendorState, + bun_refusal: Option<&crate::commands::bun_preflight::BunVendorRefusal>, + takeover_blocked: &dyn Fn(&str) -> bool, + (pipenv_version, installed_sites): ( + &tokio::sync::OnceCell>, + &vendor::pypi::InstalledSiteListings, + ), +) -> Vec { + // Each loop candidate that reaches its backend, in loop order. + let mut reaching: Vec<(&str, &PatchRecord, &Path)> = Vec::new(); + let mut handled_bases: HashSet = HashSet::new(); + for (purl, staged) in all_packages { + let source = staged.as_source(fetched_holders, deferred_holders); + let deferred = match staged { + StagedSource::Deferred(at) => Some(&deferred_holders[*at]), + _ => None, + }; + let is_variant_eco = + Ecosystem::from_purl(purl).is_some_and(|e| e.supports_release_variants()); + let candidates: Vec = if is_variant_eco { + let base = strip_purl_qualifiers(purl).to_string(); + if !handled_bases.insert(base.clone()) { + continue; + } + variant_groups + .get(&base) + .cloned() + .unwrap_or_else(|| vec![base]) + } else { + vec![purl.clone()] + }; + for candidate in &candidates { + let Some((candidate, record)) = records.get_key_value(candidate) else { + continue; + }; + // The loop's installed-variant probe (see there). + let probe_applicable = is_variant_eco + && !matches!(Ecosystem::from_purl(candidate), Some(Ecosystem::Maven)); + let ledger_answers_probe = deferred.is_some_and(|d| d.outcome().is_none()) + && lookup_entry(&ledger.entries, candidate).is_some_and(|e| e.uuid == record.uuid); + if probe_applicable && !force && !ledger_answers_probe { + if let Some((file, info)) = representative_file(&record.files) { + if matches!(source, PackageSource::Deferred(_)) { + continue; + } + let Ok(dir) = source.materialize().await else { + continue; + }; + let status = verify_file_patch(dir, file, info).await.status; + if !variant_matches_installed(Some(&status)) { + continue; + } + } + } + if bun_refusal.is_some_and(|r| r.applies_to(candidate)) { + continue; + } + if socket_patch_core::patch::redirect::redirect_revert_supported(candidate) + && takeover_blocked(candidate) + { + continue; + } + // The npm backends re-wire a committed artifact the ledger + // anchors at this uuid without asking the service. + if candidate.starts_with("pkg:npm/") + && ledger.entries.values().any(|e| { + e.ecosystem == "npm" && e.uuid == record.uuid && !e.artifact.sha256.is_empty() + }) + { + continue; + } + // A purl the ledger already records at this record's uuid is a + // re-run, which every backend's in-sync hot path answers without + // the service; proving it costs the verification of the whole + // committed artifact, which the loop's own call repeats. Left + // out of the plan: should the artifact need rebuilding after all, + // the loop fetches it live. + if lookup_entry(&ledger.entries, candidate).is_some_and(|e| e.uuid == record.uuid) { + continue; + } + reaching.push((candidate.as_str(), record, source.path())); + } + } + + let npm: Vec<(&str, &PatchRecord)> = reaching + .iter() + .filter(|(purl, _, _)| purl.starts_with("pkg:npm/")) + .map(|(purl, record, _)| (*purl, *record)) + .collect(); + let mut npm_verdicts = if npm.is_empty() { + Vec::new() + } else { + vendor::npm_flavor::preflight_packages(cwd, &npm).await + } + .into_iter(); + // One gate at a time: several at once would each hold their own parse + // of the project's lockfiles (a cargo gate clones the whole Cargo.lock + // document), which a monorepo pays for in peak memory. + let mut planned = Vec::new(); + for (purl, record, source_path) in &reaching { + let download = if purl.starts_with("pkg:npm/") { + npm_verdicts + .next() + .filter(|verdict| verdict.is_ok()) + .map(|_| { + socket_patch_core::api::client::PlannedDownload::archive(record.uuid.clone()) + }) + } else { + vendor::service_preflight( + purl, + source_path, + cwd, + record, + pipenv_version, + installed_sites, + ) + .await + }; + planned.extend(download); + } + planned +} + /// The vendoring engine, decoupled from the manifest file. `records` is the /// purl → [`PatchRecord`] view to vendor: `manifest.patches` for the /// manifest-driven `vendor` command, or the in-memory record map @@ -1159,6 +1712,30 @@ pub(crate) async fn vendor_records( // command and `scan --vendor` pass `Some(_)`, honoring `--vendor-source`. service: Option<&VendorServiceConfig>, ledger: std::io::Result, +) -> bool { + vendor_records_reusing( + common, records, sources, detached, force, env, service, ledger, None, + ) + .await +} + +/// [`vendor_records`], resolving npm packages from `prior` — the npm half +/// of a crawl this process made earlier with the same options, over a tree +/// nothing has touched since (`scan`'s own crawl) — instead of walking +/// `node_modules` again: its roots feed the targeted lookup and its +/// packages the alias identity fallback. `None` (or a snapshot taken with +/// other options) crawls as before. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn vendor_records_reusing( + common: &GlobalArgs, + records: &HashMap, + sources: &PatchSources<'_>, + detached: bool, + force: bool, + env: &mut Envelope, + service: Option<&VendorServiceConfig>, + ledger: std::io::Result, + prior: Option<&NpmCrawlSnapshot>, ) -> bool { let mut has_errors = false; // Lockfile flavors the backends wired THIS run (from the returned ledger @@ -1216,13 +1793,20 @@ pub(crate) async fn vendor_records( // first (and pushing its warnings into the envelope) only to fail the // run afterwards. let mut state = match ledger { - Ok(s) => s, + Ok(s) => std::sync::Arc::new(s), Err(e) => { env.mark_error(EnvelopeError::new("vendor_state_unreadable", e.to_string())); report_state_unreadable(common, &e); return true; } }; + // Pre-stage trees a previous run left behind (it crashed or was + // interrupted between staging an archive and settling): nothing of + // this run is staged yet, and the caller holds the apply lock, so every + // one on disk is stale scratch. A dry run deletes nothing. + if !common.dry_run { + vendor::prestage::sweep_stale(&common.cwd).await; + } let crawler_options = CrawlerOptions { cwd: common.cwd.clone(), @@ -1240,12 +1824,16 @@ pub(crate) async fn vendor_records( // registry download, and (for gem) a HashMap-order platform coin-flip. // The rollback variant fans each base path back out to every qualified // manifest purl (same invariant as `find_manifest_package_paths`). - let mut all_packages = find_packages_for_rollback( + let mut all_packages: HashMap = find_packages_for_rollback_reusing( &vendorable_partition, &crawler_options, common.silent || common.json, + prior, ) - .await; + .await + .into_iter() + .map(|(purl, dir)| (purl, StagedSource::Installed(dir))) + .collect(); // An npm alias is installed under its dependency key, not its actual // package name. The targeted resolver probes canonical paths; before @@ -1257,8 +1845,12 @@ pub(crate) async fn vendor_records( .flatten() .filter(|p| !all_packages.contains_key(*p)) .collect(); - for (purl, paths) in npm_paths_by_identity(&crawler_options, &missing_npm).await { - all_packages.insert(purl, paths[0].clone()); + let by_identity = match prior.and_then(|p| p.packages_for(&crawler_options)) { + Some(installed) => npm_paths_by_identity_in(installed, &missing_npm), + None => npm_paths_by_identity(&crawler_options, &missing_npm).await, + }; + for (purl, paths) in by_identity { + all_packages.insert(purl, StagedSource::Installed(paths[0].clone())); } // ── Auto-fetch: lockfile-resolved packages with no installed copy ──── @@ -1270,16 +1862,21 @@ pub(crate) async fn vendor_records( // installed copy (it keys off lock entries). The holders keep the // tempdirs alive until the dispatch loop below has staged from them. let mut fetched_holders: Vec = Vec::new(); + // Sources whose download is deferred to the backend branch that reads + // them (see the plan below), held by index like `fetched_holders`. + let mut deferred_holders: Vec = Vec::new(); // Fetch failures must keep their distinct Failed event; this set // suppresses the later duplicate `package_not_installed` skip. let mut fetch_failed: HashSet = HashSet::new(); // The lockfile inventory (every recognized lockfile parsed) — a local // read, fine offline — built lazily at the first site that consumes it - // and shared by the registry-fetch rung below and the `--offline` - // "the lockfile resolves it" detail at the end, so a run parses the - // lockfiles at most once (and not at all when every missing purl - // stages from its committed artifact or nothing is missing). - let mut inventory: Option> = None; + // and shared by the registry-fetch rung below, the deferred fetches and + // the `--offline` "the lockfile resolves it" detail at the end, so a run + // parses the lockfiles at most once (and not at all when every missing + // purl stages from its committed artifact, every deferred source is + // answered by its backend's hot path, or nothing is missing). + let inventory: Arc>> = + Arc::new(tokio::sync::OnceCell::new()); { let missing: Vec = vendorable .iter() @@ -1288,93 +1885,305 @@ pub(crate) async fn vendor_records( .collect(); if !missing.is_empty() { let client = registry_fetch::build_registry_client(); - // Artifact-staging path: an already-vendored purl with no - // installed copy (fresh clone) stages from its own committed - // artifact, sha256-verified against the ledger — offline-safe, - // no registry traffic. - for purl in &missing { - let ledger_entry = lookup_entry(&state.entries, purl); - if let Some(entry) = ledger_entry - .filter(|e| e.ecosystem == "npm" && e.artifact.path.ends_with(".tgz")) - { - let tgz = common.cwd.join(&entry.artifact.path); - if tokio::fs::metadata(&tgz).await.is_err() { - // The committed artifact is GONE (gitignored or - // deleted): not corruption — fall through to the - // registry ladder, which recovers the pre-vendor - // resolution from the ledger and rebuilds. - record_warning( - env, - purl, - &VendorWarning::new( - "vendor_artifact_missing", - format!( - "the committed vendored artifact {} is missing; \ - recovering the registry resolution to rebuild it", - entry.artifact.path - ), - ), - common, - ); - } else { - match registry_fetch::stage_local_artifact(&tgz, &entry.artifact.sha256) - .await - { - Ok(staged) => { - all_packages.insert(purl.clone(), staged.dir().to_path_buf()); - fetched_holders.push(staged); - continue; - } - Err(registry_fetch::FetchError::Failed(detail)) => { - // A PRESENT-but-corrupt committed artifact is - // worth a loud failure — silently re-vendoring - // over it would mask the corruption. - fetch_failed.insert(purl.clone()); - let detail = format!( - "{detail}; run `socket-patch repair` to rebuild the \ - vendored artifact" - ); - env.record( - PatchEvent::new(PatchAction::Failed, purl.clone()) - .with_error("vendor_fetch_failed", detail.clone()), - ); - report_vendor_failure(common, purl, &detail); - continue; - } - Err(registry_fetch::FetchError::Unverifiable(_)) => { - // No recorded hash (legacy ledger) — fall - // through to the lockfile/registry path. - } - } - } + // Two passes over `missing`, so the registry fetches can run + // concurrently while every event, warning and stderr line still + // lands in `missing` order, exactly as the one-purl-at-a-time + // loop emitted them. Pass 1 decides each purl's local rungs (the + // committed-artifact staging and the offline stop: local and + // read-only, so deciding them early changes nothing) without + // emitting anything; the purls left for the registry are then + // fetched at most `registry_concurrency` at a time, in order, and + // pass 2 emits every purl's outcome in turn. + let mut rungs: Vec<(Option, MissingRung)> = { + let mut rungs = Vec::with_capacity(missing.len()); + for purl in &missing { + rungs + .push(missing_local_rung(common, lookup_entry(&state.entries, purl)).await); } - if common.offline { - // The enriched skip detail lands below in the unmatched - // pass (the purl stays unmatched). + rungs + }; + // Downloads nothing may need, deferred to the backend branch + // that reads the pristine tree (see `DeferredPackage`): + // + // * a purl the ledger already covers — its entry records this + // record's patch uuid and the committed artifact is on disk — + // is what the backend's in-sync hot path answers from those + // committed bytes alone, so a re-run makes no registry request + // (and succeeds with no network at all). `--force` may + // rebuild anyway, so it keeps the eager fetch. + // * a cargo crate the patch service can serve: the backend reads + // the pristine tree only once `cargo_service_copy` falls back + // to the local build. Only a crate the registry ladder COULD + // fetch (see `pristine_fetch_is_verifiable`) — a git, path or + // custom-registry crate keeps the eager rung, whose + // `vendor_fetch_unverifiable` refusal is what keeps a + // crates.io patch off a crate that does not come from + // crates.io. + // + // A backend that does reach its pristine tree fetches it then, + // through the same ladder, and the loop reports the fetch as the + // eager one would have (see `deferred_miss`). + let service_enabled = service.is_some_and(VendorServiceConfig::service_enabled); + for (purl, (_, rung)) in missing.iter().zip(rungs.iter_mut()) { + if !matches!(rung, MissingRung::Fetch | MissingRung::Offline) { continue; } - if inventory.is_none() { - inventory = Some(lock_inventory::inventory_project(&common.cwd).await); + let covered = !force + && match records.get(purl) { + Some(record) => { + ledger_covers(&common.cwd, lookup_entry(&state.entries, purl), record) + .await + } + None => false, + }; + let cargo_via_service = service_enabled + && matches!(rung, MissingRung::Fetch) + && Ecosystem::from_purl(purl) == Some(Ecosystem::Cargo) + && pristine_fetch_is_verifiable( + &common.cwd, + inventory + .get_or_init(|| lock_inventory::inventory_project(&common.cwd)) + .await, + purl, + lookup_entry(&state.entries, purl), + ) + .await; + if covered || cargo_via_service { + *rung = MissingRung::Deferred; } - let inv = inventory.as_deref().expect("filled just above"); - match fetch_pristine_package(&common.cwd, inv, &client, purl, ledger_entry).await { - PristineFetch::Fetched(fetched) => { - record_warning( - env, - purl, - &VendorWarning::new( - "vendor_fetched_missing", - format!( - "{} is not installed; fetched the pristine artifact \ - from {} (integrity verified) and vendored from that \ - copy — the project tree was not touched", - normalize_purl(purl), - fetched.url - ), + } + // A missing npm or cargo purl its backend refuses on the + // project's lock text alone (see `vendor::lock_text_refusals`: + // the pnpm / yarn classic / yarn berry gates, cargo's locked + // version) is deferred rather than fetched: the backend refuses + // it — at its turn, in its own words — before anything reads + // the source, so the refusal costs no registry request. A purl + // the hosted redirect ledger claims keeps the eager fetch: its + // takeover reverts the hosted lock edits first, which rewrites + // the text the gates read (and a malformed redirect ledger + // defers nothing). + let lock_candidates: Vec<(&str, &str)> = missing + .iter() + .zip(&rungs) + .filter(|(_, (_, rung))| matches!(rung, MissingRung::Fetch)) + .filter_map(|(purl, _)| { + records + .get(purl) + .map(|record| (purl.as_str(), record.uuid.as_str())) + }) + .filter(|(purl, _)| { + matches!( + vendor::ecosystem_dir_for_purl(purl), + Some("npm") | Some("cargo") + ) + }) + .collect(); + if !lock_candidates.is_empty() { + let claimed: Option> = + match socket_patch_core::patch::redirect::load_redirect_state(&common.cwd).await + { + Ok(Some(state)) => { + Some(state.records.keys().map(|k| canonical_purl(k)).collect()) + } + Ok(None) => Some(Vec::new()), + Err(_) => None, + }; + if let Some(claimed) = claimed { + let unclaimed: Vec<(&str, &str)> = lock_candidates + .into_iter() + .filter(|(purl, _)| !claimed.contains(&canonical_purl(purl))) + .collect(); + // Only a purl the ladder would really fetch (a + // verifiable registry resolution): one with no source + // at all keeps the loop's `package_not_installed` skip. + // These purls have no installed copy, so none is + // looked for. + let refused = lock_refusals_reaching_backend( + &common.cwd, + vendor::lock_text_refusals(&common.cwd, &unclaimed).await, + &state.entries, + |_| async { HashSet::new() }, + ) + .await; + for (purl, (_, rung)) in missing.iter().zip(rungs.iter_mut()) { + if matches!(rung, MissingRung::Fetch) && refused.contains_key(purl) { + *rung = MissingRung::Deferred; + } + } + } + } + // A NOT-INSTALLED gem can only be vendored through the patch + // service. The bundler path source the gem backend wires needs + // the eval-able stub gemspec rubygems writes into + // `/specifications/` at INSTALL time; a fetched `.gem` + // carries its gemspec only as YAML in `metadata.gz`, which is + // exactly why the service serves a converted `gem-stub-gemspec` + // second artifact. With the service off (`--vendor-source build`, + // or no config at all) the fetched copy is unusable, so the + // backend refused `gem_spec_missing` — AFTER paying for the + // download, on every run. Refuse before the download instead, + // with the same code and a detail that names the real remedy. + // The backend keeps its own refusal as the backstop for every + // other route into it. + // + // Scoped to the purls a DOWNLOAD would actually happen for, + // mirroring `fetch_pristine_package`'s own `fetchable` filter: a + // gem the lock cannot VERIFY (no `CHECKSUMS` section) is never + // fetched and keeps its `vendor_fetch_unverifiable` warning plus + // the calm `package_not_installed` skip; a gem the ledger already + // holds is the already-vendored fresh-clone case the backend's + // hot path confirms without a stub gemspec; a gem that resolves + // from nowhere keeps the calm skip. Refusing first also means a + // refusal the backend would have reached earlier on the fetched + // copy (an uneditable Gemfile declaration, say) now reports as + // `gem_spec_missing` instead; either way the package fails. + // A dry run never refused: the backend's verify-only preview + // runs on the fetched copy, so it keeps the eager fetch. + if !service_enabled + && !common.dry_run + && missing + .iter() + .zip(&rungs) + .any(|(p, (_, r))| p.starts_with("pkg:gem/") && r.needs_registry()) + { + let inv = inventory + .get_or_init(|| lock_inventory::inventory_project(&common.cwd)) + .await; + for (purl, (_, rung)) in missing.iter().zip(rungs.iter_mut()) { + if purl.starts_with("pkg:gem/") + && rung.needs_registry() + && lookup_entry(&state.entries, purl).is_none() + && lock_inventory::lookup(inv, purl) + .is_some_and(|e| e.integrity != lock_inventory::LockIntegrity::None) + { + *rung = MissingRung::GemBuildRefused; + } + } + } + // Parsed only when some purl reaches the registry rung (the + // serial loop's lazy first use). + let inv: &[lock_inventory::LockfileEntry] = + if rungs.iter().any(|(_, r)| r.needs_registry()) { + inventory + .get_or_init(|| lock_inventory::inventory_project(&common.cwd)) + .await + } else { + &[] + }; + let (cwd, client_ref, ledger) = (&common.cwd, &client, &state.entries); + let to_fetch: Vec<&String> = missing + .iter() + .zip(&rungs) + .filter(|(_, (_, rung))| rung.needs_registry()) + .map(|(purl, _)| purl) + .collect(); + let mut pristine = std::pin::pin!(ordered_concurrent( + to_fetch, + registry_concurrency(), + |purl| fetch_pristine_package( + cwd, + inv, + client_ref, + purl, + lookup_entry(ledger, purl) + ), + )); + for (purl, (artifact_missing, rung)) in missing.iter().zip(rungs) { + if let Some(artifact) = artifact_missing { + // The committed artifact is GONE (gitignored or + // deleted): not corruption — fall through to the + // registry ladder, which recovers the pre-vendor + // resolution from the ledger and rebuilds. + record_warning( + env, + purl, + &VendorWarning::new( + "vendor_artifact_missing", + format!( + "the committed vendored artifact {artifact} is missing; \ + recovering the registry resolution to rebuild it" ), + ), + common, + ); + } + let fetched = match rung { + MissingRung::Staged(staged) => { + all_packages + .insert(purl.clone(), StagedSource::Fetched(fetched_holders.len())); + fetched_holders.push(staged); + continue; + } + MissingRung::StageFailed(detail) => { + // A PRESENT-but-corrupt committed artifact is + // worth a loud failure — silently re-vendoring + // over it would mask the corruption. + fetch_failed.insert(purl.clone()); + let detail = format!( + "{detail}; run `socket-patch repair` to rebuild the \ + vendored artifact" + ); + env.record( + PatchEvent::new(PatchAction::Failed, purl.clone()) + .with_error("vendor_fetch_failed", detail.clone()), + ); + report_vendor_failure(common, purl, &detail); + continue; + } + MissingRung::Deferred => { + all_packages + .insert(purl.clone(), StagedSource::Deferred(deferred_holders.len())); + deferred_holders.push(deferred_pristine_package( common, + &inventory, + &client, + purl, + lookup_entry(&state.entries, purl), + )); + continue; + } + MissingRung::GemBuildRefused => { + fetch_failed.insert(purl.clone()); + // The backend's own refusal text, word for word. + let detail = format!( + "no local stub gemspec for {} (a path source cannot be wired \ + without one); install the gem or use --vendor-source=service", + strip_purl_qualifiers(purl).trim_start_matches("pkg:gem/") + ); + env.record( + PatchEvent::new(PatchAction::Failed, purl.clone()) + .with_error("gem_spec_missing", detail.clone()), ); - all_packages.insert(purl.clone(), fetched.dir().to_path_buf()); + report_vendor_failure(common, purl, &detail); + continue; + } + // The enriched skip detail lands below in the unmatched + // pass (the purl stays unmatched). + MissingRung::Offline => continue, + MissingRung::Fetch => match pristine.next().await { + Some(fetched) => fetched, + // Unreachable: `to_fetch` holds one fetch per + // `needs_registry` rung, and this is the only arm + // that consumes one. A live fetch keeps the outcome + // right if the two ever fall out of step. + None => { + debug_assert!(false, "pristine prefetch plan out of step at {purl}"); + fetch_pristine_package( + cwd, + inv, + client_ref, + purl, + lookup_entry(ledger, purl), + ) + .await + } + }, + }; + match fetched { + PristineFetch::Fetched(fetched) => { + record_fetched_missing(env, common, purl, &fetched.url); + all_packages + .insert(purl.clone(), StagedSource::Fetched(fetched_holders.len())); fetched_holders.push(fetched); } PristineFetch::NoSource => { @@ -1472,16 +2281,92 @@ pub(crate) async fn vendor_records( let berry_takeover_refusal: tokio::sync::OnceCell> = tokio::sync::OnceCell::new(); let pipenv_version = tokio::sync::OnceCell::new(); + let installed_sites = vendor::pypi::InstalledSiteListings::default(); let mut dry_in_sync: u32 = 0; // Sorted, so per-package lines print in the same order every run. - let mut all_packages: Vec<(String, std::path::PathBuf)> = all_packages.into_iter().collect(); - all_packages.sort(); + // Purls are unique keys, so ordering on the purl alone is the order the + // `(purl, path)` sort produced. + let mut all_packages: Vec<(String, StagedSource)> = all_packages.into_iter().collect(); + all_packages.sort_by(|a, b| a.0.cmp(&b.0)); // Progress over the per-package engine calls (download, pack, lockfile // rewrite): shown only while an engine call runs, so every per-package // line prints on a clean line. let mut status = StatusLine::stderr(common.json, common.silent); let total = all_packages.len(); - for (index, (purl, pkg_path)) in all_packages.iter().enumerate() { + // Service downloads, fetched ahead of this serial loop (the wiring and + // every write stay here, in order). The plan is EXACT — the records the + // loop will ask the service for, in loop order, across every ecosystem: + // past the variant probe, the Bun refusal and the takeover gate below, + // and past every refusal the backend raises before its first service + // call (evaluated here with the backend's own gates — npm's + // `preflight_packages`, the others' `service_preflight`), and not + // answered by a backend's in-sync hot path or a committed-artifact + // reuse (those never ask the service). A download grant can start a + // server-side build and counts against quota, so a package the loop + // refuses is never granted on its behalf. The plan stays advisory — + // the breaker and every outcome are still decided at the loop's own + // call (see `VendorPrefetch`). Asking `wants_prefetch` first keeps the + // walk off the runs that would drop the plan anyway (`--vendor-source + // build`, `--offline`, one request at a time). + let service_prefetch = match service.filter(|cfg| !common.dry_run && cfg.wants_prefetch()) { + Some(cfg) => { + let takeover_blocked = |purl: &str| { + redirect_ledger_corrupt.is_some() + || redirect_ledger.as_ref().is_some_and(|l| { + l.records + .keys() + .any(|k| canonical_purl(k) == canonical_purl(purl)) + }) + }; + let planned = plan_service_downloads( + &common.cwd, + force, + &all_packages, + (&fetched_holders, &deferred_holders), + &variant_groups, + records, + &state, + bun_refusal.as_ref(), + &takeover_blocked, + (&pipenv_version, &installed_sites), + ) + .await; + cfg.prefetch_archives(planned) + } + None => None, + }; + // The source of the purl the loop has just left. Its archive is what a + // fetched source holds to be able to write its tree, and `all_packages` + // gives each holder to exactly one purl — so once the loop moves on, + // nothing reads it again, and a run that fetched 110 artifacts need not + // carry all 110 to the end of the loop. + let mut spent: Option> = None; + // Deferred sources whose fetch the loop has already reported. + let mut deferred_fetch_reported: HashSet = HashSet::new(); + // Group commit: from here until the loop ends, every backend's + // lockfile / manifest / config edits, the takeover's hosted reverts and + // the per-package ledger saves are captured in memory — every read in + // the loop sees them — and written to disk ONCE, after the loop (see + // `socket_patch_core::utils::group_commit`). A run that never reaches + // the commit (a crash, a panic) leaves the pre-run lockfiles and ledgers + // on disk; the artifacts it wrote are orphans the next run re-vendors + // over. The replaced uuid dirs of re-vendored packages are swept only + // after the commit, since until then the committed wiring still names + // them. Dry runs write nothing and capture nothing. + let group = (!common.dry_run + && !socket_patch_core::utils::failpoint::switched_off("group_commit")) + .then(|| GroupCommit::begin(&common.cwd)); + let mut stale_artifacts: Vec = Vec::new(); + for (index, (purl, staged)) in all_packages.iter().enumerate() { + if let Some(done) = spent.take() { + done.release(); + } + let pkg_source = staged.as_source(&fetched_holders, &deferred_holders); + let deferred = match staged { + StagedSource::Deferred(at) => Some(&deferred_holders[*at]), + _ => None, + }; + spent = Some(pkg_source); let is_variant_eco = Ecosystem::from_purl(purl).is_some_and(|e| e.supports_release_variants()); let candidates: Vec = if is_variant_eco { @@ -1514,14 +2399,57 @@ pub(crate) async fn vendor_records( // to select), so the probe is inapplicable and is skipped for it. let probe_applicable = is_variant_eco && !matches!(Ecosystem::from_purl(candidate), Some(Ecosystem::Maven)); - if probe_applicable && !force { + // A deferred source was deferred because the ledger covers the + // purl at this record's uuid: the variant the ledger vendored is + // the installed one's by construction, so it answers the probe + // without downloading the pristine tree just to read one file. + let ledger_answers_probe = deferred.is_some_and(|d| d.outcome().is_none()) + && lookup_entry(&state.entries, candidate).is_some_and(|e| e.uuid == record.uuid); + if probe_applicable && !force && !ledger_answers_probe { // The representative must be a file that MODIFIES existing // content: a new file (empty beforeHash) verifies `Ready` // against any environment, so it can neither identify nor // disqualify a variant. Same deterministic pick as apply / // core's `select_installed_variants`. let first = match representative_file(&record.files) { - Some((f, info)) => Some(verify_file_patch(pkg_path, f, info).await.status), + Some((f, info)) => match pkg_source.materialize().await { + Ok(dir) => Some(verify_file_patch(dir, f, info).await.status), + // A deferred download that produced nothing is + // reported as the eager fetch would have reported it. + Err(_) if deferred.is_some_and(|d| matches!(d.outcome(), Some(Err(_)))) => { + if let Some(Some(Err(miss))) = deferred.map(DeferredPackage::outcome) { + deferred_miss( + env, + common, + purl, + miss, + &candidates, + &mut fetch_failed, + ); + } + break; + } + // Not a variant verdict: the tree could not be + // WRITTEN at all — a full or unwritable `$TMPDIR`, + // no file descriptors. The eager fetch hit that + // while fetching and reported it; reading it as a + // variant that does not match would file the purl + // under `package_not_installed` ("no installed + // package found on disk") and lose the cause. + // One failure for the SOURCE, keyed on the purl the + // fetch was issued for — the eager fetch raised it + // once, before the variants were ever fanned out. + Err(detail) => { + env.record( + PatchEvent::new(PatchAction::Failed, purl.clone()) + .with_error("vendor_fetch_failed", detail.clone()), + ); + report_vendor_failure(common, purl, &format!("fetch failed: {detail}")); + fetch_failed.insert(purl.clone()); + fetch_failed.extend(candidates.iter().cloned()); + break; + } + }, None => None, }; if !variant_matches_installed(first.as_ref()) { @@ -1788,7 +2716,7 @@ pub(crate) async fn vendor_records( )); let outcome = dispatch_vendor_one( candidate, - pkg_path, + pkg_source, &common.cwd, record, sources, @@ -1797,10 +2725,32 @@ pub(crate) async fn vendor_records( force, service, &pipenv_version, + &installed_sites, ) .await; status.finish(); + // A deferred source whose backend needed the pristine tree after + // all fetched it inside the call. Report that fetch as the eager + // ladder did — ahead of this package's own outcome — once per + // source; a fetch that produced nothing replaces the outcome. + if let Some(fetch) = deferred.and_then(DeferredPackage::outcome) { + match fetch { + Ok(fetched) => { + if deferred_fetch_reported.insert(purl.clone()) { + record_fetched_missing(env, common, purl, &fetched.url); + } + } + Err(miss) => { + deferred_miss(env, common, purl, miss, &candidates, &mut fetch_failed); + for c in &candidates { + matched.remove(c); + } + break; + } + } + } + match outcome { None => { env.record( @@ -1923,16 +2873,80 @@ pub(crate) async fn vendor_records( if let Some(flavor) = entry.flavor.as_deref() { wired_flavors.insert(flavor.to_string()); } - has_errors |= persist_vendor_entry( + let (save_failed, stale) = record_vendor_entry( common, env, &mut state, candidate, entry, detached, record, ) .await; + has_errors |= save_failed; + socket_patch_core::utils::failpoint::hit("vendor_package_recorded"); + if let Some(stale) = stale { + if group.is_some() { + stale_artifacts.push(stale); + } else { + sweep_stale_artifact(common, env, &state, stale).await; + } + } } } } } } + // The loop is done with the service: detach the download plan (and stop + // what is still in flight), then remove whatever it staged that no + // backend claimed — a download the breaker skipped or the loop passed + // over. Never earlier: the loop's own unwinds prune empty vendor levels, + // and a concurrent removal could race them. + drop(service_prefetch); + vendor::prestage::settle().await; + + // Every backend has staged what it needed, so the fetch tempdirs can + // go. Dropping them removes whatever was extracted into them — a + // recursive delete that belongs off the runtime thread. + if !fetched_holders.is_empty() || !deferred_holders.is_empty() { + let _ = + tokio::task::spawn_blocking(move || drop((fetched_holders, deferred_holders))).await; + } + + // The run's one commit of every lockfile, manifest, config and ledger + // the loop changed. The packages that succeeded are committed even when + // others failed — a failed package's backend already put back what it + // had touched, in the captured state — so a completed run ends exactly + // where committing after every package would have left it. + if let Some(group) = group { + socket_patch_core::utils::failpoint::hit("vendor_group_commit"); + match group.commit().await { + Ok(_) => { + for stale in stale_artifacts { + sweep_stale_artifact(common, env, &state, stale).await; + } + } + Err(e) => { + has_errors = true; + let detail = if socket_patch_core::utils::group_commit::is_pending(&e) { + // Some files were replaced and could not be put back: + // the journal left behind makes the next locked command + // finish the commit. + format!( + "could not commit the vendored lockfile, manifest and ledger edits: \ + {e}; the commit is journaled and the next socket-patch command in \ + this project finishes it" + ) + } else { + format!( + "could not commit the vendored lockfile, manifest and ledger edits: \ + {e}; the project's lockfiles and .socket/vendor/state.json are \ + unchanged" + ) + }; + if !common.json { + eprintln!("Error: {detail}"); + } + env.mark_error(EnvelopeError::new("vendor_commit_failed", detail)); + } + } + } + // Manifest entries that targeted in-scope ecosystems but had no // installed package on disk (and could not be auto-fetched). let mut unmatched: Vec = vendorable @@ -1954,10 +2968,9 @@ pub(crate) async fn vendor_records( // the inventory is a local file read, allowed offline (and reused // when the fetch rung above already built it). let lock_resolvable: HashSet = if common.offline { - if inventory.is_none() { - inventory = Some(lock_inventory::inventory_project(&common.cwd).await); - } - let entries = inventory.as_deref().expect("filled just above"); + let entries = inventory + .get_or_init(|| lock_inventory::inventory_project(&common.cwd)) + .await; unmatched .iter() .filter(|p| lock_inventory::lookup(entries, p).is_some()) @@ -2548,7 +3561,7 @@ mod dispatch_tests { assert_eq!(service.source, VendorSource::Service); let outcome = dispatch_vendor_one( "pkg:maven/org.apache.logging.log4j/log4j-core@2.17.0", - tmp.path(), + tmp.path().into(), tmp.path(), &record, &sources, @@ -2557,6 +3570,7 @@ mod dispatch_tests { false, Some(&service), &tokio::sync::OnceCell::new(), + &Default::default(), ) .await; // The backend itself may refuse (nothing is installed in the @@ -2570,6 +3584,102 @@ mod dispatch_tests { } } +#[cfg(test)] +mod plan_gate_tests { + use super::*; + use socket_patch_core::manifest::schema::PatchFileInfo; + + const UUID_A: &str = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + const UUID_B: &str = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; + const UUID_C: &str = "cccccccc-cccc-4ccc-8ccc-cccccccccccc"; + + fn record(uuid: &str) -> PatchRecord { + PatchRecord { + uuid: uuid.to_string(), + exported_at: String::new(), + files: HashMap::from([( + "index.php".to_string(), + PatchFileInfo { + before_hash: "1".repeat(64), + after_hash: "2".repeat(64), + }, + )]), + vulnerabilities: HashMap::new(), + description: String::new(), + license: String::new(), + tier: String::new(), + } + } + + /// The download plan runs every record through its backend's own gate: + /// a package the backend refuses before its first service call is + /// never planned, wherever it sits in the loop order. Here the refused + /// composer package (`psr/http-message`, absent from composer.lock) + /// sorts BETWEEN the two locked ones, so a plan that skipped the gate + /// would name it at a position the prefetch reaches ahead of the loop. + #[tokio::test] + async fn the_plan_leaves_out_a_package_its_backend_refuses_first() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + std::fs::write(root.join("composer.json"), r#"{"require":{}}"#).unwrap(); + let locked: Vec = [("psr/cache", "1.0.0"), ("psr/log", "3.0.2")] + .iter() + .map(|(name, version)| { + serde_json::json!({ + "name": name, "version": version, + "dist": {"type": "zip", "url": format!("https://example.invalid/{name}.zip"), + "reference": "abc", "shasum": ""}, + "type": "library" + }) + }) + .collect(); + std::fs::write( + root.join("composer.lock"), + serde_json::to_vec_pretty(&serde_json::json!({ + "content-hash": "x", "packages": locked, "packages-dev": [] + })) + .unwrap(), + ) + .unwrap(); + let packages = [ + ("pkg:composer/psr/cache@1.0.0", "psr/cache", UUID_A), + ("pkg:composer/psr/http-message@1.1.0", "psr/http-message", UUID_B), + ("pkg:composer/psr/log@3.0.2", "psr/log", UUID_C), + ]; + let mut all_packages: Vec<(String, StagedSource)> = Vec::new(); + let mut records: HashMap = HashMap::new(); + for (purl, name, uuid) in packages { + let dir = root.join("vendor").join(name); + std::fs::create_dir_all(&dir).unwrap(); + all_packages.push((purl.to_string(), StagedSource::Installed(dir))); + records.insert(purl.to_string(), record(uuid)); + } + let planned = plan_service_downloads( + root, + false, + &all_packages, + (&[], &[]), + &HashMap::new(), + &records, + &VendorState::default(), + None, + &|_| false, + ( + &tokio::sync::OnceCell::new(), + &vendor::pypi::InstalledSiteListings::default(), + ), + ) + .await; + let uuids: Vec<&str> = planned.iter().map(|d| d.uuid.as_str()).collect(); + assert_eq!( + uuids, + vec![UUID_A, UUID_C], + "one planned download per package the loop asks the service for, in loop \ + order, and none for the package its backend refuses first" + ); + } +} + #[cfg(test)] mod warning_counting_tests { use super::*; diff --git a/crates/socket-patch-cli/src/commands/vex.rs b/crates/socket-patch-cli/src/commands/vex.rs index 720a48389..76292e535 100644 --- a/crates/socket-patch-cli/src/commands/vex.rs +++ b/crates/socket-patch-cli/src/commands/vex.rs @@ -35,7 +35,7 @@ use crate::commands::vex_sources::{ self, Plan, Sources, RECORD_MISMATCH, RECORD_UNAVAILABLE, REDIRECT_UNWIRED, VENDOR_UNWIRED, WIRING_CONFLICT, }; -use crate::ecosystem_dispatch::{collapse_to_first, find_manifest_package_copies}; +use crate::ecosystem_dispatch::{collapse_to_first, find_manifest_package_copies_reusing}; use crate::json_envelope::{Command, Envelope, EnvelopeError, PatchAction, PatchEvent, RunWarning}; use crate::ui::plural; @@ -182,6 +182,7 @@ impl VexEmbedArgs { // Embedded callers skip VEX entirely under `--dry-run`. dry_run: false, product_flag: "--vex-product", + npm_prior: None, } } } @@ -215,6 +216,14 @@ pub(crate) struct VexBuildParams { /// The flag that carried `product`, named in the non-IRI advisory /// (`--product` standalone, `--vex-product` embedded). pub product_flag: &'static str, + /// Embedded hosted `scan --vex` only: scan's npm crawl of the same + /// tree earlier in this process. The installed-copy lookups take the + /// npm `node_modules` roots (and, for the identity fallback, the + /// crawled packages) from it instead of walking the tree again; each + /// root is still searched as before, so copy choice and order are + /// unchanged. Ignored when taken with other crawler options. The + /// standalone `vex` passes `None` and walks the tree. + pub npm_prior: Option, } /// Successful result of [`generate_vex`]. @@ -333,6 +342,7 @@ pub async fn run(args: VexArgs) -> i32 { known_stale: Vec::new(), dry_run: args.common.dry_run, product_flag: "--product", + npm_prior: None, }; let manifest_path = args.common.resolved_manifest_path(); @@ -572,7 +582,9 @@ async fn generate_vex( let purls: Vec = manifest.patches.keys().cloned().collect(); // ONE installed-tree lookup: the first copy of every purl for the // record check, every copy of the hosted ones below. - let copies = find_manifest_package_copies(&purls, common, quiet).await; + let copies = + find_manifest_package_copies_reusing(&purls, common, quiet, params.npm_prior.as_ref()) + .await; let package_paths = collapse_to_first(copies.clone()); let go_patches = synthesize_go_patches(common, manifest, &plan.vendor_entries).await; // Hosted-basis purls are judged by the copies their build CONSUMES @@ -580,9 +592,13 @@ async fn generate_vex( // maven's suffixed version; every copy where hosted and registry // bytes share a location) — never by a pristine sibling the // crawler's first match may be. See `vex_consumed`. - let hosted = - crate::commands::vex_consumed::hosted_consumed_copies(common, &plan.hosted, &copies) - .await; + let hosted = crate::commands::vex_consumed::hosted_consumed_copies( + common, + &plan.hosted, + &copies, + params.npm_prior.as_ref(), + ) + .await; let vendor = VendorContext { project_root: common.cwd.clone(), entries: plan.vendor_entries.clone(), @@ -1821,3 +1837,216 @@ mod tests { } } } + +/// H3: embedded hosted `scan --vex` takes the npm roots and crawled +/// packages from scan's crawl instead of walking the tree again; the VEX +/// document is byte-identical (modulo its per-run timestamps) to the one +/// the tree walk produces. +#[cfg(test)] +mod npm_prior_tests { + use super::*; + use socket_patch_core::crawlers::CrawlerOptions; + use socket_patch_core::hash::git_sha256::compute_git_sha256_from_bytes; + use socket_patch_core::manifest::schema::{PatchFileInfo, PatchRecord, VulnerabilityInfo}; + use socket_patch_core::patch::redirect::RedirectState; + use std::collections::HashMap; + + const UUID: &str = "9f6b2c4e-1d3a-4f6b-8c2d-7e5a9b1c3d5f"; + const TOKEN: &str = "11111111-2222-4333-8444-555555555555"; + + fn put(root: &Path, rel: &str, bytes: &[u8]) { + let path = root.join(rel); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, bytes).unwrap(); + } + + fn npm_pkg(root: &Path, dir: &str, name: &str, index: &[u8]) { + put( + root, + &format!("{dir}/package.json"), + format!(r#"{{ "name": "{name}", "version": "1.3.0" }}"#).as_bytes(), + ); + put(root, &format!("{dir}/index.js"), index); + } + + /// A hosted left-pad@1.3.0 redirect (ledger record + pinned + /// package-lock wiring) installed as a root copy, a nested copy, a + /// workspace member's alias (`packages/a/node_modules/lp`) and a member + /// copy found only by its identity (`apps/web/node_modules/@me/lp`). + fn fixture(root: &Path, patched: &[u8], alias: &[u8]) { + put( + root, + "package.json", + br#"{ "name": "app", "version": "1.0.0", "dependencies": { "left-pad": "1.3.0" } }"#, + ); + npm_pkg(root, "node_modules/left-pad", "left-pad", patched); + npm_pkg( + root, + "node_modules/dep/node_modules/left-pad", + "left-pad", + patched, + ); + npm_pkg(root, "packages/a/node_modules/lp", "left-pad", alias); + npm_pkg(root, "apps/web/node_modules/@me/lp", "left-pad", patched); + let url = format!( + "https://patch.socket.dev/patch/npm/left-pad/1.3.0/{TOKEN}/{UUID}/left-pad-1.3.0.tgz" + ); + put( + root, + "package-lock.json", + serde_json::json!({ + "name": "app", "version": "1.0.0", "lockfileVersion": 3, "requires": true, + "packages": { + "": { "name": "app", "version": "1.0.0" }, + "node_modules/left-pad": { + "version": "1.3.0", + "resolved": url, + "integrity": "sha512-UEFUQ0hFRHBhdGNoZWRQQVRDSEVEcGF0Y2hlZA==", + }, + }, + }) + .to_string() + .as_bytes(), + ); + let record = PatchRecord { + uuid: UUID.to_string(), + exported_at: "2024-01-01T00:00:00Z".to_string(), + files: HashMap::from([( + "package/index.js".to_string(), + PatchFileInfo { + before_hash: "a".repeat(64), + after_hash: compute_git_sha256_from_bytes(patched), + }, + )]), + vulnerabilities: HashMap::from([( + "GHSA-rdir-1111".to_string(), + VulnerabilityInfo { + cves: vec!["CVE-2024-1".to_string()], + summary: "s".to_string(), + severity: "high".to_string(), + description: "d".to_string(), + }, + )]), + description: "p".to_string(), + license: "MIT".to_string(), + tier: "free".to_string(), + }; + let mut state = RedirectState::new(); + state + .records + .insert("pkg:npm/left-pad@1.3.0".to_string(), record); + put( + root, + ".socket/vendor/redirect-state.json", + serde_json::to_string_pretty(&state).unwrap().as_bytes(), + ); + } + + /// The document with its per-run timestamps blanked. + fn untimed(path: &Path) -> serde_json::Value { + fn strip(v: &mut serde_json::Value) { + match v { + serde_json::Value::Object(map) => { + if map.contains_key("timestamp") { + map.insert("timestamp".into(), serde_json::Value::Null); + } + map.values_mut().for_each(strip); + } + serde_json::Value::Array(items) => items.iter_mut().for_each(strip), + _ => {} + } + } + let mut doc: serde_json::Value = + serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap(); + strip(&mut doc); + doc + } + + async fn run( + common: &GlobalArgs, + out: &Path, + prior: Option, + ) -> ( + serde_json::Value, + usize, + Vec, + serde_json::Value, + ) { + let params = VexBuildParams { + output: Some(out.to_path_buf()), + product: Some("pkg:npm/app@1.0.0".into()), + no_verify: false, + doc_id: Some("urn:uuid:00000000-0000-4000-8000-000000000000".into()), + compact: false, + assume_applied: Vec::new(), + known_stale: Vec::new(), + dry_run: false, + product_flag: "--vex-product", + npm_prior: prior, + }; + let manifest_path = common.resolved_manifest_path(); + match generate_vex_from_manifest_path(common, ¶ms, &manifest_path).await { + Ok(s) => ( + untimed(out), + s.statements, + s.failed, + serde_json::to_value(&s.warnings).unwrap(), + ), + Err(e) => ( + serde_json::json!({ "error": e.code }), + 0, + Vec::new(), + serde_json::to_value(e.embedded_warnings()).unwrap(), + ), + } + } + + #[tokio::test] + async fn embedded_vex_from_the_crawl_snapshot_matches_the_tree_walk() { + let patched = &b"module.exports = 'patched'\n"[..]; + let pristine = &b"module.exports = 'pristine'\n"[..]; + for (label, alias, want_statements) in [ + ("every copy patched", patched, 1), + ("stale alias", pristine, 0), + ] { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + fixture(&root, patched, alias); + let common = GlobalArgs { + cwd: root.clone(), + json: true, + no_telemetry: true, + ..GlobalArgs::default() + }; + let options = CrawlerOptions { + cwd: root.clone(), + global: false, + global_prefix: None, + }; + let (_, _, _, snapshot) = + crate::ecosystem_dispatch::crawl_ecosystems_with_npm(&options, None).await; + let snapshot = snapshot.expect("npm crawled"); + + let walked = run(&common, &tmp.path().join("walked.json"), None).await; + let reused = run( + &common, + &tmp.path().join("reused.json"), + Some(snapshot.clone()), + ) + .await; + assert_eq!(walked, reused, "{label}"); + assert_eq!(walked.1, want_statements, "{label}: {:?}", walked); + + // A snapshot taken with other crawler options is ignored. + let other = CrawlerOptions { + cwd: tmp.path().to_path_buf(), + global: false, + global_prefix: None, + }; + let (_, _, _, foreign) = + crate::ecosystem_dispatch::crawl_ecosystems_with_npm(&other, None).await; + let ignored = run(&common, &tmp.path().join("ignored.json"), foreign).await; + assert_eq!(walked, ignored, "{label} (foreign snapshot)"); + } + } +} diff --git a/crates/socket-patch-cli/src/commands/vex_consumed.rs b/crates/socket-patch-cli/src/commands/vex_consumed.rs index b8319f4e9..b38a056ae 100644 --- a/crates/socket-patch-cli/src/commands/vex_consumed.rs +++ b/crates/socket-patch-cli/src/commands/vex_consumed.rs @@ -52,18 +52,24 @@ use socket_patch_core::vex::HostedCopies; use crate::args::GlobalArgs; use crate::commands::vex_sources::HostedWiring; -use crate::ecosystem_dispatch::{npm_paths_by_identity, partition_purls}; +use crate::ecosystem_dispatch::{ + npm_paths_by_identity, npm_paths_by_identity_in, partition_purls, NpmCrawlSnapshot, +}; /// Resolve [`HostedCopies`] for every hosted-basis purl of `hosted` (see the /// module docs), under the same crawler options and `--ecosystems` scope as /// the installed-tree lookup. `installed` is that lookup's every-copy /// result ([`crate::ecosystem_dispatch::find_manifest_package_copies`] over /// the record view, which holds every hosted purl): the shared-location -/// ecosystems read it instead of crawling the tree a second time. +/// ecosystems read it instead of crawling the tree a second time. `prior` +/// (embedded hosted `scan --vex` only) is scan's npm crawl of the same +/// tree: the alias walk takes its `node_modules` roots and the identity +/// fallback its packages instead of walking the tree again. pub(crate) async fn hosted_consumed_copies( common: &GlobalArgs, hosted: &BTreeMap, installed: &HashMap>, + prior: Option<&NpmCrawlSnapshot>, ) -> HashMap { let mut out = HashMap::new(); if hosted.is_empty() { @@ -90,10 +96,17 @@ pub(crate) async fn hosted_consumed_copies( .filter_map(|purl| Some((purl.clone(), installed.get(purl)?.clone()))) .collect(); let mut aliases = match shared.get(&Ecosystem::Npm) { - Some(npm) => npm_alias_copies(&options, npm).await, + Some(npm) => npm_alias_copies_reusing(&options, npm, prior).await, None => HashMap::new(), }; - npm_identity_fallback(shared.get(&Ecosystem::Npm), &options, &mut all, &aliases).await; + npm_identity_fallback_reusing( + shared.get(&Ecosystem::Npm), + &options, + &mut all, + &aliases, + prior, + ) + .await; for purl in shared.values().flatten() { let mut paths = all.remove(purl).unwrap_or_default(); paths.extend(aliases.remove(purl).unwrap_or_default()); @@ -158,9 +171,22 @@ const ALIAS_WALK_MAX_DIRS: usize = 200_000; /// traversed. A plain `--global` run is not walked: its roots come from /// spawning every package manager again, and the identity fallback covers /// an alias that is the only global copy. +#[cfg(test)] async fn npm_alias_copies( options: &CrawlerOptions, purls: &[String], +) -> HashMap> { + npm_alias_copies_reusing(options, purls, None).await +} + +/// [`npm_alias_copies`], taking the importer `node_modules` roots from +/// `prior` when it was crawled with `options` (the same roots +/// `NpmCrawler::get_node_modules_paths` returns) instead of walking the tree +/// for them; the per-root BFS below is unchanged. +async fn npm_alias_copies_reusing( + options: &CrawlerOptions, + purls: &[String], + prior: Option<&NpmCrawlSnapshot>, ) -> HashMap> { let wanted: HashMap<(String, String), &String> = purls .iter() @@ -176,10 +202,13 @@ async fn npm_alias_copies( if options.global && options.global_prefix.is_none() { return out; } - let roots = NpmCrawler::new() - .get_node_modules_paths(options) - .await - .unwrap_or_default(); + let roots = match prior.and_then(|p| p.roots_for(options)) { + Some(roots) => roots.to_vec(), + None => NpmCrawler::new() + .get_node_modules_paths(options) + .await + .unwrap_or_default(), + }; let mut queue = std::collections::VecDeque::from(roots); let mut visited = 0usize; while let Some(nm) = queue.pop_front() { @@ -251,11 +280,25 @@ async fn real_subdirs(dir: &Path) -> Vec<(PathBuf, String)> { /// hash-verified ("installed evidence wins"). Resolve every npm purl the /// targeted lookup missed by the installed `package.json` identity instead /// — the same fallback `vendor` uses before declaring a package missing. +#[cfg(test)] async fn npm_identity_fallback( npm: Option<&Vec>, options: &CrawlerOptions, all: &mut HashMap>, aliases: &HashMap>, +) { + npm_identity_fallback_reusing(npm, options, all, aliases, None).await +} + +/// [`npm_identity_fallback`], answering from `prior`'s crawled packages +/// when it was crawled with `options` (the whole `NpmCrawler::crawl_all` +/// output for them) instead of crawling again. +async fn npm_identity_fallback_reusing( + npm: Option<&Vec>, + options: &CrawlerOptions, + all: &mut HashMap>, + aliases: &HashMap>, + prior: Option<&NpmCrawlSnapshot>, ) { let missing: Vec<&String> = npm .into_iter() @@ -264,7 +307,10 @@ async fn npm_identity_fallback( all.get(*purl).is_none_or(Vec::is_empty) && aliases.get(*purl).is_none_or(Vec::is_empty) }) .collect(); - all.extend(npm_paths_by_identity(options, &missing).await); + match prior.and_then(|p| p.packages_for(options)) { + Some(installed) => all.extend(npm_paths_by_identity_in(installed, &missing)), + None => all.extend(npm_paths_by_identity(options, &missing).await), + } } // ── golang ─────────────────────────────────────────────────────────────── @@ -590,6 +636,70 @@ mod tests { assert!(!all.contains_key(&purls[0]), "{all:?}"); } + /// H3: the identity fallback answered from the crawl snapshot finds the + /// same copies as crawling again — here an alias installed through a + /// symlink (yarn's pnpm linker, `npm link`), which neither the targeted + /// lookup nor the alias walk (it skips symlinks) finds, among other + /// crawled packages so it is not the snapshot's first entry. + #[cfg(unix)] + #[tokio::test] + async fn npm_identity_fallback_from_the_snapshot_matches_the_crawl() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + write_pkg(root, "node_modules/aaa", "aaa", "1.0.0"); + write_pkg(root, "node_modules/left-pad", "left-pad", "1.2.0"); + write_pkg(root, "node_modules/zzz", "zzz", "1.0.0"); + // Two such packages, so an answer drawn from only part of the + // snapshot (whatever its directory order) cannot match. + for (store, link, name) in [ + ("store/a", "node_modules/lp", "left-pad"), + ("store/b", "node_modules/odd", "is-odd"), + ] { + write_pkg(root, store, name, "1.3.0"); + std::os::unix::fs::symlink(root.join(store), root.join(link)).unwrap(); + } + let options = local(root); + let purls = vec![ + "pkg:npm/left-pad@1.3.0".to_string(), + "pkg:npm/is-odd@1.3.0".to_string(), + "pkg:npm/absent@2.0.0".to_string(), + ]; + let aliases = npm_alias_copies(&options, &purls).await; + assert!( + aliases.is_empty(), + "the alias walk skips symlinks: {aliases:?}" + ); + + let (_, _, _, snapshot) = + crate::ecosystem_dispatch::crawl_ecosystems_with_npm(&options, None).await; + let snapshot = snapshot.expect("npm crawled"); + assert!( + snapshot.packages_for(&options).is_some_and(|p| p.len() > 1), + "several crawled packages" + ); + + let mut walked: HashMap> = HashMap::new(); + npm_identity_fallback(Some(&purls), &options, &mut walked, &aliases).await; + let mut reused: HashMap> = HashMap::new(); + npm_identity_fallback_reusing( + Some(&purls), + &options, + &mut reused, + &aliases, + Some(&snapshot), + ) + .await; + assert_eq!(reused, walked); + for purl in &purls[..2] { + assert_eq!( + reused.get(purl).map(Vec::len), + Some(1), + "{purl}: {reused:?}" + ); + } + assert!(!reused.contains_key("pkg:npm/absent@2.0.0"), "{reused:?}"); + } + /// Only dirs whose install key differs from the package name are /// aliases; scoped keys, nested trees and scoped targets are walked; /// hidden dirs, other versions and symlinks are not copies. diff --git a/crates/socket-patch-cli/src/commands/vex_sources.rs b/crates/socket-patch-cli/src/commands/vex_sources.rs index 77b69e3c0..52d95f225 100644 --- a/crates/socket-patch-cli/src/commands/vex_sources.rs +++ b/crates/socket-patch-cli/src/commands/vex_sources.rs @@ -69,11 +69,15 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use futures_util::StreamExt; + use socket_patch_core::api::client::{ - build_proxy_fallback_client, get_api_client_with_overrides, is_fallback_candidate, + build_proxy_fallback_client, get_api_client_with_overrides, hold_back_debug, + is_fallback_candidate, }; use socket_patch_core::manifest::schema::{PatchManifest, PatchRecord}; use socket_patch_core::patch::redirect::RedirectState; +use socket_patch_core::utils::concurrent::{api_concurrency, ordered_concurrent}; use socket_patch_core::utils::purl::strip_purl_qualifiers; use socket_patch_core::vendor::state::{lookup_entry_kv, VendorArtifact, VendorEntry, VendorState}; use socket_patch_core::vex::discover::{ @@ -103,10 +107,22 @@ pub(crate) const REDIRECT_UNWIRED: &str = "redirect_unwired"; /// candidate for that package attests. pub(crate) const WIRING_CONFLICT: &str = "wiring_conflict"; -/// Bound on concurrent patch-view fetches (one GET per uuid; the view -/// carries blob content, so it is heavy) — the batch fallback's limit. +/// Ceiling on concurrent patch-view fetches (one GET per uuid; the view +/// carries blob content, so it is heavy). const FETCH_CONCURRENCY: usize = 10; +/// The in-flight cap for the record fetch. +/// +/// These are patch-API requests — `scan --vex` makes them too — so the +/// documented escape hatch has to reach them like it reaches every other +/// window: an operator behind something that caps in-flight requests per +/// client sets `SOCKET_API_CONCURRENCY=1` and gets one view at a time. +/// [`FETCH_CONCURRENCY`] is this window's own ceiling on top of that, for +/// the size of a view. +fn fetch_concurrency(use_public_proxy: bool) -> usize { + api_concurrency(use_public_proxy).min(FETCH_CONCURRENCY) +} + /// Everything `vex` reads, loaded once by the caller (which owns the /// corrupt-ledger hard errors). pub(crate) struct Sources { @@ -902,30 +918,38 @@ async fn fetch_records( loop { let mut auth_refused: Vec = Vec::new(); let mut auth_error: Option = None; - for chunk in pending.chunks(FETCH_CONCURRENCY) { - status.set(format!( - "Fetching {}... ({done}/{total})", - if total == 1 { - "the patch record" - } else { - "patch records" - } + // A sliding window of at most FETCH_CONCURRENCY views in flight + // (it used to wait for each whole chunk of that size to drain + // before starting the next), consumed in `pending` order. + // + // That IS an observable change, the one in this area: the chunked + // JoinSet folded each chunk in COMPLETION order, so which refusal + // was reported as `auth_error` (printed in the fallback note) and + // the order of the retried `pending` list were a race. They now + // follow `pending` order — deterministic, and the same order the + // notes above already came out in. + { + let client = &client; + let mut views = std::pin::pin!(ordered_concurrent( + pending.iter(), + fetch_concurrency(use_public_proxy), + |uuid| async move { (uuid, hold_back_debug(client.fetch_patch(uuid)).await) }, )); - let mut set = tokio::task::JoinSet::new(); - for uuid in chunk { - let client = client.clone(); - let uuid = uuid.clone(); - set.spawn(async move { - let result = client.fetch_patch(&uuid).await; - (uuid, result) - }); - } - while let Some(joined) = set.join_next().await { - let Ok((uuid, result)) = joined else { - continue; + loop { + status.set(format!( + "Fetching {}... ({done}/{total})", + if total == 1 { + "the patch record" + } else { + "patch records" + } + )); + let Some((uuid, result)) = views.next().await else { + break; }; + let uuid = uuid.clone(); done += 1; - match result { + match result.release() { Ok(Some(view)) => { out.insert( uuid, @@ -1028,6 +1052,102 @@ mod tests { } } + /// `scan --vex` reaches this window, so the documented escape hatch + /// has to reach it too: `SOCKET_API_CONCURRENCY=1` means one view at a + /// time here as well. Serial: `SOCKET_*` is process-global. + #[test] + #[serial_test::serial] + fn socket_api_concurrency_paces_the_record_fetch() { + use socket_patch_core::utils::concurrent::API_CONCURRENCY_ENV; + let orig = std::env::var(API_CONCURRENCY_ENV).ok(); + std::env::remove_var(API_CONCURRENCY_ENV); + // A view is heavy, so this window's own ceiling binds by default + // on the authenticated API (whose cap is above it). + assert_eq!(fetch_concurrency(false), FETCH_CONCURRENCY); + assert_eq!(fetch_concurrency(true), 4); + + std::env::set_var(API_CONCURRENCY_ENV, "1"); + assert_eq!(fetch_concurrency(false), 1); + assert_eq!(fetch_concurrency(true), 1); + + // Turned up past this window's own ceiling, the ceiling holds. + std::env::set_var(API_CONCURRENCY_ENV, "32"); + assert_eq!(fetch_concurrency(false), FETCH_CONCURRENCY); + + match orig { + Some(v) => std::env::set_var(API_CONCURRENCY_ENV, v), + None => std::env::remove_var(API_CONCURRENCY_ENV), + } + } + + /// The record fetch folds in `pending` order, not in the order the + /// server happens to answer: the FIRST refusal in `pending` is the one + /// reported in the fallback note, even when it answers last, and the + /// refused uuids are retried against the proxy in that same order. + /// (The chunked JoinSet this replaced folded by completion, so which + /// refusal was reported was a race.) + #[tokio::test] + #[serial_test::serial] + async fn refused_records_fold_in_pending_order_not_completion_order() { + use wiremock::matchers::{method, path as wm_path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let api = MockServer::start().await; + let proxy = MockServer::start().await; + // U1 is first in `pending` and answers LAST; the two refusals are + // different statuses, so the reported one is identifiable. + for (uuid, status, delay) in [(U1, 401, 300u64), (U2, 403, 0)] { + Mock::given(method("GET")) + .and(wm_path(format!("/v0/orgs/acme/patches/view/{uuid}"))) + .respond_with( + ResponseTemplate::new(status) + .set_delay(std::time::Duration::from_millis(delay)), + ) + .mount(&api) + .await; + // The proxy retry serves both, so the run still ends with both + // records: only the ORDER of the refusal report is at stake. + Mock::given(method("GET")) + .and(wm_path(format!("/patch/view/{uuid}"))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "uuid": uuid, + "purl": "pkg:npm/vexorder@1.0.0", + "publishedAt": "2026-01-01T00:00:00Z", + "files": {}, + "vulnerabilities": {}, + "description": "", + "license": "MIT", + "tier": "free", + }))) + .mount(&proxy) + .await; + } + + let tmp = tempfile::tempdir().unwrap(); + let common = GlobalArgs { + cwd: tmp.path().to_path_buf(), + json: true, + api_url: Some(api.uri()), + api_token: Some("sktsec_placeholder_value_for_tests_api".into()), + org: Some("acme".into()), + proxy_url: Some(proxy.uri()), + ..GlobalArgs::default() + }; + let mut notes: Vec = Vec::new(); + let out = fetch_records(&common, &[U1.to_string(), U2.to_string()], &mut notes).await; + + let fallback = notes + .iter() + .find(|n| n.code == NOTE_API_AUTH_FALLBACK) + .unwrap_or_else(|| panic!("no fallback note: {notes:?}")); + assert!( + fallback.detail.contains("Unauthorized") && !fallback.detail.contains("Forbidden"), + "the first refusal in `pending` order must be the reported one: {}", + fallback.detail + ); + assert!(out.contains_key(U1) && out.contains_key(U2), "{out:?}"); + } + fn discovery(refs: Vec) -> Discovery { let mut d = Discovery::default(); for r in refs { diff --git a/crates/socket-patch-cli/src/ecosystem_dispatch.rs b/crates/socket-patch-cli/src/ecosystem_dispatch.rs index da32594c8..999823180 100644 --- a/crates/socket-patch-cli/src/ecosystem_dispatch.rs +++ b/crates/socket-patch-cli/src/ecosystem_dispatch.rs @@ -7,6 +7,7 @@ use std::path::PathBuf; use crate::args::GlobalArgs; +use socket_patch_core::crawlers::walk_pool; use socket_patch_core::crawlers::CargoCrawler; use socket_patch_core::crawlers::ComposerCrawler; use socket_patch_core::crawlers::DenoCrawler; @@ -243,11 +244,16 @@ fn passthrough_purls(purls: &[String]) -> Vec { /// inserts the crawler-returned PURL with first-wins semantics. It is /// applied to the release-variant ecosystems (PyPI / RubyGems / Maven), /// which are also queried with deduped base PURLs. +/// +/// `npm_roots`, when given, are the `node_modules` roots an earlier crawl of +/// the same options and (untouched) tree walked — used instead of walking +/// the tree for them again ([`NpmRootsCrawler`]). async fn dispatch_find( partitioned: &HashMap>, options: &CrawlerOptions, silent: bool, variant_merge: MergeFn, + npm_roots: Option<&[PathBuf]>, ) -> HashMap> { let mut out: HashMap> = HashMap::new(); @@ -257,7 +263,7 @@ async fn dispatch_find( eco = Ecosystem::Npm, options = options, silent = silent, - crawler = NpmCrawler, + crawler = NpmRootsCrawler { roots: npm_roots }, get_paths = get_node_modules_paths, using_label = "global npm packages", err_label = "npm packages", @@ -418,7 +424,7 @@ pub async fn find_all_packages_for_purls( // `merge_qualified`'s `push_path`. Single-copy ecosystems keep true // first-wins via their own `merge_first_wins` wiring in // `dispatch_find`. - dispatch_find(partitioned, options, silent, merge_variant_copies).await + dispatch_find(partitioned, options, silent, merge_variant_copies, None).await } /// Multi-copy variant of `find_packages_for_rollback` (qualified-aware @@ -428,7 +434,7 @@ pub async fn find_all_packages_for_rollback( options: &CrawlerOptions, silent: bool, ) -> HashMap> { - dispatch_find(partitioned, options, silent, merge_qualified).await + dispatch_find(partitioned, options, silent, merge_qualified, None).await } /// Qualified-aware PURL resolution for rollback, vendor, repair and @@ -445,7 +451,94 @@ pub async fn find_packages_for_rollback( options: &CrawlerOptions, silent: bool, ) -> HashMap { - collapse_to_first(find_all_packages_for_rollback(partitioned, options, silent).await) + find_packages_for_rollback_reusing(partitioned, options, silent, None).await +} + +/// [`find_packages_for_rollback`], taking the npm `node_modules` roots from +/// `prior` (a crawl of the same options earlier in this process, over a +/// tree nothing has touched since) instead of walking the tree for them +/// again. Only the root discovery is reused: each root is still searched +/// by `find_by_purls`, so copy choice and order are unchanged. A snapshot +/// taken with other options is ignored. +pub async fn find_packages_for_rollback_reusing( + partitioned: &HashMap>, + options: &CrawlerOptions, + silent: bool, + prior: Option<&NpmCrawlSnapshot>, +) -> HashMap { + let npm_roots = prior + .filter(|p| p.taken_with(options)) + .map(|p| p.roots.as_slice()); + collapse_to_first(dispatch_find(partitioned, options, silent, merge_qualified, npm_roots).await) +} + +/// The npm half of one [`crawl_ecosystems_with_npm`] run: the packages +/// the npm crawler found (its whole output, in crawl order) and the +/// `node_modules` roots it walked, with the options they were taken with. +/// Handed from `scan`'s crawl to its vendor step so the vendor engine does +/// not walk the same untouched tree again ([`npm_paths_by_identity_in`], +/// [`find_packages_for_rollback_reusing`]). +#[derive(Debug, Clone)] +pub struct NpmCrawlSnapshot { + cwd: PathBuf, + global: bool, + global_prefix: Option, + roots: Vec, + packages: Vec, +} + +impl NpmCrawlSnapshot { + /// Whether this snapshot was crawled with exactly `options`. + fn taken_with(&self, options: &CrawlerOptions) -> bool { + // Destructured, not field-by-field: a new crawler option that + // changes what the crawler walks has to be answered here, and the + // compiler is what asks. Everything this snapshot stands in for + // was crawled with these options and nothing else. + let CrawlerOptions { + cwd, + global, + global_prefix, + } = options; + self.cwd == *cwd && self.global == *global && self.global_prefix == *global_prefix + } + + /// The crawled npm packages, when crawled with exactly `options`. + pub(crate) fn packages_for(&self, options: &CrawlerOptions) -> Option<&[CrawledPackage]> { + self.taken_with(options).then_some(self.packages.as_slice()) + } + + /// The `node_modules` roots the crawl walked — exactly what + /// `NpmCrawler::get_node_modules_paths` returns for the same options and + /// tree — when crawled with exactly `options`. + pub(crate) fn roots_for(&self, options: &CrawlerOptions) -> Option<&[PathBuf]> { + self.taken_with(options).then_some(self.roots.as_slice()) + } +} + +/// [`NpmCrawler`] for [`dispatch_find`], answering its root discovery from +/// an earlier crawl's roots when it has them. +struct NpmRootsCrawler<'a> { + roots: Option<&'a [PathBuf]>, +} + +impl NpmRootsCrawler<'_> { + async fn get_node_modules_paths( + &self, + options: &CrawlerOptions, + ) -> Result, std::io::Error> { + match self.roots { + Some(roots) => Ok(roots.to_vec()), + None => NpmCrawler.get_node_modules_paths(options).await, + } + } + + async fn find_by_purls( + &self, + node_modules_path: &std::path::Path, + purls: &[String], + ) -> Result>, std::io::Error> { + NpmCrawler.find_by_purls(node_modules_path, purls).await + } } /// The installed copy of each npm purl in `purls`, found by its @@ -463,11 +556,21 @@ pub(crate) async fn npm_paths_by_identity( options: &CrawlerOptions, purls: &[&String], ) -> HashMap> { - let mut out = HashMap::new(); if purls.is_empty() { - return out; + return HashMap::new(); } let installed = NpmCrawler::new().crawl_all(options).await; + npm_paths_by_identity_in(&installed, purls) +} + +/// [`npm_paths_by_identity`] over an npm crawl already in hand (the whole +/// output of `NpmCrawler::crawl_all` for the same options, over a tree +/// nothing has touched since) instead of crawling again. +pub(crate) fn npm_paths_by_identity_in( + installed: &[CrawledPackage], + purls: &[&String], +) -> HashMap> { + let mut out = HashMap::new(); for purl in purls { let want = canonical_purl(purl); let paths: Vec = installed @@ -511,6 +614,22 @@ pub async fn find_manifest_package_copies( purls: &[String], common: &GlobalArgs, quiet: bool, +) -> HashMap> { + find_manifest_package_copies_reusing(purls, common, quiet, None).await +} + +/// [`find_manifest_package_copies`], taking the npm `node_modules` roots +/// from `prior` (a crawl of the same options earlier in this process, over +/// a tree whose directories nothing has touched since) instead of walking +/// the tree for them again — the every-copy twin of +/// [`find_packages_for_rollback_reusing`]. Only the root discovery is +/// reused: each root is still searched by `find_by_purls`, so copy choice +/// and order are unchanged. A snapshot taken with other options is ignored. +pub async fn find_manifest_package_copies_reusing( + purls: &[String], + common: &GlobalArgs, + quiet: bool, + prior: Option<&NpmCrawlSnapshot>, ) -> HashMap> { let partitioned = partition_purls(purls, common.ecosystems.as_deref()); let crawler_options = CrawlerOptions { @@ -518,7 +637,25 @@ pub async fn find_manifest_package_copies( global: common.global, global_prefix: common.global_prefix.clone(), }; - find_all_packages_for_rollback(&partitioned, &crawler_options, quiet).await + let npm_roots = prior.and_then(|p| p.roots_for(&crawler_options)); + dispatch_find( + &partitioned, + &crawler_options, + quiet, + merge_qualified, + npm_roots, + ) + .await +} + +/// Box the future `make` returns, constructing it inside this (non-async) +/// frame so the caller's poll frame only ever holds the pointer. +fn boxed<'a, T, F, Fut>(make: F) -> std::pin::Pin + 'a>> +where + F: FnOnce() -> Fut, + Fut: std::future::Future + 'a, +{ + Box::pin(make()) } /// Crawl all ecosystems and return all packages, per-ecosystem counts and @@ -533,31 +670,174 @@ pub async fn crawl_all_ecosystems( HashMap, Option, ) { - let mut all_packages = Vec::new(); - let mut counts: HashMap = HashMap::new(); + crawl_ecosystems(options, None).await +} + +/// [`crawl_all_ecosystems`] over only the ecosystems `only` names +/// (`--ecosystems` spellings; `None` crawls every one). A crawler that is +/// not selected never runs: it contributes no packages and no `counts` +/// entry. Each crawler reports only its own ecosystem's purls, so the +/// selected ecosystems' packages, their order and their counts are exactly +/// the full crawl's. +pub async fn crawl_ecosystems( + options: &CrawlerOptions, + only: Option<&[String]>, +) -> ( + Vec, + HashMap, + Option, +) { + let (packages, counts, skipped_config_path, _) = crawl_every_ecosystem(options, only).await; + (packages, counts, skipped_config_path) +} + +/// [`crawl_all_ecosystems`], also handing back the npm half of the crawl as +/// an [`NpmCrawlSnapshot`] (its packages are the leading `counts[Npm]` +/// entries of the package list). +#[cfg(test)] +pub async fn crawl_all_ecosystems_with_npm( + options: &CrawlerOptions, +) -> ( + Vec, + HashMap, + Option, + NpmCrawlSnapshot, +) { + let (packages, counts, skipped_config_path, snapshot) = + crawl_ecosystems_with_npm(options, None).await; + let snapshot = snapshot.expect("a crawl of every ecosystem crawls npm"); + (packages, counts, skipped_config_path, snapshot) +} + +/// [`crawl_ecosystems`], also handing back the npm half of the crawl as an +/// [`NpmCrawlSnapshot`] — `None` when `only` leaves npm out, so nothing +/// mistakes the skipped crawl for an empty `node_modules`. +pub async fn crawl_ecosystems_with_npm( + options: &CrawlerOptions, + only: Option<&[String]>, +) -> ( + Vec, + HashMap, + Option, + Option, +) { + let (packages, counts, skipped_config_path, npm_roots) = + crawl_every_ecosystem(options, only).await; + let snapshot = counts + .get(&Ecosystem::Npm) + .map(|&npm_count| NpmCrawlSnapshot { + cwd: options.cwd.clone(), + global: options.global, + global_prefix: options.global_prefix.clone(), + roots: npm_roots, + packages: packages[..npm_count].to_vec(), + }); + (packages, counts, skipped_config_path, snapshot) +} +/// Whether a crawl limited to `only` visits `eco` (`None`: every one). +fn crawl_selects(only: Option<&[String]>, eco: Ecosystem) -> bool { + only.is_none_or(|list| list.iter().any(|name| name == eco.cli_name())) +} + +/// The crawl behind the entry points above; the fourth element is the npm +/// crawler's `node_modules` roots (empty when npm was not selected). +async fn crawl_every_ecosystem( + options: &CrawlerOptions, + only: Option<&[String]>, +) -> ( + Vec, + HashMap, + Option, + Vec, +) { + // The nine crawlers are independent (none prints, none mutates shared + // state), so they run concurrently; their blocking walks and + // subprocesses sit on the blocking pool. Results are consumed in the + // fixed order below, so packages and counts are exactly the serial + // run's. Each future is heap-allocated through `boxed` (constructed + // in that helper's frame) so joining nine does not grow the caller's + // poll frame by their combined size. Under a tight descriptor limit + // they run one at a time instead, keeping the serial run's descriptor + // profile (see `walk_pool`): a crawler treats a failed open as an + // absent dir, so extra concurrent descriptors could silently drop + // packages there. A crawler `only` leaves out resolves to its empty + // result without running. macro_rules! crawl { - ($eco:expr, $crawler:expr) => {{ - let pkgs = $crawler.crawl_all(options).await; - counts.insert($eco, pkgs.len()); - all_packages.extend(pkgs); - }}; + ($eco:expr, $crawl:expr) => { + boxed(move || async move { + if crawl_selects(only, $eco) { + Some($crawl.await) + } else { + None + } + }) + }; } + let (npm, pypi, cargo, gems, golang, maven, composer, nuget, deno) = + if walk_pool::fd_limit_is_tight() { + ( + crawl!(Ecosystem::Npm, NpmCrawler.crawl_all_with_roots(options)).await, + crawl!(Ecosystem::Pypi, PythonCrawler.crawl_all(options)).await, + crawl!(Ecosystem::Cargo, CargoCrawler.crawl_all(options)).await, + crawl!( + Ecosystem::Gem, + RubyCrawler.crawl_all_with_discovery(options) + ) + .await, + crawl!(Ecosystem::Golang, GoCrawler.crawl_all(options)).await, + crawl!(Ecosystem::Maven, MavenCrawler.crawl_all(options)).await, + crawl!(Ecosystem::Composer, ComposerCrawler.crawl_all(options)).await, + crawl!(Ecosystem::Nuget, NuGetCrawler.crawl_all(options)).await, + crawl!(Ecosystem::Deno, DenoCrawler.crawl_all(options)).await, + ) + } else { + tokio::join!( + crawl!(Ecosystem::Npm, NpmCrawler.crawl_all_with_roots(options)), + crawl!(Ecosystem::Pypi, PythonCrawler.crawl_all(options)), + crawl!(Ecosystem::Cargo, CargoCrawler.crawl_all(options)), + crawl!( + Ecosystem::Gem, + RubyCrawler.crawl_all_with_discovery(options) + ), + crawl!(Ecosystem::Golang, GoCrawler.crawl_all(options)), + crawl!(Ecosystem::Maven, MavenCrawler.crawl_all(options)), + crawl!(Ecosystem::Composer, ComposerCrawler.crawl_all(options)), + crawl!(Ecosystem::Nuget, NuGetCrawler.crawl_all(options)), + crawl!(Ecosystem::Deno, DenoCrawler.crawl_all(options)), + ) + }; + let (npm, npm_roots) = match npm { + Some((packages, roots)) => (Some(packages), roots), + None => (None, Vec::new()), + }; + let (gems, gem_discovery) = match gems { + Some((packages, discovery)) => (Some(packages), discovery), + None => (None, None), + }; - crawl!(Ecosystem::Npm, NpmCrawler); - crawl!(Ecosystem::Pypi, PythonCrawler); - crawl!(Ecosystem::Cargo, CargoCrawler); - let (gems, gem_discovery) = RubyCrawler.crawl_all_with_discovery(options).await; - counts.insert(Ecosystem::Gem, gems.len()); - all_packages.extend(gems); - crawl!(Ecosystem::Golang, GoCrawler); - crawl!(Ecosystem::Maven, MavenCrawler); - crawl!(Ecosystem::Composer, ComposerCrawler); - crawl!(Ecosystem::Nuget, NuGetCrawler); - crawl!(Ecosystem::Deno, DenoCrawler); + let mut all_packages = Vec::new(); + let mut counts: HashMap = HashMap::new(); + for (eco, pkgs) in [ + (Ecosystem::Npm, npm), + (Ecosystem::Pypi, pypi), + (Ecosystem::Cargo, cargo), + (Ecosystem::Gem, gems), + (Ecosystem::Golang, golang), + (Ecosystem::Maven, maven), + (Ecosystem::Composer, composer), + (Ecosystem::Nuget, nuget), + (Ecosystem::Deno, deno), + ] { + let Some(pkgs) = pkgs else { + continue; + }; + counts.insert(eco, pkgs.len()); + all_packages.extend(pkgs); + } let skipped_config_path = gem_discovery.and_then(|d| d.skipped_config_path); - (all_packages, counts, skipped_config_path) + (all_packages, counts, skipped_config_path, npm_roots) } #[cfg(test)] @@ -1337,6 +1617,321 @@ mod tests { } } + /// The vendor engine's reuse of scan's npm crawl is an oracle-equal + /// substitute: over one tree (a hoisted dep, a nested duplicate, an + /// alias install, a workspace member's own `node_modules`), the + /// snapshot's roots and packages equal what the engine's own discovery + /// and identity crawl find, and both lookups built on it answer + /// exactly as the crawling ones do. A snapshot taken with other + /// options is never used. + #[tokio::test] + async fn npm_crawl_snapshot_matches_the_crawls_it_replaces() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + let write = |dir: &std::path::Path, name: &str, version: &str| { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write( + dir.join("package.json"), + format!(r#"{{"name":"{name}","version":"{version}"}}"#), + ) + .unwrap(); + }; + std::fs::write( + root.join("package.json"), + r#"{"name":"root","version":"1.0.0","workspaces":["packages/*"]}"#, + ) + .unwrap(); + write(&root.join("node_modules/foo"), "foo", "1.0.0"); + write(&root.join("node_modules/bar"), "bar", "2.0.0"); + write( + &root.join("node_modules/bar/node_modules/foo"), + "foo", + "0.9.0", + ); + write(&root.join("node_modules/@s/qux"), "@s/qux", "4.0.0"); + // `"lp": "npm:left-pad@1.3.0"` installs under the alias key. + write(&root.join("node_modules/lp"), "left-pad", "1.3.0"); + write(&root.join("packages/app"), "app", "0.1.0"); + write(&root.join("packages/app/node_modules/baz"), "baz", "3.0.0"); + write(&root.join("packages/app/node_modules/foo"), "foo", "0.9.0"); + let options = local_options(root.to_path_buf()); + + let (packages, counts, _, snapshot) = crawl_all_ecosystems_with_npm(&options).await; + let npm_count = counts[&Ecosystem::Npm]; + let pairs = |pkgs: &[CrawledPackage]| -> Vec<(String, PathBuf)> { + pkgs.iter() + .map(|p| (p.purl.clone(), p.path.clone())) + .collect() + }; + assert_eq!( + snapshot.roots, + NpmCrawler.get_node_modules_paths(&options).await.unwrap() + ); + assert_eq!( + pairs(snapshot.packages_for(&options).unwrap()), + pairs(&NpmCrawler.crawl_all(&options).await) + ); + assert_eq!(pairs(&snapshot.packages), pairs(&packages[..npm_count])); + assert!(snapshot.roots.len() >= 2, "roots={:?}", snapshot.roots); + + let purls: Vec = [ + "pkg:npm/foo@1.0.0", + "pkg:npm/foo@0.9.0", + "pkg:npm/bar@2.0.0", + "pkg:npm/baz@3.0.0", + "pkg:npm/%40s/qux@4.0.0", + "pkg:npm/left-pad@1.3.0", + "pkg:npm/absent@9.9.9", + ] + .map(String::from) + .to_vec(); + let partitioned = partition_purls(&purls, None); + let crawled = find_packages_for_rollback(&partitioned, &options, true).await; + let reused = + find_packages_for_rollback_reusing(&partitioned, &options, true, Some(&snapshot)).await; + assert_eq!(reused, crawled); + assert!(crawled.contains_key("pkg:npm/baz@3.0.0"), "{crawled:?}"); + + let missing: Vec<&String> = purls.iter().filter(|p| !crawled.contains_key(*p)).collect(); + assert!( + missing.iter().any(|p| p.contains("left-pad")), + "{missing:?}" + ); + let by_crawl = npm_paths_by_identity(&options, &missing).await; + let by_snapshot = + npm_paths_by_identity_in(snapshot.packages_for(&options).unwrap(), &missing); + assert_eq!(by_snapshot, by_crawl); + assert_eq!( + by_crawl.get("pkg:npm/left-pad@1.3.0"), + Some(&vec![root.join("node_modules/lp")]) + ); + + let elsewhere = local_options(root.join("packages/app")); + assert!(snapshot.packages_for(&elsewhere).is_none()); + let app_purls = vec!["pkg:npm/foo@1.0.0".to_string()]; + let app_partitioned = partition_purls(&app_purls, None); + assert_eq!( + find_packages_for_rollback_reusing(&app_partitioned, &elsewhere, true, Some(&snapshot)) + .await, + find_packages_for_rollback(&app_partitioned, &elsewhere, true).await, + "a snapshot of another root must not answer for this one" + ); + } + + /// Stage one installed package for EVERY ecosystem under `root`, the + /// dir handed to all nine crawlers verbatim as `--global-prefix`. Each + /// layout is the one that crawler's own global-prefix tests use, and + /// they do not collide: a cargo crate dir has no `lib/` (so it is no + /// gem), a gem dir has no `Cargo.toml`, a NuGet id dir carries its + /// version as a child rather than a `-` suffix, and the jsr scope dir + /// holds no package.json. + fn stage_every_ecosystem(root: &std::path::Path) { + let dir = |path: PathBuf| std::fs::create_dir_all(path).unwrap(); + let file = |path: PathBuf, body: String| { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, body).unwrap(); + }; + + // npm: //package.json + for (name, version) in [("zeta", "1.0.0"), ("alpha", "2.0.0"), ("mid", "3.0.0")] { + file( + root.join(name).join("package.json"), + format!(r#"{{"name":"{name}","version":"{version}"}}"#), + ); + } + // pypi: /-.dist-info/METADATA + for (name, version) in [("requests", "2.31.0"), ("attrs", "23.1.0")] { + file( + root.join(format!("{name}-{version}.dist-info")) + .join("METADATA"), + format!("Name: {name}\nVersion: {version}\n"), + ); + } + // cargo: /-/Cargo.toml + for (name, version) in [("serde", "1.0.0"), ("anyhow", "1.0.75")] { + file( + root.join(format!("{name}-{version}")).join("Cargo.toml"), + format!("[package]\nname = \"{name}\"\nversion = \"{version}\"\n"), + ); + } + // gem: /-/ verified by a .gemspec (a `lib/` + // would also make NuGet's legacy `.` reading of the + // same dir stick). + file( + root.join("rgem-2.0.0").join("rgem.gemspec"), + "Gem::Specification.new\n".to_string(), + ); + // golang: //@/ + dir(root.join("example.com").join("gomod@v1.2.3")); + // maven: ////-.pom + file( + root.join("org") + .join("example") + .join("mlib") + .join("4.0.0") + .join("mlib-4.0.0.pom"), + "org.example\ + mlib4.0.0" + .to_string(), + ); + // composer: the prefix IS the vendor dir — its metadata plus the + // install dir, which crawl_all requires to exist. + dir(root.join("acme").join("phplib")); + file( + root.join("composer").join("installed.json"), + serde_json::json!({"packages": [{"name": "acme/phplib", "version": "3.0.0"}]}) + .to_string(), + ); + // nuget: /// verified by a lib/ + dir(root.join("nugetlib").join("5.0.0").join("lib")); + // deno (jsr): /@/// + dir(root.join("@denoscope").join("jsrlib").join("6.0.0")); + } + + /// A crawl scoped to some ecosystems runs only their crawlers — a + /// skipped one leaves no `counts` entry, the pin that it never ran — + /// and yields exactly the full crawl's packages of those ecosystems, + /// in the full crawl's order, with the full crawl's counts. Scoped to + /// every ecosystem (or `None`) it IS the full crawl. The npm snapshot + /// exists exactly when npm was crawled. + #[tokio::test(flavor = "multi_thread")] + async fn scoped_crawl_is_the_full_crawl_filtered_to_its_ecosystems() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + stage_every_ecosystem(root); + let options = CrawlerOptions { + cwd: root.to_path_buf(), + global: false, + global_prefix: Some(root.to_path_buf()), + }; + let key = |p: &CrawledPackage| (p.purl.clone(), p.path.clone()); + let (full, full_counts, _, full_snapshot) = crawl_all_ecosystems_with_npm(&options).await; + assert_eq!(full_counts.len(), Ecosystem::all().len()); + + let every: Vec = Ecosystem::all() + .iter() + .map(|e| e.cli_name().to_string()) + .collect(); + let mut scopes: Vec> = every.iter().map(|e| vec![e.clone()]).collect(); + scopes.push(vec!["maven".into(), "npm".into()]); + scopes.push(vec!["pypi".into(), "deno".into(), "gem".into()]); + scopes.push(every.clone()); + for scope in &scopes { + let selected = |eco: &Ecosystem| scope.iter().any(|name| name == eco.cli_name()); + let (packages, counts, _, snapshot) = + crawl_ecosystems_with_npm(&options, Some(scope)).await; + let expected: Vec<_> = full + .iter() + .filter(|p| Ecosystem::from_purl(&p.purl).is_some_and(|e| selected(&e))) + .map(key) + .collect(); + assert_eq!( + packages.iter().map(key).collect::>(), + expected, + "{scope:?}" + ); + let expected_counts: HashMap = full_counts + .iter() + .filter(|(eco, _)| selected(eco)) + .map(|(eco, n)| (*eco, *n)) + .collect(); + assert_eq!(counts, expected_counts, "{scope:?}"); + assert_eq!( + snapshot.is_some(), + scope.iter().any(|name| name == "npm"), + "{scope:?}" + ); + if let Some(snapshot) = snapshot { + assert_eq!(snapshot.roots, full_snapshot.roots, "{scope:?}"); + assert_eq!( + snapshot.packages.iter().map(key).collect::>(), + full_snapshot.packages.iter().map(key).collect::>(), + "{scope:?}" + ); + } + } + + let (unscoped, unscoped_counts, _) = crawl_ecosystems(&options, None).await; + assert_eq!( + unscoped.iter().map(key).collect::>(), + full.iter().map(key).collect::>() + ); + assert_eq!(unscoped_counts, full_counts); + } + + /// The concurrent crawl must yield exactly the serial run's packages, + /// in the fixed ecosystem order, with the same counts. A + /// `--global-prefix` root is handed to every crawler verbatim, so one + /// polyglot dir exercises every ecosystem at once — and it has to: + /// an ecosystem that finds nothing contributes nothing to the + /// concatenation, so its POSITION in the consumption array is + /// unobservable and a reordering would ship silently. That order is + /// shipped behavior: `scan` chunks the crawl-ordered purls into + /// batches, so it decides batch composition, the `batch N/M failed` + /// warning text and order, and `last_batch_error`. + #[tokio::test(flavor = "multi_thread")] + async fn crawl_all_ecosystems_matches_serial_order() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + stage_every_ecosystem(root); + let options = CrawlerOptions { + cwd: root.to_path_buf(), + global: false, + global_prefix: Some(root.to_path_buf()), + }; + + let (packages, counts, _) = crawl_all_ecosystems(&options).await; + + let mut serial: Vec = Vec::new(); + let mut serial_counts: HashMap = HashMap::new(); + macro_rules! serial { + ($eco:expr, $pkgs:expr) => {{ + let pkgs = $pkgs; + serial_counts.insert($eco, pkgs.len()); + serial.extend(pkgs); + }}; + } + serial!(Ecosystem::Npm, NpmCrawler.crawl_all(&options).await); + serial!(Ecosystem::Pypi, PythonCrawler.crawl_all(&options).await); + serial!(Ecosystem::Cargo, CargoCrawler.crawl_all(&options).await); + serial!(Ecosystem::Gem, RubyCrawler.crawl_all(&options).await); + serial!(Ecosystem::Golang, GoCrawler.crawl_all(&options).await); + serial!(Ecosystem::Maven, MavenCrawler.crawl_all(&options).await); + serial!( + Ecosystem::Composer, + ComposerCrawler.crawl_all(&options).await + ); + serial!(Ecosystem::Nuget, NuGetCrawler.crawl_all(&options).await); + serial!(Ecosystem::Deno, DenoCrawler.crawl_all(&options).await); + + let key = |p: &CrawledPackage| (p.purl.clone(), p.path.clone()); + assert_eq!( + packages.iter().map(key).collect::>(), + serial.iter().map(key).collect::>() + ); + assert_eq!(counts, serial_counts); + // Non-vacuous for EVERY ecosystem, or the ones that found nothing + // are pinned only by this test's own copy of the order. + for eco in [ + Ecosystem::Npm, + Ecosystem::Pypi, + Ecosystem::Cargo, + Ecosystem::Gem, + Ecosystem::Golang, + Ecosystem::Maven, + Ecosystem::Composer, + Ecosystem::Nuget, + Ecosystem::Deno, + ] { + assert!( + counts.get(&eco).is_some_and(|&n| n >= 1), + "{eco:?} found nothing — its position is unobservable: {counts:?}" + ); + } + assert!(counts[&Ecosystem::Npm] >= 3, "{counts:?}"); + assert!(counts[&Ecosystem::Pypi] >= 2, "{counts:?}"); + } + /// Deno is the ONE dispatch branch no other test drives end-to-end /// (lcov: every other ecosystem's `scan_ecosystem!` invocation has /// executed, deno's never has). Stage the JSR cache layout diff --git a/crates/socket-patch-cli/src/ui/mod.rs b/crates/socket-patch-cli/src/ui/mod.rs index ed0b70d62..ae9f7225c 100644 --- a/crates/socket-patch-cli/src/ui/mod.rs +++ b/crates/socket-patch-cli/src/ui/mod.rs @@ -21,7 +21,7 @@ use std::io::IsTerminal; use crate::args::GlobalArgs; -pub(crate) use prompt::{confirm, confirm_or_proceed}; +pub(crate) use prompt::{confirm, confirm_or_proceed, confirm_waits}; pub use prompt::{select_one, SelectError}; pub(crate) use status::StatusLine; pub(crate) use text::{plural, truncate}; diff --git a/crates/socket-patch-cli/src/ui/prompt.rs b/crates/socket-patch-cli/src/ui/prompt.rs index 86b76e06b..a48df963c 100644 --- a/crates/socket-patch-cli/src/ui/prompt.rs +++ b/crates/socket-patch-cli/src/ui/prompt.rs @@ -34,12 +34,23 @@ pub(crate) fn confirm(prompt: &str, default_yes: bool, common: &GlobalArgs) -> b Ask { default_yes, non_interactive_answer: default_yes, - interactive: io::stdin().is_terminal(), + // The same question [`confirm_waits`] answers — asked through + // it, so the two cannot drift. + interactive: confirm_waits(common), silent: common.silent, }, ) } +/// Whether [`confirm`] would stop and wait for a person to answer — a +/// caller's clue that the world may change while it does (`scan` reuses a +/// crawl across the prompt only when it does not wait). Derived from +/// `confirm` itself rather than hand-copied at the call site: the drift +/// that matters is the unsafe direction, a wait nobody accounted for. +pub(crate) fn confirm_waits(common: &GlobalArgs) -> bool { + !(common.yes || common.json) && io::stdin().is_terminal() +} + /// A default-**no** confirmation that still proceeds when nobody can be /// asked (stdin not a terminal): `setup`'s mutation gate. `--yes`/`--json` /// proceed without asking. @@ -531,4 +542,51 @@ mod tests { Err(SelectError::Cancelled) )); } + + /// `confirm_waits` is what `scan` plans its crawl reuse around, so it + /// must never say "no wait" for a case `confirm` would stop on. Over + /// the whole `{yes, json}` cube with this process's stdin (a pipe + /// under the test harness, so never a terminal), it says no wait — + /// and `confirm` indeed answers from its default without reading a + /// byte, whatever is on stdin. + #[test] + fn confirm_waits_agrees_with_confirm_over_the_flag_cube() { + for (yes, json) in [(false, false), (true, false), (false, true), (true, true)] { + let common = GlobalArgs { + yes, + json, + silent: true, + ..GlobalArgs::default() + }; + assert!(!confirm_waits(&common), "yes={yes} json={json}"); + for default_yes in [true, false] { + assert_eq!( + confirm("go?", default_yes, &common), + default_yes, + "yes={yes} json={json} default={default_yes}" + ); + } + } + } + + /// The other half of the same contract, on the one input `confirm` + /// takes that a test can vary: when nobody is waiting, the answer is + /// the caller's default, never whatever happens to be on stdin. + #[test] + fn a_prompt_nobody_waits_on_never_reads_the_answer() { + let mut out = Vec::new(); + let answered = confirm_with( + &mut &b"n\n"[..], + &mut out, + "go?", + Ask { + default_yes: true, + non_interactive_answer: true, + interactive: false, + silent: true, + }, + ); + assert!(answered); + assert!(out.is_empty(), "{}", String::from_utf8_lossy(&out)); + } } diff --git a/crates/socket-patch-cli/tests/cli_parse_scan.rs b/crates/socket-patch-cli/tests/cli_parse_scan.rs index 4c846c38f..40fd8610f 100644 --- a/crates/socket-patch-cli/tests/cli_parse_scan.rs +++ b/crates/socket-patch-cli/tests/cli_parse_scan.rs @@ -6,7 +6,9 @@ //! //! Two defaults are especially load-bearing and explicitly asserted: //! -//! * `--batch-size` defaults to `100`. Downstream API batching assumes this. +//! * `--batch-size` has no parse-time default: unset, `scan` picks it per +//! endpoint at run time (500 on the authenticated API, 100 on the public +//! proxy), so an explicit value must stay distinguishable from none. //! * `--download-mode` defaults to `"diff"`. This diverges from `repair`'s //! default and is a silent-regression risk if flipped. @@ -110,7 +112,10 @@ fn defaults_match_contract() { let args = parse_scan(&[]); // Critical load-bearing defaults. - assert_eq!(args.batch_size, 100, "--batch-size default is 100"); + assert_eq!( + args.batch_size, None, + "--batch-size has no parse-time default (resolved per endpoint at run time)" + ); assert_eq!( args.common.download_mode, "diff", "--download-mode default is \"diff\"" @@ -264,14 +269,14 @@ fn api_token_flag() { #[serial_test::serial] fn batch_size_500() { let args = parse_scan(&["--batch-size", "500"]); - assert_eq!(args.batch_size, 500); + assert_eq!(args.batch_size, Some(500)); } #[test] #[serial_test::serial] fn batch_size_1() { let args = parse_scan(&["--batch-size", "1"]); - assert_eq!(args.batch_size, 1); + assert_eq!(args.batch_size, Some(1)); } #[test] @@ -281,7 +286,7 @@ fn batch_size_0_parses() { // a command-level concern, not a parser concern. Lock in that the parser // itself does not reject it. let args = parse_scan(&["--batch-size", "0"]); - assert_eq!(args.batch_size, 0); + assert_eq!(args.batch_size, Some(0)); } #[test] @@ -707,7 +712,7 @@ fn legacy_mode_spellings_still_parse() { /// `Debug` derive) are formatted individually. fn snap(a: &ScanArgs) -> String { format!( - "{:?} paths={:?} batch_size={} apply={} prune={} sync={} vendor={} detached={} \ + "{:?} paths={:?} batch_size={:?} apply={} prune={} sync={} vendor={} detached={} \ redirect={} mode={:?} all_releases={} vex={:?} vex_product={:?} \ vex_no_verify={} vex_doc_id={:?} vex_compact={}", a.common, diff --git a/crates/socket-patch-cli/tests/covgap_commands_vendor.rs b/crates/socket-patch-cli/tests/covgap_commands_vendor.rs index 5785b504c..745a85260 100644 --- a/crates/socket-patch-cli/tests/covgap_commands_vendor.rs +++ b/crates/socket-patch-cli/tests/covgap_commands_vendor.rs @@ -958,11 +958,15 @@ async fn reconcile_state_write_failure_reports_failed_after_removal() { ); } -/// A vendor run whose per-package `save_state` fails after the backend -/// already wrote the artifact and rewired the lock: the package's -/// `Applied` event stands, a `vendor_state_write_failed` failure rides -/// beside it, and the run exits 1 (the crash-consistency contract of the -/// per-package save). +/// A vendor run whose ledger cannot be written (`.socket/vendor` refuses +/// writes) after the backend already wrote the artifact: the run's ONE +/// commit of the lock rewire and the ledger fails as a whole, so neither is +/// written — the lock keeps its pre-run bytes, no ledger appears — and the +/// run exits 1 with the top-level `vendor_commit_failed` error. The +/// package's `Applied` event still reports what the backend did; the +/// artifact is an orphan the next run re-vendors over. (Before the group +/// commit this was a per-package `vendor_state_write_failed` next to an +/// already-rewired lock.) #[cfg(unix)] #[tokio::test] async fn vendor_state_write_failure_reports_failed_event() { @@ -977,22 +981,32 @@ async fn vendor_state_write_failure_reports_failed_event() { assert_eq!(code, 1, "{env:#}"); let applied = find_event(&env, "applied", None); assert_eq!(applied["purl"], PURL, "the backend vendored: {env:#}"); - let failed = find_event(&env, "failed", Some("vendor_state_write_failed")); - assert_eq!(failed["purl"], PURL, "{env:#}"); + assert_eq!(env["error"]["code"], "vendor_commit_failed", "{env:#}"); assert!( fx.tgz_path().is_file(), "the artifact the backend wrote is on disk" ); assert!(!fx.state_path().exists(), "the ledger write failed"); + assert_eq!( + fx.lock_bytes(), + fx.original_lock, + "the lock rewire is committed with the ledger or not at all" + ); } /// A hosted redirect record whose revert succeeds but whose ledger update -/// cannot be persisted (`.socket/vendor` read-only): the purl fails CLOSED -/// with `redirect_ledger_write_failed` and is NOT vendored — a ledger -/// still claiming reverted wiring must stop the takeover. +/// cannot be persisted (`.socket/vendor` read-only). The takeover's revert, +/// its redirect-ledger drop, the vendor rewire and the vendor ledger are +/// committed together, so the failed commit leaves ALL of them as found: +/// the lock untouched, the redirect ledger byte-identical (still claiming +/// only wiring that is still there), no vendor ledger — never a redirect +/// ledger claiming reverted wiring. The run exits 1 with +/// `vendor_commit_failed`. (Before the group commit the takeover persisted +/// the redirect ledger on its own and failed the purl closed with +/// `redirect_ledger_write_failed` before vendoring it.) #[cfg(unix)] #[test] -fn redirect_ledger_write_failure_fails_takeover_purl_closed() { +fn redirect_ledger_write_failure_commits_nothing() { let fx = npm_fixture(); std::fs::create_dir_all(fx.vendor_dir()).unwrap(); let before_hash = compute_git_sha256_from_bytes(ORIG_INDEX); @@ -1011,25 +1025,28 @@ fn redirect_ledger_write_failure_fails_takeover_purl_closed() { let (code, env) = vendor_cli(fx.root(), &[]); assert_eq!(code, 1, "{env:#}"); - let failed = find_event(&env, "failed", Some("redirect_ledger_write_failed")); - assert_eq!(failed["purl"], PURL); + assert_eq!(env["error"]["code"], "vendor_commit_failed", "{env:#}"); assert!( - failed["error"] + env["error"]["message"] .as_str() - .is_some_and(|d| d.contains("could not") && d.contains("redirect-state.json")), + .is_some_and(|d| d.contains("could not commit")), "{env:#}" ); assert_eq!( fx.lock_bytes(), fx.original_lock, - "no vendor rewire happened" + "no vendor rewire is committed" ); - assert!(!fx.tgz_path().exists(), "the purl must not be vendored"); + assert!(!fx.state_path().exists(), "no vendor ledger is committed"); assert_eq!( std::fs::read(fx.redirect_state_path()).unwrap(), ledger_bytes, "the unpersistable ledger is left exactly as found" ); + assert!( + !fx.vendor_dir().join(".commit-journal.json").exists(), + "the failed commit leaves no journal behind" + ); } // ───────────────────────────────────────────────────────────────────── diff --git a/crates/socket-patch-cli/tests/crawl_fd_limit_e2e.rs b/crates/socket-patch-cli/tests/crawl_fd_limit_e2e.rs new file mode 100644 index 000000000..d492e3e6a --- /dev/null +++ b/crates/socket-patch-cli/tests/crawl_fd_limit_e2e.rs @@ -0,0 +1,300 @@ +//! The crawl under a tight `RLIMIT_NOFILE` must inventory exactly what it +//! does with ample descriptors. +//! +//! Every crawler treats a failed `read_dir`/open — `EMFILE` included — as +//! an absent directory, so a crawl that holds more descriptors at once +//! than the old sequential walk (parallel walk threads, crawlers running +//! concurrently) would silently drop packages under a limit the old walk +//! handled. Below the walk pool's tight-limit threshold the crawl keeps +//! the sequential descriptor profile; this suite pins that by scanning the +//! same tree under a tight `ulimit -n` and under the inherited limit and +//! requiring byte-identical JSON. The npm-only tree pins the single walk +//! thread (the sequential walk scans it fully at 14; with one walk thread +//! per CPU it lost most of it at 16); the multi-ecosystem tree pins the +//! crawlers running one at a time. +//! +//! Descriptors are the only resource with a budget here. Peak memory also +//! scales with the walk thread count now (one package.json read per +//! thread, uncapped); nothing pins that beyond the thread count's own +//! ceiling — see the `walk_pool` module docs. +#![cfg(unix)] + +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +use serde_json::Value; + +fn binary() -> PathBuf { + env!("CARGO_BIN_EXE_socket-patch").into() +} + +fn write_package(dir: &Path, name: &str, version: &str) { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write( + dir.join("package.json"), + format!(r#"{{"name":"{name}","version":"{version}"}}"#), + ) + .unwrap(); +} + +/// A tree wide enough that parallel walk threads each hold a descriptor +/// at once: a root `node_modules` with nested and scoped packages, a pnpm +/// virtual store, and workspace packages with their own `node_modules`. +/// Returns the number of distinct packages it holds. +fn build_tree(root: &Path) -> usize { + std::fs::write( + root.join("package.json"), + r#"{"name":"root","version":"0.0.0"}"#, + ) + .unwrap(); + let mut count = 0; + let nm = root.join("node_modules"); + for i in 0..60 { + let pkg = nm.join(format!("pkg{i}")); + write_package(&pkg, &format!("pkg{i}"), "1.0.0"); + count += 1; + for j in 0..3 { + let name = format!("nested{i}-{j}"); + write_package(&pkg.join("node_modules").join(&name), &name, "2.0.0"); + count += 1; + } + } + for i in 0..15 { + let name = format!("scoped{i}"); + write_package( + &nm.join("@scope").join(&name), + &format!("@scope/{name}"), + "3.0.0", + ); + count += 1; + } + for i in 0..30 { + let name = format!("stored{i}"); + write_package( + &nm.join(".pnpm") + .join(format!("{name}@4.0.0")) + .join("node_modules") + .join(&name), + &name, + "4.0.0", + ); + count += 1; + } + for w in 0..10 { + let ws_nm = root + .join("packages") + .join(format!("ws{w}")) + .join("node_modules"); + for i in 0..10 { + let name = format!("ws{w}-dep{i}"); + write_package(&ws_nm.join(&name), &name, "5.0.0"); + count += 1; + } + } + count +} + +/// [`build_tree`] plus installs for three more ecosystems — a Python +/// virtualenv, a Bundler `vendor/bundle` and a Composer `vendor/` — so the +/// crawlers that run alongside npm hold descriptors of their own: run +/// concurrently instead of one at a time, they need more than a tight +/// limit leaves. Returns the number of distinct packages it holds. +fn build_multi_ecosystem_tree(root: &Path) -> usize { + let mut count = build_tree(root); + let site = root + .join(".venv") + .join("lib") + .join("python3.11") + .join("site-packages"); + for i in 0..40 { + let dist = site.join(format!("pydist{i}-1.0.{i}.dist-info")); + std::fs::create_dir_all(&dist).unwrap(); + std::fs::write( + dist.join("METADATA"), + format!("Metadata-Version: 2.1\nName: pydist{i}\nVersion: 1.0.{i}\n\n"), + ) + .unwrap(); + count += 1; + } + let gems = root + .join("vendor") + .join("bundle") + .join("ruby") + .join("3.2.0"); + for i in 0..40 { + let gem = gems.join("gems").join(format!("rgem{i}-2.0.{i}")); + std::fs::create_dir_all(gem.join("lib")).unwrap(); + std::fs::write(gem.join("lib").join(format!("rgem{i}.rb")), "").unwrap(); + count += 1; + } + std::fs::create_dir_all(gems.join("specifications")).unwrap(); + let composer = root.join("vendor").join("composer"); + std::fs::create_dir_all(&composer).unwrap(); + let mut installed = Vec::new(); + for i in 0..20 { + let name = format!("acme/lib{i}"); + std::fs::create_dir_all(root.join("vendor").join(&name)).unwrap(); + installed.push(serde_json::json!({"name": name, "version": format!("3.0.{i}")})); + count += 1; + } + std::fs::write(root.join("composer.json"), "{}").unwrap(); + std::fs::write( + composer.join("installed.json"), + serde_json::json!({ "packages": installed }).to_string(), + ) + .unwrap(); + count +} + +/// `scan --json` against an unreachable API (the crawl still runs and the +/// JSON still reports what it found), optionally under `ulimit -n`. +fn scan(root: &Path, nofile: Option) -> Output { + let mut script = String::new(); + if let Some(limit) = nofile { + script.push_str(&format!("ulimit -n {limit} || exit 99; ")); + } + script.push_str(r#"exec "$0" "$@""#); + let mut cmd = Command::new("/bin/sh"); + cmd.arg("-c") + .arg(script) + .arg(binary()) + .args([ + "scan", + "--json", + "--no-telemetry", + "--api-url", + "http://127.0.0.1:1", + "--api-token", + "x", + "--org", + "test-org", + ]) + .current_dir(root); + for (key, _) in std::env::vars_os() { + let name = key.to_string_lossy(); + if name.starts_with("SOCKET_") && !name.contains("TELEMETRY") && name != "SOCKET_NO_CONFIG" + { + cmd.env_remove(&key); + } + } + // Keep the Python and Bundler discovery on the fixture's own installs. + for key in [ + "VIRTUAL_ENV", + "BUNDLE_PATH", + "BUNDLE_APP_CONFIG", + "GEM_HOME", + "GEM_PATH", + ] { + cmd.env_remove(key); + } + cmd.output().unwrap() +} + +/// A dropped package is a blown descriptor budget, not a JSON diff: say +/// so before the byte-for-byte comparison does, while the limit that +/// produced it is still in hand. Silent when the tight run printed no +/// JSON at all — the stdout comparison then reports it, with stderr. +fn assert_scanned_the_same(tight: &Output, ample_json: &Value, limit: u32) { + let Ok(tight_json) = serde_json::from_slice::(&tight.stdout) else { + return; + }; + assert_eq!( + tight_json["scannedPackages"], + ample_json["scannedPackages"], + "ulimit -n {limit} dropped packages — the crawl's descriptor budget regressed; \ + stderr:\n{}", + String::from_utf8_lossy(&tight.stderr) + ); +} + +#[test] +fn tight_descriptor_limit_scans_the_same_packages() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + let expected = build_tree(root); + + let ample = scan(root, None); + let tight = scan(root, Some(16)); + assert_ne!(tight.status.code(), Some(99), "ulimit -n 16 was refused"); + + let ample_json: Value = serde_json::from_slice(&le.stdout).unwrap_or_else(|e| { + panic!( + "ample-limit scan printed no JSON ({e}); stderr:\n{}", + String::from_utf8_lossy(&le.stderr) + ) + }); + assert_eq!( + ample_json["scannedPackages"].as_u64(), + Some(expected as u64), + "{ample_json}" + ); + assert_scanned_the_same(&tight, &le_json, 16); + assert_eq!( + String::from_utf8_lossy(&tight.stdout), + String::from_utf8_lossy(&le.stdout), + "tight-limit stderr:\n{}", + String::from_utf8_lossy(&tight.stderr) + ); + assert_eq!(tight.status.code(), ample.status.code()); +} + +/// The same, with every crawler finding packages: the tight-limit run +/// must crawl the ecosystems one at a time, as the sequential dispatch +/// did, or the concurrently running crawlers' descriptors crowd each other +/// out and packages go missing. At 16 that is headroom; at 12 (checked on +/// macOS, where it was measured: the sequential dispatch scans this tree +/// fully down to 11, while running the crawlers concurrently loses 40-80 +/// of its packages at 12) it is what pins the one-at-a-time dispatch. +/// +/// 12 therefore leaves the serial dispatch exactly ONE descriptor of +/// slack: the measured cliff is 11 (complete) / 10 (445 of 485 packages). +/// Anything that holds one more descriptor open across the crawl — a +/// config read, a cert store, a log file — fails this test, so the +/// package-count assertion below runs first and names the budget instead +/// of leaving a 40-package JSON diff. +#[test] +fn tight_descriptor_limit_scans_every_ecosystem_the_same() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + let expected = build_multi_ecosystem_tree(root); + + let ample = scan(root, None); + let ample_json: Value = serde_json::from_slice(&le.stdout).unwrap_or_else(|e| { + panic!( + "ample-limit scan printed no JSON ({e}); stderr:\n{}", + String::from_utf8_lossy(&le.stderr) + ) + }); + assert_eq!( + ample_json["scannedPackages"].as_u64(), + Some(expected as u64), + "{ample_json}" + ); + + let limits: &[u32] = if cfg!(target_os = "macos") { + // The concurrent crawlers' overlap is timing-dependent: repeat. + &[16, 12, 12, 12] + } else { + &[16] + }; + for &limit in limits { + let tight = scan(root, Some(limit)); + assert_ne!( + tight.status.code(), + Some(99), + "ulimit -n {limit} was refused" + ); + assert_scanned_the_same(&tight, &le_json, limit); + assert_eq!( + String::from_utf8_lossy(&tight.stdout), + String::from_utf8_lossy(&le.stdout), + "ulimit -n {limit} stderr:\n{}", + String::from_utf8_lossy(&tight.stderr) + ); + assert_eq!( + tight.status.code(), + ample.status.code(), + "ulimit -n {limit}" + ); + } +} diff --git a/crates/socket-patch-cli/tests/e2e_yarn_legacy_cachekey_refusal_build.rs b/crates/socket-patch-cli/tests/e2e_yarn_legacy_cachekey_refusal_build.rs index 7d23c4660..00c81f027 100644 --- a/crates/socket-patch-cli/tests/e2e_yarn_legacy_cachekey_refusal_build.rs +++ b/crates/socket-patch-cli/tests/e2e_yarn_legacy_cachekey_refusal_build.rs @@ -406,9 +406,12 @@ async fn refusal_case(tag: &str, yarn_pm: &str, compression_zero: bool, expected ); eprintln!("({tag}) HOSTED REFUSAL OK"); - // 4. VENDORED refusal: per-package failed event, exit 1, partial_failure, - // zero mutations, no vendor artifacts. The download block proves the - // refusal fires at the WIRING step, not by failing discovery. + // 4. VENDORED refusal: exit 1, partial_failure, zero mutations, no + // vendor artifacts. The cacheKey gate reads only the lock, so the + // vendored download phase refuses the package BEFORE fetching its + // view (a package that will be refused costs no network): a failed + // download record carrying the backend's code and words, and nothing + // left for the vendor step. let vendored_args: Vec = api_args("vendored"); let vendored_argv: Vec<&str> = vendored_args.iter().map(String::as_str).collect(); let (code, stdout, stderr) = run_socket(&proj, &vendored_argv); @@ -422,28 +425,43 @@ async fn refusal_case(tag: &str, yarn_pm: &str, compression_zero: bool, expected }); assert_eq!(env["status"], "partial_failure", "({tag}) envelope: {env}"); assert_eq!( - env["download"]["downloaded"], 1, - "({tag}) the patch must download fine — the refusal is at the wiring step: {env}" + (&env["download"]["downloaded"], &env["download"]["failed"]), + (&serde_json::json!(0), &serde_json::json!(1)), + "({tag}) the lock-text refusal fires before the view fetch: {env}" ); - let events = env["vendor"]["events"] + let failed = env["download"]["patches"] .as_array() - .unwrap_or_else(|| panic!("({tag}) vendor.events must be an array: {env}")); - let failed = events - .iter() - .find(|e| e["action"] == "failed" && e["purl"] == PURL) - .unwrap_or_else(|| panic!("({tag}) expected a per-package failed event for {PURL}: {env}")); + .and_then(|p| p.iter().find(|r| r["purl"] == PURL)) + .unwrap_or_else(|| panic!("({tag}) expected a download record for {PURL}: {env}")); + assert_eq!(failed["action"], "failed", "({tag}) {failed}"); assert_eq!( failed["errorCode"], "vendor_yarn_berry_cache_unsupported", "({tag}) the refusal must be the CODE, not human text: {failed}" ); - assert_eq!( - env["vendor"]["summary"]["failed"], 1, - "({tag}) vendor summary: {env}" + assert!( + failed["error"] + .as_str() + .unwrap_or_default() + .contains(&format!("`{expected_cache_key}`")), + "({tag}) the refusal detail is the backend's own, naming the cacheKey: {failed}" ); assert_eq!( env["vendor"]["summary"]["applied"], 0, "({tag}) nothing may be vendored: {env}" ); + let vendor_events_for_purl: Vec<&serde_json::Value> = env["vendor"]["events"] + .as_array() + .map(|events| events.iter().filter(|e| e["purl"] == PURL).collect()) + .unwrap_or_default(); + assert!( + vendor_events_for_purl.is_empty(), + "({tag}) the vendor step must emit nothing for the package refused in the \ + download phase: {env}" + ); + assert_eq!( + env["vendor"]["summary"]["failed"], 0, + "({tag}) the refusal is not double-counted by the vendor step: {env}" + ); assert_eq!( std::fs::read(&lock_path).unwrap(), lock_before, diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/cargo/wired/.socket/vendor/state.json b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/cargo/wired/.socket/vendor/state.json new file mode 100644 index 000000000..4f84a03ed --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/cargo/wired/.socket/vendor/state.json @@ -0,0 +1,113 @@ +{ + "version": 1, + "entries": { + "pkg:cargo/alpha@1.0.0": { + "ecosystem": "cargo", + "basePurl": "pkg:cargo/alpha@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000001", + "artifact": { + "path": ".socket/vendor/cargo/11111111-1111-4111-8111-000000000001/alpha-1.0.0" + }, + "wiring": [ + { + "file": "Cargo.toml", + "kind": "cargo_patch_entry", + "action": "added", + "key": "alpha-socket-11111111", + "new": ".socket/vendor/cargo/11111111-1111-4111-8111-000000000001/alpha-1.0.0" + }, + { + "file": "Cargo.lock", + "kind": "cargo_lock_entry", + "action": "rewritten", + "key": "alpha@1.0.0", + "original": { + "source": "registry+https://github.com/rust-lang/crates.io-index", + "checksum": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + } + ], + "lock": { + "source": "registry+https://github.com/rust-lang/crates.io-index", + "checksum": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "record": { + "uuid": "11111111-1111-4111-8111-000000000001", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "package/src/lib.rs": { + "beforeHash": "288e50844f8b06d1428944c2cf984bd9749f34aa6d6482919a6cc83ed16d0bd3", + "afterHash": "eda8d84af85be378b1c97ca9bf97db8ba9e2b73502d1144956d88cc7f7d8aaf0" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + }, + "pkg:cargo/beta@1.0.0": { + "ecosystem": "cargo", + "basePurl": "pkg:cargo/beta@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000002", + "artifact": { + "path": ".socket/vendor/cargo/11111111-1111-4111-8111-000000000002/beta-1.0.0" + }, + "wiring": [ + { + "file": "Cargo.toml", + "kind": "cargo_patch_entry", + "action": "added", + "key": "beta-socket-11111111", + "new": ".socket/vendor/cargo/11111111-1111-4111-8111-000000000002/beta-1.0.0" + }, + { + "file": "Cargo.lock", + "kind": "cargo_lock_entry", + "action": "rewritten", + "key": "beta@1.0.0", + "original": { + "source": "registry+https://github.com/rust-lang/crates.io-index", + "checksum": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + } + ], + "lock": { + "source": "registry+https://github.com/rust-lang/crates.io-index", + "checksum": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "record": { + "uuid": "11111111-1111-4111-8111-000000000002", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "package/src/lib.rs": { + "beforeHash": "6c1e663d109d4f226179fd0ace6064071334aa58f7df3f47b57b71a8df98ba28", + "afterHash": "c88454177f0821faf61ff7601f190addb45899edfaf6758141682f538909c77f" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + } + } +} diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/cargo/wired/Cargo.lock b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/cargo/wired/Cargo.lock new file mode 100644 index 000000000..03f60496e --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/cargo/wired/Cargo.lock @@ -0,0 +1,19 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "alpha" +version = "1.0.0+socket.11111111-1111-4111-8111-000000000001" + +[[package]] +name = "beta" +version = "1.0.0+socket.11111111-1111-4111-8111-000000000002" + +[[package]] +name = "app" +version = "0.1.0" +dependencies = [ + "alpha", + "beta", +] diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/cargo/wired/Cargo.toml b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/cargo/wired/Cargo.toml new file mode 100644 index 000000000..95b34f9e0 --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/cargo/wired/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "app" +version = "0.1.0" + +[dependencies] +alpha = "1" +beta = "1" + +[patch.crates-io] +alpha-socket-11111111 = { package = "alpha", path = ".socket/vendor/cargo/11111111-1111-4111-8111-000000000001/alpha-1.0.0" } +beta-socket-11111111 = { package = "beta", path = ".socket/vendor/cargo/11111111-1111-4111-8111-000000000002/beta-1.0.0" } diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/composer/wired/.socket/vendor/state.json b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/composer/wired/.socket/vendor/state.json new file mode 100644 index 000000000..31dd50cec --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/composer/wired/.socket/vendor/state.json @@ -0,0 +1,141 @@ +{ + "version": 1, + "entries": { + "pkg:composer/fx/alpha@1.0.0": { + "ecosystem": "composer", + "basePurl": "pkg:composer/fx/alpha@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000001", + "artifact": { + "path": ".socket/vendor/composer/11111111-1111-4111-8111-000000000001/fx/alpha@1.0.0" + }, + "wiring": [ + { + "file": "composer.lock", + "kind": "composer_lock_package", + "action": "rewritten", + "key": "packages:fx/alpha", + "original": { + "name": "fx/alpha", + "version": "1.0.0", + "source": { + "type": "git", + "url": "https://github.com/fx/alpha.git", + "reference": "aaaa" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/fx/alpha/zipball/aaaa", + "reference": "aaaa", + "shasum": "" + }, + "type": "library" + }, + "new": { + "name": "fx/alpha", + "version": "1.0.0", + "dist": { + "type": "path", + "url": ".socket/vendor/composer/11111111-1111-4111-8111-000000000001/fx/alpha@1.0.0", + "reference": "11111111-1111-4111-8111-000000000001" + }, + "transport-options": { + "symlink": false + }, + "type": "library" + } + } + ], + "record": { + "uuid": "11111111-1111-4111-8111-000000000001", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "src/Lib.php": { + "beforeHash": "78b3130524b747d69f0f73976b69af77fdb540b61014435f7ede15b5f388505b", + "afterHash": "76ec0f946f1b365b8eb01cd36f7d517309649502886cb5b6587fec526ed71535" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + }, + "pkg:composer/fx/beta@1.0.0": { + "ecosystem": "composer", + "basePurl": "pkg:composer/fx/beta@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000002", + "artifact": { + "path": ".socket/vendor/composer/11111111-1111-4111-8111-000000000002/fx/beta@1.0.0" + }, + "wiring": [ + { + "file": "composer.lock", + "kind": "composer_lock_package", + "action": "rewritten", + "key": "packages:fx/beta", + "original": { + "name": "fx/beta", + "version": "1.0.0", + "source": { + "type": "git", + "url": "https://github.com/fx/beta.git", + "reference": "aaaa" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/fx/beta/zipball/aaaa", + "reference": "aaaa", + "shasum": "" + }, + "type": "library" + }, + "new": { + "name": "fx/beta", + "version": "1.0.0", + "dist": { + "type": "path", + "url": ".socket/vendor/composer/11111111-1111-4111-8111-000000000002/fx/beta@1.0.0", + "reference": "11111111-1111-4111-8111-000000000002" + }, + "transport-options": { + "symlink": false + }, + "type": "library" + } + } + ], + "record": { + "uuid": "11111111-1111-4111-8111-000000000002", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "src/Lib.php": { + "beforeHash": "6647d84bc9d78ca7485e3579d2ac9a195f02f40c941946258b5fd249a1578ef7", + "afterHash": "3a1b30750274c070fa015f0b1205fc1b958a96ecd69d79c15ce7a87b1ddce606" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + } + } +} diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/composer/wired/composer.lock b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/composer/wired/composer.lock new file mode 100644 index 000000000..a0cecf52d --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/composer/wired/composer.lock @@ -0,0 +1,37 @@ +{ + "_readme": [ + "This file locks the dependencies of your project to a known state" + ], + "content-hash": "7a59d114f58e9b02546b21d7e57430d3", + "packages": [ + { + "name": "fx/alpha", + "version": "1.0.0", + "dist": { + "type": "path", + "url": ".socket/vendor/composer/11111111-1111-4111-8111-000000000001/fx/alpha@1.0.0", + "reference": "11111111-1111-4111-8111-000000000001" + }, + "transport-options": { + "symlink": false + }, + "type": "library" + }, + { + "name": "fx/beta", + "version": "1.0.0", + "dist": { + "type": "path", + "url": ".socket/vendor/composer/11111111-1111-4111-8111-000000000002/fx/beta@1.0.0", + "reference": "11111111-1111-4111-8111-000000000002" + }, + "transport-options": { + "symlink": false + }, + "type": "library" + } + ], + "packages-dev": [], + "minimum-stability": "stable", + "plugin-api-version": "2.6.0" +} diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/gem/wired/.socket/vendor/state.json b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/gem/wired/.socket/vendor/state.json new file mode 100644 index 000000000..db80ec6f3 --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/gem/wired/.socket/vendor/state.json @@ -0,0 +1,129 @@ +{ + "version": 1, + "entries": { + "pkg:gem/alpha@1.0.0": { + "ecosystem": "gem", + "basePurl": "pkg:gem/alpha@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000001", + "artifact": { + "path": ".socket/vendor/gem/11111111-1111-4111-8111-000000000001/alpha-1.0.0", + "fileInventory": { + "alpha.gemspec": "95d0c65ffb7f1901607b734359ba09358be57261f3e9edf54cfd5f9a2ba05c60", + "lib/alpha.rb": "044fcb19093086b3d8167fd2dc1e1f6a969863d0e1bc90b62c0636782c2dd989" + } + }, + "wiring": [ + { + "file": "Gemfile", + "kind": "gemfile_line", + "action": "rewritten", + "key": "alpha", + "original": "gem \"alpha\"", + "new": "gem \"alpha\", \"1.0.0\", path: \".socket/vendor/gem/11111111-1111-4111-8111-000000000001/alpha-1.0.0\"" + }, + { + "file": "Gemfile.lock", + "kind": "gemfile_lock_spec", + "action": "rewritten", + "key": "alpha", + "original": [ + " alpha (1.0.0)", + " alpha" + ], + "new": [ + "PATH", + " remote: .socket/vendor/gem/11111111-1111-4111-8111-000000000001/alpha-1.0.0", + " specs:", + " alpha (1.0.0)", + " alpha (= 1.0.0)!" + ] + } + ], + "record": { + "uuid": "11111111-1111-4111-8111-000000000001", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "lib/alpha.rb": { + "beforeHash": "c9f4439e663855223be4f07be1641fecea39b5bf0bd0bd2036e8e80f996b3055", + "afterHash": "dad65038aa044e65110b925b244f15d3853e478a35630ea37642448d0dcd20dc" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + }, + "pkg:gem/beta@1.0.0": { + "ecosystem": "gem", + "basePurl": "pkg:gem/beta@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000002", + "artifact": { + "path": ".socket/vendor/gem/11111111-1111-4111-8111-000000000002/beta-1.0.0", + "fileInventory": { + "beta.gemspec": "39927495c69e36646adfb3e5848e636eea6b7ba4ed21d0eec15b7267391f8d93", + "lib/beta.rb": "7294d9f6fb1a5bdd7fd6cec972d3c89a1212b9f233a0e12703077e7cd8e62ae4" + } + }, + "wiring": [ + { + "file": "Gemfile", + "kind": "gemfile_line", + "action": "rewritten", + "key": "beta", + "original": "gem \"beta\"", + "new": "gem \"beta\", \"1.0.0\", path: \".socket/vendor/gem/11111111-1111-4111-8111-000000000002/beta-1.0.0\"" + }, + { + "file": "Gemfile.lock", + "kind": "gemfile_lock_spec", + "action": "rewritten", + "key": "beta", + "original": [ + " beta (1.0.0)", + " beta" + ], + "new": [ + "PATH", + " remote: .socket/vendor/gem/11111111-1111-4111-8111-000000000002/beta-1.0.0", + " specs:", + " beta (1.0.0)", + " beta (= 1.0.0)!" + ] + } + ], + "record": { + "uuid": "11111111-1111-4111-8111-000000000002", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "lib/beta.rb": { + "beforeHash": "a2167bcd355f6c9559c1d1703ddb866512c5f559a4e2e7c7081534ff0c9c81b7", + "afterHash": "7ca15e52fd628b0423d45383462380154be6979f8084e7aa3153e251591867f5" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + } + } +} diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/gem/wired/Gemfile b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/gem/wired/Gemfile new file mode 100644 index 000000000..d8cae51bb --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/gem/wired/Gemfile @@ -0,0 +1,3 @@ +source "https://rubygems.org" +gem "alpha", "1.0.0", path: ".socket/vendor/gem/11111111-1111-4111-8111-000000000001/alpha-1.0.0" +gem "beta", "1.0.0", path: ".socket/vendor/gem/11111111-1111-4111-8111-000000000002/beta-1.0.0" diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/gem/wired/Gemfile.lock b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/gem/wired/Gemfile.lock new file mode 100644 index 000000000..9dea33999 --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/gem/wired/Gemfile.lock @@ -0,0 +1,23 @@ +PATH + remote: .socket/vendor/gem/11111111-1111-4111-8111-000000000001/alpha-1.0.0 + specs: + alpha (1.0.0) + +PATH + remote: .socket/vendor/gem/11111111-1111-4111-8111-000000000002/beta-1.0.0 + specs: + beta (1.0.0) + +GEM + remote: https://rubygems.org/ + specs: + +PLATFORMS + ruby + +DEPENDENCIES + alpha (= 1.0.0)! + beta (= 1.0.0)! + +BUNDLED WITH + 2.5.3 diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/golang/wired/.socket/vendor/state.json b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/golang/wired/.socket/vendor/state.json new file mode 100644 index 000000000..125c8e104 --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/golang/wired/.socket/vendor/state.json @@ -0,0 +1,85 @@ +{ + "version": 1, + "entries": { + "pkg:golang/github.com/fx/alpha@v1.0.0": { + "ecosystem": "golang", + "basePurl": "pkg:golang/github.com/fx/alpha@v1.0.0", + "uuid": "11111111-1111-4111-8111-000000000001", + "artifact": { + "path": ".socket/vendor/golang/11111111-1111-4111-8111-000000000001/github.com/fx/alpha@v1.0.0" + }, + "wiring": [ + { + "file": "go.mod", + "kind": "go_replace", + "action": "added", + "key": "github.com/fx/alpha", + "new": "./.socket/vendor/golang/11111111-1111-4111-8111-000000000001/github.com/fx/alpha@v1.0.0" + } + ], + "record": { + "uuid": "11111111-1111-4111-8111-000000000001", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "package/alpha.go": { + "beforeHash": "f9a7a720c337f89c4603ba86ad11f1ffa9faba094ea13c440950247a86421a65", + "afterHash": "f9982be3d2b2789be8d5d3202414b4ae57f43fc83379a5ca9f0466c1f595c140" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + }, + "pkg:golang/github.com/fx/beta@v1.0.0": { + "ecosystem": "golang", + "basePurl": "pkg:golang/github.com/fx/beta@v1.0.0", + "uuid": "11111111-1111-4111-8111-000000000002", + "artifact": { + "path": ".socket/vendor/golang/11111111-1111-4111-8111-000000000002/github.com/fx/beta@v1.0.0" + }, + "wiring": [ + { + "file": "go.mod", + "kind": "go_replace", + "action": "added", + "key": "github.com/fx/beta", + "new": "./.socket/vendor/golang/11111111-1111-4111-8111-000000000002/github.com/fx/beta@v1.0.0" + } + ], + "record": { + "uuid": "11111111-1111-4111-8111-000000000002", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "package/beta.go": { + "beforeHash": "6e20e1faedb1581301d4dc08ff12af667d6f9c5e0ca74c178073cb91b605e971", + "afterHash": "ee07242ebcfb6651bc05997fc26d1124dbe3c58790dac8ec3d6ebf4ceb8df0cc" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + } + } +} diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/golang/wired/go.mod b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/golang/wired/go.mod new file mode 100644 index 000000000..e085cd309 --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/golang/wired/go.mod @@ -0,0 +1,12 @@ +module example.com/app + +go 1.21 + +require ( + github.com/fx/alpha v1.0.0 + github.com/fx/beta v1.0.0 +) + +replace github.com/fx/alpha v1.0.0 => ./.socket/vendor/golang/11111111-1111-4111-8111-000000000001/github.com/fx/alpha@v1.0.0 + +replace github.com/fx/beta v1.0.0 => ./.socket/vendor/golang/11111111-1111-4111-8111-000000000002/github.com/fx/beta@v1.0.0 diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/maven/wired/.socket/vendor/state.json b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/maven/wired/.socket/vendor/state.json new file mode 100644 index 000000000..df29360af --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/maven/wired/.socket/vendor/state.json @@ -0,0 +1,91 @@ +{ + "version": 1, + "entries": { + "pkg:maven/org.fx/alpha@1.0.0": { + "ecosystem": "maven", + "basePurl": "pkg:maven/org.fx/alpha@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000001", + "artifact": { + "path": ".socket/vendor/maven/11111111-1111-4111-8111-000000000001/org/fx/alpha/1.0.0/alpha-1.0.0.jar", + "sha256": "25372d12e4d57fa98e8aaac80bbed4948478d187ec161469b6a1c8051efcd641", + "size": 417 + }, + "wiring": [ + { + "file": "pom.xml", + "kind": "maven_pom_repository", + "action": "added", + "key": "socket-patch-vendor-11111111-1111-4111-8111-000000000001", + "original": "\n\n 4.0.0\n com.example\n app\n 1.0.0\n \n \n org.fx\n alpha\n 1.0.0\n \n \n org.fx\n beta\n 1.0.0\n \n \n org.filler\n filler00\n 1.0.0\n \n \n org.filler\n filler01\n 1.0.1\n \n \n org.filler\n filler02\n 1.0.2\n \n \n org.filler\n filler03\n 1.0.3\n \n \n org.filler\n filler04\n 1.0.4\n \n \n org.filler\n filler05\n 1.0.5\n \n \n org.filler\n filler06\n 1.0.6\n \n \n org.filler\n filler07\n 1.0.7\n \n \n org.filler\n filler08\n 1.0.8\n \n \n org.filler\n filler09\n 1.0.9\n \n \n org.filler\n filler10\n 1.0.10\n \n \n org.filler\n filler11\n 1.0.11\n \n \n org.filler\n filler12\n 1.0.12\n \n \n org.filler\n filler13\n 1.0.13\n \n \n org.filler\n filler14\n 1.0.14\n \n \n org.filler\n filler15\n 1.0.15\n \n \n org.filler\n filler16\n 1.0.16\n \n \n org.filler\n filler17\n 1.0.17\n \n \n org.filler\n filler18\n 1.0.18\n \n \n org.filler\n filler19\n 1.0.19\n \n \n org.filler\n filler20\n 1.0.20\n \n \n org.filler\n filler21\n 1.0.21\n \n \n org.filler\n filler22\n 1.0.22\n \n \n org.filler\n filler23\n 1.0.23\n \n \n org.filler\n filler24\n 1.0.24\n \n \n org.filler\n filler25\n 1.0.25\n \n \n org.filler\n filler26\n 1.0.26\n \n \n org.filler\n filler27\n 1.0.27\n \n \n org.filler\n filler28\n 1.0.28\n \n \n org.filler\n filler29\n 1.0.29\n \n \n org.filler\n filler30\n 1.0.30\n \n \n org.filler\n filler31\n 1.0.31\n \n \n org.filler\n filler32\n 1.0.32\n \n \n org.filler\n filler33\n 1.0.33\n \n \n org.filler\n filler34\n 1.0.34\n \n \n org.filler\n filler35\n 1.0.35\n \n \n org.filler\n filler36\n 1.0.36\n \n \n org.filler\n filler37\n 1.0.37\n \n \n org.filler\n filler38\n 1.0.38\n \n \n org.filler\n filler39\n 1.0.39\n \n \n org.filler\n filler40\n 1.0.40\n \n \n org.filler\n filler41\n 1.0.41\n \n \n org.filler\n filler42\n 1.0.42\n \n \n org.filler\n filler43\n 1.0.43\n \n \n org.filler\n filler44\n 1.0.44\n \n \n org.filler\n filler45\n 1.0.45\n \n \n org.filler\n filler46\n 1.0.46\n \n \n org.filler\n filler47\n 1.0.47\n \n \n org.filler\n filler48\n 1.0.48\n \n \n org.filler\n filler49\n 1.0.49\n \n \n org.filler\n filler50\n 1.0.50\n \n \n org.filler\n filler51\n 1.0.51\n \n \n org.filler\n filler52\n 1.0.52\n \n \n org.filler\n filler53\n 1.0.53\n \n \n org.filler\n filler54\n 1.0.54\n \n \n org.filler\n filler55\n 1.0.55\n \n \n org.filler\n filler56\n 1.0.56\n \n \n org.filler\n filler57\n 1.0.57\n \n \n org.filler\n filler58\n 1.0.58\n \n \n org.filler\n filler59\n 1.0.59\n \n \n\n", + "new": "\n\n 4.0.0\n com.example\n app\n 1.0.0\n \n \n org.fx\n alpha\n 1.0.0\n \n \n org.fx\n beta\n 1.0.0\n \n \n org.filler\n filler00\n 1.0.0\n \n \n org.filler\n filler01\n 1.0.1\n \n \n org.filler\n filler02\n 1.0.2\n \n \n org.filler\n filler03\n 1.0.3\n \n \n org.filler\n filler04\n 1.0.4\n \n \n org.filler\n filler05\n 1.0.5\n \n \n org.filler\n filler06\n 1.0.6\n \n \n org.filler\n filler07\n 1.0.7\n \n \n org.filler\n filler08\n 1.0.8\n \n \n org.filler\n filler09\n 1.0.9\n \n \n org.filler\n filler10\n 1.0.10\n \n \n org.filler\n filler11\n 1.0.11\n \n \n org.filler\n filler12\n 1.0.12\n \n \n org.filler\n filler13\n 1.0.13\n \n \n org.filler\n filler14\n 1.0.14\n \n \n org.filler\n filler15\n 1.0.15\n \n \n org.filler\n filler16\n 1.0.16\n \n \n org.filler\n filler17\n 1.0.17\n \n \n org.filler\n filler18\n 1.0.18\n \n \n org.filler\n filler19\n 1.0.19\n \n \n org.filler\n filler20\n 1.0.20\n \n \n org.filler\n filler21\n 1.0.21\n \n \n org.filler\n filler22\n 1.0.22\n \n \n org.filler\n filler23\n 1.0.23\n \n \n org.filler\n filler24\n 1.0.24\n \n \n org.filler\n filler25\n 1.0.25\n \n \n org.filler\n filler26\n 1.0.26\n \n \n org.filler\n filler27\n 1.0.27\n \n \n org.filler\n filler28\n 1.0.28\n \n \n org.filler\n filler29\n 1.0.29\n \n \n org.filler\n filler30\n 1.0.30\n \n \n org.filler\n filler31\n 1.0.31\n \n \n org.filler\n filler32\n 1.0.32\n \n \n org.filler\n filler33\n 1.0.33\n \n \n org.filler\n filler34\n 1.0.34\n \n \n org.filler\n filler35\n 1.0.35\n \n \n org.filler\n filler36\n 1.0.36\n \n \n org.filler\n filler37\n 1.0.37\n \n \n org.filler\n filler38\n 1.0.38\n \n \n org.filler\n filler39\n 1.0.39\n \n \n org.filler\n filler40\n 1.0.40\n \n \n org.filler\n filler41\n 1.0.41\n \n \n org.filler\n filler42\n 1.0.42\n \n \n org.filler\n filler43\n 1.0.43\n \n \n org.filler\n filler44\n 1.0.44\n \n \n org.filler\n filler45\n 1.0.45\n \n \n org.filler\n filler46\n 1.0.46\n \n \n org.filler\n filler47\n 1.0.47\n \n \n org.filler\n filler48\n 1.0.48\n \n \n org.filler\n filler49\n 1.0.49\n \n \n org.filler\n filler50\n 1.0.50\n \n \n org.filler\n filler51\n 1.0.51\n \n \n org.filler\n filler52\n 1.0.52\n \n \n org.filler\n filler53\n 1.0.53\n \n \n org.filler\n filler54\n 1.0.54\n \n \n org.filler\n filler55\n 1.0.55\n \n \n org.filler\n filler56\n 1.0.56\n \n \n org.filler\n filler57\n 1.0.57\n \n \n org.filler\n filler58\n 1.0.58\n \n \n org.filler\n filler59\n 1.0.59\n \n \n \n \n socket-patch-vendor-11111111-1111-4111-8111-000000000001\n file://${project.basedir}/.socket/vendor/maven/11111111-1111-4111-8111-000000000001\n \n true\n fail\n \n \n false\n \n \n \n\n" + } + ], + "record": { + "uuid": "11111111-1111-4111-8111-000000000001", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "META-INF/NOTICE.txt": { + "beforeHash": "9c1c75c055fc0bd7fd044bc59a0ef590253ba217cb27b2dda195d9d7ee8c7a4e", + "afterHash": "029771b8d7fa637cc2276a5e70845bc70135b2b26cda46b530cd4803153c7230" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + }, + "pkg:maven/org.fx/beta@1.0.0": { + "ecosystem": "maven", + "basePurl": "pkg:maven/org.fx/beta@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000002", + "artifact": { + "path": ".socket/vendor/maven/11111111-1111-4111-8111-000000000002/org/fx/beta/1.0.0/beta-1.0.0.jar", + "sha256": "5e891a2eabcb537d7bdeb0540e02d5cc16ab0930d207a4a8e687c37980fda0f5", + "size": 416 + }, + "wiring": [ + { + "file": "pom.xml", + "kind": "maven_pom_repository", + "action": "added", + "key": "socket-patch-vendor-11111111-1111-4111-8111-000000000002", + "original": "\n\n 4.0.0\n com.example\n app\n 1.0.0\n \n \n org.fx\n alpha\n 1.0.0\n \n \n org.fx\n beta\n 1.0.0\n \n \n org.filler\n filler00\n 1.0.0\n \n \n org.filler\n filler01\n 1.0.1\n \n \n org.filler\n filler02\n 1.0.2\n \n \n org.filler\n filler03\n 1.0.3\n \n \n org.filler\n filler04\n 1.0.4\n \n \n org.filler\n filler05\n 1.0.5\n \n \n org.filler\n filler06\n 1.0.6\n \n \n org.filler\n filler07\n 1.0.7\n \n \n org.filler\n filler08\n 1.0.8\n \n \n org.filler\n filler09\n 1.0.9\n \n \n org.filler\n filler10\n 1.0.10\n \n \n org.filler\n filler11\n 1.0.11\n \n \n org.filler\n filler12\n 1.0.12\n \n \n org.filler\n filler13\n 1.0.13\n \n \n org.filler\n filler14\n 1.0.14\n \n \n org.filler\n filler15\n 1.0.15\n \n \n org.filler\n filler16\n 1.0.16\n \n \n org.filler\n filler17\n 1.0.17\n \n \n org.filler\n filler18\n 1.0.18\n \n \n org.filler\n filler19\n 1.0.19\n \n \n org.filler\n filler20\n 1.0.20\n \n \n org.filler\n filler21\n 1.0.21\n \n \n org.filler\n filler22\n 1.0.22\n \n \n org.filler\n filler23\n 1.0.23\n \n \n org.filler\n filler24\n 1.0.24\n \n \n org.filler\n filler25\n 1.0.25\n \n \n org.filler\n filler26\n 1.0.26\n \n \n org.filler\n filler27\n 1.0.27\n \n \n org.filler\n filler28\n 1.0.28\n \n \n org.filler\n filler29\n 1.0.29\n \n \n org.filler\n filler30\n 1.0.30\n \n \n org.filler\n filler31\n 1.0.31\n \n \n org.filler\n filler32\n 1.0.32\n \n \n org.filler\n filler33\n 1.0.33\n \n \n org.filler\n filler34\n 1.0.34\n \n \n org.filler\n filler35\n 1.0.35\n \n \n org.filler\n filler36\n 1.0.36\n \n \n org.filler\n filler37\n 1.0.37\n \n \n org.filler\n filler38\n 1.0.38\n \n \n org.filler\n filler39\n 1.0.39\n \n \n org.filler\n filler40\n 1.0.40\n \n \n org.filler\n filler41\n 1.0.41\n \n \n org.filler\n filler42\n 1.0.42\n \n \n org.filler\n filler43\n 1.0.43\n \n \n org.filler\n filler44\n 1.0.44\n \n \n org.filler\n filler45\n 1.0.45\n \n \n org.filler\n filler46\n 1.0.46\n \n \n org.filler\n filler47\n 1.0.47\n \n \n org.filler\n filler48\n 1.0.48\n \n \n org.filler\n filler49\n 1.0.49\n \n \n org.filler\n filler50\n 1.0.50\n \n \n org.filler\n filler51\n 1.0.51\n \n \n org.filler\n filler52\n 1.0.52\n \n \n org.filler\n filler53\n 1.0.53\n \n \n org.filler\n filler54\n 1.0.54\n \n \n org.filler\n filler55\n 1.0.55\n \n \n org.filler\n filler56\n 1.0.56\n \n \n org.filler\n filler57\n 1.0.57\n \n \n org.filler\n filler58\n 1.0.58\n \n \n org.filler\n filler59\n 1.0.59\n \n \n \n \n socket-patch-vendor-11111111-1111-4111-8111-000000000001\n file://${project.basedir}/.socket/vendor/maven/11111111-1111-4111-8111-000000000001\n \n true\n fail\n \n \n false\n \n \n \n\n", + "new": "\n\n 4.0.0\n com.example\n app\n 1.0.0\n \n \n org.fx\n alpha\n 1.0.0\n \n \n org.fx\n beta\n 1.0.0\n \n \n org.filler\n filler00\n 1.0.0\n \n \n org.filler\n filler01\n 1.0.1\n \n \n org.filler\n filler02\n 1.0.2\n \n \n org.filler\n filler03\n 1.0.3\n \n \n org.filler\n filler04\n 1.0.4\n \n \n org.filler\n filler05\n 1.0.5\n \n \n org.filler\n filler06\n 1.0.6\n \n \n org.filler\n filler07\n 1.0.7\n \n \n org.filler\n filler08\n 1.0.8\n \n \n org.filler\n filler09\n 1.0.9\n \n \n org.filler\n filler10\n 1.0.10\n \n \n org.filler\n filler11\n 1.0.11\n \n \n org.filler\n filler12\n 1.0.12\n \n \n org.filler\n filler13\n 1.0.13\n \n \n org.filler\n filler14\n 1.0.14\n \n \n org.filler\n filler15\n 1.0.15\n \n \n org.filler\n filler16\n 1.0.16\n \n \n org.filler\n filler17\n 1.0.17\n \n \n org.filler\n filler18\n 1.0.18\n \n \n org.filler\n filler19\n 1.0.19\n \n \n org.filler\n filler20\n 1.0.20\n \n \n org.filler\n filler21\n 1.0.21\n \n \n org.filler\n filler22\n 1.0.22\n \n \n org.filler\n filler23\n 1.0.23\n \n \n org.filler\n filler24\n 1.0.24\n \n \n org.filler\n filler25\n 1.0.25\n \n \n org.filler\n filler26\n 1.0.26\n \n \n org.filler\n filler27\n 1.0.27\n \n \n org.filler\n filler28\n 1.0.28\n \n \n org.filler\n filler29\n 1.0.29\n \n \n org.filler\n filler30\n 1.0.30\n \n \n org.filler\n filler31\n 1.0.31\n \n \n org.filler\n filler32\n 1.0.32\n \n \n org.filler\n filler33\n 1.0.33\n \n \n org.filler\n filler34\n 1.0.34\n \n \n org.filler\n filler35\n 1.0.35\n \n \n org.filler\n filler36\n 1.0.36\n \n \n org.filler\n filler37\n 1.0.37\n \n \n org.filler\n filler38\n 1.0.38\n \n \n org.filler\n filler39\n 1.0.39\n \n \n org.filler\n filler40\n 1.0.40\n \n \n org.filler\n filler41\n 1.0.41\n \n \n org.filler\n filler42\n 1.0.42\n \n \n org.filler\n filler43\n 1.0.43\n \n \n org.filler\n filler44\n 1.0.44\n \n \n org.filler\n filler45\n 1.0.45\n \n \n org.filler\n filler46\n 1.0.46\n \n \n org.filler\n filler47\n 1.0.47\n \n \n org.filler\n filler48\n 1.0.48\n \n \n org.filler\n filler49\n 1.0.49\n \n \n org.filler\n filler50\n 1.0.50\n \n \n org.filler\n filler51\n 1.0.51\n \n \n org.filler\n filler52\n 1.0.52\n \n \n org.filler\n filler53\n 1.0.53\n \n \n org.filler\n filler54\n 1.0.54\n \n \n org.filler\n filler55\n 1.0.55\n \n \n org.filler\n filler56\n 1.0.56\n \n \n org.filler\n filler57\n 1.0.57\n \n \n org.filler\n filler58\n 1.0.58\n \n \n org.filler\n filler59\n 1.0.59\n \n \n \n \n socket-patch-vendor-11111111-1111-4111-8111-000000000001\n file://${project.basedir}/.socket/vendor/maven/11111111-1111-4111-8111-000000000001\n \n true\n fail\n \n \n false\n \n \n \n socket-patch-vendor-11111111-1111-4111-8111-000000000002\n file://${project.basedir}/.socket/vendor/maven/11111111-1111-4111-8111-000000000002\n \n true\n fail\n \n \n false\n \n \n \n\n" + } + ], + "record": { + "uuid": "11111111-1111-4111-8111-000000000002", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "META-INF/NOTICE.txt": { + "beforeHash": "141122ba74dc5ad6bfe8467e43181a225fd0e1b962167357020896b710fecc51", + "afterHash": "3e08c51cf3fff46cdcebf34b7a72d8ee48d3f2ffa67d9edb9c84bf4130a2ec08" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + } + } +} diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/maven/wired/pom.xml b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/maven/wired/pom.xml new file mode 100644 index 000000000..f86d73851 --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/maven/wired/pom.xml @@ -0,0 +1,343 @@ + + + 4.0.0 + com.example + app + 1.0.0 + + + org.fx + alpha + 1.0.0 + + + org.fx + beta + 1.0.0 + + + org.filler + filler00 + 1.0.0 + + + org.filler + filler01 + 1.0.1 + + + org.filler + filler02 + 1.0.2 + + + org.filler + filler03 + 1.0.3 + + + org.filler + filler04 + 1.0.4 + + + org.filler + filler05 + 1.0.5 + + + org.filler + filler06 + 1.0.6 + + + org.filler + filler07 + 1.0.7 + + + org.filler + filler08 + 1.0.8 + + + org.filler + filler09 + 1.0.9 + + + org.filler + filler10 + 1.0.10 + + + org.filler + filler11 + 1.0.11 + + + org.filler + filler12 + 1.0.12 + + + org.filler + filler13 + 1.0.13 + + + org.filler + filler14 + 1.0.14 + + + org.filler + filler15 + 1.0.15 + + + org.filler + filler16 + 1.0.16 + + + org.filler + filler17 + 1.0.17 + + + org.filler + filler18 + 1.0.18 + + + org.filler + filler19 + 1.0.19 + + + org.filler + filler20 + 1.0.20 + + + org.filler + filler21 + 1.0.21 + + + org.filler + filler22 + 1.0.22 + + + org.filler + filler23 + 1.0.23 + + + org.filler + filler24 + 1.0.24 + + + org.filler + filler25 + 1.0.25 + + + org.filler + filler26 + 1.0.26 + + + org.filler + filler27 + 1.0.27 + + + org.filler + filler28 + 1.0.28 + + + org.filler + filler29 + 1.0.29 + + + org.filler + filler30 + 1.0.30 + + + org.filler + filler31 + 1.0.31 + + + org.filler + filler32 + 1.0.32 + + + org.filler + filler33 + 1.0.33 + + + org.filler + filler34 + 1.0.34 + + + org.filler + filler35 + 1.0.35 + + + org.filler + filler36 + 1.0.36 + + + org.filler + filler37 + 1.0.37 + + + org.filler + filler38 + 1.0.38 + + + org.filler + filler39 + 1.0.39 + + + org.filler + filler40 + 1.0.40 + + + org.filler + filler41 + 1.0.41 + + + org.filler + filler42 + 1.0.42 + + + org.filler + filler43 + 1.0.43 + + + org.filler + filler44 + 1.0.44 + + + org.filler + filler45 + 1.0.45 + + + org.filler + filler46 + 1.0.46 + + + org.filler + filler47 + 1.0.47 + + + org.filler + filler48 + 1.0.48 + + + org.filler + filler49 + 1.0.49 + + + org.filler + filler50 + 1.0.50 + + + org.filler + filler51 + 1.0.51 + + + org.filler + filler52 + 1.0.52 + + + org.filler + filler53 + 1.0.53 + + + org.filler + filler54 + 1.0.54 + + + org.filler + filler55 + 1.0.55 + + + org.filler + filler56 + 1.0.56 + + + org.filler + filler57 + 1.0.57 + + + org.filler + filler58 + 1.0.58 + + + org.filler + filler59 + 1.0.59 + + + + + socket-patch-vendor-11111111-1111-4111-8111-000000000001 + file://${project.basedir}/.socket/vendor/maven/11111111-1111-4111-8111-000000000001 + + true + fail + + + false + + + + socket-patch-vendor-11111111-1111-4111-8111-000000000002 + file://${project.basedir}/.socket/vendor/maven/11111111-1111-4111-8111-000000000002 + + true + fail + + + false + + + + diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/npm/wired/.socket/vendor/state.json b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/npm/wired/.socket/vendor/state.json new file mode 100644 index 000000000..bfbc26ade --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/npm/wired/.socket/vendor/state.json @@ -0,0 +1,109 @@ +{ + "version": 1, + "entries": { + "pkg:npm/alpha@1.0.0": { + "ecosystem": "npm", + "basePurl": "pkg:npm/alpha@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000001", + "artifact": { + "path": ".socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz", + "sha256": "9750489c2d19b3ada3dde4a408b894e0f63cc73b9e0c0bcd80a8abef5e4f82c4", + "size": 178 + }, + "wiring": [ + { + "file": "package-lock.json", + "kind": "npm_lock_entry", + "action": "rewritten", + "key": "node_modules/alpha", + "original": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/alpha/-/alpha-1.0.0.tgz", + "integrity": "sha512-orig==" + }, + "new": { + "version": "1.0.0", + "resolved": "file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz", + "integrity": "sha512-kmYLZ8QKvcFKA3NQaxXFnIwOjKLrsfUVj5XeSUIOlaYzy5VKZ+ZbIaRpwvt7crrAd5x6llAVcTVLvWt+QAdZnA==" + } + } + ], + "flavor": "package-lock", + "record": { + "uuid": "11111111-1111-4111-8111-000000000001", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "package/index.js": { + "beforeHash": "4717263f295eed420954cedf06613ed753d8ed50ab048a60aea61c43d3b9ad9d", + "afterHash": "bc15782a928574cb0d3864cfa343d525c642e845a470e3e64aafeba864635284" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + }, + "pkg:npm/beta@1.0.0": { + "ecosystem": "npm", + "basePurl": "pkg:npm/beta@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000002", + "artifact": { + "path": ".socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz", + "sha256": "15a787bf25ccefc1d643efe9b157a0703dad1837d3d6fef1dad2177738ca0aac", + "size": 177 + }, + "wiring": [ + { + "file": "package-lock.json", + "kind": "npm_lock_entry", + "action": "rewritten", + "key": "node_modules/beta", + "original": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/beta/-/beta-1.0.0.tgz", + "integrity": "sha512-orig==" + }, + "new": { + "version": "1.0.0", + "resolved": "file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz", + "integrity": "sha512-qdpYNuREio4Ne2pCoWYKEfqDrKR7gNVvJEi6UhkognzwSBA9PvcgPrVhGDFWlMeerfGwZzVfmHd8p6zabtkO2A==" + } + } + ], + "flavor": "package-lock", + "record": { + "uuid": "11111111-1111-4111-8111-000000000002", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "package/index.js": { + "beforeHash": "64e197dfafd58f0f80310669f32563a0bd078317decf19152868b6a5d20ede42", + "afterHash": "0c82d6e011aad77f6178863228991fb143135c15123ce7e17d7c6d700614b273" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + } + } +} diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/npm/wired/package-lock.json b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/npm/wired/package-lock.json new file mode 100644 index 000000000..401970e6d --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/npm/wired/package-lock.json @@ -0,0 +1,26 @@ +{ + "name": "fixture", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "fixture", + "version": "1.0.0", + "dependencies": { + "alpha": "1.0.0", + "beta": "1.0.0" + } + }, + "node_modules/alpha": { + "version": "1.0.0", + "resolved": "file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz", + "integrity": "sha512-kmYLZ8QKvcFKA3NQaxXFnIwOjKLrsfUVj5XeSUIOlaYzy5VKZ+ZbIaRpwvt7crrAd5x6llAVcTVLvWt+QAdZnA==" + }, + "node_modules/beta": { + "version": "1.0.0", + "resolved": "file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz", + "integrity": "sha512-qdpYNuREio4Ne2pCoWYKEfqDrKR7gNVvJEi6UhkognzwSBA9PvcgPrVhGDFWlMeerfGwZzVfmHd8p6zabtkO2A==" + } + } +} diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/nuget/reverted/nuget.config b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/nuget/reverted/nuget.config new file mode 100644 index 000000000..38a5012d4 --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/nuget/reverted/nuget.config @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/nuget/wired/.socket/vendor/state.json b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/nuget/wired/.socket/vendor/state.json new file mode 100644 index 000000000..2fa68334b --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/nuget/wired/.socket/vendor/state.json @@ -0,0 +1,121 @@ +{ + "version": 1, + "entries": { + "pkg:nuget/Fx.Alpha@1.0.0": { + "ecosystem": "nuget", + "basePurl": "pkg:nuget/Fx.Alpha@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000001", + "artifact": { + "path": ".socket/vendor/nuget/11111111-1111-4111-8111-000000000001/fx.alpha.1.0.0.nupkg", + "sha256": "7775dcdcad772d88a77852489b5c6790d22d9d1383638da6888fd67e35e15130", + "size": 450 + }, + "wiring": [ + { + "file": "nuget.config", + "kind": "nuget_config_source", + "action": "rewritten", + "key": "socket-patch-11111111-1111-4111-8111-000000000001", + "original": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", + "new": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n" + }, + { + "file": "nuget.config", + "kind": "nuget_config_mapping", + "action": "added", + "key": "Fx.Alpha", + "new": " \n \n \n" + }, + { + "file": "packages.lock.json", + "kind": "nuget_lock_entry", + "action": "rewritten", + "key": "Fx.Alpha", + "original": "WvjSxPwHWUhuw5uHwuU7DSaI2iD+KAAhc1wgPJmhUIWcEPgSQgjgeJ0mbK3DGdBCayS4j3g4bX3xVxsfBUQimw==", + "new": "oJs+8+IeSAWWoJLiUglQ9cM9oZFqEwgcROMfvL3UCIpfxatepCF6wpasspgir9k59m6mUnhEfhwpIZAVGyGl1A==" + } + ], + "record": { + "uuid": "11111111-1111-4111-8111-000000000001", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "LICENSE.md": { + "beforeHash": "a113315d903cb8d12abebf88dca65557dde6f7e67f8bab1be0f3339e37dbb56b", + "afterHash": "c09bd2fa258994ecf531a17c2eb77b4a25ee0af90174dd66b53735a2fbf95260" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + }, + "pkg:nuget/Fx.Beta@1.0.0": { + "ecosystem": "nuget", + "basePurl": "pkg:nuget/Fx.Beta@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000002", + "artifact": { + "path": ".socket/vendor/nuget/11111111-1111-4111-8111-000000000002/fx.beta.1.0.0.nupkg", + "sha256": "bc06ab471673ab4369b56f94ca99212a4aa0a578eb062f811d4bb0efcc6fdcef", + "size": 446 + }, + "wiring": [ + { + "file": "nuget.config", + "kind": "nuget_config_source", + "action": "rewritten", + "key": "socket-patch-11111111-1111-4111-8111-000000000002", + "original": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", + "new": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n" + }, + { + "file": "nuget.config", + "kind": "nuget_config_mapping", + "action": "added", + "key": "Fx.Beta", + "new": " \n \n \n" + }, + { + "file": "packages.lock.json", + "kind": "nuget_lock_entry", + "action": "rewritten", + "key": "Fx.Beta", + "original": "LRYXwnxkETVJsNm6fm1wKrIroTJHlGqiGeSY8/kyIRecUE/SoEfQ6aBadCT7N7NK/m3dvNTfZJQcaSs7oAXgVQ==", + "new": "ocZmaRMoSfAmYajxMOZrx5xeCA/igWX0A/QbKOBcR9QTI6tP/H0cWUrYIyVzSY5SVgqzVkyCbZiqhisZtrcNVw==" + } + ], + "record": { + "uuid": "11111111-1111-4111-8111-000000000002", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "LICENSE.md": { + "beforeHash": "6911ab6df108f90ead8d7182f2d1112c913023a02cd438c7fbf581938bffef3e", + "afterHash": "867fe755204c3c2fa125f706d8e882f74f90ab2f5e5879ac36ce6464d5c8d691" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + } + } +} diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/nuget/wired/nuget.config b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/nuget/wired/nuget.config new file mode 100644 index 000000000..21648d88b --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/nuget/wired/nuget.config @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/nuget/wired/packages.lock.json b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/nuget/wired/packages.lock.json new file mode 100644 index 000000000..d9fdde5f1 --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/nuget/wired/packages.lock.json @@ -0,0 +1,19 @@ +{ + "version": 1, + "dependencies": { + "net8.0": { + "Fx.Alpha": { + "type": "Direct", + "requested": "[1.0.0, )", + "resolved": "1.0.0", + "contentHash": "oJs+8+IeSAWWoJLiUglQ9cM9oZFqEwgcROMfvL3UCIpfxatepCF6wpasspgir9k59m6mUnhEfhwpIZAVGyGl1A==" + }, + "Fx.Beta": { + "type": "Direct", + "requested": "[1.0.0, )", + "resolved": "1.0.0", + "contentHash": "ocZmaRMoSfAmYajxMOZrx5xeCA/igWX0A/QbKOBcR9QTI6tP/H0cWUrYIyVzSY5SVgqzVkyCbZiqhisZtrcNVw==" + } + } + } +} \ No newline at end of file diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/reverted/package.json b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/reverted/package.json new file mode 100644 index 000000000..0c21e32ea --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/reverted/package.json @@ -0,0 +1,12 @@ +{ + "name": "fixture", + "version": "1.0.0", + "private": true, + "dependencies": { + "alpha": "1.0.0", + "beta": "1.0.0" + }, + "pnpm": { + "overrides": {} + } +} diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/reverted/pnpm-workspace.yaml b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/reverted/pnpm-workspace.yaml new file mode 100644 index 000000000..36659cdc6 --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/reverted/pnpm-workspace.yaml @@ -0,0 +1,3 @@ +packages: + - '.' +overrides: diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/wired/.socket/vendor/state.json b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/wired/.socket/vendor/state.json new file mode 100644 index 000000000..88cec8469 --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/wired/.socket/vendor/state.json @@ -0,0 +1,207 @@ +{ + "version": 1, + "entries": { + "pkg:npm/alpha@1.0.0": { + "ecosystem": "npm", + "basePurl": "pkg:npm/alpha@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000001", + "artifact": { + "path": ".socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz", + "sha256": "9750489c2d19b3ada3dde4a408b894e0f63cc73b9e0c0bcd80a8abef5e4f82c4", + "size": 178 + }, + "wiring": [ + { + "file": "package.json", + "kind": "pnpm_pkg_override", + "action": "added", + "key": "alpha@1.0.0", + "new": "file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz" + }, + { + "file": "pnpm-lock.yaml", + "kind": "pnpm_lock_overrides", + "action": "added", + "key": "alpha@1.0.0", + "new": "file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz" + }, + { + "file": "pnpm-lock.yaml", + "kind": "pnpm_lock_importer_dep", + "action": "rewritten", + "key": ".|alpha", + "original": { + "specifier": "1.0.0", + "version": "1.0.0" + }, + "new": { + "specifier": "file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz", + "version": "file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz" + } + }, + { + "file": "pnpm-lock.yaml", + "kind": "pnpm_lock_package", + "action": "rewritten", + "key": "alpha@1.0.0", + "original": [ + " alpha@1.0.0:", + " resolution: {integrity: sha512-orig==}" + ], + "new": [ + " alpha@file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz:", + " resolution: {integrity: sha512-kmYLZ8QKvcFKA3NQaxXFnIwOjKLrsfUVj5XeSUIOlaYzy5VKZ+ZbIaRpwvt7crrAd5x6llAVcTVLvWt+QAdZnA==, tarball: file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz}", + " version: 1.0.0" + ] + }, + { + "file": "pnpm-lock.yaml", + "kind": "pnpm_lock_snapshot", + "action": "rewritten", + "key": "alpha@1.0.0", + "original": [ + " alpha@1.0.0: {}" + ], + "new": [ + " alpha@file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz: {}" + ] + }, + { + "file": "pnpm-workspace.yaml", + "kind": "pnpm_ws_override", + "action": "added", + "key": "alpha@1.0.0", + "new": "file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz" + } + ], + "flavor": "pnpm", + "pnpm": { + "createdOverridesTable": true, + "createdPnpmTable": true, + "createdWorkspaceFile": true + }, + "record": { + "uuid": "11111111-1111-4111-8111-000000000001", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "package/index.js": { + "beforeHash": "4717263f295eed420954cedf06613ed753d8ed50ab048a60aea61c43d3b9ad9d", + "afterHash": "bc15782a928574cb0d3864cfa343d525c642e845a470e3e64aafeba864635284" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + }, + "pkg:npm/beta@1.0.0": { + "ecosystem": "npm", + "basePurl": "pkg:npm/beta@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000002", + "artifact": { + "path": ".socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz", + "sha256": "15a787bf25ccefc1d643efe9b157a0703dad1837d3d6fef1dad2177738ca0aac", + "size": 177 + }, + "wiring": [ + { + "file": "package.json", + "kind": "pnpm_pkg_override", + "action": "added", + "key": "beta@1.0.0", + "new": "file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz" + }, + { + "file": "pnpm-lock.yaml", + "kind": "pnpm_lock_overrides", + "action": "added", + "key": "beta@1.0.0", + "new": "file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz" + }, + { + "file": "pnpm-lock.yaml", + "kind": "pnpm_lock_importer_dep", + "action": "rewritten", + "key": ".|beta", + "original": { + "specifier": "1.0.0", + "version": "1.0.0" + }, + "new": { + "specifier": "file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz", + "version": "file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz" + } + }, + { + "file": "pnpm-lock.yaml", + "kind": "pnpm_lock_package", + "action": "rewritten", + "key": "beta@1.0.0", + "original": [ + " beta@1.0.0:", + " resolution: {integrity: sha512-orig==}" + ], + "new": [ + " beta@file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz:", + " resolution: {integrity: sha512-qdpYNuREio4Ne2pCoWYKEfqDrKR7gNVvJEi6UhkognzwSBA9PvcgPrVhGDFWlMeerfGwZzVfmHd8p6zabtkO2A==, tarball: file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz}", + " version: 1.0.0" + ] + }, + { + "file": "pnpm-lock.yaml", + "kind": "pnpm_lock_snapshot", + "action": "rewritten", + "key": "beta@1.0.0", + "original": [ + " beta@1.0.0: {}" + ], + "new": [ + " beta@file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz: {}" + ] + }, + { + "file": "pnpm-workspace.yaml", + "kind": "pnpm_ws_override", + "action": "added", + "key": "beta@1.0.0", + "new": "file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz" + } + ], + "flavor": "pnpm", + "pnpm": {}, + "record": { + "uuid": "11111111-1111-4111-8111-000000000002", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "package/index.js": { + "beforeHash": "64e197dfafd58f0f80310669f32563a0bd078317decf19152868b6a5d20ede42", + "afterHash": "0c82d6e011aad77f6178863228991fb143135c15123ce7e17d7c6d700614b273" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + } + } +} diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/wired/package.json b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/wired/package.json new file mode 100644 index 000000000..007544fee --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/wired/package.json @@ -0,0 +1,15 @@ +{ + "name": "fixture", + "version": "1.0.0", + "private": true, + "dependencies": { + "alpha": "1.0.0", + "beta": "1.0.0" + }, + "pnpm": { + "overrides": { + "alpha@1.0.0": "file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz", + "beta@1.0.0": "file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz" + } + } +} diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/wired/pnpm-lock.yaml b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/wired/pnpm-lock.yaml new file mode 100644 index 000000000..03d512d80 --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/wired/pnpm-lock.yaml @@ -0,0 +1,29 @@ +lockfileVersion: '9.0' + +overrides: + alpha@1.0.0: file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz + beta@1.0.0: file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz + +importers: + .: + dependencies: + alpha: + specifier: file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz + version: file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz + beta: + specifier: file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz + version: file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz + +packages: + alpha@file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz: + resolution: {integrity: sha512-kmYLZ8QKvcFKA3NQaxXFnIwOjKLrsfUVj5XeSUIOlaYzy5VKZ+ZbIaRpwvt7crrAd5x6llAVcTVLvWt+QAdZnA==, tarball: file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz} + version: 1.0.0 + + beta@file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz: + resolution: {integrity: sha512-qdpYNuREio4Ne2pCoWYKEfqDrKR7gNVvJEi6UhkognzwSBA9PvcgPrVhGDFWlMeerfGwZzVfmHd8p6zabtkO2A==, tarball: file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz} + version: 1.0.0 + +snapshots: + alpha@file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz: {} + + beta@file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz: {} diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/wired/pnpm-workspace.yaml b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/wired/pnpm-workspace.yaml new file mode 100644 index 000000000..a14976921 --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pnpm/wired/pnpm-workspace.yaml @@ -0,0 +1,5 @@ +packages: + - '.' +overrides: + alpha@1.0.0: file:.socket/vendor/npm/11111111-1111-4111-8111-000000000001/alpha-1.0.0.tgz + beta@1.0.0: file:.socket/vendor/npm/11111111-1111-4111-8111-000000000002/beta-1.0.0.tgz diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pylock/wired/.socket/vendor/state.json b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pylock/wired/.socket/vendor/state.json new file mode 100644 index 000000000..56e0044f3 --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pylock/wired/.socket/vendor/state.json @@ -0,0 +1,93 @@ +{ + "version": 1, + "entries": { + "pkg:pypi/alpha@1.0.0": { + "ecosystem": "pypi", + "basePurl": "pkg:pypi/alpha@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000001", + "artifact": { + "path": ".socket/vendor/pypi/11111111-1111-4111-8111-000000000001/alpha-1.0.0-py3-none-any.whl", + "sha256": "9064609f1885c0bffb2e651f3339a6833010a9d388d2dadb5ddc2a629129574d", + "size": 837 + }, + "wiring": [ + { + "file": "pylock.toml", + "kind": "python_lock_document", + "action": "rewritten", + "key": "alpha", + "original": "lock-version = \"1.0\"\ncreated-by = \"fixture\"\n\n[[packages]]\nname = \"filler00\"\nversion = \"1.0.0\"\nwheels = [{ url = \"https://files.example/filler00-1.0.0-py3-none-any.whl\", hashes = { sha256 = \"0000000000000000000000000000000000000000000000000000000000000000\" } }]\n\n[[packages]]\nname = \"filler01\"\nversion = \"1.0.1\"\nwheels = [{ url = \"https://files.example/filler01-1.0.1-py3-none-any.whl\", hashes = { sha256 = \"0101010101010101010101010101010101010101010101010101010101010101\" } }]\n\n[[packages]]\nname = \"filler02\"\nversion = \"1.0.2\"\nwheels = [{ url = \"https://files.example/filler02-1.0.2-py3-none-any.whl\", hashes = { sha256 = \"0202020202020202020202020202020202020202020202020202020202020202\" } }]\n\n[[packages]]\nname = \"filler03\"\nversion = \"1.0.3\"\nwheels = [{ url = \"https://files.example/filler03-1.0.3-py3-none-any.whl\", hashes = { sha256 = \"0303030303030303030303030303030303030303030303030303030303030303\" } }]\n\n[[packages]]\nname = \"filler04\"\nversion = \"1.0.4\"\nwheels = [{ url = \"https://files.example/filler04-1.0.4-py3-none-any.whl\", hashes = { sha256 = \"0404040404040404040404040404040404040404040404040404040404040404\" } }]\n\n[[packages]]\nname = \"filler05\"\nversion = \"1.0.5\"\nwheels = [{ url = \"https://files.example/filler05-1.0.5-py3-none-any.whl\", hashes = { sha256 = \"0505050505050505050505050505050505050505050505050505050505050505\" } }]\n\n[[packages]]\nname = \"filler06\"\nversion = \"1.0.6\"\nwheels = [{ url = \"https://files.example/filler06-1.0.6-py3-none-any.whl\", hashes = { sha256 = \"0606060606060606060606060606060606060606060606060606060606060606\" } }]\n\n[[packages]]\nname = \"filler07\"\nversion = \"1.0.7\"\nwheels = [{ url = \"https://files.example/filler07-1.0.7-py3-none-any.whl\", hashes = { sha256 = \"0707070707070707070707070707070707070707070707070707070707070707\" } }]\n\n[[packages]]\nname = \"filler08\"\nversion = \"1.0.8\"\nwheels = [{ url = \"https://files.example/filler08-1.0.8-py3-none-any.whl\", hashes = { sha256 = \"0808080808080808080808080808080808080808080808080808080808080808\" } }]\n\n[[packages]]\nname = \"filler09\"\nversion = \"1.0.9\"\nwheels = [{ url = \"https://files.example/filler09-1.0.9-py3-none-any.whl\", hashes = { sha256 = \"0909090909090909090909090909090909090909090909090909090909090909\" } }]\n\n[[packages]]\nname = \"filler10\"\nversion = \"1.0.10\"\nwheels = [{ url = \"https://files.example/filler10-1.0.10-py3-none-any.whl\", hashes = { sha256 = \"1010101010101010101010101010101010101010101010101010101010101010\" } }]\n\n[[packages]]\nname = \"filler11\"\nversion = \"1.0.11\"\nwheels = [{ url = \"https://files.example/filler11-1.0.11-py3-none-any.whl\", hashes = { sha256 = \"1111111111111111111111111111111111111111111111111111111111111111\" } }]\n\n[[packages]]\nname = \"filler12\"\nversion = \"1.0.12\"\nwheels = [{ url = \"https://files.example/filler12-1.0.12-py3-none-any.whl\", hashes = { sha256 = \"1212121212121212121212121212121212121212121212121212121212121212\" } }]\n\n[[packages]]\nname = \"filler13\"\nversion = \"1.0.13\"\nwheels = [{ url = \"https://files.example/filler13-1.0.13-py3-none-any.whl\", hashes = { sha256 = \"1313131313131313131313131313131313131313131313131313131313131313\" } }]\n\n[[packages]]\nname = \"filler14\"\nversion = \"1.0.14\"\nwheels = [{ url = \"https://files.example/filler14-1.0.14-py3-none-any.whl\", hashes = { sha256 = \"1414141414141414141414141414141414141414141414141414141414141414\" } }]\n\n[[packages]]\nname = \"filler15\"\nversion = \"1.0.15\"\nwheels = [{ url = \"https://files.example/filler15-1.0.15-py3-none-any.whl\", hashes = { sha256 = \"1515151515151515151515151515151515151515151515151515151515151515\" } }]\n\n[[packages]]\nname = \"filler16\"\nversion = \"1.0.16\"\nwheels = [{ url = \"https://files.example/filler16-1.0.16-py3-none-any.whl\", hashes = { sha256 = \"1616161616161616161616161616161616161616161616161616161616161616\" } }]\n\n[[packages]]\nname = \"filler17\"\nversion = \"1.0.17\"\nwheels = [{ url = \"https://files.example/filler17-1.0.17-py3-none-any.whl\", hashes = { sha256 = \"1717171717171717171717171717171717171717171717171717171717171717\" } }]\n\n[[packages]]\nname = \"filler18\"\nversion = \"1.0.18\"\nwheels = [{ url = \"https://files.example/filler18-1.0.18-py3-none-any.whl\", hashes = { sha256 = \"1818181818181818181818181818181818181818181818181818181818181818\" } }]\n\n[[packages]]\nname = \"filler19\"\nversion = \"1.0.19\"\nwheels = [{ url = \"https://files.example/filler19-1.0.19-py3-none-any.whl\", hashes = { sha256 = \"1919191919191919191919191919191919191919191919191919191919191919\" } }]\n\n[[packages]]\nname = \"filler20\"\nversion = \"1.0.20\"\nwheels = [{ url = \"https://files.example/filler20-1.0.20-py3-none-any.whl\", hashes = { sha256 = \"2020202020202020202020202020202020202020202020202020202020202020\" } }]\n\n[[packages]]\nname = \"filler21\"\nversion = \"1.0.21\"\nwheels = [{ url = \"https://files.example/filler21-1.0.21-py3-none-any.whl\", hashes = { sha256 = \"2121212121212121212121212121212121212121212121212121212121212121\" } }]\n\n[[packages]]\nname = \"filler22\"\nversion = \"1.0.22\"\nwheels = [{ url = \"https://files.example/filler22-1.0.22-py3-none-any.whl\", hashes = { sha256 = \"2222222222222222222222222222222222222222222222222222222222222222\" } }]\n\n[[packages]]\nname = \"filler23\"\nversion = \"1.0.23\"\nwheels = [{ url = \"https://files.example/filler23-1.0.23-py3-none-any.whl\", hashes = { sha256 = \"2323232323232323232323232323232323232323232323232323232323232323\" } }]\n\n[[packages]]\nname = \"filler24\"\nversion = \"1.0.24\"\nwheels = [{ url = \"https://files.example/filler24-1.0.24-py3-none-any.whl\", hashes = { sha256 = \"2424242424242424242424242424242424242424242424242424242424242424\" } }]\n\n[[packages]]\nname = \"filler25\"\nversion = \"1.0.25\"\nwheels = [{ url = \"https://files.example/filler25-1.0.25-py3-none-any.whl\", hashes = { sha256 = \"2525252525252525252525252525252525252525252525252525252525252525\" } }]\n\n[[packages]]\nname = \"filler26\"\nversion = \"1.0.26\"\nwheels = [{ url = \"https://files.example/filler26-1.0.26-py3-none-any.whl\", hashes = { sha256 = \"2626262626262626262626262626262626262626262626262626262626262626\" } }]\n\n[[packages]]\nname = \"filler27\"\nversion = \"1.0.27\"\nwheels = [{ url = \"https://files.example/filler27-1.0.27-py3-none-any.whl\", hashes = { sha256 = \"2727272727272727272727272727272727272727272727272727272727272727\" } }]\n\n[[packages]]\nname = \"filler28\"\nversion = \"1.0.28\"\nwheels = [{ url = \"https://files.example/filler28-1.0.28-py3-none-any.whl\", hashes = { sha256 = \"2828282828282828282828282828282828282828282828282828282828282828\" } }]\n\n[[packages]]\nname = \"filler29\"\nversion = \"1.0.29\"\nwheels = [{ url = \"https://files.example/filler29-1.0.29-py3-none-any.whl\", hashes = { sha256 = \"2929292929292929292929292929292929292929292929292929292929292929\" } }]\n\n[[packages]]\nname = \"filler30\"\nversion = \"1.0.30\"\nwheels = [{ url = \"https://files.example/filler30-1.0.30-py3-none-any.whl\", hashes = { sha256 = \"3030303030303030303030303030303030303030303030303030303030303030\" } }]\n\n[[packages]]\nname = \"filler31\"\nversion = \"1.0.31\"\nwheels = [{ url = \"https://files.example/filler31-1.0.31-py3-none-any.whl\", hashes = { sha256 = \"3131313131313131313131313131313131313131313131313131313131313131\" } }]\n\n[[packages]]\nname = \"filler32\"\nversion = \"1.0.32\"\nwheels = [{ url = \"https://files.example/filler32-1.0.32-py3-none-any.whl\", hashes = { sha256 = \"3232323232323232323232323232323232323232323232323232323232323232\" } }]\n\n[[packages]]\nname = \"filler33\"\nversion = \"1.0.33\"\nwheels = [{ url = \"https://files.example/filler33-1.0.33-py3-none-any.whl\", hashes = { sha256 = \"3333333333333333333333333333333333333333333333333333333333333333\" } }]\n\n[[packages]]\nname = \"filler34\"\nversion = \"1.0.34\"\nwheels = [{ url = \"https://files.example/filler34-1.0.34-py3-none-any.whl\", hashes = { sha256 = \"3434343434343434343434343434343434343434343434343434343434343434\" } }]\n\n[[packages]]\nname = \"filler35\"\nversion = \"1.0.35\"\nwheels = [{ url = \"https://files.example/filler35-1.0.35-py3-none-any.whl\", hashes = { sha256 = \"3535353535353535353535353535353535353535353535353535353535353535\" } }]\n\n[[packages]]\nname = \"filler36\"\nversion = \"1.0.36\"\nwheels = [{ url = \"https://files.example/filler36-1.0.36-py3-none-any.whl\", hashes = { sha256 = \"3636363636363636363636363636363636363636363636363636363636363636\" } }]\n\n[[packages]]\nname = \"filler37\"\nversion = \"1.0.37\"\nwheels = [{ url = \"https://files.example/filler37-1.0.37-py3-none-any.whl\", hashes = { sha256 = \"3737373737373737373737373737373737373737373737373737373737373737\" } }]\n\n[[packages]]\nname = \"filler38\"\nversion = \"1.0.38\"\nwheels = [{ url = \"https://files.example/filler38-1.0.38-py3-none-any.whl\", hashes = { sha256 = \"3838383838383838383838383838383838383838383838383838383838383838\" } }]\n\n[[packages]]\nname = \"filler39\"\nversion = \"1.0.39\"\nwheels = [{ url = \"https://files.example/filler39-1.0.39-py3-none-any.whl\", hashes = { sha256 = \"3939393939393939393939393939393939393939393939393939393939393939\" } }]\n\n[[packages]]\nname = \"alpha\"\nversion = \"1.0.0\"\nwheels = [{ url = \"https://files.example/alpha-1.0.0-py3-none-any.whl\", hashes = { sha256 = \"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\" } }]\n\n[[packages]]\nname = \"beta\"\nversion = \"1.0.0\"\nwheels = [{ url = \"https://files.example/beta-1.0.0-py3-none-any.whl\", hashes = { sha256 = \"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\" } }]\n", + "new": "lock-version = \"1.0\"\ncreated-by = \"fixture\"\n\n[[packages]]\nname = \"filler00\"\nversion = \"1.0.0\"\nwheels = [{ url = \"https://files.example/filler00-1.0.0-py3-none-any.whl\", hashes = { sha256 = \"0000000000000000000000000000000000000000000000000000000000000000\" } }]\n\n[[packages]]\nname = \"filler01\"\nversion = \"1.0.1\"\nwheels = [{ url = \"https://files.example/filler01-1.0.1-py3-none-any.whl\", hashes = { sha256 = \"0101010101010101010101010101010101010101010101010101010101010101\" } }]\n\n[[packages]]\nname = \"filler02\"\nversion = \"1.0.2\"\nwheels = [{ url = \"https://files.example/filler02-1.0.2-py3-none-any.whl\", hashes = { sha256 = \"0202020202020202020202020202020202020202020202020202020202020202\" } }]\n\n[[packages]]\nname = \"filler03\"\nversion = \"1.0.3\"\nwheels = [{ url = \"https://files.example/filler03-1.0.3-py3-none-any.whl\", hashes = { sha256 = \"0303030303030303030303030303030303030303030303030303030303030303\" } }]\n\n[[packages]]\nname = \"filler04\"\nversion = \"1.0.4\"\nwheels = [{ url = \"https://files.example/filler04-1.0.4-py3-none-any.whl\", hashes = { sha256 = \"0404040404040404040404040404040404040404040404040404040404040404\" } }]\n\n[[packages]]\nname = \"filler05\"\nversion = \"1.0.5\"\nwheels = [{ url = \"https://files.example/filler05-1.0.5-py3-none-any.whl\", hashes = { sha256 = \"0505050505050505050505050505050505050505050505050505050505050505\" } }]\n\n[[packages]]\nname = \"filler06\"\nversion = \"1.0.6\"\nwheels = [{ url = \"https://files.example/filler06-1.0.6-py3-none-any.whl\", hashes = { sha256 = \"0606060606060606060606060606060606060606060606060606060606060606\" } }]\n\n[[packages]]\nname = \"filler07\"\nversion = \"1.0.7\"\nwheels = [{ url = \"https://files.example/filler07-1.0.7-py3-none-any.whl\", hashes = { sha256 = \"0707070707070707070707070707070707070707070707070707070707070707\" } }]\n\n[[packages]]\nname = \"filler08\"\nversion = \"1.0.8\"\nwheels = [{ url = \"https://files.example/filler08-1.0.8-py3-none-any.whl\", hashes = { sha256 = \"0808080808080808080808080808080808080808080808080808080808080808\" } }]\n\n[[packages]]\nname = \"filler09\"\nversion = \"1.0.9\"\nwheels = [{ url = \"https://files.example/filler09-1.0.9-py3-none-any.whl\", hashes = { sha256 = \"0909090909090909090909090909090909090909090909090909090909090909\" } }]\n\n[[packages]]\nname = \"filler10\"\nversion = \"1.0.10\"\nwheels = [{ url = \"https://files.example/filler10-1.0.10-py3-none-any.whl\", hashes = { sha256 = \"1010101010101010101010101010101010101010101010101010101010101010\" } }]\n\n[[packages]]\nname = \"filler11\"\nversion = \"1.0.11\"\nwheels = [{ url = \"https://files.example/filler11-1.0.11-py3-none-any.whl\", hashes = { sha256 = \"1111111111111111111111111111111111111111111111111111111111111111\" } }]\n\n[[packages]]\nname = \"filler12\"\nversion = \"1.0.12\"\nwheels = [{ url = \"https://files.example/filler12-1.0.12-py3-none-any.whl\", hashes = { sha256 = \"1212121212121212121212121212121212121212121212121212121212121212\" } }]\n\n[[packages]]\nname = \"filler13\"\nversion = \"1.0.13\"\nwheels = [{ url = \"https://files.example/filler13-1.0.13-py3-none-any.whl\", hashes = { sha256 = \"1313131313131313131313131313131313131313131313131313131313131313\" } }]\n\n[[packages]]\nname = \"filler14\"\nversion = \"1.0.14\"\nwheels = [{ url = \"https://files.example/filler14-1.0.14-py3-none-any.whl\", hashes = { sha256 = \"1414141414141414141414141414141414141414141414141414141414141414\" } }]\n\n[[packages]]\nname = \"filler15\"\nversion = \"1.0.15\"\nwheels = [{ url = \"https://files.example/filler15-1.0.15-py3-none-any.whl\", hashes = { sha256 = \"1515151515151515151515151515151515151515151515151515151515151515\" } }]\n\n[[packages]]\nname = \"filler16\"\nversion = \"1.0.16\"\nwheels = [{ url = \"https://files.example/filler16-1.0.16-py3-none-any.whl\", hashes = { sha256 = \"1616161616161616161616161616161616161616161616161616161616161616\" } }]\n\n[[packages]]\nname = \"filler17\"\nversion = \"1.0.17\"\nwheels = [{ url = \"https://files.example/filler17-1.0.17-py3-none-any.whl\", hashes = { sha256 = \"1717171717171717171717171717171717171717171717171717171717171717\" } }]\n\n[[packages]]\nname = \"filler18\"\nversion = \"1.0.18\"\nwheels = [{ url = \"https://files.example/filler18-1.0.18-py3-none-any.whl\", hashes = { sha256 = \"1818181818181818181818181818181818181818181818181818181818181818\" } }]\n\n[[packages]]\nname = \"filler19\"\nversion = \"1.0.19\"\nwheels = [{ url = \"https://files.example/filler19-1.0.19-py3-none-any.whl\", hashes = { sha256 = \"1919191919191919191919191919191919191919191919191919191919191919\" } }]\n\n[[packages]]\nname = \"filler20\"\nversion = \"1.0.20\"\nwheels = [{ url = \"https://files.example/filler20-1.0.20-py3-none-any.whl\", hashes = { sha256 = \"2020202020202020202020202020202020202020202020202020202020202020\" } }]\n\n[[packages]]\nname = \"filler21\"\nversion = \"1.0.21\"\nwheels = [{ url = \"https://files.example/filler21-1.0.21-py3-none-any.whl\", hashes = { sha256 = \"2121212121212121212121212121212121212121212121212121212121212121\" } }]\n\n[[packages]]\nname = \"filler22\"\nversion = \"1.0.22\"\nwheels = [{ url = \"https://files.example/filler22-1.0.22-py3-none-any.whl\", hashes = { sha256 = \"2222222222222222222222222222222222222222222222222222222222222222\" } }]\n\n[[packages]]\nname = \"filler23\"\nversion = \"1.0.23\"\nwheels = [{ url = \"https://files.example/filler23-1.0.23-py3-none-any.whl\", hashes = { sha256 = \"2323232323232323232323232323232323232323232323232323232323232323\" } }]\n\n[[packages]]\nname = \"filler24\"\nversion = \"1.0.24\"\nwheels = [{ url = \"https://files.example/filler24-1.0.24-py3-none-any.whl\", hashes = { sha256 = \"2424242424242424242424242424242424242424242424242424242424242424\" } }]\n\n[[packages]]\nname = \"filler25\"\nversion = \"1.0.25\"\nwheels = [{ url = \"https://files.example/filler25-1.0.25-py3-none-any.whl\", hashes = { sha256 = \"2525252525252525252525252525252525252525252525252525252525252525\" } }]\n\n[[packages]]\nname = \"filler26\"\nversion = \"1.0.26\"\nwheels = [{ url = \"https://files.example/filler26-1.0.26-py3-none-any.whl\", hashes = { sha256 = \"2626262626262626262626262626262626262626262626262626262626262626\" } }]\n\n[[packages]]\nname = \"filler27\"\nversion = \"1.0.27\"\nwheels = [{ url = \"https://files.example/filler27-1.0.27-py3-none-any.whl\", hashes = { sha256 = \"2727272727272727272727272727272727272727272727272727272727272727\" } }]\n\n[[packages]]\nname = \"filler28\"\nversion = \"1.0.28\"\nwheels = [{ url = \"https://files.example/filler28-1.0.28-py3-none-any.whl\", hashes = { sha256 = \"2828282828282828282828282828282828282828282828282828282828282828\" } }]\n\n[[packages]]\nname = \"filler29\"\nversion = \"1.0.29\"\nwheels = [{ url = \"https://files.example/filler29-1.0.29-py3-none-any.whl\", hashes = { sha256 = \"2929292929292929292929292929292929292929292929292929292929292929\" } }]\n\n[[packages]]\nname = \"filler30\"\nversion = \"1.0.30\"\nwheels = [{ url = \"https://files.example/filler30-1.0.30-py3-none-any.whl\", hashes = { sha256 = \"3030303030303030303030303030303030303030303030303030303030303030\" } }]\n\n[[packages]]\nname = \"filler31\"\nversion = \"1.0.31\"\nwheels = [{ url = \"https://files.example/filler31-1.0.31-py3-none-any.whl\", hashes = { sha256 = \"3131313131313131313131313131313131313131313131313131313131313131\" } }]\n\n[[packages]]\nname = \"filler32\"\nversion = \"1.0.32\"\nwheels = [{ url = \"https://files.example/filler32-1.0.32-py3-none-any.whl\", hashes = { sha256 = \"3232323232323232323232323232323232323232323232323232323232323232\" } }]\n\n[[packages]]\nname = \"filler33\"\nversion = \"1.0.33\"\nwheels = [{ url = \"https://files.example/filler33-1.0.33-py3-none-any.whl\", hashes = { sha256 = \"3333333333333333333333333333333333333333333333333333333333333333\" } }]\n\n[[packages]]\nname = \"filler34\"\nversion = \"1.0.34\"\nwheels = [{ url = \"https://files.example/filler34-1.0.34-py3-none-any.whl\", hashes = { sha256 = \"3434343434343434343434343434343434343434343434343434343434343434\" } }]\n\n[[packages]]\nname = \"filler35\"\nversion = \"1.0.35\"\nwheels = [{ url = \"https://files.example/filler35-1.0.35-py3-none-any.whl\", hashes = { sha256 = \"3535353535353535353535353535353535353535353535353535353535353535\" } }]\n\n[[packages]]\nname = \"filler36\"\nversion = \"1.0.36\"\nwheels = [{ url = \"https://files.example/filler36-1.0.36-py3-none-any.whl\", hashes = { sha256 = \"3636363636363636363636363636363636363636363636363636363636363636\" } }]\n\n[[packages]]\nname = \"filler37\"\nversion = \"1.0.37\"\nwheels = [{ url = \"https://files.example/filler37-1.0.37-py3-none-any.whl\", hashes = { sha256 = \"3737373737373737373737373737373737373737373737373737373737373737\" } }]\n\n[[packages]]\nname = \"filler38\"\nversion = \"1.0.38\"\nwheels = [{ url = \"https://files.example/filler38-1.0.38-py3-none-any.whl\", hashes = { sha256 = \"3838383838383838383838383838383838383838383838383838383838383838\" } }]\n\n[[packages]]\nname = \"filler39\"\nversion = \"1.0.39\"\nwheels = [{ url = \"https://files.example/filler39-1.0.39-py3-none-any.whl\", hashes = { sha256 = \"3939393939393939393939393939393939393939393939393939393939393939\" } }]\n\n[[packages]]\nname = \"alpha\"\nversion = \"1.0.0\"\narchive = { path = \".socket/vendor/pypi/11111111-1111-4111-8111-000000000001/alpha-1.0.0-py3-none-any.whl\", hashes = { sha256 = \"9064609f1885c0bffb2e651f3339a6833010a9d388d2dadb5ddc2a629129574d\" } }\n\n[[packages]]\nname = \"beta\"\nversion = \"1.0.0\"\nwheels = [{ url = \"https://files.example/beta-1.0.0-py3-none-any.whl\", hashes = { sha256 = \"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\" } }]\n" + } + ], + "flavor": "python-lock", + "record": { + "uuid": "11111111-1111-4111-8111-000000000001", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "alpha.py": { + "beforeHash": "490ce2042e030d4655a2fddaf5f85d98215f64866fe924652851388b80b32d66", + "afterHash": "3171463aa7c6f59eb47b7ef9c888b14739f6da2be164d5b6ebfdb6b5bdb6fb21" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + }, + "pkg:pypi/beta@1.0.0": { + "ecosystem": "pypi", + "basePurl": "pkg:pypi/beta@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000002", + "artifact": { + "path": ".socket/vendor/pypi/11111111-1111-4111-8111-000000000002/beta-1.0.0-py3-none-any.whl", + "sha256": "5f2338b259764f7a63e5a9234c9b18d7ccd556bd76f6b31da48c4e2018c0e17e", + "size": 824 + }, + "wiring": [ + { + "file": "pylock.toml", + "kind": "python_lock_document", + "action": "rewritten", + "key": "beta", + "original": "lock-version = \"1.0\"\ncreated-by = \"fixture\"\n\n[[packages]]\nname = \"filler00\"\nversion = \"1.0.0\"\nwheels = [{ url = \"https://files.example/filler00-1.0.0-py3-none-any.whl\", hashes = { sha256 = \"0000000000000000000000000000000000000000000000000000000000000000\" } }]\n\n[[packages]]\nname = \"filler01\"\nversion = \"1.0.1\"\nwheels = [{ url = \"https://files.example/filler01-1.0.1-py3-none-any.whl\", hashes = { sha256 = \"0101010101010101010101010101010101010101010101010101010101010101\" } }]\n\n[[packages]]\nname = \"filler02\"\nversion = \"1.0.2\"\nwheels = [{ url = \"https://files.example/filler02-1.0.2-py3-none-any.whl\", hashes = { sha256 = \"0202020202020202020202020202020202020202020202020202020202020202\" } }]\n\n[[packages]]\nname = \"filler03\"\nversion = \"1.0.3\"\nwheels = [{ url = \"https://files.example/filler03-1.0.3-py3-none-any.whl\", hashes = { sha256 = \"0303030303030303030303030303030303030303030303030303030303030303\" } }]\n\n[[packages]]\nname = \"filler04\"\nversion = \"1.0.4\"\nwheels = [{ url = \"https://files.example/filler04-1.0.4-py3-none-any.whl\", hashes = { sha256 = \"0404040404040404040404040404040404040404040404040404040404040404\" } }]\n\n[[packages]]\nname = \"filler05\"\nversion = \"1.0.5\"\nwheels = [{ url = \"https://files.example/filler05-1.0.5-py3-none-any.whl\", hashes = { sha256 = \"0505050505050505050505050505050505050505050505050505050505050505\" } }]\n\n[[packages]]\nname = \"filler06\"\nversion = \"1.0.6\"\nwheels = [{ url = \"https://files.example/filler06-1.0.6-py3-none-any.whl\", hashes = { sha256 = \"0606060606060606060606060606060606060606060606060606060606060606\" } }]\n\n[[packages]]\nname = \"filler07\"\nversion = \"1.0.7\"\nwheels = [{ url = \"https://files.example/filler07-1.0.7-py3-none-any.whl\", hashes = { sha256 = \"0707070707070707070707070707070707070707070707070707070707070707\" } }]\n\n[[packages]]\nname = \"filler08\"\nversion = \"1.0.8\"\nwheels = [{ url = \"https://files.example/filler08-1.0.8-py3-none-any.whl\", hashes = { sha256 = \"0808080808080808080808080808080808080808080808080808080808080808\" } }]\n\n[[packages]]\nname = \"filler09\"\nversion = \"1.0.9\"\nwheels = [{ url = \"https://files.example/filler09-1.0.9-py3-none-any.whl\", hashes = { sha256 = \"0909090909090909090909090909090909090909090909090909090909090909\" } }]\n\n[[packages]]\nname = \"filler10\"\nversion = \"1.0.10\"\nwheels = [{ url = \"https://files.example/filler10-1.0.10-py3-none-any.whl\", hashes = { sha256 = \"1010101010101010101010101010101010101010101010101010101010101010\" } }]\n\n[[packages]]\nname = \"filler11\"\nversion = \"1.0.11\"\nwheels = [{ url = \"https://files.example/filler11-1.0.11-py3-none-any.whl\", hashes = { sha256 = \"1111111111111111111111111111111111111111111111111111111111111111\" } }]\n\n[[packages]]\nname = \"filler12\"\nversion = \"1.0.12\"\nwheels = [{ url = \"https://files.example/filler12-1.0.12-py3-none-any.whl\", hashes = { sha256 = \"1212121212121212121212121212121212121212121212121212121212121212\" } }]\n\n[[packages]]\nname = \"filler13\"\nversion = \"1.0.13\"\nwheels = [{ url = \"https://files.example/filler13-1.0.13-py3-none-any.whl\", hashes = { sha256 = \"1313131313131313131313131313131313131313131313131313131313131313\" } }]\n\n[[packages]]\nname = \"filler14\"\nversion = \"1.0.14\"\nwheels = [{ url = \"https://files.example/filler14-1.0.14-py3-none-any.whl\", hashes = { sha256 = \"1414141414141414141414141414141414141414141414141414141414141414\" } }]\n\n[[packages]]\nname = \"filler15\"\nversion = \"1.0.15\"\nwheels = [{ url = \"https://files.example/filler15-1.0.15-py3-none-any.whl\", hashes = { sha256 = \"1515151515151515151515151515151515151515151515151515151515151515\" } }]\n\n[[packages]]\nname = \"filler16\"\nversion = \"1.0.16\"\nwheels = [{ url = \"https://files.example/filler16-1.0.16-py3-none-any.whl\", hashes = { sha256 = \"1616161616161616161616161616161616161616161616161616161616161616\" } }]\n\n[[packages]]\nname = \"filler17\"\nversion = \"1.0.17\"\nwheels = [{ url = \"https://files.example/filler17-1.0.17-py3-none-any.whl\", hashes = { sha256 = \"1717171717171717171717171717171717171717171717171717171717171717\" } }]\n\n[[packages]]\nname = \"filler18\"\nversion = \"1.0.18\"\nwheels = [{ url = \"https://files.example/filler18-1.0.18-py3-none-any.whl\", hashes = { sha256 = \"1818181818181818181818181818181818181818181818181818181818181818\" } }]\n\n[[packages]]\nname = \"filler19\"\nversion = \"1.0.19\"\nwheels = [{ url = \"https://files.example/filler19-1.0.19-py3-none-any.whl\", hashes = { sha256 = \"1919191919191919191919191919191919191919191919191919191919191919\" } }]\n\n[[packages]]\nname = \"filler20\"\nversion = \"1.0.20\"\nwheels = [{ url = \"https://files.example/filler20-1.0.20-py3-none-any.whl\", hashes = { sha256 = \"2020202020202020202020202020202020202020202020202020202020202020\" } }]\n\n[[packages]]\nname = \"filler21\"\nversion = \"1.0.21\"\nwheels = [{ url = \"https://files.example/filler21-1.0.21-py3-none-any.whl\", hashes = { sha256 = \"2121212121212121212121212121212121212121212121212121212121212121\" } }]\n\n[[packages]]\nname = \"filler22\"\nversion = \"1.0.22\"\nwheels = [{ url = \"https://files.example/filler22-1.0.22-py3-none-any.whl\", hashes = { sha256 = \"2222222222222222222222222222222222222222222222222222222222222222\" } }]\n\n[[packages]]\nname = \"filler23\"\nversion = \"1.0.23\"\nwheels = [{ url = \"https://files.example/filler23-1.0.23-py3-none-any.whl\", hashes = { sha256 = \"2323232323232323232323232323232323232323232323232323232323232323\" } }]\n\n[[packages]]\nname = \"filler24\"\nversion = \"1.0.24\"\nwheels = [{ url = \"https://files.example/filler24-1.0.24-py3-none-any.whl\", hashes = { sha256 = \"2424242424242424242424242424242424242424242424242424242424242424\" } }]\n\n[[packages]]\nname = \"filler25\"\nversion = \"1.0.25\"\nwheels = [{ url = \"https://files.example/filler25-1.0.25-py3-none-any.whl\", hashes = { sha256 = \"2525252525252525252525252525252525252525252525252525252525252525\" } }]\n\n[[packages]]\nname = \"filler26\"\nversion = \"1.0.26\"\nwheels = [{ url = \"https://files.example/filler26-1.0.26-py3-none-any.whl\", hashes = { sha256 = \"2626262626262626262626262626262626262626262626262626262626262626\" } }]\n\n[[packages]]\nname = \"filler27\"\nversion = \"1.0.27\"\nwheels = [{ url = \"https://files.example/filler27-1.0.27-py3-none-any.whl\", hashes = { sha256 = \"2727272727272727272727272727272727272727272727272727272727272727\" } }]\n\n[[packages]]\nname = \"filler28\"\nversion = \"1.0.28\"\nwheels = [{ url = \"https://files.example/filler28-1.0.28-py3-none-any.whl\", hashes = { sha256 = \"2828282828282828282828282828282828282828282828282828282828282828\" } }]\n\n[[packages]]\nname = \"filler29\"\nversion = \"1.0.29\"\nwheels = [{ url = \"https://files.example/filler29-1.0.29-py3-none-any.whl\", hashes = { sha256 = \"2929292929292929292929292929292929292929292929292929292929292929\" } }]\n\n[[packages]]\nname = \"filler30\"\nversion = \"1.0.30\"\nwheels = [{ url = \"https://files.example/filler30-1.0.30-py3-none-any.whl\", hashes = { sha256 = \"3030303030303030303030303030303030303030303030303030303030303030\" } }]\n\n[[packages]]\nname = \"filler31\"\nversion = \"1.0.31\"\nwheels = [{ url = \"https://files.example/filler31-1.0.31-py3-none-any.whl\", hashes = { sha256 = \"3131313131313131313131313131313131313131313131313131313131313131\" } }]\n\n[[packages]]\nname = \"filler32\"\nversion = \"1.0.32\"\nwheels = [{ url = \"https://files.example/filler32-1.0.32-py3-none-any.whl\", hashes = { sha256 = \"3232323232323232323232323232323232323232323232323232323232323232\" } }]\n\n[[packages]]\nname = \"filler33\"\nversion = \"1.0.33\"\nwheels = [{ url = \"https://files.example/filler33-1.0.33-py3-none-any.whl\", hashes = { sha256 = \"3333333333333333333333333333333333333333333333333333333333333333\" } }]\n\n[[packages]]\nname = \"filler34\"\nversion = \"1.0.34\"\nwheels = [{ url = \"https://files.example/filler34-1.0.34-py3-none-any.whl\", hashes = { sha256 = \"3434343434343434343434343434343434343434343434343434343434343434\" } }]\n\n[[packages]]\nname = \"filler35\"\nversion = \"1.0.35\"\nwheels = [{ url = \"https://files.example/filler35-1.0.35-py3-none-any.whl\", hashes = { sha256 = \"3535353535353535353535353535353535353535353535353535353535353535\" } }]\n\n[[packages]]\nname = \"filler36\"\nversion = \"1.0.36\"\nwheels = [{ url = \"https://files.example/filler36-1.0.36-py3-none-any.whl\", hashes = { sha256 = \"3636363636363636363636363636363636363636363636363636363636363636\" } }]\n\n[[packages]]\nname = \"filler37\"\nversion = \"1.0.37\"\nwheels = [{ url = \"https://files.example/filler37-1.0.37-py3-none-any.whl\", hashes = { sha256 = \"3737373737373737373737373737373737373737373737373737373737373737\" } }]\n\n[[packages]]\nname = \"filler38\"\nversion = \"1.0.38\"\nwheels = [{ url = \"https://files.example/filler38-1.0.38-py3-none-any.whl\", hashes = { sha256 = \"3838383838383838383838383838383838383838383838383838383838383838\" } }]\n\n[[packages]]\nname = \"filler39\"\nversion = \"1.0.39\"\nwheels = [{ url = \"https://files.example/filler39-1.0.39-py3-none-any.whl\", hashes = { sha256 = \"3939393939393939393939393939393939393939393939393939393939393939\" } }]\n\n[[packages]]\nname = \"alpha\"\nversion = \"1.0.0\"\narchive = { path = \".socket/vendor/pypi/11111111-1111-4111-8111-000000000001/alpha-1.0.0-py3-none-any.whl\", hashes = { sha256 = \"9064609f1885c0bffb2e651f3339a6833010a9d388d2dadb5ddc2a629129574d\" } }\n\n[[packages]]\nname = \"beta\"\nversion = \"1.0.0\"\nwheels = [{ url = \"https://files.example/beta-1.0.0-py3-none-any.whl\", hashes = { sha256 = \"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\" } }]\n", + "new": "lock-version = \"1.0\"\ncreated-by = \"fixture\"\n\n[[packages]]\nname = \"filler00\"\nversion = \"1.0.0\"\nwheels = [{ url = \"https://files.example/filler00-1.0.0-py3-none-any.whl\", hashes = { sha256 = \"0000000000000000000000000000000000000000000000000000000000000000\" } }]\n\n[[packages]]\nname = \"filler01\"\nversion = \"1.0.1\"\nwheels = [{ url = \"https://files.example/filler01-1.0.1-py3-none-any.whl\", hashes = { sha256 = \"0101010101010101010101010101010101010101010101010101010101010101\" } }]\n\n[[packages]]\nname = \"filler02\"\nversion = \"1.0.2\"\nwheels = [{ url = \"https://files.example/filler02-1.0.2-py3-none-any.whl\", hashes = { sha256 = \"0202020202020202020202020202020202020202020202020202020202020202\" } }]\n\n[[packages]]\nname = \"filler03\"\nversion = \"1.0.3\"\nwheels = [{ url = \"https://files.example/filler03-1.0.3-py3-none-any.whl\", hashes = { sha256 = \"0303030303030303030303030303030303030303030303030303030303030303\" } }]\n\n[[packages]]\nname = \"filler04\"\nversion = \"1.0.4\"\nwheels = [{ url = \"https://files.example/filler04-1.0.4-py3-none-any.whl\", hashes = { sha256 = \"0404040404040404040404040404040404040404040404040404040404040404\" } }]\n\n[[packages]]\nname = \"filler05\"\nversion = \"1.0.5\"\nwheels = [{ url = \"https://files.example/filler05-1.0.5-py3-none-any.whl\", hashes = { sha256 = \"0505050505050505050505050505050505050505050505050505050505050505\" } }]\n\n[[packages]]\nname = \"filler06\"\nversion = \"1.0.6\"\nwheels = [{ url = \"https://files.example/filler06-1.0.6-py3-none-any.whl\", hashes = { sha256 = \"0606060606060606060606060606060606060606060606060606060606060606\" } }]\n\n[[packages]]\nname = \"filler07\"\nversion = \"1.0.7\"\nwheels = [{ url = \"https://files.example/filler07-1.0.7-py3-none-any.whl\", hashes = { sha256 = \"0707070707070707070707070707070707070707070707070707070707070707\" } }]\n\n[[packages]]\nname = \"filler08\"\nversion = \"1.0.8\"\nwheels = [{ url = \"https://files.example/filler08-1.0.8-py3-none-any.whl\", hashes = { sha256 = \"0808080808080808080808080808080808080808080808080808080808080808\" } }]\n\n[[packages]]\nname = \"filler09\"\nversion = \"1.0.9\"\nwheels = [{ url = \"https://files.example/filler09-1.0.9-py3-none-any.whl\", hashes = { sha256 = \"0909090909090909090909090909090909090909090909090909090909090909\" } }]\n\n[[packages]]\nname = \"filler10\"\nversion = \"1.0.10\"\nwheels = [{ url = \"https://files.example/filler10-1.0.10-py3-none-any.whl\", hashes = { sha256 = \"1010101010101010101010101010101010101010101010101010101010101010\" } }]\n\n[[packages]]\nname = \"filler11\"\nversion = \"1.0.11\"\nwheels = [{ url = \"https://files.example/filler11-1.0.11-py3-none-any.whl\", hashes = { sha256 = \"1111111111111111111111111111111111111111111111111111111111111111\" } }]\n\n[[packages]]\nname = \"filler12\"\nversion = \"1.0.12\"\nwheels = [{ url = \"https://files.example/filler12-1.0.12-py3-none-any.whl\", hashes = { sha256 = \"1212121212121212121212121212121212121212121212121212121212121212\" } }]\n\n[[packages]]\nname = \"filler13\"\nversion = \"1.0.13\"\nwheels = [{ url = \"https://files.example/filler13-1.0.13-py3-none-any.whl\", hashes = { sha256 = \"1313131313131313131313131313131313131313131313131313131313131313\" } }]\n\n[[packages]]\nname = \"filler14\"\nversion = \"1.0.14\"\nwheels = [{ url = \"https://files.example/filler14-1.0.14-py3-none-any.whl\", hashes = { sha256 = \"1414141414141414141414141414141414141414141414141414141414141414\" } }]\n\n[[packages]]\nname = \"filler15\"\nversion = \"1.0.15\"\nwheels = [{ url = \"https://files.example/filler15-1.0.15-py3-none-any.whl\", hashes = { sha256 = \"1515151515151515151515151515151515151515151515151515151515151515\" } }]\n\n[[packages]]\nname = \"filler16\"\nversion = \"1.0.16\"\nwheels = [{ url = \"https://files.example/filler16-1.0.16-py3-none-any.whl\", hashes = { sha256 = \"1616161616161616161616161616161616161616161616161616161616161616\" } }]\n\n[[packages]]\nname = \"filler17\"\nversion = \"1.0.17\"\nwheels = [{ url = \"https://files.example/filler17-1.0.17-py3-none-any.whl\", hashes = { sha256 = \"1717171717171717171717171717171717171717171717171717171717171717\" } }]\n\n[[packages]]\nname = \"filler18\"\nversion = \"1.0.18\"\nwheels = [{ url = \"https://files.example/filler18-1.0.18-py3-none-any.whl\", hashes = { sha256 = \"1818181818181818181818181818181818181818181818181818181818181818\" } }]\n\n[[packages]]\nname = \"filler19\"\nversion = \"1.0.19\"\nwheels = [{ url = \"https://files.example/filler19-1.0.19-py3-none-any.whl\", hashes = { sha256 = \"1919191919191919191919191919191919191919191919191919191919191919\" } }]\n\n[[packages]]\nname = \"filler20\"\nversion = \"1.0.20\"\nwheels = [{ url = \"https://files.example/filler20-1.0.20-py3-none-any.whl\", hashes = { sha256 = \"2020202020202020202020202020202020202020202020202020202020202020\" } }]\n\n[[packages]]\nname = \"filler21\"\nversion = \"1.0.21\"\nwheels = [{ url = \"https://files.example/filler21-1.0.21-py3-none-any.whl\", hashes = { sha256 = \"2121212121212121212121212121212121212121212121212121212121212121\" } }]\n\n[[packages]]\nname = \"filler22\"\nversion = \"1.0.22\"\nwheels = [{ url = \"https://files.example/filler22-1.0.22-py3-none-any.whl\", hashes = { sha256 = \"2222222222222222222222222222222222222222222222222222222222222222\" } }]\n\n[[packages]]\nname = \"filler23\"\nversion = \"1.0.23\"\nwheels = [{ url = \"https://files.example/filler23-1.0.23-py3-none-any.whl\", hashes = { sha256 = \"2323232323232323232323232323232323232323232323232323232323232323\" } }]\n\n[[packages]]\nname = \"filler24\"\nversion = \"1.0.24\"\nwheels = [{ url = \"https://files.example/filler24-1.0.24-py3-none-any.whl\", hashes = { sha256 = \"2424242424242424242424242424242424242424242424242424242424242424\" } }]\n\n[[packages]]\nname = \"filler25\"\nversion = \"1.0.25\"\nwheels = [{ url = \"https://files.example/filler25-1.0.25-py3-none-any.whl\", hashes = { sha256 = \"2525252525252525252525252525252525252525252525252525252525252525\" } }]\n\n[[packages]]\nname = \"filler26\"\nversion = \"1.0.26\"\nwheels = [{ url = \"https://files.example/filler26-1.0.26-py3-none-any.whl\", hashes = { sha256 = \"2626262626262626262626262626262626262626262626262626262626262626\" } }]\n\n[[packages]]\nname = \"filler27\"\nversion = \"1.0.27\"\nwheels = [{ url = \"https://files.example/filler27-1.0.27-py3-none-any.whl\", hashes = { sha256 = \"2727272727272727272727272727272727272727272727272727272727272727\" } }]\n\n[[packages]]\nname = \"filler28\"\nversion = \"1.0.28\"\nwheels = [{ url = \"https://files.example/filler28-1.0.28-py3-none-any.whl\", hashes = { sha256 = \"2828282828282828282828282828282828282828282828282828282828282828\" } }]\n\n[[packages]]\nname = \"filler29\"\nversion = \"1.0.29\"\nwheels = [{ url = \"https://files.example/filler29-1.0.29-py3-none-any.whl\", hashes = { sha256 = \"2929292929292929292929292929292929292929292929292929292929292929\" } }]\n\n[[packages]]\nname = \"filler30\"\nversion = \"1.0.30\"\nwheels = [{ url = \"https://files.example/filler30-1.0.30-py3-none-any.whl\", hashes = { sha256 = \"3030303030303030303030303030303030303030303030303030303030303030\" } }]\n\n[[packages]]\nname = \"filler31\"\nversion = \"1.0.31\"\nwheels = [{ url = \"https://files.example/filler31-1.0.31-py3-none-any.whl\", hashes = { sha256 = \"3131313131313131313131313131313131313131313131313131313131313131\" } }]\n\n[[packages]]\nname = \"filler32\"\nversion = \"1.0.32\"\nwheels = [{ url = \"https://files.example/filler32-1.0.32-py3-none-any.whl\", hashes = { sha256 = \"3232323232323232323232323232323232323232323232323232323232323232\" } }]\n\n[[packages]]\nname = \"filler33\"\nversion = \"1.0.33\"\nwheels = [{ url = \"https://files.example/filler33-1.0.33-py3-none-any.whl\", hashes = { sha256 = \"3333333333333333333333333333333333333333333333333333333333333333\" } }]\n\n[[packages]]\nname = \"filler34\"\nversion = \"1.0.34\"\nwheels = [{ url = \"https://files.example/filler34-1.0.34-py3-none-any.whl\", hashes = { sha256 = \"3434343434343434343434343434343434343434343434343434343434343434\" } }]\n\n[[packages]]\nname = \"filler35\"\nversion = \"1.0.35\"\nwheels = [{ url = \"https://files.example/filler35-1.0.35-py3-none-any.whl\", hashes = { sha256 = \"3535353535353535353535353535353535353535353535353535353535353535\" } }]\n\n[[packages]]\nname = \"filler36\"\nversion = \"1.0.36\"\nwheels = [{ url = \"https://files.example/filler36-1.0.36-py3-none-any.whl\", hashes = { sha256 = \"3636363636363636363636363636363636363636363636363636363636363636\" } }]\n\n[[packages]]\nname = \"filler37\"\nversion = \"1.0.37\"\nwheels = [{ url = \"https://files.example/filler37-1.0.37-py3-none-any.whl\", hashes = { sha256 = \"3737373737373737373737373737373737373737373737373737373737373737\" } }]\n\n[[packages]]\nname = \"filler38\"\nversion = \"1.0.38\"\nwheels = [{ url = \"https://files.example/filler38-1.0.38-py3-none-any.whl\", hashes = { sha256 = \"3838383838383838383838383838383838383838383838383838383838383838\" } }]\n\n[[packages]]\nname = \"filler39\"\nversion = \"1.0.39\"\nwheels = [{ url = \"https://files.example/filler39-1.0.39-py3-none-any.whl\", hashes = { sha256 = \"3939393939393939393939393939393939393939393939393939393939393939\" } }]\n\n[[packages]]\nname = \"alpha\"\nversion = \"1.0.0\"\narchive = { path = \".socket/vendor/pypi/11111111-1111-4111-8111-000000000001/alpha-1.0.0-py3-none-any.whl\", hashes = { sha256 = \"9064609f1885c0bffb2e651f3339a6833010a9d388d2dadb5ddc2a629129574d\" } }\n\n[[packages]]\nname = \"beta\"\nversion = \"1.0.0\"\narchive = { path = \".socket/vendor/pypi/11111111-1111-4111-8111-000000000002/beta-1.0.0-py3-none-any.whl\", hashes = { sha256 = \"5f2338b259764f7a63e5a9234c9b18d7ccd556bd76f6b31da48c4e2018c0e17e\" } }\n" + } + ], + "flavor": "python-lock", + "record": { + "uuid": "11111111-1111-4111-8111-000000000002", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "beta.py": { + "beforeHash": "4dcafa0c7d504862a459d3a13ae317f906e0e083275ed64890c3fcdc40a88ed1", + "afterHash": "9b25de3f9bfcccfa6d08a5f0983bdcb83b2f3911513a4e6d9d0f6e718f46464b" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + } + } +} diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pylock/wired/pylock.toml b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pylock/wired/pylock.toml new file mode 100644 index 000000000..b0ae99bc2 --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pylock/wired/pylock.toml @@ -0,0 +1,212 @@ +lock-version = "1.0" +created-by = "fixture" + +[[packages]] +name = "filler00" +version = "1.0.0" +wheels = [{ url = "https://files.example/filler00-1.0.0-py3-none-any.whl", hashes = { sha256 = "0000000000000000000000000000000000000000000000000000000000000000" } }] + +[[packages]] +name = "filler01" +version = "1.0.1" +wheels = [{ url = "https://files.example/filler01-1.0.1-py3-none-any.whl", hashes = { sha256 = "0101010101010101010101010101010101010101010101010101010101010101" } }] + +[[packages]] +name = "filler02" +version = "1.0.2" +wheels = [{ url = "https://files.example/filler02-1.0.2-py3-none-any.whl", hashes = { sha256 = "0202020202020202020202020202020202020202020202020202020202020202" } }] + +[[packages]] +name = "filler03" +version = "1.0.3" +wheels = [{ url = "https://files.example/filler03-1.0.3-py3-none-any.whl", hashes = { sha256 = "0303030303030303030303030303030303030303030303030303030303030303" } }] + +[[packages]] +name = "filler04" +version = "1.0.4" +wheels = [{ url = "https://files.example/filler04-1.0.4-py3-none-any.whl", hashes = { sha256 = "0404040404040404040404040404040404040404040404040404040404040404" } }] + +[[packages]] +name = "filler05" +version = "1.0.5" +wheels = [{ url = "https://files.example/filler05-1.0.5-py3-none-any.whl", hashes = { sha256 = "0505050505050505050505050505050505050505050505050505050505050505" } }] + +[[packages]] +name = "filler06" +version = "1.0.6" +wheels = [{ url = "https://files.example/filler06-1.0.6-py3-none-any.whl", hashes = { sha256 = "0606060606060606060606060606060606060606060606060606060606060606" } }] + +[[packages]] +name = "filler07" +version = "1.0.7" +wheels = [{ url = "https://files.example/filler07-1.0.7-py3-none-any.whl", hashes = { sha256 = "0707070707070707070707070707070707070707070707070707070707070707" } }] + +[[packages]] +name = "filler08" +version = "1.0.8" +wheels = [{ url = "https://files.example/filler08-1.0.8-py3-none-any.whl", hashes = { sha256 = "0808080808080808080808080808080808080808080808080808080808080808" } }] + +[[packages]] +name = "filler09" +version = "1.0.9" +wheels = [{ url = "https://files.example/filler09-1.0.9-py3-none-any.whl", hashes = { sha256 = "0909090909090909090909090909090909090909090909090909090909090909" } }] + +[[packages]] +name = "filler10" +version = "1.0.10" +wheels = [{ url = "https://files.example/filler10-1.0.10-py3-none-any.whl", hashes = { sha256 = "1010101010101010101010101010101010101010101010101010101010101010" } }] + +[[packages]] +name = "filler11" +version = "1.0.11" +wheels = [{ url = "https://files.example/filler11-1.0.11-py3-none-any.whl", hashes = { sha256 = "1111111111111111111111111111111111111111111111111111111111111111" } }] + +[[packages]] +name = "filler12" +version = "1.0.12" +wheels = [{ url = "https://files.example/filler12-1.0.12-py3-none-any.whl", hashes = { sha256 = "1212121212121212121212121212121212121212121212121212121212121212" } }] + +[[packages]] +name = "filler13" +version = "1.0.13" +wheels = [{ url = "https://files.example/filler13-1.0.13-py3-none-any.whl", hashes = { sha256 = "1313131313131313131313131313131313131313131313131313131313131313" } }] + +[[packages]] +name = "filler14" +version = "1.0.14" +wheels = [{ url = "https://files.example/filler14-1.0.14-py3-none-any.whl", hashes = { sha256 = "1414141414141414141414141414141414141414141414141414141414141414" } }] + +[[packages]] +name = "filler15" +version = "1.0.15" +wheels = [{ url = "https://files.example/filler15-1.0.15-py3-none-any.whl", hashes = { sha256 = "1515151515151515151515151515151515151515151515151515151515151515" } }] + +[[packages]] +name = "filler16" +version = "1.0.16" +wheels = [{ url = "https://files.example/filler16-1.0.16-py3-none-any.whl", hashes = { sha256 = "1616161616161616161616161616161616161616161616161616161616161616" } }] + +[[packages]] +name = "filler17" +version = "1.0.17" +wheels = [{ url = "https://files.example/filler17-1.0.17-py3-none-any.whl", hashes = { sha256 = "1717171717171717171717171717171717171717171717171717171717171717" } }] + +[[packages]] +name = "filler18" +version = "1.0.18" +wheels = [{ url = "https://files.example/filler18-1.0.18-py3-none-any.whl", hashes = { sha256 = "1818181818181818181818181818181818181818181818181818181818181818" } }] + +[[packages]] +name = "filler19" +version = "1.0.19" +wheels = [{ url = "https://files.example/filler19-1.0.19-py3-none-any.whl", hashes = { sha256 = "1919191919191919191919191919191919191919191919191919191919191919" } }] + +[[packages]] +name = "filler20" +version = "1.0.20" +wheels = [{ url = "https://files.example/filler20-1.0.20-py3-none-any.whl", hashes = { sha256 = "2020202020202020202020202020202020202020202020202020202020202020" } }] + +[[packages]] +name = "filler21" +version = "1.0.21" +wheels = [{ url = "https://files.example/filler21-1.0.21-py3-none-any.whl", hashes = { sha256 = "2121212121212121212121212121212121212121212121212121212121212121" } }] + +[[packages]] +name = "filler22" +version = "1.0.22" +wheels = [{ url = "https://files.example/filler22-1.0.22-py3-none-any.whl", hashes = { sha256 = "2222222222222222222222222222222222222222222222222222222222222222" } }] + +[[packages]] +name = "filler23" +version = "1.0.23" +wheels = [{ url = "https://files.example/filler23-1.0.23-py3-none-any.whl", hashes = { sha256 = "2323232323232323232323232323232323232323232323232323232323232323" } }] + +[[packages]] +name = "filler24" +version = "1.0.24" +wheels = [{ url = "https://files.example/filler24-1.0.24-py3-none-any.whl", hashes = { sha256 = "2424242424242424242424242424242424242424242424242424242424242424" } }] + +[[packages]] +name = "filler25" +version = "1.0.25" +wheels = [{ url = "https://files.example/filler25-1.0.25-py3-none-any.whl", hashes = { sha256 = "2525252525252525252525252525252525252525252525252525252525252525" } }] + +[[packages]] +name = "filler26" +version = "1.0.26" +wheels = [{ url = "https://files.example/filler26-1.0.26-py3-none-any.whl", hashes = { sha256 = "2626262626262626262626262626262626262626262626262626262626262626" } }] + +[[packages]] +name = "filler27" +version = "1.0.27" +wheels = [{ url = "https://files.example/filler27-1.0.27-py3-none-any.whl", hashes = { sha256 = "2727272727272727272727272727272727272727272727272727272727272727" } }] + +[[packages]] +name = "filler28" +version = "1.0.28" +wheels = [{ url = "https://files.example/filler28-1.0.28-py3-none-any.whl", hashes = { sha256 = "2828282828282828282828282828282828282828282828282828282828282828" } }] + +[[packages]] +name = "filler29" +version = "1.0.29" +wheels = [{ url = "https://files.example/filler29-1.0.29-py3-none-any.whl", hashes = { sha256 = "2929292929292929292929292929292929292929292929292929292929292929" } }] + +[[packages]] +name = "filler30" +version = "1.0.30" +wheels = [{ url = "https://files.example/filler30-1.0.30-py3-none-any.whl", hashes = { sha256 = "3030303030303030303030303030303030303030303030303030303030303030" } }] + +[[packages]] +name = "filler31" +version = "1.0.31" +wheels = [{ url = "https://files.example/filler31-1.0.31-py3-none-any.whl", hashes = { sha256 = "3131313131313131313131313131313131313131313131313131313131313131" } }] + +[[packages]] +name = "filler32" +version = "1.0.32" +wheels = [{ url = "https://files.example/filler32-1.0.32-py3-none-any.whl", hashes = { sha256 = "3232323232323232323232323232323232323232323232323232323232323232" } }] + +[[packages]] +name = "filler33" +version = "1.0.33" +wheels = [{ url = "https://files.example/filler33-1.0.33-py3-none-any.whl", hashes = { sha256 = "3333333333333333333333333333333333333333333333333333333333333333" } }] + +[[packages]] +name = "filler34" +version = "1.0.34" +wheels = [{ url = "https://files.example/filler34-1.0.34-py3-none-any.whl", hashes = { sha256 = "3434343434343434343434343434343434343434343434343434343434343434" } }] + +[[packages]] +name = "filler35" +version = "1.0.35" +wheels = [{ url = "https://files.example/filler35-1.0.35-py3-none-any.whl", hashes = { sha256 = "3535353535353535353535353535353535353535353535353535353535353535" } }] + +[[packages]] +name = "filler36" +version = "1.0.36" +wheels = [{ url = "https://files.example/filler36-1.0.36-py3-none-any.whl", hashes = { sha256 = "3636363636363636363636363636363636363636363636363636363636363636" } }] + +[[packages]] +name = "filler37" +version = "1.0.37" +wheels = [{ url = "https://files.example/filler37-1.0.37-py3-none-any.whl", hashes = { sha256 = "3737373737373737373737373737373737373737373737373737373737373737" } }] + +[[packages]] +name = "filler38" +version = "1.0.38" +wheels = [{ url = "https://files.example/filler38-1.0.38-py3-none-any.whl", hashes = { sha256 = "3838383838383838383838383838383838383838383838383838383838383838" } }] + +[[packages]] +name = "filler39" +version = "1.0.39" +wheels = [{ url = "https://files.example/filler39-1.0.39-py3-none-any.whl", hashes = { sha256 = "3939393939393939393939393939393939393939393939393939393939393939" } }] + +[[packages]] +name = "alpha" +version = "1.0.0" +archive = { path = ".socket/vendor/pypi/11111111-1111-4111-8111-000000000001/alpha-1.0.0-py3-none-any.whl", hashes = { sha256 = "9064609f1885c0bffb2e651f3339a6833010a9d388d2dadb5ddc2a629129574d" } } + +[[packages]] +name = "beta" +version = "1.0.0" +archive = { path = ".socket/vendor/pypi/11111111-1111-4111-8111-000000000002/beta-1.0.0-py3-none-any.whl", hashes = { sha256 = "5f2338b259764f7a63e5a9234c9b18d7ccd556bd76f6b31da48c4e2018c0e17e" } } diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pypi-requirements/wired/.socket/vendor/state.json b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pypi-requirements/wired/.socket/vendor/state.json new file mode 100644 index 000000000..d15ff4b2e --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pypi-requirements/wired/.socket/vendor/state.json @@ -0,0 +1,97 @@ +{ + "version": 1, + "entries": { + "pkg:pypi/alpha@1.0.0": { + "ecosystem": "pypi", + "basePurl": "pkg:pypi/alpha@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000001", + "artifact": { + "path": ".socket/vendor/pypi/11111111-1111-4111-8111-000000000001/alpha-1.0.0-py3-none-any.whl", + "sha256": "9064609f1885c0bffb2e651f3339a6833010a9d388d2dadb5ddc2a629129574d", + "size": 837 + }, + "wiring": [ + { + "file": "requirements.txt", + "kind": "requirements_line", + "action": "rewritten", + "key": "requirements.txt:1", + "original": [ + "alpha==1.0.0" + ], + "new": "./.socket/vendor/pypi/11111111-1111-4111-8111-000000000001/alpha-1.0.0-py3-none-any.whl --hash=sha256:9064609f1885c0bffb2e651f3339a6833010a9d388d2dadb5ddc2a629129574d # socket-patch vendor: alpha==1.0.0" + } + ], + "flavor": "requirements", + "record": { + "uuid": "11111111-1111-4111-8111-000000000001", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "alpha.py": { + "beforeHash": "490ce2042e030d4655a2fddaf5f85d98215f64866fe924652851388b80b32d66", + "afterHash": "3171463aa7c6f59eb47b7ef9c888b14739f6da2be164d5b6ebfdb6b5bdb6fb21" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + }, + "pkg:pypi/beta@1.0.0": { + "ecosystem": "pypi", + "basePurl": "pkg:pypi/beta@1.0.0", + "uuid": "11111111-1111-4111-8111-000000000002", + "artifact": { + "path": ".socket/vendor/pypi/11111111-1111-4111-8111-000000000002/beta-1.0.0-py3-none-any.whl", + "sha256": "5f2338b259764f7a63e5a9234c9b18d7ccd556bd76f6b31da48c4e2018c0e17e", + "size": 824 + }, + "wiring": [ + { + "file": "requirements.txt", + "kind": "requirements_line", + "action": "rewritten", + "key": "requirements.txt:2", + "original": [ + "beta==1.0.0" + ], + "new": "./.socket/vendor/pypi/11111111-1111-4111-8111-000000000002/beta-1.0.0-py3-none-any.whl --hash=sha256:5f2338b259764f7a63e5a9234c9b18d7ccd556bd76f6b31da48c4e2018c0e17e # socket-patch vendor: beta==1.0.0" + } + ], + "flavor": "requirements", + "record": { + "uuid": "11111111-1111-4111-8111-000000000002", + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + "beta.py": { + "beforeHash": "4dcafa0c7d504862a459d3a13ae317f906e0e083275ed64890c3fcdc40a88ed1", + "afterHash": "9b25de3f9bfcccfa6d08a5f0983bdcb83b2f3911513a4e6d9d0f6e718f46464b" + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": [ + "CVE-2026-0001" + ], + "summary": "s", + "severity": "high", + "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + } + } + } +} diff --git a/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pypi-requirements/wired/requirements.txt b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pypi-requirements/wired/requirements.txt new file mode 100644 index 000000000..0b248e89f --- /dev/null +++ b/crates/socket-patch-cli/tests/fixtures/legacy-ledgers/pypi-requirements/wired/requirements.txt @@ -0,0 +1,2 @@ +./.socket/vendor/pypi/11111111-1111-4111-8111-000000000001/alpha-1.0.0-py3-none-any.whl --hash=sha256:9064609f1885c0bffb2e651f3339a6833010a9d388d2dadb5ddc2a629129574d # socket-patch vendor: alpha==1.0.0 +./.socket/vendor/pypi/11111111-1111-4111-8111-000000000002/beta-1.0.0-py3-none-any.whl --hash=sha256:5f2338b259764f7a63e5a9234c9b18d7ccd556bd76f6b31da48c4e2018c0e17e # socket-patch vendor: beta==1.0.0 diff --git a/crates/socket-patch-cli/tests/hosted_wheel_metadata_order.rs b/crates/socket-patch-cli/tests/hosted_wheel_metadata_order.rs new file mode 100644 index 000000000..c6fe0ab6d --- /dev/null +++ b/crates/socket-patch-cli/tests/hosted_wheel_metadata_order.rs @@ -0,0 +1,638 @@ +//! `scan --mode hosted` over a uv project with SEVERAL pypi wheel patches: +//! the hosted wheel metadata each native uv rewrite embeds is fetched +//! concurrently, but every outcome must fold in dep order — the +//! `python_metadata_unavailable` skips come out in the same order the old +//! one-at-a-time loop produced, whatever order the downloads finish in. +//! +//! The first failing wheel is served SLOWLY and the second fails at once, +//! so a fold in completion order would swap them. Request arrivals are +//! recorded too, so a regression to one-at-a-time fetching (which keeps the +//! order but loses the overlap) fails as well. +//! +//! Runs the built binary as a subprocess (`common::run_with_env`) against a +//! wiremock patch API. Unix-only: the fabricated `.venv` uses the POSIX +//! site-packages layout. + +#![cfg(unix)] + +use std::path::Path; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use serde_json::{json, Value}; +use wiremock::matchers::{method, path, path_regex}; +use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate}; + +#[path = "common/mod.rs"] +mod common; + +const ORG: &str = "test-org"; + +/// (name, version, patch uuid) — purl order is name order. +const PKGS: [(&str, &str, &str); 4] = [ + ("aaa-pkg", "1.0.0", "11111111-1111-4111-8111-111111111111"), + ("bbb-pkg", "2.0.0", "22222222-2222-4222-8222-222222222222"), + ("ccc-pkg", "3.0.0", "33333333-3333-4333-8333-333333333333"), + ("ddd-pkg", "4.0.0", "44444444-4444-4444-8444-444444444444"), +]; + +fn purl(name: &str, version: &str) -> String { + format!("pkg:pypi/{name}@{version}") +} + +fn wheel_file(name: &str, version: &str) -> String { + format!("{}-{version}-py3-none-any.whl", name.replace('-', "_")) +} + +/// A minimal but valid wheel: `-.dist-info/METADATA` with the +/// three required core-metadata headers. Returns the bytes and their sha256. +fn build_wheel(name: &str, version: &str) -> (Vec, String) { + use std::io::Write as _; + let mut buf = std::io::Cursor::new(Vec::new()); + { + let mut writer = zip::ZipWriter::new(&mut buf); + let opts = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Stored); + let dist = name.replace('-', "_"); + writer + .start_file(format!("{dist}-{version}.dist-info/METADATA"), opts) + .unwrap(); + writer + .write_all( + format!("Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n\n").as_bytes(), + ) + .unwrap(); + writer.finish().unwrap(); + } + let bytes = buf.into_inner(); + let sha = common::sha256_hex(&bytes); + (bytes, sha) +} + +fn write_uv_project(root: &Path) { + std::fs::create_dir_all(root).unwrap(); + let deps: Vec = PKGS + .iter() + .map(|(name, version, _)| format!("\"{name}=={version}\"")) + .collect(); + std::fs::write( + root.join("pyproject.toml"), + format!( + "[project]\nname = \"socket-uv-order-fixture\"\nversion = \"0.1.0\"\nrequires-python = \">=3.9\"\ndependencies = [{}]\n", + deps.join(", ") + ), + ) + .unwrap(); + let mut lock = String::from( + "version = 1\nrevision = 3\nrequires-python = \">=3.9\"\n\n[[package]]\nname = \"socket-uv-order-fixture\"\nversion = \"0.1.0\"\nsource = { virtual = \".\" }\ndependencies = [\n", + ); + for (name, _, _) in PKGS { + lock.push_str(&format!(" {{ name = \"{name}\" }},\n")); + } + lock.push_str("]\n\n[package.metadata]\nrequires-dist = ["); + lock.push_str( + &PKGS + .iter() + .map(|(name, version, _)| { + format!("{{ name = \"{name}\", specifier = \"=={version}\" }}") + }) + .collect::>() + .join(", "), + ); + lock.push_str("]\n"); + for (name, version, _) in PKGS { + let file = wheel_file(name, version); + lock.push_str(&format!( + "\n[[package]]\nname = \"{name}\"\nversion = \"{version}\"\nsource = {{ registry = \"https://pypi.org/simple\" }}\nwheels = [\n {{ url = \"https://files.pythonhosted.org/packages/xx/{file}\", hash = \"sha256:{}\" }},\n]\n", + "0".repeat(64) + )); + } + std::fs::write(root.join("uv.lock"), lock).unwrap(); + let site = root + .join(".venv") + .join("lib") + .join("python3.11") + .join("site-packages"); + for (name, version, _) in PKGS { + let dist = site.join(format!("{}-{version}.dist-info", name.replace('-', "_"))); + std::fs::create_dir_all(&dist).unwrap(); + std::fs::write( + dist.join("METADATA"), + format!("Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n"), + ) + .unwrap(); + } +} + +/// When each wheel request ARRIVED at the mock (before its response delay). +type Arrivals = Arc>>; + +/// Serves `template` and records the request's arrival under `name`. +struct RecordArrival { + name: &'static str, + template: ResponseTemplate, + arrivals: Arrivals, +} + +impl Respond for RecordArrival { + fn respond(&self, _: &Request) -> ResponseTemplate { + self.arrivals + .lock() + .unwrap() + .push((self.name, Instant::now())); + self.template.clone() + } +} + +/// A serve host that handles ONE download at a time: a wheel request that +/// arrives while another is still being answered gets `429 Retry-After: 1` +/// (a per-token rate limiter / CDN cap). Records every arrival and status. +struct OneAtATime { + name: &'static str, + body: Vec, + busy_until: Arc>, + log: Arc>>, +} + +impl Respond for OneAtATime { + fn respond(&self, _: &Request) -> ResponseTemplate { + let now = Instant::now(); + let mut busy_until = self.busy_until.lock().unwrap(); + let (status, template) = if now < *busy_until { + ( + 429, + ResponseTemplate::new(429).insert_header("Retry-After", "1"), + ) + } else { + let delay = Duration::from_millis(200); + *busy_until = now + delay; + ( + 200, + ResponseTemplate::new(200) + .set_body_bytes(self.body.clone()) + .set_delay(delay), + ) + }; + self.log.lock().unwrap().push((self.name, status)); + template + } +} + +/// A serve host that 503s a wheel's first `fail_first` requests and serves +/// it from then on — the shape that catches a retry budget being spent +/// twice: one `attempts`-deep budget never reaches request `fail_first + 1`. +struct Flapping { + name: &'static str, + body: Vec, + fail_first: usize, + log: Arc>>, +} + +impl Respond for Flapping { + fn respond(&self, _: &Request) -> ResponseTemplate { + let mut log = self.log.lock().unwrap(); + let seen = log.iter().filter(|(n, _)| *n == self.name).count(); + let (status, template) = if seen < self.fail_first { + (503, ResponseTemplate::new(503)) + } else { + ( + 200, + ResponseTemplate::new(200).set_body_bytes(self.body.clone()), + ) + }; + log.push((self.name, status)); + template + } +} + +/// Which wheel host [`mock_api`] mounts. +enum WheelHost { + /// `aaa` and `ccc` serve valid bytes (reversed delays), `bbb` is a SLOW + /// 404 and `ddd` serves bytes that do not match the granted sha256. + Mixed(Arrivals), + /// Every wheel is valid, behind a [`OneAtATime`] host. + OneAtATime(Arc>>), + /// Every wheel is valid, but `bbb` is behind a [`Flapping`] host that + /// 503s its first `VENDOR_ATTEMPTS` requests. + Flapping(Arc>>), +} + +/// `VendorRetryPolicy::default().attempts` — the requests one wheel +/// download may cost the host, however the fan-out splits it up. +const VENDOR_ATTEMPTS: usize = 3; + +/// Discovery, per-package search and the reference grants for all of PKGS, +/// with the wheels served per [`WheelHost::Mixed`]. Returns the +/// wheel-request arrival log. +async fn mock_api(server: &MockServer) -> Arrivals { + let arrivals: Arrivals = Arc::default(); + mock_api_with(server, WheelHost::Mixed(arrivals.clone())).await; + arrivals +} + +async fn mock_api_with(server: &MockServer, host: WheelHost) { + let busy_until = Arc::new(Mutex::new(Instant::now())); + let patch = |name: &str, version: &str, uuid: &str| { + json!({ + "uuid": uuid, "purl": purl(name, version), "tier": "free", + "cveIds": [], "ghsaIds": [], "severity": "high", + "title": format!("{name} fixture") + }) + }; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/batch"))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "packages": PKGS.iter().map(|(name, version, uuid)| json!({ + "purl": purl(name, version), + "patches": [patch(name, version, uuid)], + })).collect::>(), + "canAccessPaidPatches": false, + }))) + .mount(server) + .await; + for (name, version, uuid) in PKGS { + Mock::given(method("GET")) + .and(path_regex(format!( + "^/v0/orgs/{ORG}/patches/by-package/.*{name}.*$" + ))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "patches": [{ + "uuid": uuid, "purl": purl(name, version), + "publishedAt": "2024-01-01T00:00:00Z", + "description": "x", "license": "MIT", "tier": "free", + "vulnerabilities": {} + }], + "canAccessPaidPatches": false, + }))) + .mount(server) + .await; + } + let mut results = serde_json::Map::new(); + for (i, (name, version, uuid)) in PKGS.into_iter().enumerate() { + let file = wheel_file(name, version); + let url = format!("{}/wheels/{file}", server.uri()); + let (bytes, sha256) = build_wheel(name, version); + let wheel = Mock::given(method("GET")).and(path(format!("/wheels/{file}"))); + match &host { + WheelHost::Mixed(arrivals) => { + let delay = Duration::from_millis([600, 900, 0, 0][i]); + let response = match name { + "bbb-pkg" => ResponseTemplate::new(404), + "ddd-pkg" => { + ResponseTemplate::new(200).set_body_bytes(b"not the granted wheel".to_vec()) + } + _ => ResponseTemplate::new(200).set_body_bytes(bytes), + }; + wheel + .respond_with(RecordArrival { + name, + template: response.set_delay(delay), + arrivals: arrivals.clone(), + }) + .mount(server) + .await; + } + WheelHost::OneAtATime(log) => { + wheel + .respond_with(OneAtATime { + name, + body: bytes, + busy_until: busy_until.clone(), + log: log.clone(), + }) + .mount(server) + .await; + } + WheelHost::Flapping(log) => { + wheel + .respond_with(Flapping { + name, + body: bytes, + fail_first: if name == "bbb-pkg" { + VENDOR_ATTEMPTS + } else { + 0 + }, + log: log.clone(), + }) + .mount(server) + .await; + } + } + results.insert( + uuid.to_string(), + json!({ + "status": "granted", + "url": url, + "purl": purl(name, version), + "artifacts": [{ "kind": "tarball", "url": url, "integrity": { "sha256": sha256 } }], + "registryOverride": null + }), + ); + } + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/package"))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "results": results }))) + .mount(server) + .await; +} + +#[tokio::test] +async fn wheel_metadata_failures_fold_in_dep_order() { + let server = MockServer::start().await; + let arrivals = mock_api(&server).await; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + write_uv_project(&root); + let lock_before = std::fs::read(root.join("uv.lock")).unwrap(); + + let cwd = root.to_str().unwrap().to_string(); + let api = server.uri(); + let (code, stdout, stderr) = common::run_with_env( + &root, + &[ + "scan", + "--mode", + "hosted", + "--dry-run", + "--json", + "--cwd", + &cwd, + "--api-url", + &api, + "--org", + ORG, + "--api-token", + "fake", + ], + &[], + ); + let doc: Value = serde_json::from_str(&stdout) + .unwrap_or_else(|e| panic!("JSON envelope ({e}):\n{stdout}\n{stderr}")); + assert_eq!(code, 0, "{doc:#}\n{stderr}"); + + let skipped: Vec<(String, String)> = doc["redirect"]["skipped"] + .as_array() + .unwrap_or_else(|| panic!("redirect.skipped: {doc:#}")) + .iter() + .filter(|s| s["reason"] == "python_metadata_unavailable") + .map(|s| { + ( + s["purl"].as_str().unwrap().to_string(), + s["uuid"].as_str().unwrap().to_string(), + ) + }) + .collect(); + assert_eq!( + skipped, + vec![ + (purl("bbb-pkg", "2.0.0"), PKGS[1].2.to_string()), + (purl("ddd-pkg", "4.0.0"), PKGS[3].2.to_string()), + ], + "metadata failures must be reported in dep order: {doc:#}" + ); + for s in doc["redirect"]["skipped"].as_array().unwrap() { + if s["reason"] == "python_metadata_unavailable" { + let detail = s["detail"].as_str().unwrap(); + assert!( + !detail.contains(&server.uri()), + "the hosted URL is redacted from the detail: {detail}" + ); + } + } + // The two good wheels still redirect; the refused two stay upstream. + assert_eq!(doc["redirect"]["redirected"], 2, "{doc:#}"); + assert_eq!( + std::fs::read(root.join("uv.lock")).unwrap(), + lock_before, + "--dry-run writes nothing" + ); + + // The fetches overlap: `bbb` is requested while `aaa`'s 600 ms response + // is still pending. A one-at-a-time loop cannot request `bbb` until + // `aaa` has been answered. (No wall-clock budget: arrival order only.) + let arrivals = arrivals.lock().unwrap().clone(); + let first = |name: &str| { + arrivals + .iter() + .find(|(n, _)| *n == name) + .map(|(_, at)| *at) + .unwrap_or_else(|| panic!("no request for {name}: {arrivals:?}")) + }; + let (aaa, bbb) = (first("aaa-pkg"), first("bbb-pkg")); + assert!( + bbb < aaa + Duration::from_millis(600), + "bbb must be requested before aaa's delayed response is due \ + (arrived {:?} after aaa): {arrivals:?}", + bbb.saturating_duration_since(aaa) + ); +} + +/// The debug stream's wheel GET lines, as `(dep name, whole line)`. +fn wheel_debug_gets(stderr: &str, api: &str) -> Vec<(String, String)> { + stderr + .lines() + .filter(|line| line.starts_with("[socket-patch debug]") && line.contains("/wheels/")) + .map(|line| { + let dep = PKGS + .iter() + .find(|(name, version, _)| line.ends_with(&wheel_file(name, version))) + .unwrap_or_else(|| panic!("unknown wheel in {line}")); + assert!(line.contains(api), "the wheel host is {api}: {line}"); + (dep.0.to_string(), line.to_string()) + }) + .collect() +} + +/// A host that serves one download at a time and 429s the rest must end up +/// with the one-at-a-time loop's outcome: every wheel's metadata fetched, +/// nothing skipped. Concurrent attempts that share a `Retry-After` wake up +/// together and collide again, so letting each one retry on its own drains +/// the budgets and drops a redirect the serial loop makes; a retryable +/// failure must hand the rest of the fan-out back to the serial loop. +/// +/// The collisions themselves are the fan-out's own doing, so this asserts on +/// the HOST's log, not only on stdout: no wheel may cost more requests than +/// the one-at-a-time loop's budget (a deferred attempt is resumed, never +/// replayed on a fresh one), and every wheel must end on a 200. The opt-in +/// debug stream must account for every one of those requests — a speculative +/// GET the host really served is reported, not censored — and must still +/// fold in dep order. +#[tokio::test] +async fn rate_limited_wheel_host_matches_the_serial_outcome() { + let server = MockServer::start().await; + let log: Arc>> = Arc::default(); + mock_api_with(&server, WheelHost::OneAtATime(log.clone())).await; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + write_uv_project(&root); + + let cwd = root.to_str().unwrap().to_string(); + let api = server.uri(); + let (code, stdout, stderr) = common::run_with_env( + &root, + &[ + "scan", + "--mode", + "hosted", + "--dry-run", + "--json", + "--cwd", + &cwd, + "--api-url", + &api, + "--org", + ORG, + "--api-token", + "fake", + ], + &[("SOCKET_DEBUG", "1")], + ); + let doc: Value = serde_json::from_str(&stdout) + .unwrap_or_else(|e| panic!("JSON envelope ({e}):\n{stdout}\n{stderr}")); + assert_eq!(code, 0, "{doc:#}\n{stderr}"); + let log = log.lock().unwrap().clone(); + assert_eq!( + doc["redirect"]["redirected"], + PKGS.len(), + "every wheel redirects, as in the serial loop: {doc:#}\nwheel requests: {log:?}" + ); + let metadata_skips: Vec<&Value> = doc["redirect"]["skipped"] + .as_array() + .map(|skipped| { + skipped + .iter() + .filter(|s| s["reason"] == "python_metadata_unavailable") + .collect() + }) + .unwrap_or_default(); + assert!( + metadata_skips.is_empty(), + "no wheel may be skipped: {metadata_skips:?}\nwheel requests: {log:?}" + ); + + // No wheel may cost more than one retry budget, and each must end on a + // 200 — a deferred first attempt is resumed, not replayed on a fresh + // budget, which would let a wheel outlive the serial loop's failures. + for (name, _, _) in PKGS { + let statuses: Vec = log + .iter() + .filter(|(n, _)| *n == name) + .map(|(_, status)| *status) + .collect(); + assert!( + !statuses.is_empty() && statuses.len() <= VENDOR_ATTEMPTS, + "{name} may cost at most {VENDOR_ATTEMPTS} requests, got {statuses:?}: {log:?}" + ); + assert_eq!( + statuses.last(), + Some(&200), + "{name} must end on the metadata the serial loop got: {log:?}" + ); + } + + // Every request the host really served is in the debug stream, and the + // stream still folds in dep order (each dep's lines in one run). + let wheel_debug = wheel_debug_gets(&stderr, &server.uri()); + assert_eq!( + wheel_debug.len(), + log.len(), + "the debug stream must account for every wheel request the host \ + served ({log:?}):\n{stderr}" + ); + let mut folded: Vec<&str> = wheel_debug.iter().map(|(dep, _)| dep.as_str()).collect(); + folded.dedup(); + assert_eq!( + folded, + PKGS.iter().map(|(name, _, _)| *name).collect::>(), + "each dep's GETs must fold together, in dep order:\n{stderr}" + ); + // Each reported retry is a real 429, and each real 429 is reported. + let failed = stderr + .lines() + .filter(|line| line.contains("vendor package download attempt")) + .count(); + assert_eq!( + failed, + log.iter().filter(|(_, status)| *status == 429).count(), + "every collision the host answered must be reported once:\n{stderr}" + ); + assert_eq!( + failed, + stderr + .lines() + .filter(|line| line.contains("vendor service retry")) + .count(), + "every reported failure must also report its Retry-After pause:\n{stderr}" + ); +} + +/// The fan-out may not hand a wheel a second retry budget. A host that 503s +/// a wheel's first `VENDOR_ATTEMPTS` requests exhausts the one-at-a-time +/// loop's budget, so that wheel is skipped and the other three redirect. +/// Spending the speculative attempt and then starting a FULL retry budget +/// would reach the host's first 200 instead — a different stdout, and in a +/// wet run a lockfile the serial loop never writes. +#[tokio::test] +async fn a_deferred_wheel_attempt_does_not_buy_a_second_retry_budget() { + let server = MockServer::start().await; + let log: Arc>> = Arc::default(); + mock_api_with(&server, WheelHost::Flapping(log.clone())).await; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + write_uv_project(&root); + + let cwd = root.to_str().unwrap().to_string(); + let api = server.uri(); + let (code, stdout, stderr) = common::run_with_env( + &root, + &[ + "scan", + "--mode", + "hosted", + "--dry-run", + "--json", + "--cwd", + &cwd, + "--api-url", + &api, + "--org", + ORG, + "--api-token", + "fake", + ], + &[], + ); + let doc: Value = serde_json::from_str(&stdout) + .unwrap_or_else(|e| panic!("JSON envelope ({e}):\n{stdout}\n{stderr}")); + assert_eq!(code, 0, "{doc:#}\n{stderr}"); + let log = log.lock().unwrap().clone(); + let bbb: Vec = log + .iter() + .filter(|(n, _)| *n == "bbb-pkg") + .map(|(_, status)| *status) + .collect(); + assert_eq!( + bbb, + vec![503; VENDOR_ATTEMPTS], + "the flapping wheel gets exactly the serial loop's budget, all of \ + it spent before the host's first 200: {log:?}" + ); + let skipped: Vec = doc["redirect"]["skipped"] + .as_array() + .unwrap_or_else(|| panic!("redirect.skipped: {doc:#}")) + .iter() + .filter(|s| s["reason"] == "python_metadata_unavailable") + .map(|s| s["purl"].as_str().unwrap().to_string()) + .collect(); + assert_eq!( + skipped, + vec![purl("bbb-pkg", "2.0.0")], + "the exhausted wheel is skipped, as in the serial loop: {doc:#}" + ); + assert_eq!( + doc["redirect"]["redirected"], + PKGS.len() - 1, + "the other three still redirect: {doc:#}" + ); +} diff --git a/crates/socket-patch-cli/tests/in_process_cargo_apply.rs b/crates/socket-patch-cli/tests/in_process_cargo_apply.rs index 13bc46300..b85ef413d 100644 --- a/crates/socket-patch-cli/tests/in_process_cargo_apply.rs +++ b/crates/socket-patch-cli/tests/in_process_cargo_apply.rs @@ -236,7 +236,7 @@ async fn cargo_fetch_scan_sync_patches_real_file() { dry_run: false, ..socket_patch_cli::args::GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: true, @@ -355,7 +355,7 @@ async fn cargo_apply_refuses_on_before_hash_mismatch() { strict: true, ..socket_patch_cli::args::GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: true, @@ -453,7 +453,7 @@ async fn cargo_crawler_finds_real_fetched_crate() { dry_run: false, ..socket_patch_cli::args::GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: false, diff --git a/crates/socket-patch-cli/tests/in_process_gem_apply.rs b/crates/socket-patch-cli/tests/in_process_gem_apply.rs index 1e70fd6e5..2f5fd7d25 100644 --- a/crates/socket-patch-cli/tests/in_process_gem_apply.rs +++ b/crates/socket-patch-cli/tests/in_process_gem_apply.rs @@ -214,7 +214,7 @@ async fn gem_install_scan_sync_patches_real_file() { dry_run: false, ..socket_patch_cli::args::GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: true, @@ -326,7 +326,7 @@ async fn gem_crawler_finds_real_installed_gem() { dry_run: false, ..socket_patch_cli::args::GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: false, diff --git a/crates/socket-patch-cli/tests/in_process_gem_multi_platform.rs b/crates/socket-patch-cli/tests/in_process_gem_multi_platform.rs index 862234a4d..15d7ebf97 100644 --- a/crates/socket-patch-cli/tests/in_process_gem_multi_platform.rs +++ b/crates/socket-patch-cli/tests/in_process_gem_multi_platform.rs @@ -232,7 +232,7 @@ fn scan_args(cwd: &Path, api_url: String, all_releases: bool) -> ScanArgs { dry_run: false, ..socket_patch_cli::args::GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), // apply (not sync) so the post-sync GC doesn't sweep beforeHash // blobs the later rollback/remove needs offline. apply: true, diff --git a/crates/socket-patch-cli/tests/in_process_pypi_apply.rs b/crates/socket-patch-cli/tests/in_process_pypi_apply.rs index d8b1d7dda..91d8c7ff3 100644 --- a/crates/socket-patch-cli/tests/in_process_pypi_apply.rs +++ b/crates/socket-patch-cli/tests/in_process_pypi_apply.rs @@ -263,7 +263,7 @@ async fn pypi_install_scan_sync_patches_real_file() { dry_run: false, ..socket_patch_cli::args::GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: true, @@ -339,7 +339,7 @@ async fn pypi_scan_then_apply_force_patches_real_file() { dry_run: false, ..socket_patch_cli::args::GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: true, @@ -448,7 +448,7 @@ async fn pypi_apply_dry_run_does_not_modify_file() { dry_run: true, ..socket_patch_cli::args::GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), apply: true, prune: false, sync: false, @@ -580,7 +580,7 @@ async fn pypi_crawler_finds_real_installed_six() { dry_run: false, ..socket_patch_cli::args::GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: false, diff --git a/crates/socket-patch-cli/tests/in_process_pypi_multi_release.rs b/crates/socket-patch-cli/tests/in_process_pypi_multi_release.rs index 8b4ec39b8..639d635ee 100644 --- a/crates/socket-patch-cli/tests/in_process_pypi_multi_release.rs +++ b/crates/socket-patch-cli/tests/in_process_pypi_multi_release.rs @@ -306,7 +306,7 @@ fn scan_args(tmp: &Path, api_url: String, all_releases: bool) -> ScanArgs { dry_run: false, ..socket_patch_cli::args::GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), // Download + apply but DON'T prune/GC: the post-sync GC sweeps // `beforeHash` blobs (only `afterHash` blobs are kept for apply), // which would force the later rollback/remove to re-fetch them diff --git a/crates/socket-patch-cli/tests/in_process_python_envs.rs b/crates/socket-patch-cli/tests/in_process_python_envs.rs index 3fa045623..29e1c4ecb 100644 --- a/crates/socket-patch-cli/tests/in_process_python_envs.rs +++ b/crates/socket-patch-cli/tests/in_process_python_envs.rs @@ -129,7 +129,7 @@ fn default_args(cwd: &Path, api_url: String) -> ScanArgs { dry_run: false, ..socket_patch_cli::args::GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: false, diff --git a/crates/socket-patch-cli/tests/in_process_redirect.rs b/crates/socket-patch-cli/tests/in_process_redirect.rs index abdb014e5..090eaa882 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect.rs @@ -49,7 +49,7 @@ fn redirect_args(cwd: &Path, api_url: String) -> ScanArgs { yes: true, ..socket_patch_cli::args::GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: false, diff --git a/crates/socket-patch-cli/tests/in_process_redirect_pdm.rs b/crates/socket-patch-cli/tests/in_process_redirect_pdm.rs index 88517a543..a42a8f6c9 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect_pdm.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect_pdm.rs @@ -76,7 +76,7 @@ fn hosted_args(cwd: &Path, api_url: String, vex: Option<&Path>) -> ScanArgs { ScanArgs { paths: Vec::new(), common: global(cwd, api_url), - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: false, diff --git a/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs b/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs index 55eca4226..1d01bc2df 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs @@ -81,7 +81,7 @@ fn hosted_args(cwd: &Path, api_url: String, vex: Option<&Path>) -> ScanArgs { ScanArgs { paths: Vec::new(), common: global(cwd, api_url), - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: false, diff --git a/crates/socket-patch-cli/tests/in_process_redirect_pnpm.rs b/crates/socket-patch-cli/tests/in_process_redirect_pnpm.rs index 069907349..3ea3791ce 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect_pnpm.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect_pnpm.rs @@ -43,7 +43,7 @@ fn hosted_args(cwd: &Path, api_url: String) -> ScanArgs { yes: true, ..socket_patch_cli::args::GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: false, diff --git a/crates/socket-patch-cli/tests/in_process_redirect_poetry.rs b/crates/socket-patch-cli/tests/in_process_redirect_poetry.rs index c8dc1fff6..58813cb2b 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect_poetry.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect_poetry.rs @@ -60,7 +60,7 @@ fn hosted_args(cwd: &Path, api_url: String, vex: Option<&Path>) -> ScanArgs { ScanArgs { paths: Vec::new(), common: global(cwd, api_url), - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: false, diff --git a/crates/socket-patch-cli/tests/in_process_remote_ecosystems_apply.rs b/crates/socket-patch-cli/tests/in_process_remote_ecosystems_apply.rs index c93613699..3bb1d39e7 100644 --- a/crates/socket-patch-cli/tests/in_process_remote_ecosystems_apply.rs +++ b/crates/socket-patch-cli/tests/in_process_remote_ecosystems_apply.rs @@ -88,7 +88,7 @@ fn default_scan_args(cwd: &Path, eco: &str, api_url: String) -> ScanArgs { dry_run: false, ..socket_patch_cli::args::GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: true, diff --git a/crates/socket-patch-cli/tests/in_process_rollback_hosted.rs b/crates/socket-patch-cli/tests/in_process_rollback_hosted.rs index dcb3f1710..4a6c17a7b 100644 --- a/crates/socket-patch-cli/tests/in_process_rollback_hosted.rs +++ b/crates/socket-patch-cli/tests/in_process_rollback_hosted.rs @@ -68,7 +68,7 @@ fn hosted_scan_args(cwd: &Path, api_url: String) -> ScanArgs { yes: true, ..socket_patch_cli::args::GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: false, diff --git a/crates/socket-patch-cli/tests/in_process_scan.rs b/crates/socket-patch-cli/tests/in_process_scan.rs index 64959fab6..cf4bf1c42 100644 --- a/crates/socket-patch-cli/tests/in_process_scan.rs +++ b/crates/socket-patch-cli/tests/in_process_scan.rs @@ -33,7 +33,7 @@ fn default_args(cwd: &Path) -> ScanArgs { dry_run: false, ..socket_patch_cli::args::GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: false, @@ -790,7 +790,7 @@ async fn scan_small_batch_size_chunks_requests() { let mut args = default_args(tmp.path()); args.common.api_url = Some(server.uri()); - args.batch_size = 1; // force 3 separate API calls + args.batch_size = Some(1); // force 3 separate API calls assert_eq!(run_scrubbed(args).await, 0); // The whole point of this test: batch_size=1 over 3 discovered packages // must produce exactly 3 separate batch requests, each carrying one @@ -1063,7 +1063,7 @@ async fn scan_batch_size_zero_does_not_panic() { write_npm_package(tmp.path(), "in-proc-scan", "1.0.0"); let mut args = default_args(tmp.path()); args.common.api_url = Some(server.uri()); - args.batch_size = 0; + args.batch_size = Some(0); // No panic, and the discovered package still reaches the batch endpoint // (proving the loop ran rather than being skipped). diff --git a/crates/socket-patch-cli/tests/in_process_vendor.rs b/crates/socket-patch-cli/tests/in_process_vendor.rs index 54ddc4268..c44786b35 100644 --- a/crates/socket-patch-cli/tests/in_process_vendor.rs +++ b/crates/socket-patch-cli/tests/in_process_vendor.rs @@ -3135,7 +3135,7 @@ snapshots: yes: true, ..GlobalArgs::default() }, - batch_size: 100, + batch_size: Some(100), apply: false, prune: false, sync: false, diff --git a/crates/socket-patch-cli/tests/scan_api_retry_e2e.rs b/crates/socket-patch-cli/tests/scan_api_retry_e2e.rs new file mode 100644 index 000000000..37613825c --- /dev/null +++ b/crates/socket-patch-cli/tests/scan_api_retry_e2e.rs @@ -0,0 +1,458 @@ +//! `scan` against a patch API that throttles (HTTP 429 / 503). +//! +//! The client retries a 429 / 503 a bounded number of times +//! (`socket_patch_core::api::retry`); a request still throttled after that +//! is a real failure in the channel its siblings already use — the +//! per-batch / per-package warning (stderr for a human run, run-level +//! `warnings[]` for `--json`), or the all-failed error when nothing +//! succeeded — never a package silently missing from the envelope. +//! +//! Every throttled answer carries `Retry-After: 0`, so the subprocess +//! retries without sleeping (the backoff arithmetic itself is pinned on a +//! virtual clock in the core crate's `api_retry_e2e.rs`). + +use std::path::Path; +use std::process::Command; + +use wiremock::matchers::{body_string_contains, method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const ORG: &str = "test-org"; +const NAMES: [&str; 6] = [ + "retry-alpha", + "retry-bravo", + "retry-charlie", + "retry-delta", + "retry-echo", + "retry-foxtrot", +]; +const VERSION: &str = "1.0.0"; + +fn purl(name: &str) -> String { + format!("pkg:npm/{name}@{VERSION}") +} + +fn uuid(idx: usize) -> String { + format!("0000000a-0000-4000-8000-{idx:012x}") +} + +fn encode_purl(purl: &str) -> String { + purl.replace(':', "%3A") + .replace('/', "%2F") + .replace('@', "%40") +} + +fn scrubbed_cli() -> Command { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_socket-patch")); + for (key, _) in std::env::vars_os() { + let name = key.to_string_lossy(); + if name.starts_with("SOCKET_") && name != "SOCKET_NO_CONFIG" { + cmd.env_remove(&key); + } + } + cmd.env_remove("VIRTUAL_ENV"); + cmd.env("SOCKET_TELEMETRY_DISABLED", "1"); + cmd +} + +fn run_scan(cwd: &Path, api: &str, args: &[&str], env: &[(&str, &str)]) -> (i32, String, String) { + let out = scrubbed_cli() + .arg("scan") + .args([ + "--cwd", + cwd.to_str().unwrap(), + "--api-url", + api, + "--api-token", + "fake-token-for-test", + "--org", + ORG, + ]) + .args(args) + // Never reach for the real proxy. + .env("SOCKET_PROXY_URL", api) + .envs(env.iter().copied()) + .output() + .expect("run socket-patch scan"); + ( + out.status.code().unwrap_or(-1), + String::from_utf8_lossy(&out.stdout).into_owned(), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) +} + +/// An npm project with every name in `NAMES` installed and locked. +fn write_project(root: &Path) { + let deps: Vec = NAMES + .iter() + .map(|n| format!(r#""{n}": "{VERSION}""#)) + .collect(); + let deps = deps.join(", "); + std::fs::write( + root.join("package.json"), + format!(r#"{{ "name": "consumer", "version": "0.0.0", "dependencies": {{ {deps} }} }}"#), + ) + .unwrap(); + let mut entries = vec![format!( + r#" "": {{ "name": "consumer", "version": "0.0.0", "dependencies": {{ {deps} }} }}"# + )]; + for name in NAMES { + let pkg = root.join("node_modules").join(name); + std::fs::create_dir_all(&pkg).unwrap(); + std::fs::write( + pkg.join("package.json"), + format!(r#"{{ "name": "{name}", "version": "{VERSION}" }}"#), + ) + .unwrap(); + std::fs::write(pkg.join("index.js"), b"module.exports = 1;\n").unwrap(); + entries.push(format!( + r#" "node_modules/{name}": {{ + "version": "{VERSION}", + "resolved": "https://registry.npmjs.org/{name}/-/{name}-{VERSION}.tgz", + "integrity": "sha512-UPSTREAM{name}==" + }}"# + )); + } + std::fs::write( + root.join("package-lock.json"), + format!( + "{{\n \"name\": \"consumer\",\n \"version\": \"0.0.0\",\n \"lockfileVersion\": 3,\n \ + \"requires\": true,\n \"packages\": {{\n{}\n }}\n}}\n", + entries.join(",\n") + ), + ) + .unwrap(); +} + +fn auth_batch_route() -> String { + format!("/v0/orgs/{ORG}/patches/batch") +} + +fn batch_entry(idx: usize) -> serde_json::Value { + let p = purl(NAMES[idx]); + serde_json::json!({ + "purl": p, + "patches": [{ + "uuid": uuid(idx), "purl": p, "tier": "free", + "cveIds": [], "ghsaIds": [], "severity": "high", + "title": format!("patch for {}", NAMES[idx]), + }] + }) +} + +fn batch_body(entries: Vec) -> serde_json::Value { + serde_json::json!({ "packages": entries, "canAccessPaidPatches": false }) +} + +fn throttled(status: u16, idx: usize) -> ResponseTemplate { + ResponseTemplate::new(status) + .insert_header("Retry-After", "0") + .set_body_string(format!("busy-{}", NAMES[idx])) +} + +/// Per-package batch mocks (`--batch-size 1`): package `idx` answers +/// `throttle(idx)` = `Some((status, times))` that many times first (`None` +/// times = forever), then its 200. +async fn mount_batches( + server: &MockServer, + throttle: impl Fn(usize) -> Option<(u16, Option)>, +) { + for (idx, name) in NAMES.iter().enumerate() { + let needle = format!("\"{}\"", purl(name)); + let body = || body_string_contains(needle.clone()); + if let Some((status, times)) = throttle(idx) { + let mock = Mock::given(method("POST")) + .and(path(auth_batch_route())) + .and(body()) + .respond_with(throttled(status, idx)) + .with_priority(1); + let mock = match times { + Some(n) => mock.up_to_n_times(n), + None => mock, + }; + mock.mount(server).await; + } + Mock::given(method("POST")) + .and(path(auth_batch_route())) + .and(body()) + .respond_with( + ResponseTemplate::new(200).set_body_json(batch_body(vec![batch_entry(idx)])), + ) + .mount(server) + .await; + } +} + +/// Per-package detail GETs for every package; `stuck` answers 429 forever. +async fn mount_details(server: &MockServer, stuck: Option) { + for (idx, name) in NAMES.iter().enumerate() { + let p = purl(name); + let route = format!("/v0/orgs/{ORG}/patches/by-package/{}", encode_purl(&p)); + let template = if Some(idx) == stuck { + throttled(429, idx) + } else { + ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "patches": [{ + "uuid": uuid(idx), "purl": p, + "publishedAt": "2024-01-01T00:00:00Z", + "description": format!("details for {name}"), + "license": "MIT", "tier": "free", "vulnerabilities": {} + }], + "canAccessPaidPatches": false, + })) + }; + Mock::given(method("GET")) + .and(path(route)) + .respond_with(template) + .mount(server) + .await; + } +} + +async fn posts_for(server: &MockServer, idx: usize) -> usize { + let needle = format!("\"{}\"", purl(NAMES[idx])); + server + .received_requests() + .await + .unwrap() + .iter() + .filter(|r| { + r.url.path() == auth_batch_route() && String::from_utf8_lossy(&r.body).contains(&needle) + }) + .count() +} + +fn json(stdout: &str) -> serde_json::Value { + serde_json::from_str(stdout).unwrap_or_else(|e| panic!("{e}: {stdout}")) +} + +fn warnings(v: &serde_json::Value, code: &str) -> Vec { + v["warnings"] + .as_array() + .map(|w| { + w.iter() + .filter(|w| w["code"] == code) + .map(|w| w["detail"].as_str().unwrap().to_string()) + .collect() + }) + .unwrap_or_default() +} + +/// 429 (and 503) answers that clear within the retries leave the `--json` +/// envelope byte-identical to a clean run's. +#[tokio::test] +async fn transient_throttling_is_invisible_in_the_envelope() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path()); + + let clean = MockServer::start().await; + mount_batches(&clean, |_| None).await; + let (code0, stdout0, stderr0) = run_scan( + tmp.path(), + &clean.uri(), + &["--json", "--batch-size", "1"], + &[], + ); + assert_eq!(code0, 0, "{stdout0}\n{stderr0}"); + assert_eq!(json(&stdout0)["packages"].as_array().unwrap().len(), 6); + + let busy = MockServer::start().await; + mount_batches(&busy, |idx| match idx % 3 { + 0 => Some((429, Some(1))), + 1 => Some((503, Some(3))), + _ => None, + }) + .await; + let (code, stdout, stderr) = run_scan( + tmp.path(), + &busy.uri(), + &["--json", "--batch-size", "1"], + &[], + ); + assert_eq!(code, 0, "{stdout}\n{stderr}"); + assert_eq!( + stdout, stdout0, + "retried answers fold exactly like clean ones" + ); + assert_eq!(posts_for(&busy, 0).await, 2); + assert_eq!(posts_for(&busy, 1).await, 4, "3 retries is the default"); + assert_eq!(posts_for(&busy, 2).await, 1); +} + +/// Batches still throttled after 3 retries: the run succeeds with the rest, +/// and each failed batch is a `warnings[]` entry in chunk order — the same +/// text a human run prints on stderr — instead of its packages silently +/// vanishing from `packages`. +#[tokio::test] +async fn exhausted_batches_surface_as_json_warnings() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path()); + let server = MockServer::start().await; + // Throttle two packages forever: one 429, one 503. + let stuck = |idx: usize| match idx { + 1 => Some((429, None)), + 4 => Some((503, None)), + _ => None, + }; + mount_batches(&server, stuck).await; + mount_details(&server, None).await; + + let (code, stdout, stderr) = run_scan( + tmp.path(), + &server.uri(), + &["--json", "--batch-size", "1"], + &[], + ); + assert_eq!(code, 0, "{stdout}\n{stderr}"); + let v = json(&stdout); + assert_eq!(v["status"], "success"); + assert_eq!(v["packagesWithPatches"], 4, "{v:#}"); + let w = warnings(&v, "api_batch_failed"); + assert_eq!(w.len(), 2, "{v:#}"); + // Chunk order is crawl order (readdir), so match either batch number. + let rate = + "failed: Rate limit exceeded (HTTP 429, gave up after 3 retries). Please try again later."; + let down = format!( + "failed: API request failed with status 503: busy-{} (gave up after 3 retries)", + NAMES[4] + ); + assert!( + w.iter() + .any(|d| d.starts_with("API batch ") && d.ends_with(rate)), + "{w:?}" + ); + assert!(w.iter().any(|d| d.ends_with(&down)), "{w:?}"); + assert!(w.iter().all(|d| d.contains(" of 6 failed: ")), "{w:?}"); + let first_batch = |d: &String| -> usize { + d["API batch ".len()..] + .split(' ') + .next() + .unwrap() + .parse() + .unwrap() + }; + assert!( + first_batch(&w[0]) < first_batch(&w[1]), + "chunk order: {w:?}" + ); + assert_eq!(posts_for(&server, 1).await, 4); + assert_eq!(posts_for(&server, 4).await, 4); + + // The human run prints the same lines as warnings on stderr. + let (code_h, _, stderr_h) = run_scan( + tmp.path(), + &server.uri(), + &["--batch-size", "1", "--dry-run"], + &[], + ); + assert_eq!(code_h, 0, "{stderr_h}"); + for d in &w { + assert!( + stderr_h.contains(&format!("Warning: {d}")), + "{d}\n{stderr_h}" + ); + } +} + +/// Every batch still throttled: the existing all-batches-failed error +/// envelope and exit 1, carrying the last chunk's error. +#[tokio::test] +async fn every_batch_exhausted_is_the_all_failed_error() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path()); + let server = MockServer::start().await; + mount_batches(&server, |_| Some((503, None))).await; + let (code, stdout, stderr) = run_scan( + tmp.path(), + &server.uri(), + &["--json", "--batch-size", "1"], + &[], + ); + assert_eq!(code, 1, "{stdout}\n{stderr}"); + let v = json(&stdout); + assert_eq!(v["status"], "error"); + let err = v["error"].as_str().unwrap(); + assert!( + err.starts_with("API request failed with status 503: busy-") + && err.ends_with(" (gave up after 3 retries)"), + "{err}" + ); + assert!(v.get("warnings").is_none(), "{v:#}"); +} + +/// `SOCKET_API_MAX_RETRIES=0` turns the retry off: one request per batch, +/// and the pre-retry error text. +#[tokio::test] +async fn max_retries_env_zero_disables_retry() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path()); + let server = MockServer::start().await; + mount_batches(&server, |idx| (idx == 2).then_some((429, Some(1)))).await; + let (code, stdout, stderr) = run_scan( + tmp.path(), + &server.uri(), + &["--json", "--batch-size", "1"], + &[("SOCKET_API_MAX_RETRIES", "0")], + ); + assert_eq!(code, 0, "{stdout}\n{stderr}"); + let v = json(&stdout); + assert_eq!(v["packagesWithPatches"], 5); + let w = warnings(&v, "api_batch_failed"); + assert_eq!(w.len(), 1, "{v:#}"); + assert!( + w[0].ends_with("failed: Rate limit exceeded. Please try again later."), + "{w:?}" + ); + assert_eq!(posts_for(&server, 2).await, 1); +} + +/// The agent flow's per-package detail fetch: a package still throttled +/// after its retries is a `patch_details_failed` warning in the `--json` +/// envelope (the human run's `could not fetch details` line), and the +/// other packages proceed. +#[tokio::test] +async fn exhausted_detail_fetch_is_a_json_warning() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path()); + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(auth_batch_route())) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(batch_body((0..NAMES.len()).map(batch_entry).collect())), + ) + .mount(&server) + .await; + let stuck = 3usize; + mount_details(&server, Some(stuck)).await; + let (code, stdout, stderr) = run_scan( + tmp.path(), + &server.uri(), + &["--json", "--apply", "--dry-run"], + &[], + ); + let v = json(&stdout); + assert_eq!(code, 0, "{v:#}\n{stderr}"); + let w = warnings(&v, "patch_details_failed"); + assert_eq!( + w, + vec![format!( + "could not fetch details for {}: Rate limit exceeded (HTTP 429, gave up after 3 \ + retries). Please try again later.", + purl(NAMES[stuck]) + )], + "{v:#}" + ); + let stuck_route = format!( + "/v0/orgs/{ORG}/patches/by-package/{}", + encode_purl(&purl(NAMES[stuck])) + ); + let gets = server + .received_requests() + .await + .unwrap() + .iter() + .filter(|r| r.url.path() == stuck_route) + .count(); + assert_eq!(gets, 4); +} diff --git a/crates/socket-patch-cli/tests/scan_batch_sizing_e2e.rs b/crates/socket-patch-cli/tests/scan_batch_sizing_e2e.rs new file mode 100644 index 000000000..55feb02ff --- /dev/null +++ b/crates/socket-patch-cli/tests/scan_batch_sizing_e2e.rs @@ -0,0 +1,283 @@ +//! How `scan` sizes its batch requests: unset, `--batch-size` follows the +//! endpoint (500 purls per POST on the authenticated API — the server's own +//! per-request maximum — and 100 on the public proxy); a given size +//! (`--batch-size` or `SOCKET_BATCH_SIZE`) wins on either; a chunk whose +//! body would exceed 256 KiB is split into consecutive smaller chunks; and a +//! mid-run downgrade to the proxy keeps the chunk boundaries the run started +//! with. +//! +//! Subprocess runs scrub the `SOCKET_*` environment (the +//! `scan_ordered_concurrency_e2e.rs::scrubbed_cli` pattern) so ambient +//! configuration cannot reroute the branch under test. + +use std::path::Path; +use std::process::Command; + +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const ORG: &str = "test-org"; +const AUTH_BATCH_ROUTE: &str = "/v0/orgs/test-org/patches/batch"; +const PROXY_BATCH_ROUTE: &str = "/patch/batch"; +/// The body cap `scan` splits at (the public proxy's own limit). +const BODY_CAP: usize = 256 * 1024; + +fn scrubbed_cli() -> Command { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_socket-patch")); + for (key, _) in std::env::vars_os() { + let name = key.to_string_lossy(); + if name.starts_with("SOCKET_") && name != "SOCKET_NO_CONFIG" { + cmd.env_remove(&key); + } + } + cmd.env_remove("VIRTUAL_ENV"); + cmd.env("SOCKET_TELEMETRY_DISABLED", "1"); + cmd +} + +/// `scan --json` in `cwd`: authenticated against `server` when `token`, +/// else token-less against `server` as the public proxy (the authenticated +/// base is unroutable, so a stray authenticated request fails the run). +fn run_scan( + cwd: &Path, + server: &str, + token: bool, + args: &[&str], + env: &[(&str, &str)], +) -> (i32, String, String) { + let mut cmd = scrubbed_cli(); + cmd.arg("scan") + .args(["--json", "--cwd", cwd.to_str().unwrap()]) + .args(args) + .env("SOCKET_NO_CONFIG", "1"); + if token { + cmd.args([ + "--api-url", + server, + "--api-token", + "fake-token-for-test", + "--org", + ORG, + ]); + } else { + cmd.args(["--api-url", "http://127.0.0.1:1", "--proxy-url", server]) + .env("SOCKET_NO_API_TOKEN", "1"); + } + for (key, value) in env { + cmd.env(key, value); + } + let out = cmd.output().expect("run socket-patch scan"); + ( + out.status.code().unwrap_or(-1), + String::from_utf8_lossy(&out.stdout).into_owned(), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) +} + +/// An npm project with `n` installed packages whose names are padded to +/// `name_len` characters. +fn write_project(root: &Path, n: usize, name_len: usize) { + std::fs::write( + root.join("package.json"), + r#"{ "name": "consumer", "version": "0.0.0" }"#, + ) + .unwrap(); + for i in 0..n { + let head = format!("pkg{i:05}-"); + let name = format!("{head}{}", "x".repeat(name_len.saturating_sub(head.len()))); + let dir = root.join("node_modules").join(&name); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write( + dir.join("package.json"), + format!(r#"{{ "name": "{name}", "version": "1.0.0" }}"#), + ) + .unwrap(); + } +} + +async fn empty_batch_server(route: &str) -> MockServer { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(route)) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "packages": [], "canAccessPaidPatches": false, + }))) + .mount(&server) + .await; + server +} + +/// Each batch POST's body, in arrival order. +async fn batch_bodies(server: &MockServer, route: &str) -> Vec { + server + .received_requests() + .await + .expect("wiremock records requests") + .into_iter() + .filter(|r| r.method.as_str() == "POST" && r.url.path() == route) + .map(|r| String::from_utf8_lossy(&r.body).into_owned()) + .collect() +} + +fn components(body: &str) -> Vec { + let v: serde_json::Value = serde_json::from_str(body).expect("batch body"); + v["components"] + .as_array() + .expect("components") + .iter() + .map(|c| c["purl"].as_str().unwrap().to_string()) + .collect() +} + +/// The chunk sizes of a run's batch POSTs, largest first (the chunks run +/// concurrently, so arrival order is not chunk order). +fn sizes(bodies: &[String]) -> Vec { + let mut sizes: Vec = bodies.iter().map(|b| components(b).len()).collect(); + sizes.sort_unstable_by(|a, b| b.cmp(a)); + sizes +} + +/// One run of `scan` with `args`, returning its batch bodies. +async fn bodies_of(root: &Path, token: bool, args: &[&str], env: &[(&str, &str)]) -> Vec { + let route = if token { + AUTH_BATCH_ROUTE + } else { + PROXY_BATCH_ROUTE + }; + let server = empty_batch_server(route).await; + let (code, stdout, stderr) = run_scan(root, &server.uri(), token, args, env); + assert_eq!(code, 0, "stdout={stdout} stderr={stderr}"); + let v: serde_json::Value = serde_json::from_str(&stdout).expect("scan --json envelope"); + assert_eq!(v["status"], "success", "{stdout}"); + batch_bodies(&server, route).await +} + +/// Unset on the authenticated API: 500 purls per POST, cut at the same +/// place a one-chunk run lists them (chunk 0 is the first 500 of the crawl +/// order, chunk 1 the rest). +#[tokio::test] +async fn authenticated_default_is_500_per_batch() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), 501, 12); + + let bodies = bodies_of(tmp.path(), true, &[], &[]).await; + assert_eq!(sizes(&bodies), vec![500, 1]); + + let whole = bodies_of(tmp.path(), true, &["--batch-size", "1000"], &[]).await; + assert_eq!(whole.len(), 1); + let order = components(&whole[0]); + let mut chunks: Vec> = bodies.iter().map(|b| components(b)).collect(); + chunks.sort_by_key(|c| std::cmp::Reverse(c.len())); + assert_eq!(chunks.concat(), order, "same purls, same order, cut at 500"); +} + +/// Unset on the public proxy: 100 purls per POST, as before. +#[tokio::test] +async fn proxy_default_stays_100_per_batch() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), 101, 12); + + let bodies = bodies_of(tmp.path(), false, &[], &[]).await; + assert_eq!(sizes(&bodies), vec![100, 1]); +} + +/// A given size wins on both endpoints, from the flag or from +/// `SOCKET_BATCH_SIZE` (which the parser honors like the flag). +#[tokio::test] +async fn an_explicit_batch_size_wins_on_either_endpoint() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), 20, 12); + + for token in [true, false] { + let flag = bodies_of(tmp.path(), token, &["--batch-size", "7"], &[]).await; + assert_eq!(sizes(&flag), vec![7, 7, 6], "flag, token={token}"); + let env = bodies_of(tmp.path(), token, &[], &[("SOCKET_BATCH_SIZE", "7")]).await; + assert_eq!(sizes(&env), vec![7, 7, 6], "env, token={token}"); + } +} + +/// A chunk whose body would pass 256 KiB is split: every POST fits, and +/// together they carry every purl exactly once. +#[tokio::test] +async fn an_oversize_chunk_is_split_at_the_body_cap() { + let tmp = tempfile::tempdir().unwrap(); + // 1,300 purls of ~226 body bytes each: ~290 KiB in one 5,000-purl chunk. + write_project(tmp.path(), 1300, 200); + + let bodies = bodies_of(tmp.path(), true, &["--batch-size", "5000"], &[]).await; + assert_eq!(bodies.len(), 2, "{:?}", sizes(&bodies)); + for body in &bodies { + assert!(body.len() <= BODY_CAP, "{} bytes", body.len()); + } + let mut all: Vec = bodies.iter().flat_map(|b| components(b)).collect(); + all.sort(); + all.dedup(); + assert_eq!(all.len(), 1300); +} + +/// A 401 from the authenticated batch endpoint downgrades the run to the +/// public proxy, which then gets the SAME 500-purl chunks: the failed chunk +/// retried as-is, the rest as cut. The downgrade does not re-chunk at the +/// proxy's own 100-purl default (every chunk is within the proxy's body cap +/// by construction), it warns once, and the run still succeeds. +#[tokio::test] +async fn a_mid_run_downgrade_keeps_the_500_purl_chunks() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), 1001, 12); + + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(AUTH_BATCH_ROUTE)) + .respond_with(ResponseTemplate::new(401)) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path(PROXY_BATCH_ROUTE)) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "packages": [], "canAccessPaidPatches": false, + }))) + .mount(&server) + .await; + + let mut cmd = scrubbed_cli(); + cmd.arg("scan") + .args(["--json", "--cwd", tmp.path().to_str().unwrap()]) + .args(["--api-url", &server.uri(), "--proxy-url", &server.uri()]) + .args(["--api-token", "fake-token-for-test", "--org", ORG]) + .env("SOCKET_NO_CONFIG", "1"); + let out = cmd.output().expect("run socket-patch scan"); + let stdout = String::from_utf8_lossy(&out.stdout); + let stderr = String::from_utf8_lossy(&out.stderr); + assert_eq!( + out.status.code(), + Some(0), + "stdout={stdout} stderr={stderr}" + ); + let v: serde_json::Value = serde_json::from_str(&stdout).expect("scan --json envelope"); + assert_eq!(v["status"], "success", "{stdout}"); + + let auth = batch_bodies(&server, AUTH_BATCH_ROUTE).await; + assert_eq!( + sizes(&auth), + vec![500], + "the first chunk goes alone, then downgrades" + ); + let proxy = batch_bodies(&server, PROXY_BATCH_ROUTE).await; + assert_eq!( + sizes(&proxy), + vec![500, 500, 1], + "the proxy gets the run's own chunks, not a 100-purl re-chunk" + ); + assert_eq!( + components(&auth[0]), + components(&proxy[0]), + "the failed chunk is retried as-is, first" + ); + assert_eq!( + stderr + .matches("falling back to public patch API proxy") + .count(), + 1, + "{stderr}" + ); +} diff --git a/crates/socket-patch-cli/tests/scan_ecosystems_scope_e2e.rs b/crates/socket-patch-cli/tests/scan_ecosystems_scope_e2e.rs new file mode 100644 index 000000000..cca601d43 --- /dev/null +++ b/crates/socket-patch-cli/tests/scan_ecosystems_scope_e2e.rs @@ -0,0 +1,283 @@ +//! `scan --ecosystems` crawls only the named ecosystems unless the run +//! garbage-collects. +//! +//! Without `--prune`/`--sync`, every output of the run is already narrowed +//! to the selected ecosystems, so the other crawlers are not run at all — +//! which by design shows in no output, so these tests do not (cannot) pin +//! the skip itself: `crawl_scope`'s unit test pins the decision and +//! `ecosystem_dispatch`'s `scoped_crawl_is_the_full_crawl_filtered_to_its_ecosystems` +//! pins that a scoped crawl never runs the other crawlers. What these pin +//! is the one output that used to see past the filter, the lockfile-only +//! count: a selected-ecosystem scan counted the OTHER ecosystems' +//! uninstalled lockfile entries too. A scoped run cannot tell whether a +//! skipped ecosystem's entry is installed, so `lockfileOnlyPackages` now +//! counts the selected ecosystems only. A GC run still crawls everything — +//! the prune judges every manifest entry against the full installed set — +//! and keeps the old count (and keeps an installed crate of an unselected +//! ecosystem out of the prune). +//! +//! The fixture: an npm project (one installed package, one lockfile-only) +//! that also carries a `Cargo.lock` whose one crates.io crate is not +//! installed (`CARGO_HOME` points at an empty dir, so the cargo crawl finds +//! nothing — hermetic, and a lockfile-only cargo entry either way). + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +fn binary() -> PathBuf { + env!("CARGO_BIN_EXE_socket-patch").into() +} + +const ORG: &str = "test-org"; +const INSTALLED_PURL: &str = "pkg:npm/installed-dep@1.0.0"; +const NPM_LOCK_ONLY: &str = "lock-only-dep"; +const CARGO_LOCK_ONLY: &str = "lockonly-crate"; +const INSTALLED_CRATE: &str = "installed-crate"; + +/// `scan --json` against `api_url` with the `SOCKET_*` environment +/// scrubbed (except the workspace-pinned `SOCKET_NO_CONFIG`), `VIRTUAL_ENV` +/// removed (the python crawler honors it first) and `CARGO_HOME` pinned to +/// an empty dir under `cwd`. +fn run_scan(cwd: &Path, api_url: &str, extra: &[&str]) -> (i32, String, String) { + let mut cmd = Command::new(binary()); + cmd.arg("scan").current_dir(cwd); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("SOCKET_") + && key.to_string_lossy() != "SOCKET_NO_CONFIG" + { + cmd.env_remove(&key); + } + } + cmd.env_remove("VIRTUAL_ENV"); + cmd.env("CARGO_HOME", cwd.join(".cargo-home")); + cmd.env("SOCKET_TELEMETRY_DISABLED", "1"); + let out = cmd + .args([ + "--json", + "--api-url", + api_url, + "--api-token", + "fake-token-for-test", + "--org", + ORG, + ]) + .args(extra) + .output() + .expect("run socket-patch"); + ( + out.status.code().unwrap_or(-1), + String::from_utf8_lossy(&out.stdout).to_string(), + String::from_utf8_lossy(&out.stderr).to_string(), + ) +} + +fn stage_project(root: &Path) { + std::fs::write( + root.join("package.json"), + r#"{ "name": "scope-root", "version": "0.0.0", + "dependencies": { "installed-dep": "^1.0.0", "lock-only-dep": "^1.0.0" } }"#, + ) + .unwrap(); + let pkg = root.join("node_modules").join("installed-dep"); + std::fs::create_dir_all(&pkg).unwrap(); + std::fs::write( + pkg.join("package.json"), + r#"{ "name": "installed-dep", "version": "1.0.0" }"#, + ) + .unwrap(); + let entry = |name: &str| { + serde_json::json!({ + "version": "1.0.0", + "resolved": format!("https://registry.npmjs.org/{name}/-/{name}-1.0.0.tgz"), + "integrity": "sha512-fake==", + "license": "MIT" + }) + }; + let lock = serde_json::json!({ + "name": "scope-root", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "scope-root", + "version": "0.0.0", + "dependencies": { "installed-dep": "^1.0.0", "lock-only-dep": "^1.0.0" } + }, + "node_modules/installed-dep": entry("installed-dep"), + "node_modules/lock-only-dep": entry(NPM_LOCK_ONLY), + } + }); + std::fs::write( + root.join("package-lock.json"), + serde_json::to_vec_pretty(&lock).unwrap(), + ) + .unwrap(); + std::fs::write( + root.join("Cargo.lock"), + format!( + "version = 3\n\n[[package]]\nname = \"{CARGO_LOCK_ONLY}\"\nversion = \"0.1.0\"\n\ + source = \"registry+https://github.com/rust-lang/crates.io-index\"\n\ + checksum = \"{}\"\n", + "ab".repeat(32) + ), + ) + .unwrap(); +} + +async fn mock_batch_empty(server: &MockServer) { + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/batch"))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "packages": [], "canAccessPaidPatches": false, + }))) + .mount(server) + .await; +} + +/// Run one scan against a fresh mock and hand back its envelope plus the +/// concatenated batch request bodies. +async fn scan(root: &Path, extra: &[&str]) -> (serde_json::Value, String) { + let server = MockServer::start().await; + mock_batch_empty(&server).await; + let (code, stdout, stderr) = run_scan(root, &server.uri(), extra); + assert_eq!(code, 0, "{extra:?}: stdout={stdout}; stderr={stderr}"); + let v: serde_json::Value = serde_json::from_str(stdout.trim()) + .unwrap_or_else(|e| panic!("{extra:?}: not a JSON envelope ({e}): {stdout}")); + let bodies = server + .received_requests() + .await + .unwrap_or_default() + .iter() + .filter(|r| r.url.path().ends_with("/patches/batch")) + .map(|r| String::from_utf8_lossy(&r.body).into_owned()) + .collect::>() + .join("\n"); + (v, bodies) +} + +/// Without a GC, `-e npm` counts, queries and flags npm alone: the +/// lockfile-only count no longer includes the cargo crate the (skipped) +/// cargo crawl would have had to vouch for. +#[tokio::test] +async fn scoped_scan_counts_only_the_selected_ecosystems() { + let tmp = tempfile::tempdir().unwrap(); + stage_project(tmp.path()); + + for extra in [ + &["-e", "npm"][..], + &["--ecosystems", "npm", "--dry-run"][..], + ] { + let (v, bodies) = scan(tmp.path(), extra).await; + assert_eq!(v["status"], "success", "{extra:?}: {v}"); + assert_eq!( + v["scannedPackages"], 2, + "{extra:?}: the installed npm package plus the npm lockfile-only one: {v}" + ); + assert_eq!( + v["lockfileOnlyPackages"], 1, + "{extra:?}: only the npm lockfile-only entry counts: {v}" + ); + assert!(bodies.contains(INSTALLED_PURL), "{extra:?}: {bodies}"); + assert!(bodies.contains(NPM_LOCK_ONLY), "{extra:?}: {bodies}"); + assert!(!bodies.contains(CARGO_LOCK_ONLY), "{extra:?}: {bodies}"); + } + + // Selecting cargo too brings its lockfile-only crate back, crawled. + let (v, bodies) = scan(tmp.path(), &["-e", "npm,cargo"]).await; + assert_eq!(v["scannedPackages"], 3, "{v}"); + assert_eq!(v["lockfileOnlyPackages"], 2, "{v}"); + assert!(bodies.contains(CARGO_LOCK_ONLY), "{bodies}"); +} + +/// A GC run (`--prune`, or `--sync`, which implies it) still crawls every +/// ecosystem, so the lockfile-only count keeps seeing past the filter +/// exactly as before, while what is counted, queried and shown stays npm. +#[tokio::test] +async fn gc_scan_keeps_the_unscoped_lockfile_only_count() { + let tmp = tempfile::tempdir().unwrap(); + stage_project(tmp.path()); + + for extra in [ + &["-e", "npm", "--prune", "--dry-run"][..], + &["-e", "npm", "--sync", "--dry-run"][..], + ] { + let (v, bodies) = scan(tmp.path(), extra).await; + assert_eq!(v["status"], "success", "{extra:?}: {v}"); + assert_eq!(v["scannedPackages"], 2, "{extra:?}: {v}"); + assert_eq!( + v["lockfileOnlyPackages"], 2, + "{extra:?}: the full crawl also counts the cargo lockfile-only crate: {v}" + ); + assert!(!bodies.contains(CARGO_LOCK_ONLY), "{extra:?}: {bodies}"); + } +} + +/// The crawl of a GC run itself is unscoped: a crate installed under +/// `CARGO_HOME` (and absent from `Cargo.lock`, so only the cargo CRAWL can +/// vouch for it) keeps its manifest entry under `-e npm --prune`, while an +/// uninstalled npm entry is still prunable. +#[tokio::test] +async fn gc_scan_crawls_the_unselected_ecosystems() { + let tmp = tempfile::tempdir().unwrap(); + stage_project(tmp.path()); + let krate = tmp + .path() + .join(".cargo-home") + .join("registry") + .join("src") + .join("index.crates.io-6f17d22bba15001f") + .join(format!("{INSTALLED_CRATE}-0.2.0")); + std::fs::create_dir_all(&krate).unwrap(); + std::fs::write( + krate.join("Cargo.toml"), + format!("[package]\nname = \"{INSTALLED_CRATE}\"\nversion = \"0.2.0\"\n"), + ) + .unwrap(); + let socket = tmp.path().join(".socket"); + std::fs::create_dir_all(&socket).unwrap(); + let record = |uuid: &str| { + serde_json::json!({ + "uuid": uuid, "exportedAt": "2024-01-01T00:00:00Z", + "files": {}, "vulnerabilities": {}, + "description": "d", "license": "MIT", "tier": "free" + }) + }; + let manifest = serde_json::json!({ "patches": { + format!("pkg:cargo/{INSTALLED_CRATE}@0.2.0"): record("11111111-1111-4111-8111-111111111111"), + "pkg:npm/orphan-npm@9.9.9": record("22222222-2222-4222-8222-222222222222"), + }}); + std::fs::write( + socket.join("manifest.json"), + serde_json::to_vec_pretty(&manifest).unwrap(), + ) + .unwrap(); + + for extra in [ + &["-e", "npm", "--prune", "--dry-run"][..], + &["-e", "npm", "--sync", "--dry-run"][..], + ] { + let (v, _) = scan(tmp.path(), extra).await; + assert_eq!( + v["gc"]["prunableManifestEntries"], + serde_json::json!(["pkg:npm/orphan-npm@9.9.9"]), + "{extra:?}: the installed crate must not read as uninstalled: {v}" + ); + } +} + +/// Without `--ecosystems` nothing changes: every ecosystem is crawled, +/// counted and queried. +#[tokio::test] +async fn unfiltered_scan_crawls_every_ecosystem() { + let tmp = tempfile::tempdir().unwrap(); + stage_project(tmp.path()); + + let (v, bodies) = scan(tmp.path(), &[]).await; + assert_eq!(v["scannedPackages"], 3, "{v}"); + assert_eq!(v["lockfileOnlyPackages"], 2, "{v}"); + assert!(bodies.contains(CARGO_LOCK_ONLY), "{bodies}"); +} diff --git a/crates/socket-patch-cli/tests/scan_ordered_concurrency_e2e.rs b/crates/socket-patch-cli/tests/scan_ordered_concurrency_e2e.rs new file mode 100644 index 000000000..4ce81b0c1 --- /dev/null +++ b/crates/socket-patch-cli/tests/scan_ordered_concurrency_e2e.rs @@ -0,0 +1,915 @@ +//! `scan`'s patch-API loops run their requests concurrently (batch POSTs, +//! per-package detail GETs, hosted record views) but must stay +//! indistinguishable from the old serial loops: results fold in input +//! order, warnings print in input order, and the authenticated-to-proxy +//! fallback replays from the exact chunk that triggered it. +//! +//! Every test here makes the server answer LATER requests FIRST (reversed +//! latencies), so an implementation that folded in completion order — or +//! that let a discarded response leak in — produces visibly different +//! output. Where a pure oracle exists, the output is also compared with a +//! zero-latency run of the same fixture. +//! +//! Subprocess runs scrub the `SOCKET_*` environment (the +//! `in_process_redirect.rs::scrubbed_cli` pattern) so ambient +//! configuration cannot reroute the branch under test. + +use std::path::Path; +use std::process::Command; +use std::time::Duration; + +use wiremock::matchers::{body_string_contains, method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const ORG: &str = "test-org"; + +/// Package names: no name is a prefix of another, so a body/path match on +/// `"{name}@"` is unambiguous. +const NAMES: [&str; 6] = [ + "conc-alpha", + "conc-bravo", + "conc-charlie", + "conc-delta", + "conc-echo", + "conc-foxtrot", +]; +const VERSION: &str = "1.0.0"; + +fn purl(name: &str) -> String { + format!("pkg:npm/{name}@{VERSION}") +} + +/// Distinct uuid per (package index, source) so the output reveals which +/// server response was folded for each package. +fn uuid(idx: usize, source: u8) -> String { + format!("{source:08x}-0000-4000-8000-{idx:012x}") +} + +const AUTH: u8 = 0xa; +const PROXY: u8 = 0xb; + +fn encode_purl(purl: &str) -> String { + purl.replace(':', "%3A") + .replace('/', "%2F") + .replace('@', "%40") +} + +fn scrubbed_cli() -> Command { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_socket-patch")); + cmd.env("SOCKET_DRY_RUN", "true") + .env("SOCKET_ECOSYSTEMS", "cargo") + .env("SOCKET_MANIFEST_PATH", "/nonexistent/manifest.json") + .env_remove("SOCKET_DRY_RUN") + .env_remove("SOCKET_ECOSYSTEMS") + .env_remove("SOCKET_MANIFEST_PATH"); + for (key, _) in std::env::vars_os() { + let name = key.to_string_lossy(); + if name.starts_with("SOCKET_") && name != "SOCKET_NO_CONFIG" { + cmd.env_remove(&key); + } + } + cmd.env_remove("VIRTUAL_ENV"); + cmd.env("SOCKET_TELEMETRY_DISABLED", "1"); + cmd +} + +/// `socket-patch scan ` in `cwd` against `api` (authenticated) with +/// the public proxy pointed at `proxy`. +fn run_scan(cwd: &Path, api: &str, proxy: &str, args: &[&str]) -> (i32, String, String) { + let out = scrubbed_cli() + .arg("scan") + .args([ + "--cwd", + cwd.to_str().unwrap(), + "--api-url", + api, + "--api-token", + "fake-token-for-test", + "--org", + ORG, + ]) + .args(args) + .env("SOCKET_PROXY_URL", proxy) + .output() + .expect("run socket-patch scan"); + ( + out.status.code().unwrap_or(-1), + String::from_utf8_lossy(&out.stdout).into_owned(), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) +} + +/// An npm project with every name in `names` installed and locked. +fn write_project(root: &Path, names: &[&str]) { + let deps: Vec = names + .iter() + .map(|n| format!(r#""{n}": "{VERSION}""#)) + .collect(); + let deps = deps.join(", "); + std::fs::write( + root.join("package.json"), + format!(r#"{{ "name": "consumer", "version": "0.0.0", "dependencies": {{ {deps} }} }}"#), + ) + .unwrap(); + let mut entries = vec![format!( + r#" "": {{ "name": "consumer", "version": "0.0.0", "dependencies": {{ {deps} }} }}"# + )]; + for name in names { + let pkg = root.join("node_modules").join(name); + std::fs::create_dir_all(&pkg).unwrap(); + std::fs::write( + pkg.join("package.json"), + format!(r#"{{ "name": "{name}", "version": "{VERSION}" }}"#), + ) + .unwrap(); + std::fs::write(pkg.join("index.js"), b"module.exports = 1;\n").unwrap(); + entries.push(format!( + r#" "node_modules/{name}": {{ + "version": "{VERSION}", + "resolved": "https://registry.npmjs.org/{name}/-/{name}-{VERSION}.tgz", + "integrity": "sha512-UPSTREAM{name}==" + }}"# + )); + } + std::fs::write( + root.join("package-lock.json"), + format!( + "{{\n \"name\": \"consumer\",\n \"version\": \"0.0.0\",\n \"lockfileVersion\": 3,\n \ + \"requires\": true,\n \"packages\": {{\n{}\n }}\n}}\n", + entries.join(",\n") + ), + ) + .unwrap(); +} + +fn batch_entry(idx: usize, source: u8) -> serde_json::Value { + let p = purl(NAMES[idx]); + serde_json::json!({ + "purl": p, + "patches": [{ + "uuid": uuid(idx, source), + "purl": p, + "tier": "free", + "cveIds": [], + "ghsaIds": [], + "severity": "high", + "title": format!("patch for {}", NAMES[idx]), + }] + }) +} + +fn batch_body(entries: Vec) -> serde_json::Value { + serde_json::json!({ "packages": entries, "canAccessPaidPatches": false }) +} + +/// One batch mock per package on `route`, matched by the package in the +/// request body: answers `status` (a 200 carries that package's patch from +/// `source`) after `delay`. +async fn mount_batch_for( + server: &MockServer, + route: &str, + idx: usize, + source: u8, + status: u16, + delay: Duration, +) { + let template = if status == 200 { + ResponseTemplate::new(200).set_body_json(batch_body(vec![batch_entry(idx, source)])) + } else { + ResponseTemplate::new(status).set_body_string(format!("boom-{}", NAMES[idx])) + }; + Mock::given(method("POST")) + .and(path(route)) + .and(body_string_contains(format!("\"{}\"", purl(NAMES[idx])))) + .respond_with(template.set_delay(delay)) + .mount(server) + .await; +} + +fn auth_batch_route() -> String { + format!("/v0/orgs/{ORG}/patches/batch") +} + +const PROXY_BATCH_ROUTE: &str = "/patch/batch"; + +async fn batch_requests(server: &MockServer, route: &str) -> Vec { + server + .received_requests() + .await + .expect("wiremock records requests") + .into_iter() + .filter(|r| r.method.as_str() == "POST" && r.url.path() == route) + .map(|r| String::from_utf8_lossy(&r.body).into_owned()) + .collect() +} + +/// The crawl order of the fixture's packages — the order `scan` chunks +/// them in. Learned from one default-batch-size run: its single batch body +/// lists every purl in that order. (Crawl order is readdir order, which the +/// test must not assume.) +async fn crawl_order(root: &Path) -> Vec { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(auth_batch_route())) + .respond_with(ResponseTemplate::new(200).set_body_json(batch_body(vec![]))) + .mount(&server) + .await; + let (code, stdout, stderr) = run_scan(root, &server.uri(), &server.uri(), &["--json"]); + assert_eq!(code, 0, "order probe: stdout={stdout} stderr={stderr}"); + let bodies = batch_requests(&server, &auth_batch_route()).await; + assert_eq!(bodies.len(), 1, "one batch expected: {bodies:?}"); + let body: serde_json::Value = serde_json::from_str(&bodies[0]).unwrap(); + let order: Vec = body["components"] + .as_array() + .unwrap() + .iter() + .map(|c| { + let p = c["purl"].as_str().unwrap(); + NAMES.iter().position(|n| purl(n) == p).unwrap() + }) + .collect(); + assert_eq!(order.len(), NAMES.len(), "every package crawled: {order:?}"); + order +} + +/// `package idx → folded patch uuids` from a `scan --json` envelope. +fn folded_uuids(stdout: &str) -> Vec<(String, Vec)> { + let v: serde_json::Value = serde_json::from_str(stdout).expect("scan --json envelope"); + v["packages"] + .as_array() + .expect("packages array") + .iter() + .map(|pkg| { + let uuids = pkg["patches"] + .as_array() + .unwrap() + .iter() + .map(|p| p["uuid"].as_str().unwrap().to_string()) + .collect(); + (pkg["purl"].as_str().unwrap().to_string(), uuids) + }) + .collect() +} + +/// Delay for the `pos`-th chunk (crawl position): later chunks answer +/// first. +fn reversed(pos: usize) -> Duration { + Duration::from_millis(60 * (NAMES.len() - pos) as u64) +} + +// --------------------------------------------------------------------------- +// Batch POSTs (A2) +// --------------------------------------------------------------------------- + +/// A 401 on the very first chunk: that chunk and every later one go to the +/// proxy, the authenticated API sees exactly the one request it saw +/// before, and the downgrade warning prints once. +#[tokio::test] +async fn batch_fallback_on_first_chunk_sends_everything_after_to_proxy() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), &NAMES[..5]); + + let auth = MockServer::start().await; + let proxy = MockServer::start().await; + Mock::given(method("POST")) + .and(path(auth_batch_route())) + .respond_with(ResponseTemplate::new(401).set_body_string("invalid token")) + .expect(1) + .mount(&auth) + .await; + for idx in 0..5 { + mount_batch_for(&proxy, PROXY_BATCH_ROUTE, idx, PROXY, 200, reversed(idx)).await; + } + + let (code, stdout, stderr) = run_scan( + tmp.path(), + &auth.uri(), + &proxy.uri(), + &["--json", "--batch-size", "1"], + ); + assert_eq!(code, 0, "stdout={stdout} stderr={stderr}"); + assert_eq!( + stderr + .matches("falling back to public patch API proxy") + .count(), + 1, + "exactly one downgrade warning: {stderr}" + ); + assert_eq!(batch_requests(&auth, &auth_batch_route()).await.len(), 1); + assert_eq!(batch_requests(&proxy, PROXY_BATCH_ROUTE).await.len(), 5); + let folded = folded_uuids(&stdout); + assert_eq!(folded.len(), 5, "{stdout}"); + for (p, uuids) in folded { + let idx = NAMES.iter().position(|n| purl(n) == p).unwrap(); + assert_eq!(uuids, vec![uuid(idx, PROXY)], "{p}"); + } +} + +/// A 401 on chunk 3 of 6 (crawl order): chunks 0-2 fold the authenticated +/// answers, chunk 3 is retried on the proxy and 4-5 are proxy-only. The +/// authenticated answers for 4-5 arrive BEFORE chunk 3's 401 (reversed +/// latencies) and must be discarded, exactly as if they had never been +/// sent. +#[tokio::test] +async fn batch_fallback_mid_run_replays_from_the_failing_chunk() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), &NAMES); + let order = crawl_order(tmp.path()).await; + + let auth = MockServer::start().await; + let proxy = MockServer::start().await; + for (pos, &idx) in order.iter().enumerate() { + let status = if pos == 3 { 401 } else { 200 }; + mount_batch_for(&auth, &auth_batch_route(), idx, AUTH, status, reversed(pos)).await; + mount_batch_for(&proxy, PROXY_BATCH_ROUTE, idx, PROXY, 200, reversed(pos)).await; + } + + let (code, stdout, stderr) = run_scan( + tmp.path(), + &auth.uri(), + &proxy.uri(), + &["--json", "--batch-size", "1"], + ); + assert_eq!(code, 0, "stdout={stdout} stderr={stderr}"); + assert_eq!( + stderr + .matches("falling back to public patch API proxy") + .count(), + 1, + "exactly one downgrade warning: {stderr}" + ); + + let mut expected: Vec<(String, Vec)> = order + .iter() + .enumerate() + .map(|(pos, &idx)| { + let source = if pos < 3 { AUTH } else { PROXY }; + (purl(NAMES[idx]), vec![uuid(idx, source)]) + }) + .collect(); + expected.sort(); + assert_eq!(folded_uuids(&stdout), expected); + + // The proxy saw exactly the replayed tail, chunk 3 onward. + let mut proxied: Vec = batch_requests(&proxy, PROXY_BATCH_ROUTE) + .await + .iter() + .map(|b| { + let v: serde_json::Value = serde_json::from_str(b).unwrap(); + v["components"][0]["purl"].as_str().unwrap().to_string() + }) + .collect(); + proxied.sort(); + let mut tail: Vec = order[3..].iter().map(|&i| purl(NAMES[i])).collect(); + tail.sort(); + assert_eq!(proxied, tail); + + // The authenticated API saw every chunk: chunk 0 alone, then the + // whole 1..6 window in flight at once. So the answers for chunks 4-5 + // really existed (they arrived before chunk 3's 401) and were + // dropped — the uuids above prove it — rather than never requested. + assert_eq!(batch_requests(&auth, &auth_batch_route()).await.len(), 6); +} + +/// The same mid-run downgrade under `--debug`: the window really does +/// dispatch chunks past the failing one to the authenticated endpoint, but +/// the chunks it then drops announce nothing. Each chunk's debug lines are +/// held back until it is folded, so the stream reads exactly as the +/// one-at-a-time loop's did — four authenticated POSTs (chunks 0-3, the +/// last being the 401) and three proxy POSTs (the replayed 3-5) — while +/// the authenticated server saw six requests. +#[tokio::test] +async fn a_dropped_batch_window_holds_back_its_debug_lines() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), &NAMES); + let order = crawl_order(tmp.path()).await; + + let auth = MockServer::start().await; + let proxy = MockServer::start().await; + for (pos, &idx) in order.iter().enumerate() { + let status = if pos == 3 { 401 } else { 200 }; + mount_batch_for(&auth, &auth_batch_route(), idx, AUTH, status, reversed(pos)).await; + mount_batch_for(&proxy, PROXY_BATCH_ROUTE, idx, PROXY, 200, reversed(pos)).await; + } + + let (code, stdout, stderr) = run_scan( + tmp.path(), + &auth.uri(), + &proxy.uri(), + &["--json", "--batch-size", "1", "--debug"], + ); + assert_eq!(code, 0, "stdout={stdout} stderr={stderr}"); + + let posted = |host: &str, route: &str| { + let needle = format!("[socket-patch debug] POST {host}{route}"); + stderr.matches(needle.as_str()).count() + }; + assert_eq!( + posted(&auth.uri(), &auth_batch_route()), + 4, + "chunks 0-3 announce themselves, the dropped 4-5 do not: {stderr}" + ); + assert_eq!( + posted(&proxy.uri(), PROXY_BATCH_ROUTE), + 3, + "the replayed tail announces itself once per chunk: {stderr}" + ); + assert_eq!( + batch_requests(&auth, &auth_batch_route()).await.len(), + 6, + "the window did dispatch the chunks whose lines were held back" + ); +} + +/// Mixed 500s in chunks 2 and 4 with reversed latencies: the per-batch +/// warnings print in chunk order, and the run still succeeds with the +/// other chunks' patches. +#[tokio::test] +async fn batch_failure_warnings_print_in_chunk_order() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), &NAMES); + let order = crawl_order(tmp.path()).await; + + let auth = MockServer::start().await; + for (pos, &idx) in order.iter().enumerate() { + let status = if pos == 2 || pos == 4 { 500 } else { 200 }; + mount_batch_for(&auth, &auth_batch_route(), idx, AUTH, status, reversed(pos)).await; + } + let all: Vec = (0..NAMES.len()).collect(); + mount_details(&auth, &all, &[], |_| Duration::ZERO).await; + + let (code, stdout, stderr) = run_scan( + tmp.path(), + &auth.uri(), + &auth.uri(), + &["--batch-size", "1", "--dry-run"], + ); + assert_eq!(code, 0, "stdout={stdout} stderr={stderr}"); + let warn2 = format!( + "Warning: API batch 3 of 6 failed: API request failed with status 500: boom-{}", + NAMES[order[2]] + ); + let warn4 = format!( + "Warning: API batch 5 of 6 failed: API request failed with status 500: boom-{}", + NAMES[order[4]] + ); + let at2 = stderr + .find(&warn2) + .unwrap_or_else(|| panic!("{warn2}\n{stderr}")); + let at4 = stderr + .find(&warn4) + .unwrap_or_else(|| panic!("{warn4}\n{stderr}")); + assert!(at2 < at4, "chunk-order warnings: {stderr}"); + assert_eq!(stderr.matches("Warning: API batch").count(), 2, "{stderr}"); +} + +/// Every chunk fails, the FIRST chunk slowest: the all-failed error still +/// carries the LAST chunk's error, as the serial loop's `last_batch_error` +/// did. +#[tokio::test] +async fn all_batches_failed_reports_the_last_chunks_error() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), &NAMES); + let order = crawl_order(tmp.path()).await; + + let auth = MockServer::start().await; + for (pos, &idx) in order.iter().enumerate() { + mount_batch_for(&auth, &auth_batch_route(), idx, AUTH, 500, reversed(pos)).await; + } + + let (code, stdout, stderr) = run_scan( + tmp.path(), + &auth.uri(), + &auth.uri(), + &["--json", "--batch-size", "1"], + ); + assert_eq!(code, 1, "stdout={stdout} stderr={stderr}"); + let v: serde_json::Value = serde_json::from_str(&stdout).unwrap(); + assert_eq!(v["status"], "error"); + assert_eq!( + v["error"].as_str().unwrap(), + format!( + "API request failed with status 500: boom-{}", + NAMES[order[5]] + ) + ); +} + +/// `scan` against `proxy` with NO API token — the common unauthenticated +/// path. `extra` env is applied on top of the scrubbed environment. The +/// authenticated base URL is unroutable, so any request that reached for it +/// fails the run loudly instead of escaping to the real API. +fn run_scan_anonymous( + cwd: &Path, + proxy: &str, + args: &[&str], + extra: &[(&str, &str)], +) -> (i32, String, String) { + let mut cmd = scrubbed_cli(); + cmd.arg("scan") + .args([ + "--cwd", + cwd.to_str().unwrap(), + "--api-url", + "http://127.0.0.1:1", + "--proxy-url", + proxy, + ]) + .args(args) + .env("SOCKET_NO_API_TOKEN", "1") + .env("SOCKET_NO_CONFIG", "1"); + for (key, value) in extra { + cmd.env(key, value); + } + let out = cmd.output().expect("run socket-patch scan"); + ( + out.status.code().unwrap_or(-1), + String::from_utf8_lossy(&out.stdout).into_owned(), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) +} + +/// Every proxy batch POST takes this long, so the chunks in flight at an +/// arrival are exactly those that arrived less than this before it. +const PROXY_CHUNK_DELAY: Duration = Duration::from_millis(400); + +/// Records each proxy batch POST's arrival, then answers its chunk's +/// packages after [`PROXY_CHUNK_DELAY`]. +struct ProxyArrivals(std::sync::Arc>>); + +impl wiremock::Respond for ProxyArrivals { + fn respond(&self, req: &wiremock::Request) -> ResponseTemplate { + self.0.lock().unwrap().push(std::time::Instant::now()); + let body: serde_json::Value = serde_json::from_slice(&req.body).expect("batch body"); + let entries: Vec = body["components"] + .as_array() + .expect("components") + .iter() + .map(|c| { + let p = c["purl"].as_str().unwrap(); + let idx = NAMES.iter().position(|n| purl(n) == p).unwrap(); + batch_entry(idx, PROXY) + }) + .collect(); + ResponseTemplate::new(200) + .set_body_json(batch_body(entries)) + .set_delay(PROXY_CHUNK_DELAY) + } +} + +/// A request is "in flight" at an arrival when it arrived less than this +/// before: its slot frees only once its answer, one delay later, is back. +fn in_flight_window() -> Duration { + PROXY_CHUNK_DELAY.mul_f32(0.8) +} + +/// Most arrivals inside one [`in_flight_window`] — a lower bound on the +/// chunks that were in flight together. +fn peak_in_flight(arrivals: &[Duration]) -> usize { + (0..arrivals.len()) + .map(|i| { + arrivals[i..] + .iter() + .take_while(|t| **t - arrivals[i] < in_flight_window()) + .count() + }) + .max() + .unwrap_or(0) +} + +/// One anonymous `scan --json --batch-size 1` over every package against a +/// proxy that records arrivals; returns `(stdout, each chunk POST's +/// arrival as an offset from the first, sorted)`. +async fn anonymous_batch_run(root: &Path, extra: &[(&str, &str)]) -> (String, Vec) { + let proxy = MockServer::start().await; + let arrivals = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + Mock::given(method("POST")) + .and(path(PROXY_BATCH_ROUTE)) + .respond_with(ProxyArrivals(std::sync::Arc::clone(&arrivals))) + .mount(&proxy) + .await; + + let (code, stdout, stderr) = + run_scan_anonymous(root, &proxy.uri(), &["--json", "--batch-size", "1"], extra); + assert_eq!(code, 0, "stdout={stdout} stderr={stderr}"); + assert_eq!( + batch_requests(&proxy, PROXY_BATCH_ROUTE).await.len(), + NAMES.len(), + "one POST per chunk" + ); + let mut arrivals = arrivals.lock().unwrap().clone(); + arrivals.sort(); + let first = arrivals[0]; + ( + stdout, + arrivals.into_iter().map(|t| t - first).collect::>(), + ) +} + +/// A token-less run is already on the proxy, so it has no authenticated +/// downgrade left to cap: its batch window opens at chunk 0 instead of +/// waiting out a round trip for a fallback that cannot fire. The second +/// chunk therefore arrives while the first is still unanswered — and +/// `SOCKET_API_CONCURRENCY=1`, the escape hatch for an endpoint that caps +/// in-flight requests, puts the very same run back on one request at a +/// time, with identical output. +#[tokio::test] +async fn proxy_batch_window_opens_at_the_first_chunk_unless_capped() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), &NAMES); + + let (concurrent_stdout, concurrent) = anonymous_batch_run(tmp.path(), &[]).await; + assert!( + concurrent[1] < in_flight_window(), + "chunk 0 must not go alone on the proxy: arrivals {concurrent:?}" + ); + assert!( + peak_in_flight(&concurrent) > 1, + "chunks must overlap: arrivals {concurrent:?}" + ); + + let (serial_stdout, serial) = + anonymous_batch_run(tmp.path(), &[("SOCKET_API_CONCURRENCY", "1")]).await; + assert_eq!( + peak_in_flight(&serial), + 1, + "a cap of 1 is the old serial loop: arrivals {serial:?}" + ); + assert_eq!( + concurrent_stdout, serial_stdout, + "the cap changes pacing, never output" + ); + let folded = folded_uuids(&concurrent_stdout); + assert_eq!(folded.len(), NAMES.len(), "{concurrent_stdout}"); + for (p, uuids) in folded { + let idx = NAMES.iter().position(|n| purl(n) == p).unwrap(); + assert_eq!(uuids, vec![uuid(idx, PROXY)], "{p}"); + } +} + +// --------------------------------------------------------------------------- +// Per-package detail GETs (A1) +// --------------------------------------------------------------------------- + +/// One batch answering every package in `idxs`, then per-package detail +/// GETs ([`mount_details`]). +async fn mount_discovery_with_details( + server: &MockServer, + idxs: &[usize], + failing: &[usize], + delay: impl Fn(usize) -> Duration, +) { + Mock::given(method("POST")) + .and(path(auth_batch_route())) + .respond_with(ResponseTemplate::new(200).set_body_json(batch_body( + idxs.iter().map(|&i| batch_entry(i, AUTH)).collect(), + ))) + .mount(server) + .await; + mount_details(server, idxs, failing, delay).await; +} + +/// Per-package detail GETs for `idxs`: `failing` ones answer 500, the rest +/// their patch; every answer is delayed by `delay(idx)`. +async fn mount_details( + server: &MockServer, + idxs: &[usize], + failing: &[usize], + delay: impl Fn(usize) -> Duration, +) { + for &idx in idxs { + let p = purl(NAMES[idx]); + let template = if failing.contains(&idx) { + ResponseTemplate::new(500).set_body_string(format!("detail-boom-{}", NAMES[idx])) + } else { + ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "patches": [{ + "uuid": uuid(idx, AUTH), "purl": p, + "publishedAt": "2024-01-01T00:00:00Z", + "description": format!("details for {}", NAMES[idx]), + "license": "MIT", "tier": "free", + "vulnerabilities": {} + }], + "canAccessPaidPatches": false, + })) + }; + Mock::given(method("GET")) + .and(path(format!( + "/v0/orgs/{ORG}/patches/by-package/{}", + encode_purl(&p) + ))) + .respond_with(template.set_delay(delay(idx))) + .mount(server) + .await; + } +} + +/// Partial detail-fetch failures with reversed latencies: the per-package +/// warnings print in package (purl-sorted) order, and the whole human +/// preview is byte-identical to a zero-latency run. +#[tokio::test] +async fn detail_fetch_warnings_and_preview_keep_package_order() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), &NAMES); + let all: Vec = (0..NAMES.len()).collect(); + let failing = [1usize, 4]; + + let run_with = |delay: fn(usize) -> Duration| { + let all = all.clone(); + let root = tmp.path().to_path_buf(); + async move { + let server = MockServer::start().await; + mount_discovery_with_details(&server, &all, &failing, delay).await; + run_scan(&root, &server.uri(), &server.uri(), &["--dry-run"]) + } + }; + // NAMES is already purl-sorted, so index order is the fold order. + let (code, stdout, stderr) = run_with(|i| Duration::from_millis(50 * (6 - i as u64))).await; + assert_eq!(code, 0, "stdout={stdout} stderr={stderr}"); + let w1 = format!("Warning: could not fetch details for {}: ", purl(NAMES[1])); + let w4 = format!("Warning: could not fetch details for {}: ", purl(NAMES[4])); + let at1 = stderr.find(&w1).unwrap_or_else(|| panic!("{w1}\n{stderr}")); + let at4 = stderr.find(&w4).unwrap_or_else(|| panic!("{w4}\n{stderr}")); + assert!(at1 < at4, "package-order warnings: {stderr}"); + + let (code0, stdout0, stderr0) = run_with(|_| Duration::ZERO).await; + assert_eq!(code0, code); + assert_eq!(stdout0, stdout, "latency must not change the preview"); + assert_eq!(stderr0, stderr, "latency must not change stderr"); +} + +/// Every detail fetch fails (reversed latencies): the one terminal error +/// names the LAST package's failure, as the serial loop's `failures.last()` +/// did. +#[tokio::test] +async fn all_detail_fetches_failed_reports_the_last_packages_error() { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), &NAMES); + let all: Vec = (0..NAMES.len()).collect(); + + let server = MockServer::start().await; + mount_discovery_with_details(&server, &all, &all, |i| { + Duration::from_millis(50 * (6 - i as u64)) + }) + .await; + let (code, stdout, stderr) = run_scan( + tmp.path(), + &server.uri(), + &server.uri(), + &["--json", "--mode", "hosted", "--dry-run"], + ); + assert_eq!(code, 1, "stdout={stdout} stderr={stderr}"); + let v: serde_json::Value = serde_json::from_str(&stdout).unwrap(); + let err = v["error"].as_str().unwrap(); + assert!( + err.starts_with("all 6 patch-detail queries failed: ") + && err.ends_with(&format!("detail-boom-{}", NAMES[5])), + "{err}" + ); +} + +// --------------------------------------------------------------------------- +// Hosted record views (A3) +// --------------------------------------------------------------------------- + +fn hosted_url(idx: usize) -> String { + let name = NAMES[idx]; + format!( + "http://patch.test/patch/npm/{name}/{VERSION}/22222222-2222-4222-8222-222222222222/{}/{name}-{VERSION}.tgz", + uuid(idx, AUTH) + ) +} + +/// Discovery + references + views for a hosted wet run over every package; +/// the `failing` views answer 500. Every by-package / view answer is delayed +/// by `delay(idx)`. +async fn mount_hosted(server: &MockServer, failing: &[usize], delay: fn(usize) -> Duration) { + let all: Vec = (0..NAMES.len()).collect(); + mount_discovery_with_details(server, &all, &[], delay).await; + let mut results = serde_json::Map::new(); + for (idx, name) in NAMES.iter().enumerate() { + results.insert( + uuid(idx, AUTH), + serde_json::json!({ + "status": "granted", + "url": hosted_url(idx), + "purl": purl(name), + "artifacts": [{ + "kind": "tarball", + "url": hosted_url(idx), + "integrity": { "sha512": format!("sha512-PATCHED{idx}==") } + }], + "registryOverride": null + }), + ); + } + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/package"))) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({ "results": results })), + ) + .mount(server) + .await; + for (idx, name) in NAMES.iter().enumerate() { + let u = uuid(idx, AUTH); + let template = if failing.contains(&idx) { + ResponseTemplate::new(500) + } else { + ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "uuid": u, + "purl": purl(name), + "publishedAt": "2024-01-01T00:00:00Z", + "files": { + "package/index.js": { + "beforeHash": format!("{idx:064x}"), + "afterHash": format!("{:064x}", idx + 100), + } + }, + "vulnerabilities": { + format!("GHSA-conc-{idx:04}-aaaa"): { + "cves": [format!("CVE-2024-{idx:04}")], + "summary": "s", "severity": "high", "description": "d" + } + }, + "description": "x", "license": "MIT", "tier": "free" + })) + }; + Mock::given(method("GET")) + .and(path(format!("/v0/orgs/{ORG}/patches/view/{u}"))) + .respond_with(template.set_delay(delay(idx))) + .mount(server) + .await; + } +} + +/// The ledger with its run timestamps blanked, for a byte comparison. +fn ledger_without_timestamps(root: &Path) -> String { + let raw = std::fs::read_to_string(root.join(".socket/vendor/redirect-state.json")).unwrap(); + let mut v: serde_json::Value = serde_json::from_str(&raw).unwrap(); + fn blank(v: &mut serde_json::Value) { + match v { + serde_json::Value::Object(map) => { + for (k, val) in map.iter_mut() { + let key = k.to_ascii_lowercase(); + if key.ends_with("at") && val.is_string() { + *val = serde_json::Value::String("".into()); + } else { + blank(val); + } + } + } + serde_json::Value::Array(items) => items.iter_mut().for_each(blank), + _ => {} + } + } + blank(&mut v); + serde_json::to_string_pretty(&v).unwrap() +} + +/// A wet hosted run where 2 of 6 record views fail, with reversed +/// latencies: the `record_fetch_failed` warnings keep `confirmed` order, +/// and stdout, the rewritten lockfile and the ledger all equal a +/// zero-latency run's. +#[tokio::test] +async fn hosted_record_fetch_failures_keep_order_and_ledger_bytes() { + let failing = [1usize, 3]; + let slow: fn(usize) -> Duration = |i| Duration::from_millis(50 * (6 - i as u64)); + let fast: fn(usize) -> Duration = |_| Duration::ZERO; + + let mut outcomes = Vec::new(); + for delay in [slow, fast] { + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path(), &NAMES); + let server = MockServer::start().await; + mount_hosted(&server, &failing, delay).await; + let (code, stdout, stderr) = run_scan( + tmp.path(), + &server.uri(), + &server.uri(), + &["--json", "--mode", "hosted", "--yes"], + ); + assert_eq!(code, 0, "stdout={stdout} stderr={stderr}"); + let lock = std::fs::read_to_string(tmp.path().join("package-lock.json")).unwrap(); + outcomes.push((stdout, lock, ledger_without_timestamps(tmp.path()))); + } + + let (stdout, lock, ledger) = &outcomes[0]; + let v: serde_json::Value = serde_json::from_str(stdout).unwrap(); + let warnings: Vec<&str> = v["redirect"]["warnings"] + .as_array() + .unwrap() + .iter() + .filter(|w| w["code"] == "record_fetch_failed") + .map(|w| w["detail"].as_str().unwrap()) + .collect(); + assert_eq!(warnings.len(), 2, "{stdout}"); + assert!(warnings[0].starts_with(&format!("{} redirected", purl(NAMES[1])))); + assert!(warnings[1].starts_with(&format!("{} redirected", purl(NAMES[3])))); + for idx in 0..NAMES.len() { + assert!(lock.contains(&hosted_url(idx)), "{lock}"); + let has_record = ledger.contains(&format!("GHSA-conc-{idx:04}-aaaa")); + assert_eq!(has_record, !failing.contains(&idx), "{ledger}"); + } + + assert_eq!(outcomes[0], outcomes[1], "latency must not change the run"); +} diff --git a/crates/socket-patch-cli/tests/scan_vendor_e2e.rs b/crates/socket-patch-cli/tests/scan_vendor_e2e.rs index 91a53a98e..242851a34 100644 --- a/crates/socket-patch-cli/tests/scan_vendor_e2e.rs +++ b/crates/socket-patch-cli/tests/scan_vendor_e2e.rs @@ -2147,3 +2147,862 @@ fn run_manifestless_tail(label: &str, checkout: &Path, pristine: Vec) { .expect("manifest-less VEX tail panicked"); }); } + +// ───────────────────── the download plan is exact ───────────────────── + +mod exact_download_plan { + //! A vendored run fetches prebuilt archives ahead of its serial wiring + //! loop, from a plan of the packages the loop will ask the service + //! for. A download grant (`POST /patches/package`) can start a + //! server-side build and counts against quota, so the plan must be + //! EXACT: a package the loop refuses before it would ask the service + //! — here a pnpm entry the backend cannot rewire — costs no grant at + //! all, while every package the loop does reach costs exactly one. + use super::*; + use wiremock::matchers::path_regex; + + const UUID_A: &str = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + const UUID_B: &str = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; + const UUID_C: &str = "cccccccc-cccc-4ccc-8ccc-cccccccccccc"; + const PACKAGES: [(&str, &str); 3] = [("pkg-a", UUID_A), ("pkg-b", UUID_B), ("pkg-c", UUID_C)]; + + fn purl(name: &str) -> String { + format!("pkg:npm/{name}@1.0.0") + } + + /// A pnpm 9 project with three installed, patched packages. `pkg-b`'s + /// snapshot key carries a peer suffix (`1.0.0(peer-x@1.0.0)`), which + /// the pnpm backend refuses as `vendor_lock_entry_unsupported` before + /// staging anything — the shape behind two of the three speculative + /// grants the plan used to issue on depscan. + fn write_pnpm_fixture(root: &Path) { + std::fs::write( + root.join("package.json"), + r#"{ "name": "plan-test", "version": "0.0.0", "dependencies": { "pkg-a": "1.0.0", "pkg-b": "1.0.0", "pkg-c": "1.0.0" } }"#, + ) + .unwrap(); + std::fs::write( + root.join("pnpm-lock.yaml"), + "lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + pkg-a: + specifier: 1.0.0 + version: 1.0.0 + pkg-b: + specifier: 1.0.0 + version: 1.0.0(peer-x@1.0.0) + pkg-c: + specifier: 1.0.0 + version: 1.0.0 + +packages: + + pkg-a@1.0.0: + resolution: {integrity: sha512-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==} + + pkg-b@1.0.0: + resolution: {integrity: sha512-BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB==} + peerDependencies: + peer-x: '*' + + pkg-c@1.0.0: + resolution: {integrity: sha512-CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC==} + +snapshots: + + pkg-a@1.0.0: {} + + pkg-b@1.0.0(peer-x@1.0.0): {} + + pkg-c@1.0.0: {} +", + ) + .unwrap(); + for (name, _) in PACKAGES { + let pkg = root.join("node_modules").join(name); + std::fs::create_dir_all(&pkg).unwrap(); + std::fs::write( + pkg.join("package.json"), + format!(r#"{{"name":"{name}","version":"1.0.0"}}"#), + ) + .unwrap(); + std::fs::write(pkg.join("index.js"), BEFORE).unwrap(); + } + } + + /// Discovery, per-package search and views for the three patches, and a + /// grant endpoint that answers `not_found` for every uuid (the loop then + /// builds locally — the grant is what this test counts). + async fn mount_three_patch_api(mock: &MockServer) { + let before_hash = git_sha256(BEFORE); + let after_hash = git_sha256(AFTER); + let packages: Vec = PACKAGES + .iter() + .map(|(name, uuid)| { + serde_json::json!({ + "purl": purl(name), + "patches": [{ + "uuid": uuid, "purl": purl(name), "tier": "free", + "cveIds": ["CVE-2026-0001"], "ghsaIds": [], "severity": "high", + "title": "plan target" + }] + }) + }) + .collect(); + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/patches/batch"))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "packages": packages, + "canAccessPaidPatches": false, + }))) + .mount(mock) + .await; + for (name, uuid) in PACKAGES { + let encoded = format!("pkg%3Anpm%2F{name}%401.0.0"); + Mock::given(method("GET")) + .and(path(format!( + "/v0/orgs/{ORG_SLUG}/patches/by-package/{encoded}" + ))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "patches": [{ + "uuid": uuid, "purl": purl(name), + "publishedAt": "2026-01-01T00:00:00Z", + "description": "plan target", "license": "MIT", "tier": "free", + "vulnerabilities": {} + }], + "canAccessPaidPatches": false, + }))) + .mount(mock) + .await; + Mock::given(method("GET")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/patches/view/{uuid}"))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "uuid": uuid, + "purl": purl(name), + "publishedAt": "2026-01-01T00:00:00Z", + "files": { + "package/index.js": { + "beforeHash": before_hash, + "afterHash": after_hash, + "blobContent": AFTER_B64, + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": ["CVE-2026-0001"], "summary": "test vuln", + "severity": "high", "description": "details" + } + }, + "description": "plan target", "license": "MIT", "tier": "free", + }))) + .mount(mock) + .await; + } + let results: serde_json::Map = PACKAGES + .iter() + .map(|(_, uuid)| { + ( + uuid.to_string(), + serde_json::json!({ "status": "not_found", "url": null, "artifacts": [] }), + ) + }) + .collect(); + Mock::given(method("POST")) + .and(path_regex(format!("^/v0/orgs/{ORG_SLUG}/patches/package$"))) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({ "results": results })), + ) + .mount(mock) + .await; + } + + /// Every uuid the run asked a download grant for, in request order + /// (one request may name several). + async fn granted_uuids(mock: &MockServer) -> Vec { + mock.received_requests() + .await + .unwrap() + .iter() + .filter(|r| { + r.method == wiremock::http::Method::POST + && r.url.path().ends_with("/patches/package") + }) + .flat_map(|r| { + let body: serde_json::Value = serde_json::from_slice(&r.body).expect("grant body"); + body["uuids"] + .as_array() + .expect("uuids array") + .iter() + .map(|u| u.as_str().unwrap().to_string()) + .collect::>() + }) + .collect() + } + + /// Composer twins of the three npm packages: `psr/cache` and `psr/log` + /// are installed AND locked; `psr/http-message` is installed but absent + /// from composer.lock, which the composer backend refuses as + /// `vendor_lock_entry_not_found` before it asks the service. It sorts + /// BETWEEN the two, in the middle of the loop (and plan) order: the + /// prefetch only ever requests positions at or past the loop's, so a + /// refused package the loop meets FIRST would be passed over before + /// any request whether the plan named it or not — only one behind a + /// granted position shows whether the gate kept it out of the plan. + const COMPOSER: [(&str, &str, &str); 3] = [ + ("pkg:composer/psr/cache@1.0.0", "psr/cache", UUID_A), + ( + "pkg:composer/psr/http-message@1.1.0", + "psr/http-message", + UUID_B, + ), + ("pkg:composer/psr/log@3.0.2", "psr/log", UUID_C), + ]; + const COMPOSER_REFUSED: &str = "psr/http-message"; + + fn write_composer_fixture(root: &Path) { + std::fs::write(root.join("composer.json"), r#"{"require":{}}"#).unwrap(); + let locked: Vec = COMPOSER + .iter() + .filter(|(_, name, _)| *name != COMPOSER_REFUSED) + .map(|(purl, name, _)| { + let version = purl.rsplit('@').next().unwrap(); + serde_json::json!({ + "name": name, "version": version, + "dist": {"type": "zip", "url": format!("https://example.invalid/{name}.zip"), + "reference": "abc", "shasum": ""}, + "type": "library" + }) + }) + .collect(); + std::fs::write( + root.join("composer.lock"), + serde_json::to_vec_pretty(&serde_json::json!({ + "content-hash": "x", "packages": locked, "packages-dev": [] + })) + .unwrap(), + ) + .unwrap(); + let installed: Vec = COMPOSER + .iter() + .map(|(purl, name, _)| { + serde_json::json!({ + "name": name, "version": purl.rsplit('@').next().unwrap(), + "install-path": format!("../{name}") + }) + }) + .collect(); + std::fs::create_dir_all(root.join("vendor/composer")).unwrap(); + std::fs::write( + root.join("vendor/composer/installed.json"), + serde_json::to_vec(&serde_json::json!({ "packages": installed })).unwrap(), + ) + .unwrap(); + for (_, name, _) in COMPOSER { + let pkg = root.join("vendor").join(name); + std::fs::create_dir_all(&pkg).unwrap(); + std::fs::write(pkg.join("index.js"), BEFORE).unwrap(); + } + } + + /// [`mount_three_patch_api`] for arbitrary `(purl, uuid)` pairs whose + /// patch rewrites `file`. + async fn mount_patch_api(mock: &MockServer, patches: &[(&str, &str)], file: &str) { + let before_hash = git_sha256(BEFORE); + let after_hash = git_sha256(AFTER); + let packages: Vec = patches + .iter() + .map(|(purl, uuid)| { + serde_json::json!({ + "purl": purl, + "patches": [{ + "uuid": uuid, "purl": purl, "tier": "free", + "cveIds": ["CVE-2026-0001"], "ghsaIds": [], "severity": "high", + "title": "plan target" + }] + }) + }) + .collect(); + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/patches/batch"))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "packages": packages, + "canAccessPaidPatches": false, + }))) + .mount(mock) + .await; + for (purl, uuid) in patches { + let encoded = purl + .replace(':', "%3A") + .replace('/', "%2F") + .replace('@', "%40"); + Mock::given(method("GET")) + .and(path(format!( + "/v0/orgs/{ORG_SLUG}/patches/by-package/{encoded}" + ))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "patches": [{ + "uuid": uuid, "purl": purl, + "publishedAt": "2026-01-01T00:00:00Z", + "description": "plan target", "license": "MIT", "tier": "free", + "vulnerabilities": {} + }], + "canAccessPaidPatches": false, + }))) + .mount(mock) + .await; + Mock::given(method("GET")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/patches/view/{uuid}"))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "uuid": uuid, + "purl": purl, + "publishedAt": "2026-01-01T00:00:00Z", + "files": { + file: { + "beforeHash": before_hash, + "afterHash": after_hash, + "blobContent": AFTER_B64, + } + }, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": ["CVE-2026-0001"], "summary": "test vuln", + "severity": "high", "description": "details" + } + }, + "description": "plan target", "license": "MIT", "tier": "free", + }))) + .mount(mock) + .await; + } + let results: serde_json::Map = patches + .iter() + .map(|(_, uuid)| { + ( + uuid.to_string(), + serde_json::json!({ "status": "not_found", "url": null, "artifacts": [] }), + ) + }) + .collect(); + Mock::given(method("POST")) + .and(path_regex(format!("^/v0/orgs/{ORG_SLUG}/patches/package$"))) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({ "results": results })), + ) + .mount(mock) + .await; + } + + /// The plan is exact beyond npm: every ecosystem's backend gate keeps + /// the packages it refuses before its first service call out of the + /// plan. Here the composer backend refuses `psr/http-message` (not in + /// composer.lock, and in the middle of the loop order, behind a + /// package the service answers) — zero grants — while the two locked + /// packages it does ask the service for cost exactly one grant each. + /// (`plan_gate_tests` in `commands/vendor.rs` pins the plan itself.) + #[tokio::test] + async fn a_composer_package_the_loop_refuses_costs_zero_grants() { + assert!( + !socket_patch_core::crawlers::walk_pool::fd_limit_is_tight(), + "the descriptor limit is too tight for the download plan to be built, so this \ + test cannot exercise the pre-flight it pins; raise `ulimit -n` and re-run" + ); + let mock = MockServer::start().await; + let patches: Vec<(&str, &str)> = COMPOSER.iter().map(|(p, _, u)| (*p, *u)).collect(); + mount_patch_api(&mock, &patches, "index.js").await; + let tmp = tempfile::tempdir().unwrap(); + write_composer_fixture(tmp.path()); + + let (_code, stdout, stderr) = run_scan_vendor(tmp.path(), &mock.uri(), &[]); + let v: serde_json::Value = serde_json::from_str(stdout.trim()) + .unwrap_or_else(|e| panic!("valid JSON: {e}\nstdout={stdout}\nstderr={stderr}")); + let events = v["vendor"]["events"].as_array().expect("vendor events"); + let event_for = |purl: &str| { + events + .iter() + .find(|e| e["purl"] == purl && e["action"] != "skipped") + .unwrap_or_else(|| panic!("no vendor event for {purl}: {v}")) + }; + assert_eq!(event_for(COMPOSER[0].0)["action"], "applied", "{v}"); + assert_eq!(event_for(COMPOSER[2].0)["action"], "applied", "{v}"); + let refused = event_for(COMPOSER[1].0); + assert_eq!(refused["action"], "failed", "{v}"); + assert_eq!(refused["errorCode"], "vendor_lock_entry_not_found", "{v}"); + + let mut granted = granted_uuids(&mock).await; + granted.sort(); + assert_eq!( + granted, + vec![UUID_A.to_string(), UUID_C.to_string()], + "exactly one grant per package the loop reaches the service for, and none \ + for the package it refuses first" + ); + } + + /// A package the loop refuses before its first service call costs ZERO + /// download grants: the plan is built from the backend's own pre-flight, + /// so `pkg-b` is never asked for, while `pkg-a` and `pkg-c` — which the + /// loop does ask for — cost exactly one grant each. `pkg-b`'s refusal + /// reads only pnpm-lock.yaml, so the download phase raises it before + /// fetching its view — the backend's code and words on a failed download + /// record — and it costs no request at all. + #[tokio::test] + async fn a_package_the_loop_refuses_costs_zero_grants() { + // The plan is only built when the run may keep more than one + // request in flight, and a tight descriptor limit pins the API + // concurrency at one whatever the environment says (the helper + // already scrubs `SOCKET_API_CONCURRENCY`). The strictly serial + // loop then trivially grants nothing for the refused package, and + // this test would pass without the pre-flight it pins ever + // running — so fail loudly rather than vacuously. + assert!( + !socket_patch_core::crawlers::walk_pool::fd_limit_is_tight(), + "the descriptor limit is too tight for the download plan to be built, so this \ + test cannot exercise the pre-flight it pins; raise `ulimit -n` and re-run" + ); + let mock = MockServer::start().await; + mount_three_patch_api(&mock).await; + let tmp = tempfile::tempdir().unwrap(); + write_pnpm_fixture(tmp.path()); + + let (_code, stdout, stderr) = run_scan_vendor(tmp.path(), &mock.uri(), &[]); + let v: serde_json::Value = serde_json::from_str(stdout.trim()) + .unwrap_or_else(|e| panic!("valid JSON: {e}\nstdout={stdout}\nstderr={stderr}")); + let events = v["vendor"]["events"].as_array().expect("vendor events"); + let event_for = |name: &str| events.iter().find(|e| e["purl"] == purl(name)); + assert_eq!( + event_for("pkg-a").expect("pkg-a event")["action"], + "applied", + "{v}" + ); + assert_eq!( + event_for("pkg-c").expect("pkg-c event")["action"], + "applied", + "{v}" + ); + assert!( + event_for("pkg-b").is_none(), + "refused before the vendor step: {v}" + ); + let refused = v["download"]["patches"] + .as_array() + .and_then(|p| p.iter().find(|r| r["purl"] == purl("pkg-b"))) + .unwrap_or_else(|| panic!("no download record for pkg-b: {v}")); + assert_eq!(refused["action"], "failed", "{v}"); + assert_eq!(refused["errorCode"], "vendor_lock_entry_unsupported", "{v}"); + assert_eq!(v["download"]["failed"], 1, "{v}"); + + let mut granted = granted_uuids(&mock).await; + granted.sort(); + assert_eq!( + granted, + vec![UUID_A.to_string(), UUID_C.to_string()], + "exactly one grant per package the loop reaches the service for, and none \ + for the package it refuses first" + ); + let viewed: Vec = mock + .received_requests() + .await + .unwrap() + .iter() + .filter(|r| r.url.path().contains("/patches/view/")) + .map(|r| r.url.path().rsplit('/').next().unwrap().to_string()) + .collect(); + assert!( + !viewed.contains(&UUID_B.to_string()), + "a package refused on lock text alone costs no view: {viewed:?}" + ); + } + + // ── Which packages the lock-text refusal reaches ──────────────────── + // + // The download phase refuses, before the view, only a package the + // vendor loop would hand to its backend: one installed on disk, or one + // the lockfile resolves to a verifiable registry source. A package with + // neither never reached its backend — the loop skips it + // `package_not_installed` — and keeps that outcome. + + const UUID_Y: &str = "dddddddd-dddd-4ddd-8ddd-dddddddddddd"; + const UUID_Z: &str = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee"; + + /// A pnpm 9 project: `pkg-a` installed and locked (wireable); `pkg-b` + /// locked behind a peer-suffixed snapshot key (refused), NOT installed; + /// `pkg-y` installed but absent from the lock (refused); `pkg-z` + /// neither installed nor locked. + fn write_pnpm_scope_fixture(root: &Path) { + std::fs::write( + root.join("package.json"), + r#"{ "name": "scope-test", "version": "0.0.0", "dependencies": { "pkg-a": "1.0.0", "pkg-b": "1.0.0" } }"#, + ) + .unwrap(); + std::fs::write( + root.join("pnpm-lock.yaml"), + "lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + pkg-a: + specifier: 1.0.0 + version: 1.0.0 + pkg-b: + specifier: 1.0.0 + version: 1.0.0(peer-x@1.0.0) + +packages: + + pkg-a@1.0.0: + resolution: {integrity: sha512-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==} + + pkg-b@1.0.0: + resolution: {integrity: sha512-BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB==} + peerDependencies: + peer-x: '*' + +snapshots: + + pkg-a@1.0.0: {} + + pkg-b@1.0.0(peer-x@1.0.0): {} +", + ) + .unwrap(); + for name in ["pkg-a", "pkg-y"] { + let pkg = root.join("node_modules").join(name); + std::fs::create_dir_all(&pkg).unwrap(); + std::fs::write( + pkg.join("package.json"), + format!(r#"{{"name":"{name}","version":"1.0.0"}}"#), + ) + .unwrap(); + std::fs::write(pkg.join("index.js"), BEFORE).unwrap(); + } + } + + const PNPM_SCOPE: [(&str, &str); 4] = [ + ("pkg:npm/pkg-a@1.0.0", UUID_A), + ("pkg:npm/pkg-b@1.0.0", UUID_B), + ("pkg:npm/pkg-y@1.0.0", UUID_Y), + ("pkg:npm/pkg-z@1.0.0", UUID_Z), + ]; + + /// Every uuid whose view the run fetched. + async fn viewed_uuids(mock: &MockServer) -> Vec { + let mut viewed: Vec = mock + .received_requests() + .await + .unwrap() + .iter() + .filter(|r| r.url.path().contains("/patches/view/")) + .map(|r| r.url.path().rsplit('/').next().unwrap().to_string()) + .collect(); + viewed.sort(); + viewed + } + + /// No request reached the (mock) registry. + async fn assert_no_registry_request(mock: &MockServer) { + let registry: Vec = mock + .received_requests() + .await + .unwrap() + .iter() + .map(|r| r.url.path().to_string()) + .filter(|p| p.starts_with("/registry/")) + .collect(); + assert!(registry.is_empty(), "no pristine fetch: {registry:?}"); + } + + fn record_for<'a>(records: &'a serde_json::Value, purl: &str) -> &'a serde_json::Value { + records + .as_array() + .and_then(|p| p.iter().find(|r| r["purl"] == purl)) + .unwrap_or_else(|| panic!("no record for {purl}: {records}")) + } + + fn events_for<'a>(v: &'a serde_json::Value, purl: &str) -> Vec<(&'a str, &'a str)> { + v["vendor"]["events"] + .as_array() + .expect("vendor events") + .iter() + .filter(|e| e["purl"] == purl) + .map(|e| { + ( + e["action"].as_str().unwrap_or_default(), + e["errorCode"].as_str().unwrap_or_default(), + ) + }) + .collect() + } + + fn run_json(root: &Path, argv: &[&str], env: &[(&str, &str)]) -> serde_json::Value { + let (_code, stdout, stderr) = run_cli_env(root, argv, env); + serde_json::from_str(stdout.trim()) + .unwrap_or_else(|e| panic!("valid JSON: {e}\nstdout={stdout}\nstderr={stderr}")) + } + + fn api_argv<'a>(mock_uri: &'a str, head: &[&'a str]) -> Vec<&'a str> { + let mut argv = head.to_vec(); + argv.extend_from_slice(&[ + "--json", + "--yes", + "--api-url", + mock_uri, + "--api-token", + "fake-token", + "--org", + ORG_SLUG, + ]); + argv + } + + /// `scan --mode vendored` over the pnpm scope fixture: the installed + /// (`pkg-y`) and the lock-resolved (`pkg-b`) refused packages fail in + /// the download phase with no view and no pristine fetch; `pkg-z`, + /// which the lock does not resolve and nothing installed, is fetched + /// and skipped `package_not_installed` by the loop exactly as before. + #[tokio::test] + async fn scan_refuses_early_only_what_reaches_the_pnpm_backend() { + let mock = MockServer::start().await; + mount_patch_api(&mock, &PNPM_SCOPE, "package/index.js").await; + let tmp = tempfile::tempdir().unwrap(); + write_pnpm_scope_fixture(tmp.path()); + let registry = format!("{}/registry", mock.uri()); + let uri = mock.uri(); + let v = run_json( + tmp.path(), + &api_argv(&uri, &["scan", "--mode", "vendored"]), + &[("SOCKET_NPM_REGISTRY", registry.as_str())], + ); + let dl = &v["download"]["patches"]; + let b = record_for(dl, "pkg:npm/pkg-b@1.0.0"); + assert_eq!( + (&b["action"], &b["errorCode"]), + ( + &serde_json::json!("failed"), + &serde_json::json!("vendor_lock_entry_unsupported") + ), + "{v}" + ); + let y = record_for(dl, "pkg:npm/pkg-y@1.0.0"); + assert_eq!( + (&y["action"], &y["errorCode"]), + ( + &serde_json::json!("failed"), + &serde_json::json!("vendor_lock_entry_not_found") + ), + "{v}" + ); + assert_eq!( + record_for(dl, "pkg:npm/pkg-z@1.0.0")["action"], + "downloaded", + "not refused early: {v}" + ); + assert_eq!( + (&v["download"]["downloaded"], &v["download"]["failed"]), + (&serde_json::json!(2), &serde_json::json!(2)), + "{v}" + ); + assert_eq!( + events_for(&v, "pkg:npm/pkg-z@1.0.0"), + vec![("skipped", "package_not_installed")], + "{v}" + ); + assert!(events_for(&v, "pkg:npm/pkg-b@1.0.0").is_empty(), "{v}"); + assert!(events_for(&v, "pkg:npm/pkg-y@1.0.0").is_empty(), "{v}"); + assert_eq!( + events_for(&v, "pkg:npm/pkg-a@1.0.0"), + vec![("applied", "")], + "{v}" + ); + assert_eq!( + viewed_uuids(&mock).await, + vec![UUID_A.to_string(), UUID_Z.to_string()] + ); + assert_no_registry_request(&mock).await; + } + + /// `get --mode vendored` (exact-versioned purls skip the + /// installed-version narrowing): a package neither installed nor locked + /// keeps the loop's `package_not_installed` skip; a lock-resolved one + /// the backend refuses fails before its view. + #[tokio::test] + async fn exact_purl_get_refuses_early_only_what_reaches_the_pnpm_backend() { + let mock = MockServer::start().await; + mount_patch_api(&mock, &PNPM_SCOPE, "package/index.js").await; + let registry = format!("{}/registry", mock.uri()); + let uri = mock.uri(); + + let tmp = tempfile::tempdir().unwrap(); + write_pnpm_scope_fixture(tmp.path()); + let v = run_json( + tmp.path(), + &api_argv(&uri, &["get", "pkg:npm/pkg-z@1.0.0", "--mode", "vendored"]), + &[("SOCKET_NPM_REGISTRY", registry.as_str())], + ); + assert_eq!( + record_for(&v["patches"], "pkg:npm/pkg-z@1.0.0")["action"], + "downloaded", + "{v}" + ); + assert_eq!(v["failed"], 0, "{v}"); + assert_eq!( + events_for(&v, "pkg:npm/pkg-z@1.0.0"), + vec![("skipped", "package_not_installed")], + "{v}" + ); + + let tmp = tempfile::tempdir().unwrap(); + write_pnpm_scope_fixture(tmp.path()); + let v = run_json( + tmp.path(), + &api_argv(&uri, &["get", "pkg:npm/pkg-b@1.0.0", "--mode", "vendored"]), + &[("SOCKET_NPM_REGISTRY", registry.as_str())], + ); + let b = record_for(&v["patches"], "pkg:npm/pkg-b@1.0.0"); + assert_eq!( + (&b["action"], &b["errorCode"]), + ( + &serde_json::json!("failed"), + &serde_json::json!("vendor_lock_entry_unsupported") + ), + "{v}" + ); + assert!(events_for(&v, "pkg:npm/pkg-b@1.0.0").is_empty(), "{v}"); + assert_eq!(viewed_uuids(&mock).await, vec![UUID_Z.to_string()]); + assert_no_registry_request(&mock).await; + } + + const CARGO_SCOPE: [(&str, &str); 2] = [ + ("pkg:cargo/cfg-if@9.9.9", UUID_Y), + ("pkg:cargo/absent-crate@1.0.0", UUID_Z), + ]; + + /// A cargo project locking `cfg-if 1.0.4`, with `cfg-if 9.9.9` in the + /// (private) registry cache — installed at a version the lock does not + /// resolve — and `absent-crate` nowhere. Returns the `CARGO_HOME`. + fn write_cargo_scope_fixture(tmp: &Path) -> (PathBuf, String) { + let root = tmp.join("proj"); + let cargo_home = tmp.join("cargo-home"); + let krate = cargo_home.join("registry/src/index.crates.io-6f17d22bba15001f/cfg-if-9.9.9"); + std::fs::create_dir_all(krate.join("src")).unwrap(); + std::fs::write(krate.join("src/lib.rs"), BEFORE).unwrap(); + std::fs::write( + krate.join("Cargo.toml"), + "[package]\nname = \"cfg-if\"\nversion = \"9.9.9\"\n", + ) + .unwrap(); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write( + root.join("Cargo.toml"), + "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[dependencies]\ncfg-if = \"1\"\n", + ) + .unwrap(); + std::fs::write( + root.join("Cargo.lock"), + format!( + "version = 4\n\n\ + [[package]]\nname = \"app\"\nversion = \"0.1.0\"\n\ + dependencies = [\n \"cfg-if\",\n]\n\n\ + [[package]]\nname = \"cfg-if\"\nversion = \"1.0.4\"\n\ + source = \"registry+https://github.com/rust-lang/crates.io-index\"\n\ + checksum = \"{}\"\n", + "9".repeat(64) + ), + ) + .unwrap(); + (root, cargo_home.to_string_lossy().into_owned()) + } + + /// cargo's `locked_version_mismatch` is refused before the view only + /// for a crate installed at the unlocked version (the loop hands it to + /// the backend); a crate the lock does not resolve and nothing + /// installed keeps the loop's `package_not_installed` skip — on + /// `scan --mode vendored` and on exact-purl `get --mode vendored`. + #[tokio::test] + async fn cargo_refuses_early_only_an_installed_crate() { + let mock = MockServer::start().await; + mount_patch_api(&mock, &CARGO_SCOPE, "package/src/lib.rs").await; + let registry = format!("{}/registry", mock.uri()); + let uri = mock.uri(); + let env_for = |home: &str| { + vec![ + ("CARGO_HOME".to_string(), home.to_string()), + ("SOCKET_CRATES_REGISTRY".to_string(), registry.clone()), + ] + }; + + let tmp = tempfile::tempdir().unwrap(); + let (root, home) = write_cargo_scope_fixture(tmp.path()); + let env = env_for(&home); + let env: Vec<(&str, &str)> = env.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect(); + let v = run_json( + &root, + &api_argv(&uri, &["scan", "--mode", "vendored"]), + &env, + ); + let dl = &v["download"]["patches"]; + let installed = record_for(dl, CARGO_SCOPE[0].0); + assert_eq!( + (&installed["action"], &installed["errorCode"]), + ( + &serde_json::json!("failed"), + &serde_json::json!("locked_version_mismatch") + ), + "{v}" + ); + assert!(events_for(&v, CARGO_SCOPE[0].0).is_empty(), "{v}"); + assert_eq!(record_for(dl, CARGO_SCOPE[1].0)["action"], "downloaded", "{v}"); + assert_eq!( + events_for(&v, CARGO_SCOPE[1].0), + vec![("skipped", "package_not_installed")], + "{v}" + ); + assert_eq!(viewed_uuids(&mock).await, vec![UUID_Z.to_string()]); + + let tmp = tempfile::tempdir().unwrap(); + let (root, home) = write_cargo_scope_fixture(tmp.path()); + let env = env_for(&home); + let env: Vec<(&str, &str)> = env.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect(); + let v = run_json( + &root, + &api_argv(&uri, &["get", CARGO_SCOPE[1].0, "--mode", "vendored"]), + &env, + ); + assert_eq!( + record_for(&v["patches"], CARGO_SCOPE[1].0)["action"], + "downloaded", + "{v}" + ); + assert_eq!( + events_for(&v, CARGO_SCOPE[1].0), + vec![("skipped", "package_not_installed")], + "{v}" + ); + let v = run_json( + &root, + &api_argv(&uri, &["get", CARGO_SCOPE[0].0, "--mode", "vendored"]), + &env, + ); + let installed = record_for(&v["patches"], CARGO_SCOPE[0].0); + assert_eq!(installed["errorCode"], "locked_version_mismatch", "{v}"); + assert_eq!( + viewed_uuids(&mock).await, + vec![UUID_Z.to_string(), UUID_Z.to_string()], + "the installed crate's view is never fetched" + ); + assert_no_registry_request(&mock).await; + } +} diff --git a/crates/socket-patch-cli/tests/telemetry_e2e.rs b/crates/socket-patch-cli/tests/telemetry_e2e.rs index 05df12bad..cbfdb3af3 100644 --- a/crates/socket-patch-cli/tests/telemetry_e2e.rs +++ b/crates/socket-patch-cli/tests/telemetry_e2e.rs @@ -46,13 +46,55 @@ fn run_cmd( extra_args: &[&str], extra_env: &[(&str, &str)], ) -> (i32, String, String) { + let out = build_cmd(cwd, api_url, subcommand, extra_args, extra_env) + .output() + .expect("run socket-patch"); + ( + out.status.code().unwrap_or(-1), + String::from_utf8_lossy(&out.stdout).to_string(), + String::from_utf8_lossy(&out.stderr).to_string(), + ) +} + +/// The [`run_cmd`] invocation, unstarted (for tests that wire its stdio). +fn build_cmd( + cwd: &Path, + api_url: &str, + subcommand: &str, + extra_args: &[&str], + extra_env: &[(&str, &str)], +) -> Command { + build_cmd_with_token( + "fake-token-for-test", + cwd, + api_url, + subcommand, + extra_args, + extra_env, + ) +} + +/// A `sktsec_<44 chars>_api`-shaped token: the client's token-shape check +/// stays quiet, so a run's stderr carries only what the command under test +/// writes (the default fake token draws a warning before anything runs). +const WELL_SHAPED_TOKEN: &str = "sktsec_00000000000000000000000000000000000000000000_api"; + +/// [`build_cmd`] with an explicit `--api-token`. +fn build_cmd_with_token( + api_token: &str, + cwd: &Path, + api_url: &str, + subcommand: &str, + extra_args: &[&str], + extra_env: &[(&str, &str)], +) -> Command { let mut args = vec![ subcommand, "--json", "--api-url", api_url, "--api-token", - "fake-token-for-test", + api_token, "--org", ORG_SLUG, ]; @@ -109,12 +151,7 @@ fn run_cmd( for (k, v) in extra_env { cmd.env(k, v); } - let out = cmd.output().expect("run socket-patch"); - ( - out.status.code().unwrap_or(-1), - String::from_utf8_lossy(&out.stdout).to_string(), - String::from_utf8_lossy(&out.stderr).to_string(), - ) + cmd } /// Count POSTs the wiremock server received against the telemetry @@ -728,3 +765,309 @@ async fn list_skips_telemetry_in_airgap_mode() { let count = telemetry_post_count(&mock, None).await; assert_eq!(count, 0, "SOCKET_OFFLINE=1 must suppress patch_listed"); } + +// --------------------------------------------------------------------------- +// scan: background sends are flushed before exit +// --------------------------------------------------------------------------- + +/// Scan sends its telemetry off the critical path but must flush it before +/// the process exits. Against a telemetry endpoint that answers only after +/// `TELEMETRY_DELAY`, each scan terminal — success, empty crawl, and +/// all-batches-failed — must still deliver exactly its one event AND stay +/// alive until the response arrives (the lower bound on wall time cannot +/// flake under load: load only makes a run slower). +#[tokio::test] +async fn scan_flushes_background_telemetry_before_exit() { + const TELEMETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(1500); + + struct Case { + label: &'static str, + batch_status: u16, + install_package: bool, + want_event: &'static str, + want_code: i32, + } + let cases = [ + Case { + label: "success", + batch_status: 200, + install_package: true, + want_event: "patch_scanned", + want_code: 0, + }, + Case { + label: "empty crawl", + batch_status: 200, + install_package: false, + want_event: "patch_scanned", + want_code: 0, + }, + Case { + label: "all batches failed", + batch_status: 500, + install_package: true, + want_event: "patch_scan_failed", + want_code: 1, + }, + ]; + + for case in cases { + let mock = MockServer::start().await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/patches/batch"))) + .respond_with(ResponseTemplate::new(case.batch_status).set_body_json( + serde_json::json!({ "packages": [], "canAccessPaidPatches": false }), + )) + .mount(&mock) + .await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/telemetry"))) + .respond_with(ResponseTemplate::new(201).set_delay(TELEMETRY_DELAY)) + .mount(&mock) + .await; + + let tmp = tempfile::tempdir().expect("tempdir"); + write_root_package_json(tmp.path()); + if case.install_package { + write_npm_package(tmp.path(), "minimist", "1.2.2"); + } + + let started = std::time::Instant::now(); + let (code, stdout, stderr) = run_cmd(tmp.path(), &mock.uri(), "scan", &[], &[]); + let elapsed = started.elapsed(); + assert_eq!( + code, case.want_code, + "{}: stdout={stdout} stderr={stderr}", + case.label + ); + assert_eq!( + telemetry_post_count(&mock, Some(case.want_event)).await, + 1, + "{}: exactly one {} event must be delivered", + case.label, + case.want_event + ); + assert_eq!( + telemetry_post_count(&mock, None).await, + 1, + "{}: no other telemetry event", + case.label + ); + assert!( + elapsed >= TELEMETRY_DELAY, + "{}: scan exited after {elapsed:?}, before its telemetry send completed", + case.label + ); + } +} + +/// A consumer that exits early (`scan | head`, `scan | true`) closes +/// stdout, and the CLI dies of SIGPIPE on its first result write (main +/// restores SIG_DFL). The background send must already be delivered by +/// then — flushed before that write, as the inline send it replaced was — +/// not lost with the process. Covers each JSON flush point: the empty-crawl +/// and all-batches-failed terminals, the plain envelope, and the hosted and +/// vendored arms (flushed at `discover_selected`). The first three print +/// right after the event fires, so they fail deterministically without the +/// flush; the hosted/vendored arms do enough work before printing that an +/// unflushed send usually wins the race anyway, so for them this is a +/// delivery check rather than a pin on the flush point. +#[tokio::test] +async fn scan_delivers_telemetry_before_writing_to_a_closed_stdout() { + const TELEMETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(800); + + struct Case { + label: &'static str, + batch_status: u16, + install_package: bool, + extra_args: &'static [&'static str], + want_event: &'static str, + } + let cases = [ + Case { + label: "plain envelope", + batch_status: 200, + install_package: true, + extra_args: &[], + want_event: "patch_scanned", + }, + Case { + label: "empty crawl", + batch_status: 200, + install_package: false, + extra_args: &[], + want_event: "patch_scanned", + }, + Case { + label: "all batches failed", + batch_status: 500, + install_package: true, + extra_args: &[], + want_event: "patch_scan_failed", + }, + Case { + label: "hosted", + batch_status: 200, + install_package: true, + extra_args: &["--mode", "hosted", "--dry-run"], + want_event: "patch_scanned", + }, + Case { + label: "vendored", + batch_status: 200, + install_package: true, + extra_args: &["--mode", "vendored", "--dry-run"], + want_event: "patch_scanned", + }, + ]; + + for case in cases { + let mock = MockServer::start().await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/patches/batch"))) + .respond_with(ResponseTemplate::new(case.batch_status).set_body_json( + serde_json::json!({ "packages": [], "canAccessPaidPatches": false }), + )) + .mount(&mock) + .await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/telemetry"))) + .respond_with(ResponseTemplate::new(201).set_delay(TELEMETRY_DELAY)) + .mount(&mock) + .await; + + let tmp = tempfile::tempdir().expect("tempdir"); + write_root_package_json(tmp.path()); + if case.install_package { + write_npm_package(tmp.path(), "minimist", "1.2.2"); + } + + let mut child = build_cmd(tmp.path(), &mock.uri(), "scan", case.extra_args, &[]) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("spawn socket-patch"); + // Close the read end before the child can write anything: its + // first stdout write now raises SIGPIPE. + drop(child.stdout.take()); + let status = child.wait().expect("wait socket-patch"); + + assert_eq!( + telemetry_post_count(&mock, Some(case.want_event)).await, + 1, + "{}: the {} event must be delivered before stdout is written \ + (exit status {status:?})", + case.label, + case.want_event + ); + assert_eq!( + telemetry_post_count(&mock, None).await, + 1, + "{}: no other telemetry event", + case.label + ); + } +} + +/// The stderr twin of the closed-stdout test above: a malformed hosted +/// redirect ledger makes a non-hosted scan warn on stderr (the lenient +/// read-only consult) right after the scan event fires, BEFORE any stdout +/// write — so with stderr closed that warning is the run's first +/// SIGPIPE-raising write, and the background send must be flushed ahead of +/// it, as the inline send it replaced always was. The plain envelope does +/// no network work between the event and the warning, so without the flush +/// the delayed send deterministically loses the race; the vendored arm is +/// covered too (its warning also precedes `discover_selected`'s flush). +#[tokio::test] +async fn scan_delivers_telemetry_before_writing_to_a_closed_stderr() { + const TELEMETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(800); + + let cases: [(&str, &[&str]); 2] = [ + ("plain envelope", &[]), + ("vendored", &["--mode", "vendored", "--dry-run"]), + ]; + for (label, extra_args) in cases { + let mock = MockServer::start().await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/patches/batch"))) + .respond_with(ResponseTemplate::new(200).set_body_json( + serde_json::json!({ "packages": [], "canAccessPaidPatches": false }), + )) + .mount(&mock) + .await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/telemetry"))) + .respond_with(ResponseTemplate::new(201).set_delay(TELEMETRY_DELAY)) + .mount(&mock) + .await; + + let tmp = tempfile::tempdir().expect("tempdir"); + write_root_package_json(tmp.path()); + write_npm_package(tmp.path(), "minimist", "1.2.2"); + let ledger_dir = tmp.path().join(".socket").join("vendor"); + std::fs::create_dir_all(&ledger_dir).expect("mkdir .socket/vendor"); + std::fs::write(ledger_dir.join("redirect-state.json"), "{ not json") + .expect("write malformed redirect ledger"); + + // Sanity: with stderr open the run does warn about the ledger, so + // the closed-stderr run below really has a write to die on. + let out = build_cmd_with_token( + WELL_SHAPED_TOKEN, + tmp.path(), + &mock.uri(), + "scan", + extra_args, + &[], + ) + .output() + .expect("run socket-patch"); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.starts_with("Warning: ") && stderr.contains("redirect-state.json"), + "{label}: the malformed redirect ledger's warning must be the \ + run's first stderr write; got: {stderr}" + ); + mock.reset().await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/patches/batch"))) + .respond_with(ResponseTemplate::new(200).set_body_json( + serde_json::json!({ "packages": [], "canAccessPaidPatches": false }), + )) + .mount(&mock) + .await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/telemetry"))) + .respond_with(ResponseTemplate::new(201).set_delay(TELEMETRY_DELAY)) + .mount(&mock) + .await; + + let mut child = build_cmd_with_token( + WELL_SHAPED_TOKEN, + tmp.path(), + &mock.uri(), + "scan", + extra_args, + &[], + ) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn socket-patch"); + // Close the read end before the child can write anything: its + // first stderr write now raises SIGPIPE. + drop(child.stderr.take()); + let status = child.wait().expect("wait socket-patch"); + + assert_eq!( + telemetry_post_count(&mock, Some("patch_scanned")).await, + 1, + "{label}: the patch_scanned event must be delivered before stderr \ + is written (exit status {status:?})" + ); + assert_eq!( + telemetry_post_count(&mock, None).await, + 1, + "{label}: no other telemetry event" + ); + } +} diff --git a/crates/socket-patch-cli/tests/vendor_crash_safety_e2e.rs b/crates/socket-patch-cli/tests/vendor_crash_safety_e2e.rs new file mode 100644 index 000000000..5bb643a06 --- /dev/null +++ b/crates/socket-patch-cli/tests/vendor_crash_safety_e2e.rs @@ -0,0 +1,344 @@ +//! Crash safety of a vendored run's writes. +//! +//! Vendored artifacts (the packed `.tgz`, the patched copy trees, the +//! marker) are written WITHOUT an fsync of their own; one durability barrier +//! syncs them before the first durable commit point (lockfile, ledger) that +//! could name them — see `socket_patch_core::utils::durability`. These tests +//! crash the real binary at that barrier through its debug-build failpoint +//! (`SOCKET_PATCH_FAILPOINT`, compiled out of release builds) and then play +//! the part of the power loss the barrier guards against by destroying the +//! un-synced artifact bytes, and prove the next run re-verifies the +//! artifact, rebuilds it and wires the project exactly as an uninterrupted +//! run would have. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; + +const UUID: &str = "9f6b2c4e-1d3a-4f6b-8c2d-7e5a9b1c3d5f"; +const PURL: &str = "pkg:npm/left-pad@1.3.0"; +const ORIG_INDEX: &[u8] = b"module.exports = () => 'orig';\n"; +const PATCHED_INDEX: &[u8] = b"module.exports = () => 'patched';\n"; + +fn git_sha256(content: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(format!("blob {}\0", content.len()).as_bytes()); + hasher.update(content); + hex::encode(hasher.finalize()) +} + +fn rel_tgz() -> String { + format!(".socket/vendor/npm/{UUID}/left-pad-1.3.0.tgz") +} + +/// An installed npm project (package-lock v3) with a manifest patch for +/// left-pad and its after-blob, so `vendor --offline` needs no network. +fn npm_project() -> tempfile::TempDir { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + let pkg = root.join("node_modules/left-pad"); + std::fs::create_dir_all(&pkg).unwrap(); + std::fs::write( + pkg.join("package.json"), + br#"{"name":"left-pad","version":"1.3.0"}"#, + ) + .unwrap(); + std::fs::write(pkg.join("index.js"), ORIG_INDEX).unwrap(); + std::fs::write( + root.join("package.json"), + br#"{"name":"fixture","version":"1.0.0","private":true}"#, + ) + .unwrap(); + let lock = json!({ + "name": "fixture", "version": "1.0.0", "lockfileVersion": 3, "requires": true, + "packages": { + "": { "name": "fixture", "version": "1.0.0", + "dependencies": { "left-pad": "^1.3.0" } }, + "node_modules/left-pad": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz", + "integrity": "sha512-orig==", + "license": "WTFPL" + } + } + }); + let mut lock_bytes = serde_json::to_vec_pretty(&lock).unwrap(); + lock_bytes.push(b'\n'); + std::fs::write(root.join("package-lock.json"), lock_bytes).unwrap(); + let manifest = json!({ "patches": { PURL: { + "uuid": UUID, "exportedAt": "2026-01-01T00:00:00Z", + "files": { "package/index.js": { + "beforeHash": git_sha256(ORIG_INDEX), "afterHash": git_sha256(PATCHED_INDEX) } }, + "vulnerabilities": {}, "description": "synthetic", "license": "MIT", "tier": "free" + }}}); + let socket = root.join(".socket"); + std::fs::create_dir_all(socket.join("blobs")).unwrap(); + std::fs::write( + socket.join("manifest.json"), + serde_json::to_vec_pretty(&manifest).unwrap(), + ) + .unwrap(); + std::fs::write( + socket.join("blobs").join(git_sha256(PATCHED_INDEX)), + PATCHED_INDEX, + ) + .unwrap(); + tmp +} + +fn binary() -> PathBuf { + env!("CARGO_BIN_EXE_socket-patch").into() +} + +/// `vendor --json --offline` through the built binary, optionally crashing +/// at `failpoint`. Returns the exit code and stdout. +fn vendor(root: &Path, failpoint: Option<&str>) -> (i32, String) { + let mut cmd = Command::new(binary()); + cmd.args(["vendor", "--json", "--offline"]) + .current_dir(root); + for (key, _) in std::env::vars() { + if key.starts_with("SOCKET_") && key != "SOCKET_NO_CONFIG" { + cmd.env_remove(key); + } + } + cmd.env("SOCKET_TELEMETRY_DISABLED", "1"); + if let Some(point) = failpoint { + cmd.env("SOCKET_PATCH_FAILPOINT", point); + } + let out = cmd.output().expect("run socket-patch vendor"); + ( + out.status.code().unwrap_or(-1), + String::from_utf8_lossy(&out.stdout).into_owned(), + ) +} + +/// Every regular file under `root` except the transient lock (relative +/// path → bytes), sorted. +fn tree(root: &Path) -> Vec<(String, Vec)> { + let mut out = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + for entry in std::fs::read_dir(&dir).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if entry.file_type().unwrap().is_dir() { + stack.push(path); + } else { + let rel = path + .strip_prefix(root) + .unwrap() + .to_string_lossy() + .replace('\\', "/"); + if rel != ".socket/apply.lock" { + out.push((rel, std::fs::read(&path).unwrap())); + } + } + } + } + out.sort(); + out +} + +/// The ledger with its timestamps masked, for comparing two runs. +fn masked_state(root: &Path) -> Value { + let mut v: Value = + serde_json::from_slice(&std::fs::read(root.join(".socket/vendor/state.json")).unwrap()) + .unwrap(); + fn mask(v: &mut Value) { + match v { + Value::Object(map) => { + for (k, val) in map.iter_mut() { + if k == "vendoredAt" || k == "exportedAt" { + *val = Value::Null; + } else { + mask(val); + } + } + } + Value::Array(items) => items.iter_mut().for_each(mask), + _ => {} + } + } + mask(&mut v); + v +} + +/// A crash at the barrier — artifacts written, nothing durable yet — leaves +/// the commit points exactly as they were: the lock still resolves the +/// registry and no ledger exists. Losing the un-synced tarball bytes +/// on top of that (what a power loss may do) costs nothing: the next run +/// re-verifies what it finds in the orphaned uuid dir, rebuilds it, and +/// ends in the same state an uninterrupted run reaches. +#[test] +fn crash_before_the_barrier_is_repaired_by_the_next_run() { + let clean = npm_project(); + let (code, stdout) = vendor(clean.path(), None); + assert_eq!(code, 0, "{stdout}"); + + let tmp = npm_project(); + let root = tmp.path(); + let lock_before = std::fs::read(root.join("package-lock.json")).unwrap(); + let (code, stdout) = vendor(root, Some("durability_barrier")); + assert_eq!(code, 86, "the failpoint must crash the run: {stdout}"); + assert!( + root.join(rel_tgz()).is_file(), + "the artifact was written before the crash" + ); + assert_eq!( + std::fs::read(root.join("package-lock.json")).unwrap(), + lock_before, + "no commit point is written ahead of the barrier" + ); + assert!(!root.join(".socket/vendor/state.json").exists()); + + // The power loss the barrier exists for: the un-synced artifact and + // marker come back empty. + std::fs::write(root.join(rel_tgz()), b"").unwrap(); + std::fs::write( + root.join(format!( + ".socket/vendor/npm/{UUID}/socket-patch.vendor.json" + )), + b"", + ) + .unwrap(); + + let (code, stdout) = vendor(root, None); + assert_eq!(code, 0, "the next run repairs: {stdout}"); + let v: Value = serde_json::from_str(&stdout).unwrap(); + assert!( + v["events"] + .as_array() + .unwrap() + .iter() + .any(|e| e["purl"] == PURL && e["action"] == "applied"), + "{v:#}" + ); + let tgz = std::fs::read(root.join(rel_tgz())).unwrap(); + let state = masked_state(root); + assert_eq!( + state["entries"][PURL]["artifact"]["sha256"], + hex::encode(Sha256::digest(&tgz)), + "the rebuilt artifact is the one the ledger records" + ); + assert_eq!( + state, + masked_state(clean.path()), + "same ledger as an uninterrupted run" + ); + let strip = |t: Vec<(String, Vec)>| -> Vec<(String, Vec)> { + t.into_iter() + .filter(|(rel, _)| { + rel != ".socket/vendor/state.json" && !rel.ends_with("socket-patch.vendor.json") + }) + .collect() + }; + assert_eq!( + strip(tree(root)), + strip(tree(clean.path())), + "same artifact and wiring as an uninterrupted run" + ); +} + +/// The same for a copy-dir artifact: the cargo backend patches a staged +/// copy (written without an fsync) and swaps it into place, then the +/// crash hits the barrier ahead of the `Cargo.toml` / `Cargo.lock` wiring +/// commit. A patched file +/// lost to the power loss is caught by the next run's afterHash check on +/// the copy, which is rebuilt and wired. +#[test] +fn crash_before_the_barrier_repairs_a_copy_dir_artifact() { + const CARGO_UUID: &str = "2b1f6c1e-8d3a-4f6b-9c2d-7e5a9b1c3d11"; + const CARGO_PURL: &str = "pkg:cargo/cfg-if@1.0.4"; + const PRISTINE: &[u8] = b"pub fn cfg() {}\n"; + const PATCHED: &[u8] = b"pub fn cfg() { /* patched */ }\n"; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + let cargo_home = tmp.path().join("cargo-home"); + let krate = cargo_home.join("registry/src/index.crates.io-6f17d22bba15001f/cfg-if-1.0.4"); + std::fs::create_dir_all(krate.join("src")).unwrap(); + std::fs::write(krate.join("src/lib.rs"), PRISTINE).unwrap(); + std::fs::write( + krate.join("Cargo.toml"), + "[package]\nname = \"cfg-if\"\nversion = \"1.0.4\"\n", + ) + .unwrap(); + std::fs::create_dir_all(root.join(".socket/blobs")).unwrap(); + let manifest_toml = + "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[dependencies]\ncfg-if = \"1\"\n"; + std::fs::write(root.join("Cargo.toml"), manifest_toml).unwrap(); + let lock = format!( + "version = 4\n\n[[package]]\nname = \"app\"\nversion = \"0.1.0\"\n\ + dependencies = [\n \"cfg-if\",\n]\n\n[[package]]\nname = \"cfg-if\"\n\ + version = \"1.0.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\n\ + checksum = \"{}\"\n", + "9".repeat(64) + ); + std::fs::write(root.join("Cargo.lock"), &lock).unwrap(); + let manifest = json!({ "patches": { CARGO_PURL: { + "uuid": CARGO_UUID, "exportedAt": "2026-01-01T00:00:00Z", + "files": { "package/src/lib.rs": { + "beforeHash": git_sha256(PRISTINE), "afterHash": git_sha256(PATCHED) } }, + "vulnerabilities": {}, "description": "synthetic", "license": "MIT", "tier": "free" + }}}); + std::fs::write( + root.join(".socket/manifest.json"), + serde_json::to_vec_pretty(&manifest).unwrap(), + ) + .unwrap(); + std::fs::write( + root.join(".socket/blobs").join(git_sha256(PATCHED)), + PATCHED, + ) + .unwrap(); + + let run = |failpoint: Option<&str>| { + let mut cmd = Command::new(binary()); + cmd.args(["vendor", "--json", "--offline"]) + .current_dir(&root) + .env("CARGO_HOME", &cargo_home) + .env("SOCKET_TELEMETRY_DISABLED", "1"); + for (key, _) in std::env::vars() { + if key.starts_with("SOCKET_") && key != "SOCKET_NO_CONFIG" { + cmd.env_remove(key); + } + } + if let Some(point) = failpoint { + cmd.env("SOCKET_PATCH_FAILPOINT", point); + } + let out = cmd.output().unwrap(); + ( + out.status.code().unwrap_or(-1), + String::from_utf8_lossy(&out.stdout).into_owned(), + ) + }; + + let (code, stdout) = run(Some("durability_barrier")); + assert_eq!(code, 86, "{stdout}"); + let copy_lib = root.join(format!( + ".socket/vendor/cargo/{CARGO_UUID}/cfg-if-1.0.4/src/lib.rs" + )); + assert_eq!(std::fs::read(©_lib).unwrap(), PATCHED); + assert!(!root.join(".cargo/config.toml").exists()); + assert_eq!( + std::fs::read_to_string(root.join("Cargo.toml")).unwrap(), + manifest_toml, + "no wiring committed before the barrier" + ); + assert_eq!( + std::fs::read_to_string(root.join("Cargo.lock")).unwrap(), + lock + ); + // The un-synced patched file comes back empty. + std::fs::write(©_lib, b"").unwrap(); + + let (code, stdout) = run(None); + assert_eq!(code, 0, "{stdout}"); + assert_eq!(std::fs::read(©_lib).unwrap(), PATCHED, "rebuilt"); + let wired = std::fs::read_to_string(root.join("Cargo.toml")).unwrap(); + assert!(wired.contains(CARGO_UUID), "{wired}"); + let (code, stdout) = run(None); + assert_eq!(code, 0, "{stdout}"); + assert!(stdout.contains("already_vendored"), "{stdout}"); +} diff --git a/crates/socket-patch-cli/tests/vendor_ecosystem_fixtures/mod.rs b/crates/socket-patch-cli/tests/vendor_ecosystem_fixtures/mod.rs new file mode 100644 index 000000000..c47a8cc5f --- /dev/null +++ b/crates/socket-patch-cli/tests/vendor_ecosystem_fixtures/mod.rs @@ -0,0 +1,824 @@ +//! Hermetic, vendorable two-package projects for every ecosystem the vendor +//! engine wires: installed (pristine) copies in a per-test store, the +//! project's own lockfiles / manifests, and a `.socket/manifest.json` plus +//! after-blobs so `vendor --offline` needs no network. Shared by the +//! group-commit equivalence and crash tests and the ledger-schema tests. +//! +//! Two packages per project on purpose: a run then commits the same +//! lockfile / config twice, which is what the group commit collapses and +//! what makes whole-file ledger snapshots chain. +#![allow(dead_code)] + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use sha2::{Digest, Sha256}; + +pub fn git_sha256(content: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(format!("blob {}\0", content.len()).as_bytes()); + hasher.update(content); + hex::encode(hasher.finalize()) +} + +/// Every ecosystem fixture, by name. +pub const ALL: &[&str] = &[ + "npm", + "pnpm", + "cargo", + "golang", + "pypi-requirements", + "pylock", + "gem", + "composer", + "maven", + "nuget", +]; + +/// One patch of a fixture: the purl, its uuid, and the one file it patches +/// (manifest key, pristine bytes, patched bytes). +pub struct Patch { + pub purl: String, + pub uuid: String, + pub file: String, + pub before: Vec, + pub after: Vec, +} + +fn patch(purl: &str, uuid: &str, file: &str, before: &[u8], after: &[u8]) -> Patch { + Patch { + purl: purl.to_string(), + uuid: uuid.to_string(), + file: file.to_string(), + before: before.to_vec(), + after: after.to_vec(), + } +} + +/// A built fixture: the project root, the store holding the installed +/// copies, the environment the binary needs to find them, and the patches. +pub struct Fixture { + _tmp: tempfile::TempDir, + pub root: PathBuf, + pub store: PathBuf, + pub env: Vec<(String, String)>, + pub patches: Vec, +} + +impl Fixture { + /// Build ecosystem `name` (one of [`ALL`]). + pub fn new(name: &str) -> Fixture { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + let store = tmp.path().join("store"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::create_dir_all(&store).unwrap(); + let (env, patches) = match name { + "npm" => npm(&root), + "pnpm" => pnpm(&root), + "cargo" => cargo(&root, &store), + "golang" => golang(&root, &store), + "pypi-requirements" => pypi_requirements(&root), + "pylock" => pylock(&root), + "gem" => gem(&root), + "composer" => composer(&root), + "maven" => maven(&root, &store), + "nuget" => nuget(&root, &store), + other => panic!("no fixture named {other}"), + }; + write_manifest(&root, &patches); + Fixture { + _tmp: tmp, + root, + store, + env, + patches, + } + } + + /// Run the built binary in the project with the fixture's environment, + /// every ambient `SOCKET_*` scrubbed and `extra_env` on top. Returns + /// the exit code, stdout and stderr. + pub fn run(&self, args: &[&str], extra_env: &[(&str, &str)]) -> (i32, String, String) { + self.run_bin(env!("CARGO_BIN_EXE_socket-patch"), args, extra_env) + } + + /// [`Self::run`] with another build of the binary (the legacy-fixture + /// generator runs the integrated base through it). + pub fn run_bin( + &self, + bin: &str, + args: &[&str], + extra_env: &[(&str, &str)], + ) -> (i32, String, String) { + let mut cmd = Command::new(bin); + cmd.args(args).current_dir(&self.root); + for (key, _) in std::env::vars() { + if key.starts_with("SOCKET_") && key != "SOCKET_NO_CONFIG" { + cmd.env_remove(key); + } + } + for var in ["VIRTUAL_ENV", "CONDA_PREFIX", "BUNDLE_PATH", "GEM_HOME"] { + cmd.env_remove(var); + } + cmd.env("SOCKET_TELEMETRY_DISABLED", "1") + .env("HOME", self.store.join("home")) + .env("CARGO_HOME", self.store.join("cargo-home")) + .env("GOMODCACHE", self.store.join("modcache")) + .env("MAVEN_REPO_LOCAL", self.store.join("m2")) + .env("NUGET_PACKAGES", self.store.join("nuget")); + for (k, v) in &self.env { + cmd.env(k, v); + } + for (k, v) in extra_env { + cmd.env(k, v); + } + let out = cmd.output().expect("run socket-patch"); + ( + out.status.code().unwrap_or(-1), + String::from_utf8_lossy(&out.stdout).into_owned(), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) + } + + /// `vendor --json --offline [extra]`. + pub fn vendor(&self, extra: &[&str], extra_env: &[(&str, &str)]) -> (i32, String, String) { + let mut args = vec!["vendor", "--json", "--offline"]; + args.extend_from_slice(extra); + self.run(&args, extra_env) + } + + /// Every regular file of the project except the transient lock file and + /// the fixture's inputs under `.socket/` (manifest, blobs), relative + /// path → bytes, sorted. + pub fn tree(&self) -> Vec<(String, Vec)> { + tree(&self.root) + } +} + +/// Every regular file under `root` except `.socket/apply.lock`, +/// `.socket/manifest.json` and `.socket/blobs/`, sorted. +pub fn tree(root: &Path) -> Vec<(String, Vec)> { + let mut out = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&dir) else { + continue; + }; + for entry in entries { + let entry = entry.unwrap(); + let path = entry.path(); + let ft = entry.file_type().unwrap(); + let rel = path + .strip_prefix(root) + .unwrap() + .to_string_lossy() + .replace('\\', "/"); + if ft.is_dir() { + if rel != ".socket/blobs" { + stack.push(path); + } + } else if ft.is_file() && rel != ".socket/apply.lock" && rel != ".socket/manifest.json" + { + out.push((rel, std::fs::read(&path).unwrap())); + } + } + } + out.sort(); + out +} + +/// [`tree`] with every vendoring timestamp masked (ledger and markers carry +/// the run's clock), for comparing two runs. +pub fn masked_tree(root: &Path) -> Vec<(String, String)> { + tree(root) + .into_iter() + .map(|(rel, bytes)| { + let text = String::from_utf8_lossy(&bytes).into_owned(); + let masked = if rel.ends_with("state.json") || rel.ends_with("socket-patch.vendor.json") + { + mask_timestamps(&text) + } else { + text + }; + (rel, masked) + }) + .collect() +} + +fn mask_timestamps(text: &str) -> String { + let Ok(mut v) = serde_json::from_str::(text) else { + return text.to_string(); + }; + fn walk(v: &mut serde_json::Value) { + match v { + serde_json::Value::Object(map) => { + for (k, val) in map.iter_mut() { + if k == "vendoredAt" { + *val = serde_json::Value::Null; + } else { + walk(val); + } + } + } + serde_json::Value::Array(items) => items.iter_mut().for_each(walk), + _ => {} + } + } + walk(&mut v); + serde_json::to_string_pretty(&v).unwrap() +} + +fn put(path: &Path, bytes: impl AsRef<[u8]>) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, bytes).unwrap(); +} + +fn write_manifest(root: &Path, patches: &[Patch]) { + let mut map = serde_json::Map::new(); + for p in patches { + map.insert( + p.purl.clone(), + serde_json::json!({ + "uuid": p.uuid, + "exportedAt": "2026-01-01T00:00:00Z", + "files": { p.file.clone(): { + "beforeHash": git_sha256(&p.before), + "afterHash": git_sha256(&p.after), + }}, + "vulnerabilities": { + "GHSA-aaaa-bbbb-cccc": { + "cves": ["CVE-2026-0001"], "summary": "s", + "severity": "high", "description": "d" + } + }, + "description": "fixture patch", + "license": "MIT", + "tier": "free" + }), + ); + put( + &root.join(".socket/blobs").join(git_sha256(&p.after)), + &p.after, + ); + } + put( + &root.join(".socket/manifest.json"), + serde_json::to_vec_pretty(&serde_json::json!({ "patches": map })).unwrap(), + ); +} + +fn zip_bytes(members: &[(&str, &[u8])]) -> Vec { + use std::io::Write as _; + let mut zw = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + // A fixed "made by" host: the zip crate stamps DOS on a Windows build, + // and the checked-in legacy-ledger fixtures carry these archives' + // hashes as a Unix build made them. + let opts = zip::write::SimpleFileOptions::default().system(zip::System::Unix); + for (name, bytes) in members { + zw.start_file(*name, opts).unwrap(); + zw.write_all(bytes).unwrap(); + } + zw.finish().unwrap().into_inner() +} + +type Built = (Vec<(String, String)>, Vec); + +// ── npm (package-lock) ───────────────────────────────────────────────── + +const U1: &str = "11111111-1111-4111-8111-000000000001"; +const U2: &str = "11111111-1111-4111-8111-000000000002"; + +fn npm_install(root: &Path, name: &str, index: &[u8]) { + let pkg = root.join("node_modules").join(name); + put( + &pkg.join("package.json"), + format!(r#"{{"name":"{name}","version":"1.0.0"}}"#), + ); + put(&pkg.join("index.js"), index); +} + +fn npm(root: &Path) -> Built { + npm_install(root, "alpha", b"module.exports = 'a';\n"); + npm_install(root, "beta", b"module.exports = 'b';\n"); + put( + &root.join("package.json"), + r#"{"name":"fixture","version":"1.0.0","private":true,"dependencies":{"alpha":"1.0.0","beta":"1.0.0"}}"#, + ); + let entry = |name: &str| { + serde_json::json!({ + "version": "1.0.0", + "resolved": format!("https://registry.npmjs.org/{name}/-/{name}-1.0.0.tgz"), + "integrity": "sha512-orig==", + }) + }; + let lock = serde_json::json!({ + "name": "fixture", "version": "1.0.0", "lockfileVersion": 3, "requires": true, + "packages": { + "": { "name": "fixture", "version": "1.0.0", + "dependencies": { "alpha": "1.0.0", "beta": "1.0.0" } }, + "node_modules/alpha": entry("alpha"), + "node_modules/beta": entry("beta"), + } + }); + let mut bytes = serde_json::to_vec_pretty(&lock).unwrap(); + bytes.push(b'\n'); + put(&root.join("package-lock.json"), bytes); + ( + Vec::new(), + vec![ + patch( + "pkg:npm/alpha@1.0.0", + U1, + "package/index.js", + b"module.exports = 'a';\n", + b"module.exports = 'A';\n", + ), + patch( + "pkg:npm/beta@1.0.0", + U2, + "package/index.js", + b"module.exports = 'b';\n", + b"module.exports = 'B';\n", + ), + ], + ) +} + +// ── pnpm (lockfile v9, pnpm >= 11 workspace mirror) ───────────────────── + +fn pnpm(root: &Path) -> Built { + npm_install(root, "alpha", b"module.exports = 'a';\n"); + npm_install(root, "beta", b"module.exports = 'b';\n"); + put( + &root.join("package.json"), + r#"{"name":"fixture","version":"1.0.0","private":true,"dependencies":{"alpha":"1.0.0","beta":"1.0.0"}}"#, + ); + put( + &root.join("pnpm-lock.yaml"), + "lockfileVersion: '9.0' + +importers: + .: + dependencies: + alpha: + specifier: 1.0.0 + version: 1.0.0 + beta: + specifier: 1.0.0 + version: 1.0.0 + +packages: + alpha@1.0.0: + resolution: {integrity: sha512-orig==} + + beta@1.0.0: + resolution: {integrity: sha512-orig==} + +snapshots: + alpha@1.0.0: {} + + beta@1.0.0: {} +", + ); + let (_, patches) = npm_patches(); + (Vec::new(), patches) +} + +fn npm_patches() -> Built { + ( + Vec::new(), + vec![ + patch( + "pkg:npm/alpha@1.0.0", + U1, + "package/index.js", + b"module.exports = 'a';\n", + b"module.exports = 'A';\n", + ), + patch( + "pkg:npm/beta@1.0.0", + U2, + "package/index.js", + b"module.exports = 'b';\n", + b"module.exports = 'B';\n", + ), + ], + ) +} + +// ── cargo ─────────────────────────────────────────────────────────────── + +fn cargo(root: &Path, store: &Path) -> Built { + let reg = store.join("cargo-home/registry/src/index.crates.io-6f17d22bba15001f"); + for name in ["alpha", "beta"] { + let krate = reg.join(format!("{name}-1.0.0")); + put( + &krate.join("Cargo.toml"), + format!("[package]\nname = \"{name}\"\nversion = \"1.0.0\"\n"), + ); + put(&krate.join("src/lib.rs"), format!("pub fn {name}() {{}}\n")); + put(&krate.join(".cargo-checksum.json"), "{\"files\":{}}"); + } + put( + &root.join("Cargo.toml"), + "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[dependencies]\nalpha = \"1\"\nbeta = \"1\"\n", + ); + let pkg = |name: &str, sum: char| { + format!( + "[[package]]\nname = \"{name}\"\nversion = \"1.0.0\"\n\ + source = \"registry+https://github.com/rust-lang/crates.io-index\"\n\ + checksum = \"{}\"\n", + sum.to_string().repeat(64) + ) + }; + put( + &root.join("Cargo.lock"), + format!( + "# This file is automatically @generated by Cargo.\n\ + # It is not intended for manual editing.\nversion = 4\n\n\ + {}\n{}\n[[package]]\nname = \"app\"\nversion = \"0.1.0\"\n\ + dependencies = [\n \"alpha\",\n \"beta\",\n]\n", + pkg("alpha", 'a'), + pkg("beta", 'b') + ), + ); + ( + Vec::new(), + vec![ + patch( + "pkg:cargo/alpha@1.0.0", + U1, + "package/src/lib.rs", + b"pub fn alpha() {}\n", + b"pub fn alpha() { /* patched */ }\n", + ), + patch( + "pkg:cargo/beta@1.0.0", + U2, + "package/src/lib.rs", + b"pub fn beta() {}\n", + b"pub fn beta() { /* patched */ }\n", + ), + ], + ) +} + +// ── golang ────────────────────────────────────────────────────────────── + +fn golang(root: &Path, store: &Path) -> Built { + for name in ["alpha", "beta"] { + let dir = store.join(format!("modcache/github.com/fx/{name}@v1.0.0")); + put( + &dir.join("go.mod"), + format!("module github.com/fx/{name}\n\ngo 1.21\n"), + ); + put( + &dir.join(format!("{name}.go")), + format!("package {name}\n\nfunc Hello() string {{ return \"hi\" }}\n"), + ); + } + put( + &root.join("go.mod"), + "module example.com/app\n\ngo 1.21\n\nrequire (\n\tgithub.com/fx/alpha v1.0.0\n\tgithub.com/fx/beta v1.0.0\n)\n", + ); + let p = |name: &str| { + patch( + &format!("pkg:golang/github.com/fx/{name}@v1.0.0"), + if name == "alpha" { U1 } else { U2 }, + &format!("package/{name}.go"), + format!("package {name}\n\nfunc Hello() string {{ return \"hi\" }}\n").as_bytes(), + format!("package {name}\n\nfunc Hello() string {{ return \"patched\" }}\n").as_bytes(), + ) + }; + (Vec::new(), vec![p("alpha"), p("beta")]) +} + +// ── pypi ──────────────────────────────────────────────────────────────── + +fn site_packages(root: &Path) -> PathBuf { + if cfg!(windows) { + root.join(".venv/Lib/site-packages") + } else { + root.join(".venv/lib/python3.12/site-packages") + } +} + +fn pypi_install(root: &Path, name: &str) { + let sp = site_packages(root); + let di = sp.join(format!("{name}-1.0.0.dist-info")); + put( + &di.join("METADATA"), + format!("Metadata-Version: 2.1\nName: {name}\nVersion: 1.0.0\n\nbody\n"), + ); + put( + &di.join("WHEEL"), + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ); + put( + &di.join("RECORD"), + format!( + "{name}.py,sha256=AAAA,20\n{name}-1.0.0.dist-info/METADATA,,\n\ + {name}-1.0.0.dist-info/WHEEL,,\n{name}-1.0.0.dist-info/RECORD,,\n" + ), + ); + put(&sp.join(format!("{name}.py")), format!("# {name}\nX = 1\n")); +} + +fn pypi_patches() -> Vec { + ["alpha", "beta"] + .iter() + .map(|name| { + patch( + &format!("pkg:pypi/{name}@1.0.0"), + if *name == "alpha" { U1 } else { U2 }, + &format!("{name}.py"), + format!("# {name}\nX = 1\n").as_bytes(), + format!("# {name}\nX = 2\n").as_bytes(), + ) + }) + .collect() +} + +fn pypi_requirements(root: &Path) -> Built { + pypi_install(root, "alpha"); + pypi_install(root, "beta"); + put( + &root.join("requirements.txt"), + "alpha==1.0.0\nbeta==1.0.0\n", + ); + (Vec::new(), pypi_patches()) +} + +fn pylock(root: &Path) -> Built { + pypi_install(root, "alpha"); + pypi_install(root, "beta"); + // Enough unrelated packages that the lock is not a toy: whole-file + // ledger snapshots are what the ledger schema compacts. + let mut lock = String::from("lock-version = \"1.0\"\ncreated-by = \"fixture\"\n"); + for i in 0..40 { + lock.push_str(&format!( + "\n[[packages]]\nname = \"filler{i:02}\"\nversion = \"1.0.{i}\"\n\ + wheels = [{{ url = \"https://files.example/filler{i:02}-1.0.{i}-py3-none-any.whl\", \ + hashes = {{ sha256 = \"{}\" }} }}]\n", + format!("{i:02}").repeat(32) + )); + } + for name in ["alpha", "beta"] { + lock.push_str(&format!( + "\n[[packages]]\nname = \"{name}\"\nversion = \"1.0.0\"\n\ + wheels = [{{ url = \"https://files.example/{name}-1.0.0-py3-none-any.whl\", \ + hashes = {{ sha256 = \"{}\" }} }}]\n", + "c".repeat(64) + )); + } + put(&root.join("pylock.toml"), lock); + (Vec::new(), pypi_patches()) +} + +// ── gem ───────────────────────────────────────────────────────────────── + +fn gem(root: &Path) -> Built { + let bundle = root.join("vendor/bundle"); + for name in ["alpha", "beta"] { + let leaf = format!("{name}-1.0.0"); + put( + &bundle + .join("gems") + .join(&leaf) + .join(format!("lib/{name}.rb")), + format!("module {}; end\n", name.to_uppercase()), + ); + put( + &bundle + .join("specifications") + .join(format!("{leaf}.gemspec")), + format!( + "Gem::Specification.new do |s|\n s.name = \"{name}\"\n \ + s.version = \"1.0.0\"\n s.summary = \"fixture\"\n \ + s.authors = [\"Socket\"]\n s.require_paths = [\"lib\"]\nend\n" + ), + ); + } + put( + &root.join("Gemfile"), + "source \"https://rubygems.org\"\ngem \"alpha\"\ngem \"beta\"\n", + ); + put( + &root.join("Gemfile.lock"), + "GEM\n remote: https://rubygems.org/\n specs:\n alpha (1.0.0)\n beta (1.0.0)\n\n\ + PLATFORMS\n ruby\n\nDEPENDENCIES\n alpha\n beta\n\nBUNDLED WITH\n 2.5.3\n", + ); + let p = |name: &str| { + patch( + &format!("pkg:gem/{name}@1.0.0"), + if name == "alpha" { U1 } else { U2 }, + &format!("lib/{name}.rb"), + format!("module {}; end\n", name.to_uppercase()).as_bytes(), + format!("module {}; SAFE = true; end\n", name.to_uppercase()).as_bytes(), + ) + }; + (Vec::new(), vec![p("alpha"), p("beta")]) +} + +// ── composer ──────────────────────────────────────────────────────────── + +fn composer(root: &Path) -> Built { + let mut installed = Vec::new(); + let mut locked = Vec::new(); + for name in ["alpha", "beta"] { + let dir = root.join(format!("vendor/fx/{name}")); + put( + &dir.join("composer.json"), + format!("{{\"name\": \"fx/{name}\"}}\n"), + ); + put(&dir.join("src/Lib.php"), format!(" Built { + let mut deps = String::new(); + for name in ["alpha", "beta"] { + let dir = store.join(format!("m2/org/fx/{name}/1.0.0")); + put( + &dir.join(format!("{name}-1.0.0.pom")), + format!( + "\n \ + 4.0.0\n org.fx\n \ + {name}\n 1.0.0\n\n" + ), + ); + put( + &dir.join(format!("{name}-1.0.0.jar")), + zip_bytes(&[ + ("META-INF/MANIFEST.MF", b"Manifest-Version: 1.0\n"), + ("META-INF/NOTICE.txt", format!("{name} notice\n").as_bytes()), + ("org/fx/Main.class", b"\xca\xfe\xba\xbe"), + ]), + ); + deps.push_str(&format!( + " \n org.fx\n \ + {name}\n 1.0.0\n \n" + )); + } + // A realistic-size project pom: whole-file ledger snapshots of it are + // what the ledger schema compacts. + let mut filler = String::new(); + for i in 0..60 { + filler.push_str(&format!( + " \n org.filler\n \ + filler{i:02}\n 1.0.{i}\n \ + \n" + )); + } + put( + &root.join("pom.xml"), + format!( + "\n\ + \n \ + 4.0.0\n com.example\n \ + app\n 1.0.0\n \ + \n{deps}{filler} \n\n" + ), + ); + let p = |name: &str| { + patch( + &format!("pkg:maven/org.fx/{name}@1.0.0"), + if name == "alpha" { U1 } else { U2 }, + "META-INF/NOTICE.txt", + format!("{name} notice\n").as_bytes(), + format!("{name} notice (patched)\n").as_bytes(), + ) + }; + (Vec::new(), vec![p("alpha"), p("beta")]) +} + +// ── nuget ─────────────────────────────────────────────────────────────── + +fn nupkg_content_hash(bytes: &[u8]) -> String { + use base64::Engine as _; + base64::engine::general_purpose::STANDARD.encode(sha2::Sha512::digest(bytes)) +} + +fn nuget(root: &Path, store: &Path) -> Built { + let mut refs = String::new(); + let mut deps = serde_json::Map::new(); + for name in ["Fx.Alpha", "Fx.Beta"] { + let lower = name.to_lowercase(); + let dir = store.join(format!("nuget/{lower}/1.0.0")); + let nuspec = format!( + "{name}\ + 1.0.0" + ); + let member = format!("{name} license\n"); + let nupkg = zip_bytes(&[ + (&format!("{name}.nuspec"), nuspec.as_bytes()), + ("LICENSE.md", member.as_bytes()), + ("lib/net8.0/Fx.dll", b"MZ dll"), + ]); + put(&dir.join(format!("{lower}.1.0.0.nupkg")), &nupkg); + put( + &dir.join(format!("{lower}.1.0.0.nupkg.sha512")), + nupkg_content_hash(&nupkg), + ); + put(&dir.join(format!("{lower}.nuspec")), &nuspec); + put(&dir.join("LICENSE.md"), &member); + put(&dir.join("lib/net8.0/Fx.dll"), b"MZ dll"); + refs.push_str(&format!( + "" + )); + deps.insert( + name.to_string(), + serde_json::json!({ + "type": "Direct", "requested": "[1.0.0, )", "resolved": "1.0.0", + "contentHash": nupkg_content_hash(&nupkg) + }), + ); + } + put( + &root.join("app.csproj"), + format!( + "net8.0\ + true\ + {refs}" + ), + ); + let lock = serde_json::json!({ + "version": 1, + "dependencies": { "net8.0": deps } + }); + put( + &root.join("packages.lock.json"), + serde_json::to_vec_pretty(&lock).unwrap(), + ); + let mut config = String::from( + "\n\n \n \ + \n \ + \n", + ); + // A realistic-size config: whole-file ledger snapshots of it are what + // the ledger schema compacts. + config.push_str(" \n"); + for i in 0..40 { + config.push_str(&format!( + " \n" + )); + } + config.push_str(" \n\n"); + put(&root.join("nuget.config"), config); + let p = |name: &str| { + patch( + &format!("pkg:nuget/{name}@1.0.0"), + if name == "Fx.Alpha" { U1 } else { U2 }, + "LICENSE.md", + format!("{name} license\n").as_bytes(), + format!("{name} license (patched)\n").as_bytes(), + ) + }; + (Vec::new(), vec![p("Fx.Alpha"), p("Fx.Beta")]) +} diff --git a/crates/socket-patch-cli/tests/vendor_group_commit_e2e.rs b/crates/socket-patch-cli/tests/vendor_group_commit_e2e.rs new file mode 100644 index 000000000..d2238428f --- /dev/null +++ b/crates/socket-patch-cli/tests/vendor_group_commit_e2e.rs @@ -0,0 +1,313 @@ +//! Group commit of a vendored run (see +//! `socket_patch_core::utils::group_commit`): every lockfile, manifest, +//! config and ledger edit of the run is written once, after the loop, +//! through a roll-forward journal. +//! +//! * **Equivalence.** For every ecosystem, a completed run ends in exactly +//! the tree the per-package commits produce — the debug build's +//! `SOCKET_PATCH_SWITCH_OFF=group_commit` switch runs that path as the +//! oracle — including a run where one package fails and the other +//! succeeds, and the `--revert` that follows. +//! * **Crash safety.** The debug build's failpoints crash the binary mid-loop, +//! at the artifact barrier, right after the journal is written, and after +//! the first journaled file is replaced. Before the journal, the project's +//! lockfiles and ledger are exactly the pre-run ones; after it, the next +//! locked command finishes the commit; either way the next run ends in +//! the uninterrupted run's tree. A journal that no longer matches the +//! files (a hand edit after the crash) is set aside whole, never +//! half-applied. + +#[path = "vendor_ecosystem_fixtures/mod.rs"] +mod fx; + +use fx::{masked_tree, Fixture}; + +const OFF: (&str, &str) = ("SOCKET_PATCH_SWITCH_OFF", "group_commit"); + +fn events(stdout: &str) -> Vec<(String, String, String)> { + let v: serde_json::Value = serde_json::from_str(stdout.trim()) + .unwrap_or_else(|e| panic!("envelope JSON: {e}\n{stdout}")); + v["events"] + .as_array() + .expect("events") + .iter() + .map(|e| { + ( + e["purl"].as_str().unwrap_or_default().to_string(), + e["action"].as_str().unwrap_or_default().to_string(), + e["errorCode"].as_str().unwrap_or_default().to_string(), + ) + }) + .collect() +} + +/// The paths whose bytes differ between two trees (or exist in one only). +fn differing(a: &[(String, String)], b: &[(String, String)]) -> Vec { + let mut out: Vec = a + .iter() + .filter(|entry| !b.contains(entry)) + .chain(b.iter().filter(|entry| !a.contains(entry))) + .map(|(rel, _)| rel.clone()) + .collect(); + out.sort(); + out.dedup(); + out +} + +/// A completed run commits the same tree as committing after every package, +/// for every ecosystem, and the revert that follows restores the same tree. +#[test] +fn group_commit_ends_where_per_package_commits_end_for_every_ecosystem() { + for eco in fx::ALL { + let grouped = Fixture::new(eco); + let oracle = Fixture::new(eco); + let pristine = masked_tree(&grouped.root); + + let (code, stdout, stderr) = grouped.vendor(&[], &[]); + let (oracle_code, oracle_stdout, oracle_stderr) = oracle.vendor(&[], &[OFF]); + assert_eq!(code, 0, "{eco}: {stdout}\n{stderr}"); + assert_eq!( + oracle_code, 0, + "{eco} (oracle): {oracle_stdout}\n{oracle_stderr}" + ); + let applied: Vec<_> = events(&stdout) + .into_iter() + .filter(|(_, action, _)| action == "applied") + .map(|(purl, ..)| purl) + .collect(); + assert_eq!( + applied.len(), + grouped.patches.len(), + "{eco}: both packages vendor: {stdout}" + ); + assert_eq!( + events(&stdout), + events(&oracle_stdout), + "{eco}: same events" + ); + assert_eq!( + masked_tree(&grouped.root), + masked_tree(&oracle.root), + "{eco}: same committed tree" + ); + assert!( + !grouped + .root + .join(".socket/vendor/.commit-journal.json") + .exists(), + "{eco}: a completed commit leaves no journal" + ); + + // In sync: a re-run writes nothing. + let before = masked_tree(&grouped.root); + let (code, stdout, _) = grouped.vendor(&[], &[]); + assert_eq!(code, 0, "{eco}: {stdout}"); + assert_eq!(masked_tree(&grouped.root), before, "{eco}: in-sync re-run"); + + let (code, stdout, stderr) = grouped.vendor(&["--revert"], &[]); + let (oracle_code, ..) = oracle.vendor(&["--revert"], &[OFF]); + assert_eq!(code, 0, "{eco} revert: {stdout}\n{stderr}"); + assert_eq!(oracle_code, 0, "{eco} revert (oracle)"); + assert_eq!( + masked_tree(&grouped.root), + masked_tree(&oracle.root), + "{eco}: same tree after --revert" + ); + // Two ecosystems keep scaffolding their revert does not remove when + // TWO packages were vendored (the emptied pnpm override tables and + // workspace file; the catch-all `` nuget adds + // to a config that had none). That predates the group commit — the + // oracle leaves the same bytes, asserted just above — so only the + // others are held to a byte-exact round trip here. + if !["pnpm", "nuget"].contains(eco) { + let after = masked_tree(&grouped.root); + assert!( + after == pristine, + "{eco}: --revert restores the pre-vendor project byte for byte; differing: {:?}", + differing(&after, &pristine) + ); + } + } +} + +/// One package fails (its patch target is missing from the installed copy, +/// which fails closed without `--force`), the other succeeds: the success is +/// committed, the failure leaves nothing, and the tree is the per-package +/// commits' tree. +#[test] +fn a_partial_failure_commits_the_packages_that_succeeded() { + for (eco, target) in [ + ("npm", "proj:node_modules/beta/index.js"), + ( + "cargo", + "store:cargo-home/registry/src/index.crates.io-6f17d22bba15001f/beta-1.0.0/src/lib.rs", + ), + ("gem", "proj:vendor/bundle/gems/beta-1.0.0/lib/beta.rb"), + ("golang", "store:modcache/github.com/fx/beta@v1.0.0/beta.go"), + ] { + let grouped = Fixture::new(eco); + let oracle = Fixture::new(eco); + for f in [&grouped, &oracle] { + let path = match target.split_once(':') { + Some(("proj", rel)) => f.root.join(rel), + Some((_, rel)) => f.store.join(rel), + None => unreachable!(), + }; + std::fs::remove_file(path).unwrap(); + } + let (code, stdout, _) = grouped.vendor(&[], &[]); + let (oracle_code, oracle_stdout, _) = oracle.vendor(&[], &[OFF]); + assert_eq!(code, oracle_code, "{eco}: {stdout}\n{oracle_stdout}"); + assert_ne!(code, 0, "{eco}: the failed package fails the run"); + let ev = events(&stdout); + assert!( + ev.iter() + .any(|(p, a, _)| *p == grouped.patches[0].purl && a == "applied"), + "{eco}: {stdout}" + ); + assert!( + !ev.iter() + .any(|(p, a, _)| *p == grouped.patches[1].purl && a == "applied"), + "{eco}: {stdout}" + ); + assert_eq!(ev, events(&oracle_stdout), "{eco}: same events"); + assert_eq!( + masked_tree(&grouped.root), + masked_tree(&oracle.root), + "{eco}: same committed tree" + ); + let state = std::fs::read_to_string(grouped.root.join(".socket/vendor/state.json")) + .unwrap_or_else(|e| panic!("{eco}: the success is committed: {e}")); + assert!(state.contains(&grouped.patches[0].purl), "{eco}"); + assert!(!state.contains(&grouped.patches[1].purl), "{eco}"); + } +} + +/// The files a crash before the journal must leave exactly as they were. +fn commit_points(f: &Fixture) -> Vec<(String, Vec)> { + f.tree() + .into_iter() + .filter(|(rel, _)| !rel.starts_with(".socket/vendor/") || rel.ends_with("state.json")) + .collect() +} + +/// Crash at `failpoint`, then let the next run finish; returns the crashed +/// fixture's final tree next to an uninterrupted run's. +fn crash_then_rerun(eco: &str, failpoint: &str, before_journal: bool) { + let clean = Fixture::new(eco); + let (code, stdout, _) = clean.vendor(&[], &[]); + assert_eq!(code, 0, "{eco}: {stdout}"); + + let f = Fixture::new(eco); + let pre = commit_points(&f); + let (code, stdout, stderr) = f.vendor(&[], &[("SOCKET_PATCH_FAILPOINT", failpoint)]); + assert_eq!( + code, 86, + "{eco}/{failpoint}: the crash fires: {stdout}\n{stderr}" + ); + let journal = f.root.join(".socket/vendor/.commit-journal.json"); + if before_journal { + assert_eq!( + commit_points(&f), + pre, + "{eco}/{failpoint}: no commit point is written before the journal" + ); + assert!(!journal.exists(), "{eco}/{failpoint}"); + } else { + assert!( + journal.is_file(), + "{eco}/{failpoint}: the journal is the commit" + ); + } + + let (code, stdout, stderr) = f.vendor(&[], &[]); + assert_eq!( + code, 0, + "{eco}/{failpoint}: the next run completes: {stdout}\n{stderr}" + ); + assert!( + !journal.exists(), + "{eco}/{failpoint}: the journal is consumed" + ); + assert_eq!( + masked_tree(&f.root), + masked_tree(&clean.root), + "{eco}/{failpoint}: the next run ends where an uninterrupted run ends" + ); +} + +#[test] +fn a_crash_mid_loop_leaves_the_pre_run_commit_points() { + for eco in ["npm", "pnpm", "cargo", "nuget"] { + crash_then_rerun(eco, "vendor_package_recorded@1", true); + } +} + +#[test] +fn a_crash_at_the_artifact_barrier_leaves_the_pre_run_commit_points() { + for eco in ["pnpm", "golang", "maven"] { + crash_then_rerun(eco, "durability_barrier", true); + } +} + +#[test] +fn a_crash_after_the_journal_is_rolled_forward_by_the_next_locked_command() { + for eco in ["pnpm", "cargo", "gem", "pypi-requirements"] { + crash_then_rerun(eco, "group_commit_journal", false); + crash_then_rerun(eco, "group_commit_file@1", false); + } +} + +/// The roll-forward runs under ANY command's lock, not just `vendor`: a +/// `vendor --revert` after the crash first finishes the commit, then +/// reverts the fully-wired project byte for byte. +#[test] +fn the_roll_forward_runs_before_any_locked_command_reads_the_files() { + let f = Fixture::new("npm"); + let pristine = masked_tree(&f.root); + let (code, ..) = f.vendor(&[], &[("SOCKET_PATCH_FAILPOINT", "group_commit_file@1")]); + assert_eq!(code, 86); + let (code, stdout, stderr) = f.vendor(&["--revert"], &[]); + assert_eq!(code, 0, "{stdout}\n{stderr}"); + assert_eq!(masked_tree(&f.root), pristine); +} + +/// A file the journal covers was edited by hand after the crash: the +/// journal matches neither side of it, so it is set aside whole (with a +/// warning), nothing of it is applied, and the run proceeds over the +/// project as it stands. +#[test] +fn a_journal_the_files_no_longer_match_is_set_aside_not_half_applied() { + let f = Fixture::new("pnpm"); + let (code, ..) = f.vendor(&[], &[("SOCKET_PATCH_FAILPOINT", "group_commit_journal")]); + assert_eq!(code, 86); + let lock_before = std::fs::read(f.root.join("pnpm-lock.yaml")).unwrap(); + let pkg = f.root.join("package.json"); + let mut edited = std::fs::read_to_string(&pkg).unwrap(); + edited.push('\n'); + std::fs::write(&pkg, &edited).unwrap(); + + let (code, stdout, stderr) = f.run(&["vendor", "--json", "--offline", "--dry-run"], &[]); + assert_eq!(code, 0, "{stdout}\n{stderr}"); + assert!( + stderr.contains("set aside"), + "the set-aside is reported: {stderr}" + ); + assert!(!f.root.join(".socket/vendor/.commit-journal.json").exists()); + let aside: Vec<_> = std::fs::read_dir(f.root.join(".socket/vendor")) + .unwrap() + .filter_map(|e| e.ok()) + .filter(|e| { + e.file_name() + .to_string_lossy() + .starts_with(".commit-journal.set-aside-") + }) + .collect(); + assert_eq!(aside.len(), 1, "the journal is kept for inspection"); + assert_eq!( + std::fs::read(f.root.join("pnpm-lock.yaml")).unwrap(), + lock_before, + "no file of the set-aside journal is applied" + ); + assert_eq!(std::fs::read_to_string(&pkg).unwrap(), edited); +} diff --git a/crates/socket-patch-cli/tests/vendor_ledger_schema_e2e.rs b/crates/socket-patch-cli/tests/vendor_ledger_schema_e2e.rs new file mode 100644 index 000000000..406a5b33f --- /dev/null +++ b/crates/socket-patch-cli/tests/vendor_ledger_schema_e2e.rs @@ -0,0 +1,248 @@ +//! The vendor ledger's version-2 snapshot edits (see +//! `socket_patch_core::vendor::ledger_snapshots`): a whole-file wiring +//! record's `new` is stored as a small edit of the same record's +//! `original`, and every older ledger keeps reverting byte for byte. +//! +//! * `legacy_ledgers_revert_byte_for_byte` replays ledgers the integrated +//! base binary wrote (version 1, inline whole-file snapshots — checked in +//! under `tests/fixtures/legacy-ledgers//`, regenerated by the +//! ignored `generate_legacy_ledger_fixtures` below) through this binary's +//! `vendor --revert`, for every ecosystem, and requires exactly the tree +//! the base binary's own revert left. +//! * `new_ledgers_compact_whole_file_snapshots_and_revert` vendors each +//! ecosystem with this binary: the ledgers that hold whole-file snapshots +//! (maven, nuget, pylock) come out as version 2 with every `new` an edit +//! far smaller than the inline text, the others stay version 1, and the +//! revert is exact. + +#[path = "vendor_ecosystem_fixtures/mod.rs"] +mod fx; + +use std::path::{Path, PathBuf}; + +use fx::{masked_tree, tree, Fixture}; + +fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/legacy-ledgers") +} + +/// The files of `after` that are new or changed relative to `before`, +/// minus the artifacts under `.socket/vendor//` (the ledger and the +/// project's wiring files are what a legacy checkout carries forward; the +/// revert deletes the artifacts either way). +fn wiring_delta( + before: &[(String, Vec)], + after: &[(String, Vec)], +) -> Vec<(String, Vec)> { + after + .iter() + .filter(|entry| !before.contains(entry)) + .filter(|(rel, _)| { + !rel.starts_with(".socket/vendor/") || rel == ".socket/vendor/state.json" + }) + .cloned() + .collect() +} + +fn write_tree(dir: &Path, files: &[(String, Vec)]) { + for (rel, bytes) in files { + let path = dir.join(rel); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, bytes).unwrap(); + } +} + +fn read_tree(dir: &Path) -> Vec<(String, Vec)> { + if !dir.exists() { + return Vec::new(); + } + tree(dir) +} + +/// Regenerate `tests/fixtures/legacy-ledgers/` with the binary named by +/// `SOCKET_PATCH_LEGACY_BIN` (the integrated base, before the version-2 +/// ledger). For each ecosystem it stores the wiring files and the ledger +/// that binary's `vendor` wrote (`wired/`), and the wiring files its +/// `vendor --revert` then left (`reverted/`). +#[test] +#[ignore = "regenerates checked-in fixtures; needs SOCKET_PATCH_LEGACY_BIN"] +fn generate_legacy_ledger_fixtures() { + let bin = std::env::var("SOCKET_PATCH_LEGACY_BIN").expect("SOCKET_PATCH_LEGACY_BIN"); + for eco in fx::ALL { + let f = Fixture::new(eco); + let pristine = tree(&f.root); + let (code, stdout, stderr) = f.run_bin(&bin, &["vendor", "--json", "--offline"], &[]); + assert_eq!(code, 0, "{eco}: {stdout}\n{stderr}"); + let wired = wiring_delta(&pristine, &tree(&f.root)); + let (code, stdout, stderr) = + f.run_bin(&bin, &["vendor", "--json", "--offline", "--revert"], &[]); + assert_eq!(code, 0, "{eco} revert: {stdout}\n{stderr}"); + let reverted = wiring_delta(&pristine, &tree(&f.root)); + let out = fixtures_dir().join(eco); + let _ = std::fs::remove_dir_all(&out); + write_tree(&out.join("wired"), &wired); + write_tree(&out.join("reverted"), &reverted); + } +} + +#[test] +fn legacy_ledgers_revert_byte_for_byte() { + for eco in fx::ALL { + let dir = fixtures_dir().join(eco); + let wired = read_tree(&dir.join("wired")); + let legacy_ledger = wired + .iter() + .find(|(rel, _)| rel == ".socket/vendor/state.json") + .unwrap_or_else(|| panic!("{eco}: fixture carries the legacy ledger")); + let ledger: serde_json::Value = serde_json::from_slice(&legacy_ledger.1).unwrap(); + assert_eq!(ledger["version"], 1, "{eco}: a version-1 ledger"); + assert!(ledger.get("snapshots").is_none(), "{eco}"); + + let f = Fixture::new(eco); + let pristine = tree(&f.root); + write_tree(&f.root, &wired); + let (code, stdout, stderr) = f.vendor(&["--revert"], &[]); + assert_eq!(code, 0, "{eco}: {stdout}\n{stderr}"); + let mut expected = pristine + .iter() + .filter(|(rel, _)| { + !read_tree(&dir.join("reverted")) + .iter() + .any(|(r, _)| r == rel) + }) + .cloned() + .collect::>(); + expected.extend(read_tree(&dir.join("reverted"))); + expected.sort(); + assert_eq!( + tree(&f.root), + expected, + "{eco}: this binary reverts a legacy ledger to exactly the base binary's result" + ); + if !["pnpm", "nuget"].contains(eco) { + assert_eq!(tree(&f.root), pristine, "{eco}: byte-exact restore"); + } + } +} + +#[test] +fn new_ledgers_compact_whole_file_snapshots_and_revert() { + for eco in fx::ALL { + let f = Fixture::new(eco); + let pristine = masked_tree(&f.root); + let (code, stdout, stderr) = f.vendor(&[], &[]); + assert_eq!(code, 0, "{eco}: {stdout}\n{stderr}"); + let bytes = std::fs::read(f.root.join(".socket/vendor/state.json")).unwrap(); + let ledger: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + let whole_file = ["maven", "nuget", "pylock"].contains(eco); + assert_eq!( + ledger["version"], + if whole_file { 2 } else { 1 }, + "{eco}: {ledger:#}" + ); + if whole_file { + let legacy: serde_json::Value = serde_json::from_slice( + &std::fs::read( + fixtures_dir() + .join(eco) + .join("wired/.socket/vendor/state.json"), + ) + .unwrap(), + ) + .unwrap(); + // The whole-file `new` texts the base binary stored inline… + let inline: usize = legacy["entries"] + .as_object() + .unwrap() + .values() + .flat_map(|e| e["wiring"].as_array().unwrap().iter()) + .filter_map(|r| r["new"].as_str()) + .filter(|t| t.len() >= 1024) + .map(str::len) + .sum(); + // …against the edits this binary stores them as. Every record + // stays self-contained: its `original` is still plain text and + // nothing lives outside the entries (an older binary re-saving + // the ledger drops unknown top-level fields). + assert!(ledger.get("snapshots").is_none(), "{eco}: {ledger:#}"); + let records: Vec<&serde_json::Value> = ledger["entries"] + .as_object() + .unwrap() + .values() + .flat_map(|e| e["wiring"].as_array().unwrap().iter()) + .collect(); + assert!( + records + .iter() + .all(|r| r["original"].get("snapshot").is_none()), + "{eco}: an original is never an edit: {ledger:#}" + ); + let stored: usize = records + .iter() + .map(|r| &r["new"]) + .filter(|v| v.get("snapshot").is_some()) + .map(|v| serde_json::to_vec(v).unwrap().len()) + .sum(); + assert!(stored > 0, "{eco}: {ledger:#}"); + assert!( + stored * 4 < inline, + "{eco}: the edits ({stored} bytes) are well under the inline \ + `new` texts ({inline} bytes)" + ); + } + + let (code, stdout, stderr) = f.vendor(&["--revert"], &[]); + assert_eq!(code, 0, "{eco} revert: {stdout}\n{stderr}"); + if !["pnpm", "nuget"].contains(eco) { + assert_eq!(masked_tree(&f.root), pristine, "{eco}: byte-exact restore"); + } else { + let reverted = read_tree(&fixtures_dir().join(eco).join("reverted")); + for (rel, bytes) in reverted { + assert_eq!( + std::fs::read(f.root.join(&rel)).unwrap(), + bytes, + "{eco}: {rel} reverts exactly as the base binary reverted it" + ); + } + } + } +} + +/// Against the integrated base: for every ecosystem this binary wires +/// exactly the files the base binary wired (the checked-in legacy +/// fixtures), and its ledger — whatever its on-disk version — loads to the +/// same entries the base's version-1 ledger loads to. +#[tokio::test] +async fn this_binary_wires_what_the_base_binary_wired() { + for eco in fx::ALL { + let base_wired = read_tree(&fixtures_dir().join(eco).join("wired")); + let f = Fixture::new(eco); + let pristine = tree(&f.root); + let (code, stdout, stderr) = f.vendor(&[], &[]); + assert_eq!(code, 0, "{eco}: {stdout}\n{stderr}"); + let wired = wiring_delta(&pristine, &tree(&f.root)); + let strip = |t: &[(String, Vec)]| -> Vec<(String, Vec)> { + t.iter() + .filter(|(rel, _)| rel != ".socket/vendor/state.json") + .cloned() + .collect() + }; + assert_eq!( + strip(&wired), + strip(&base_wired), + "{eco}: the same wiring files, byte for byte" + ); + let base_dir = tempfile::tempdir().unwrap(); + write_tree(base_dir.path(), &base_wired); + let base_state = socket_patch_core::vendor::load_state(base_dir.path()) + .await + .unwrap(); + let state = socket_patch_core::vendor::load_state(&f.root) + .await + .unwrap(); + assert_eq!( + state.entries, base_state.entries, + "{eco}: the same ledger entries" + ); + } +} diff --git a/crates/socket-patch-cli/tests/vendor_pristine_fetch_order_e2e.rs b/crates/socket-patch-cli/tests/vendor_pristine_fetch_order_e2e.rs new file mode 100644 index 000000000..5c6f5db82 --- /dev/null +++ b/crates/socket-patch-cli/tests/vendor_pristine_fetch_order_e2e.rs @@ -0,0 +1,228 @@ +//! `vendor` over several lockfile-only packages: the pristine registry +//! fetches run concurrently, and every package's outcome must still be +//! exactly the one-at-a-time loop's — here with the registry answering the +//! later packages first and a mix of verified, tampered and unverifiable +//! lock entries. Mock registry + a real npm lockfile fixture, driven +//! through the built binary. + +use std::path::PathBuf; +use std::process::Command; +use std::time::Duration; + +use sha2::{Digest, Sha256}; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +fn binary() -> PathBuf { + env!("CARGO_BIN_EXE_socket-patch").into() +} + +const BEFORE: &[u8] = b"before\n"; +const AFTER: &[u8] = b"after\n"; + +fn git_sha256(content: &[u8]) -> String { + let header = format!("blob {}\0", content.len()); + let mut hasher = Sha256::new(); + hasher.update(header.as_bytes()); + hasher.update(content); + hex::encode(hasher.finalize()) +} + +fn sri_of(bytes: &[u8]) -> String { + use base64::Engine as _; + use sha2::Sha512; + format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(Sha512::digest(bytes)) + ) +} + +/// A pristine registry tarball whose index.js carries the BEFORE bytes. +fn pristine_tgz(name: &str) -> Vec { + let mut builder = tar::Builder::new(flate2::write::GzEncoder::new( + Vec::new(), + flate2::Compression::default(), + )); + let pkg_json = format!(r#"{{"name":"{name}","version":"1.0.0"}}"#); + for (path, bytes) in [ + ("package/package.json", pkg_json.as_bytes()), + ("package/index.js", BEFORE), + ] { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder.append_data(&mut header, path, bytes).unwrap(); + } + builder.into_inner().unwrap().finish().unwrap() +} + +/// How the lockfile records one package. +enum Lock { + /// The registry tarball's real integrity. + Verified, + /// An integrity the served bytes do not match (tampered). + Tampered, + /// No integrity at all (unverifiable). + Missing, +} + +#[tokio::test] +async fn lockfile_only_packages_fetch_concurrently_with_serial_outcomes() { + let mock = MockServer::start().await; + // Earlier packages answer last. + let packages: [(&str, Lock, u64); 5] = [ + ("pa", Lock::Verified, 500), + ("pb", Lock::Tampered, 400), + ("pc", Lock::Verified, 300), + ("pd", Lock::Missing, 200), + ("pe", Lock::Verified, 0), + ]; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + let mut deps = serde_json::Map::new(); + let mut lock_packages = serde_json::Map::new(); + let mut patches = serde_json::Map::new(); + for (i, (name, lock, delay)) in packages.iter().enumerate() { + let tgz = pristine_tgz(name); + let tgz_path = format!("/{name}/-/{name}-1.0.0.tgz"); + let mut entry = serde_json::json!({ + "version": "1.0.0", + "resolved": format!("{}{tgz_path}", mock.uri()), + }); + match lock { + Lock::Verified => entry["integrity"] = sri_of(&tgz).into(), + Lock::Tampered => entry["integrity"] = sri_of(b"other bytes").into(), + Lock::Missing => {} + } + Mock::given(method("GET")) + .and(path(tgz_path)) + .respond_with( + ResponseTemplate::new(200) + .set_body_bytes(tgz) + .set_delay(Duration::from_millis(*delay)), + ) + .mount(&mock) + .await; + deps.insert(name.to_string(), "^1.0.0".into()); + lock_packages.insert(format!("node_modules/{name}"), entry); + patches.insert( + format!("pkg:npm/{name}@1.0.0"), + serde_json::json!({ + "uuid": format!("{i:08x}-1111-4111-8111-111111111111"), + "exportedAt": "2026-01-01T00:00:00Z", + "files": { "package/index.js": { + "beforeHash": git_sha256(BEFORE), + "afterHash": git_sha256(AFTER), + }}, + "vulnerabilities": {}, + "description": "synthetic", + "license": "MIT", + "tier": "free" + }), + ); + } + let manifest = + serde_json::json!({ "name": "order-test", "version": "0.0.0", "dependencies": deps }); + std::fs::write(root.join("package.json"), manifest.to_string()).unwrap(); + lock_packages.insert( + String::new(), + serde_json::json!({ "name": "order-test", "version": "0.0.0", "dependencies": deps }), + ); + let lock = serde_json::json!({ + "name": "order-test", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": lock_packages, + }); + std::fs::write( + root.join("package-lock.json"), + serde_json::to_vec_pretty(&lock).unwrap(), + ) + .unwrap(); + let socket = root.join(".socket"); + std::fs::create_dir_all(socket.join("blobs")).unwrap(); + std::fs::write( + socket.join("manifest.json"), + serde_json::to_vec_pretty(&serde_json::json!({ "patches": patches })).unwrap(), + ) + .unwrap(); + std::fs::write(socket.join("blobs").join(git_sha256(AFTER)), AFTER).unwrap(); + + let out = Command::new(binary()) + .args(["vendor", "--json", "--vendor-source", "build"]) + .current_dir(root) + .env("SOCKET_TELEMETRY_DISABLED", "1") + .output() + .expect("run vendor"); + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + let v: serde_json::Value = serde_json::from_str(stdout.trim()).unwrap_or_else(|e| { + panic!( + "vendor --json must emit JSON: {e}\n{stdout}\n{}", + String::from_utf8_lossy(&out.stderr) + ) + }); + assert_ne!( + out.status.code(), + Some(0), + "the tampered entry fails: {v:#}" + ); + let events = v["events"].as_array().unwrap(); + let has = |purl: &str, action: &str, code: Option<&str>| { + events.iter().any(|e| { + e["purl"] == purl && e["action"] == action && code.is_none_or(|c| e["errorCode"] == c) + }) + }; + for name in ["pa", "pc", "pe"] { + let purl = format!("pkg:npm/{name}@1.0.0"); + assert!(has(&purl, "applied", None), "{purl}: {v:#}"); + assert!( + events + .iter() + .any(|e| e["purl"] == purl.as_str() && e["errorCode"] == "vendor_fetched_missing"), + "{purl}: {v:#}" + ); + assert!(root + .join(format!( + ".socket/vendor/npm/{}/{name}-1.0.0.tgz", + patches[&purl]["uuid"].as_str().unwrap() + )) + .is_file()); + } + assert!( + has("pkg:npm/pb@1.0.0", "failed", Some("vendor_fetch_failed")), + "{v:#}" + ); + assert!( + events + .iter() + .any(|e| e["purl"] == "pkg:npm/pd@1.0.0" + && e["errorCode"] == "vendor_fetch_unverifiable"), + "{v:#}" + ); + assert!( + !has("pkg:npm/pb@1.0.0", "skipped", Some("package_not_installed")), + "no duplicate not-installed skip for a failed fetch: {v:#}" + ); + // Every package's fetch-phase outcome is reported exactly once. + let fetch_phase: Vec<&str> = events + .iter() + .filter(|e| { + matches!( + e["errorCode"].as_str(), + Some("vendor_fetched_missing" | "vendor_fetch_unverifiable") + ) || (e["action"] == "failed" && e["errorCode"] == "vendor_fetch_failed") + }) + .map(|e| e["purl"].as_str().unwrap()) + .collect(); + assert_eq!(fetch_phase.len(), 5, "{v:#}"); + assert!(!root.join("node_modules").exists()); + let requests = mock.received_requests().await.unwrap(); + assert_eq!( + requests.len(), + 4, + "one GET per fetchable entry (the integrity-less one is refused \ + before the network): {requests:?}" + ); +} diff --git a/crates/socket-patch-cli/tests/vendor_rerun_no_network_e2e.rs b/crates/socket-patch-cli/tests/vendor_rerun_no_network_e2e.rs new file mode 100644 index 000000000..10efe03bd --- /dev/null +++ b/crates/socket-patch-cli/tests/vendor_rerun_no_network_e2e.rs @@ -0,0 +1,1248 @@ +//! An already-vendored project re-runs `vendor` with no network. +//! +//! The fresh-clone case: `.socket/vendor/` and the wired lockfile are +//! committed, the package itself is not installed (no `bundle install`, no +//! venv, an empty module/registry cache). The missing purl used to go +//! through the pristine-source ladder BEFORE the backend's in-sync check — +//! recovering the pre-vendor registry resolution from the ledger and +//! downloading the pristine artifact just to hand the backend a tree it +//! never reads on that path. With no network the download failed and so +//! did the run, for every already-vendored pypi, cargo, go and +//! lockfile-only gem package. +//! +//! The ladder now defers that download to the backend branch that reads +//! the tree, for a purl whose ledger entry records the record's patch uuid +//! and whose committed artifact is on disk. The backend's hot path answers +//! from the committed bytes, so the re-run is green (`already_vendored`), +//! makes no registry request, and no longer reports a +//! `vendor_fetched_missing` fetch it did not need — both with the registry +//! unreachable and under `--offline`. +//! +//! Hermetic: every registry base and the patch API point at a +//! guaranteed-dead local endpoint, and patch staging reads `.socket/blobs`. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use sha2::{Digest, Sha256}; + +fn binary() -> PathBuf { + env!("CARGO_BIN_EXE_socket-patch").into() +} + +fn git_sha256(content: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(format!("blob {}\0", content.len()).as_bytes()); + hasher.update(content); + hex::encode(hasher.finalize()) +} + +/// A guaranteed-unreachable local endpoint: bind an ephemeral port, then +/// release it, so every request fails fast with connection-refused. +fn dead_endpoint() -> String { + let port = std::net::TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port(); + format!("http://127.0.0.1:{port}") +} + +/// `.socket/manifest.json` with one patch and its after-blob. +fn write_manifest(root: &Path, purl: &str, uuid: &str, file: &str, before: &[u8], after: &[u8]) { + let socket = root.join(".socket"); + std::fs::create_dir_all(socket.join("blobs")).unwrap(); + let manifest = serde_json::json!({ + "patches": { + purl: { + "uuid": uuid, + "exportedAt": "2026-01-01T00:00:00Z", + "files": { + file: { + "beforeHash": git_sha256(before), + "afterHash": git_sha256(after), + } + }, + "vulnerabilities": {}, + "description": "synthetic offline re-run patch", + "license": "MIT", + "tier": "free" + } + } + }); + std::fs::write( + socket.join("manifest.json"), + serde_json::to_vec_pretty(&manifest).unwrap(), + ) + .unwrap(); + std::fs::write(socket.join("blobs").join(git_sha256(after)), after).unwrap(); +} + +/// `vendor --json --vendor-source build` through the built binary with +/// every ambient `SOCKET_*` var scrubbed, the API and every registry base +/// pointed at `dead`, and `env` on top. +fn run_vendor( + root: &Path, + dead: &str, + extra: &[&str], + env: &[(&str, &str)], +) -> (i32, serde_json::Value, String) { + let mut cmd = Command::new(binary()); + cmd.args([ + "vendor", + "--json", + "--vendor-source", + "build", + "--api-url", + dead, + "--proxy-url", + dead, + "--api-token", + "fake-token", + "--org", + "test-org", + ]) + .args(extra) + .current_dir(root); + for (key, _) in std::env::vars() { + if key.starts_with("SOCKET_") && key != "SOCKET_NO_CONFIG" { + cmd.env_remove(key); + } + } + for var in ["VIRTUAL_ENV", "CONDA_PREFIX", "BUNDLE_PATH", "GEM_HOME"] { + cmd.env_remove(var); + } + cmd.env("SOCKET_TELEMETRY_DISABLED", "1") + .env("SOCKET_CRATES_REGISTRY", dead) + .env("SOCKET_GOPROXY", dead) + .env("SOCKET_PYPI_JSON_API", dead) + .env("SOCKET_NPM_REGISTRY", dead) + .env("GOFLAGS", "-mod=mod") + .envs(env.iter().copied()); + let out = cmd.output().expect("run socket-patch vendor"); + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); + let v: serde_json::Value = serde_json::from_str(stdout.trim()) + .unwrap_or_else(|e| panic!("vendor --json must emit JSON: {e}\n{stdout}\n{stderr}")); + (out.status.code().unwrap_or(-1), v, stderr) +} + +/// `(action, errorCode)` of every event for `purl`, in order. +fn purl_events<'a>(v: &'a serde_json::Value, purl: &str) -> Vec<(&'a str, &'a str)> { + v["events"] + .as_array() + .expect("events array") + .iter() + .filter(|e| e["purl"] == purl) + .map(|e| { + ( + e["action"].as_str().unwrap_or_default(), + e["errorCode"].as_str().unwrap_or_default(), + ) + }) + .collect() +} + +/// Vendor with the package installed, take the installed copy away, then +/// re-run twice — registry unreachable, and `--offline` — and require the +/// in-sync outcome with nothing fetched and nothing rewritten. +fn assert_rerun_green_without_network( + root: &Path, + purl: &str, + uninstall: impl FnOnce(&Path), + env: &[(&str, &str)], +) { + let dead = dead_endpoint(); + let (code, v, stderr) = run_vendor(root, &dead, &[], env); + assert_eq!( + code, 0, + "run 1 vendors the installed package: {v:#}\n{stderr}" + ); + assert!( + purl_events(&v, purl).contains(&("applied", "")), + "run 1 must vendor {purl}: {v:#}" + ); + let snapshot = snapshot_tree(root); + + // Fresh clone: the committed artifact + wired lock, nothing installed. + uninstall(root); + let snapshot_after_uninstall = snapshot_tree(root); + + for extra in [&[][..], &["--offline"][..]] { + let (code, v, stderr) = run_vendor(root, &dead, extra, env); + assert_eq!( + code, 0, + "an in-sync re-run needs no network ({extra:?}): {v:#}\n{stderr}" + ); + assert_eq!(v["status"], "success", "{extra:?}: {v:#}"); + assert_eq!( + purl_events(&v, purl), + vec![("skipped", "already_vendored")], + "the hot path answers from the committed artifact; no fetch is \ + attempted or reported ({extra:?}): {v:#}" + ); + assert_eq!( + snapshot_tree(root), + snapshot_after_uninstall, + "an in-sync re-run writes nothing ({extra:?})" + ); + } + // The run-1 artifact and wiring are what the re-runs found in place. + for (rel, bytes) in &snapshot { + if rel.starts_with(".socket/") { + assert_eq!( + std::fs::read(root.join(rel)).ok().as_ref(), + Some(bytes), + "{rel} changed across the re-runs" + ); + } + } +} + +/// Every regular file under `root` (relative path → bytes), sorted. +fn snapshot_tree(root: &Path) -> Vec<(String, Vec)> { + let mut out = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + for entry in std::fs::read_dir(&dir).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + let ft = entry.file_type().unwrap(); + if ft.is_dir() { + stack.push(path); + } else if ft.is_file() { + let rel = path + .strip_prefix(root) + .unwrap() + .to_string_lossy() + .replace('\\', "/"); + out.push((rel, std::fs::read(&path).unwrap())); + } + } + } + out.sort(); + out +} + +// ── pypi ──────────────────────────────────────────────────────────────── + +const SIX_PURL: &str = "pkg:pypi/six@1.16.0"; + +/// A requirements project with `six` installed in `.venv` and a patch for +/// it in the manifest. +fn write_six_project(root: &Path, uuid: &str) { + const PURL: &str = SIX_PURL; + const ORIG: &[u8] = b"# six\nVERSION = '1.16.0'\n"; + const PATCHED: &[u8] = b"# six\nVERSION = '1.16.0'\nSAFE = True\n"; + // A hash-pinned requirement: the ledger-recovered pre-vendor line is + // then fetchable, so the old ladder really went to the registry. + std::fs::write( + root.join("requirements.txt"), + format!("six==1.16.0 --hash=sha256:{}\n", "a".repeat(64)), + ) + .unwrap(); + let sp = if cfg!(windows) { + root.join(".venv/Lib/site-packages") + } else { + root.join(".venv/lib/python3.12/site-packages") + }; + let di = sp.join("six-1.16.0.dist-info"); + std::fs::create_dir_all(&di).unwrap(); + std::fs::write(sp.join("six.py"), ORIG).unwrap(); + std::fs::write( + di.join("METADATA"), + "Metadata-Version: 2.1\nName: six\nVersion: 1.16.0\n\nbody\n", + ) + .unwrap(); + std::fs::write( + di.join("WHEEL"), + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py2-none-any\nTag: py3-none-any\n", + ) + .unwrap(); + std::fs::write( + di.join("RECORD"), + "six.py,sha256=AAAA,20\nsix-1.16.0.dist-info/METADATA,,\n\ + six-1.16.0.dist-info/WHEEL,,\nsix-1.16.0.dist-info/RECORD,,\n", + ) + .unwrap(); + write_manifest(root, PURL, uuid, "six.py", ORIG, PATCHED); +} + +#[test] +fn pypi_rerun_without_network_is_in_sync() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + write_six_project(root, "2b1f6c1e-8d3a-4f6b-9c2d-7e5a9b1c3d01"); + assert_rerun_green_without_network( + root, + SIX_PURL, + |root| std::fs::remove_dir_all(root.join(".venv")).unwrap(), + &[], + ); +} + +/// The deferral trusts a committed FILE artifact only while it hashes to +/// its ledger pin (the pypi in-sync check looks only for the wheel's +/// presence). A tampered wheel on a fresh clone keeps the eager ladder, so +/// with no network the run fails as it always did instead of calling the +/// garbage in sync. +#[test] +fn pypi_rerun_over_a_tampered_wheel_is_not_called_in_sync() { + const UUID: &str = "2b1f6c1e-8d3a-4f6b-9c2d-7e5a9b1c3d09"; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + write_six_project(root, UUID); + let dead = dead_endpoint(); + let (code, v, stderr) = run_vendor(root, &dead, &[], &[]); + assert_eq!(code, 0, "{v:#}\n{stderr}"); + std::fs::remove_dir_all(root.join(".venv")).unwrap(); + let uuid_dir = root.join(format!(".socket/vendor/pypi/{UUID}")); + let wheel = std::fs::read_dir(&uuid_dir) + .unwrap() + .filter_map(Result::ok) + .map(|e| e.path()) + .find(|p| p.extension().is_some_and(|x| x == "whl")) + .expect("the committed wheel"); + std::fs::write(&wheel, b"garbage").unwrap(); + + for extra in [&[][..], &["--offline"][..]] { + let (code, v, stderr) = run_vendor(root, &dead, extra, &[]); + assert_eq!(code, 1, "{extra:?}: {v:#}\n{stderr}"); + // Exactly what the eager ladder reports for it (the integrated + // base binary's events on this fixture). + let expected = if extra.is_empty() { + vec![ + ("skipped", "vendor_fetch_unverifiable"), + ("skipped", "package_not_installed"), + ] + } else { + vec![("skipped", "package_not_installed")] + }; + assert_eq!(purl_events(&v, SIX_PURL), expected, "{extra:?}: {v:#}"); + } + assert_eq!(std::fs::read(&wheel).unwrap(), b"garbage"); +} + +// ── cargo ─────────────────────────────────────────────────────────────── + +#[test] +fn cargo_rerun_without_network_is_in_sync() { + const PURL: &str = "pkg:cargo/cfg-if@1.0.4"; + const UUID: &str = "2b1f6c1e-8d3a-4f6b-9c2d-7e5a9b1c3d02"; + const PRISTINE: &[u8] = b"pub fn cfg() {}\n"; + const PATCHED: &[u8] = b"pub fn cfg() { /* patched */ }\n"; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + let cargo_home = tmp.path().join("cargo-home"); + let krate = cargo_home.join("registry/src/index.crates.io-6f17d22bba15001f/cfg-if-1.0.4"); + std::fs::create_dir_all(krate.join("src")).unwrap(); + std::fs::write(krate.join("src/lib.rs"), PRISTINE).unwrap(); + std::fs::write( + krate.join("Cargo.toml"), + "[package]\nname = \"cfg-if\"\nversion = \"1.0.4\"\n", + ) + .unwrap(); + std::fs::write(krate.join(".cargo-checksum.json"), "{\"files\":{}}").unwrap(); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write( + root.join("Cargo.toml"), + "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[dependencies]\ncfg-if = \"1\"\n", + ) + .unwrap(); + std::fs::write( + root.join("Cargo.lock"), + format!( + "# This file is automatically @generated by Cargo.\n\ + # It is not intended for manual editing.\n\ + version = 4\n\n\ + [[package]]\nname = \"app\"\nversion = \"0.1.0\"\n\ + dependencies = [\n \"cfg-if\",\n]\n\n\ + [[package]]\nname = \"cfg-if\"\nversion = \"1.0.4\"\n\ + source = \"registry+https://github.com/rust-lang/crates.io-index\"\n\ + checksum = \"{}\"\n", + "9".repeat(64) + ), + ) + .unwrap(); + write_manifest(&root, PURL, UUID, "package/src/lib.rs", PRISTINE, PATCHED); + + let home = cargo_home.to_string_lossy().into_owned(); + assert_rerun_green_without_network( + &root, + PURL, + |_| std::fs::remove_dir_all(&krate).unwrap(), + &[("CARGO_HOME", home.as_str())], + ); +} + +// ── golang ────────────────────────────────────────────────────────────── + +/// The go.sum `h1:` dirhash of a module zip holding `files` under the +/// `@/` prefix. +fn go_h1(module: &str, version: &str, files: &[(&str, &[u8])]) -> String { + use base64::Engine as _; + let mut lines: Vec = files + .iter() + .map(|(name, bytes)| { + format!( + "{} {module}@{version}/{name}\n", + hex::encode(Sha256::digest(bytes)) + ) + }) + .collect(); + lines.sort(); + let summary = Sha256::digest(lines.concat().as_bytes()); + format!( + "h1:{}", + base64::engine::general_purpose::STANDARD.encode(summary) + ) +} + +#[test] +fn golang_rerun_without_network_is_in_sync() { + const MODULE: &str = "github.com/foo/bar"; + const VERSION: &str = "v1.4.2"; + const PURL: &str = "pkg:golang/github.com/foo/bar@v1.4.2"; + const UUID: &str = "2b1f6c1e-8d3a-4f6b-9c2d-7e5a9b1c3d03"; + const PRISTINE: &[u8] = b"package bar\n\nfunc Hello() string { return \"hi\" }\n"; + const PATCHED: &[u8] = b"package bar\n\nfunc Hello() string { return \"patched\" }\n"; + const MOD: &[u8] = b"module github.com/foo/bar\n\ngo 1.21\n"; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + let modcache = tmp.path().join("modcache"); + let module_dir = modcache.join(format!("{MODULE}@{VERSION}")); + std::fs::create_dir_all(&module_dir).unwrap(); + std::fs::write(module_dir.join("bar.go"), PRISTINE).unwrap(); + std::fs::write(module_dir.join("go.mod"), MOD).unwrap(); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write( + root.join("go.mod"), + format!("module example.com/app\n\ngo 1.21\n\nrequire {MODULE} {VERSION}\n"), + ) + .unwrap(); + // A verifiable go.sum pin: the old ladder fetched it from GOPROXY. + std::fs::write( + root.join("go.sum"), + format!( + "{MODULE} {VERSION} {}\n", + go_h1(MODULE, VERSION, &[("bar.go", PRISTINE), ("go.mod", MOD)]) + ), + ) + .unwrap(); + write_manifest(&root, PURL, UUID, "package/bar.go", PRISTINE, PATCHED); + + let cache = modcache.to_string_lossy().into_owned(); + assert_rerun_green_without_network( + &root, + PURL, + |_| std::fs::remove_dir_all(&module_dir).unwrap(), + &[("GOMODCACHE", cache.as_str())], + ); +} + +// ── gem (lockfile-only) ───────────────────────────────────────────────── + +const GEM_NAME: &str = "socketfixturegem"; +const GEM_VERSION: &str = "1.0.0"; +const GEM_PURL: &str = "pkg:gem/socketfixturegem@1.0.0"; +const GEM_LIB: &str = "lib/socketfixturegem.rb"; +const GEM_PRISTINE: &[u8] = b"module SocketFixtureGem; VERSION = '1.0.0'; end\n"; +const GEM_PATCHED: &[u8] = b"module SocketFixtureGem; VERSION = '1.0.0'; SAFE = true; end\n"; + +/// Gemfile + a bundler >= 2.6 Gemfile.lock (CHECKSUMS pin, so the gem is +/// fetchable from `remote`) + the manifest. +fn write_gem_project(root: &Path, remote: &str, uuid: &str) { + write_gem_project_with(root, remote, uuid, &"b".repeat(64)); +} + +/// [`write_gem_project`] with the lock's `CHECKSUMS` sha256 given. +fn write_gem_project_with(root: &Path, remote: &str, uuid: &str, checksum: &str) { + std::fs::write( + root.join("Gemfile"), + format!("source \"{remote}\"\ngem \"{GEM_NAME}\"\n"), + ) + .unwrap(); + std::fs::write( + root.join("Gemfile.lock"), + format!( + "GEM\n remote: {remote}\n specs:\n {GEM_NAME} ({GEM_VERSION})\n\n\ + PLATFORMS\n ruby\n\n\ + DEPENDENCIES\n {GEM_NAME}\n\n\ + CHECKSUMS\n {GEM_NAME} ({GEM_VERSION}) sha256={}\n\n\ + BUNDLED WITH\n 2.6.2\n", + checksum + ), + ) + .unwrap(); + write_manifest(root, GEM_PURL, uuid, GEM_LIB, GEM_PRISTINE, GEM_PATCHED); +} + +/// A `bundle install --path vendor/bundle` deployment of the gem: the +/// unpacked gem and the eval-able stub rubygems writes beside it. +fn install_gem(root: &Path) { + let leaf = format!("{GEM_NAME}-{GEM_VERSION}"); + let bundle = root.join("vendor").join("bundle"); + let gem_dir = bundle.join("gems").join(&leaf); + std::fs::create_dir_all(gem_dir.join("lib")).unwrap(); + std::fs::write(gem_dir.join(GEM_LIB), GEM_PRISTINE).unwrap(); + let specs = bundle.join("specifications"); + std::fs::create_dir_all(&specs).unwrap(); + std::fs::write( + specs.join(format!("{leaf}.gemspec")), + format!( + "Gem::Specification.new do |s|\n s.name = \"{GEM_NAME}\"\n \ + s.version = \"{GEM_VERSION}\"\n s.summary = \"a synthetic fixture gem\"\n \ + s.authors = [\"Socket\"]\n s.require_paths = [\"lib\"]\nend\n" + ), + ) + .unwrap(); +} + +#[test] +fn gem_rerun_without_network_is_in_sync() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + let remote = dead_endpoint(); + write_gem_project(root, &remote, "2b1f6c1e-8d3a-4f6b-9c2d-7e5a9b1c3d04"); + install_gem(root); + + assert_rerun_green_without_network( + root, + GEM_PURL, + |root| std::fs::remove_dir_all(root.join("vendor")).unwrap(), + &[], + ); +} + +/// A not-installed gem that a lock CAN verify, and that no ledger entry +/// covers, cannot be vendored by a local build (no stub gemspec comes with +/// a downloaded `.gem`): build mode refuses it `gem_spec_missing` BEFORE +/// downloading it, instead of downloading it and then refusing. +#[tokio::test] +async fn gem_build_mode_refuses_a_lockfile_only_gem_before_downloading_it() { + use wiremock::{matchers::method, Mock, MockServer, ResponseTemplate}; + let registry = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(b"not a gem".to_vec())) + .mount(®istry) + .await; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + write_gem_project( + root, + ®istry.uri(), + "2b1f6c1e-8d3a-4f6b-9c2d-7e5a9b1c3d05", + ); + + let (code, v, stderr) = run_vendor(root, &dead_endpoint(), &[], &[]); + + assert_eq!(code, 1, "{v:#}\n{stderr}"); + assert!( + registry + .received_requests() + .await + .unwrap_or_default() + .is_empty(), + "build mode cannot use a fetched gem, so it must not download one" + ); + assert_eq!( + purl_events(&v, GEM_PURL), + vec![("failed", "gem_spec_missing")], + "{v:#}" + ); + assert!(!root.join(".socket/vendor").exists(), "nothing is written"); +} + +/// A `.gem`: an uncompressed outer tar holding `metadata.gz` and a +/// `data.tar.gz` of `data_files` at its root. +fn make_gem(data_files: &[(&str, &[u8])]) -> Vec { + use std::io::Write as _; + let gz = |bytes: &[u8]| { + let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + enc.write_all(bytes).unwrap(); + enc.finish().unwrap() + }; + let tar = |files: &[(&str, &[u8])]| { + let mut builder = tar::Builder::new(Vec::new()); + for (rel, content) in files { + let mut header = tar::Header::new_gnu(); + header.set_size(content.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder.append_data(&mut header, rel, *content).unwrap(); + } + builder.into_inner().unwrap() + }; + let data = gz(&tar(data_files)); + let metadata = + gz(format!("--- !ruby/object:Gem::Specification\nname: {GEM_NAME}\n").as_bytes()); + tar(&[("metadata.gz", &metadata), ("data.tar.gz", &data)]) +} + +/// The build-mode refusal happens before the download only on a wet run: a +/// `--dry-run` never refused, and still fetches the gem and previews it +/// (`vendor_fetched_missing` + `verified`, exit 0) as it always did. +#[tokio::test] +async fn gem_build_mode_dry_run_still_fetches_and_previews() { + use wiremock::{matchers::method, Mock, MockServer, ResponseTemplate}; + let gem = make_gem(&[(GEM_LIB, GEM_PRISTINE)]); + let checksum = hex::encode(Sha256::digest(&gem)); + let registry = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(gem)) + .mount(®istry) + .await; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + write_gem_project_with( + root, + ®istry.uri(), + "2b1f6c1e-8d3a-4f6b-9c2d-7e5a9b1c3d0a", + &checksum, + ); + + let (code, v, stderr) = run_vendor(root, &dead_endpoint(), &["--dry-run"], &[]); + + assert_eq!(code, 0, "{v:#}\n{stderr}"); + assert_eq!( + purl_events(&v, GEM_PURL), + vec![("skipped", "vendor_fetched_missing"), ("verified", "")], + "{v:#}" + ); + assert!( + !root.join(".socket/vendor").exists(), + "a dry run writes nothing" + ); +} + +/// The deferral only moves the download; it never hides one the backend +/// needs. A committed copy that no longer verifies is rebuilt from the +/// pristine source, so that re-run fetches — and with the registry +/// unreachable reports the same `vendor_fetch_failed` the eager ladder +/// did, leaving the tampered copy for `repair`; under `--offline` the same +/// calm `package_not_installed` skip. +#[test] +fn cargo_rerun_over_a_drifted_copy_still_fetches_and_reports_it() { + const PURL: &str = "pkg:cargo/cfg-if@1.0.4"; + const UUID: &str = "2b1f6c1e-8d3a-4f6b-9c2d-7e5a9b1c3d06"; + const PRISTINE: &[u8] = b"pub fn cfg() {}\n"; + const PATCHED: &[u8] = b"pub fn cfg() { /* patched */ }\n"; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + let cargo_home = tmp.path().join("cargo-home"); + let krate = cargo_home.join("registry/src/index.crates.io-6f17d22bba15001f/cfg-if-1.0.4"); + std::fs::create_dir_all(krate.join("src")).unwrap(); + std::fs::write(krate.join("src/lib.rs"), PRISTINE).unwrap(); + std::fs::write( + krate.join("Cargo.toml"), + "[package]\nname = \"cfg-if\"\nversion = \"1.0.4\"\n", + ) + .unwrap(); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write( + root.join("Cargo.toml"), + "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[dependencies]\ncfg-if = \"1\"\n", + ) + .unwrap(); + std::fs::write( + root.join("Cargo.lock"), + format!( + "version = 4\n\n\ + [[package]]\nname = \"app\"\nversion = \"0.1.0\"\n\ + dependencies = [\n \"cfg-if\",\n]\n\n\ + [[package]]\nname = \"cfg-if\"\nversion = \"1.0.4\"\n\ + source = \"registry+https://github.com/rust-lang/crates.io-index\"\n\ + checksum = \"{}\"\n", + "9".repeat(64) + ), + ) + .unwrap(); + write_manifest(&root, PURL, UUID, "package/src/lib.rs", PRISTINE, PATCHED); + let home = cargo_home.to_string_lossy().into_owned(); + let env = [("CARGO_HOME", home.as_str())]; + let dead = dead_endpoint(); + + let (code, v, stderr) = run_vendor(&root, &dead, &[], &env); + assert_eq!(code, 0, "{v:#}\n{stderr}"); + std::fs::remove_dir_all(&krate).unwrap(); + let copy_lib = root.join(format!( + ".socket/vendor/cargo/{UUID}/cfg-if-1.0.4/src/lib.rs" + )); + std::fs::write(©_lib, b"tampered\n").unwrap(); + + let (code, v, stderr) = run_vendor(&root, &dead, &[], &env); + assert_eq!(code, 1, "{v:#}\n{stderr}"); + assert_eq!( + purl_events(&v, PURL), + vec![("failed", "vendor_fetch_failed")], + "{v:#}" + ); + assert_eq!(std::fs::read(©_lib).unwrap(), b"tampered\n"); + + let (code, v, stderr) = run_vendor(&root, &dead, &["--offline"], &env); + assert_eq!(code, 1, "{v:#}\n{stderr}"); + assert_eq!( + purl_events(&v, PURL), + vec![("skipped", "package_not_installed")], + "{v:#}" + ); +} + +// ── cargo: the service path needs no pristine source ───────────────────── + +/// A `.crate`: a tar.gz with a single `{prefix}/` top-level dir. +fn make_crate(prefix: &str, files: &[(&str, &[u8])]) -> Vec { + use std::io::Write as _; + let mut builder = tar::Builder::new(Vec::new()); + for (rel, content) in files { + let mut header = tar::Header::new_gnu(); + header.set_size(content.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, format!("{prefix}/{rel}"), *content) + .unwrap(); + } + let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + enc.write_all(&builder.into_inner().unwrap()).unwrap(); + enc.finish().unwrap() +} + +/// A lockfile-only cargo crate the patch service serves prebuilt: the +/// backend reads the pristine source only once `cargo_service_copy` falls +/// back to the local build, so the registry download is deferred until +/// then — here, never. (The eager ladder downloaded and verified the +/// pristine `.crate` first, and reported `vendor_fetched_missing` for it.) +#[tokio::test] +async fn cargo_service_vendor_never_downloads_the_pristine_crate() { + use base64::Engine as _; + use sha2::Sha512; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + const PURL: &str = "pkg:cargo/cfg-if@1.0.4"; + const UUID: &str = "2b1f6c1e-8d3a-4f6b-9c2d-7e5a9b1c3d07"; + const PRISTINE: &[u8] = b"pub fn cfg() {}\n"; + const PATCHED: &[u8] = b"pub fn cfg() { /* patched */ }\n"; + const TOML: &[u8] = b"[package]\nname = \"cfg-if\"\nversion = \"1.0.4\"\n"; + + let registry = MockServer::start().await; + let pristine = make_crate( + "cfg-if-1.0.4", + &[("Cargo.toml", TOML), ("src/lib.rs", PRISTINE)], + ); + let checksum = hex::encode(Sha256::digest(&pristine)); + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(pristine)) + .mount(®istry) + .await; + + let api = MockServer::start().await; + let prebuilt = make_crate( + "cfg-if-1.0.4", + &[("Cargo.toml", TOML), ("src/lib.rs", PATCHED)], + ); + let sha512 = format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(Sha512::digest(&prebuilt)) + ); + let serve_path = format!("/patch/cargo/cfg-if/1.0.4/tok/{UUID}/cfg-if-1.0.4.crate"); + let serve_url = format!("{}{serve_path}", api.uri()); + Mock::given(method("POST")) + .and(path("/v0/orgs/acme/patches/package")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "results": { UUID: { + "status": "granted", "url": serve_url, "purl": PURL, + "artifacts": [{ "kind": "tarball", "url": serve_url, + "integrity": { "sha512": sha512 } }] + }} + }))) + .mount(&api) + .await; + Mock::given(method("GET")) + .and(path(serve_path)) + .respond_with(ResponseTemplate::new(200).set_body_bytes(prebuilt)) + .mount(&api) + .await; + + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write( + root.join("Cargo.toml"), + "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[dependencies]\ncfg-if = \"1\"\n", + ) + .unwrap(); + std::fs::write( + root.join("Cargo.lock"), + format!( + "version = 4\n\n\ + [[package]]\nname = \"app\"\nversion = \"0.1.0\"\n\ + dependencies = [\n \"cfg-if\",\n]\n\n\ + [[package]]\nname = \"cfg-if\"\nversion = \"1.0.4\"\n\ + source = \"registry+https://github.com/rust-lang/crates.io-index\"\n\ + checksum = \"{checksum}\"\n" + ), + ) + .unwrap(); + write_manifest(&root, PURL, UUID, "package/src/lib.rs", PRISTINE, PATCHED); + let empty_home = tmp.path().join("cargo-home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let mut cmd = Command::new(binary()); + cmd.args([ + "vendor", + "--json", + "--vendor-source", + "auto", + "--api-url", + &api.uri(), + "--api-token", + "sktsec_placeholder_value_for_tests_api", + "--org", + "acme", + ]) + .current_dir(&root); + for (key, _) in std::env::vars() { + if key.starts_with("SOCKET_") && key != "SOCKET_NO_CONFIG" { + cmd.env_remove(key); + } + } + let out = cmd + .env("SOCKET_TELEMETRY_DISABLED", "1") + .env("SOCKET_CRATES_REGISTRY", registry.uri()) + .env("CARGO_HOME", &empty_home) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&out.stdout); + let v: serde_json::Value = serde_json::from_str(stdout.trim()) + .unwrap_or_else(|e| panic!("{e}: {stdout}\n{}", String::from_utf8_lossy(&out.stderr))); + + assert_eq!(out.status.code(), Some(0), "{v:#}"); + let events = purl_events(&v, PURL); + assert!(events.contains(&("applied", "")), "{v:#}"); + assert!( + !events.contains(&("skipped", "vendor_fetched_missing")), + "no pristine fetch happened, so none is reported: {v:#}" + ); + assert!( + registry + .received_requests() + .await + .unwrap_or_default() + .is_empty(), + "the service served the crate; the registry was never asked" + ); + assert_eq!( + std::fs::read(root.join(format!( + ".socket/vendor/cargo/{UUID}/cfg-if-1.0.4/src/lib.rs" + ))) + .unwrap(), + PATCHED + ); +} + +/// The deferral behind the service moves only WHEN the pristine crate is +/// downloaded, never WHICH crates are vendored. A crate the lock resolves +/// from a git fork or a custom registry (no crates.io checksum) has no +/// verifiable pristine source, so the eager ladder refuses it +/// `vendor_fetch_unverifiable` + `package_not_installed` and touches +/// nothing — the crates.io patch the service serves must not replace it. +#[tokio::test] +async fn cargo_service_never_vendors_a_git_or_custom_registry_crate() { + use base64::Engine as _; + use sha2::Sha512; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + const PURL: &str = "pkg:cargo/cfg-if@1.0.4"; + const UUID: &str = "2b1f6c1e-8d3a-4f6b-9c2d-7e5a9b1c3d08"; + const PRISTINE: &[u8] = b"pub fn cfg() {}\n"; + const PATCHED: &[u8] = b"pub fn cfg() { /* patched */ }\n"; + const TOML: &[u8] = b"[package]\nname = \"cfg-if\"\nversion = \"1.0.4\"\n"; + + let registry = MockServer::start().await; + let api = MockServer::start().await; + let prebuilt = make_crate( + "cfg-if-1.0.4", + &[("Cargo.toml", TOML), ("src/lib.rs", PATCHED)], + ); + let sha512 = format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(Sha512::digest(&prebuilt)) + ); + let serve_path = format!("/patch/cargo/cfg-if/1.0.4/tok/{UUID}/cfg-if-1.0.4.crate"); + let serve_url = format!("{}{serve_path}", api.uri()); + Mock::given(method("POST")) + .and(path("/v0/orgs/acme/patches/package")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "results": { UUID: { + "status": "granted", "url": serve_url, "purl": PURL, + "artifacts": [{ "kind": "tarball", "url": serve_url, + "integrity": { "sha512": sha512 } }] + }} + }))) + .mount(&api) + .await; + Mock::given(method("GET")) + .and(path(serve_path)) + .respond_with(ResponseTemplate::new(200).set_body_bytes(prebuilt)) + .mount(&api) + .await; + + for source in [ + "git+https://example.com/fork/cfg-if#abc123", + "registry+https://my-registry.example/index", + ] { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write( + root.join("Cargo.toml"), + "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n\ + [dependencies]\ncfg-if = { git = \"https://example.com/fork/cfg-if\" }\n", + ) + .unwrap(); + let lock = format!( + "version = 4\n\n\ + [[package]]\nname = \"app\"\nversion = \"0.1.0\"\n\ + dependencies = [\n \"cfg-if\",\n]\n\n\ + [[package]]\nname = \"cfg-if\"\nversion = \"1.0.4\"\n\ + source = \"{source}\"\n" + ); + std::fs::write(root.join("Cargo.lock"), &lock).unwrap(); + write_manifest(&root, PURL, UUID, "package/src/lib.rs", PRISTINE, PATCHED); + let empty_home = tmp.path().join("cargo-home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let mut cmd = Command::new(binary()); + cmd.args([ + "vendor", + "--json", + "--vendor-source", + "auto", + "--api-url", + &api.uri(), + "--api-token", + "sktsec_placeholder_value_for_tests_api", + "--org", + "acme", + ]) + .current_dir(&root); + for (key, _) in std::env::vars() { + if key.starts_with("SOCKET_") && key != "SOCKET_NO_CONFIG" { + cmd.env_remove(key); + } + } + let out = cmd + .env("SOCKET_TELEMETRY_DISABLED", "1") + .env("SOCKET_CRATES_REGISTRY", registry.uri()) + .env("CARGO_HOME", &empty_home) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&out.stdout); + let v: serde_json::Value = serde_json::from_str(stdout.trim()) + .unwrap_or_else(|e| panic!("{e}: {stdout}\n{}", String::from_utf8_lossy(&out.stderr))); + + assert_eq!(out.status.code(), Some(1), "{source}: {v:#}"); + assert_eq!( + purl_events(&v, PURL), + vec![ + ("skipped", "vendor_fetch_unverifiable"), + ("skipped", "package_not_installed"), + ], + "{source}: {v:#}" + ); + assert_eq!( + std::fs::read_to_string(root.join("Cargo.lock")).unwrap(), + lock, + "{source}: the lock keeps its provenance" + ); + assert!( + !root.join(".cargo").exists(), + "{source}: no [patch] written" + ); + assert!( + !root.join(".socket/vendor").exists(), + "{source}: nothing vendored" + ); + } + assert!( + registry + .received_requests() + .await + .unwrap_or_default() + .is_empty(), + "an unverifiable crate is never downloaded" + ); +} + +/// A deferred fetch that does run — a drifted committed copy rebuilt from a +/// reachable registry — reports its `vendor_fetched_missing` just ahead of +/// the package's own event, exactly where the eager ladder reported it, and +/// downloads the pristine crate once. +#[tokio::test] +async fn cargo_rerun_over_a_drifted_copy_reports_the_deferred_fetch_first() { + use wiremock::{matchers::method, Mock, MockServer, ResponseTemplate}; + const PURL: &str = "pkg:cargo/cfg-if@1.0.4"; + const UUID: &str = "2b1f6c1e-8d3a-4f6b-9c2d-7e5a9b1c3d0b"; + const PRISTINE: &[u8] = b"pub fn cfg() {}\n"; + const PATCHED: &[u8] = b"pub fn cfg() { /* patched */ }\n"; + const TOML: &[u8] = b"[package]\nname = \"cfg-if\"\nversion = \"1.0.4\"\n"; + let pristine = make_crate( + "cfg-if-1.0.4", + &[("Cargo.toml", TOML), ("src/lib.rs", PRISTINE)], + ); + let checksum = hex::encode(Sha256::digest(&pristine)); + let registry = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(pristine)) + .mount(®istry) + .await; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write( + root.join("Cargo.toml"), + "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[dependencies]\ncfg-if = \"1\"\n", + ) + .unwrap(); + std::fs::write( + root.join("Cargo.lock"), + format!( + "version = 4\n\n\ + [[package]]\nname = \"app\"\nversion = \"0.1.0\"\n\ + dependencies = [\n \"cfg-if\",\n]\n\n\ + [[package]]\nname = \"cfg-if\"\nversion = \"1.0.4\"\n\ + source = \"registry+https://github.com/rust-lang/crates.io-index\"\n\ + checksum = \"{checksum}\"\n" + ), + ) + .unwrap(); + write_manifest(&root, PURL, UUID, "package/src/lib.rs", PRISTINE, PATCHED); + let empty_home = tmp.path().join("cargo-home"); + std::fs::create_dir_all(&empty_home).unwrap(); + let home = empty_home.to_string_lossy().into_owned(); + let uri = registry.uri(); + let env = [ + ("CARGO_HOME", home.as_str()), + ("SOCKET_CRATES_REGISTRY", uri.as_str()), + ]; + let dead = dead_endpoint(); + let gets = || async { registry.received_requests().await.unwrap_or_default().len() }; + + let (code, v, stderr) = run_vendor(&root, &dead, &[], &env); + assert_eq!(code, 0, "{v:#}\n{stderr}"); + let copy_lib = root.join(format!( + ".socket/vendor/cargo/{UUID}/cfg-if-1.0.4/src/lib.rs" + )); + std::fs::write(©_lib, b"tampered\n").unwrap(); + let before = gets().await; + + let (code, v, stderr) = run_vendor(&root, &dead, &[], &env); + assert_eq!(code, 0, "{v:#}\n{stderr}"); + assert_eq!( + purl_events(&v, PURL), + vec![ + ("skipped", "vendor_fetched_missing"), + ("applied", ""), + ("skipped", "vendor_artifact_rebuilt"), + ], + "the fetch is reported first, then the package's own events: {v:#}" + ); + assert_eq!(gets().await, before + 1, "one pristine download"); + assert_eq!(std::fs::read(©_lib).unwrap(), PATCHED, "rebuilt"); +} + +/// A lockfile-only pnpm package its backend refuses on the lock text alone +/// (a peer-suffixed snapshot key it cannot rewire) is never fetched: the +/// pristine download is deferred to the backend, which refuses before it +/// reads anything, so the run makes no registry request and reports the +/// real reason. With the registry unreachable, the old eager fetch failed +/// first (`vendor_fetch_failed`) and hid it. +#[test] +fn a_lockfile_only_package_refused_on_lock_text_is_never_fetched() { + const UUID: &str = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; + const PURL: &str = "pkg:npm/pkg-b@1.0.0"; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + std::fs::write( + root.join("package.json"), + r#"{ "name": "t", "version": "0.0.0", "dependencies": { "pkg-b": "1.0.0" } }"#, + ) + .unwrap(); + let sri = format!("sha512-{}==", "A".repeat(86)); + std::fs::write( + root.join("pnpm-lock.yaml"), + format!( + "lockfileVersion: '9.0'\n\nsettings:\n autoInstallPeers: true\n \ + excludeLinksFromLockfile: false\n\nimporters:\n\n .:\n dependencies:\n \ + pkg-b:\n specifier: 1.0.0\n version: 1.0.0(peer-x@1.0.0)\n\n\ + packages:\n\n pkg-b@1.0.0:\n resolution: {{integrity: {sri}}}\n \ + peerDependencies:\n peer-x: '*'\n\nsnapshots:\n\n \ + pkg-b@1.0.0(peer-x@1.0.0): {{}}\n" + ), + ) + .unwrap(); + write_manifest( + root, + PURL, + UUID, + "package/index.js", + b"before\n", + b"after\n", + ); + let dead = dead_endpoint(); + + let (code, v, stderr) = run_vendor(root, &dead, &[], &[]); + assert_eq!(code, 1, "a refused package fails the run: {v:#}\n{stderr}"); + assert_eq!( + purl_events(&v, PURL), + vec![("failed", "vendor_lock_entry_unsupported")], + "the backend's own refusal, with no fetch before it: {v:#}\n{stderr}" + ); + assert!(!root.join(".socket/vendor").exists(), "nothing vendored"); +} + +/// The early deferral above is only for a package the pristine-source +/// ladder would really fetch (a verifiable lock resolution). A package the +/// lock does not resolve and nothing installed has no source at all: the +/// loop skips it `package_not_installed`, as it always did — never the +/// backend's lock refusal. pnpm (absent from the lock) and cargo (a crate +/// absent from Cargo.lock, and one at a version it does not lock). +#[test] +fn a_package_absent_from_the_lock_keeps_the_not_installed_skip() { + const UUID: &str = "cccccccc-cccc-4ccc-8ccc-cccccccccccc"; + let dead = dead_endpoint(); + + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + std::fs::write( + root.join("package.json"), + r#"{ "name": "t", "version": "0.0.0", "dependencies": {} }"#, + ) + .unwrap(); + std::fs::write( + root.join("pnpm-lock.yaml"), + "lockfileVersion: '9.0'\n\nsettings:\n autoInstallPeers: true\n \ + excludeLinksFromLockfile: false\n\nimporters:\n\n .: {}\n", + ) + .unwrap(); + const NPM_PURL: &str = "pkg:npm/pkg-z@1.0.0"; + write_manifest( + root, + NPM_PURL, + UUID, + "package/index.js", + b"before\n", + b"after\n", + ); + let (_code, v, stderr) = run_vendor(root, &dead, &[], &[]); + assert_eq!( + purl_events(&v, NPM_PURL), + vec![("skipped", "package_not_installed")], + "{v:#}\n{stderr}" + ); + + for purl in ["pkg:cargo/absent-crate@1.0.0", "pkg:cargo/cfg-if@9.9.9"] { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + let cargo_home = tmp.path().join("cargo-home"); + std::fs::create_dir_all(cargo_home.join("registry/src")).unwrap(); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write( + root.join("Cargo.toml"), + "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[dependencies]\ncfg-if = \"1\"\n", + ) + .unwrap(); + std::fs::write( + root.join("Cargo.lock"), + format!( + "version = 4\n\n\ + [[package]]\nname = \"app\"\nversion = \"0.1.0\"\n\ + dependencies = [\n \"cfg-if\",\n]\n\n\ + [[package]]\nname = \"cfg-if\"\nversion = \"1.0.4\"\n\ + source = \"registry+https://github.com/rust-lang/crates.io-index\"\n\ + checksum = \"{}\"\n", + "9".repeat(64) + ), + ) + .unwrap(); + write_manifest( + &root, + purl, + UUID, + "package/src/lib.rs", + b"before\n", + b"after\n", + ); + let home = cargo_home.to_string_lossy().into_owned(); + let (_code, v, stderr) = + run_vendor(&root, &dead, &[], &[("CARGO_HOME", home.as_str())]); + assert_eq!( + purl_events(&v, purl), + vec![("skipped", "package_not_installed")], + "{purl}: {v:#}\n{stderr}" + ); + } +} + +/// A `.socket-prestage` tree a crashed or interrupted run left +/// behind is swept at the start of the next wet vendor loop — whatever +/// that run vendors, `--offline` and `--vendor-source build` included — +/// while a `--dry-run` deletes nothing. +#[test] +fn a_stale_prestage_tree_is_swept_by_the_next_wet_run_only() { + const UUID: &str = "cccccccc-cccc-4ccc-8ccc-cccccccccccc"; + const OLD: &str = "dddddddd-dddd-4ddd-8ddd-dddddddddddd"; + let dead = dead_endpoint(); + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + std::fs::write( + root.join("package.json"), + r#"{ "name": "t", "version": "0.0.0", "dependencies": {} }"#, + ) + .unwrap(); + write_manifest( + root, + "pkg:npm/pkg-z@1.0.0", + UUID, + "package/index.js", + b"before\n", + b"after\n", + ); + let litter = [ + format!(".socket/vendor/cargo/{OLD}/foo-1.0.0.socket-prestage"), + format!(".socket/vendor/composer/{OLD}/psr/log@3.0.2.socket-prestage"), + ]; + let plant = || { + for dir in &litter { + std::fs::create_dir_all(root.join(dir).join("src")).unwrap(); + std::fs::write(root.join(dir).join("src/lib.rs"), b"stale").unwrap(); + } + }; + plant(); + + let (_code, v, stderr) = run_vendor(root, &dead, &["--dry-run"], &[]); + for dir in &litter { + assert!(root.join(dir).exists(), "a dry run deletes nothing: {dir}\n{v:#}\n{stderr}"); + } + assert!( + !v.to_string().contains("socket-prestage"), + "and reports nothing about it: {v:#}" + ); + + for extra in [&["--offline"][..], &[][..]] { + let (_code, v, stderr) = run_vendor(root, &dead, extra, &[]); + for dir in &litter { + assert!(!root.join(dir).exists(), "{extra:?} sweeps {dir}\n{v:#}\n{stderr}"); + } + assert!( + !root.join(format!(".socket/vendor/composer/{OLD}")).exists(), + "{extra:?}: the levels only the leftover kept alive go too" + ); + assert!( + !v.to_string().contains("socket-prestage"), + "the sweep is silent: {v:#}" + ); + plant(); + } +} diff --git a/crates/socket-patch-core/Cargo.toml b/crates/socket-patch-core/Cargo.toml index 26b97ca4f..98757345b 100644 --- a/crates/socket-patch-core/Cargo.toml +++ b/crates/socket-patch-core/Cargo.toml @@ -7,6 +7,15 @@ license.workspace = true repository.workspace = true readme = "README.md" +[features] +# Compiles the crash-injection failpoints (`utils::failpoint`) into builds +# without debug assertions too. Debug builds always have them; this exists +# so the crash-safety e2e suites also run against the optimized binary the +# `test-release` CI job builds — the CLI enables it from its +# [dev-dependencies] only, so a `cargo build --release` / `cargo install` +# binary never carries it. +failpoints = [] + [dependencies] serde = { workspace = true } serde_json = { workspace = true } @@ -15,13 +24,16 @@ sha1 = { workspace = true } hex = { workspace = true } reqwest = { workspace = true } tokio = { workspace = true } +futures-util = { workspace = true } thiserror = { workspace = true } walkdir = { workspace = true } uuid = { workspace = true } regex = { workspace = true } +aho-corasick = { workspace = true } toml_edit = { workspace = true } once_cell = { workspace = true } qbsdiff = { workspace = true } +rayon = { workspace = true } tar = { workspace = true } flate2 = { workspace = true } fs2 = { workspace = true } @@ -40,6 +52,18 @@ libc = { workspace = true } [target.'cfg(windows)'.dependencies] self-replace = { workspace = true } +# sha2 0.10 compiles its aarch64 SHA-256 hardware path only under the `asm` +# feature (x86/x86_64 already select SHA-NI at runtime without it); the +# soft fallback is 3-5x slower on every git-sha256 in apply, verify and +# vendor. The digests are identical. The hardware path is sha2's own +# in-crate intrinsics (runtime-detected, soft fallback on CPUs without the +# crypto extension); the feature also builds and links sha2-asm, which +# aarch64 never calls — sha2 0.11 needs neither the feature nor a C +# toolchain for it. Windows-on-ARM is left on the soft path: sha2-asm's +# GNU-syntax assembly does not assemble under the MSVC toolchain. +[target.'cfg(all(target_arch = "aarch64", not(target_env = "msvc")))'.dependencies] +sha2 = { workspace = true, features = ["asm"] } + # All ecosystems (npm, PyPI, Ruby gems, Go, Cargo, NuGet, Maven, Composer, # Deno) are unconditionally compiled in AND enabled at runtime — there are # no ecosystem feature gates and no runtime env gates (the old diff --git a/crates/socket-patch-core/src/api/client.rs b/crates/socket-patch-core/src/api/client.rs index ba3ff5963..b0bc04249 100644 --- a/crates/socket-patch-core/src/api/client.rs +++ b/crates/socket-patch-core/src/api/client.rs @@ -12,7 +12,13 @@ use serde::Serialize; // different ones came to exist; there is now exactly one, in `api::ranking`. use crate::api::ranking::severity_order as get_severity_order; use crate::api::ranking::{cmp_batch_infos, cmp_search_results}; +use crate::api::retry::{ + is_retryable_status, jitter_sample as retry_jitter, parse_retry_after, ApiRetry, + ApiRetryPolicy, RetryHooks, +}; use crate::api::types::*; +use crate::api::vendor_prefetch::VendorPrefetch; +pub use crate::api::vendor_prefetch::VendorPrefetchGuard; use crate::constants::USER_AGENT as USER_AGENT_VALUE; use crate::utils::env_compat::{is_debug_enabled, is_offline_env, proxy_url_from_env}; use crate::utils::notice::{notice_once, Notice}; @@ -76,11 +82,86 @@ fn status_error(head: &str, status: StatusCode, text: &str) -> String { /// Log debug messages when debug mode is enabled. fn debug_log(message: &str) { - if is_debug_enabled() { + if is_debug_enabled() && !defer_debug_line(message) { eprintln!("[socket-patch debug] {}", message); } } +/// Hold `message` back for [`with_deferred_debug`]'s caller when the current +/// task runs inside it; `false` (print it now) otherwise. +fn defer_debug_line(message: &str) -> bool { + DEFERRED_DEBUG + .try_with(|lines| { + lines + .borrow_mut() + .push(format!("[socket-patch debug] {}", message)); + }) + .is_ok() +} + +tokio::task_local! { + /// Set around a speculative (concurrent) request so its debug lines are + /// held back and the caller can print them, or drop them, in the order a + /// one-at-a-time loop would have produced. + static DEFERRED_DEBUG: std::cell::RefCell>; +} + +/// Run `fut` with its [`debug_log`] lines captured instead of printed; +/// returns its output and the captured lines, in emission order. +pub(crate) async fn with_deferred_debug( + fut: impl std::future::Future, +) -> (T, Vec) { + DEFERRED_DEBUG + .scope(std::cell::RefCell::new(Vec::new()), async move { + let out = fut.await; + (out, DEFERRED_DEBUG.with(|lines| lines.take())) + }) + .await +} + +/// Print lines captured by [`with_deferred_debug`] (already prefixed). +pub(crate) fn flush_deferred_debug(lines: Vec) { + for line in lines { + eprintln!("{line}"); + } +} + +/// A request's output fetched ahead of the loop that consumes it, with its +/// `--debug` lines held back until [`Self::release`] — call that where the +/// one-at-a-time loop would have issued the request, so the debug stream +/// keeps the serial interleaving with the loop's own stderr lines. Dropping +/// it unreleased discards the lines (the serial loop never made the call). +#[derive(Debug)] +pub struct HeldBack { + value: T, + debug: Vec, +} + +impl HeldBack { + /// The output, without releasing the debug lines. + pub(crate) fn peek(&self) -> &T { + &self.value + } + + /// The output, mutably, without releasing the debug lines. + pub(crate) fn peek_mut(&mut self) -> &mut T { + &mut self.value + } + + /// The output, printing the held-back debug lines first. + pub fn release(self) -> T { + flush_deferred_debug(self.debug); + self.value + } +} + +/// Run `fut` (a request made ahead of its turn) with its debug lines held +/// back; see [`HeldBack`]. +pub async fn hold_back_debug(fut: impl std::future::Future) -> HeldBack { + let (value, debug) = with_deferred_debug(fut).await; + HeldBack { value, debug } +} + /// Options for constructing an [`ApiClient`]. #[derive(Debug, Clone)] pub struct ApiClientOptions { @@ -118,8 +199,45 @@ pub struct ApiClient { /// retryable failure (transport / 429 / 5xx after every retry) — the /// run-level circuit breaker. Shared by clones: one CLI run, one count. vendor_outage: Arc, + /// In-flight slots for the public proxy's batch path — the + /// `/patch/batch` POSTs and the legacy per-package GETs they degrade + /// to. Shared by clones, so concurrent [`Self::search_patches_batch`] + /// calls on one client (scan's batch windows) stay within + /// [`PROXY_BATCH_PATH_CONCURRENCY`] requests in total: the peak the + /// serial batch loop reached, never that peak times the window. + proxy_batch_slots: Arc, + /// The vendor loop's download plan, while one is attached + /// ([`Self::prefetch_vendor_packages`]): [`Self::fetch_vendor_package`] + /// takes a planned uuid's outcome from it instead of requesting it. + /// Shared by clones, like the breaker it defers to. + vendor_prefetch: Arc>>>, + /// Bounded 429 / 503 retry for the JSON calls ([`crate::api::retry`]): + /// the policy, the run-wide retry window (shared by clones, and by + /// every default client in the process) and the clock hooks. + api_retry: ApiRetry, +} + +/// A JSON request's answer after [`ApiClient::send_json_request`]'s retry +/// loop: the live response, or a 429 / 503 that is final (retries off, +/// exhausted, refused by the caller's predicate, a `Retry-After` over the +/// cap, or past the run's retry window) with +/// its body already read. +enum Sent { + Response(reqwest::Response), + Throttled { + status: StatusCode, + text: String, + /// Why no further retry was made, for the error message; `None` + /// when retries are off (the message stays the pre-retry one). + gave_up: Option, + }, } +/// Most requests the public proxy's batch path keeps in flight per client: +/// the legacy per-package fallback's fan-out (one call runs its PURLs in +/// groups of this size), and the cap all concurrent calls share. +const PROXY_BATCH_PATH_CONCURRENCY: usize = 10; + /// Retry policy for the vendoring service's package-reference POST and /// archive GET: `attempts` tries in total, exponential delays from `base` /// with ±25% jitter, each capped at `max_delay` (a `Retry-After` in seconds @@ -181,7 +299,7 @@ impl VendorRetryPolicy { /// Consecutive retryable vendor-service failures after which the rest of the /// run skips the service without any I/O (`auto` then builds locally, /// `service` fails closed — the existing miss policy). -const VENDOR_BREAKER_THRESHOLD: u32 = 2; +pub(crate) const VENDOR_BREAKER_THRESHOLD: u32 = 2; /// A jitter sample in `[0, 1)` from std's randomly keyed hasher (no RNG /// dependency; the quality needed here is "not synchronized"). @@ -271,9 +389,30 @@ impl ApiClient { org_slug: options.org_slug, vendor_retry: VendorRetryPolicy::default(), vendor_outage: Arc::new(AtomicU32::new(0)), + proxy_batch_slots: Arc::new(tokio::sync::Semaphore::new(PROXY_BATCH_PATH_CONCURRENCY)), + vendor_prefetch: Arc::new(std::sync::Mutex::new(None)), + api_retry: ApiRetry::from_env(), } } + /// Override the JSON calls' 429 / 503 retry policy and clock hooks + /// (tests inject a recording sleep and a fixed jitter seed). The client + /// and its clones get a fresh retry window of `policy.retry_window`, + /// detached from the process-wide one; [`ApiRetryPolicy::none`] turns + /// retries off. + pub fn with_api_retry(mut self, policy: ApiRetryPolicy, hooks: RetryHooks) -> Self { + self.api_retry = ApiRetry::with_policy(policy, hooks); + self + } + + /// Wait for a [`Self::proxy_batch_slots`] slot; held until dropped. + async fn proxy_batch_slot(&self) -> tokio::sync::OwnedSemaphorePermit { + Arc::clone(&self.proxy_batch_slots) + .acquire_owned() + .await + .expect("proxy_batch_slots is never closed") + } + /// Override the vendoring-service retry policy (tests; a policy of /// [`VendorRetryPolicy::none`] makes a single attempt). pub fn with_vendor_retry(mut self, policy: VendorRetryPolicy) -> Self { @@ -281,6 +420,12 @@ impl ApiClient { self } + /// The run-level breaker's consecutive-failure count (tests). + #[cfg(test)] + pub(crate) fn vendor_outage_count(&self) -> u32 { + self.vendor_outage.load(Ordering::Relaxed) + } + /// Returns the API token, if set. pub fn api_token(&self) -> Option<&String> { self.api_token.as_ref() @@ -291,8 +436,99 @@ impl ApiClient { self.org_slug.as_ref() } + /// Whether this client talks to the public patch proxy (vs. the + /// authenticated org API) — picks the concurrency cap + /// ([`crate::utils::concurrent::api_concurrency`]). + pub fn uses_public_proxy(&self) -> bool { + self.use_public_proxy + } + // ── Internal helpers ────────────────────────────────────────────── + /// Send one JSON request (`build` makes a fresh builder per attempt), + /// retrying 429 / 503 per [`crate::api::retry`]. `label` (`"GET "`) + /// names the request in `--debug` lines and keys its jitter. + /// `retryable` can veto a retry after seeing the body (the proxy's + /// permanent "Patch API is not configured" 503, which every JSON path + /// vetoes). Transport errors are never retried. + async fn send_json_request( + &self, + label: &str, + build: impl Fn() -> reqwest::RequestBuilder, + retryable: impl Fn(StatusCode, &str) -> bool, + ) -> Result { + let retry = &self.api_retry; + let max = retry.policy.max_retries; + let mut retries = 0u32; + loop { + let resp = build().send().await.map_err(|e| { + ApiError::Network(format!("Network error: {}", network_error_detail(&e))) + })?; + let status = resp.status(); + if !is_retryable_status(status) { + return Ok(Sent::Response(resp)); + } + let retry_after = parse_retry_after(resp.headers(), (retry.hooks.now_unix_secs)()); + let text = resp.text().await.unwrap_or_default(); + let throttled = |gave_up: Option| { + Ok(Sent::Throttled { + status, + text: text.clone(), + gave_up: gave_up.filter(|_| max > 0), + }) + }; + if !retryable(status, &text) { + return throttled(None); + } + if retries >= max { + return throttled(Some(format!( + "gave up after {}", + if retries == 1 { + "1 retry".to_string() + } else { + format!("{retries} retries") + } + ))); + } + let next = retries + 1; + let Some(delay) = retry.policy.delay( + next, + retry_after, + retry_jitter(retry.hooks.jitter_seed, label, next), + ) else { + // A server asking for more than the cap will refuse an + // earlier retry too: report now instead of waiting. + let why = format!( + "Retry-After {} s exceeds the {} s retry cap", + retry_after.unwrap_or_default().as_secs(), + retry.policy.max_retry_after.as_secs() + ); + debug_log(&format!( + "{label} returned {}; not retrying: {why}", + status.as_u16() + )); + return throttled(Some(why)); + }; + if !retry.reserve(delay) { + let why = format!( + "the run's {} s retry window has closed", + retry.policy.retry_window.as_secs() + ); + debug_log(&format!( + "{label} returned {}; not retrying: {why}", + status.as_u16() + )); + return throttled(Some(why)); + } + debug_log(&format!( + "{label} returned {}; retry {next}/{max} in {delay:?}", + status.as_u16() + )); + (retry.hooks.sleep)(delay).await; + retries = next; + } + } + /// Internal GET that deserialises JSON. Returns `Ok(None)` on 404. async fn get_json( &self, @@ -301,11 +537,14 @@ impl ApiClient { let url = format!("{}{}", self.api_url, path); debug_log(&format!("GET {}", url)); - let resp = self.client.get(&url).send().await.map_err(|e| { - ApiError::Network(format!("Network error: {}", network_error_detail(&e))) - })?; - - Self::handle_json_response(resp, self.use_public_proxy).await + let sent = self + .send_json_request( + &format!("GET {url}"), + || self.client.get(&url), + |status, text| !is_patch_api_unconfigured(status, text), + ) + .await?; + Self::handle_json_response(sent, self.use_public_proxy).await } /// Internal POST that deserialises JSON. Returns `Ok(None)` on 404. @@ -317,25 +556,34 @@ impl ApiClient { let url = format!("{}{}", self.api_url, path); debug_log(&format!("POST {}", url)); - let resp = self - .client - .post(&url) - .header(header::CONTENT_TYPE, "application/json") - .json(body) - .send() - .await - .map_err(|e| { - ApiError::Network(format!("Network error: {}", network_error_detail(&e))) - })?; - - Self::handle_json_response(resp, self.use_public_proxy).await + let sent = self + .send_json_request( + &format!("POST {url}"), + || { + self.client + .post(&url) + .header(header::CONTENT_TYPE, "application/json") + .json(body) + }, + |status, text| !is_patch_api_unconfigured(status, text), + ) + .await?; + Self::handle_json_response(sent, self.use_public_proxy).await } /// Map an HTTP response to `Ok(Some(T))`, `Ok(None)` (404), or `Err`. async fn handle_json_response( - resp: reqwest::Response, + sent: Sent, use_public_proxy: bool, ) -> Result, ApiError> { + let resp = match sent { + Sent::Response(resp) => resp, + Sent::Throttled { + status, + text, + gave_up, + } => return Err(throttled_error(status, &text, use_public_proxy, gave_up)), + }; let status = resp.status(); if status == StatusCode::OK { @@ -541,42 +789,66 @@ impl ApiClient { /// Auth / rate-limit statuses are classified via `classify_auth_error` /// exactly like the JSON transport — 401/403 keep feeding /// `is_fallback_candidate` and 429 stays visible — and any other - /// failure (including over-capacity 503s) surfaces as an error. + /// failure (including over-capacity 503s) surfaces as an error. A 429 + /// or over-capacity 503 is first retried per [`crate::api::retry`]; + /// the permanent "not configured" 503 degrades at once. async fn proxy_batch_post( &self, purls: &[String], ) -> Result, ApiError> { let url = format!("{}/patch/batch", self.api_url); - debug_log(&format!("POST {}", url)); - let body = BatchSearchBody::new(purls); - let resp = self - .client - .post(&url) - .header(header::CONTENT_TYPE, "application/json") - .json(&body) - .send() - .await - .map_err(|e| { - ApiError::Network(format!("Network error: {}", network_error_detail(&e))) - })?; - - let status = resp.status(); - - if status == StatusCode::OK { - let parsed = resp - .json::() - .await - .map_err(|e| ApiError::Parse(format!("Failed to parse response: {}", e)))?; - return Ok(Some(parsed)); - } - - if let Some(err) = classify_auth_error(status, true) { - return Err(err); - } + // Held until this call returns, response body read. + let _slot = self.proxy_batch_slot().await; + // Logged AFTER the permit, not before: the line announces a request + // that is about to go out, and a caller queued behind the proxy's + // in-flight limit would otherwise print it and then wait. The slot + // stays held through any 429 / 503 retry wait: a throttled proxy + // is exactly when the other callers should not pile on. + debug_log(&format!("POST {}", url)); + let sent = self + .send_json_request( + &format!("POST {url}"), + || { + self.client + .post(&url) + .header(header::CONTENT_TYPE, "application/json") + .json(&body) + }, + // "Patch API is not configured" is permanent: degrade to + // the per-package path at once instead of retrying it. + |status, text| !is_batch_unsupported(status, text), + ) + .await?; - let text = resp.text().await.unwrap_or_default(); + let (status, text) = match sent { + Sent::Response(resp) => { + let status = resp.status(); + if status == StatusCode::OK { + let parsed = resp + .json::() + .await + .map_err(|e| ApiError::Parse(format!("Failed to parse response: {}", e)))?; + return Ok(Some(parsed)); + } + if let Some(err) = classify_auth_error(status, true) { + return Err(err); + } + (status, resp.text().await.unwrap_or_default()) + } + Sent::Throttled { + status, + text, + gave_up, + } => { + if is_batch_unsupported(status, &text) { + (status, text) + } else { + return Err(throttled_error(status, &text, true, gave_up)); + } + } + }; let fallback_reason = if is_batch_unsupported(status, &text) { Some("proxy batch endpoint unavailable") } else if status == StatusCode::BAD_REQUEST { @@ -607,31 +879,40 @@ impl ApiClient { /// proxy gained `POST /patch/batch`, this is the legacy path for /// deployments that predate it. /// - /// Processes PURLs in batches of `CONCURRENCY_LIMIT` to avoid - /// overwhelming the server while remaining efficient. + /// Processes PURLs in batches of `PROXY_BATCH_PATH_CONCURRENCY` to + /// avoid overwhelming the server while remaining efficient; each GET + /// also takes a [`Self::proxy_batch_slots`] slot, so concurrent calls + /// on one client share that cap instead of multiplying it. async fn search_patches_batch_via_individual_queries( &self, purls: &[String], ) -> Result { - const CONCURRENCY_LIMIT: usize = 10; - // Collect all (purl, response) pairs let mut all_results: Vec<(String, Option)> = Vec::new(); + // The first (in input order) package still throttled after its + // retries. Its error fails the whole call — a per-package miss is + // swallowed for an unresolvable PURL, but swallowing a throttle + // would drop the package from the scan without a word. + let mut throttled: Option<(usize, ApiError)> = None; - for chunk in purls.chunks(CONCURRENCY_LIMIT) { + for (chunk_idx, chunk) in purls.chunks(PROXY_BATCH_PATH_CONCURRENCY).enumerate() { // Use tokio::JoinSet for concurrent execution within each chunk let mut join_set = tokio::task::JoinSet::new(); - for purl in chunk { + for (offset, purl) in chunk.iter().enumerate() { + let index = chunk_idx * PROXY_BATCH_PATH_CONCURRENCY + offset; let purl = purl.clone(); let client = self.clone(); join_set.spawn(async move { + let slot = client.proxy_batch_slot().await; let resp = client.search_patches_by_package(&purl).await; + drop(slot); match resp { - Ok(r) => (purl, Some(r)), + Ok(r) => (index, purl, Ok(Some(r))), + Err(e) if is_throttle_error(&e) => (index, purl, Err(e)), Err(e) => { debug_log(&format!("Error fetching patches for {}: {}", purl, e)); - (purl, None) + (index, purl, Ok(None)) } } }); @@ -639,13 +920,21 @@ impl ApiClient { while let Some(result) = join_set.join_next().await { match result { - Ok(pair) => all_results.push(pair), + Ok((_, purl, Ok(resp))) => all_results.push((purl, resp)), + Ok((index, _, Err(e))) => { + if throttled.as_ref().is_none_or(|(first, _)| index < *first) { + throttled = Some((index, e)); + } + } Err(e) => { debug_log(&format!("Task join error: {}", e)); } } } } + if let Some((_, e)) = throttled { + return Err(e); + } // Convert the individual SearchResponse results into the batch shape. Ok(assemble_batch_from_individual(all_results)) @@ -842,9 +1131,28 @@ impl ApiClient { this run" ))); } - let (outcome, retryable_failure) = self - .fetch_vendor_package_once(uuid, free_only, vendor_url, patch_server_url) - .await; + // A download the attached plan already fetched stands in for the + // live requests; everything around it (the breaker check above, the + // counter update below) runs here, in call order, as before. + let plan = self + .vendor_prefetch + .lock() + .ok() + .and_then(|slot| slot.clone()); + let prefetched = match plan { + Some(plan) => { + plan.take(self, uuid, free_only, vendor_url, patch_server_url) + .await + } + None => None, + }; + let (outcome, retryable_failure) = match prefetched { + Some(fetched) => fetched.release(), + None => { + self.fetch_vendor_package_once(uuid, free_only, vendor_url, patch_server_url) + .await + } + }; match &outcome { VendorServiceOutcome::Failed(_) if retryable_failure => { self.vendor_outage.fetch_add(1, Ordering::Relaxed); @@ -857,9 +1165,75 @@ impl ApiClient { outcome } + /// Attach a download plan: `uuids` are the packages the vendor loop is + /// expected to download from the service, in loop order, with these + /// request parameters. Until the guard drops, the loop's + /// [`Self::fetch_vendor_package`] call for a planned uuid takes an + /// outcome fetched ahead of it (at most `window` in flight, and at + /// most `window` requests ahead of the loop) — see + /// [`super::vendor_prefetch`] for why nothing observable changes, and + /// what the plan may cost (it is exact: the CLI gates it with the + /// backends' own pre-flights, [`crate::vendor::service_preflight`] and + /// [`crate::vendor::npm_flavor::preflight_packages`], so it names only + /// the downloads the loop will ask for). A uuid this method refuses + /// without any I/O is dropped from the plan, so the prefetch never + /// sends what the loop's own call would not. The plan replaces any plan + /// already attached. + pub fn prefetch_vendor_packages( + &self, + uuids: Vec, + free_only: bool, + vendor_url: Option<&str>, + patch_server_url: Option<&str>, + window: usize, + ) -> VendorPrefetchGuard { + self.prefetch_vendor_downloads( + uuids.into_iter().map(PlannedDownload::archive).collect(), + free_only, + vendor_url, + patch_server_url, + window, + usize::MAX, + ) + } + + /// [`Self::prefetch_vendor_packages`] with a planned secondary artifact + /// per download (see [`PlannedDownload::secondary`]) and a bound on the + /// fetched archive bytes waiting for the loop: while `byte_budget` bytes + /// are held, only the download the loop is waiting on may start. + pub fn prefetch_vendor_downloads( + &self, + downloads: Vec, + free_only: bool, + vendor_url: Option<&str>, + patch_server_url: Option<&str>, + window: usize, + byte_budget: usize, + ) -> VendorPrefetchGuard { + let planned: Vec = downloads + .into_iter() + .filter(|d| is_valid_uuid(&d.uuid)) + .collect(); + let plan = Arc::new(VendorPrefetch::new( + planned, + free_only, + vendor_url, + patch_server_url, + window, + byte_budget, + )); + if let Ok(mut slot) = self.vendor_prefetch.lock() { + *slot = Some(Arc::clone(&plan)); + } + VendorPrefetchGuard { + slot: Arc::clone(&self.vendor_prefetch), + plan, + } + } + /// [`Self::fetch_vendor_package`] without the breaker: the outcome, and /// whether a `Failed` one was a retryable (availability) failure. - async fn fetch_vendor_package_once( + pub(crate) async fn fetch_vendor_package_once( &self, uuid: &str, free_only: bool, @@ -952,6 +1326,7 @@ impl ApiClient { kind: a.kind.clone(), url, integrity_sri: normalize_sha512_sri(sha512), + prefetched: None, }); } } @@ -965,6 +1340,7 @@ impl ApiClient { dirhash_h1: artifact.integrity.dirhash_h1.clone(), source_url: download_url, secondary_artifacts, + prestaged: Default::default(), })) } (ServeDownload::NotFound, _) => done(VendorServiceOutcome::Unavailable( @@ -1127,15 +1503,23 @@ impl ApiClient { /// [`VendorRetryPolicy`]; the flag says whether a final `Failed` was a /// retryable (availability) failure. async fn download_vendor_archive_retrying(&self, url: &str) -> (ServeDownload, bool) { + self.download_vendor_archive_from(url, 1).await + } + + /// [`Self::download_vendor_archive_retrying`] starting at attempt + /// `attempt` — 2 when attempt 1 was made elsewhere and deferred (see + /// [`Self::download_artifact_resuming`]), so a split download still + /// costs the policy's `attempts` requests, not one budget per split. + async fn download_vendor_archive_from( + &self, + url: &str, + mut attempt: u32, + ) -> (ServeDownload, bool) { let attempts = self.vendor_retry.attempts.max(1); - let mut attempt = 1; loop { match self.download_vendor_archive_once(url).await { (ServeDownload::Failed(e), Some(retry_after)) if attempt < attempts => { - debug_log(&format!( - "vendor package download attempt {attempt} failed: {e}" - )); - self.vendor_backoff(attempt, retry_after).await; + self.vendor_download_retry(attempt, &e, retry_after).await; attempt += 1; } (outcome, hint) => return (outcome, hint.is_some()), @@ -1143,6 +1527,20 @@ impl ApiClient { } } + /// Report attempt `attempt`'s retryable failure and pause before the + /// next one. + async fn vendor_download_retry( + &self, + attempt: u32, + e: &ApiError, + retry_after: Option, + ) { + debug_log(&format!( + "vendor package download attempt {attempt} failed: {e}" + )); + self.vendor_backoff(attempt, retry_after).await; + } + /// [`Self::download_vendor_archive_retrying`] without the flag. async fn download_vendor_archive(&self, url: &str) -> ServeDownload { self.download_vendor_archive_retrying(url).await.0 @@ -1251,19 +1649,69 @@ impl ApiClient { /// integrity. A 404/410/408 surfaces as an error (a secondary the /// reference promised should be present). pub(crate) async fn download_artifact(&self, url: &str) -> Result, ApiError> { - match self.download_vendor_archive(url).await { - ServeDownload::Ok(bytes) => Ok(bytes), - ServeDownload::NotFound => Err(ApiError::Other(format!("artifact not found: {url}"))), - ServeDownload::Pending => { - Err(ApiError::Other(format!("artifact still building: {url}"))) + artifact_download_result(self.download_vendor_archive(url).await, url) + } + + /// The first attempt of [`Self::download_artifact`] alone: `Ok` with + /// exactly the result `download_artifact` would return when that attempt + /// settles it (success, or a failure it would not retry), `Err` with the + /// budget the attempt left unspent when it would retry. That `Err` goes + /// to [`Self::download_artifact_resuming`], which spends the REST of the + /// budget — so however a caller splits the two, the host sees the single + /// request sequence `download_artifact` would have made. + pub(crate) async fn download_artifact_first_attempt( + &self, + url: &str, + ) -> Result, ApiError>, DeferredAttempt> { + match self.download_vendor_archive_once(url).await { + (ServeDownload::Failed(error), Some(retry_after)) + if self.vendor_retry.attempts.max(1) > 1 => + { + Err(DeferredAttempt { error, retry_after }) } - ServeDownload::Failed(e) => Err(e), + (outcome, _) => Ok(artifact_download_result(outcome, url)), } } + + /// [`Self::download_artifact`] resumed from the attempt + /// [`Self::download_artifact_first_attempt`] deferred: that attempt's + /// failure is reported and its `Retry-After` waited out exactly where + /// the retry loop would have, then the remaining attempts run. + pub(crate) async fn download_artifact_resuming( + &self, + url: &str, + deferred: DeferredAttempt, + ) -> Result, ApiError> { + self.vendor_download_retry(1, &deferred.error, deferred.retry_after) + .await; + artifact_download_result(self.download_vendor_archive_from(url, 2).await.0, url) + } +} + +/// The unspent remainder of a retry budget: a first attempt that failed +/// retryably and was set aside, carrying the failure its retry still owes a +/// debug line and the `Retry-After` it still owes a pause. Only +/// [`ApiClient::download_artifact_first_attempt`] makes one (and only when +/// the policy has an attempt left to give), and only +/// [`ApiClient::download_artifact_resuming`] spends it. +#[derive(Debug)] +pub(crate) struct DeferredAttempt { + error: ApiError, + retry_after: Option, } // ── Free functions ──────────────────────────────────────────────────── +/// [`ApiClient::download_artifact`]'s mapping of a serve outcome. +fn artifact_download_result(outcome: ServeDownload, url: &str) -> Result, ApiError> { + match outcome { + ServeDownload::Ok(bytes) => Ok(bytes), + ServeDownload::NotFound => Err(ApiError::Other(format!("artifact not found: {url}"))), + ServeDownload::Pending => Err(ApiError::Other(format!("artifact still building: {url}"))), + ServeDownload::Failed(e) => Err(e), + } +} + /// Cap on a single prebuilt-archive download (defensive bound against a /// runaway / hostile serve response). Generous enough for any real package. const MAX_VENDOR_PACKAGE_BYTES: u64 = 256 * 1024 * 1024; @@ -1285,6 +1733,9 @@ pub(crate) struct FetchedVendorPackage { /// each with a host-rewritten URL + normalized sha512, for a backend to /// download + verify lazily via [`ApiClient::download_artifact`]. pub secondary_artifacts: Vec, + /// What the vendor prefetch plan already did with these bytes ahead of + /// the backend (see [`crate::vendor::prestage`]). + pub prestaged: crate::vendor::prestage::Prestaged, } /// A non-tarball served artifact reference (e.g. `gem-stub-gemspec`): its kind, @@ -1295,6 +1746,65 @@ pub(crate) struct SecondaryArtifact { pub url: String, /// Normalized `sha512-` of the artifact bytes. pub integrity_sri: String, + /// The download a vendor prefetch plan already made for it, taken by + /// the backend's own call in its place (see [`PlannedDownload`]). + pub prefetched: Option, +} + +/// One download in a vendor prefetch plan (see +/// [`ApiClient::prefetch_vendor_downloads`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PlannedDownload { + /// The patch uuid the vendor loop will ask the service for. + pub uuid: String, + /// The kind of served secondary artifact (e.g. `gem-stub-gemspec`) the + /// loop's backend downloads right after the verified archive. The plan + /// fetches it along with the archive — only when the archive is ready, + /// passes its integrity checks and the service served that kind, the + /// backend's own conditions — and the backend takes it in its place. + pub secondary: Option, + /// What to do with the archive once it has landed and passed its + /// integrity checks, ahead of the backend: extract it next to the + /// backend's stage, or run its afterHash check (see + /// [`crate::vendor::prestage`]). Built by the backends' plan gates. + pub stage: Option, +} + +impl PlannedDownload { + /// A plain archive download: no secondary artifact, nothing staged. + pub fn archive(uuid: String) -> Self { + Self { + uuid, + secondary: None, + stage: None, + } + } +} + +/// A secondary artifact's download made ahead of the backend's call, with +/// its debug lines held back until that call takes it. Shared, so the +/// artifact reference stays `Clone`; taken at most once. +#[derive(Clone)] +pub(crate) struct PrefetchedSecondary(Arc>>); + +/// A download's bytes (or failure), debug lines held back. +pub(crate) type HeldDownload = HeldBack, ApiError>>; + +impl PrefetchedSecondary { + pub(crate) fn new(downloaded: HeldDownload) -> Self { + Self(Arc::new(std::sync::Mutex::new(Some(downloaded)))) + } + + /// The download, once; `None` after it was taken. + pub(crate) fn take(&self) -> Option { + self.0.lock().ok().and_then(|mut slot| slot.take()) + } +} + +impl std::fmt::Debug for PrefetchedSecondary { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("PrefetchedSecondary") + } } /// Outcome of [`ApiClient::fetch_vendor_package`]. @@ -1738,6 +2248,55 @@ fn classify_auth_error(status: StatusCode, use_public_proxy: bool) -> Option bool { + matches!( + e, + ApiError::RateLimited(_) | ApiError::ServiceUnavailable(_) + ) +} + +/// The public proxy's permanent "Patch API is not configured" 503 +/// (patch endpoints disabled on that deployment). Not throttling: no JSON +/// path retries it, and it stays the plain [`ApiError::Other`] status +/// error it always was. +fn is_patch_api_unconfigured(status: StatusCode, body: &str) -> bool { + status == StatusCode::SERVICE_UNAVAILABLE && body.contains("Patch API is not configured") +} + +/// The error for a 429 / 503 the retry loop left final: 429 → +/// [`ApiError::RateLimited`], an over-capacity 503 → +/// [`ApiError::ServiceUnavailable`] with the status and body, naming why +/// the loop stopped (`gave_up`, e.g. "gave up after 3 retries") when +/// retries were on. The permanent "not configured" 503 keeps its +/// pre-retry [`ApiError::Other`]. +fn throttled_error( + status: StatusCode, + text: &str, + use_public_proxy: bool, + gave_up: Option, +) -> ApiError { + match (classify_auth_error(status, use_public_proxy), gave_up) { + (Some(ApiError::RateLimited(_)), Some(why)) => ApiError::RateLimited(format!( + "Rate limit exceeded (HTTP 429, {why}). Please try again later." + )), + (Some(err), _) => err, + (None, why) => { + let msg = status_error("API request failed with status", status, text); + if is_patch_api_unconfigured(status, text) { + return ApiError::Other(msg); + } + ApiError::ServiceUnavailable(match why { + Some(why) => format!("{msg} ({why})"), + None => msg, + }) + } + } +} + /// Decide whether a public-proxy response to `POST /patch/batch` means the /// endpoint is unsupported on that deployment, in which case /// [`ApiClient::search_patches_batch`] degrades to per-package GETs (which @@ -1756,7 +2315,7 @@ fn classify_auth_error(status: StatusCode, use_public_proxy: bool) -> Option bool { match status { StatusCode::BAD_REQUEST => body.contains("Unsupported endpoint"), - StatusCode::SERVICE_UNAVAILABLE => body.contains("Patch API is not configured"), + StatusCode::SERVICE_UNAVAILABLE => is_patch_api_unconfigured(status, body), // A deployment / CDN layer with no route for POST /patch/batch. StatusCode::NOT_FOUND | StatusCode::METHOD_NOT_ALLOWED => true, _ => false, @@ -1990,6 +2549,14 @@ pub enum ApiError { #[error("{0}")] RateLimited(String), + /// An HTTP 503 (over capacity, maintenance) still failing after the + /// bounded retry ([`crate::api::retry`]). The text is the plain status + /// error (`API request failed with status 503: `, plus the retry + /// note). The proxy's permanent "Patch API is not configured" 503 is + /// [`ApiError::Other`] instead: it is not throttling. + #[error("{0}")] + ServiceUnavailable(String), + #[error("{0}")] InvalidHash(String), @@ -4076,6 +4643,117 @@ mod vendor_retry_tests { .with_vendor_retry(policy) } + /// `download_artifact_first_attempt` settles exactly what + /// `download_artifact` would return on its first attempt, and defers + /// (one request, no backoff) wherever `download_artifact` would retry — + /// unless the policy has no retry to give, where it settles. + #[tokio::test] + async fn first_attempt_settles_or_defers_like_download_artifact() { + let server = MockServer::start().await; + for (route, status) in [("/ok", 200), ("/gone", 404), ("/busy", 429), ("/down", 503)] { + Mock::given(method("GET")) + .and(path(route)) + .respond_with(ResponseTemplate::new(status).set_body_bytes(BYTES.to_vec())) + .mount(&server) + .await; + } + let url = |route: &str| format!("{}{route}", server.uri()); + let retrying = client(&server.uri(), fast()); + + let ok = retrying.download_artifact_first_attempt(&url("/ok")).await; + assert_eq!(ok.expect("200 settles").expect("200 is Ok"), BYTES); + let gone = retrying + .download_artifact_first_attempt(&url("/gone")) + .await + .expect("404 is terminal, so it settles") + .expect_err("404 is an error"); + let full = retrying + .download_artifact(&url("/gone")) + .await + .expect_err("404 is an error"); + assert_eq!(gone.to_string(), full.to_string()); + for route in ["/busy", "/down"] { + assert!( + retrying + .download_artifact_first_attempt(&url(route)) + .await + .is_err(), + "{route} is retried by download_artifact, so it defers" + ); + } + let gets = |route: &'static str| { + let server = &server; + async move { + server + .received_requests() + .await + .unwrap_or_default() + .iter() + .filter(|r| r.url.path() == route) + .count() + } + }; + assert_eq!(gets("/busy").await, 1, "a deferral makes one request"); + assert_eq!(gets("/down").await, 1, "a deferral makes one request"); + + let single = client(&server.uri(), VendorRetryPolicy::none()); + let settled = single + .download_artifact_first_attempt(&url("/down")) + .await + .expect("with no retry budget the first attempt is final") + .expect_err("503 is an error"); + let full = single + .download_artifact(&url("/down")) + .await + .expect_err("503 is an error"); + assert_eq!(settled.to_string(), full.to_string()); + } + + /// A deferral RESUMES the budget: a first attempt plus + /// `download_artifact_resuming` costs the host exactly the requests one + /// `download_artifact` costs — never the deferred attempt plus a fresh + /// budget, which would give a flapping host one extra try and turn the + /// serial loop's failure into a success. + #[tokio::test] + async fn resuming_a_deferral_spends_one_budget_not_two() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/down")) + .respond_with(ResponseTemplate::new(503).set_body_bytes(BYTES.to_vec())) + .mount(&server) + .await; + let url = format!("{}/down", server.uri()); + let gets = || async { + server + .received_requests() + .await + .unwrap_or_default() + .iter() + .filter(|r| r.url.path() == "/down") + .count() + }; + let client = client(&server.uri(), fast()); + + let whole = client.download_artifact(&url).await.expect_err("503"); + let serial = gets().await; + assert_eq!(serial, fast().attempts as usize, "the policy's attempts"); + + let deferred = client + .download_artifact_first_attempt(&url) + .await + .expect_err("503 defers"); + let resumed = client + .download_artifact_resuming(&url, deferred) + .await + .expect_err("503"); + assert_eq!(resumed.to_string(), whole.to_string(), "same final error"); + assert_eq!( + gets().await - serial, + serial, + "the split download costs the same budget as the whole one" + ); + } + fn granted(server: &MockServer, uuid: &str) -> ResponseTemplate { let url = format!("{}{SERVE}", server.uri()); let sri = format!( @@ -4691,3 +5369,110 @@ mod authenticated_batch_tests { assert!(result.can_access_paid_patches); } } + +#[cfg(test)] +mod proxy_batch_path_cap_tests { + //! Concurrent `search_patches_batch` calls on one public-proxy client + //! (scan's batch windows) must share the batch path's in-flight cap, + //! not multiply it: when every chunk degrades to the legacy per-package + //! GETs, the proxy sees at most `PROXY_BATCH_PATH_CONCURRENCY` of them + //! at once — what the serial batch loop peaked at. + use super::*; + use std::sync::Mutex; + use std::time::Instant; + use wiremock::matchers::{method, path, path_regex}; + use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate}; + + /// Every by-package answer takes this long, so the requests in flight + /// at an arrival are exactly those that arrived less than this before. + const GET_DELAY: Duration = Duration::from_millis(300); + + /// Records each by-package GET's arrival time, then answers it (empty, + /// after [`GET_DELAY`]). + struct Arrivals(Arc>>); + + impl Respond for Arrivals { + fn respond(&self, _: &Request) -> ResponseTemplate { + self.0.lock().unwrap().push(Instant::now()); + ResponseTemplate::new(200) + .set_body_json(serde_json::json!({ + "patches": [], + "canAccessPaidPatches": false, + })) + .set_delay(GET_DELAY) + } + } + + /// Most GETs whose arrivals fall inside one window shorter than + /// [`GET_DELAY`] — a lower bound on the peak in flight that a capped + /// run cannot exceed (a slot frees only when its answer, `GET_DELAY` + /// after its arrival, is back). + fn peak_in_flight(arrivals: &[Instant]) -> usize { + let mut sorted = arrivals.to_vec(); + sorted.sort(); + let window = GET_DELAY.mul_f32(0.8); + (0..sorted.len()) + .map(|i| { + sorted[i..] + .iter() + .take_while(|t| t.duration_since(sorted[i]) < window) + .count() + }) + .max() + .unwrap_or(0) + } + + #[tokio::test] + async fn concurrent_batches_share_the_legacy_fallback_cap() { + let server = MockServer::start().await; + // A validation 400 for every chunk: each degrades to per-package + // GETs (one exotic PURL per chunk is enough in the wild). + Mock::given(method("POST")) + .and(path("/patch/batch")) + .respond_with(ResponseTemplate::new(400).set_body_string("bad purl")) + .mount(&server) + .await; + let arrivals = Arc::new(Mutex::new(Vec::new())); + Mock::given(method("GET")) + .and(path_regex("^/patch/by-package/")) + .respond_with(Arrivals(Arc::clone(&arrivals))) + .mount(&server) + .await; + + let client = ApiClient::new(ApiClientOptions { + api_url: server.uri(), + api_token: None, + use_public_proxy: true, + org_slug: None, + }); + // Four windows of 10 PURLs, as scan's proxy batch windows run them. + let chunks: Vec> = (0..4) + .map(|c| { + (0..PROXY_BATCH_PATH_CONCURRENCY) + .map(|i| format!("pkg:npm/cap-{c}-{i}@1.0.0")) + .collect() + }) + .collect(); + let results = futures_util::future::join_all( + chunks + .iter() + .map(|chunk| client.search_patches_batch(chunk)), + ) + .await; + for result in results { + let response = result.expect("the per-package path swallows nothing here"); + assert!(response.packages.is_empty()); + } + + let arrivals = arrivals.lock().unwrap(); + assert_eq!(arrivals.len(), 40, "one GET per PURL"); + let peak = peak_in_flight(&arrivals); + assert!( + peak <= PROXY_BATCH_PATH_CONCURRENCY, + "{peak} by-package GETs in flight at once; the cap is \ + {PROXY_BATCH_PATH_CONCURRENCY}" + ); + // And the cap is reached, not undershot: the calls still overlap. + assert_eq!(peak, PROXY_BATCH_PATH_CONCURRENCY); + } +} diff --git a/crates/socket-patch-core/src/api/mod.rs b/crates/socket-patch-core/src/api/mod.rs index ab918e7a5..b8f682cd8 100644 --- a/crates/socket-patch-core/src/api/mod.rs +++ b/crates/socket-patch-core/src/api/mod.rs @@ -2,4 +2,6 @@ pub mod blob_fetcher; pub mod client; pub mod date; pub mod ranking; +pub mod retry; pub mod types; +pub(crate) mod vendor_prefetch; diff --git a/crates/socket-patch-core/src/api/retry.rs b/crates/socket-patch-core/src/api/retry.rs new file mode 100644 index 000000000..e9b7ec4c2 --- /dev/null +++ b/crates/socket-patch-core/src/api/retry.rs @@ -0,0 +1,468 @@ +//! Bounded retry for the patch API's JSON calls on HTTP 429 / 503. +//! +//! `scan` keeps up to 32 patch-API requests in flight, so a throttling +//! server (or a proxy / WAF in front of it) answering `429 Too Many +//! Requests` or `503 Service Unavailable` is expected, not exotic. Each +//! such answer is retried a bounded number of times before it becomes the +//! request's error: +//! +//! - at most [`ApiRetryPolicy::max_retries`] retries per request (3 by +//! default, [`API_MAX_RETRIES_ENV`] overrides, `0` disables); +//! - the wait honors the server's `Retry-After` (delta-seconds or an +//! HTTP-date). One longer than [`ApiRetryPolicy::max_retry_after`] +//! (30 s) is not waited out at all: the answer is final at once (retrying +//! early would only be refused again). One shorter than the jittered +//! first backoff step (`Retry-After: 0`, a date already past) waits that +//! step instead, so a server saying "now" is not hammered; +//! - without a `Retry-After` the wait is exponential from +//! [`ApiRetryPolicy::base_delay`] (500 ms, 1 s, 2 s, ... capped at +//! [`ApiRetryPolicy::max_backoff`]) with "equal jitter" — each wait lands +//! in `[d/2, d)`, the sample derived deterministically from a seed, the +//! request and the retry number; +//! - all retries share one run-wide WALL-CLOCK window +//! ([`ApiRetryPolicy::retry_window`], 60 s) that opens with the run's +//! first retry: a retry whose wait would end after the window closes is +//! refused and the answer is final. Concurrent requests wait in parallel, +//! so 32 throttled requests each still get their retries (their waits +//! overlap rather than add up), while a throttled run as a whole adds at +//! most about the window's length of waiting. +//! +//! Nothing else is retried here: other 4xx (401/403 keep driving the proxy +//! fallback), other 5xx and transport errors surface on the first answer, +//! exactly as before. Retries happen inside each request's future, so the +//! callers' ordered folds (`ordered_concurrent`) see the same sequence of +//! results a clean run produces. + +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, OnceLock}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use reqwest::header::{HeaderMap, RETRY_AFTER}; +use reqwest::StatusCode; + +/// Env override for [`ApiRetryPolicy::max_retries`]: a non-negative +/// integer, clamped to [`MAX_RETRIES_CEILING`]; `0` turns retries off. An +/// unset, empty or unparsable value keeps the default. +pub const API_MAX_RETRIES_ENV: &str = "SOCKET_API_MAX_RETRIES"; + +/// The most retries [`API_MAX_RETRIES_ENV`] can ask for per request. +pub const MAX_RETRIES_CEILING: u32 = 10; + +/// Retry policy for the patch API's JSON calls (see the module docs). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ApiRetryPolicy { + /// Retries per request after the first attempt (`0` = never retry). + pub max_retries: u32, + /// First backoff step without a `Retry-After`; doubles per retry. + pub base_delay: Duration, + /// Cap on one backoff step (before jitter halves its lower bound). + pub max_backoff: Duration, + /// Longest `Retry-After` the loop waits out; a longer one makes the + /// answer final at once. + pub max_retry_after: Duration, + /// Wall-clock window, opened by the first retry of the run (of the + /// clients sharing it), within which every retry's wait must end. + pub retry_window: Duration, +} + +impl Default for ApiRetryPolicy { + fn default() -> Self { + Self { + max_retries: 3, + base_delay: Duration::from_millis(500), + max_backoff: Duration::from_secs(8), + max_retry_after: Duration::from_secs(30), + retry_window: Duration::from_secs(60), + } + } +} + +impl ApiRetryPolicy { + /// No retries: every 429 / 503 is final on the first answer (the + /// behavior before retries existed). + pub fn none() -> Self { + Self { + max_retries: 0, + ..Self::default() + } + } + + /// The default policy with [`API_MAX_RETRIES_ENV`] applied. + pub fn from_env() -> Self { + let mut policy = Self::default(); + if let Some(n) = max_retries_override(std::env::var(API_MAX_RETRIES_ENV).ok().as_deref()) { + policy.max_retries = n; + } + policy + } + + /// The pause before retry number `retry` (1-based), `jitter` in + /// `[0, 1)`: the server's `Retry-After` when it sent one (floored at the + /// jittered first backoff step), otherwise the jittered exponential + /// step. `None` when the `Retry-After` exceeds + /// [`Self::max_retry_after`]: the caller gives up instead of waiting. + pub fn delay( + &self, + retry: u32, + retry_after: Option, + jitter: f64, + ) -> Option { + // Equal jitter: [step/2, step). + let jittered = |step: Duration| { + let half = step / 2; + half + half.mul_f64(jitter.clamp(0.0, 1.0)) + }; + if let Some(after) = retry_after { + if after > self.max_retry_after { + return None; + } + return Some(after.max(jittered(self.base_delay.min(self.max_backoff)))); + } + let step = self + .base_delay + .saturating_mul(1u32 << retry.saturating_sub(1).min(16)) + .min(self.max_backoff); + Some(jittered(step)) + } +} + +/// [`API_MAX_RETRIES_ENV`]'s value as a retry count, or `None` to keep the +/// default (unset, empty, or not a non-negative integer). +fn max_retries_override(raw: Option<&str>) -> Option { + let n = raw?.trim().parse::().ok()?; + Some(n.min(u64::from(MAX_RETRIES_CEILING)) as u32) +} + +/// Is this answer one the retry loop may repeat? (429 and 503 only.) +pub fn is_retryable_status(status: StatusCode) -> bool { + matches!( + status, + StatusCode::TOO_MANY_REQUESTS | StatusCode::SERVICE_UNAVAILABLE + ) +} + +/// A `Retry-After` header as a wait from `now_unix_secs`: delta-seconds +/// (`Retry-After: 7`) or an HTTP-date (`Retry-After: Fri, 27 Mar 2026 +/// 19:12:42 GMT`; a date already past waits zero). `None` when absent or +/// unreadable — the caller then backs off exponentially. +pub fn parse_retry_after(headers: &HeaderMap, now_unix_secs: u64) -> Option { + let raw = headers.get(RETRY_AFTER)?.to_str().ok()?.trim(); + if raw.bytes().all(|b| b.is_ascii_digit()) { + return raw.parse::().ok().map(Duration::from_secs); + } + // Only the date forms: a signed or fractional number is malformed. + if raw.starts_with(['-', '+']) || raw.parse::().is_ok() { + return None; + } + let at = crate::api::date::parse_timestamp_secs(raw)?; + Some(Duration::from_secs(at.saturating_sub(now_unix_secs))) +} + +/// The jitter sample in `[0, 1)` for retry `retry` of the request `key` +/// under `seed`: a SplitMix64 mix, so a given seed replays the same waits +/// (tests) while distinct requests and processes stay desynchronized. +pub fn jitter_sample(seed: u64, key: &str, retry: u32) -> f64 { + // FNV-1a over the key, folded with the seed and the retry number. + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in key.bytes() { + h ^= u64::from(b); + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + let mut z = seed ^ h ^ u64::from(retry).wrapping_mul(0x9e37_79b9_7f4a_7c15); + z = z.wrapping_add(0x9e37_79b9_7f4a_7c15); + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^= z >> 31; + (z >> 11) as f64 / (1u64 << 53) as f64 +} + +/// The retry loop's sleep, injectable so tests run on a virtual clock. +pub type RetrySleep = + Arc Pin + Send>> + Send + Sync>; + +/// The retry loop's view of time and randomness. [`Default`] is the real +/// thing (tokio's sleep, the system and monotonic clocks, a per-process +/// random seed); tests substitute a recorder and fixed values. +#[derive(Clone)] +pub struct RetryHooks { + /// Waits out one retry delay. + pub sleep: RetrySleep, + /// Now, as UNIX seconds (resolves an HTTP-date `Retry-After`). + pub now_unix_secs: Arc u64 + Send + Sync>, + /// Now on a monotonic clock, as time since an arbitrary fixed epoch + /// (the [`ApiRetryPolicy::retry_window`] is measured on it). Every + /// client sharing a window must share the epoch; the default is one + /// per process. + pub monotonic_now: Arc Duration + Send + Sync>, + /// Seed for [`jitter_sample`]. + pub jitter_seed: u64, +} + +impl Default for RetryHooks { + fn default() -> Self { + Self { + sleep: Arc::new(|d| Box::pin(tokio::time::sleep(d))), + now_unix_secs: Arc::new(|| { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_secs()) + }), + monotonic_now: Arc::new(process_monotonic_now), + jitter_seed: process_seed(), + } + } +} + +impl std::fmt::Debug for RetryHooks { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RetryHooks") + .field("jitter_seed", &self.jitter_seed) + .finish_non_exhaustive() + } +} + +/// One random seed per process (std's randomly keyed hasher; no RNG +/// dependency). +fn process_seed() -> u64 { + use std::hash::{BuildHasher as _, Hasher as _}; + static SEED: OnceLock = OnceLock::new(); + *SEED.get_or_init(|| { + let mut hasher = std::collections::hash_map::RandomState::new().build_hasher(); + hasher.write_u64(0x5eed); + hasher.finish() + }) +} + +/// Time since this process's fixed monotonic epoch (the default +/// [`RetryHooks::monotonic_now`]). +fn process_monotonic_now() -> Duration { + static EPOCH: OnceLock = OnceLock::new(); + EPOCH.get_or_init(Instant::now).elapsed() +} + +/// When the retry window opened (on [`RetryHooks::monotonic_now`]'s +/// clock), set by the first retry; shared by every client holding it. +type WindowStart = Arc>; + +/// The run-wide retry window every default client shares: one CLI run is +/// one process, and a run builds several clients (discovery, download, +/// the proxy fallback), so the window is process-wide rather than +/// per-client. +fn process_window() -> WindowStart { + static WINDOW: OnceLock = OnceLock::new(); + Arc::clone(WINDOW.get_or_init(WindowStart::default)) +} + +/// A client's retry state: the policy, the shared window, the hooks. +#[derive(Debug, Clone)] +pub(crate) struct ApiRetry { + pub(crate) policy: ApiRetryPolicy, + /// When the run's first retry opened the window. + window_start: WindowStart, + pub(crate) hooks: RetryHooks, +} + +impl ApiRetry { + /// The default: [`ApiRetryPolicy::from_env`] on the process window. + pub(crate) fn from_env() -> Self { + Self { + policy: ApiRetryPolicy::from_env(), + window_start: process_window(), + hooks: RetryHooks::default(), + } + } + + /// `policy` on a FRESH window of its own (tests, or a caller that wants + /// isolation from the process window). + pub(crate) fn with_policy(policy: ApiRetryPolicy, hooks: RetryHooks) -> Self { + Self { + policy, + window_start: WindowStart::default(), + hooks, + } + } + + /// May a retry wait `delay` now? The first call opens the window; + /// `true` iff the wait ends within [`ApiRetryPolicy::retry_window`] of + /// that opening. Waits running in parallel all fit as long as each + /// ends in time — only wall-clock time counts, not summed waiting. + pub(crate) fn reserve(&self, delay: Duration) -> bool { + let now = (self.hooks.monotonic_now)(); + let start = *self.window_start.get_or_init(|| now); + now.saturating_add(delay) <= start.saturating_add(self.policy.retry_window) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use reqwest::header::HeaderValue; + + fn headers(v: &str) -> HeaderMap { + let mut h = HeaderMap::new(); + h.insert(RETRY_AFTER, HeaderValue::from_str(v).unwrap()); + h + } + + #[test] + fn retry_after_delta_seconds() { + assert_eq!( + parse_retry_after(&headers("7"), 0), + Some(Duration::from_secs(7)) + ); + assert_eq!(parse_retry_after(&headers(" 0 "), 0), Some(Duration::ZERO)); + } + + #[test] + fn retry_after_http_date_is_relative_to_now() { + // Fri, 27 Mar 2026 19:12:42 GMT = 1774638762. + let at = 1_774_638_762u64; + let h = headers("Fri, 27 Mar 2026 19:12:42 GMT"); + assert_eq!( + parse_retry_after(&h, at - 12), + Some(Duration::from_secs(12)) + ); + // A date in the past waits zero. + assert_eq!(parse_retry_after(&h, at + 100), Some(Duration::ZERO)); + } + + #[test] + fn retry_after_malformed_or_absent_is_none() { + assert_eq!(parse_retry_after(&HeaderMap::new(), 0), None); + for bad in ["-3", "+3", "1.5", "soon", ""] { + assert_eq!(parse_retry_after(&headers(bad), 0), None, "{bad:?}"); + } + } + + #[test] + fn delay_caps_retry_after_and_jitters_backoff_into_the_upper_half() { + let p = ApiRetryPolicy::default(); + assert_eq!( + p.delay(1, Some(Duration::from_secs(120)), 0.9), + None, + "a Retry-After over the cap is not waited out" + ); + assert_eq!( + p.delay(1, Some(Duration::from_secs(30)), 0.9), + Some(Duration::from_secs(30)), + "the cap itself is still waited" + ); + assert_eq!( + p.delay(2, Some(Duration::from_secs(3)), 0.9), + Some(Duration::from_secs(3)), + "Retry-After ignores jitter and the backoff step" + ); + // Retry-After: 0 (or a past date) is floored at the jittered first + // step, whatever the retry number. + for retry in [1u32, 3] { + assert_eq!( + p.delay(retry, Some(Duration::ZERO), 0.0), + Some(Duration::from_millis(250)) + ); + assert_eq!( + p.delay(retry, Some(Duration::from_millis(100)), 0.5), + Some(Duration::from_millis(375)) + ); + } + // Steps 500 ms, 1 s, 2 s, 4 s, 8 s, 8 s: each wait in [step/2, step). + for (retry, step_ms) in [(1u32, 500u64), (2, 1000), (3, 2000), (5, 8000), (9, 8000)] { + let lo = p.delay(retry, None, 0.0).unwrap(); + let hi = p.delay(retry, None, 0.999_999).unwrap(); + assert_eq!(lo, Duration::from_millis(step_ms / 2), "retry {retry}"); + assert!(hi < Duration::from_millis(step_ms), "retry {retry}: {hi:?}"); + assert!(hi > lo); + } + } + + #[test] + fn jitter_is_deterministic_per_seed_key_and_retry() { + let a = jitter_sample(42, "GET /x", 1); + assert_eq!(a, jitter_sample(42, "GET /x", 1)); + assert_ne!(a, jitter_sample(43, "GET /x", 1)); + assert_ne!(a, jitter_sample(42, "GET /y", 1)); + assert_ne!(a, jitter_sample(42, "GET /x", 2)); + for i in 0..1000u32 { + let j = jitter_sample(u64::from(i), "k", i % 4); + assert!((0.0..1.0).contains(&j), "{j}"); + } + } + + #[test] + fn max_retries_env_parsing() { + assert_eq!(max_retries_override(None), None); + assert_eq!(max_retries_override(Some("")), None); + assert_eq!(max_retries_override(Some("x")), None); + assert_eq!(max_retries_override(Some("-1")), None); + assert_eq!(max_retries_override(Some("0")), Some(0)); + assert_eq!(max_retries_override(Some(" 5 ")), Some(5)); + assert_eq!( + max_retries_override(Some("1000")), + Some(MAX_RETRIES_CEILING) + ); + } + + /// A hand-driven monotonic clock (milliseconds). + fn manual_clock() -> (RetryHooks, Arc) { + let ms = Arc::new(std::sync::atomic::AtomicU64::new(1_000)); + let read = Arc::clone(&ms); + let hooks = RetryHooks { + monotonic_now: Arc::new(move || { + Duration::from_millis(read.load(std::sync::atomic::Ordering::SeqCst)) + }), + ..RetryHooks::default() + }; + (hooks, ms) + } + + #[test] + fn the_window_counts_wall_clock_not_summed_waits() { + use std::sync::atomic::Ordering::SeqCst; + let (hooks, clock) = manual_clock(); + let retry = ApiRetry::with_policy( + ApiRetryPolicy { + retry_window: Duration::from_secs(3), + ..ApiRetryPolicy::default() + }, + hooks, + ); + // Parallel waits at one instant: each ends within 3 s of the + // window's opening, so all fit though they sum to far more. + for _ in 0..32 { + assert!(retry.reserve(Duration::from_secs(2))); + } + assert!( + retry.reserve(Duration::from_secs(3)), + "ends exactly at the close" + ); + assert!( + !retry.reserve(Duration::from_millis(3_001)), + "ends after it" + ); + // 2.5 s later: only a wait ending by the 3 s mark fits. + clock.fetch_add(2_500, SeqCst); + assert!(retry.reserve(Duration::from_millis(500))); + assert!(!retry.reserve(Duration::from_millis(501))); + // Clones share the window; once it has closed nothing fits. + clock.fetch_add(1_000, SeqCst); + let clone = retry.clone(); + assert!(!clone.reserve(Duration::ZERO)); + // A fresh policy client opens a window of its own. + let (hooks, _) = manual_clock(); + let fresh = ApiRetry::with_policy(ApiRetryPolicy::default(), hooks); + assert!(fresh.reserve(Duration::from_secs(60))); + assert!(!fresh.reserve(Duration::from_millis(60_001))); + } + + #[test] + fn only_429_and_503_are_retryable() { + assert!(is_retryable_status(StatusCode::TOO_MANY_REQUESTS)); + assert!(is_retryable_status(StatusCode::SERVICE_UNAVAILABLE)); + for s in [400u16, 401, 403, 404, 408, 500, 502, 504] { + assert!( + !is_retryable_status(StatusCode::from_u16(s).unwrap()), + "{s}" + ); + } + } +} diff --git a/crates/socket-patch-core/src/api/vendor_prefetch.rs b/crates/socket-patch-core/src/api/vendor_prefetch.rs new file mode 100644 index 000000000..ff57163c6 --- /dev/null +++ b/crates/socket-patch-core/src/api/vendor_prefetch.rs @@ -0,0 +1,1356 @@ +//! Vendor-service downloads fetched ahead of the serial vendor loop. +//! +//! The vendor engine wires one package at a time (lockfile and ledger +//! writes stay serial, in sorted order), and each package's service path +//! makes two round trips — the package-reference POST and the archive GET +//! ([`ApiClient::fetch_vendor_package`]). A run that downloads many +//! prebuilt archives paid those round trips back to back. A +//! [`VendorPrefetch`] plan names the uuids the loop is expected to +//! download, in loop order; a background task fetches them ahead of the +//! loop and the loop's own call for a planned uuid takes the fetched +//! outcome instead of making the requests. +//! +//! Nothing observable may change, so the plan is advisory and every +//! decision stays at consumption time, in loop order: +//! +//! * The run-level circuit breaker is evaluated exactly as before — the +//! prefetch never touches its counter. A call the breaker skips never +//! consults the plan (its fetched outcome, if any, is discarded along +//! with its debug lines), and a consumed outcome updates the counter as +//! the live request would have. Outage messages therefore match the +//! serial loop package for package. +//! * A call for a uuid the plan does not hold (or holds only behind the +//! point already consumed) makes the live requests. Planned uuids the +//! loop never asks for (a flavor refused the package first) are dropped +//! when a later one is taken. +//! * Each prefetched request's `--debug` lines are held back and printed +//! when the loop takes its outcome, where the serial request would have +//! printed them. +//! +//! ## What the plan may cost +//! +//! A download grant is a real request against the service — it can start +//! a server-side build and counts against quota — so the plan is EXACT: +//! the CLI names only the packages the loop will ask the service for. It +//! evaluates every refusal a backend raises before its first service call +//! with the backend's own gates — for every ecosystem +//! ([`crate::vendor::service_preflight`], npm's +//! [`crate::vendor::npm_flavor::preflight_packages`]) — and leaves out the +//! re-runs a backend answers without the service, so a package the loop +//! refuses is never granted on its behalf (on a depscan vendored run, 71 +//! grants — the serial loop's own count). The task is still bounded in +//! requests, not just in time, as a second line of defence should a plan +//! ever name a position the loop then passes over: +//! +//! * It only ever requests plan positions in `[at, at + reach)`, where +//! `at` is the position the loop has reached and `reach` is one until +//! the service has answered once, then slow-starts: [`SLOW_START`] (4) +//! after the first answer, one more per good answer up to `window` (the +//! API's in-flight cap), and back to [`SLOW_START`] on an availability +//! failure. So it never runs more than `window` requests ahead of the +//! loop — and only as far as the service's recent answers earned — a +//! run that stops +//! consulting the plan (every remaining package refused, or the loop +//! finishing) leaves at most `window` requests outstanding, and a plan +//! the loop never consults makes no request at all. +//! * It never requests a position the loop has already passed, and starts +//! nothing more once it has seen +//! [`super::client::VENDOR_BREAKER_THRESHOLD`] consecutive availability +//! failures of its own — so a service that is down from the first +//! package costs exactly the retry ladders the serial loop paid before +//! its own breaker opened, not a window of them. What is already in +//! flight when it stops is still drained and delivered: the loop needs +//! those packages, and abandoning them would make it re-issue requests +//! the plan has already paid for. +//! * It never requests PAST a position whose own fetch was an +//! availability failure until the loop has consumed that position. So +//! an outage part-way down a list the window has already widened over +//! costs what the serial loop paid, as long as the task is running +//! ahead of the loop (the usual case: the loop stops to write between +//! packages). Only when the loop has caught up with the window — every +//! package up to it granted, and the first failure the whole window's — +//! can it spend up to `reach - 1` retry ladders the serial loop, one +//! failure from opening its own breaker, would not have spent; `reach` +//! is at most `window`, and reaches it only after that many good +//! answers in a row. +//! +//! Memory is bounded too: at most `window` downloads are in flight, and +//! while the fetched archives waiting for the loop add up to the plan's +//! byte budget, only the position the loop is at may start. The budget +//! gates STARTING a download, not its bytes: the downloads already in +//! flight when it is reached still land, so the held bytes can exceed it +//! by up to `window - 1` archives (whatever their size: the budget bounds +//! new downloads, not the ones already running). Trees a [`PrestageRecipe`] extracts from those archives land +//! on disk, not in memory, and are not counted against the budget at all — +//! they are bounded by the plan (one per planned directory-shaped +//! download, removed as soon as their package is passed over or the loop +//! ends) and by the [`crate::vendor::prestage`] pool, not by size. +//! +//! A planned download may name a secondary artifact (the gem stub +//! gemspec) its backend fetches right after a verified archive; the task +//! fetches it along with the archive, under the backend's own conditions, +//! and the backend's call takes it (see +//! [`super::client::PlannedDownload`]). +//! +//! Outcomes are delivered as they finish, not in plan order: a passed-over +//! download must never hold up the package the loop is actually waiting +//! for (that would make the loop slower than serial, on a request serial +//! never made). `take` puts an early outcome aside until its own call. +//! Dropping the [`VendorPrefetchGuard`] detaches the plan and aborts the +//! task. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; +use std::sync::Arc; + +use futures_util::StreamExt; + +use super::client::{ + hold_back_debug, ApiClient, HeldBack, PlannedDownload, PrefetchedSecondary, + VendorServiceOutcome, VENDOR_BREAKER_THRESHOLD, +}; +use crate::vendor::lock_inventory::LockIntegrity; +use crate::vendor::prestage::PrestageRecipe; +use crate::vendor::registry_fetch::{artifact_matches_integrity, verify_go_h1}; + +/// Where the task's reach opens once the service has answered, and where +/// it falls back to after an availability failure (see +/// [`Lookahead::grow`]). +const SLOW_START: usize = 4; + +/// The reach after one more good answer: [`SLOW_START`] at first, then one +/// more per answer, never past `window`. +fn next_reach(reach: usize, window: usize) -> usize { + let window = window.max(1); + if reach < SLOW_START { + SLOW_START.min(window) + } else { + (reach + 1).min(window) + } +} + +/// One fetched outcome: `(outcome, retryable failure)` as +/// `fetch_vendor_package_once` returned it, debug lines held back. +type Fetched = HeldBack<(VendorServiceOutcome, bool)>; + +/// A planned run of vendor-service downloads; see the module docs. +#[derive(Debug)] +pub(crate) struct VendorPrefetch { + /// The request parameters every planned call must match. + free_only: bool, + vendor_url: Option, + patch_server_url: Option, + /// Planned uuids, in the order the vendor loop consumes them. + planned: Vec, + /// Per planned uuid, what rides its download (see [`PlannedDownload`]): + /// the served secondary artifact the loop's backend downloads right + /// after a verified archive (the gem stub gemspec), and the recipe + /// that stages the archive ahead of the backend. + riders: Vec, + /// Most downloads in flight at once, and the most the task may run + /// ahead of the loop. + window: usize, + /// What the task is allowed to request, shared with it. + look: Arc, + state: tokio::sync::Mutex, +} + +/// The window of plan positions the task may request: `[at, at + reach)`, +/// and never past `barrier`. All three move — `at` as the loop consumes, +/// `reach` once the service has answered, `barrier` as the service fails +/// and as the loop consumes the failure — so the speculation is bounded in +/// REQUESTS both by what the loop has actually reached and by what the +/// service is actually answering (see the module docs). Gating on the +/// position rather than on a count of permits keeps the order +/// deterministic: the futures are polled in whatever order the unordered +/// pool likes, so a counter would hand the opening request to an arbitrary +/// position. +#[derive(Debug)] +struct Lookahead { + /// The plan position the loop is at; everything below it was passed + /// over and must never be requested. + at: AtomicUsize, + /// How far past `at` the task may run: one until the service has + /// answered once, then [`SLOW_START`], growing by one per good answer + /// up to the whole window and falling back on availability failures. + reach: AtomicUsize, + /// Lowest position whose own fetch was an availability failure and + /// that the loop has not consumed yet; nothing past it is started + /// (see [`Self::failed`]). `usize::MAX` while the service is healthy. + barrier: AtomicUsize, + /// Consecutive retryable failures the TASK has seen, in the order its + /// own requests answered. Purely a stop signal for the speculation — + /// the observable breaker is the client's, folded at consumption time. + failures: AtomicU32, + /// Archive bytes fetched and not yet taken (or passed over) by the loop. + held: AtomicUsize, + /// While `held` is at or above this, only the position the loop is at + /// may start: the window bounds how many archives are in flight, this + /// bounds how many finished ones wait in memory for the loop. + budget: usize, + /// Set once `failures` reached the threshold: nothing more is STARTED. + /// Sticky, unlike `failures` itself — a success draining out from + /// behind the failures resets the count, and must not let the + /// speculation resume against a service the loop is giving up on. + stopped: AtomicBool, + /// Woken whenever any of the four above moves. + moved: tokio::sync::Notify, +} + +impl Lookahead { + fn new(budget: usize) -> Self { + Self { + held: AtomicUsize::new(0), + budget, + at: AtomicUsize::new(0), + // Opens at one request: a service that is down from the first + // package then costs what the serial loop cost. + reach: AtomicUsize::new(1), + barrier: AtomicUsize::new(usize::MAX), + failures: AtomicU32::new(0), + stopped: AtomicBool::new(false), + moved: tokio::sync::Notify::new(), + } + } + + /// The loop has reached plan position `position`. + fn arrive(&self, position: usize) { + // Past the failure the barrier stands at: the loop consumed that + // package and went on, so its breaker did not end the run and the + // task may speculate again. (A failure landing concurrently just + // re-sets the line; all of this is advisory, and every outcome is + // still decided at the loop's own call.) + if position > self.barrier.load(Ordering::Relaxed) { + self.barrier.store(usize::MAX, Ordering::Relaxed); + } + self.at.store(position, Ordering::Relaxed); + self.moved.notify_waiters(); + } + + /// The service answered: the task may run further ahead — slow start. + /// The first answer opens the reach to [`SLOW_START`]; every later one + /// adds a position, up to the full `window` (so the reach roughly + /// doubles per round of answers, as a TCP congestion window does). A + /// service that stops answering well only ever faces a reach its own + /// recent answers earned. + fn grow(&self, window: usize) { + let _ = self + .reach + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |reach| { + Some(next_reach(reach, window)) + }); + self.moved.notify_waiters(); + } + + /// An availability failure: the reach falls back to [`SLOW_START`] + /// (never below one), and grows again only with fresh answers. + fn shrink(&self) { + let _ = self + .reach + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |reach| { + Some(reach.clamp(1, SLOW_START)) + }); + self.moved.notify_waiters(); + } + + /// Plan position `index` failed to reach the service: nothing past it + /// is started until the loop has consumed it. The loop is about to + /// meet a failing service at that package and one more failure stops + /// the task, so speculating past it aims retry ladders at a struggling + /// host for packages the serial loop — one failure from opening its + /// own breaker — asked nothing for. + /// + /// Lowest failure wins: a later position's failure must not move the + /// line past an earlier one the loop has yet to reach. + fn failed(&self, index: usize) { + self.barrier.fetch_min(index, Ordering::Relaxed); + self.moved.notify_waiters(); + } + + /// `bytes` of fetched archive left memory (taken or passed over). + fn drained(&self, bytes: usize) { + self.held.fetch_sub(bytes, Ordering::Relaxed); + self.moved.notify_waiters(); + } + + /// The task's own breaker opened: start nothing more, for good. + fn stop(&self) { + self.stopped.store(true, Ordering::Relaxed); + self.moved.notify_waiters(); + } + + /// Wait until plan position `index` may be requested; `false` when it + /// never may be (the loop passed it, or the task's breaker opened). + async fn admits(&self, index: usize) -> bool { + loop { + let notified = self.moved.notified(); + tokio::pin!(notified); + // Armed before the check, so a move between the two is not lost. + notified.as_mut().enable(); + if index < self.at.load(Ordering::Relaxed) || self.stopped.load(Ordering::Relaxed) { + return false; + } + let at = self.at.load(Ordering::Relaxed); + if index <= self.barrier.load(Ordering::Relaxed) + && index < at + self.reach.load(Ordering::Relaxed) + && (index == at || self.held.load(Ordering::Relaxed) < self.budget) + { + return true; + } + notified.await; + } + } +} + +#[derive(Debug, Default)] +struct PrefetchState { + /// First plan position not yet consumed or passed over — where the + /// next lookup starts, so a repeated call never waits on an outcome + /// already taken. + next: usize, + /// Outcomes from the task, tagged with their plan position, as they + /// finish. `None` until the loop's first planned call starts the task. + rx: Option>, + task: Option>, + /// Outcomes that answered before the loop asked for them, by position. + ready: HashMap, +} + +impl Drop for PrefetchState { + fn drop(&mut self) { + if let Some(task) = &self.task { + task.abort(); + } + } +} + +impl VendorPrefetch { + pub(crate) fn new( + planned: Vec, + free_only: bool, + vendor_url: Option<&str>, + patch_server_url: Option<&str>, + window: usize, + byte_budget: usize, + ) -> Self { + let (planned, riders) = planned + .into_iter() + .map(|d| { + ( + d.uuid, + Riders { + secondary: d.secondary, + stage: d.stage, + }, + ) + }) + .unzip(); + Self { + free_only, + vendor_url: vendor_url.map(str::to_string), + patch_server_url: patch_server_url.map(str::to_string), + planned, + riders, + window: window.max(1), + look: Arc::new(Lookahead::new(byte_budget)), + state: tokio::sync::Mutex::new(PrefetchState::default()), + } + } + + /// The prefetched outcome of the loop's call for `uuid`, or `None` to + /// make the live requests. Plan positions before `uuid`'s are passed + /// over (their outcomes discarded unreleased). + pub(crate) async fn take( + &self, + client: &ApiClient, + uuid: &str, + free_only: bool, + vendor_url: Option<&str>, + patch_server_url: Option<&str>, + ) -> Option { + if free_only != self.free_only + || vendor_url != self.vendor_url.as_deref() + || patch_server_url != self.patch_server_url.as_deref() + { + return None; + } + let mut state = self.state.lock().await; + let from = state.next; + let position = from + + self.planned[from..] + .iter() + .position(|planned| planned == uuid)?; + // Pass over every position before this one: the task must not + // request them, anything they already answered is dropped + // (unreleased, with its debug lines), and the loop's arrival here + // is what lets the task run one position further. + state.next = position + 1; + // Everything BELOW this position was passed over; this position's + // own outcome, if it already answered, is the one being taken. + let look = &self.look; + state.ready.retain(|index, fetched| { + let keep = *index >= position; + if !keep { + look.drained(archive_bytes(fetched)); + } + keep + }); + self.look.arrive(position); + if state.task.is_none() { + self.start(&mut state, client, position); + } + let PrefetchState { rx, ready, .. } = &mut *state; + if let Some(fetched) = ready.remove(&position) { + self.look.drained(archive_bytes(&fetched)); + return Some(fetched); + } + let rx = rx.as_mut()?; + loop { + match rx.recv().await { + Some((index, fetched)) if index == position => { + self.look.drained(archive_bytes(&fetched)); + return Some(fetched); + } + // A later position answered first: keep it for its own + // call. An earlier one was passed over — drop it here. + Some((index, fetched)) => { + if index > position { + ready.insert(index, fetched); + } else { + self.look.drained(archive_bytes(&fetched)); + } + } + // The task stopped (its breaker, or the plan ran out) and + // this position never came: fetch live from here on. + _ => break, + } + } + state.rx = None; + state.next = self.planned.len(); + self.look.arrive(self.planned.len()); + None + } + + /// Spawn the task fetching `planned[from..]`, at most `window` at once. + fn start(&self, state: &mut PrefetchState, client: &ApiClient, from: usize) { + let (tx, rx) = tokio::sync::mpsc::channel(self.window); + let client = client.clone(); + let planned: Vec<(String, Riders)> = self.planned[from..] + .iter() + .cloned() + .zip(self.riders[from..].iter().cloned()) + .collect(); + let (free_only, window) = (self.free_only, self.window); + let vendor_url = self.vendor_url.clone(); + let patch_server_url = self.patch_server_url.clone(); + let look = Arc::clone(&self.look); + state.task = Some(tokio::spawn(async move { + let (client, vendor_url, patch_server_url) = + (&client, vendor_url.as_deref(), patch_server_url.as_deref()); + let look = &look; + // UNORDERED on purpose, unlike every folding loop in the CLI: + // nothing observable is folded here, and a passed-over + // download must not delay the one the loop is waiting for. + // `take` puts each outcome back on its own call. + let mut fetched = + std::pin::pin!(futures_util::stream::iter(planned.into_iter().enumerate()) + .map( + move |(offset, (uuid, riders)): (usize, (String, Riders))| async move { + let index = from + offset; + if !look.admits(index).await { + return None; + } + let mut held = hold_back_debug(client.fetch_vendor_package_once( + &uuid, + free_only, + vendor_url, + patch_server_url, + )) + .await; + ride_along(client, &mut held, riders).await; + Some((index, held)) + } + ) + .buffer_unordered(window)); + while let Some(item) = fetched.next().await { + let Some((index, held)) = item else { continue }; + // Counted before the send, released by `take`. + look.held.fetch_add(archive_bytes(&held), Ordering::Relaxed); + let availability_failure = + matches!(held.peek(), (VendorServiceOutcome::Failed(_), true)); + if availability_failure { + look.failures.fetch_add(1, Ordering::Relaxed); + // Set BEFORE the send below, which can yield: the next + // poll of the stream is what pulls a new position in, + // and it must already see the line. + look.shrink(); + look.failed(index); + } else { + if !matches!(held.peek(), (VendorServiceOutcome::Failed(_), false)) { + // Anything but a failure proves the service is up. A + // NON-retryable failure (auth, parse) says nothing + // about availability either way, so it neither counts + // nor resets — exactly the client breaker's rule. + look.failures.store(0, Ordering::Relaxed); + } + look.grow(window); + } + if look.failures.load(Ordering::Relaxed) >= VENDOR_BREAKER_THRESHOLD { + // Start nothing more — but keep draining. The requests + // already in flight are for packages BEHIND the + // failures, which the loop has yet to reach and will + // otherwise re-issue live; dropping the stream here + // would make the outage cost the service each of them + // twice. Positions not yet started cost nothing: + // `admits` refuses them without a request. + look.stop(); + } + if tx.send((index, held)).await.is_err() { + return; + } + } + })); + state.rx = Some(rx); + } +} + +/// The archive bytes a fetched outcome holds in memory. +fn archive_bytes(fetched: &Fetched) -> usize { + match fetched.peek() { + (VendorServiceOutcome::Ready(pkg), _) => pkg.tarball.len(), + _ => 0, + } +} + +/// What rides one planned download (see [`PlannedDownload`]). +#[derive(Debug, Clone)] +struct Riders { + secondary: Option, + stage: Option, +} + +/// Do what rides a planned download — exactly when the loop's backend +/// would get that far: the archive is ready and passes the integrity checks +/// `fetch_verified_archive` runs before handing it over. +/// +/// * The secondary artifact of `kind`, when the service served one (the +/// first, as `fetch_verified_secondary` picks), is downloaded with its +/// debug lines held on the artifact for `fetch_verified_secondary` to +/// take. +/// * The stage recipe runs on the bytes (see [`crate::vendor::prestage`]); +/// what it produced rides the archive to the backend. +async fn ride_along(client: &ApiClient, held: &mut Fetched, riders: Riders) { + let Riders { secondary, stage } = riders; + if secondary.is_none() && stage.is_none() { + return; + } + let (VendorServiceOutcome::Ready(pkg), _) = held.peek_mut() else { + return; + }; + let intact = artifact_matches_integrity( + &pkg.tarball, + "", + &LockIntegrity::Sri(pkg.integrity_sri.clone()), + ) + .is_ok() + && pkg + .dirhash_h1 + .as_deref() + .is_none_or(|h1| verify_go_h1(&pkg.tarball, h1).is_ok()); + if !intact { + return; + } + if let Some(kind) = secondary { + if let Some(artifact) = pkg.secondary_artifacts.iter_mut().find(|a| a.kind == kind) { + let downloaded = hold_back_debug(client.download_artifact(&artifact.url)).await; + artifact.prefetched = Some(PrefetchedSecondary::new(downloaded)); + } + } + if let Some(recipe) = stage { + let (bytes, prestaged) = + crate::vendor::prestage::run(&recipe, std::mem::take(&mut pkg.tarball)).await; + pkg.tarball = bytes; + pkg.prestaged = prestaged; + } +} + +/// Keeps a [`VendorPrefetch`] plan attached to its client; dropping it +/// detaches the plan and aborts any downloads still in flight. +#[must_use = "the plan is detached when the guard drops"] +#[derive(Debug)] +pub struct VendorPrefetchGuard { + pub(crate) slot: Arc>>>, + /// The plan this guard attached. Only that one is detached on drop, so + /// a guard outliving a plan attached after it cannot take the newer + /// plan down with it. + pub(crate) plan: Arc, +} + +impl Drop for VendorPrefetchGuard { + fn drop(&mut self) { + if let Ok(mut slot) = self.slot.lock() { + if slot + .as_ref() + .is_some_and(|attached| Arc::ptr_eq(attached, &self.plan)) + { + slot.take(); + } + } + } +} + +#[cfg(test)] +mod tests { + //! Oracle tests: every call sequence yields, call for call, exactly the + //! outcomes (and final breaker count) of the same sequence with no plan + //! attached — the serial loop. + use super::*; + use crate::api::client::{ApiClientOptions, VendorRetryPolicy}; + use base64::Engine as _; + use sha2::{Digest as _, Sha512}; + use std::time::Duration; + use wiremock::matchers::{body_partial_json, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + const POST_PATH: &str = "/v0/orgs/acme/patches/package"; + + /// How the service answers one uuid. + #[derive(Clone, Copy)] + enum Script { + /// Granted, with this delay on the POST. + Granted(u64), + /// 503 on every POST attempt (a retryable availability failure). + Down, + /// 403 (a non-retryable failure: says nothing about availability). + Forbidden, + Pending, + NotFound, + } + + fn uuid(i: usize) -> String { + format!("{i:08x}-0000-4000-8000-{i:012x}") + } + + fn client(uri: &str) -> ApiClient { + ApiClient::new(ApiClientOptions { + api_url: uri.to_string(), + api_token: Some("sktsec_placeholder_value_for_tests_api".into()), + use_public_proxy: false, + org_slug: Some("acme".into()), + }) + .with_vendor_retry(VendorRetryPolicy { + attempts: 3, + base: Duration::from_millis(1), + max_delay: Duration::from_millis(5), + ..VendorRetryPolicy::default() + }) + } + + async fn serve(scripts: &[Script]) -> MockServer { + let server = MockServer::start().await; + for (i, script) in scripts.iter().enumerate() { + let u = uuid(i); + let serve_path = format!("/serve/{u}.tgz"); + let bytes = u.as_bytes().to_vec(); + let post = |resp: ResponseTemplate| { + Mock::given(method("POST")) + .and(path(POST_PATH)) + .and(body_partial_json( + serde_json::json!({ "uuids": [u.clone()] }), + )) + .respond_with(resp) + }; + let status_body = |status: &str| { + ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "results": { u.clone(): { "status": status, "url": null, "artifacts": [] } } + })) + }; + let resp = match *script { + Script::Granted(delay) => { + let url = format!("{}{serve_path}", server.uri()); + let sri = format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(Sha512::digest(&bytes)) + ); + ResponseTemplate::new(200) + .set_body_json(serde_json::json!({ + "results": { u.clone(): { "status": "granted", "url": url, + "artifacts": [{ "kind": "tarball", "url": url, + "integrity": { "sha512": sri } }] } } + })) + .set_delay(Duration::from_millis(delay)) + } + Script::Down => ResponseTemplate::new(503), + Script::Forbidden => ResponseTemplate::new(403), + Script::Pending => status_body("pending_build"), + Script::NotFound => status_body("not_found"), + }; + post(resp).mount(&server).await; + Mock::given(method("GET")) + .and(path(serve_path)) + .respond_with(ResponseTemplate::new(200).set_body_bytes(bytes)) + .mount(&server) + .await; + } + server + } + + fn summary(outcome: &VendorServiceOutcome) -> String { + match outcome { + VendorServiceOutcome::Ready(pkg) => format!( + "ready {} {} {}", + String::from_utf8_lossy(&pkg.tarball), + pkg.integrity_sri, + pkg.source_url + ), + VendorServiceOutcome::Pending => "pending".to_string(), + VendorServiceOutcome::Unavailable(reason) => format!("unavailable {reason}"), + VendorServiceOutcome::Failed(e) => format!("failed {e}"), + } + } + + /// Requests the mock server saw, as `"METHOD /path"`. + async fn request_log(server: &MockServer) -> Vec { + server + .received_requests() + .await + .unwrap() + .iter() + .map(|r| format!("{} {}", r.method, r.url.path())) + .collect() + } + + /// Run `calls` (indices into the scripted uuids) one at a time, with + /// `plan` attached when given; the per-call outcomes and final count. + async fn run( + server: &MockServer, + plan: Option<&[usize]>, + calls: &[usize], + ) -> (Vec, u32) { + let c = client(&server.uri()); + let _guard = plan.map(|plan| { + c.prefetch_vendor_packages( + plan.iter().map(|&i| uuid(i)).collect(), + false, + None, + None, + 4, + ) + }); + let mut out = Vec::new(); + for &i in calls { + out.push(summary( + &c.fetch_vendor_package(&uuid(i), false, None, None).await, + )); + } + (out, c.vendor_outage_count()) + } + + async fn assert_matches_serial(scripts: &[Script], plan: &[usize], calls: &[usize]) { + let server = serve(scripts).await; + let serial = run(&server, None, calls).await; + let planned = run(&server, Some(plan), calls).await; + assert_eq!(planned, serial); + } + + /// Later packages answer first (reversed latencies); every outcome + /// still lands on its own call. + #[tokio::test] + async fn outcomes_land_on_their_own_calls_under_reversed_latencies() { + let scripts: Vec