From 7115db4303d75452897381facd022a66b85091d1 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Sun, 27 Sep 2026 17:44:28 -0400 Subject: [PATCH 01/66] Stop writing the hosted redirect ledger from scan Hosted scan keeps its edits and records in memory only; the lockfiles are the record of a redirect. Replays the parked WIP (which was snapshotted on an older tree) as just its hosted.rs delta. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commands/scan/hosted.rs | 130 ++++-------------- 1 file changed, 28 insertions(+), 102 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 156f8bf40..e250d1a24 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -1409,15 +1409,12 @@ pub(crate) async fn run_redirect_selected( // The apply lock (see `acquire_hosted_lock`), taken only by a WET run // that holds at least one granted reference — the only runs that can // write anything: the takeover pre-reverts (lockfiles + the vendored - // ledger), the redirect-ledger merge and the lockfile writes. Dry runs + // ledger) and the lockfile writes. Dry runs // and zero-grant runs never touch `.socket/`, so they never lock (a // preview must not create `.socket/`, flip to `lock_held` under a - // concurrent wet run, or fail on a read-only checkout). Acquired BEFORE - // the ledger load so load → merge → persist is one critical section - // (rollback's rule: a ledger a run will persist is loaded under the - // lock) and held to the end of the function. Read below: it also gates - // the corrupt-ledger quarantine, the one write the load itself can make. - let lock: Option = if !common.dry_run && !candidates.is_empty() { + // concurrent wet run, or fail on a read-only checkout). Held to the end + // of the function. + let _lock: Option = if !common.dry_run && !candidates.is_empty() { match acquire_hosted_lock(common, &mut scan_result) { Ok(guard) => Some(guard), Err(code) => return code, @@ -1426,36 +1423,13 @@ pub(crate) async fn run_redirect_selected( None }; - // Load the existing redirect ledger before any file changes, including - // Cargo takeover reverts. It stores the originals a future revert needs, so - // a malformed (torn/hand-mangled) ledger must abort the run while the - // project is still untouched, or the merge below would silently - // overwrite that revert data. The malformed file is moved aside to - // redirect-state.json.corrupt (never clobbered) — but only by a run - // holding the apply lock; a dry run or zero-grant run reports the same - // error and moves nothing, so `.socket/vendor/` is never mutated - // lock-free. - // - // Held as the ONE in-memory ledger for the whole run: the write below - // merges into it in place, the stale-install probes read its records - // (persisted ones are the fallback when this run's /patches/view fetch - // fails transiently), and the takeover classification at the end reads - // the merged state. - let mut ledger = - match socket_patch_core::patch::redirect::load_redirect_state(&common.cwd).await { - Ok(state) => state.unwrap_or_else(RedirectState::new), - Err(mut corrupt) => { - if lock.is_some() { - corrupt.quarantine().await; - } - let message = corrupt.to_string(); - eprintln!("{}", format_error_line(&message)); - if common.json { - emit_json_error(scan_result.take(), &message); - } - return 1; - } - }; + // v5 hosted mode keeps no ledger: the lockfiles are the only record of + // a redirect (vex, list, vendor and rollback read the hosted pins from + // them). This run's edits and records still collect here in memory, + // for the edit rebasing below, the stale-install probes and the + // takeover classification. A pre-v5 ledger on disk is left untouched: + // it only goes stale, and replaying a stale edit fails closed. + let mut ledger = RedirectState::new(); // The vendored ledger, loaded ONCE per run (under the same lock, so no // other writer can move the on-disk file under it): the takeover below // mutates it in place per reverted purl (saving after each), and the @@ -2817,23 +2791,10 @@ pub(crate) async fn run_redirect_selected( status.finish(); } - // Whether this run persisted the redirect ledger (human next steps). - let mut ledger_written = false; if !common.dry_run { - // Ledger (mirrors the vendor state.json shape): recorded edits for a - // future revert + the patch records (file hashes + vulnerabilities) so - // a post-install `socket-patch vex` can attest the redirected patches. - // MERGE with any existing ledger rather than overwriting: an idempotent - // re-run produces no new edits (the lockfile already points at the - // hosted patch), and clobbering the file would lose the original - // pre-redirect values a future revert needs. New edits APPEND (revert - // walks them in reverse), skipping byte-identical re-plans from a - // retried partial failure; records are keyed by PURL, newest wins. - // - // Persisted BEFORE the project files, and atomically (stage + fsync + - // rename): a crash between the two leaves a complete ledger whose - // originals match files never rewritten, never rewritten files whose - // originals reached no ledger. + // Fold this run's edits and records into the in-memory ledger. + // New edits append, skipping byte-identical re-plans; records are + // keyed by purl, newest wins. if !rewrite.edits.is_empty() || !records.is_empty() { // Older ledgers carry `"mode": "redirect"`; normalize on rewrite // (the loader accepts either). @@ -2933,20 +2894,6 @@ pub(crate) async fn run_redirect_selected( } } ledger.records.extend(records); - // The ledger is the only revert path and the VEX record store — - // a swallowed write failure would let the lockfile writes below - // proceed with no revert data persisted while reporting success. - let saved = - socket_patch_core::patch::redirect::save_redirect_state(&common.cwd, &ledger).await; - ledger_written = saved.is_ok(); - if let Err(e) = saved { - let message = format!("failed to write .socket/vendor/redirect-state.json: {e}"); - eprintln!("{}", format_error_line(&message)); - if common.json { - emit_json_error(scan_result.take(), &message); - } - return 1; - } } for (rel, content) in rewrite .files @@ -3312,7 +3259,7 @@ pub(crate) async fn run_redirect_selected( } if !common.dry_run { for line in - format_next_steps(&human_files, ledger_written, !takeover_migrated.is_empty()) + format_next_steps(&human_files, !takeover_migrated.is_empty()) { println!("{line}"); } @@ -3614,30 +3561,17 @@ fn join_names(names: &[String], max: usize) -> String { /// Next steps after a wet run that rewrote files (stdout, after the /// summary — the same place vendored mode prints its own): commit the -/// ledger and the rewritten files, reinstall so the installed tree picks -/// up the patched artifacts, then verify with `vex`. After a -/// vendored→hosted takeover (`vendored_removed`) the commit also has to -/// carry the deleted vendored ledger entries and artifacts, so the whole -/// `.socket/vendor/` directory is named instead of the redirect ledger. -fn format_next_steps( - files: &[String], - ledger_written: bool, - vendored_removed: bool, -) -> Vec { +/// rewritten files, reinstall so the installed tree picks up the patched +/// artifacts, then verify with `vex`. After a vendored→hosted takeover +/// (`vendored_removed`) the commit also has to carry the deleted vendored +/// ledger entries and artifacts. +fn format_next_steps(files: &[String], vendored_removed: bool) -> Vec { if files.is_empty() && !vendored_removed { return Vec::new(); } let mut commit: Vec = Vec::new(); if vendored_removed { - commit.push(if ledger_written { - ".socket/vendor/ (the redirect ledger, plus the removed vendored ledger entries and \ - artifacts)" - .to_string() - } else { - ".socket/vendor/ (the removed vendored ledger entries and artifacts)".to_string() - }); - } else if ledger_written { - commit.push(".socket/vendor/redirect-state.json".to_string()); + commit.push(".socket/vendor/ (the removed vendored ledger entries and artifacts)".to_string()); } commit.extend(files.iter().cloned()); let npm = files @@ -5379,14 +5313,12 @@ mod tests { } #[test] - fn next_steps_name_the_ledger_files_and_reinstall() { - assert!(format_next_steps(&[], true, false).is_empty()); + fn next_steps_name_the_rewritten_files_and_reinstall() { + assert!(format_next_steps(&[], false).is_empty()); assert_eq!( - format_next_steps(&["package-lock.json".to_string()], true, false), + format_next_steps(&["package-lock.json".to_string()], false), vec![ - "Commit .socket/vendor/redirect-state.json and package-lock.json to keep the \ - redirect." - .to_string(), + "Commit package-lock.json to keep the redirect.".to_string(), "Reinstall from the updated lockfile (e.g. `npm ci`) so the installed packages \ pick up the patched artifacts, then run `socket-patch vex` to verify them." .to_string(), @@ -5398,7 +5330,6 @@ mod tests { "pnpm-workspace.yaml".to_string(), ], false, - false, ); assert_eq!( steps[0], @@ -5409,13 +5340,13 @@ mod tests { #[test] fn next_steps_add_the_vlt_ci_line_only_for_a_rewritten_vlt_lock() { - let steps = format_next_steps(&["vlt-lock.json".to_string()], true, false); + let steps = format_next_steps(&["vlt-lock.json".to_string()], false); assert_eq!( steps.last().map(String::as_str), Some("vlt: commit vlt-lock.json; CI should run `vlt ci`") ); assert!( - !format_next_steps(&["package-lock.json".to_string()], true, false) + !format_next_steps(&["package-lock.json".to_string()], false) .iter() .any(|s| s.starts_with("vlt:")) ); @@ -5593,12 +5524,7 @@ mod tests { #[test] fn next_steps_after_a_takeover_name_the_removed_vendored_state() { assert_eq!( - format_next_steps(&["package-lock.json".to_string()], true, true)[0], - "Commit .socket/vendor/ (the redirect ledger, plus the removed vendored ledger \ - entries and artifacts) and package-lock.json to keep the redirect." - ); - assert_eq!( - format_next_steps(&["pnpm-lock.yaml".to_string()], false, true)[0], + format_next_steps(&["pnpm-lock.yaml".to_string()], true)[0], "Commit .socket/vendor/ (the removed vendored ledger entries and artifacts) and \ pnpm-lock.yaml to keep the redirect." ); From f0e39217d69c34f010eb12b5f543ca557fe1dc5c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 22:24:26 +0000 Subject: [PATCH 02/66] Restore hosted pins to their upstream registry entries instead of replaying the ledger Imports the stopped local WS1 agent's work-in-progress (backup/local-v5- ledger-free-hosted): core patch::redirect::upstream re-resolves npm-family, cargo and golang registry entries for every hosted pin vex::discover finds in the lockfiles, and rollback/remove/vendor route their hosted legs through it instead of the redirect ledger. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018tncU7u7FK48tubFNA12UQ --- .../socket-patch-cli/src/commands/remove.rs | 298 +++---- .../socket-patch-cli/src/commands/rollback.rs | 374 +++------ .../socket-patch-cli/src/commands/vendor.rs | 365 ++++----- .../src/patch/redirect/mod.rs | 1 + .../src/patch/redirect/pnpm.rs | 33 + .../src/patch/redirect/upstream/cargo.rs | 380 +++++++++ .../src/patch/redirect/upstream/client.rs | 308 ++++++++ .../src/patch/redirect/upstream/golang.rs | 113 +++ .../src/patch/redirect/upstream/mod.rs | 416 ++++++++++ .../src/patch/redirect/upstream/npm.rs | 742 ++++++++++++++++++ .../src/patch/redirect/vlt_heal.rs | 23 + .../socket-patch-core/src/vendor/berry_zip.rs | 2 +- crates/socket-patch-core/src/vendor/mod.rs | 2 +- .../src/vendor/registry_fetch.rs | 14 +- .../socket-patch-core/src/vex/discover/mod.rs | 15 + .../tests/upstream_restore_golden.rs | 340 ++++++++ 16 files changed, 2744 insertions(+), 682 deletions(-) create mode 100644 crates/socket-patch-core/src/patch/redirect/upstream/cargo.rs create mode 100644 crates/socket-patch-core/src/patch/redirect/upstream/client.rs create mode 100644 crates/socket-patch-core/src/patch/redirect/upstream/golang.rs create mode 100644 crates/socket-patch-core/src/patch/redirect/upstream/mod.rs create mode 100644 crates/socket-patch-core/src/patch/redirect/upstream/npm.rs create mode 100644 crates/socket-patch-core/tests/upstream_restore_golden.rs diff --git a/crates/socket-patch-cli/src/commands/remove.rs b/crates/socket-patch-cli/src/commands/remove.rs index 9ce954018..8bb5412a6 100644 --- a/crates/socket-patch-cli/src/commands/remove.rs +++ b/crates/socket-patch-cli/src/commands/remove.rs @@ -3,9 +3,7 @@ use socket_patch_core::api::client::get_api_client_with_overrides; use socket_patch_core::manifest::cleanup_blobs::format_bytes; use socket_patch_core::manifest::operations::{read_manifest, write_manifest}; use socket_patch_core::manifest::schema::PatchManifest; -use socket_patch_core::patch::redirect::{ - load_redirect_state, persist_redirect_state, RedirectState, REDIRECT_STATE_REL, -}; +use socket_patch_core::patch::redirect::upstream::HostedPin; use socket_patch_core::telemetry::{track_patch_remove_failed, track_patch_removed}; use socket_patch_core::utils::purl::patch_matches; use socket_patch_core::vendor::{ @@ -38,15 +36,15 @@ fn vendor_entries_matching(state: &VendorState, identifier: &str) -> Vec<(String matches } -/// Hosted redirect records matching a remove identifier, sorted. -fn hosted_records_matching(state: &RedirectState, identifier: &str) -> Vec { - let mut matches: Vec = state - .records +/// The lockfiles' hosted pins matching a remove identifier (by purl or +/// patch uuid), sorted by purl. +fn hosted_pins_matching(pins: &[HostedPin], identifier: &str) -> Vec { + let mut matches: Vec = pins .iter() - .filter(|(purl, rec)| patch_matches(purl, &rec.uuid, identifier)) - .map(|(purl, _)| purl.clone()) + .filter(|pin| patch_matches(&pin.purl, &pin.uuid, identifier)) + .cloned() .collect(); - matches.sort(); + matches.sort_by(|a, b| a.purl.cmp(&b.purl)); matches } @@ -340,22 +338,22 @@ pub async fn run(args: RemoveArgs) -> i32 { let cwd = &args.common.cwd; // ── state discovery ───────────────────────────────────────────────── - // A ledger-only project (vendored mode keeps its records in the vendor - // ledger, hosted mode in the redirect ledger — neither writes a - // manifest) proceeds manifest-less: `remove` is the per-purl exit path - // for those entries. Only cheap EXISTENCE probes run before the lock — - // they decide the truly-empty error path, which never locks (a bare - // project must not see `.socket/` created and pruned again). The - // stores themselves are loaded under the lock below. + // A manifest-less project (vendored mode keeps its records in the + // vendor ledger; hosted mode keeps none — its lockfile pins are the + // record) proceeds manifest-less: `remove` is the per-purl exit path + // for those entries. Only cheap probes run before the lock — they + // decide the truly-empty error path, which never locks (a bare project + // must not see `.socket/` created and pruned again). The vendor ledger + // is loaded under the lock below; the hosted pins come from read-only + // lockfile discovery (the restore re-reads every file under the lock). let manifest_missing = tokio::fs::metadata(&manifest_path).await.is_err(); + let hosted_pins: Vec = + HostedPin::all(&crate::commands::discover_wiring(&args.common, cwd).await); if manifest_missing { let vendor_ledger_exists = tokio::fs::metadata(cwd.join(VENDOR_STATE_REL)) .await .is_ok(); - let redirect_ledger_exists = tokio::fs::metadata(cwd.join(REDIRECT_STATE_REL)) - .await - .is_ok(); - if !vendor_ledger_exists && !redirect_ledger_exists { + if !vendor_ledger_exists && hosted_pins.is_empty() { emit_error_envelope( args.common.json, args.common.dry_run, @@ -453,23 +451,18 @@ pub async fn run(args: RemoveArgs) -> i32 { } } - // Hosted-only patches likewise have no manifest entry — the - // redirect ledger is their only persistence, and `remove` is - // their per-purl exit path (the unwind IS the removal). An - // unreadable ledger falls through to `not_found`: nothing is - // mutated on that path. - if let Ok(Some(redirect_state)) = load_redirect_state(cwd).await { - let hosted_matches = hosted_records_matching(&redirect_state, &args.identifier); - if !hosted_matches.is_empty() { - return remove_hosted_only( - &args, - hosted_matches, - redirect_state, - api_token.as_deref(), - org_slug.as_deref(), - ) - .await; - } + // Hosted-only patches likewise have no manifest entry — their + // lockfile pins are their only persistence, and `remove` is their + // per-purl exit path (restoring the upstream entry IS the removal). + let hosted_matches = hosted_pins_matching(&hosted_pins, &args.identifier); + if !hosted_matches.is_empty() { + return remove_hosted_only( + &args, + hosted_matches, + api_token.as_deref(), + org_slug.as_deref(), + ) + .await; } emit_not_found( @@ -516,8 +509,8 @@ pub async fn run(args: RemoveArgs) -> i32 { // `--dry-run` previews without mutating, so there is nothing to // confirm — skip the prompt (matching the global contract row: // "Preview, no mutations"). The prompt names every leg the removal - // will touch: the redirect ledger is probed read-only here (the legs - // below re-load it and decide for real). + // will touch: the hosted pins come from the read-only discovery above + // (the legs below decide for real). if !args.common.dry_run { let (vendored, hosted) = if args.skip_rollback { (0, 0) @@ -526,12 +519,7 @@ pub async fn run(args: RemoveArgs) -> i32 { .as_ref() .map(|st| vendor_entries_matching(st, &args.identifier).len()) .unwrap_or(0); - let hosted = load_redirect_state(cwd) - .await - .ok() - .flatten() - .map(|st| hosted_records_matching(&st, &args.identifier).len()) - .unwrap_or(0); + let hosted = hosted_pins_matching(&hosted_pins, &args.identifier).len(); (vendored, hosted) }; let prompt = remove_prompt( @@ -745,75 +733,50 @@ pub async fn run(args: RemoveArgs) -> i32 { } // ── hosted leg ────────────────────────────────────────────────────── - // An identifier can also (or only) match hosted records in the - // redirect ledger. Supported ecosystems (cargo, npm-family, golang) unwind - // per-purl; when the identifier covers EVERY record the whole-ledger - // replay serves the rest; otherwise unsupported targets fail closed - // BEFORE the manifest mutation. A corrupt ledger skips the leg with a - // warning (the identifier may still match other stores). + // An identifier can also (or only) match hosted pins in the lockfiles. + // Each is restored to its default upstream registry entry; a pin that + // cannot be fails closed BEFORE the manifest mutation. // `--skip-rollback` leaves hosted wiring untouched, like the vendor - // wiring above; `--preserve-state` still unwinds — hosted has no + // wiring above; `--preserve-state` still restores — hosted has no // preservable local state. let mut hosted_reverted_events: Vec = Vec::new(); - // The hosted leg's run-level advisories (e.g. - // `redirect_npmrc_allow_remote_modified`): printed as they arrive, + // The hosted leg's run-level advisories: printed as they arrive, // carried into the success envelope's `warnings[]`. let mut hosted_leg_warnings: Vec<(String, String)> = Vec::new(); if !args.skip_rollback { - match load_redirect_state(cwd).await { - Err(e) => { - if loud { - eprintln!( - "Warning: cannot read the hosted redirect ledger ({e}); hosted \ - redirects were not examined" - ); + let hosted_matches = hosted_pins_matching(&hosted_pins, &args.identifier); + if !hosted_matches.is_empty() { + let leg = match unwind_hosted(&args.common, &hosted_matches).await { + Ok(leg) => { + hosted_leg_warnings.extend(leg.warnings.iter().cloned()); + leg } - } - Ok(None) => {} - Ok(Some(mut redirect_state)) => { - let hosted_matches = hosted_records_matching(&redirect_state, &args.identifier); - if !hosted_matches.is_empty() { - let leg = - match unwind_hosted(&args.common, &hosted_matches, &mut redirect_state) - .await - { - Ok(leg) => { - hosted_leg_warnings.extend(leg.warnings.iter().cloned()); - leg - } - Err(err) => { - let (code, msg) = hosted_unwind_error(err, true); - emit_error_envelope( - args.common.json, - args.common.dry_run, - code, - msg, - ); - return 1; - } - }; - if args.preserve_state && !leg.reverted.is_empty() && loud { - eprintln!( - "Note: hosted redirects have no preservable local state; \ - their ledger records were dropped with the unwound wiring." - ); - } - // `run_hosted_leg` printed one line per unwound purl. - printed_progress |= loud && !leg.reverted.is_empty(); - let hosted_action = if args.common.dry_run { - PatchAction::Verified - } else { - PatchAction::Removed - }; - for purl in &leg.reverted { - hosted_reverted_events.push( - PatchEvent::new(hosted_action, purl.clone()).with_reason( - "hosted_reverted", - "hosted lockfile redirect unwound on remove", - ), - ); - } + Err(err) => { + let (code, msg) = hosted_unwind_error(err, true); + emit_error_envelope(args.common.json, args.common.dry_run, code, msg); + return 1; } + }; + if args.preserve_state && !leg.reverted.is_empty() && loud { + eprintln!( + "Note: hosted wiring has no preservable local state; its lockfile pins \ + now resolve upstream." + ); + } + // `run_hosted_leg` printed one line per restored purl. + printed_progress |= loud && !leg.reverted.is_empty(); + let hosted_action = if args.common.dry_run { + PatchAction::Verified + } else { + PatchAction::Removed + }; + for purl in &leg.reverted { + hosted_reverted_events.push( + PatchEvent::new(hosted_action, purl.clone()).with_reason( + "hosted_reverted", + "hosted lockfile pin restored to the upstream registry on remove", + ), + ); } } } @@ -1299,86 +1262,54 @@ async fn revert_vendored_matches( Ok(leg) } -/// Why a hosted unwind stopped. Each caller renders its own message (the +/// Why a hosted unwind stopped: a pin the upstream restore refused (or a +/// write failure). Each caller renders its own message (the /// manifest-backed path adds that the manifest was not touched). -enum HostedUnwindError { - /// The ledger could not be persisted after the reverts flushed. - Persist(String), - /// Scoped targets whose ecosystem has no per-purl hosted revert. - Unsupported(Vec), - /// A per-purl revert (or the whole-ledger replay) refused. - Failed { what: String, why: String }, +struct HostedUnwindError { + what: String, + why: String, } -/// Unwind the hosted redirect records in `hosted_matches` and persist the -/// ledger — FIRST, failure or not: the per-purl reverts flush lockfile -/// writes as they go, so an early error return without persisting would -/// strand already-reverted purls' records in the on-disk ledger (lockfiles -/// and ledger desynced; `list`/VEX attest dead wiring). When the matches -/// cover EVERY record the whole-ledger replay serves the ecosystems without -/// a per-purl revert. Shared by the manifest-backed and hosted-only remove -/// paths. +/// Restore the hosted pins in `hosted_matches` to their upstream registry +/// entries. Nothing is written unless every pin resolved (the restore is +/// all-or-nothing per pin, and a refused pin fails the remove). Shared by +/// the manifest-backed and hosted-only remove paths. async fn unwind_hosted( common: &GlobalArgs, - hosted_matches: &[String], - state: &mut RedirectState, + hosted_matches: &[HostedPin], ) -> Result { - let replay_eligible = state.records.keys().all(|p| hosted_matches.contains(p)); - let before = (state.edits.len(), state.records.len()); - let leg = run_hosted_leg(common, hosted_matches, state, replay_eligible).await; + let leg = run_hosted_leg(common, hosted_matches).await; // Printed as soon as the leg returns, so a human run that then fails // still says what it did to the files. print_hosted_leg_warnings(common, &leg.warnings); - if !common.dry_run && (state.edits.len(), state.records.len()) != before { - if let Err(e) = persist_redirect_state(&common.cwd, state).await { - return Err(HostedUnwindError::Persist(e.to_string())); - } - } - if !leg.unsupported.is_empty() { - return Err(HostedUnwindError::Unsupported(leg.unsupported)); - } if let Some((what, why)) = leg.failed.first().cloned() { - return Err(HostedUnwindError::Failed { what, why }); + return Err(HostedUnwindError { what, why }); + } + if let Some(warning) = super::rollback::retire_legacy_redirect_ledger(common).await { + print_hosted_leg_warnings(common, std::slice::from_ref(&warning)); } Ok(leg) } /// Error code + message for a stopped hosted unwind. fn hosted_unwind_error(err: HostedUnwindError, manifest_backed: bool) -> (&'static str, String) { - let note = if manifest_backed { - " The manifest was not modified." - } else { - "" - }; - match err { - HostedUnwindError::Persist(e) => ( - "hosted_revert_failed", - format!("failed to persist the hosted redirect ledger: {e}"), - ), - HostedUnwindError::Unsupported(purls) => ( - "hosted_revert_unsupported", + let HostedUnwindError { what, why } = err; + ( + "hosted_revert_failed", + if manifest_backed { format!( - "no per-purl hosted-redirect revert exists for: {}. Run an unscoped \ - `socket-patch rollback` to unwind ALL hosted redirects, or re-run \ - `scan --mode hosted` to normalize.{note}", - purls.join(", ") - ), - ), - HostedUnwindError::Failed { what, why } => ( - "hosted_revert_failed", - if manifest_backed { - format!("could not unwind hosted redirect for {what}: {why}.{note}") - } else { - format!("could not unwind hosted redirect for {what}: {why}") - }, - ), - } + "could not restore {what} to its upstream registry entry: {why}. The manifest \ + was not modified." + ) + } else { + format!("could not restore {what} to its upstream registry entry: {why}") + }, + ) } -/// Remove path for identifiers that match ONLY hosted redirect records -/// (no manifest entry, no vendor-ledger entry): confirm, unwind each -/// record's lockfile wiring, drop it from the redirect ledger, and report -/// `Removed`/`hosted_reverted` events. Like the ledger-only vendored path, +/// Remove path for identifiers that match ONLY hosted lockfile pins (no +/// manifest entry, no vendor-ledger entry): confirm, restore each pin's +/// upstream registry entry, and report `Removed`/`hosted_reverted` events. Like the ledger-only vendored path, /// the unwind IS the removal, so events go through `env.record` and bump /// `summary.removed`. `--skip-rollback` is refused (with no manifest /// entry to delete, removing a hosted patch can only mean unwinding its @@ -1386,8 +1317,7 @@ fn hosted_unwind_error(err: HostedUnwindError, manifest_backed: bool) -> (&'stat /// preservable local state. async fn remove_hosted_only( args: &RemoveArgs, - hosted_matches: Vec, - mut redirect_state: RedirectState, + hosted_matches: Vec, api_token: Option<&str>, org_slug: Option<&str>, ) -> i32 { @@ -1398,8 +1328,8 @@ async fn remove_hosted_only( args.common.dry_run, "hosted_state_retained", format!( - "{} matches only hosted redirect records; removing one means unwinding \ - its lockfile redirect, which --skip-rollback prevents", + "{} matches only hosted lockfile pins; removing one means restoring its \ + upstream registry entry, which --skip-rollback prevents", args.identifier ), ); @@ -1420,8 +1350,8 @@ async fn remove_hosted_only( "will be" } ); - for purl in &hosted_matches { - eprintln!(" - {purl}"); + for pin in &hosted_matches { + eprintln!(" - {}", pin.purl); } eprintln!(); } @@ -1442,24 +1372,10 @@ async fn remove_hosted_only( return 0; } - let leg = match unwind_hosted(&args.common, &hosted_matches, &mut redirect_state).await { + let leg = match unwind_hosted(&args.common, &hosted_matches).await { Ok(leg) => leg, Err(err) => { - match &err { - HostedUnwindError::Unsupported(_) => { - track_patch_remove_failed( - "hosted redirect revert unsupported", - api_token, - org_slug, - ) - .await; - } - HostedUnwindError::Failed { .. } => { - track_patch_remove_failed("hosted redirect revert failed", api_token, org_slug) - .await; - } - HostedUnwindError::Persist(_) => {} - } + track_patch_remove_failed("hosted redirect revert failed", api_token, org_slug).await; let (code, msg) = hosted_unwind_error(err, false); emit_error_envelope(args.common.json, args.common.dry_run, code, msg); return 1; @@ -1482,7 +1398,7 @@ async fn remove_hosted_only( for purl in &leg.reverted { env.record(PatchEvent::new(action, purl.clone()).with_reason( "hosted_reverted", - "hosted lockfile redirect unwound on remove", + "hosted lockfile pin restored to the upstream registry on remove", )); } if args.common.json { diff --git a/crates/socket-patch-cli/src/commands/rollback.rs b/crates/socket-patch-cli/src/commands/rollback.rs index 14806e79b..287d873cf 100644 --- a/crates/socket-patch-cli/src/commands/rollback.rs +++ b/crates/socket-patch-cli/src/commands/rollback.rs @@ -16,6 +16,7 @@ use socket_patch_core::patch::rollback::{ }; use socket_patch_core::telemetry::{track_patch_rollback_failed, track_patch_rolled_back}; use socket_patch_core::utils::purl::{patch_matches, strip_purl_qualifiers}; +use socket_patch_core::patch::redirect::upstream::HostedPin; use socket_patch_core::vendor::{save_state, RevertOpts, VendorState, VendorWarning}; use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; @@ -176,12 +177,7 @@ pub(crate) fn as_question(text: &str) -> String { /// The default (destructive) rollback's confirmation prompt, naming only /// the legs that have work. -fn rollback_prompt( - manifest: usize, - vendored: usize, - hosted: usize, - leftover_edits: usize, -) -> String { +fn rollback_prompt(manifest: usize, vendored: usize, hosted: usize) -> String { let mut clauses: Vec = Vec::new(); if manifest > 0 { clauses.push(format!( @@ -206,17 +202,8 @@ fn rollback_prompt( } if hosted > 0 { clauses.push(format!( - "unwind {}", - plural(hosted, "hosted redirect", "hosted redirects") - )); - } else if leftover_edits > 0 { - clauses.push(format!( - "replay {}", - plural( - leftover_edits, - "leftover hosted redirect edit", - "leftover hosted redirect edits" - ) + "restore {} to the upstream registry", + plural(hosted, "hosted package", "hosted packages") )); } as_question(&join_clauses(&clauses)) @@ -1023,35 +1010,34 @@ async fn run_vendored_leg( out } -/// Unwind the in-scope hosted redirects: per-purl reverts where they -/// exist (cargo, npm-family, golang), and — when the scope covers the ENTIRE -/// record set — the whole-ledger reverse replay for everything else. -/// Mutates `state`; the caller persists on wet runs. -pub(crate) async fn run_hosted_leg( - common: &GlobalArgs, - purls: &[String], - state: &mut socket_patch_core::patch::redirect::RedirectState, - replay_eligible: bool, -) -> HostedLegOutcome { - use socket_patch_core::patch::redirect::{ - redirect_revert_supported, revert_redirect_purl, revert_remaining_redirect_edits, +/// The patch-server origins that count as hosted, besides Socket's own: +/// the operator's `--patch-server-url` (discovery's allowlist). +pub(crate) fn patch_server_origins(common: &GlobalArgs) -> Vec { + common + .patch_server_url + .iter() + .filter(|url| !url.trim().is_empty()) + .cloned() + .collect() +} + +/// Restore the in-scope hosted pins to their default upstream registry +/// entries (core `patch::redirect::upstream`): v5 hosted mode keeps no +/// ledger, so each pin's lock entry is re-resolved from the registry, and a +/// pin that cannot be is refused with the `git checkout` remedy. Shared +/// with remove's hosted leg. +pub(crate) async fn run_hosted_leg(common: &GlobalArgs, pins: &[HostedPin]) -> HostedLegOutcome { + use socket_patch_core::patch::redirect::upstream::{ + restore_upstream, PinStatus, RestoreOptions, }; let mut out = HostedLegOutcome::default(); - // The vlt nodes the unwound purls pin, read before the revert drops - // their edits: the heal below invalidates the patched installed copies - // once the registry pins are back. - let vlt_scope: Vec = if replay_eligible { - purls - .iter() - .cloned() - .chain(state.records.keys().cloned()) - .collect() - } else { - purls.to_vec() - }; - // FIFO-safe: a FIFO or device planted at the lock path must fail this - // read at once, not block the rollback in open(2). + if pins.is_empty() { + return out; + } + // The vlt nodes the restored pins pin, read before the restore rewrites + // them: the heal below invalidates the patched installed copies once the + // registry pins are back. let vlt_lock = socket_patch_core::utils::fs::read_regular_to_string( &common .cwd @@ -1059,126 +1045,91 @@ pub(crate) async fn run_hosted_leg( ) .await .ok(); - let vlt_targets = socket_patch_core::patch::redirect::vlt_heal::ledger_targets( - state, - &vlt_scope, - vlt_lock.as_deref(), - ); - // When the whole-ledger replay will run anyway (the scope covers every - // record), npm purls on Bun projects defer to it so all lockfile edits - // are staged together atomically. A scoped unwind of one of several - // Bun records uses the per-purl revert to restore only its package. - let has_bun_edits = state.edits.iter().any(|e| { - matches!( - e.kind.as_str(), - "redirect_bun_lock_package" | "redirect_bun_lockb_package" - ) - }); - let mut deferred_to_replay: Vec = Vec::new(); - for purl in purls { - let defer_bun = has_bun_edits && purl.starts_with("pkg:npm/") && replay_eligible; - if !defer_bun && redirect_revert_supported(purl) { - match revert_redirect_purl(&common.cwd, state, purl, common.dry_run).await { - Ok(revert) => { - for (code, detail) in &revert.warnings { - out.warnings.push((code.clone(), detail.clone())); - } - if !common.json && !common.silent { - if common.dry_run { - println!("Would unwind hosted redirect for {purl}"); - } else { - println!("Unwound hosted redirect for {purl}"); - } - } - out.edited_files - .extend(revert.reverted_files.iter().cloned()); - out.reverted.push(purl.clone()); - } - Err(e) => { - if !common.json { - eprintln!("Error: Failed to unwind hosted redirect for {purl}: {e}"); - } - out.failed.push((purl.clone(), e)); - } - } - } else if replay_eligible { - deferred_to_replay.push(purl.clone()); - } else { - if !common.json { - eprintln!( - "Error: Cannot unwind hosted redirect for {purl}: no per-purl revert exists for \ - this ecosystem. Run an unscoped `socket-patch rollback` to unwind ALL \ - hosted redirects, or re-run `scan --mode hosted` to normalize." - ); - } - out.unsupported.push(purl.clone()); - } - } - // The whole-ledger replay runs when the scope covers every record - // (however it was spelled), and also as the "last one out turns off - // the lights" pass — per-purl reverts never claim the non-package - // shared settings edits (such as pnpm trustLockfile), so an emptied - // record map with leftover edits replays them here too. (The npm - // `.npmrc` `allow-remote=all` edit is the one exception: the per-purl - // npm revert of the LAST package-lock entry unwinds it itself, so a - // scoped rollback leaves no loosened policy behind while other - // ecosystems' records remain.) - if replay_eligible || (state.records.is_empty() && !state.edits.is_empty()) { - let replay = revert_remaining_redirect_edits(&common.cwd, state, common.dry_run).await; - for refusal in &replay.refusals { - let files: Vec<&str> = refusal.files.iter().map(String::as_str).collect(); - let why = format!("{} ({})", refusal.reason, files.join(", ")); - if !common.json { - eprintln!( - "Error: Cannot unwind hosted redirect edits ({}): {why}", - refusal.group - ); - } - out.failed.push((format!("group:{}", refusal.group), why)); - } - out.warnings.extend(replay.warnings.iter().cloned()); - out.edited_files - .extend(replay.reverted_files.iter().cloned()); - // Deferred purls succeeded iff the replay dropped their records. - for purl in deferred_to_replay { - if replay.dropped_records.iter().any(|p| p == &purl) { + let origins = patch_server_origins(common); + let purls: Vec = pins.iter().map(|p| p.purl.clone()).collect(); + let vlt_targets = vlt_lock + .as_deref() + .map(|lock| { + socket_patch_core::patch::redirect::vlt_heal::lock_targets(lock, &origins, &purls) + }) + .unwrap_or_default(); + let opts = RestoreOptions { + dry_run: common.dry_run, + offline: common.offline, + patch_server_origins: origins, + }; + let outcome = restore_upstream(&common.cwd, pins, &opts).await; + for pin in &outcome.pins { + match &pin.status { + PinStatus::Restored => { if !common.json && !common.silent { if common.dry_run { - println!("Would unwind hosted redirect for {purl}"); + println!("Would restore {} to its upstream registry entry", pin.purl); } else { - println!("Unwound hosted redirect for {purl}"); + println!("Restored {} to its upstream registry entry", pin.purl); } } - out.reverted.push(purl); - } else if !out.failed.iter().any(|(p, _)| p.starts_with("group:")) { - let why = "hosted redirect edits could not be replayed"; + out.reverted.push(pin.purl.clone()); + } + PinStatus::Refused(why) => { // Errors print even under --silent: this drives exit 1. if !common.json { - eprintln!("Error: Failed to unwind hosted redirect for {purl}: {why}"); + eprintln!("Error: {}", capitalize_first(why)); } - out.failed.push((purl, why.into())); + out.failed.push((pin.purl.clone(), why.clone())); } } } - // A target's registry pins are back when its purl reverted, or when the - // whole-ledger replay committed the vlt group: groups commit on their - // own, so another lock's refusal (a drifted package-lock.json) leaves - // the restored vlt-lock.json pins restored. - let vlt_group_refused = out.failed.iter().any(|(p, _)| p == "group:vlt"); + if let Some(e) = &outcome.flush_error { + let why = format!("writing the restored lockfiles failed: {e}"); + if !common.json { + eprintln!("Error: {}", capitalize_first(&why)); + } + out.failed.push(("files".to_string(), why)); + } + out.warnings.extend( + outcome + .warnings + .iter() + .map(|(code, detail)| (code.to_string(), detail.clone())), + ); + out.edited_files.extend(outcome.reverted_files.iter().cloned()); let unwound: Vec<_> = vlt_targets .into_iter() - .filter(|t| { - out.reverted.iter().any(|p| { - socket_patch_core::utils::purl::canonical_purl(p) - == socket_patch_core::utils::purl::canonical_purl(&t.purl) - }) || (replay_eligible && !vlt_group_refused) - }) + .filter(|t| out.reverted.iter().any(|p| p == &t.purl)) .collect(); out.warnings .extend(crate::commands::scan::vlt_rollback_heal(common, &unwound).await); out } +/// Delete a pre-v5 hosted ledger once no hosted pin is left for it to +/// describe (v5 never writes it; it is read only for migration). A wet run +/// only; a failure is a warning (the file is inert). +pub(crate) async fn retire_legacy_redirect_ledger(common: &GlobalArgs) -> Option<(String, String)> { + let path = common + .cwd + .join(socket_patch_core::patch::redirect::REDIRECT_STATE_REL); + if common.dry_run || tokio::fs::symlink_metadata(&path).await.is_err() { + return None; + } + let remaining = crate::commands::discover_wiring(common, &common.cwd).await; + if !HostedPin::all(&remaining).is_empty() { + return None; + } + match socket_patch_core::utils::fs::remove_file(&path).await { + Ok(()) => None, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, + Err(e) => Some(( + "legacy_redirect_ledger_kept".to_string(), + format!( + "could not delete the pre-v5 hosted ledger {}: {e}", + socket_patch_core::patch::redirect::REDIRECT_STATE_REL + ), + )), + } +} + pub async fn run(args: RollbackArgs) -> i32 { apply_env_toggles(&args.common); @@ -1237,10 +1188,10 @@ pub async fn run(args: RollbackArgs) -> i32 { let cwd = args.common.cwd.clone(); // ── state discovery ───────────────────────────────────────────────── - // Rollback infers what to undo from the three state stores: the - // manifest (in-place/agent patches), the vendor ledger (vendored - // patches), and the redirect ledger (hosted lockfile redirects). A - // missing manifest is not fatal when a ledger holds work. + // Rollback infers what to undo from three sources: the manifest + // (in-place/agent patches), the vendor ledger (vendored patches), and + // the lockfiles themselves (hosted pins — v5 hosted mode keeps no + // ledger). A missing manifest is not fatal when either holds work. // // Only cheap EXISTENCE probes happen before the lock (they decide the // truly-empty error path, which never locks: acquiring would create @@ -1253,12 +1204,12 @@ pub async fn run(args: RollbackArgs) -> i32 { let vendor_ledger_exists = tokio::fs::metadata(cwd.join(".socket/vendor/state.json")) .await .is_ok(); - let redirect_ledger_exists = - tokio::fs::metadata(cwd.join(socket_patch_core::patch::redirect::REDIRECT_STATE_REL)) - .await - .is_ok(); + // The hosted pins the lockfiles wire (read-only discovery; the restore + // re-reads every file under the lock before it writes). + let hosted_pins: Vec = + HostedPin::all(&crate::commands::discover_wiring(&args.common, &cwd).await); - if manifest_missing && !vendor_ledger_exists && !redirect_ledger_exists { + if manifest_missing && !vendor_ledger_exists && hosted_pins.is_empty() { // Ledger-less but still wired? (a deleted/uncommitted state.json // with lockfiles still consuming `.socket/vendor/` artifacts is a // supported recovery state — `repair` reconstructs the ledger.) @@ -1308,9 +1259,7 @@ pub async fn run(args: RollbackArgs) -> i32 { // each exactly once: the agent leg below receives the manifest and the // vendor-ownership key set instead of re-reading them. let vendor_state_result = socket_patch_core::vendor::load_state(&cwd).await; - let redirect_state_result = socket_patch_core::patch::redirect::load_redirect_state(&cwd).await; let vendor_corrupt = vendor_state_result.is_err(); - let redirect_corrupt = redirect_state_result.is_err(); // An unreadable ledger degrades to "nothing vendored" for the in-place // leg (its own containment is the `vendor_state_unreadable` exit below). let vendored_keys: HashSet = vendor_state_result @@ -1355,14 +1304,10 @@ pub async fn run(args: RollbackArgs) -> i32 { } Err(_) => Vec::new(), }; - let redirect_records: Vec<(String, String)> = match &redirect_state_result { - Ok(Some(s)) => s - .records - .iter() - .map(|(purl, rec)| (purl.clone(), rec.uuid.clone())) - .collect(), - _ => Vec::new(), - }; + let redirect_records: Vec<(String, String)> = hosted_pins + .iter() + .map(|pin| (pin.purl.clone(), pin.uuid.clone())) + .collect(); let scoped = !identifiers.is_empty() || !path_scope.is_empty(); @@ -1528,25 +1473,6 @@ pub async fn run(args: RollbackArgs) -> i32 { }); } - // The whole-ledger hosted replay (which covers the ecosystems without - // a per-purl revert) runs only when the scope covers EVERY record — - // however the scope was spelled. - let replay_eligible = match &redirect_state_result { - // A records-EMPTY ledger (degraded record-fetch-failed runs leave - // edits without records) is vacuously "covered" by any scope; only - // an UNSCOPED run may replay those leftover edits — a scoped - // rollback of an unrelated purl must not unwind live redirects it - // was never asked about. `--ecosystems` counts as a scope here: - // recordless edits carry no purl to narrow by, so an eco-narrowed - // run leaves them to an unscoped rollback rather than replaying - // other ecosystems' edits behind the filter's back. - Ok(Some(s)) => { - (!s.records.is_empty() || (!scoped && args.common.ecosystems.is_none())) - && s.records.keys().all(|p| hosted_scope.contains(p)) - } - _ => false, - }; - // Corrupt-ledger containment: a corrupt store fails ONLY the legs that // need it; the agent leg still restores files (emergency restores are // never blocked by an unrelated corrupt ledger). Cleanup/GC also skip @@ -1564,40 +1490,14 @@ pub async fn run(args: RollbackArgs) -> i32 { ), )); } - if redirect_corrupt { - run_warnings.push(( - "redirect_state_unreadable".into(), - // The core error already carries the recovery steps; only say - // what this run skipped. - format!( - "the hosted leg was skipped: cannot read the hosted redirect ledger: {}", - redirect_state_result - .as_ref() - .expect_err("checked corrupt above") - ), - )); - } // ── confirmation ──────────────────────────────────────────────────── // The default run deletes manifest entries, vendored artifacts, ledger // records, and unused blobs — prompt once, remove-style. Auto-accepted // under --yes/--json/non-TTY; skipped for previews and for // --preserve-state runs (which delete no local state). - // Leftover hosted edits an eligible replay would unwind even with no - // in-scope records (degraded record-fetch-failed ledgers): they are - // work — and prompt-worthy mutation — too. - let hosted_leftover_edits = if replay_eligible { - match &redirect_state_result { - Ok(Some(st)) => st.edits.len(), - _ => 0, - } - } else { - 0 - }; - let has_work = !manifest_scope.is_empty() - || !vendor_scope.is_empty() - || !hosted_scope.is_empty() - || hosted_leftover_edits > 0; + let has_work = + !manifest_scope.is_empty() || !vendor_scope.is_empty() || !hosted_scope.is_empty(); // Everything in scope was filtered out by `--ecosystems`: say so, // instead of the misleading "No patches found in manifest". let eco_filtered_everything = !has_work && scope_before_eco_filter > 0; @@ -1614,12 +1514,7 @@ pub async fn run(args: RollbackArgs) -> i32 { if has_work && !args.common.dry_run && !args.preserve_state { // Compose only the clauses that apply, so a hosted-only run never // claims manifest entries it does not have. - let prompt = rollback_prompt( - manifest_scope.len(), - vendor_scope.len(), - hosted_scope.len(), - hosted_leftover_edits, - ); + let prompt = rollback_prompt(manifest_scope.len(), vendor_scope.len(), hosted_scope.len()); if !crate::ui::confirm(&prompt, true, &args.common) { if !args.common.json && !args.common.silent { println!("Rollback cancelled."); @@ -1675,33 +1570,15 @@ pub async fn run(args: RollbackArgs) -> i32 { } // ── hosted leg ─────────────────────────────────────────────── - let mut hosted_leg = HostedLegOutcome::default(); - if !redirect_corrupt { - if let Ok(Some(existing)) = &redirect_state_result { - let mut st = existing.clone(); - let before = (st.edits.len(), st.records.len()); - let mut purls: Vec = hosted_scope.iter().cloned().collect(); - purls.sort(); - if !purls.is_empty() || (replay_eligible && !st.edits.is_empty()) { - hosted_leg = - run_hosted_leg(&args.common, &purls, &mut st, replay_eligible).await; - let changed = (st.edits.len(), st.records.len()) != before; - if !args.common.dry_run && changed { - if let Err(e) = - socket_patch_core::patch::redirect::persist_redirect_state( - &cwd, &st, - ) - .await - { - let msg = - format!("failed to persist the hosted redirect ledger: {e}"); - if !args.common.json { - eprintln!("Error: {}", capitalize_first(&msg)); - } - hosted_leg.failed.push(("ledger".to_string(), msg)); - } - } - } + let in_scope: Vec = hosted_pins + .iter() + .filter(|pin| hosted_scope.contains(&pin.purl)) + .cloned() + .collect(); + let hosted_leg = run_hosted_leg(&args.common, &in_scope).await; + if hosted_leg.failed.is_empty() { + if let Some(warning) = retire_legacy_redirect_ledger(&args.common).await { + run_warnings.push(warning); } } @@ -1846,8 +1723,8 @@ pub async fn run(args: RollbackArgs) -> i32 { if args.preserve_state && !hosted_leg.reverted.is_empty() { run_warnings.push(( "hosted_state_not_preservable".into(), - "hosted redirects have no preservable local state: their ledger \ - records were dropped with the unwound wiring; re-run \ + "hosted wiring has no preservable local state: the lockfile pins are \ + the only record, and they now resolve upstream; re-run \ `scan --mode hosted` to re-wire" .into(), )); @@ -1889,9 +1766,7 @@ pub async fn run(args: RollbackArgs) -> i32 { .filter(|(code, _)| { !matches!( code.as_str(), - "vendor_state_unreadable" - | "redirect_state_unreadable" - | "reinstall_required" + "vendor_state_unreadable" | "reinstall_required" ) }) .chain(hosted_leg.warnings.iter()) @@ -1928,7 +1803,6 @@ pub async fn run(args: RollbackArgs) -> i32 { && hosted_leg.failed.is_empty() && hosted_leg.unsupported.is_empty() && !vendor_corrupt - && !redirect_corrupt && manifest_write_failed.is_none(); let rolled_back_count = results .iter() @@ -2157,7 +2031,7 @@ pub async fn run(args: RollbackArgs) -> i32 { eprintln!("Error: Kept vendored state for {key}: {reason}"); } for (code, detail) in &run_warnings { - if code == "vendor_state_unreadable" || code == "redirect_state_unreadable" { + if code == "vendor_state_unreadable" { eprintln!("Error ({code}): {}", capitalize_first(detail)); } } diff --git a/crates/socket-patch-cli/src/commands/vendor.rs b/crates/socket-patch-cli/src/commands/vendor.rs index 5da61dd9b..11293a373 100644 --- a/crates/socket-patch-cli/src/commands/vendor.rs +++ b/crates/socket-patch-cli/src/commands/vendor.rs @@ -1918,10 +1918,9 @@ pub(crate) async fn vendor_records_reusing( // version) is deferred rather than fetched: the backend refuses // it — at its turn, in its own words — before anything reads // the source, so the refusal costs no registry request. A purl - // the hosted redirect ledger claims keeps the eager fetch: its - // takeover reverts the hosted lock edits first, which rewrites - // the text the gates read (and a malformed redirect ledger - // defers nothing). + // the lockfiles pin hosted keeps the eager fetch: its takeover + // restores the upstream lock entry first, which rewrites the + // text the gates read. let lock_candidates: Vec<(&str, &str)> = missing .iter() .zip(&rungs) @@ -1939,15 +1938,14 @@ pub(crate) async fn vendor_records_reusing( }) .collect(); if !lock_candidates.is_empty() { - let claimed: Option> = - match socket_patch_core::patch::redirect::load_redirect_state(&common.cwd).await - { - Ok(Some(state)) => { - Some(state.records.keys().map(|k| canonical_purl(k)).collect()) - } - Ok(None) => Some(Vec::new()), - Err(_) => None, - }; + let claimed: Option> = Some( + socket_patch_core::patch::redirect::upstream::HostedPin::all( + &crate::commands::discover_wiring(common, &common.cwd).await, + ) + .into_iter() + .map(|pin| canonical_purl(&pin.purl)) + .collect(), + ); if let Some(claimed) = claimed { let unclaimed: Vec<(&str, &str)> = lock_candidates .into_iter() @@ -2195,17 +2193,19 @@ pub(crate) async fn vendor_records_reusing( let mut matched: HashSet = HashSet::new(); let mut handled_bases: HashSet = HashSet::new(); - // The hosted redirect ledger, for cross-mode takeovers: vendoring a purl - // it still claims must revert the hosted edits FIRST (see the dispatch - // loop below). Loaded once; mutated + persisted per reverted purl. With - // a MALFORMED ledger a claimed purl is indistinguishable from an - // unclaimed one, so every takeover-capable purl fails closed; other - // purls proceed. - let (mut redirect_ledger, redirect_ledger_corrupt) = - match socket_patch_core::patch::redirect::load_redirect_state(&common.cwd).await { - Ok(state) => (state, None), - Err(corrupt) => (None, Some(corrupt)), - }; + // The lockfiles' hosted pins, for cross-mode takeovers: vendoring a purl + // the lockfiles still pin hosted must restore its upstream registry + // entry FIRST (see the dispatch loop below). Discovered once, before any + // write of this run. + let hosted_pins: Vec = + socket_patch_core::patch::redirect::upstream::HostedPin::all( + &crate::commands::discover_wiring(common, &common.cwd).await, + ); + let hosted_pin_of = |purl: &str| { + hosted_pins + .iter() + .find(|pin| canonical_purl(&pin.purl) == canonical_purl(purl)) + }; // Yarn berry takeover preflight (see // `socket_patch_core::vendor::yarn_berry_vendor_preflight`): the berry @@ -2242,14 +2242,7 @@ pub(crate) async fn vendor_records_reusing( // is still decided at the loop's own call (see `VendorPrefetch`). let service_prefetch = match service.filter(|cfg| !common.dry_run && cfg.wants_prefetch()) { Some(cfg) => { - let takeover_blocked = |purl: &str| { - redirect_ledger_corrupt.is_some() - || redirect_ledger.as_ref().is_some_and(|l| { - l.records - .keys() - .any(|k| canonical_purl(k) == canonical_purl(purl)) - }) - }; + let takeover_blocked = |purl: &str| hosted_pin_of(purl).is_some(); let planned = plan_service_downloads( &common.cwd, force, @@ -2402,42 +2395,25 @@ pub(crate) async fn vendor_records_reusing( continue; } - // Cross-mode takeover: vendoring over a LIVE hosted redirect - // must first revert the hosted edits from the redirect ledger. - // Cargo: `[patch.crates-io]` only patches crates-io-sourced - // deps, so vendoring on top of the hosted registry pin leaves the - // project unbuildable. npm family: without the pre-revert the - // vendor ledger records the grant-tokenized HOSTED lock fragment - // as its pre-vendor original. In every ecosystem the pre-revert - // hands the vendor detach the PRISTINE registry fragment to - // record. A purl whose hosted edits cannot be cleanly reverted is - // REFUSED; the cargo backend's `hosted_redirect_live` guard - // backstops states with no usable ledger. - if socket_patch_core::patch::redirect::redirect_revert_supported(candidate) { - if let Some(corrupt) = &redirect_ledger_corrupt { - has_errors = true; - env.record( - PatchEvent::new(PatchAction::Failed, candidate.clone()).with_error( - "redirect_ledger_corrupt", - format!( - "cannot vendor over a possibly-live hosted redirect: \ - {corrupt}" - ), - ), - ); - report_vendor_failure(common, candidate, &corrupt.to_string()); - continue; - } - let claimed = redirect_ledger.as_ref().is_some_and(|l| { - l.records - .keys() - .any(|k| canonical_purl(k) == canonical_purl(candidate)) - }); + // Cross-mode takeover: vendoring over a LIVE hosted pin must + // first restore the upstream registry entry (v5 keeps no hosted + // ledger: the entry is re-resolved from the registry). Cargo: + // `[patch.crates-io]` only patches crates-io-sourced deps, so + // vendoring on top of the hosted registry pin leaves the project + // unbuildable. npm family: without the restore the vendor ledger + // records the grant-tokenized HOSTED lock fragment as its + // pre-vendor original. In every ecosystem the restore hands the + // vendor detach the PRISTINE registry entry to record. A purl + // whose upstream entry cannot be restored is REFUSED; the cargo + // backend's `hosted_redirect_live` guard backstops the rest. + let hosted_pin = hosted_pin_of(candidate) + .filter(|_| socket_patch_core::patch::redirect::redirect_revert_supported(candidate)); + if let Some(pin) = hosted_pin { // The refusal the berry backend would raise after the - // revert, raised HERE instead — the same `failed` event, + // restore, raised HERE instead — the same `failed` event, // code and detail, in the dry run and the wet run alike — - // so the hosted wiring and redirect ledger stay untouched. - if claimed && candidate.starts_with("pkg:npm/") { + // so the hosted wiring stays untouched. + if candidate.starts_with("pkg:npm/") { let refusal = berry_takeover_refusal .get_or_init(|| { socket_patch_core::vendor::yarn_berry_vendor_preflight(&common.cwd) @@ -2453,177 +2429,102 @@ pub(crate) async fn vendor_records_reusing( continue; } } - if claimed && common.dry_run { - // Probe the takeover exactly as the wet run would (a dry - // revert on a throwaway clone), so the preview never - // promises a takeover the wet run then refuses. - let mut probe = redirect_ledger.clone().expect("claimed implies Some"); - match socket_patch_core::patch::redirect::revert_redirect_purl( - &common.cwd, - &mut probe, + let origins = crate::commands::rollback::patch_server_origins(common); + let vlt_lock = socket_patch_core::utils::fs::read_regular_to_string( + &common + .cwd + .join(socket_patch_core::constants::npm_family::VLT_LOCK), + ) + .await + .ok(); + let targets = vlt_lock + .as_deref() + .map(|lock| { + socket_patch_core::patch::redirect::vlt_heal::lock_targets( + lock, + &origins, + std::slice::from_ref(candidate), + ) + }) + .unwrap_or_default(); + let restore = socket_patch_core::patch::redirect::upstream::restore_upstream( + &common.cwd, + std::slice::from_ref(pin), + &socket_patch_core::patch::redirect::upstream::RestoreOptions { + dry_run: common.dry_run, + offline: common.offline, + patch_server_origins: origins, + }, + ) + .await; + let refusal = restore + .refused() + .map(|(_, why)| why.to_string()) + .next() + .or_else(|| restore.flush_error.clone()); + if let Some(detail) = refusal { + has_errors = true; + env.record( + PatchEvent::new(PatchAction::Failed, candidate.clone()).with_error( + "redirect_revert_failed", + format!("cannot vendor over the live hosted pin: {detail}"), + ), + ); + report_vendor_failure( + common, candidate, - true, - ) - .await - { - Ok(revert) => { - record_warning( - env, - candidate, - &VendorWarning::new( - "vendor_would_revert_redirect", - format!( - "{} is hosted-redirected; a non-dry-run vendor will \ - revert the hosted redirect edits first, then vendor \ - (mode takeover)", - normalize_purl(candidate) - ), - ), - common, - ); - // The backend preview below reads the lock from - // disk, where the hosted wiring is still live. - // Bun's hosted rewrite REPLACES the entry's - // `name@version` spec, so the backend would refuse - // a `vendor_lock_entry_not_found` the wet run never - // sees: the advisory already states the plan, so - // the preview stops here. - if revert - .reverted_files - .iter() - .any(|f| f == "bun.lock" || f == "bun.lockb") - { - continue; - } - } - Err(detail) => { - has_errors = true; - env.record( - PatchEvent::new(PatchAction::Failed, candidate.clone()).with_error( - "redirect_revert_failed", - format!( - "cannot vendor over the live hosted redirect: \ - {detail}" - ), - ), - ); - report_vendor_failure( - common, - candidate, - &format!("cannot revert the hosted redirect: {detail}"), - ); - continue; - } - } - } else if claimed { - let ledger = redirect_ledger.as_mut().expect("claimed implies Some"); - let vlt_lock = socket_patch_core::utils::fs::read_regular_to_string( - &common - .cwd - .join(socket_patch_core::constants::npm_family::VLT_LOCK), - ) - .await - .ok(); - let targets = socket_patch_core::patch::redirect::vlt_heal::ledger_targets( - ledger, - std::slice::from_ref(candidate), - vlt_lock.as_deref(), + &format!("cannot restore the upstream entry: {detail}"), ); - match socket_patch_core::patch::redirect::revert_redirect_purl( - &common.cwd, - ledger, + continue; + } + for (code, detail) in &restore.warnings { + record_warning(env, candidate, &VendorWarning::new(*code, detail.clone()), common); + } + if common.dry_run { + record_warning( + env, candidate, - false, - ) - .await + &VendorWarning::new( + "vendor_would_revert_redirect", + format!( + "{} is hosted; a non-dry-run vendor will restore its upstream \ + registry entry first, then vendor (mode takeover)", + normalize_purl(candidate) + ), + ), + common, + ); + // The backend preview below reads the lock from disk, + // where the hosted wiring is still live. Bun's hosted + // rewrite REPLACES the entry's `name@version` spec, so the + // backend would refuse a `vendor_lock_entry_not_found` + // the wet run never sees: the advisory already states the + // plan, so the preview stops here. + if restore + .reverted_files + .iter() + .any(|f| f == "bun.lock" || f == "bun.lockb") { - Ok(revert) => { - // Advisories from the same transaction (a - // redirect-created `.npmrc` modified since — - // kept, only the `allow-remote=all` line removed). - for (code, detail) in &revert.warnings { - if code == "redirect_npmrc_allow_remote_modified" { - record_warning( - env, - candidate, - &VendorWarning::new( - "redirect_npmrc_allow_remote_modified", - detail.clone(), - ), - common, - ); - } - } - if let Err(e) = - socket_patch_core::patch::redirect::persist_redirect_state( - &common.cwd, - ledger, - ) - .await - { - // The hosted edits are reverted but the ledger - // still claims them; vendoring now would leave - // a ledger asserting wiring that is gone. Fail - // closed for this purl. - has_errors = true; - let detail = format!( - "reverted the hosted redirect but could not update \ - .socket/vendor/redirect-state.json: {e}" - ); - report_vendor_failure(common, candidate, &detail); - env.record( - PatchEvent::new(PatchAction::Failed, candidate.clone()) - .with_error("redirect_ledger_write_failed", detail), - ); - continue; - } - let reverted_what = if candidate.starts_with("pkg:cargo/") { - "the hosted edits (Cargo.toml registry pin, Cargo.lock \ - source/checksum, registries block)" - } else if candidate.starts_with("pkg:golang/") { - "the hosted edits (go.mod replace, the socket module's go.sum \ - lines, the pruned upstream go.sum lines)" - } else { - "the hosted lockfile edits back to their pre-redirect \ - registry values" - }; - if !targets.is_empty() { - vlt_takeover_targets.insert(candidate.clone(), targets); - } - record_warning( - env, - candidate, - &VendorWarning::new( - "vendor_takeover_reverted_redirect", - format!( - "{} was hosted-redirected; reverted {reverted_what} \ - and dropped the redirect-ledger record before \ - vendoring (mode takeover)", - normalize_purl(candidate) - ), - ), - common, - ); - } - Err(detail) => { - has_errors = true; - env.record( - PatchEvent::new(PatchAction::Failed, candidate.clone()).with_error( - "redirect_revert_failed", - format!( - "cannot vendor over the live hosted redirect: \ - {detail}" - ), - ), - ); - report_vendor_failure( - common, - candidate, - &format!("cannot revert the hosted redirect: {detail}"), - ); - continue; - } + continue; } + } else { + if !targets.is_empty() { + vlt_takeover_targets.insert(candidate.clone(), targets); + } + record_warning( + env, + candidate, + &VendorWarning::new( + "vendor_takeover_reverted_redirect", + format!( + "{} was hosted; restored its upstream registry entry ({}) \ + before vendoring (mode takeover)", + normalize_purl(candidate), + restore.reverted_files.join(", ") + ), + ), + common, + ); } } diff --git a/crates/socket-patch-core/src/patch/redirect/mod.rs b/crates/socket-patch-core/src/patch/redirect/mod.rs index 9f03dbaa4..aa38491c5 100644 --- a/crates/socket-patch-core/src/patch/redirect/mod.rs +++ b/crates/socket-patch-core/src/patch/redirect/mod.rs @@ -60,6 +60,7 @@ mod rewrite_oracle_support; mod staged; mod state; mod takeover; +pub mod upstream; pub mod vlt; pub mod vlt_heal; pub mod vlt_preflight; diff --git a/crates/socket-patch-core/src/patch/redirect/pnpm.rs b/crates/socket-patch-core/src/patch/redirect/pnpm.rs index a5fa6a4de..90371b462 100644 --- a/crates/socket-patch-core/src/patch/redirect/pnpm.rs +++ b/crates/socket-patch-core/src/patch/redirect/pnpm.rs @@ -192,6 +192,39 @@ impl<'a> Resolution<'a> { } } +impl Resolution<'_> { + /// The default-registry spelling of this resolution: `integrity` only + /// (pnpm omits `tarball` for a package the configured registry serves), + /// every other field kept in place — the inverse of + /// [`Resolution::rewrite`] for the v5 upstream restore. + pub fn restore(&self, integrity: &str) -> String { + let scalar = if integrity + .chars() + .any(|c| c.is_whitespace() || matches!(c, ',' | '[' | ']' | '{' | '}' | '\'' | '"')) + { + serde_json::to_string(integrity).expect("string serializes") + } else { + integrity.to_string() + }; + let mut fields = vec![format!("integrity: {scalar}")]; + fields.extend( + self.fields + .iter() + .filter(|(k, _)| !matches!(*k, "integrity" | "tarball")) + .map(|(k, v)| format!("{k}: {v}")), + ); + if self.block { + format!( + "{} {}", + self.newline, + fields.join(&format!("{} ", self.newline)) + ) + } else { + format!("{{{}}}", fields.join(", ")) + } + } +} + /// The key line every `packages:` entry's resolution map starts at. const RESOLUTION_KEY: &str = " resolution:"; diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/cargo.rs b/crates/socket-patch-core/src/patch/redirect/upstream/cargo.rs new file mode 100644 index 000000000..fdf10c876 --- /dev/null +++ b/crates/socket-patch-core/src/patch/redirect/upstream/cargo.rs @@ -0,0 +1,380 @@ +//! Cargo upstream restore: `Cargo.lock` entries back on crates.io (source + +//! the index's checksum), every `Cargo.toml` declaration loses its +//! `registry = "socket-patch-"` pin, and the project cargo config +//! drops the `[registries.socket-patch-]` block nothing references +//! any more. +//! +//! The hosted rewriter only ever pins crates.io dependencies (it refuses a +//! dep declared against any other registry), so the upstream source is +//! always crates.io's. A declaration's original spelling is not recorded: +//! the shorthand `name = { version = "…", registry = "…" }` the rewriter +//! produces from `name = "…"` collapses back to that shorthand, and every +//! other form just loses the pin. + +use std::collections::{BTreeMap, BTreeSet}; + +use regex::Regex; + +use super::{Ctx, FormatResult, HostedPin, View}; + +/// How Cargo.lock names crates.io (cargo keeps this spelling even when it +/// fetches over the sparse protocol). +const CRATES_IO_SOURCE: &str = "registry+https://github.com/rust-lang/crates.io-index"; + +/// The project cargo configs, in cargo's read preference. +const CARGO_CONFIGS: [&str; 2] = [".cargo/config", ".cargo/config.toml"]; + +fn registry_name(uuid: &str) -> String { + format!("socket-patch-{uuid}") +} + +struct LockHit { + uuid: String, + name: String, + version: String, + /// The hosted source string (`sparse+https://…/index/`). + source: String, +} + +fn quoted_field(block: &str, field: &str) -> Option { + block.lines().find_map(|l| { + let rest = l.strip_prefix(field)?.trim_start().strip_prefix('=')?; + let v = rest.trim(); + v.strip_prefix('"')?.strip_suffix('"').map(str::to_string) + }) +} + +pub(crate) async fn restore( + view: &mut View<'_>, + pins: &[&HostedPin], + _files: &[String], + ctx: &Ctx<'_>, +) -> FormatResult { + let mut result = FormatResult::default(); + let by_uuid: BTreeMap<&str, &HostedPin> = pins.iter().map(|p| (p.uuid.as_str(), *p)).collect(); + + // ── Cargo.lock ── + let lock_raw = match view.read("Cargo.lock").await { + Ok(t) => t, + Err(e) => { + for pin in pins { + result.refuse(&pin.uuid, e.clone()); + } + return result; + } + }; + if let Some(raw) = lock_raw { + let crlf = raw.contains("\r\n"); + let mut lock = raw.replace("\r\n", "\n"); + let mut hits: Vec = Vec::new(); + let mut from = 0; + while let Some((start, end)) = super::super::next_lock_block(&lock, from) { + from = end.max(start + 1); + let block = &lock[start..end]; + let Some(source) = quoted_field(block, "source") else { + continue; + }; + let Some(uuid) = ctx.hosted_uuid(&source) else { + continue; + }; + if !by_uuid.contains_key(uuid.as_str()) { + continue; + } + match (quoted_field(block, "name"), quoted_field(block, "version")) { + (Some(name), Some(version)) => hits.push(LockHit { + uuid, + name, + version, + source, + }), + _ => result.refuse(&uuid, "its Cargo.lock entry names no crate and version"), + } + } + let lookups = hits.iter().map(|h| async move { + ( + h.uuid.clone(), + ctx.client.cargo_cksum(&h.name, &h.version).await, + ) + }); + let cksums: BTreeMap> = + futures_util::future::join_all(lookups).await.into_iter().collect(); + let mut changed = false; + for hit in &hits { + let cksum = match cksums.get(&hit.uuid) { + Some(Ok(c)) => c.clone(), + Some(Err(why)) => { + result.refuse(&hit.uuid, format!("{}@{}: {why}", hit.name, hit.version)); + continue; + } + None => continue, + }; + if result.refused.contains_key(&hit.uuid) { + continue; + } + // The entry's own source + checksum lines. + let mut from = 0; + while let Some((start, end)) = super::super::next_lock_block(&lock, from) { + from = end.max(start + 1); + let block = lock[start..end].to_string(); + if quoted_field(&block, "source").as_deref() != Some(hit.source.as_str()) + || quoted_field(&block, "name").as_deref() != Some(hit.name.as_str()) + || quoted_field(&block, "version").as_deref() != Some(hit.version.as_str()) + { + continue; + } + let rebuilt: Vec = block + .split('\n') + .map(|l| { + if l.starts_with("source = ") { + format!("source = \"{CRATES_IO_SOURCE}\"") + } else if l.starts_with("checksum = ") { + format!("checksum = \"{cksum}\"") + } else { + l.to_string() + } + }) + .collect(); + lock.replace_range(start..end, &rebuilt.join("\n")); + from = start + 1; + } + // Dependents' full-id references and the v1 `[metadata]` key. + lock = lock.replace(&format!("({})", hit.source), &format!("({CRATES_IO_SOURCE})")); + let metadata_key = format!( + "\"checksum {} {} ({CRATES_IO_SOURCE})\" = \"", + hit.name, hit.version + ); + let rebuilt: Vec = lock + .split('\n') + .map(|l| match l.strip_prefix(&metadata_key) { + Some(_) => format!("{metadata_key}{cksum}\""), + None => l.to_string(), + }) + .collect(); + lock = rebuilt.join("\n"); + result.handled.insert(hit.uuid.clone()); + changed = true; + } + if changed { + view.write( + "Cargo.lock", + if crlf { lock.replace('\n', "\r\n") } else { lock }, + ); + } + } + + // ── Cargo.toml pins ── + let root = view.root().to_path_buf(); + let mut manifests: Vec = vec!["Cargo.toml".to_string()]; + manifests.extend( + tokio::task::spawn_blocking(move || crate::utils::cargo_workspace::member_manifests(&root)) + .await + .unwrap_or_default(), + ); + for rel in &manifests { + let Ok(Some(text)) = view.read(rel).await else { + continue; + }; + let mut changed = false; + let mut out: Vec = Vec::new(); + for line in text.split('\n') { + let mut kept = Some(line.to_string()); + for pin in pins { + if result.refused.contains_key(&pin.uuid) { + continue; + } + let reg = registry_name(&pin.uuid); + let Some(current) = kept.as_deref() else { + break; + }; + if !current.contains(&format!("\"{reg}\"")) { + continue; + } + match unpin_line(current, ®) { + Some(next) => { + kept = next; + changed = true; + result.handled.insert(pin.uuid.clone()); + } + None => result.refuse( + &pin.uuid, + format!("{rel} pins it with a declaration socket-patch cannot unpin"), + ), + } + } + if let Some(line) = kept { + out.push(line); + } + } + if changed { + view.write(rel, out.join("\n")); + } + } + + // ── cargo config registry blocks ── + let mut referenced: BTreeSet = BTreeSet::new(); + for rel in manifests.iter().map(String::as_str).chain(["Cargo.lock"]) { + if let Ok(Some(text)) = view.read(rel).await { + for pin in pins { + let reg = registry_name(&pin.uuid); + if text.contains(®) || lock_names_index(&text, &pin.uuid) { + referenced.insert(pin.uuid.clone()); + } + } + } + } + for rel in CARGO_CONFIGS { + let Ok(Some(config)) = view.read(rel).await else { + continue; + }; + let mut next = config.clone(); + for pin in pins { + if result.refused.contains_key(&pin.uuid) || referenced.contains(&pin.uuid) { + continue; + } + if let Some(removed) = remove_registry_block(&next, ®istry_name(&pin.uuid)) { + next = removed; + result.handled.insert(pin.uuid.clone()); + } + } + if next != config { + if next.trim().is_empty() { + view.remove(rel); + } else { + view.write(rel, next); + } + } + } + let _ = by_uuid; + result +} + +/// Whether a Cargo.lock still sources a crate from the hosted index of +/// `uuid` (a pin this pass did not restore). +fn lock_names_index(text: &str, uuid: &str) -> bool { + text.contains(&format!("/{uuid}/index/")) +} + +static SHORTHAND_RE: std::sync::LazyLock = std::sync::LazyLock::new(|| { + Regex::new( + r#"^(\s*[^=\s][^=]*?\s*=\s*)\{ version = "([^"]+)", registry = "(socket-patch-[0-9a-fA-F-]{36})" \}(.*)$"#, + ) + .expect("static shorthand regex is valid") +}); + +/// The line with its `registry = ""` pin removed: `Some(None)` drops +/// the whole line (the table form's inserted `registry = …` line), +/// `Some(Some(line))` is the unpinned declaration, `None` a spelling this +/// restore does not recognize. +fn unpin_line(line: &str, reg: &str) -> Option> { + let trimmed = line.trim(); + let body = trimmed.split('#').next().unwrap_or(trimmed).trim(); + if body == format!("registry = \"{reg}\"") { + return Some(None); + } + if let Some(c) = SHORTHAND_RE.captures(line) { + if &c[3] == reg { + return Some(Some(format!("{}\"{}\"{}", &c[1], &c[2], &c[4]))); + } + } + let open = line.find('{')?; + let close = open + line[open..].find('}')?; + let inner = &line[open + 1..close]; + let pin_re = Regex::new(&format!( + r#",?\s*registry\s*=\s*"{}"\s*,?"#, + regex::escape(reg) + )) + .expect("registry pin regex is valid"); + let m = pin_re.find(inner)?; + let mut rest = String::new(); + rest.push_str(inner[..m.start()].trim_end()); + let tail = inner[m.end()..].trim_start(); + if !tail.is_empty() { + if !rest.trim().is_empty() { + rest.push_str(", "); + } + rest.push_str(tail.trim_end()); + } + let rest = rest.trim().trim_end_matches(',').trim(); + let rebuilt = if rest.is_empty() { + format!("{}{{}}{}", &line[..open], &line[close + 1..]) + } else { + format!("{}{{ {rest} }}{}", &line[..open], &line[close + 1..]) + }; + Some(Some(rebuilt)) +} + +/// The config with its `[registries.]` block (and the blank separator +/// the rewriter put before it) removed; `None` when absent. +fn remove_registry_block(config: &str, reg: &str) -> Option { + let crlf = config.contains("\r\n"); + let lf = config.replace("\r\n", "\n"); + let header = format!("[registries.{reg}]"); + let lines: Vec<&str> = lf.split('\n').collect(); + let i = lines.iter().position(|l| l.trim() == header)?; + let mut end = lines.len(); + for (j, l) in lines.iter().enumerate().skip(i + 1) { + if l.trim_start().starts_with('[') { + end = j; + break; + } + } + while end > i + 1 && lines[end - 1].trim().is_empty() { + end -= 1; + } + let fragment = format!("{}\n", lines[i..end].join("\n")); + let removed = super::super::replay::remove_appended_cargo_block(&lf, &fragment) + .or_else(|| { + // The block ends the file with no final newline. + super::super::replay::remove_appended_cargo_block(&lf, fragment.trim_end_matches('\n')) + })?; + Some(if crlf { + removed.replace('\n', "\r\n") + } else { + removed + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + const REG: &str = "socket-patch-55555555-5555-5555-5555-555555555555"; + + #[test] + fn shorthand_collapses_back() { + let line = format!("serde = {{ version = \"1.0.190\", registry = \"{REG}\" }}"); + assert_eq!( + unpin_line(&line, REG), + Some(Some("serde = \"1.0.190\"".to_string())) + ); + } + + #[test] + fn inline_table_loses_only_the_pin() { + let line = format!( + "serde = {{ version = \"1\", features = [\"derive\"], registry = \"{REG}\" }} # hi" + ); + assert_eq!( + unpin_line(&line, REG), + Some(Some( + "serde = { version = \"1\", features = [\"derive\"] } # hi".to_string() + )) + ); + } + + #[test] + fn table_form_line_is_dropped() { + assert_eq!(unpin_line(&format!("registry = \"{REG}\""), REG), Some(None)); + } + + #[test] + fn appended_block_leaves_the_config_byte_identical() { + let original = "[net]\ngit-fetch-with-cli = true\n"; + let hosted = format!( + "{original}\n[registries.{REG}]\nindex = \"sparse+https://patch.socket.dev/x/index/\"\n" + ); + assert_eq!(remove_registry_block(&hosted, REG).as_deref(), Some(original)); + let created = format!("[registries.{REG}]\nindex = \"sparse+https://x/\"\n"); + assert_eq!(remove_registry_block(&created, REG).as_deref(), Some("")); + } +} diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/client.rs b/crates/socket-patch-core/src/patch/redirect/upstream/client.rs new file mode 100644 index 000000000..0d8f9ef08 --- /dev/null +++ b/crates/socket-patch-core/src/patch/redirect/upstream/client.rs @@ -0,0 +1,308 @@ +//! Registry lookups for the upstream restore: what a default-registry lock +//! entry pins, re-resolved from the public registry (each base overridable +//! by the same env vars the vendored fetch honors, so tests and mirrors can +//! point it elsewhere). + +use std::collections::HashMap; + +use serde_json::Value; +use tokio::sync::Mutex; + +use crate::vendor::registry_fetch::{build_registry_client, npm_registry_base, RegistryClient}; + +/// An npm version's `dist` block. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NpmDist { + /// The registry's tarball URL for the version. + pub tarball: String, + /// The SRI `dist.integrity` (sha512 on every modern publish). + pub integrity: Option, + /// The hex sha1 `dist.shasum`. + pub shasum: Option, +} + +/// A Go module version's two go.sum hashes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct GoSums { + /// `h1:` of the module zip. + pub zip_h1: String, + /// `h1:` of the module's go.mod. + pub mod_h1: String, +} + +/// The sparse crates.io index; override with `SOCKET_CRATES_INDEX`. +pub(crate) const DEFAULT_CRATES_INDEX: &str = "https://index.crates.io"; + +fn crates_index_base() -> String { + std::env::var("SOCKET_CRATES_INDEX") + .ok() + .map(|v| v.trim_end_matches('/').to_string()) + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| DEFAULT_CRATES_INDEX.to_string()) +} + +/// The sparse-index path of a crate (`cargo`'s `index_path` layout). +fn crates_index_path(name: &str) -> String { + let lower = name.to_ascii_lowercase(); + match lower.len() { + 1 => format!("1/{lower}"), + 2 => format!("2/{lower}"), + 3 => format!("3/{}/{lower}", &lower[..1]), + _ => format!("{}/{}/{lower}", &lower[..2], &lower[2..4]), + } +} + +/// The offline refusal every lookup returns under `--offline`. +pub(crate) const OFFLINE: &str = + "the upstream entry must be re-resolved from the registry, and this run is offline"; + +type Cache = Mutex>>; + +/// One client per restore run; every lookup is cached (success and +/// failure alike) so a pin wired in several files costs one request. +pub(crate) struct UpstreamClient { + http: RegistryClient, + offline: bool, + npm: Cache, + npm_tarballs: Cache>, + cargo: Cache, + go: Cache, +} + +impl UpstreamClient { + pub(crate) fn new(offline: bool) -> Self { + UpstreamClient { + http: build_registry_client(), + offline, + npm: Mutex::default(), + npm_tarballs: Mutex::default(), + cargo: Mutex::default(), + go: Mutex::default(), + } + } + + async fn get_json(&self, url: &str) -> Result { + let resp = self + .http + .get(url) + .header("accept", "application/json") + .send() + .await + .map_err(|e| format!("GET {url}: {e}"))?; + let status = resp.status(); + if !status.is_success() { + return Err(format!("GET {url}: HTTP {status}")); + } + let text = resp + .text() + .await + .map_err(|e| format!("reading {url}: {e}"))?; + serde_json::from_str(&text).map_err(|e| format!("{url} is not JSON: {e}")) + } + + async fn get_text(&self, url: &str) -> Result { + let bytes = crate::vendor::registry_fetch::download(&self.http, url).await?; + String::from_utf8(bytes).map_err(|_| format!("{url} is not UTF-8")) + } + + /// `dist` of `name@version` from the npm registry's version document. + pub(crate) async fn npm_dist(&self, name: &str, version: &str) -> Result { + let key = (name.to_string(), version.to_string()); + if let Some(hit) = self.npm.lock().await.get(&key) { + return hit.clone(); + } + let result = self.fetch_npm_dist(name, version).await; + self.npm.lock().await.insert(key, result.clone()); + result + } + + async fn fetch_npm_dist(&self, name: &str, version: &str) -> Result { + if self.offline { + return Err(OFFLINE.to_string()); + } + let encoded_name = name.replace('/', "%2f"); + let url = format!( + "{}/{encoded_name}/{}", + npm_registry_base(), + crate::utils::uri::encode_uri_component(version) + ); + let doc = self.get_json(&url).await?; + let dist = doc + .get("dist") + .ok_or_else(|| format!("{url} carries no `dist` block"))?; + let str_field = |k: &str| dist.get(k).and_then(Value::as_str).map(str::to_string); + let tarball = str_field("tarball") + .ok_or_else(|| format!("{url} carries no `dist.tarball`"))?; + Ok(NpmDist { + tarball, + integrity: str_field("integrity"), + shasum: str_field("shasum"), + }) + } + + /// The verified upstream tarball bytes of `name@version` (checked + /// against the registry's `dist.integrity`). + pub(crate) async fn npm_tarball(&self, name: &str, version: &str) -> Result, String> { + let key = (name.to_string(), version.to_string()); + if let Some(hit) = self.npm_tarballs.lock().await.get(&key) { + return hit.clone(); + } + let result = async { + let dist = self.npm_dist(name, version).await?; + let integrity = dist + .integrity + .clone() + .ok_or_else(|| format!("the registry records no integrity for {name}@{version}"))?; + let bytes = crate::vendor::registry_fetch::download(&self.http, &dist.tarball).await?; + crate::vendor::registry_fetch::verify_sri(&bytes, &integrity)?; + Ok(bytes) + } + .await; + self.npm_tarballs.lock().await.insert(key, result.clone()); + result + } + + /// The crates.io `cksum` (sha256 hex of the `.crate`) of + /// `name@version`, from the sparse index. + pub(crate) async fn cargo_cksum(&self, name: &str, version: &str) -> Result { + let key = (name.to_string(), version.to_string()); + if let Some(hit) = self.cargo.lock().await.get(&key) { + return hit.clone(); + } + let result = async { + if self.offline { + return Err(OFFLINE.to_string()); + } + let url = format!("{}/{}", crates_index_base(), crates_index_path(name)); + let text = self.get_text(&url).await?; + for line in text.lines().filter(|l| !l.trim().is_empty()) { + let Ok(row) = serde_json::from_str::(line) else { + continue; + }; + if row.get("vers").and_then(Value::as_str) == Some(version) { + return row + .get("cksum") + .and_then(Value::as_str) + .filter(|c| crate::utils::digest::is_hex64_lower(c)) + .map(str::to_string) + .ok_or_else(|| format!("{url} lists {version} without a checksum")); + } + } + Err(format!("{url} does not list {name} {version}")) + } + .await; + self.cargo.lock().await.insert(key, result.clone()); + result + } + + /// The go.sum hashes of `module@version`, computed from the module + /// proxy's `.zip` and `.mod` the way `go` computes them. + pub(crate) async fn go_sums(&self, module: &str, version: &str) -> Result { + let key = (module.to_string(), version.to_string()); + if let Some(hit) = self.go.lock().await.get(&key) { + return hit.clone(); + } + let result = async { + if self.offline { + return Err(OFFLINE.to_string()); + } + if let Some(sumdb) = gosumdb_base(module) { + let url = format!( + "{sumdb}/lookup/{}@{}", + crate::crawlers::go_crawler::encode_module_path(module), + crate::crawlers::go_crawler::encode_module_path(version) + ); + let text = self.get_text(&url).await?; + let zip_key = format!("{module} {version} "); + let mod_key = format!("{module} {version}/go.mod "); + let pick = |key: &str| { + text.lines() + .find_map(|l| l.strip_prefix(key)) + .map(|h| h.trim().to_string()) + .filter(|h| crate::vendor::go_sum_edit::is_h1_dirhash(h)) + }; + return match (pick(&zip_key), pick(&mod_key)) { + (Some(zip_h1), Some(mod_h1)) => Ok(GoSums { zip_h1, mod_h1 }), + _ => Err(format!("{url} does not list both go.sum lines of {module} {version}")), + }; + } + let proxy = crate::vendor::registry_fetch::goproxy_base(module)?; + let escaped = crate::crawlers::go_crawler::encode_module_path(module); + let escaped_version = crate::crawlers::go_crawler::encode_module_path(version); + let base = format!("{proxy}/{escaped}/@v/{escaped_version}"); + let zip = crate::vendor::registry_fetch::download(&self.http, &format!("{base}.zip")) + .await?; + let zip_h1 = crate::vendor::registry_fetch::go_h1_of_zip(&zip)?; + let go_mod = + crate::vendor::registry_fetch::download(&self.http, &format!("{base}.mod")).await?; + Ok(GoSums { + zip_h1, + mod_h1: go_mod_h1(&go_mod), + }) + } + .await; + self.go.lock().await.insert(key, result.clone()); + result + } +} + +/// Go's checksum database, `sum.golang.org`; `SOCKET_GOSUMDB_URL` names +/// another (tests, mirrors). +pub(crate) const DEFAULT_GOSUMDB: &str = "https://sum.golang.org"; + +/// The checksum database go would consult for `module`, or `None` when go +/// would not (`GOSUMDB=off`, or the module matches `GONOSUMDB` / +/// `GOPRIVATE`) — the hashes are then computed from the module proxy's +/// bytes instead. An explicit `SOCKET_GOSUMDB_URL` always wins. +fn gosumdb_base(module: &str) -> Option { + if let Ok(v) = std::env::var("SOCKET_GOSUMDB_URL") { + let v = v.trim().trim_end_matches('/').to_string(); + if !v.is_empty() { + return Some(v); + } + } + let nonempty = |key: &str| std::env::var(key).ok().filter(|v| !v.trim().is_empty()); + if nonempty("GOSUMDB").is_some_and(|v| v.trim() == "off") { + return None; + } + if let Some(patterns) = nonempty("GONOSUMDB").or_else(|| nonempty("GOPRIVATE")) { + if crate::vendor::registry_fetch::go_match_prefix_patterns(&patterns, module) { + return None; + } + } + Some(DEFAULT_GOSUMDB.to_string()) +} + +/// x/mod `dirhash.Hash1` over the single file `go.mod` — the `/go.mod h1:` +/// go.sum line. +pub(crate) fn go_mod_h1(go_mod: &[u8]) -> String { + use base64::Engine as _; + use sha2::{Digest, Sha256}; + let file_sum = hex::encode(Sha256::digest(go_mod)); + let summary = format!("{file_sum} go.mod\n"); + format!( + "h1:{}", + base64::engine::general_purpose::STANDARD.encode(Sha256::digest(summary.as_bytes())) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn crates_index_paths_follow_cargo_layout() { + assert_eq!(crates_index_path("a"), "1/a"); + assert_eq!(crates_index_path("ab"), "2/ab"); + assert_eq!(crates_index_path("abc"), "3/a/abc"); + assert_eq!(crates_index_path("Serde"), "se/rd/serde"); + } + + #[test] + fn go_mod_h1_matches_the_x_mod_recipe() { + // `module example.com/m\n` — cross-checked with `go mod download + // -json` output for a one-line module file. + let h1 = go_mod_h1(b"module example.com/m\n"); + assert!(h1.starts_with("h1:") && h1.ends_with('='), "{h1}"); + } +} diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/golang.rs b/crates/socket-patch-core/src/patch/redirect/upstream/golang.rs new file mode 100644 index 000000000..4ce16051f --- /dev/null +++ b/crates/socket-patch-core/src/patch/redirect/upstream/golang.rs @@ -0,0 +1,113 @@ +//! Go upstream restore: drop the hosted `replace M v => patch.socket.dev/ +//! gopatch/ …` directive and the socket module's go.sum lines, and put +//! the upstream module's two go.sum lines back (re-derived from the module +//! proxy exactly as `go` hashes them) — the pair the hosted rewriter pruned. +//! +//! A directive the hosted run took over from the user (a pre-existing +//! `replace` it superseded) is not recorded anywhere, so the restore always +//! lands on the plain upstream module; the refusal message of a formats the +//! restore cannot handle names the checkout remedy instead. + +use super::{Ctx, FormatResult, HostedPin, View}; +use crate::vendor::go_mod_edit::{ + hosted_module_uuid, parse_replace_entries, remove_replace_entry, ReplaceOwner, +}; +use crate::vendor::go_sum_edit::{reinsert_lines, remove_module_prefix_lines}; + +pub(crate) async fn restore( + view: &mut View<'_>, + pins: &[&HostedPin], + _files: &[String], + ctx: &Ctx<'_>, +) -> FormatResult { + let mut result = FormatResult::default(); + let go_mod = match view.read("go.mod").await { + Ok(Some(text)) => text, + Ok(None) => { + for pin in pins { + result.refuse(&pin.uuid, "go.mod no longer exists"); + } + return result; + } + Err(e) => { + for pin in pins { + result.refuse(&pin.uuid, e.clone()); + } + return result; + } + }; + let entries = parse_replace_entries(&go_mod); + // (uuid, module, version, socket module path) per hosted directive. + let mut hits: Vec<(String, String, String, String)> = Vec::new(); + for pin in pins { + let Some((module, version)) = pin.name_version() else { + result.refuse(&pin.uuid, format!("{} is not a golang purl", pin.purl)); + continue; + }; + let directive = entries.iter().find(|e| { + e.module == module + && e.rhs_module + .as_deref() + .and_then(hosted_module_uuid) + .is_some_and(|u| u == pin.uuid) + }); + let Some(directive) = directive else { + continue; + }; + let version = directive.version.clone().unwrap_or(version); + let socket_module = directive.rhs_module.clone().unwrap_or_default(); + hits.push((pin.uuid.clone(), module, version, socket_module)); + } + if hits.is_empty() { + return result; + } + let lookups = hits.iter().map(|(uuid, module, version, _)| async move { + (uuid.clone(), ctx.client.go_sums(module, version).await) + }); + let sums: std::collections::BTreeMap> = + futures_util::future::join_all(lookups).await.into_iter().collect(); + + let mut go_mod_next = go_mod.clone(); + let mut go_sum = view.read("go.sum").await.ok().flatten(); + let go_sum_original = go_sum.clone(); + for (uuid, module, version, socket_module) in &hits { + let sums = match sums.get(uuid) { + Some(Ok(s)) => s, + Some(Err(why)) => { + result.refuse(uuid, format!("{module}@{version}: {why}")); + continue; + } + None => continue, + }; + match remove_replace_entry(&go_mod_next, module, ReplaceOwner::Hosted) { + Ok(Some(next)) => go_mod_next = next, + Ok(None) => {} + Err(e) => { + result.refuse(uuid, format!("go.mod: {e}")); + continue; + } + } + if let Some(text) = go_sum.as_deref() { + let mut next = remove_module_prefix_lines(text, socket_module) + .unwrap_or_else(|| text.to_string()); + let upstream = format!( + "{module} {version} {}\n{module} {version}/go.mod {}\n", + sums.zip_h1, sums.mod_h1 + ); + if let Some(reinserted) = reinsert_lines(&next, &upstream) { + next = reinserted; + } + go_sum = Some(next); + } + result.handled.insert(uuid.clone()); + } + if go_mod_next != go_mod { + view.write("go.mod", go_mod_next); + } + if go_sum != go_sum_original { + if let Some(text) = go_sum { + view.write("go.sum", text); + } + } + result +} diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/mod.rs b/crates/socket-patch-core/src/patch/redirect/upstream/mod.rs new file mode 100644 index 000000000..7c5636d00 --- /dev/null +++ b/crates/socket-patch-core/src/patch/redirect/upstream/mod.rs @@ -0,0 +1,416 @@ +//! Hosted → upstream restore: the v5 unwind of a hosted redirect. +//! +//! v5 hosted mode keeps no ledger (`scan`/`get --mode hosted` write only +//! lockfile edits), so an unwind cannot replay recorded fragments. Instead, +//! for every hosted pin the lockfiles carry (`vex::discover`'s hosted refs: +//! purl + patch uuid + the files wiring it), this module rewrites the lock +//! entry back to the DEFAULT UPSTREAM registry entry for `name@version`, +//! re-resolving whatever the entry pins (tarball URL, integrity, checksum) +//! from the public registry: the npm registry's version document, the +//! crates.io sparse index, the Go module proxy, and so on. +//! +//! Where that is impossible — a format whose entry carries fields only the +//! package manager can compute, a binary lockfile, an offline run, a +//! registry that does not answer — the pin is REFUSED with a message naming +//! the remedy (`git checkout -- `). A refusal is all-or-nothing +//! per pin: a pin refused in one of its files is restored in none of them, +//! so no pin is ever left half hosted. +//! +//! Nothing reaches disk until every pin resolved (the staged view below), +//! and a dry run resolves everything exactly like a wet run — network +//! lookups included — and skips only the flush. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use crate::vex::discover::{Discovery, PatchedRef, WiringMode}; + +mod cargo; +mod client; +mod golang; +mod npm; + +pub(crate) use client::UpstreamClient; + +use super::staged::{flush_staged, read_rel, Staged, StagedBytes}; + +/// One hosted pin to restore: a `(purl, patch uuid)` pair and the +/// root-relative files lockfile discovery found it wired in. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostedPin { + /// Canonical base purl (no qualifiers), as discovery spells it. + pub purl: String, + /// The hosted patch uuid. + pub uuid: String, + /// Root-relative files that wire this pin, sorted and deduplicated. + pub files: Vec, +} + +impl HostedPin { + /// Group a discovery's HOSTED refs into pins, one per `(purl, uuid)`. + pub fn from_refs<'a>(refs: impl IntoIterator) -> Vec { + let mut grouped: BTreeMap<(String, String), BTreeSet> = BTreeMap::new(); + for r in refs { + if r.mode != WiringMode::Hosted { + continue; + } + grouped + .entry((r.purl.clone(), r.uuid.clone())) + .or_default() + .insert(r.source_file.to_string_lossy().replace('\\', "/")); + } + grouped + .into_iter() + .map(|((purl, uuid), files)| HostedPin { + purl, + uuid, + files: files.into_iter().collect(), + }) + .collect() + } + + /// Every hosted pin a discovery holds. + pub fn all(discovery: &Discovery) -> Vec { + Self::from_refs(&discovery.refs) + } + + /// `(name, version)` of the purl, percent-decoded. + pub(crate) fn name_version(&self) -> Option<(String, String)> { + let (_, name, version) = crate::utils::purl::purl_parts(&self.purl)?; + Some((name, version)) + } +} + +/// Knobs for [`restore_upstream`]. +#[derive(Debug, Clone, Default)] +pub struct RestoreOptions { + /// Resolve everything, write nothing. + pub dry_run: bool, + /// No network: every pin whose restore needs a registry lookup is + /// refused with the checkout remedy. + pub offline: bool, + /// Extra patch-server origins whose URLs count as hosted (the + /// operator's `--patch-server-url`), exactly as discovery takes them. + pub patch_server_origins: Vec, +} + +/// What happened to one pin. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PinStatus { + /// Every file wiring the pin now resolves the upstream registry entry + /// (or would, on a dry run). + Restored, + /// Nothing was changed for this pin. The message names the remedy. + Refused(String), +} + +/// One pin's outcome. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PinResult { + pub purl: String, + pub uuid: String, + pub status: PinStatus, + /// The files that wired the pin. + pub files: Vec, +} + +/// What [`restore_upstream`] did. +#[derive(Debug, Default)] +pub struct RestoreOutcome { + /// One entry per input pin, in input order. + pub pins: Vec, + /// Root-relative files rewritten or removed (or that would be, on a + /// dry run), sorted. + pub reverted_files: Vec, + /// Advisory `(code, detail)` pairs. + pub warnings: Vec<(&'static str, String)>, + /// A write failure after every pin resolved: some files may have + /// landed. `None` on a clean flush (and always on a dry run). + pub flush_error: Option, +} + +impl RestoreOutcome { + pub fn restored(&self) -> impl Iterator { + self.pins + .iter() + .filter(|p| p.status == PinStatus::Restored) + } + + pub fn refused(&self) -> impl Iterator { + self.pins.iter().filter_map(|p| match &p.status { + PinStatus::Refused(why) => Some((p, why.as_str())), + PinStatus::Restored => None, + }) + } +} + +/// The remedy every refusal names: restore the file from version control. +pub fn checkout_remedy(files: &[String]) -> String { + if files.is_empty() { + return "restore the lockfile from version control (`git checkout -- `)" + .to_string(); + } + format!( + "restore it from version control instead (`git checkout -- {}`)", + files.join(" ") + ) +} + +/// The staged project view the restorers work over: reads fall through to +/// disk, writes stay in memory until [`restore_upstream`] flushes them. +pub(crate) struct View<'a> { + root: &'a Path, + staged: Staged, + /// Original on-disk text of every file read, so a write that restores + /// the exact original bytes is not reported as a change. + originals: BTreeMap>, +} + +impl<'a> View<'a> { + fn new(root: &'a Path) -> Self { + View { + root, + staged: Staged::new(), + originals: BTreeMap::new(), + } + } + + pub(crate) fn root(&self) -> &Path { + self.root + } + + /// The current (staged) text of `rel`; `Ok(None)` when absent. + pub(crate) async fn read(&mut self, rel: &str) -> Result, String> { + if let Some(pending) = self.staged.get(rel) { + return Ok(pending.clone()); + } + if let Some(original) = self.originals.get(rel) { + return Ok(original.clone()); + } + let text = read_rel(self.root, rel).await?; + self.originals.insert(rel.to_string(), text.clone()); + Ok(text) + } + + pub(crate) fn write(&mut self, rel: &str, content: String) { + self.staged.insert(rel.to_string(), Some(content)); + } + + pub(crate) fn remove(&mut self, rel: &str) { + self.staged.insert(rel.to_string(), None); + } + + /// Files whose staged state differs from what was read from disk. + fn changed(&self) -> Staged { + self.staged + .iter() + .filter(|(rel, pending)| self.originals.get(*rel) != Some(*pending)) + .map(|(rel, pending)| (rel.clone(), pending.clone())) + .collect() + } +} + +/// Per-format restore result, merged across formats by the driver. +#[derive(Debug, Default)] +pub(crate) struct FormatResult { + /// Pins (by uuid) this format refused, with the reason (the driver + /// appends the remedy). + pub refused: BTreeMap, + /// Pins (by uuid) this format found wired and restored in the view. + pub handled: BTreeSet, + pub warnings: Vec<(&'static str, String)>, +} + +impl FormatResult { + pub(crate) fn refuse(&mut self, uuid: &str, why: impl Into) { + self.refused + .entry(uuid.to_string()) + .or_insert_with(|| why.into()); + } + + fn merge(&mut self, other: FormatResult) { + for (uuid, why) in other.refused { + self.refused.entry(uuid).or_insert(why); + } + self.handled.extend(other.handled); + self.warnings.extend(other.warnings); + } +} + +/// Shared context handed to every format restorer. +pub(crate) struct Ctx<'a> { + pub client: &'a UpstreamClient, + pub origins: &'a [String], +} + +impl Ctx<'_> { + /// The hosted patch uuid `url` names, under the same host allowlist + /// discovery applies. + pub(crate) fn hosted_uuid(&self, url: &str) -> Option { + super::hosted_patch_uuid(url, self.origins) + } +} + +/// The lock formats a pin's files belong to. One restorer per format sees +/// every in-scope pin wired in that format's files at once. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Format { + NpmLock, + YarnLock, + PnpmLock, + BunLock, + Cargo, + Golang, + Unsupported, +} + +fn format_of(rel: &str) -> Format { + let leaf = rel.rsplit('/').next().unwrap_or(rel); + match leaf { + "package-lock.json" | "npm-shrinkwrap.json" => Format::NpmLock, + "yarn.lock" => Format::YarnLock, + "pnpm-lock.yaml" | "shrinkwrap.yaml" => Format::PnpmLock, + "bun.lock" => Format::BunLock, + "Cargo.toml" | "Cargo.lock" | "config.toml" | "config" => Format::Cargo, + "go.mod" | "go.sum" | "go.work" => Format::Golang, + _ => Format::Unsupported, + } +} + +/// Restore every pin in `pins` to its default upstream registry entry. +/// See the module docs for the contract. +pub async fn restore_upstream( + root: &Path, + pins: &[HostedPin], + opts: &RestoreOptions, +) -> RestoreOutcome { + let client = UpstreamClient::new(opts.offline); + let ctx = Ctx { + client: &client, + origins: &opts.patch_server_origins, + }; + + // Pins refused so far (uuid → reason). Each pass restores the pins not + // yet refused over a FRESH view; a pass that refuses a new pin is rerun + // without it, so the final view restores exactly the surviving set. + let mut refused: BTreeMap = BTreeMap::new(); + let (view, result) = loop { + let active: Vec<&HostedPin> = pins + .iter() + .filter(|p| !refused.contains_key(&p.uuid)) + .collect(); + let mut view = View::new(root); + let result = restore_pass(&mut view, &active, &ctx).await; + let mut grew = false; + for (uuid, why) in &result.refused { + if !refused.contains_key(uuid) { + refused.insert(uuid.clone(), why.clone()); + grew = true; + } + } + // A pin no restorer claimed has wiring nothing here can unwind. + for pin in &active { + if !result.handled.contains(&pin.uuid) && !refused.contains_key(&pin.uuid) { + refused.insert( + pin.uuid.clone(), + format!( + "no hosted wiring for {} was found in {} that socket-patch can \ + restore to the upstream registry entry", + pin.purl, + if pin.files.is_empty() { + "the lockfiles".to_string() + } else { + pin.files.join(", ") + } + ), + ); + grew = true; + } + } + if !grew { + break (view, result); + } + }; + + let changed = view.changed(); + let reverted_files: BTreeSet = changed.keys().cloned().collect(); + let flush_error = if opts.dry_run || changed.is_empty() { + None + } else { + flush_staged(root, &changed, &StagedBytes::new()).await.err() + }; + + let pins_out = pins + .iter() + .map(|pin| PinResult { + purl: pin.purl.clone(), + uuid: pin.uuid.clone(), + status: match refused.get(&pin.uuid) { + Some(why) => PinStatus::Refused(format!( + "cannot restore {} to its upstream registry entry: {why}; {}", + pin.purl, + checkout_remedy(&pin.files) + )), + None => PinStatus::Restored, + }, + files: pin.files.clone(), + }) + .collect(); + + RestoreOutcome { + pins: pins_out, + reverted_files: reverted_files.into_iter().collect(), + warnings: result.warnings, + flush_error, + } +} + +/// One restore pass over `view` for the `active` pins. +async fn restore_pass(view: &mut View<'_>, active: &[&HostedPin], ctx: &Ctx<'_>) -> FormatResult { + let mut by_format: BTreeMap, BTreeSet)> = BTreeMap::new(); + for pin in active { + for file in &pin.files { + let slot = by_format.entry(format_of(file)).or_default(); + if !slot.0.iter().any(|p| p.uuid == pin.uuid) { + slot.0.push(pin); + } + slot.1.insert(file.clone()); + } + } + let mut out = FormatResult::default(); + for (format, (pins, files)) in by_format { + let files: Vec = files.into_iter().collect(); + let result = match format { + Format::NpmLock => npm::restore_npm_locks(view, &pins, &files, ctx).await, + Format::YarnLock => npm::restore_yarn_locks(view, &pins, &files, ctx).await, + Format::PnpmLock => npm::restore_pnpm_locks(view, &pins, &files, ctx).await, + Format::BunLock => npm::restore_bun_locks(view, &pins, &files, ctx).await, + Format::Cargo => cargo::restore(view, &pins, &files, ctx).await, + Format::Golang => golang::restore(view, &pins, &files, ctx).await, + Format::Unsupported => { + let mut r = FormatResult::default(); + for pin in &pins { + let unsupported: Vec<&str> = pin + .files + .iter() + .filter(|f| format_of(f) == Format::Unsupported) + .map(String::as_str) + .collect(); + r.refuse( + &pin.uuid, + format!( + "socket-patch cannot re-derive the upstream entry in {}", + unsupported.join(", ") + ), + ); + } + r + } + }; + out.merge(result); + } + // The npm-family side settings a hosted run may have written, once no + // npm-family lock entry needs them any more. + npm::cleanup_side_config(view, ctx, &mut out).await; + out +} diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/npm.rs b/crates/socket-patch-core/src/patch/redirect/upstream/npm.rs new file mode 100644 index 000000000..00f001c16 --- /dev/null +++ b/crates/socket-patch-core/src/patch/redirect/upstream/npm.rs @@ -0,0 +1,742 @@ +//! npm-family upstream restores: package-lock.json / npm-shrinkwrap.json, +//! yarn.lock (classic and berry), pnpm-lock.yaml, bun.lock — plus the +//! npm-family side settings a hosted run writes (`.npmrc` +//! `allow-remote=all`, pnpm-workspace.yaml `trustLockfile: true`). +//! +//! Every restorer rewrites ONLY entries whose resolution is a hosted URL +//! naming one of the in-scope patch uuids; every other byte of the file is +//! the file's own. The upstream values come from the npm registry's +//! version document (`dist.tarball`, `dist.integrity`, `dist.shasum`). + +use std::collections::{BTreeMap, BTreeSet}; + +use regex::Regex; +use serde_json::Value; + +use super::client::NpmDist; +use super::{Ctx, FormatResult, HostedPin, View}; +use crate::utils::line_endings::{to_lf, LineEndings}; + +/// The pins by uuid. +fn by_uuid<'p>(pins: &[&'p HostedPin]) -> BTreeMap<&'p str, &'p HostedPin> { + pins.iter().map(|p| (p.uuid.as_str(), *p)).collect() +} + +/// Resolve the dist of every `(uuid, name, version)` wanted, concurrently. +/// A failed lookup refuses its pin. +async fn fetch_dists( + wanted: &BTreeSet<(String, String, String)>, + ctx: &Ctx<'_>, + result: &mut FormatResult, +) -> BTreeMap<(String, String), NpmDist> { + let lookups = wanted.iter().map(|(uuid, name, version)| async move { + ( + uuid.clone(), + name.clone(), + version.clone(), + ctx.client.npm_dist(name, version).await, + ) + }); + let mut out = BTreeMap::new(); + for (uuid, name, version, dist) in futures_util::future::join_all(lookups).await { + match dist { + Ok(dist) => { + out.insert((name, version), dist); + } + Err(why) => result.refuse(&uuid, format!("{name}@{version}: {why}")), + } + } + out +} + +// ── package-lock.json / npm-shrinkwrap.json ───────────────────────────────── + +/// One hosted entry of an npm lock: its JSON pointer, and what it stands for. +struct NpmHit { + pointer: String, + uuid: String, + name: String, + version: String, +} + +fn npm_lock_hits(lock: &Value, ctx: &Ctx<'_>) -> Vec { + let mut hits = Vec::new(); + if let Some(packages) = lock.get("packages").and_then(Value::as_object) { + for (key, entry) in packages { + let Some((_, key_name)) = key.rsplit_once("node_modules/") else { + continue; + }; + let Some(uuid) = entry + .get("resolved") + .and_then(Value::as_str) + .and_then(|u| ctx.hosted_uuid(u)) + else { + continue; + }; + let name = entry + .get("name") + .and_then(Value::as_str) + .unwrap_or(key_name) + .to_string(); + let Some(version) = entry.get("version").and_then(Value::as_str) else { + continue; + }; + hits.push(NpmHit { + pointer: format!("/packages/{}", json_pointer_escape(key)), + uuid, + name, + version: version.to_string(), + }); + } + } + if let Some(deps) = lock.get("dependencies").and_then(Value::as_object) { + v2_hits(deps, "/dependencies", ctx, &mut hits, 0); + } + hits +} + +fn v2_hits( + deps: &serde_json::Map, + prefix: &str, + ctx: &Ctx<'_>, + hits: &mut Vec, + depth: usize, +) { + if depth > 64 { + return; + } + for (name, entry) in deps { + let pointer = format!("{prefix}/{}", json_pointer_escape(name)); + if let (Some(uuid), Some(version)) = ( + entry + .get("resolved") + .and_then(Value::as_str) + .and_then(|u| ctx.hosted_uuid(u)), + entry.get("version").and_then(Value::as_str), + ) { + hits.push(NpmHit { + pointer: pointer.clone(), + uuid, + name: name.clone(), + version: version.to_string(), + }); + } + if let Some(nested) = entry.get("dependencies").and_then(Value::as_object) { + v2_hits(nested, &format!("{pointer}/dependencies"), ctx, hits, depth + 1); + } + } +} + +fn json_pointer_escape(key: &str) -> String { + key.replace('~', "~0").replace('/', "~1") +} + +pub(crate) async fn restore_npm_locks( + view: &mut View<'_>, + pins: &[&HostedPin], + files: &[String], + ctx: &Ctx<'_>, +) -> FormatResult { + let mut result = FormatResult::default(); + let pins = by_uuid(pins); + for rel in files { + let Some(text) = read_or_refuse(view, rel, &pins, &mut result).await else { + continue; + }; + let Ok(mut lock) = serde_json::from_str::(&text) else { + refuse_all_in(&pins, rel, &mut result, format!("{rel} is not valid JSON")); + continue; + }; + let hits: Vec = npm_lock_hits(&lock, ctx) + .into_iter() + .filter(|h| pins.contains_key(h.uuid.as_str())) + .collect(); + let wanted: BTreeSet<(String, String, String)> = hits + .iter() + .map(|h| (h.uuid.clone(), h.name.clone(), h.version.clone())) + .collect(); + let dists = fetch_dists(&wanted, ctx, &mut result).await; + let mut changed = false; + for hit in &hits { + if result.refused.contains_key(&hit.uuid) { + continue; + } + let Some(dist) = dists.get(&(hit.name.clone(), hit.version.clone())) else { + continue; + }; + let Some(integrity) = dist.integrity.as_deref() else { + result.refuse( + &hit.uuid, + format!( + "the registry records no integrity for {}@{}", + hit.name, hit.version + ), + ); + continue; + }; + let Some(entry) = lock.pointer_mut(&hit.pointer).and_then(Value::as_object_mut) + else { + continue; + }; + entry.insert("resolved".into(), Value::String(dist.tarball.clone())); + entry.insert("integrity".into(), Value::String(integrity.to_string())); + result.handled.insert(hit.uuid.clone()); + changed = true; + } + if changed { + view.write(rel, super::super::serialize_json(&lock)); + } + } + result +} + +/// Read `rel` through the view; a missing or unreadable file refuses every +/// pin discovery found in it. +async fn read_or_refuse( + view: &mut View<'_>, + rel: &str, + pins: &BTreeMap<&str, &HostedPin>, + result: &mut FormatResult, +) -> Option { + match view.read(rel).await { + Ok(Some(text)) => Some(text), + Ok(None) => { + refuse_all_in(pins, rel, result, format!("{rel} no longer exists")); + None + } + Err(e) => { + refuse_all_in(pins, rel, result, e); + None + } + } +} + +fn refuse_all_in( + pins: &BTreeMap<&str, &HostedPin>, + rel: &str, + result: &mut FormatResult, + why: String, +) { + for pin in pins.values() { + if pin.files.iter().any(|f| f == rel) { + result.refuse(&pin.uuid, why.clone()); + } + } +} + +// ── yarn.lock ──────────────────────────────────────────────────────────────── + +/// yarn v1's default registry host, used for a classic lock's `resolved` +/// unless `SOCKET_NPM_REGISTRY` names another. +const YARN_CLASSIC_REGISTRY: &str = "https://registry.yarnpkg.com"; + +fn yarn_classic_tarball(dist: &NpmDist) -> String { + if std::env::var("SOCKET_NPM_REGISTRY").is_ok_and(|v| !v.trim().is_empty()) { + return dist.tarball.clone(); + } + match dist + .tarball + .strip_prefix(crate::vendor::registry_fetch::DEFAULT_NPM_REGISTRY) + { + Some(rest) => format!("{YARN_CLASSIC_REGISTRY}{rest}"), + None => dist.tarball.clone(), + } +} + +pub(crate) async fn restore_yarn_locks( + view: &mut View<'_>, + pins: &[&HostedPin], + files: &[String], + ctx: &Ctx<'_>, +) -> FormatResult { + let mut result = FormatResult::default(); + let pins = by_uuid(pins); + for rel in files { + let Some(raw) = read_or_refuse(view, rel, &pins, &mut result).await else { + continue; + }; + if super::super::is_berry_lock(&raw) { + restore_berry(view, rel, &raw, &pins, ctx, &mut result).await; + } else { + restore_classic(view, rel, &raw, &pins, ctx, &mut result).await; + } + } + result +} + +async fn restore_classic( + view: &mut View<'_>, + rel: &str, + raw: &str, + pins: &BTreeMap<&str, &HostedPin>, + ctx: &Ctx<'_>, + result: &mut FormatResult, +) { + let eol = LineEndings::of(raw); + if eol == LineEndings::Mixed { + refuse_all_in(pins, rel, result, format!("{rel} mixes line endings")); + return; + } + let content = to_lf(raw); + let mut blocks: Vec = content.split("\n\n").map(String::from).collect(); + let resolved_re = + Regex::new(r#"\n {2}resolved "([^"]*)""#).expect("static resolved-line regex is valid"); + let integrity_re = + Regex::new(r"\n {2}integrity [^\n]*").expect("static integrity-line regex is valid"); + let version_re = + Regex::new(r#"\n {2}version "([^"]*)""#).expect("static version-line regex is valid"); + + // (block index, uuid, name, version) per hosted block. + let mut hits: Vec<(usize, String, String, String)> = Vec::new(); + for (i, block) in blocks.iter().enumerate() { + let Some(uuid) = resolved_re + .captures(block) + .and_then(|c| ctx.hosted_uuid(&c[1])) + else { + continue; + }; + if !pins.contains_key(uuid.as_str()) { + continue; + } + let name = super::super::yarn_classic_block_head(block).and_then(|(_, n)| n); + let version = version_re.captures(block).map(|c| c[1].to_string()); + match (name, version) { + (Some(name), Some(version)) => hits.push((i, uuid, name, version)), + _ => result.refuse( + &uuid, + format!("a {rel} entry wiring it names no single package and version"), + ), + } + } + let wanted = hits + .iter() + .map(|(_, u, n, v)| (u.clone(), n.clone(), v.clone())) + .collect(); + let dists = fetch_dists(&wanted, ctx, result).await; + let mut changed = false; + for (i, uuid, name, version) in hits { + if result.refused.contains_key(&uuid) { + continue; + } + let Some(dist) = dists.get(&(name.clone(), version.clone())) else { + continue; + }; + let Some(integrity) = dist.integrity.as_deref() else { + result.refuse( + &uuid, + format!("the registry records no integrity for {name}@{version}"), + ); + continue; + }; + let frag = dist + .shasum + .as_deref() + .map(|s| format!("#{s}")) + .unwrap_or_default(); + let resolved = format!( + "\n resolved \"{}{frag}\"", + yarn_classic_tarball(dist).replace('$', "$$") + ); + let mut block = resolved_re + .replace(&blocks[i], resolved.as_str()) + .into_owned(); + if integrity_re.is_match(&block) { + block = integrity_re + .replace(&block, format!("\n integrity {integrity}").as_str()) + .into_owned(); + } + blocks[i] = block; + result.handled.insert(uuid); + changed = true; + } + if changed { + view.write(rel, eol.restore(&blocks.join("\n\n")).into_owned()); + } +} + +async fn restore_berry( + view: &mut View<'_>, + rel: &str, + raw: &str, + pins: &BTreeMap<&str, &HostedPin>, + ctx: &Ctx<'_>, + result: &mut FormatResult, +) { + use crate::vendor::yarn_classic_lock::{split_berry_key_patterns, split_pattern}; + + let (bom, body) = match raw.strip_prefix('\u{feff}') { + Some(rest) => ("\u{feff}", rest), + None => ("", raw), + }; + let yarnrc_rel = match rel.rsplit_once('/') { + Some((dir, _)) => format!("{dir}/.yarnrc.yml"), + None => ".yarnrc.yml".to_string(), + }; + let yarnrc = view.read(&yarnrc_rel).await.ok().flatten(); + if let Err(w) = super::super::preflight_yarn_berry_hosted(raw, yarnrc.as_deref()) { + refuse_all_in(pins, rel, result, w.detail); + return; + } + let eol = LineEndings::of(body); + let content = to_lf(body).into_owned(); + let mut blocks: Vec = content.split("\n\n").map(String::from).collect(); + let resolution_re = Regex::new(r#"\n {2}resolution: "([^"]*)""#) + .expect("static resolution-line regex is valid"); + let checksum_re = + Regex::new(r"\n {2}checksum: [^\n]*").expect("static checksum-line regex is valid"); + let version_re = + Regex::new(r"\n {2}version: ([^\n]*)").expect("static version-line regex is valid"); + + let mut hits: Vec<(usize, String, String, String)> = Vec::new(); + for (i, block) in blocks.iter().enumerate() { + let Some(resolution) = resolution_re.captures(block).map(|c| c[1].to_string()) else { + continue; + }; + let Some((_, archive)) = resolution.split_once("::__archiveUrl=") else { + continue; + }; + let archive = archive.split('&').next().unwrap_or(archive); + let Some(uuid) = ctx.hosted_uuid(archive) else { + continue; + }; + if !pins.contains_key(uuid.as_str()) { + continue; + } + let key = block + .lines() + .next() + .and_then(|l| l.strip_suffix(':')) + .unwrap_or(""); + let patterns = split_berry_key_patterns(key); + let names: BTreeSet = patterns + .iter() + .filter_map(|p| split_pattern(p).map(|(n, _)| n.to_string())) + .collect(); + let version = version_re + .captures(block) + .map(|c| c[1].trim().trim_matches('"').to_string()); + match (names.len(), names.into_iter().next(), version) { + (1, Some(name), Some(version)) => hits.push((i, uuid, name.to_string(), version)), + _ => result.refuse( + &uuid, + format!("a {rel} entry wiring it names no single package and version"), + ), + } + } + let mut changed = false; + for (i, uuid, name, version) in hits { + if result.refused.contains_key(&uuid) { + continue; + } + let checksum = match ctx.client.npm_tarball(&name, &version).await.and_then(|tgz| { + crate::vendor::berry_zip::berry_cache_checksum_10c0(&tgz, &name) + }) { + Ok(c) => crate::vendor::yarn_berry_lock::checksum_in_lock_spelling(&content, &c), + Err(why) => { + result.refuse(&uuid, format!("{name}@{version}: {why}")); + continue; + } + }; + let resolution = format!("\n resolution: \"{name}@npm:{version}\"").replace('$', "$$"); + let mut block = resolution_re + .replace(&blocks[i], resolution.as_str()) + .into_owned(); + if checksum_re.is_match(&block) { + block = checksum_re + .replace(&block, format!("\n checksum: {checksum}").as_str()) + .into_owned(); + } + blocks[i] = block; + result.handled.insert(uuid); + changed = true; + } + if changed { + view.write( + rel, + format!("{bom}{}", eol.restore(&blocks.join("\n\n"))), + ); + } +} + +// ── pnpm-lock.yaml ─────────────────────────────────────────────────────────── + +pub(crate) async fn restore_pnpm_locks( + view: &mut View<'_>, + pins: &[&HostedPin], + files: &[String], + ctx: &Ctx<'_>, +) -> FormatResult { + use super::super::pnpm; + + let mut result = FormatResult::default(); + let pins = by_uuid(pins); + for rel in files { + let Some(text) = read_or_refuse(view, rel, &pins, &mut result).await else { + continue; + }; + // (resolution range, uuid, name, version, rebuilt-without-integrity) + let mut hits: Vec<(std::ops::Range, String, String, String)> = Vec::new(); + for entry in pnpm::entries(&text) { + let Some(resolution) = pnpm::resolution(&entry) else { + continue; + }; + let Some(uuid) = resolution.tarball().and_then(|t| ctx.hosted_uuid(t)) else { + continue; + }; + let Some(pin) = pins.get(uuid.as_str()) else { + continue; + }; + let Some((name, version)) = pin.name_version() else { + result.refuse(&uuid, format!("{} is not an npm purl", pin.purl)); + continue; + }; + if pnpm::suffix(entry.key, &name, &version).is_none() { + result.refuse( + &uuid, + format!( + "the {rel} entry `{}` wiring it is not {name}@{version}", + entry.key + ), + ); + continue; + } + hits.push((resolution.range.clone(), uuid, name, version)); + } + let wanted = hits + .iter() + .map(|(_, u, n, v)| (u.clone(), n.clone(), v.clone())) + .collect(); + let dists = fetch_dists(&wanted, ctx, &mut result).await; + let mut splices: Vec<(std::ops::Range, String)> = Vec::new(); + let mut handled: Vec = Vec::new(); + for entry in pnpm::entries(&text) { + let Some(resolution) = pnpm::resolution(&entry) else { + continue; + }; + let Some((_, uuid, name, version)) = + hits.iter().find(|(r, ..)| *r == resolution.range) + else { + continue; + }; + if result.refused.contains_key(uuid) { + continue; + } + let Some(integrity) = dists + .get(&(name.clone(), version.clone())) + .and_then(|d| d.integrity.clone()) + else { + result.refuse( + uuid, + format!("the registry records no integrity for {name}@{version}"), + ); + continue; + }; + splices.push((resolution.range.clone(), resolution.restore(&integrity))); + handled.push(uuid.clone()); + } + // A refusal recorded after a splice was planned (a second instance + // of the same pin) drops that pin's splices too. + let splices: Vec<_> = splices + .into_iter() + .zip(&handled) + .filter(|(_, u)| !result.refused.contains_key(*u)) + .map(|(s, _)| s) + .collect(); + if splices.is_empty() { + continue; + } + let mut out = String::with_capacity(text.len()); + let mut cursor = 0; + let mut sorted = splices; + sorted.sort_by_key(|(r, _)| r.start); + for (range, replacement) in sorted { + out.push_str(&text[cursor..range.start]); + out.push_str(&replacement); + cursor = range.end; + } + out.push_str(&text[cursor..]); + for uuid in handled { + if !result.refused.contains_key(&uuid) { + result.handled.insert(uuid); + } + } + view.write(rel, out); + } + result +} + +// ── bun.lock ───────────────────────────────────────────────────────────────── + +pub(crate) async fn restore_bun_locks( + view: &mut View<'_>, + pins: &[&HostedPin], + files: &[String], + ctx: &Ctx<'_>, +) -> FormatResult { + use crate::vendor::bun_lock_text::{decode_json_string, split_name_spec}; + + let mut result = FormatResult::default(); + let pins = by_uuid(pins); + for rel in files { + let Some(text) = read_or_refuse(view, rel, &pins, &mut result).await else { + continue; + }; + let (mut lines, entries) = match super::super::parse_bun_hosted_lock(&text) { + Ok(parsed) => parsed, + Err(w) => { + refuse_all_in(&pins, rel, &mut result, w.detail); + continue; + } + }; + // (line, uuid, name, version, deps) + let mut hits: Vec<(usize, String, String, String, String)> = Vec::new(); + for entry in &entries { + if !matches!(entry.elems.len(), 2 | 3) || !entry.elems[1].starts_with('{') { + continue; + } + let Some(spec) = entry.elems.first().and_then(|e| decode_json_string(e)) else { + continue; + }; + let Some((name, url)) = split_name_spec(&spec) else { + continue; + }; + let Some(uuid) = ctx.hosted_uuid(url) else { + continue; + }; + let Some(pin) = pins.get(uuid.as_str()) else { + continue; + }; + match pin.name_version() { + Some((pin_name, version)) if pin_name == name => hits.push(( + entry.line_idx, + uuid, + name.to_string(), + version, + entry.elems[1].clone(), + )), + _ => result.refuse( + &uuid, + format!("the {rel} entry `{}` wiring it is not {}", entry.key, pin.purl), + ), + } + } + let wanted = hits + .iter() + .map(|(_, u, n, v, _)| (u.clone(), n.clone(), v.clone())) + .collect(); + let dists = fetch_dists(&wanted, ctx, &mut result).await; + let mut changed = false; + for (line_idx, uuid, name, version, deps) in hits { + if result.refused.contains_key(&uuid) { + continue; + } + let Some(integrity) = dists + .get(&(name.clone(), version.clone())) + .and_then(|d| d.integrity.clone()) + else { + result.refuse( + &uuid, + format!("the registry records no integrity for {name}@{version}"), + ); + continue; + }; + let Some(entry) = entries.iter().find(|e| e.line_idx == line_idx) else { + continue; + }; + let original = &lines[line_idx]; + let cr = if original.ends_with('\r') { "\r" } else { "" }; + let json = |s: &str| serde_json::to_string(s).expect("a str serializes to JSON"); + lines[line_idx] = format!( + "{indent}{key}: [{spec}, \"\", {deps}, {integrity}]{comma}{cr}", + indent = entry.indent, + key = entry.key_raw, + spec = json(&format!("{name}@{version}")), + integrity = json(&integrity), + comma = if entry.trailing_comma { "," } else { "" }, + ); + result.handled.insert(uuid); + changed = true; + } + if changed { + view.write(rel, lines.join("\n")); + } + } + result +} + +// ── side settings ──────────────────────────────────────────────────────────── + +/// Whether any npm / pnpm lock in the view still resolves a hosted URL. +async fn still_hosted(view: &mut View<'_>, rels: &[&str], ctx: &Ctx<'_>) -> bool { + for rel in rels { + let Ok(Some(text)) = view.read(rel).await else { + continue; + }; + if !crate::vex::discover::hosted_uuids_in_text(&text, ctx.origins).is_empty() { + return true; + } + } + false +} + +/// Remove the `.npmrc` / pnpm-workspace.yaml a hosted run CREATED (still +/// byte-identical to the scaffold) once no lock entry needs it; warn about +/// the setting when the file carries other content too (the line may be +/// the user's own, so it is never removed from a file the user wrote). +pub(crate) async fn cleanup_side_config( + view: &mut View<'_>, + ctx: &Ctx<'_>, + result: &mut FormatResult, +) { + use super::super::npmrc::{NPMRC_ALLOW_REMOTE_LINE, NPMRC_CREATED, NPMRC_REL}; + + let restored_npm_lock = view.staged.keys().any(|k| { + matches!( + k.rsplit('/').next(), + Some("package-lock.json" | "npm-shrinkwrap.json") + ) + }); + if restored_npm_lock + && !still_hosted(view, &["package-lock.json", "npm-shrinkwrap.json"], ctx).await + { + if let Ok(Some(npmrc)) = view.read(NPMRC_REL).await { + if npmrc == NPMRC_CREATED { + view.remove(NPMRC_REL); + } else if npmrc + .lines() + .any(|l| l.trim() == NPMRC_ALLOW_REMOTE_LINE) + { + result.warnings.push(( + "npm_allow_remote_left", + format!( + "{NPMRC_REL} keeps `{NPMRC_ALLOW_REMOTE_LINE}`, which hosted mode may \ + have added; no lock entry needs it any more, so remove the line if \ + nothing else does" + ), + )); + } + } + } + + let restored_pnpm = view + .staged + .keys() + .any(|k| k == "pnpm-lock.yaml"); + if restored_pnpm && !still_hosted(view, &["pnpm-lock.yaml"], ctx).await { + const WORKSPACE: &str = "pnpm-workspace.yaml"; + if let Ok(Some(ws)) = view.read(WORKSPACE).await { + if ws == "packages:\n - '.'\ntrustLockfile: true\n" { + view.remove(WORKSPACE); + } else if ws.lines().any(|l| l.trim_end() == "trustLockfile: true") { + result.warnings.push(( + "pnpm_trust_lockfile_left", + format!( + "{WORKSPACE} keeps `trustLockfile: true`, which hosted mode may have \ + added; no lock entry needs it any more, so remove the line if nothing \ + else does" + ), + )); + } + } + } +} diff --git a/crates/socket-patch-core/src/patch/redirect/vlt_heal.rs b/crates/socket-patch-core/src/patch/redirect/vlt_heal.rs index 5bbeb1ac7..96645fd41 100644 --- a/crates/socket-patch-core/src/patch/redirect/vlt_heal.rs +++ b/crates/socket-patch-core/src/patch/redirect/vlt_heal.rs @@ -458,6 +458,29 @@ pub async fn invalidate(root: &Path, state: &InstallState, stale: &[String]) -> out } +/// The Socket-hosted vlt nodes of `lock` (read BEFORE a restore rewrites +/// it) that stand for one of `purls` — the ledger-free rollback heal's +/// targets. No patch record rides along (v5 keeps no hosted ledger), so the +/// heal judges each installed copy against the lock's own pins. +pub fn lock_targets(lock: &str, origins: &[String], purls: &[String]) -> Vec { + let wanted: Vec = purls.iter().map(|p| canonical_purl(p)).collect(); + socket_owned_instances(lock, origins) + .into_iter() + .filter_map(|instance| { + let purl = crate::utils::purl::npm_purl(&instance.name, &instance.version)?; + let canon = canonical_purl(&purl); + let at = wanted.iter().position(|w| *w == canon)?; + Some(LedgerTarget { + purl: purls[at].clone(), + dep_id: instance.dep_id, + name: instance.name, + record: None, + flags: instance.flags, + }) + }) + .collect() +} + /// The vlt nodes the ledger's `redirect_vlt_lock_node` edits name for each /// of `purls`, with the purl's patch record. With the pre-revert `lock`, /// a node vlt re-keyed (a new peer context) is named by the DepID its pin diff --git a/crates/socket-patch-core/src/vendor/berry_zip.rs b/crates/socket-patch-core/src/vendor/berry_zip.rs index 17cab5202..0cd59e626 100644 --- a/crates/socket-patch-core/src/vendor/berry_zip.rs +++ b/crates/socket-patch-core/src/vendor/berry_zip.rs @@ -64,7 +64,7 @@ const MODE_FILE_EXEC: u32 = 0o100755; /// — `./`, `..`, `//`, absolute — and duplicate paths) is an `Err` — a wrong /// checksum would brick the user's `yarn install` with a YN0018, so we never /// guess. -pub(super) fn berry_cache_checksum_10c0( +pub(crate) fn berry_cache_checksum_10c0( tgz_bytes: &[u8], package_ident: &str, ) -> Result { diff --git a/crates/socket-patch-core/src/vendor/mod.rs b/crates/socket-patch-core/src/vendor/mod.rs index 9dae4b77d..387a60c0b 100644 --- a/crates/socket-patch-core/src/vendor/mod.rs +++ b/crates/socket-patch-core/src/vendor/mod.rs @@ -48,7 +48,7 @@ pub mod path; pub mod state; -mod berry_zip; +pub(crate) mod berry_zip; mod bun_binary; pub mod bun_lock; pub(crate) mod bun_lock_text; diff --git a/crates/socket-patch-core/src/vendor/registry_fetch.rs b/crates/socket-patch-core/src/vendor/registry_fetch.rs index 533ad8107..3035d89bf 100644 --- a/crates/socket-patch-core/src/vendor/registry_fetch.rs +++ b/crates/socket-patch-core/src/vendor/registry_fetch.rs @@ -1071,7 +1071,7 @@ async fn fetch_gem( /// hash, and Pipfile.lock, which records every release file's hash). pub const DEFAULT_PYPI_JSON_API: &str = "https://pypi.org/pypi"; -fn pypi_json_api_base() -> String { +pub(crate) fn pypi_json_api_base() -> String { std::env::var("SOCKET_PYPI_JSON_API") .ok() .map(|v| v.trim_end_matches('/').to_string()) @@ -1211,7 +1211,7 @@ async fn fetch_pypi( /// crates.io static download host; override with `SOCKET_CRATES_REGISTRY`. pub const DEFAULT_CRATES_REGISTRY: &str = "https://static.crates.io/crates"; -fn crates_registry_base() -> String { +pub(crate) fn crates_registry_base() -> String { std::env::var("SOCKET_CRATES_REGISTRY") .ok() .map(|v| v.trim_end_matches('/').to_string()) @@ -1261,7 +1261,7 @@ pub const DEFAULT_GOPROXY: &str = "https://proxy.golang.org"; /// the module matches GONOPROXY (defaulting to GOPRIVATE). Falling back to a /// public proxy there would send a private module path off the machine. /// A non-empty `SOCKET_GOPROXY` is an explicit choice and always wins. -fn goproxy_base(module: &str) -> Result { +pub(crate) fn goproxy_base(module: &str) -> Result { if let Ok(v) = std::env::var("SOCKET_GOPROXY") { let v = v.trim_end_matches('/').to_string(); if !v.is_empty() { @@ -1304,7 +1304,7 @@ fn goproxy_base(module: &str) -> Result { /// glob match a leading path-element prefix of `target`? A glob with /// syntax this matcher does not implement (`[...]`, `\`) counts as a /// match, so an unrecognized private pattern never leaks a module path. -fn go_match_prefix_patterns(globs: &str, target: &str) -> bool { +pub(crate) fn go_match_prefix_patterns(globs: &str, target: &str) -> bool { globs .split(',') .map(str::trim) @@ -1339,7 +1339,7 @@ fn go_glob_match(pattern: &[u8], name: &[u8]) -> bool { /// /// Runs in the ecosystem-agnostic service-download path whenever the /// service reports a `dirhashH1`. -fn go_h1_of_zip(bytes: &[u8]) -> Result { +pub(crate) fn go_h1_of_zip(bytes: &[u8]) -> Result { Ok(walk_module_zip(bytes, None)?.h1) } @@ -1863,7 +1863,7 @@ pub async fn stage_local_dir_artifact( /// Capped download. http(s) only; the cap is enforced on the declared /// Content-Length AND the actual stream (a lying server cannot blow past /// it). -async fn download(client: &reqwest::Client, url: &str) -> Result, String> { +pub(crate) async fn download(client: &reqwest::Client, url: &str) -> Result, String> { if !(url.starts_with("https://") || url.starts_with("http://")) { return Err(format!("refusing non-http(s) artifact URL `{url}`")); } @@ -2012,7 +2012,7 @@ fn verify_integrity(bytes: &[u8], integrity: &LockIntegrity) -> Result<(), Fetch /// legacy package unvendorable whenever the prebuilt-artifact service misses. /// The bare-hex twin of this trust /// decision already lives in the `LockIntegrity::Sha1Hex` arm above. -fn verify_sri(bytes: &[u8], sri: &str) -> Result<(), String> { +pub(crate) fn verify_sri(bytes: &[u8], sri: &str) -> Result<(), String> { let mut best: Option<(u8, &str, &str)> = None; for token in sri.split_whitespace() { let Some((algo, b64)) = token.split_once('-') else { diff --git a/crates/socket-patch-core/src/vex/discover/mod.rs b/crates/socket-patch-core/src/vex/discover/mod.rs index 34b3c3efc..5f895fca7 100644 --- a/crates/socket-patch-core/src/vex/discover/mod.rs +++ b/crates/socket-patch-core/src/vex/discover/mod.rs @@ -916,6 +916,21 @@ pub(crate) fn socket_patch_name_uuid(name: &str, vendored: bool) -> Option BTreeSet<(String, WiringMode)> { + socket_identities_inner(text, origins) +} + +/// The hosted patch uuids `text` mentions, under the same recognition +/// rules as discovery's sweep (the upstream restore's "does anything in +/// this file still need the hosted-side setting" probe). +pub(crate) fn hosted_uuids_in_text(text: &str, origins: &[String]) -> BTreeSet { + socket_identities(text, origins) + .into_iter() + .filter(|(_, mode)| *mode == WiringMode::Hosted) + .map(|(uuid, _)| uuid) + .collect() +} + +fn socket_identities_inner(text: &str, origins: &[String]) -> BTreeSet<(String, WiringMode)> { let mut found = BTreeSet::new(); let norm = decode_escapes(text) .replace("\\/", "/") diff --git a/crates/socket-patch-core/tests/upstream_restore_golden.rs b/crates/socket-patch-core/tests/upstream_restore_golden.rs new file mode 100644 index 000000000..c9791785a --- /dev/null +++ b/crates/socket-patch-core/tests/upstream_restore_golden.rs @@ -0,0 +1,340 @@ +//! The v5 hosted unwind, pinned against the shared redirect goldens: each +//! case's `expected/` tree (what `scan --mode hosted` writes) restored to +//! the upstream registry entry must give back the `input/` bytes — with the +//! upstream values served by a mock registry that knows only what `input/` +//! pins, exactly as the public registry would answer. +//! +//! A case is exercised when its rewrite is invertible without a ledger (the +//! list below names the ones whose original spelling is not derivable, and +//! why); every other case must round-trip byte for byte. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; + +use serial_test::serial; +use socket_patch_core::patch::redirect::upstream::{ + restore_upstream, HostedPin, PinStatus, RestoreOptions, +}; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +fn fixtures() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/redirect") +} + +fn walk(dir: &Path) -> BTreeMap { + let mut out = BTreeMap::new(); + if !dir.is_dir() { + return out; + } + for entry in walkdir::WalkDir::new(dir).into_iter().filter_map(Result::ok) { + if entry.file_type().is_file() { + let rel = entry + .path() + .strip_prefix(dir) + .unwrap() + .to_string_lossy() + .replace('\\', "/"); + out.insert(rel, fs::read_to_string(entry.path()).unwrap()); + } + } + out +} + +/// Tokens of `pattern` that `input` holds and `expected` does not: the +/// upstream values the hosted rewrite replaced. +fn vanished(input: &BTreeMap, expected: &BTreeMap, re: &str) -> Vec { + let re = regex::Regex::new(re).unwrap(); + let all = |files: &BTreeMap| -> BTreeSet { + files + .values() + .flat_map(|t| re.captures_iter(t).map(|c| c[1].to_string()).collect::>()) + .collect() + }; + let after = all(expected); + let mut out: Vec = all(input).into_iter().filter(|t| !after.contains(t)).collect(); + out.sort(); + out +} + +struct Case { + dir: PathBuf, + input: BTreeMap, + expected: BTreeMap, + overrides: Vec, +} + +fn load(flavor: &str) -> Vec { + let root = fixtures().join(flavor); + let mut dirs: Vec = fs::read_dir(&root) + .unwrap() + .map(|e| e.unwrap().path()) + .filter(|p| p.join("input").is_dir() && p.join("expected").is_dir()) + .collect(); + dirs.sort(); + dirs.into_iter() + .map(|dir| { + let input = walk(&dir.join("input")); + // `expected/` holds only the files the rewrite changed. + let mut expected = input.clone(); + expected.extend(walk(&dir.join("expected"))); + let overrides = serde_json::from_str( + &fs::read_to_string(dir.join("overrides.json")).unwrap(), + ) + .unwrap(); + Case { + dir, + input, + expected, + overrides, + } + }) + // A rewrite happened, from a pristine tree (re-redirect cases start + // hosted: there is no upstream `input/` to come back to). + .filter(|c| { + c.input != c.expected && !c.input.values().any(|t| t.contains("patch.socket.dev")) + }) + .collect() +} + +async fn run_case(case: &Case) -> (BTreeMap, Vec<(String, PinStatus)>) { + let tmp = tempfile::tempdir().unwrap(); + for (rel, text) in &case.expected { + let p = tmp.path().join(rel); + fs::create_dir_all(p.parent().unwrap()).unwrap(); + fs::write(p, text).unwrap(); + } + let discovery = socket_patch_core::vex::discover_patched_refs(tmp.path()).await; + let pins = HostedPin::all(&discovery); + let outcome = restore_upstream(tmp.path(), &pins, &RestoreOptions::default()).await; + assert!(outcome.flush_error.is_none(), "{:?}", outcome.flush_error); + let statuses = outcome + .pins + .iter() + .map(|p| (p.purl.clone(), p.status.clone())) + .collect(); + (walk(tmp.path()), statuses) +} + +fn assert_round_trip(case: &Case, after: &BTreeMap, statuses: &[(String, PinStatus)]) { + assert!(!statuses.is_empty(), "{}: discovery found no hosted pin", case.dir.display()); + for (purl, status) in statuses { + assert_eq!(*status, PinStatus::Restored, "{}: {purl}", case.dir.display()); + } + for (rel, want) in &case.input { + assert_eq!( + after.get(rel), + Some(want), + "{}: {rel} did not round-trip", + case.dir.display() + ); + } + let extra: Vec<&String> = after.keys().filter(|k| !case.input.contains_key(*k)).collect(); + assert!(extra.is_empty(), "{}: left behind {extra:?}", case.dir.display()); +} + +/// Sets env vars for the guard's lifetime (tests using it are `#[serial]`). +struct EnvGuard(Vec<(String, Option)>); + +impl EnvGuard { + fn set(vars: &[(&str, String)]) -> Self { + let saved = vars + .iter() + .map(|(k, _)| (k.to_string(), std::env::var(k).ok())) + .collect(); + for (k, v) in vars { + std::env::set_var(k, v); + } + EnvGuard(saved) + } +} + +impl Drop for EnvGuard { + fn drop(&mut self) { + for (k, v) in self.0.drain(..) { + match v { + Some(v) => std::env::set_var(&k, v), + None => std::env::remove_var(&k), + } + } + } +} + +/// Serve an npm version document for every single-override npm case. +async fn npm_mock(case: &Case) -> MockServer { + let server = MockServer::start().await; + let integrity = vanished(&case.input, &case.expected, r"(sha512-[A-Za-z0-9+/=]+)"); + let shasum = vanished(&case.input, &case.expected, r"#([0-9a-f]{40})\b"); + let tarballs = vanished( + &case.input, + &case.expected, + r##"(https://registry\.[a-z]+\.(?:org|com)/[^"#\s]+\.tgz)"##, + ); + for o in &case.overrides { + let name = match o["namespace"].as_str() { + Some(ns) if !ns.is_empty() => format!("{ns}/{}", o["name"].as_str().unwrap()), + _ => o["name"].as_str().unwrap().to_string(), + }; + let version = o["version"].as_str().unwrap(); + let leaf = name.rsplit('/').next().unwrap(); + let tarball = tarballs + .iter() + .find(|t| t.ends_with(&format!("/{leaf}-{version}.tgz"))) + .cloned() + .unwrap_or_else(|| format!("https://registry.npmjs.org/{name}/-/{leaf}-{version}.tgz")); + let mut dist = serde_json::json!({ "tarball": tarball }); + if let [only] = integrity.as_slice() { + dist["integrity"] = only.clone().into(); + } + if let [only] = shasum.as_slice() { + dist["shasum"] = only.clone().into(); + } + Mock::given(method("GET")) + .and(path(format!("/{}/{version}", name.replace('/', "%2f")))) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(serde_json::json!({ "name": name, "version": version, "dist": dist })), + ) + .mount(&server) + .await; + } + server +} + +async fn npm_flavor(flavor: &str, not_invertible: &[&str]) { + let mut ran = 0; + for case in load(flavor) { + let name = case.dir.file_name().unwrap().to_string_lossy().into_owned(); + if not_invertible.contains(&name.as_str()) { + continue; + } + let server = npm_mock(&case).await; + let _env = EnvGuard::set(&[("SOCKET_NPM_REGISTRY", server.uri())]); + let (after, statuses) = run_case(&case).await; + assert_round_trip(&case, &after, &statuses); + ran += 1; + } + assert!(ran > 0, "{flavor}: no case ran"); +} + +#[tokio::test] +#[serial] +async fn package_lock_goldens_round_trip() { + npm_flavor("npm/package-lock-v3", &[]).await; +} + +#[tokio::test] +#[serial] +async fn yarn_classic_goldens_round_trip() { + npm_flavor("npm/yarn-classic", &[]).await; +} + +#[tokio::test] +#[serial] +async fn pnpm_goldens_round_trip() { + // nested-rush-lock: a nested (Rush) lock is outside root-only discovery. + npm_flavor("npm/pnpm", &["nested-rush-lock"]).await; +} + +#[tokio::test] +#[serial] +async fn bun_goldens_round_trip() { + // custom-registry: the entry's registry slot is not recorded; the + // restore writes the default registry (""). scoped-package: its artifact + // leaf names another package, so discovery (rightly) claims no pin. + npm_flavor("npm/bun", &["custom-registry", "scoped-package"]).await; +} + +/// `(name, version, checksum)` of every checksummed entry of a Cargo.lock. +fn cargo_checksums(lock: &str) -> Vec<(String, String, String)> { + let mut out = Vec::new(); + for block in lock.split("[[package]]").skip(1) { + let field = |k: &str| { + block.lines().find_map(|l| { + l.strip_prefix(&format!("{k} = \"")) + .and_then(|r| r.strip_suffix('"')) + .map(str::to_string) + }) + }; + if let (Some(n), Some(v), Some(c)) = (field("name"), field("version"), field("checksum")) { + out.push((n, v, c)); + } + } + out +} + +#[tokio::test] +#[serial] +async fn cargo_goldens_round_trip() { + let mut ran = 0; + for case in load("cargo/cargo") { + let server = MockServer::start().await; + let Some(lock) = case.input.get("Cargo.lock") else { + continue; + }; + let mut by_crate: BTreeMap> = BTreeMap::new(); + for (n, v, c) in cargo_checksums(lock) { + by_crate + .entry(n.clone()) + .or_default() + .push(serde_json::json!({ "name": n, "vers": v, "cksum": c }).to_string()); + } + for (name, rows) in by_crate { + let lower = name.to_ascii_lowercase(); + let index_path = match lower.len() { + 1 => format!("/1/{lower}"), + 2 => format!("/2/{lower}"), + 3 => format!("/3/{}/{lower}", &lower[..1]), + _ => format!("/{}/{}/{lower}", &lower[..2], &lower[2..4]), + }; + Mock::given(method("GET")) + .and(path(index_path)) + .respond_with(ResponseTemplate::new(200).set_body_string(rows.join("\n"))) + .mount(&server) + .await; + } + let _env = EnvGuard::set(&[("SOCKET_CRATES_INDEX", server.uri())]); + let (after, statuses) = run_case(&case).await; + assert_round_trip(&case, &after, &statuses); + ran += 1; + } + assert!(ran > 0); +} + +#[tokio::test] +#[serial] +async fn golang_goldens_round_trip() { + let mut ran = 0; + for case in load("golang/gomod") { + let server = MockServer::start().await; + let sum = case.input.get("go.sum").cloned().unwrap_or_default(); + let mut by_module: BTreeMap> = BTreeMap::new(); + for line in sum.lines() { + let mut parts = line.split_whitespace(); + let (Some(m), Some(v)) = (parts.next(), parts.next()) else { + continue; + }; + let v = v.trim_end_matches("/go.mod"); + by_module + .entry(format!("{m}@{v}")) + .or_default() + .push(line.to_string()); + } + for (id, lines) in by_module { + Mock::given(method("GET")) + .and(path(format!("/lookup/{id}"))) + .respond_with(ResponseTemplate::new(200).set_body_string(format!( + "12345\n{}\n\ngo.sum database tree\n", + lines.join("\n") + ))) + .mount(&server) + .await; + } + let _env = EnvGuard::set(&[("SOCKET_GOSUMDB_URL", server.uri())]); + let (after, statuses) = run_case(&case).await; + assert_round_trip(&case, &after, &statuses); + ran += 1; + } + assert!(ran > 0); +} From 03643e09026b482166e1b0e9467436f7b2ed1a5c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 22:38:45 +0000 Subject: [PATCH 03/66] Update rollback prompt unit test for the upstream-restore wording Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018tncU7u7FK48tubFNA12UQ --- .../socket-patch-cli/src/commands/rollback.rs | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/rollback.rs b/crates/socket-patch-cli/src/commands/rollback.rs index 287d873cf..d69acca11 100644 --- a/crates/socket-patch-cli/src/commands/rollback.rs +++ b/crates/socket-patch-cli/src/commands/rollback.rs @@ -4545,30 +4545,29 @@ mod tests { #[test] fn rollback_prompt_singular_plural_and_clauses() { assert_eq!( - rollback_prompt(1, 0, 0, 0), + rollback_prompt(1, 0, 0), "Roll back 1 patch and remove it from the local manifest?" ); assert_eq!( - rollback_prompt(2, 0, 0, 0), + rollback_prompt(2, 0, 0), "Roll back 2 patches and remove them from the local manifest?" ); - // Never "..., and unwind" after an inner "and". + // Never "..., and restore" after an inner "and". assert_eq!( - rollback_prompt(1, 0, 1, 0), - "Roll back 1 patch, remove it from the local manifest, and unwind 1 hosted \ - redirect?" + rollback_prompt(1, 0, 1), + "Roll back 1 patch, remove it from the local manifest, and restore 1 hosted \ + package to the upstream registry?" ); - assert_eq!(rollback_prompt(0, 0, 3, 0), "Unwind 3 hosted redirects?"); assert_eq!( - rollback_prompt(0, 0, 0, 1), - "Replay 1 leftover hosted redirect edit?" + rollback_prompt(0, 0, 3), + "Restore 3 hosted packages to the upstream registry?" ); assert_eq!( - rollback_prompt(0, 1, 0, 0), + rollback_prompt(0, 1, 0), "Delete 1 vendored artifact and its ledger record?" ); assert_eq!( - rollback_prompt(0, 2, 0, 0), + rollback_prompt(0, 2, 0), "Delete 2 vendored artifacts and their ledger records?" ); } From 61d719160cc6be9cfa167f3d10e82dda8e7348e3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 22:50:30 +0000 Subject: [PATCH 04/66] Derive list, scan updates and takeover state from lockfile hosted pins v5 keeps no hosted ledger, so every reader that consulted .socket/vendor/redirect-state.json now reads the hosted pins lockfile discovery finds: - list shows each hosted pin with the lockfiles wiring it (details.lockfiles); a pre-v5 ledger only supplies the details of pins it still describes. - scan's updates[] fold, redirectState block and the agent-flow hosted_wiring_retained probe read the pins. - The hosted-over-vendored takeover classifier reads the pins; the vendored-over-hosted ledger reconcile (vendor_supersedes_redirect) is gone: once the lock routes a package to .socket/vendor/ no hosted state is left to go stale. - vex treats a malformed pre-v5 redirect ledger as an advisory. scan/mod.rs unit tests still need rewriting (WIP). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018tncU7u7FK48tubFNA12UQ --- crates/socket-patch-cli/src/commands/list.rs | 239 +++++--- crates/socket-patch-cli/src/commands/mod.rs | 61 ++- .../src/commands/scan/discovery.rs | 152 +++--- .../src/commands/scan/hosted.rs | 28 +- .../socket-patch-cli/src/commands/scan/mod.rs | 511 ++++-------------- .../src/commands/scan/vendor_flow.rs | 8 +- .../socket-patch-cli/src/commands/vendor.rs | 8 +- crates/socket-patch-cli/src/commands/vex.rs | 32 +- .../tests/in_process_rollback_hosted.rs | 48 +- 9 files changed, 469 insertions(+), 618 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/list.rs b/crates/socket-patch-cli/src/commands/list.rs index 6bbc5e76b..19fc7f845 100644 --- a/crates/socket-patch-cli/src/commands/list.rs +++ b/crates/socket-patch-cli/src/commands/list.rs @@ -3,7 +3,8 @@ use std::path::Path; use clap::Args; use socket_patch_core::manifest::operations::read_manifest; use socket_patch_core::manifest::schema::{PatchManifest, PatchRecord}; -use socket_patch_core::patch::redirect::{RedirectState, REDIRECT_STATE_REL}; +use socket_patch_core::patch::redirect::upstream::HostedPin; +use socket_patch_core::patch::redirect::RedirectState; use socket_patch_core::telemetry::track_patch_listed; use socket_patch_core::vendor::state::{VendorEntry, VENDOR_STATE_REL}; @@ -26,9 +27,9 @@ pub struct ListArgs { enum Source { /// A `.socket/manifest.json` entry (agent mode). Manifest, - /// A hosted redirect-ledger record: `scan --mode hosted` records its - /// patches ONLY in `.socket/vendor/redirect-state.json` and never - /// writes the manifest. + /// A hosted pin the lockfiles wire: `scan --mode hosted` keeps no + /// ledger and never writes the manifest, so the lockfiles are the only + /// record of a hosted patch. Hosted, /// A vendor-ledger record: vendored mode is manifest-free, so every /// `scan`/`get --mode vendored` patch lives ONLY in @@ -39,14 +40,14 @@ enum Source { Vendored, } -/// The `(mode, ledger)` label pair for a ledger-sourced record — the shared -/// constant labels, never a ledger's own opaque `mode` string (see -/// `HOSTED_MODE_LABEL`'s docs) — or `None` for a manifest entry. Shared by -/// the JSON `details` and the human `Mode:` line. +/// The `(mode, ledger)` label pair for a vendor-ledger record — the shared +/// constant label, never a ledger's own opaque `mode` string (see +/// `HOSTED_MODE_LABEL`'s docs) — or `None` for a manifest entry or a hosted +/// pin (which has no ledger; see [`ListEntry::lockfiles`]). Shared by the +/// JSON `details` and the human `Mode:` line. fn ledger_label(source: Source) -> Option<(&'static str, &'static str)> { match source { - Source::Manifest => None, - Source::Hosted => Some((crate::commands::HOSTED_MODE_LABEL, REDIRECT_STATE_REL)), + Source::Manifest | Source::Hosted => None, Source::Vendored => Some((crate::commands::VENDORED_MODE_LABEL, VENDOR_STATE_REL)), } } @@ -56,12 +57,62 @@ struct ListEntry<'a> { purl: &'a str, record: &'a PatchRecord, source: Source, + /// The lockfiles wiring a hosted pin (empty for the other sources). + lockfiles: &'a [String], +} + +/// A hosted pin as `list` shows it: the lockfiles wiring it, and its +/// record — from a pre-v5 redirect ledger when one still describes this +/// exact pin (read for migration only), else just the uuid (the details +/// live on the API; `vex` fetches them). +pub(crate) struct HostedListing { + pub purl: String, + pub record: PatchRecord, + pub lockfiles: Vec, +} + +impl HostedListing { + /// One listing per hosted pin in `pins`, detailed from `legacy` where + /// it records the same purl and uuid. + pub(crate) fn from_pins(pins: &[HostedPin], legacy: Option<&RedirectState>) -> Vec { + let canon = |p: &str| { + socket_patch_core::utils::purl::normalize_purl( + socket_patch_core::utils::purl::strip_purl_qualifiers(p), + ) + .into_owned() + }; + pins.iter() + .map(|pin| { + let record = legacy + .and_then(|l| { + l.records + .iter() + .find(|(k, r)| canon(k) == canon(&pin.purl) && r.uuid == pin.uuid) + .map(|(_, r)| r.clone()) + }) + .unwrap_or_else(|| PatchRecord { + uuid: pin.uuid.clone(), + exported_at: String::new(), + files: Default::default(), + vulnerabilities: Default::default(), + description: String::new(), + license: String::new(), + tier: String::new(), + }); + HostedListing { + purl: pin.purl.clone(), + record, + lockfiles: pin.files.clone(), + } + }) + .collect() + } } /// Every listable record from all three stores, in a stable order: by /// PURL, then manifest < hosted < vendored when one purl appears in more /// than one. The record maps (`HashMap` manifest and vendor ledger / -/// `BTreeMap` redirect ledger) never impose an order shared consumers could +/// hosted pins) never impose an order shared consumers could /// diff, so the sort here is the contract. Only vendor entries whose /// embedded record stands on its own fold in /// ([`crate::commands::vendor_record_is_unowned`], the rule `vex` attests @@ -72,7 +123,7 @@ struct ListEntry<'a> { /// purl. A legacy entry with no embedded record never folds in. fn combined_entries<'a>( manifest: Option<&'a PatchManifest>, - redirect: Option<&'a RedirectState>, + hosted: &'a [HostedListing], vendor: Option<&'a std::collections::HashMap>, ) -> Vec> { let mut entries: Vec> = Vec::new(); @@ -81,15 +132,15 @@ fn combined_entries<'a>( purl, record, source: Source::Manifest, + lockfiles: &[], })); } - if let Some(redirect) = redirect { - entries.extend(redirect.records.iter().map(|(purl, record)| ListEntry { - purl, - record, - source: Source::Hosted, - })); - } + entries.extend(hosted.iter().map(|h| ListEntry { + purl: &h.purl, + record: &h.record, + source: Source::Hosted, + lockfiles: &h.lockfiles, + })); if let Some(vendor) = vendor { entries.extend(vendor.iter().filter_map(|(purl, entry)| { let record = entry @@ -100,6 +151,7 @@ fn combined_entries<'a>( purl, record, source: Source::Vendored, + lockfiles: &[], }) })); } @@ -162,6 +214,10 @@ fn build_list_envelope(entries: &[ListEntry<'_>]) -> Envelope { details["mode"] = serde_json::json!(mode); details["ledger"] = serde_json::json!(ledger); } + if entry.source == Source::Hosted { + details["mode"] = serde_json::json!(crate::commands::HOSTED_MODE_LABEL); + details["lockfiles"] = serde_json::json!(entry.lockfiles); + } env.record( PatchEvent::new(PatchAction::Discovered, entry.purl.to_string()) @@ -248,6 +304,13 @@ fn format_entry(entry: &ListEntry<'_>, color: bool) -> String { // Same labeling rule as the JSON details. lines.push(format!(" Mode: {mode} (recorded in {ledger})")); } + if entry.source == Source::Hosted { + lines.push(format!( + " Mode: {} (wired in {})", + crate::commands::HOSTED_MODE_LABEL, + sanitize(&entry.lockfiles.join(", ")) + )); + } lines.extend(field(" ", "Tier", &patch.tier)); lines.extend(field(" ", "License", &patch.license)); lines.extend(field(" ", "Exported", &patch.exported_at)); @@ -334,22 +397,20 @@ pub async fn run(args: ListArgs) -> i32 { } }; - // Hosted-mode patches live ONLY in the redirect ledger and vendored-mode - // patches ONLY in the vendor ledger, so `list` consults both alongside - // the manifest — leniently (a malformed ledger degrades to "nothing to - // consult", surfaced on stderr unless --silent; the write paths - // hard-error on it instead), and always from the SAME project as the - // manifest (`project_root` steps out of the manifest's `.socket/`): - // with `--manifest-path` pointing at another project, reading the LOCAL - // cwd's ledgers would interleave two projects' patch state (and a local - // ledger could suppress the flagged project's manifest_not_found). + // Hosted-mode patches live ONLY in the lockfiles (v5 keeps no hosted + // ledger) and vendored-mode patches ONLY in the vendor ledger, so + // `list` consults both alongside the manifest — always from the SAME + // project as the manifest (`project_root` steps out of the manifest's + // `.socket/`): with `--manifest-path` pointing at another project, + // reading the LOCAL cwd's state would interleave two projects' patches. // - // Under --json a corrupt redirect ledger rides the envelope's - // `warnings[]` (stdout is the machine channel; a stderr-only warning - // would vanish for JSON consumers), the same split `update` uses. + // A pre-v5 redirect ledger is read (never written) only to detail the + // hosted pins it still describes; a malformed one degrades to "nothing + // to consult", surfaced on stderr unless --silent, or in the envelope's + // `warnings[]` under --json. let project_root = args.common.project_root(); let mut warnings: Vec = Vec::new(); - let redirect_state = + let legacy_redirect = match socket_patch_core::patch::redirect::load_redirect_state(&project_root).await { Ok(state) => state, Err(corrupt) => { @@ -364,16 +425,19 @@ pub async fn run(args: ListArgs) -> i32 { None } }; + let hosted = HostedListing::from_pins( + &HostedPin::all(&crate::commands::discover_wiring(&args.common, &project_root).await), + legacy_redirect.as_ref(), + ); let vendor_state = crate::commands::load_vendor_state_lenient(&project_root, args.common.silent).await; - // `combined_entries` folds only ledger RECORDS in (an edits-only - // redirect ledger — post-takeover residue / a degraded record-fetch- - // failed run — and a record-less legacy vendor entry assert no - // patches), so entry emptiness is the whole exit predicate. + // `combined_entries` folds only real records in (a record-less legacy + // vendor entry asserts no patch), so entry emptiness is the whole exit + // predicate. let entries = combined_entries( manifest.as_ref(), - redirect_state.as_ref(), + &hosted, vendor_state.as_ref().map(|s| &s.entries), ); if manifest.is_none() && entries.is_empty() { @@ -422,11 +486,10 @@ mod tests { use socket_patch_core::manifest::schema::{PatchFileInfo, PatchRecord, VulnerabilityInfo}; use std::collections::HashMap; - /// Envelope for a manifest-only listing (no redirect ledger) — the shape - /// most tests below need; the hosted tests call `combined_entries` - /// directly with a `RedirectState`. + /// Envelope for a manifest-only listing (no hosted pins, no vendor + /// ledger) — the shape most tests below need. fn manifest_envelope(manifest: &PatchManifest) -> Envelope { - build_list_envelope(&combined_entries(Some(manifest), None, None)) + build_list_envelope(&combined_entries(Some(manifest), &[], None)) } fn sample_manifest() -> PatchManifest { @@ -632,25 +695,30 @@ mod tests { assert_eq!(paths, vec!["z/a.js", "z/b.js"]); } - /// Hosted redirect-ledger records fold into the envelope labeled apart - /// from manifest entries: `details.mode` / `details.ledger` ride the - /// hosted events ONLY (additive keys), and the global purl sort holds - /// with the manifest entry first when one purl appears in both stores. + fn hosted(purl: &str, record: PatchRecord) -> HostedListing { + HostedListing { + purl: purl.to_string(), + record, + lockfiles: vec!["package-lock.json".to_string()], + } + } + + /// Hosted pins fold into the envelope labeled apart from manifest + /// entries: `details.mode` / `details.lockfiles` ride the hosted events + /// ONLY (additive keys), and the global purl sort holds with the + /// manifest entry first when one purl appears in both stores. #[test] - fn hosted_ledger_records_are_labeled_and_interleaved() { + fn hosted_pins_are_labeled_and_interleaved() { let manifest = sample_manifest(); - let mut redirect = RedirectState::new(); let mut hosted_record = manifest.patches["pkg:npm/minimist@1.2.2"].clone(); hosted_record.uuid = "22222222-2222-4222-8222-222222222222".to_string(); // Same purl as the manifest entry (coexistence) + a distinct one. - redirect - .records - .insert("pkg:npm/minimist@1.2.2".to_string(), hosted_record.clone()); - redirect - .records - .insert("pkg:npm/aaa-hosted@1.0.0".to_string(), hosted_record); + let pins = vec![ + hosted("pkg:npm/minimist@1.2.2", hosted_record.clone()), + hosted("pkg:npm/aaa-hosted@1.0.0", hosted_record), + ]; - let env = build_list_envelope(&combined_entries(Some(&manifest), Some(&redirect), None)); + let env = build_list_envelope(&combined_entries(Some(&manifest), &pins, None)); let v: serde_json::Value = serde_json::from_str(&env.to_pretty_json()).unwrap(); assert_eq!(v["summary"]["discovered"], 3); let events = v["events"].as_array().unwrap(); @@ -678,22 +746,53 @@ mod tests { "manifest entries must NOT carry the hosted labels: {v}" ); assert_eq!( - events[0]["details"]["ledger"], - ".socket/vendor/redirect-state.json" + events[0]["details"]["lockfiles"], + serde_json::json!(["package-lock.json"]) + ); + assert!( + events[0]["details"].get("ledger").is_none(), + "a hosted pin names no ledger: {v}" ); } + /// A pre-v5 redirect ledger details only the pins it records with the + /// same uuid; any other pin lists with its uuid alone. + #[test] + fn legacy_ledger_details_only_matching_pins() { + let manifest = sample_manifest(); + let record = manifest.patches["pkg:npm/minimist@1.2.2"].clone(); + let mut legacy = RedirectState::new(); + legacy + .records + .insert("pkg:npm/minimist@1.2.2".to_string(), record.clone()); + let pin = |purl: &str, uuid: &str| HostedPin { + purl: purl.to_string(), + uuid: uuid.to_string(), + files: vec!["yarn.lock".to_string()], + }; + let listings = HostedListing::from_pins( + &[ + pin("pkg:npm/minimist@1.2.2", &record.uuid), + pin("pkg:npm/other@1.0.0", "33333333-3333-4333-8333-333333333333"), + ], + Some(&legacy), + ); + assert_eq!(listings[0].record, record); + assert_eq!(listings[1].record.uuid, "33333333-3333-4333-8333-333333333333"); + assert!(listings[1].record.vulnerabilities.is_empty()); + assert_eq!(listings[1].lockfiles, vec!["yarn.lock".to_string()]); + } + /// A hosted-only listing (no manifest at all) — the shape a purely /// hosted-wired project produces. #[test] fn hosted_only_entries_build_a_success_envelope() { let manifest = sample_manifest(); - let mut redirect = RedirectState::new(); - redirect.records.insert( - "pkg:npm/minimist@1.2.2".to_string(), + let pins = vec![hosted( + "pkg:npm/minimist@1.2.2", manifest.patches["pkg:npm/minimist@1.2.2"].clone(), - ); - let env = build_list_envelope(&combined_entries(None, Some(&redirect), None)); + )]; + let env = build_list_envelope(&combined_entries(None, &pins, None)); let v: serde_json::Value = serde_json::from_str(&env.to_pretty_json()).unwrap(); assert_eq!(v["status"], "success"); assert_eq!(v["summary"]["discovered"], 1); @@ -728,10 +827,7 @@ mod tests { fn vendored_ledger_records_are_labeled_and_sorted_last() { let manifest = sample_manifest(); let record = manifest.patches["pkg:npm/minimist@1.2.2"].clone(); - let mut redirect = RedirectState::new(); - redirect - .records - .insert("pkg:npm/minimist@1.2.2".to_string(), record.clone()); + let pins = vec![hosted("pkg:npm/minimist@1.2.2", record.clone())]; let mut detached = record.clone(); detached.uuid = "44444444-4444-4444-8444-444444444444".to_string(); let mut vendor = HashMap::new(); @@ -751,7 +847,7 @@ mod tests { let env = build_list_envelope(&combined_entries( Some(&manifest), - Some(&redirect), + &pins, Some(&vendor), )); let v: serde_json::Value = serde_json::from_str(&env.to_pretty_json()).unwrap(); @@ -786,7 +882,7 @@ mod tests { "the ledger's embedded record is the one listed: {v}" ); - let only = build_list_envelope(&combined_entries(None, None, Some(&vendor))); + let only = build_list_envelope(&combined_entries(None, &[], Some(&vendor))); let v: serde_json::Value = serde_json::from_str(&only.to_pretty_json()).unwrap(); assert_eq!(v["status"], "success", "{v}"); assert_eq!(v["summary"]["discovered"], 2, "{v}"); @@ -850,7 +946,7 @@ mod tests { }; assert_eq!( - listed(&combined_entries(Some(&manifest), None, Some(&vendor))), + listed(&combined_entries(Some(&manifest), &[], Some(&vendor))), vec![ ( "pkg:npm/left-pad@1.3.0".to_string(), @@ -867,7 +963,7 @@ mod tests { ); // No manifest at all: every fallback copy stands on its own. - let only = listed(&combined_entries(None, None, Some(&vendor))); + let only = listed(&combined_entries(None, &[], Some(&vendor))); assert_eq!(only.len(), 3, "{only:?}"); assert!( only.iter().all(|(_, mode, _)| mode == "vendored"), @@ -899,6 +995,7 @@ mod tests { purl, record, source: Source::Manifest, + lockfiles: &[], } } @@ -975,13 +1072,13 @@ mod tests { fn format_listing_counts_and_separates_entries() { assert_eq!(format_listing(&[], false), "No patches found in manifest."); let manifest = sample_manifest(); - let one = combined_entries(Some(&manifest), None, None); + let one = combined_entries(Some(&manifest), &[], None); let out = format_listing(&one, false); assert!(out.starts_with("Found 1 patch:\n\nPackage: "), "{out}"); assert!(!out.ends_with('\n'), "no trailing blank line: {out:?}"); let multi = multi_entry_manifest(); - let many = combined_entries(Some(&multi), None, None); + let many = combined_entries(Some(&multi), &[], None); let out = format_listing(&many, false); assert!( out.starts_with(&format!("Found {} patches:\n\n", many.len())), diff --git a/crates/socket-patch-cli/src/commands/mod.rs b/crates/socket-patch-cli/src/commands/mod.rs index ddda23e26..1bf1769c9 100644 --- a/crates/socket-patch-cli/src/commands/mod.rs +++ b/crates/socket-patch-cli/src/commands/mod.rs @@ -62,35 +62,48 @@ pub(crate) async fn discover_wiring( socket_patch_core::vex::discover_patched_refs_with(root, &opts).await } -/// Read-only lenient load of the hosted redirect ledger: missing → `None` -/// (a fresh start); malformed → `None` with the corruption surfaced on -/// stderr unless `silent`. This is the "read-only consumers may degrade a -/// malformed ledger to nothing-to-consult, but must surface it" posture -/// from `load_redirect_state`'s contract — the warning is advisory -/// (muted by `--silent`, "errors only"), because every path that would -/// WRITE or ATTEST from the ledger hard-errors on the same corruption -/// instead. Used by scan's empty-discovery `redirectState` consult; the -/// main-path consult inlines the same posture so it can flush telemetry -/// before the warning. -pub(crate) async fn load_redirect_state_lenient( - cwd: &Path, - silent: bool, -) -> Option { - match socket_patch_core::patch::redirect::load_redirect_state(cwd).await { - Ok(state) => state, - Err(corrupt) => { - if !silent { - eprintln!("Warning: {corrupt}"); - } - None - } +/// The project's hosted state, v5-style: v5 hosted mode keeps no ledger, +/// so the hosted pins [`discover_wiring`] finds in the lockfiles are the +/// whole record. Shaped as a [`RedirectState`] for the readers that classify +/// hosted against vendored state (one uuid-only record per pinned purl, no +/// edits) — it is never persisted. +/// +/// [`RedirectState`]: socket_patch_core::patch::redirect::RedirectState +pub(crate) async fn hosted_state_from_lockfiles( + common: &crate::args::GlobalArgs, + root: &Path, +) -> socket_patch_core::patch::redirect::RedirectState { + hosted_state_from_pins(&socket_patch_core::patch::redirect::upstream::HostedPin::all( + &discover_wiring(common, root).await, + )) +} + +/// [`hosted_state_from_lockfiles`] over already-discovered pins. A purl +/// pinned to several uuids (different lockfiles) keeps the first. +pub(crate) fn hosted_state_from_pins( + pins: &[socket_patch_core::patch::redirect::upstream::HostedPin], +) -> socket_patch_core::patch::redirect::RedirectState { + let mut state = socket_patch_core::patch::redirect::RedirectState::new(); + for pin in pins { + state + .records + .entry(pin.purl.clone()) + .or_insert_with(|| socket_patch_core::manifest::schema::PatchRecord { + uuid: pin.uuid.clone(), + exported_at: String::new(), + files: Default::default(), + vulnerabilities: Default::default(), + description: String::new(), + license: String::new(), + tier: String::new(), + }); } + state } /// Read-only lenient load of the vendor ledger (`.socket/vendor/state.json`): /// missing → an empty ledger; malformed/unreadable → `None` with the -/// problem surfaced on stderr unless `silent`. The vendor twin of -/// [`load_redirect_state_lenient`], with the same posture: a read-only +/// problem surfaced on stderr unless `silent`. A read-only /// consumer (`list`) degrades a broken ledger to nothing-to-consult but /// must say so, while every path that writes or attests from it fails /// closed instead. diff --git a/crates/socket-patch-cli/src/commands/scan/discovery.rs b/crates/socket-patch-cli/src/commands/scan/discovery.rs index d9d51ea74..642547aad 100644 --- a/crates/socket-patch-cli/src/commands/scan/discovery.rs +++ b/crates/socket-patch-cli/src/commands/scan/discovery.rs @@ -398,62 +398,50 @@ pub(super) async fn preverify_vendor_baselines( (mismatched, views) } -/// Fold both ledgers' patch records into the manifest view update detection -/// consults. Hosted mode records purl→uuid ONLY in -/// `.socket/vendor/redirect-state.json` and vendored mode ONLY in -/// `.socket/vendor/state.json`, so without this fold a pure hosted or -/// vendored project's `updates[]` would always be empty. Precedence on a -/// collision: manifest > redirect ledger > vendor ledger (matching VEX's -/// candidate merge in `commands::vex_sources`), then the lockfile's hosted -/// pins (`hosted_pins`, uuid only). Vendor entries are keyed by their -/// manifest-form ledger key (`detect_updates` bridges the spellings); a -/// legacy entry without an embedded record contributes its uuid alone. -/// Borrows the manifest untouched when nothing else contributes. +/// Fold the hosted pins and the vendor ledger's patch records into the +/// manifest view update detection consults. Hosted mode records purl→uuid +/// ONLY in the lockfiles (`hosted_pins`, uuid only; v5 keeps no hosted +/// ledger) and vendored mode ONLY in `.socket/vendor/state.json`, so without +/// this fold a pure hosted or vendored project's `updates[]` would always +/// be empty. Precedence on a collision: manifest > hosted pins > vendor +/// ledger (the live lock over a possibly superseded vendored entry). Vendor +/// entries are keyed by their manifest-form ledger key (`detect_updates` +/// bridges the spellings); a legacy entry without an embedded record +/// contributes its uuid alone. Borrows the manifest untouched when nothing +/// else contributes. pub(super) fn merge_ledger_records_for_updates<'a>( manifest: Option<&'a PatchManifest>, - redirect: Option<&socket_patch_core::patch::redirect::RedirectState>, vendor: Option<&VendorState>, hosted_pins: &[(String, String)], ) -> Option> { - let redirect_records = redirect.map(|s| &s.records).filter(|r| !r.is_empty()); let vendor_entries = vendor.map(|s| &s.entries).filter(|e| !e.is_empty()); - if redirect_records.is_none() && vendor_entries.is_none() && hosted_pins.is_empty() { + if vendor_entries.is_none() && hosted_pins.is_empty() { return manifest.map(Cow::Borrowed); } + let uuid_only = |uuid: &str| PatchRecord { + uuid: uuid.to_string(), + exported_at: String::new(), + files: HashMap::new(), + vulnerabilities: HashMap::new(), + description: String::new(), + license: String::new(), + tier: String::new(), + }; let mut merged = manifest.cloned().unwrap_or_default(); - for (purl, record) in redirect_records.into_iter().flatten() { + for (purl, uuid) in hosted_pins { merged .patches .entry(purl.clone()) - .or_insert_with(|| record.clone()); + .or_insert_with(|| uuid_only(uuid)); } for (purl, entry) in vendor_entries.into_iter().flatten() { merged.patches.entry(purl.clone()).or_insert_with(|| { - entry.record.clone().unwrap_or_else(|| PatchRecord { - uuid: entry.uuid.clone(), - exported_at: String::new(), - files: HashMap::new(), - vulnerabilities: HashMap::new(), - description: String::new(), - license: String::new(), - tier: String::new(), - }) + entry + .record + .clone() + .unwrap_or_else(|| uuid_only(&entry.uuid)) }); } - for (purl, uuid) in hosted_pins { - merged - .patches - .entry(purl.clone()) - .or_insert_with(|| PatchRecord { - uuid: uuid.clone(), - exported_at: String::new(), - files: HashMap::new(), - vulnerabilities: HashMap::new(), - description: String::new(), - license: String::new(), - tier: String::new(), - }); - } Some(Cow::Owned(merged)) } @@ -905,16 +893,16 @@ mod tests { } // ---- merge_ledger_records_for_updates ----------------------------------- - // Hosted mode records patches ONLY in the redirect ledger and vendored - // mode ONLY in the vendor ledger — these pin that ledger-only projects - // still surface `updates[]` (the documented CI signal) through the - // merged manifest view. + // Hosted mode records patches ONLY in the lockfiles (the hosted pins) and + // vendored mode ONLY in the vendor ledger — these pin that manifest-less + // projects still surface `updates[]` (the documented CI signal) through + // the merged manifest view. - fn ledger_with(entries: &[(&str, &str)]) -> socket_patch_core::patch::redirect::RedirectState { - let mut state = socket_patch_core::patch::redirect::RedirectState::new(); - let manifest = crate::commands::scan::tests::manifest_with(entries); - state.records.extend(manifest.patches); - state + fn pins(entries: &[(&str, &str)]) -> Vec<(String, String)> { + entries + .iter() + .map(|(purl, uuid)| (purl.to_string(), uuid.to_string())) + .collect() } /// A vendor ledger with one entry per `(key, uuid, detached)`: detached @@ -946,12 +934,12 @@ mod tests { } #[test] - fn ledger_only_project_reports_superseding_patch_in_updates() { - // Pure hosted project: NO .socket/manifest.json, one redirected patch - // recorded in the ledger; discovery now offers a different (newer) - // uuid. The merged view must make detect_updates flag it. - let ledger = ledger_with(&[("pkg:npm/foo@1.0", "uuid-old")]); - let merged = merge_ledger_records_for_updates(None, Some(&ledger), None, &[]); + fn hosted_only_project_reports_superseding_patch_in_updates() { + // Pure hosted project: NO .socket/manifest.json, one hosted pin in + // the lockfile; discovery now offers a different (newer) uuid. The + // merged view must make detect_updates flag it. + let hosted = pins(&[("pkg:npm/foo@1.0", "uuid-old")]); + let merged = merge_ledger_records_for_updates(None, None, &hosted); let pkgs = vec![batch_with("pkg:npm/foo@1.0", &["uuid-new"])]; let updates = detect_updates(merged.as_deref(), &pkgs); assert_eq!(updates.len(), 1); @@ -967,7 +955,7 @@ mod tests { // record still contributes its uuid — all detection reads. for detached in [true, false] { let vendor = vendor_ledger_with(&[("pkg:npm/foo@1.0", "uuid-old", detached)]); - let merged = merge_ledger_records_for_updates(None, None, Some(&vendor), &[]); + let merged = merge_ledger_records_for_updates(None, Some(&vendor), &[]); let pkgs = vec![batch_with("pkg:npm/foo@1.0", &["uuid-new"])]; let updates = detect_updates(merged.as_deref(), &pkgs); assert_eq!(updates.len(), 1, "detached={detached}"); @@ -976,16 +964,16 @@ mod tests { } // Still the top offer — no nag. let vendor = vendor_ledger_with(&[("pkg:npm/foo@1.0", "uuid-a", true)]); - let merged = merge_ledger_records_for_updates(None, None, Some(&vendor), &[]); + let merged = merge_ledger_records_for_updates(None, Some(&vendor), &[]); let pkgs = vec![batch_with("pkg:npm/foo@1.0", &["uuid-a"])]; assert!(detect_updates(merged.as_deref(), &pkgs).is_empty()); } #[test] - fn ledger_record_matching_the_candidate_is_not_an_update() { - // The redirected patch is still the top offer — no nag. - let ledger = ledger_with(&[("pkg:npm/foo@1.0", "uuid-a")]); - let merged = merge_ledger_records_for_updates(None, Some(&ledger), None, &[]); + fn hosted_pin_matching_the_candidate_is_not_an_update() { + // The hosted patch is still the top offer — no nag. + let hosted = pins(&[("pkg:npm/foo@1.0", "uuid-a")]); + let merged = merge_ledger_records_for_updates(None, None, &hosted); let pkgs = vec![batch_with("pkg:npm/foo@1.0", &["uuid-a"])]; assert!(detect_updates(merged.as_deref(), &pkgs).is_empty()); } @@ -994,33 +982,32 @@ mod tests { fn manifest_entry_wins_a_collision_with_a_ledger_record() { // A PURL present in every store is manifest-owned (same precedence as // VEX's candidate merge): the manifest's uuid is the "old" side; - // between the ledgers, the redirect record wins. + // between the other two, the live hosted pin wins over the vendor + // ledger's (possibly superseded) entry. let manifest = crate::commands::scan::tests::manifest_with(&[("pkg:npm/foo@1.0", "uuid-manifest")]); - let ledger = ledger_with(&[("pkg:npm/foo@1.0", "uuid-ledger")]); + let hosted = pins(&[("pkg:npm/foo@1.0", "uuid-pin")]); let vendor = vendor_ledger_with(&[("pkg:npm/foo@1.0", "uuid-vendor", true)]); - let merged = - merge_ledger_records_for_updates(Some(&manifest), Some(&ledger), Some(&vendor), &[]); + let merged = merge_ledger_records_for_updates(Some(&manifest), Some(&vendor), &hosted); let pkgs = vec![batch_with("pkg:npm/foo@1.0", &["uuid-new"])]; let updates = detect_updates(merged.as_deref(), &pkgs); assert_eq!(updates.len(), 1); assert_eq!(updates[0].old_uuid, "uuid-manifest"); - let merged = merge_ledger_records_for_updates(None, Some(&ledger), Some(&vendor), &[]); + let merged = merge_ledger_records_for_updates(None, Some(&vendor), &hosted); let updates = detect_updates(merged.as_deref(), &pkgs); - assert_eq!(updates[0].old_uuid, "uuid-ledger"); + assert_eq!(updates[0].old_uuid, "uuid-pin"); } #[test] - fn ledger_and_manifest_cover_disjoint_purls() { - // A mixed project (some deps applied via manifest, some hosted via - // the redirect ledger, some vendored) gets update detection across - // every store. + fn hosted_pins_and_manifest_cover_disjoint_purls() { + // A mixed project (some deps applied via manifest, some hosted in + // the lockfile, some vendored) gets update detection across every + // store. let manifest = crate::commands::scan::tests::manifest_with(&[("pkg:npm/foo@1.0", "uuid-f1")]); - let ledger = ledger_with(&[("pkg:npm/bar@2.0", "uuid-b1")]); + let hosted = pins(&[("pkg:npm/bar@2.0", "uuid-b1")]); let vendor = vendor_ledger_with(&[("pkg:npm/baz@3.0", "uuid-z1", true)]); - let merged = - merge_ledger_records_for_updates(Some(&manifest), Some(&ledger), Some(&vendor), &[]); + let merged = merge_ledger_records_for_updates(Some(&manifest), Some(&vendor), &hosted); let pkgs = vec![ batch_with("pkg:npm/foo@1.0", &["uuid-f2"]), batch_with("pkg:npm/bar@2.0", &["uuid-b2"]), @@ -1035,28 +1022,25 @@ mod tests { } #[test] - fn absent_or_empty_ledgers_leave_the_manifest_view_untouched() { - assert!(merge_ledger_records_for_updates(None, None, None, &[]).is_none()); - let pins = vec![("pkg:npm/foo@1.0.0".to_string(), "uuid-pin".to_string())]; - let merged = merge_ledger_records_for_updates(None, None, None, &pins).expect("pinned"); + fn absent_or_empty_stores_leave_the_manifest_view_untouched() { + assert!(merge_ledger_records_for_updates(None, None, &[]).is_none()); + let hosted = pins(&[("pkg:npm/foo@1.0.0", "uuid-pin")]); + let merged = merge_ledger_records_for_updates(None, None, &hosted).expect("pinned"); assert_eq!(merged.patches["pkg:npm/foo@1.0.0"].uuid, "uuid-pin"); - let empty = socket_patch_core::patch::redirect::RedirectState::new(); let empty_vendor = VendorState::new(); - assert!( - merge_ledger_records_for_updates(None, Some(&empty), Some(&empty_vendor), &[]).is_none() - ); + assert!(merge_ledger_records_for_updates(None, Some(&empty_vendor), &[]).is_none()); let manifest = crate::commands::scan::tests::manifest_with(&[("pkg:npm/foo@1.0", "uuid-a")]); - let merged = merge_ledger_records_for_updates(Some(&manifest), Some(&empty), None, &[]) + let merged = merge_ledger_records_for_updates(Some(&manifest), Some(&empty_vendor), &[]) .expect("manifest present"); assert!( matches!(merged, Cow::Borrowed(_)), - "empty ledgers must not clone the manifest" + "empty stores must not clone the manifest" ); assert_eq!( merged.patches.len(), manifest.patches.len(), - "an empty ledger adds nothing" + "an empty vendor ledger adds nothing" ); } diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index e250d1a24..bf3cbbaf5 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -2996,23 +2996,23 @@ pub(crate) async fn run_redirect_selected( }; // Cross-mode takeover: a committed vendored ledger (`.socket/vendor/state.json`) - // may still claim package(s) this project also has a hosted redirect ledger - // for — their tarballs would then be orphaned and that ledger stale. But the - // overlap alone does NOT prove hosted won: only warn for the package(s) the - // LIVE lockfile actually routes to the hosted patch server (see - // `classify_overlap_takeover`), so a dry-run / no-op over a lock that still - // points at the vendored files stays silent instead of pointing cleanup at - // the live vendored ledger. The takeover pre-revert above already - // reconciled what it could; this only warns (JSON `warnings[]` and - // stderr) about any overlap left, WITHOUT deleting the other ledger. - // Classified over this run's in-memory ledgers — the redirect ledger as - // merged and persisted above, the vendored ledger as the takeover left - // it — so a non-dry-run reflects this run without re-reading either file. + // may still claim package(s) the lockfiles now pin hosted — their + // tarballs would then be orphaned and that ledger stale. But the overlap + // alone does NOT prove hosted won: only warn for the package(s) the LIVE + // lockfile actually routes to the hosted patch server (see + // `classify_overlap_takeover`), so a dry-run / no-op over a lock that + // still points at the vendored files stays silent instead of pointing + // cleanup at the live vendored ledger. The takeover pre-revert above + // already reconciled what it could; this only warns (JSON `warnings[]` + // and stderr) about any overlap left, WITHOUT deleting the other ledger. + // Classified over the lockfiles as this run left them and the vendored + // ledger as the takeover left it. let mut takeover_warnings: Vec = Vec::new(); + let hosted_now = crate::commands::hosted_state_from_lockfiles(common, &common.cwd).await; let superseded = super::classify_overlap_takeover_with( common, &common.cwd, - Some(&ledger), + Some(&hosted_now), vendor_state.as_ref().ok(), ) .await @@ -3020,7 +3020,7 @@ pub(crate) async fn run_redirect_selected( if !superseded.is_empty() { takeover_warnings.push(serde_json::json!({ "code": super::REDIRECT_SUPERSEDES_VENDORED, - "detail": super::mode_takeover_detail(&superseded, /*current_is_hosted=*/ true), + "detail": super::mode_takeover_detail(&superseded), })); } diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index 20ab9896f..30d15bd87 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -21,7 +21,7 @@ use socket_patch_core::telemetry::{ spawn_patch_scan_failed, spawn_patch_scanned, PendingTelemetry, }; use socket_patch_core::utils::concurrent::{api_concurrency_for, ordered_concurrent}; -use socket_patch_core::utils::purl::{normalize_purl, purl_name_version, strip_purl_qualifiers}; +use socket_patch_core::utils::purl::{normalize_purl, strip_purl_qualifiers}; use socket_patch_core::vendor::VendorState; use socket_patch_core::vex::discover::{LedgerLiveness, WiringMode}; use std::collections::{HashMap, HashSet}; @@ -687,19 +687,6 @@ async fn fetch_patch_details( (results, failures) } -/// The human hosted arm's stand-in for the lenient loader's advisory: a -/// malformed redirect ledger the engine would report as a hard error, on a -/// run that returned BEFORE the engine (empty discovery, nothing -/// downloadable, a detail-fetch failure). Read-only — -/// the file is never moved; `--silent` mutes it like every advisory. -fn warn_unreported_corrupt_ledger(common: &crate::args::GlobalArgs, corrupt: Option<&str>) { - if let Some(corrupt) = corrupt { - if !common.silent { - eprintln!("Warning: {corrupt}"); - } - } -} - /// Fold a [`discover_selected`] failure into a JSON caller's `result` and /// print it. The discovery counts already in `result` stay — they were /// computed from the (successful) batch phase — while `status`/`error` @@ -816,33 +803,29 @@ fn download_run<'a>(args: &ScanArgs, api_client: &'a ApiClient) -> DownloadRun<' } // --------------------------------------------------------------------------- -// Cross-mode ledger takeover detection (hosted ⇄ vendored) +// Cross-mode takeover detection (hosted over vendored) // --------------------------------------------------------------------------- // -// Hosted mode writes `.socket/vendor/redirect-state.json`; vendored mode -// writes `.socket/vendor/state.json` (+ committed tarballs). Switching a -// project's mode rewires the lockfile but leaves the OLD mode's ledger on -// disk asserting wiring that is no longer live, which misleads anything -// auditing a ledger (including `vex`). Detect the overlap so each flow can -// warn. The VENDORED flows clean the superseded redirect-ledger halves -// themselves (always announced); the HOSTED direction stays warn-only -// (removing a vendored entry deletes committed artifacts — `remove -// `'s job). +// Vendored mode writes `.socket/vendor/state.json` (+ committed tarballs); +// hosted mode keeps no ledger — its lockfile pins are the only record. +// Redirecting a vendored package to the hosted patch server rewires the +// lockfile but leaves the vendored ledger entry on disk asserting wiring that +// is no longer live, which misleads anything auditing the ledger (including +// `vex`). Detect the overlap so the hosted flow can warn (removing a vendored +// entry deletes committed artifacts — `remove `'s job). The reverse +// direction needs no advisory: once the lock routes a package to +// `.socket/vendor/`, no hosted state is left to go stale. // -// The overlap only proves BOTH ledgers name the package, not which won. The -// takeover DIRECTION comes from the current lockfile wiring per package -// (`classify_overlap_takeover`), never from which command is running; -// remediation points at the ledger that does NOT match the live lock, and a -// package the lock proves neither way stays silent. +// The overlap only proves the vendored ledger and a hosted pin both name the +// package, not which won. The takeover DIRECTION comes from the current +// lockfile wiring per package (`classify_overlap_takeover`), never from +// which command is running, and a package the lock proves neither way stays +// silent. /// Warning code emitted by the HOSTED flow when it just redirected package(s) /// a committed vendored ledger still claims (its tarballs are now orphaned). pub(super) const REDIRECT_SUPERSEDES_VENDORED: &str = "redirect_supersedes_vendored"; -/// Warning code emitted by the VENDORED flow when it just vendored package(s) -/// a committed hosted redirect ledger still claims. -pub(super) const VENDOR_SUPERSEDES_REDIRECT: &str = "vendor_supersedes_redirect"; - /// Warning code + detail emitted when `--prune` is combined with /// `--mode hosted`: the hosted flow runs no GC, so the flag would otherwise /// be silently dropped. `--prune` stays accepted (CLI_CONTRACT.md: an @@ -854,12 +837,12 @@ pub(super) const REDIRECT_PRUNE_IGNORED_DETAIL: &str = runs no GC sweep of `.socket/` state; run `scan --mode agent --prune` or \ `scan --mode vendored --prune` to garbage-collect"; -/// The PURLs claimed by BOTH the hosted redirect ledger -/// (`.socket/vendor/redirect-state.json`) and the vendored state ledger -/// (`.socket/vendor/state.json`), sorted, over already-loaded ledgers. A -/// non-empty result means exactly one of the two ledgers is stale for each -/// PURL (a lockfile entry can point only one way). `None`, an empty vendor -/// ledger, or disjoint ledgers (a legitimate split) yield no overlap. +/// The PURLs claimed by BOTH a hosted pin (`redirect`, the lockfiles' +/// hosted state — see [`crate::commands::hosted_state_from_lockfiles`]) and +/// the vendored state ledger (`.socket/vendor/state.json`), sorted. A +/// non-empty result means one of the two is stale for each PURL (a +/// lockfile entry can point only one way). `None`, an empty vendor ledger, +/// or disjoint states (a legitimate split) yield no overlap. fn overlap_from_states( redirect: Option<&socket_patch_core::patch::redirect::RedirectState>, vendor: &VendorState, @@ -867,54 +850,23 @@ fn overlap_from_states( let Some(redirect) = redirect else { return Vec::new(); }; - if vendor.entries.is_empty() { + if vendor.entries.is_empty() || redirect.records.is_empty() { return Vec::new(); } - // Canonicalize both sides (drop qualifiers, percent-decode) so the API - // purl form the redirect records carry matches the vendor entry's base - // purl — mirrors `vendored_ledger_supplement`. + // Canonicalize both sides (drop qualifiers, percent-decode) so the + // hosted pin's purl matches the vendor entry's base purl — mirrors + // `vendored_ledger_supplement`. let canon = |p: &str| normalize_purl(strip_purl_qualifiers(p)).into_owned(); let mut vendor_purls: std::collections::BTreeSet = std::collections::BTreeSet::new(); for (key, entry) in &vendor.entries { vendor_purls.insert(canon(key)); vendor_purls.insert(canon(&entry.base_purl)); } - if !redirect.records.is_empty() { - let redirect_purls: std::collections::BTreeSet = - redirect.records.keys().map(|p| canon(p)).collect(); - return redirect_purls - .intersection(&vendor_purls) - .cloned() - .collect(); - } - // The records map can be EMPTY while the ledger still asserts stale lock - // wiring (every per-uuid record fetch failed: `record_fetch_failed`), so - // fall back to matching the vendored purls against the recorded edit - // keys — npm `node_modules/` (possibly nested), pnpm/yarn/cargo/uv - // `@` (vlt `@~`), bun - // `/`, gem/composer/pypi bare ``. Name-level matching - // can over-claim across versions, but `classify_overlap_takeover` still - // requires the live lock to prove one side before anything is reported. - if redirect.edits.is_empty() { - return Vec::new(); - } - vendor_purls - .into_iter() - .filter(|purl| { - let Some((name, version)) = purl_name_version(strip_purl_qualifiers(purl)) else { - return false; - }; - redirect - .edits - .iter() - .filter_map(|e| e.key.as_deref()) - .any(|key| { - key == name - || key == format!("{name}@{version}") - || key.starts_with(&format!("{name}@{version}~")) - || key.ends_with(&format!("/{name}")) - }) - }) + let redirect_purls: std::collections::BTreeSet = + redirect.records.keys().map(|p| canon(p)).collect(); + redirect_purls + .intersection(&vendor_purls) + .cloned() .collect() } @@ -926,7 +878,7 @@ fn overlap_from_states( /// / `Discovery::vendor_entry_live`). `redirect` holds the overlap PURLs the /// lock routes to the hosted patch server (the vendored ledger entry is /// stale); `vendored` holds those it routes to a committed -/// `.socket/vendor//` artifact (the redirect record is stale). +/// `.socket/vendor//` artifact. /// /// A PURL the lock proves NEITHER way — a dry-run/no-op that did not rewire it, /// a half-migrated lock naming both, or an ecosystem whose live spec we cannot @@ -936,27 +888,25 @@ fn overlap_from_states( pub(super) struct OverlapTakeover { /// Overlap PURLs whose vendored ledger is stale (lock points hosted). pub redirect: Vec, - /// Overlap PURLs whose redirect ledger is stale (lock points vendored). + /// Overlap PURLs the lock routes to the vendored artifact. pub vendored: Vec, } +/// [`classify_overlap_takeover_with`] over the on-disk state: the +/// lockfiles' hosted pins and the committed vendored ledger. +#[cfg(test)] pub(super) async fn classify_overlap_takeover(common: &GlobalArgs, cwd: &Path) -> OverlapTakeover { - // Both ledgers loaded ONCE here. A malformed ledger classifies like a - // missing one, matching `overlap_from_states` (this path only - // feeds takeover warnings; corruption is a hard error on the + // A malformed vendor ledger classifies like a missing one (this path + // only feeds takeover warnings; corruption is a hard error on the // write/attest paths). - let redirect = socket_patch_core::patch::redirect::load_redirect_state(cwd) - .await - .ok() - .flatten(); + let redirect = crate::commands::hosted_state_from_lockfiles(common, cwd).await; let vendor = socket_patch_core::vendor::load_state(cwd).await.ok(); - classify_overlap_takeover_with(common, cwd, redirect.as_ref(), vendor.as_ref()).await + classify_overlap_takeover_with(common, cwd, Some(&redirect), vendor.as_ref()).await } -/// [`classify_overlap_takeover`] over already-loaded ledgers (the hosted -/// engine must classify against its in-memory post-merge / post-takeover -/// copies, never a pre-takeover snapshot); still reads the LIVE lockfiles -/// in `cwd`. `None` for either ledger yields no overlap. +/// [`classify_overlap_takeover`] over already-loaded state (the hosted +/// engine classifies against its post-takeover vendor ledger); still reads +/// the LIVE lockfiles in `cwd`. `None` for either yields no overlap. pub(super) async fn classify_overlap_takeover_with( common: &GlobalArgs, cwd: &Path, @@ -984,10 +934,8 @@ pub(super) async fn classify_overlap_takeover_with( .entry(canon(&entry.base_purl)) .or_insert(entry); } - // The hosted proof needs the redirect ledger too: each record's patch - // uuid (embedded in every hosted artifact URL, whatever the host) and - // the lockfiles the redirect actually edited. A non-empty overlap - // proves the ledger is `Some`. + // Each hosted pin's patch uuid (embedded in every hosted artifact URL, + // whatever the host). A non-empty overlap proves `redirect` is `Some`. let mut redirect_uuid_by_purl: std::collections::HashMap = std::collections::HashMap::new(); for (key, record) in redirect.iter().flat_map(|r| &r.records) { @@ -996,12 +944,10 @@ pub(super) async fn classify_overlap_takeover_with( .or_insert(record.uuid.as_str()); } let discovery = crate::commands::discover_wiring(common, cwd).await; - let mut liveness = LedgerLiveness::new(cwd, &discovery, redirect); + let mut liveness = LedgerLiveness::new(cwd, &discovery, None); for purl in overlap { let hosted_live = match redirect_uuid_by_purl.get(&purl) { Some(uuid) => liveness.redirect_record(&purl, uuid).await, - // An edits-only ledger (every record fetch failed) names no - // uuid: the lock's own hosted wiring of the package decides. None => discovery.wires_package(&purl, WiringMode::Hosted), }; let vendored_live = match vendor_by_purl.get(&purl) { @@ -1021,159 +967,42 @@ pub(super) async fn classify_overlap_takeover_with( out } -/// Human-readable detail for a mode-takeover warning naming the displaced -/// package(s). `current_is_hosted` selects the direction: `true` when a -/// hosted redirect displaced a vendored ledger, `false` when a vendored run -/// displaced a hosted redirect ledger. +/// Human-readable detail for the hosted-over-vendored takeover warning +/// ([`REDIRECT_SUPERSEDES_VENDORED`]) naming the displaced package(s). /// /// The warning fires PER PACKAGE, so the remediation is per-package and -/// non-destructive: never delete a whole ledger file or a whole -/// `.socket/vendor//` tree, which may still carry LIVE data for -/// packages this takeover did not touch (other records VEX reads, the -/// pre-redirect originals that are the only revert data, other vendored -/// uuid dirs). -/// -/// It must also be COMPLETE per package, or it does not converge: -/// -/// * The vendored direction names the package's `edits` entry alongside its -/// `records` entry: `overlap_from_states` falls back to edit KEYS once -/// `records` is empty, so a records-only cleanup keeps this warning firing. -/// * The hosted direction states `socket-patch remove`'s full blast radius, -/// including the package's `.socket/manifest.json` entry. -pub(super) fn mode_takeover_detail(superseded: &[String], current_is_hosted: bool) -> String { +/// non-destructive: never delete a whole `.socket/vendor//` tree, which +/// may still carry LIVE data for packages this takeover did not touch. It +/// states `socket-patch remove`'s full blast radius, including the +/// package's `.socket/manifest.json` entry. +pub(super) fn mode_takeover_detail(superseded: &[String]) -> String { let list = superseded.join(", "); - if current_is_hosted { - // NEVER offer deleting the `.socket/vendor//` tree here: for - // cargo the leftover `[patch.crates-io]` entry still points at that - // tree, and deleting it hard-fails every cargo invocation ("failed to - // load source for dependency"). Nor `vendor --revert`, which unwinds - // EVERY vendored package including the ones still live in the - // lockfile — `remove ` is the per-package equivalent. - format!( - "hosted redirect superseded the vendored ledger for: {list}. \ - `.socket/vendor/state.json` still claims these package(s) and their \ - committed artifacts under `.socket/vendor/` are now orphaned — the \ - lockfile points at the hosted patch server, not the vendored files. \ - Clean up per package: run `socket-patch remove ` for each \ - package listed above, so audits and VEX do not read superseded \ - wiring. It drops that package's vendored ledger entry and its own \ - `.socket/vendor///` artifact directory, AND deletes that \ - package's now-superseded `.socket/manifest.json` entry — that entry \ - describes the vendored delivery, while the live hosted patch is \ - recorded in `.socket/vendor/redirect-state.json`, which `remove` \ - never touches. In-place file rollback is skipped for vendor-owned \ - package(s), so the installed tree is left as the lockfile wires it; \ - preview with `--dry-run` first. Do not delete the whole \ - `.socket/vendor//` tree and do not run `vendor --revert`: \ - other vendored package(s) may still be live in the lockfile and \ - would break or be mass-reverted." - ) - } else { - // NEVER advise deleting the redirect ledger by hand: it may hold the - // only revert data (FileEdit originals) and VEX records for OTHER - // packages that are still hosted-redirected. - format!( - "vendored artifacts superseded the hosted redirect ledger for: {list}. \ - `.socket/vendor/redirect-state.json` still records a hosted redirect for \ - these package(s), but the lockfile now points at the committed \ - `.socket/vendor/` files. The vendored flows (`socket-patch vendor`, \ - `scan --mode vendored`) reconcile npm-family and cargo package(s) \ - automatically on their next non-dry run, dropping both halves of \ - each superseded entry — the `records` entry AND its matching \ - `edits` (cargo additionally reverts the stale hosted edits on disk \ - first). For other package(s), or if the automatic reconciliation \ - could not run, clean up by hand: delete only these package(s)' \ - entries under `records` AND their matching entries under `edits`, \ - so audits and VEX do not read superseded wiring. \ - Both halves matter: the leftover `edits` are that package's stale \ - pre-redirect originals, which a later redirect revert would replay \ - over the live vendored wiring — and an `edits` entry left behind \ - still names the package, so a ledger whose last record you just \ - deleted keeps reading as superseded and this warning keeps firing. \ - Do not delete the ledger file itself: it may still hold live \ - redirect records for other package(s), plus the recorded \ - pre-redirect lockfile originals (`edits`) a future revert needs \ - for them." - ) - } -} - -/// Detail for the vendored-direction takeover warning on the run that -/// RECONCILED the ledger in place (non-dry-run, npm-family): past tense, -/// stating what was dropped and where the revert data now lives. The code -/// stays `vendor_supersedes_redirect` (codes are stable; only the detail -/// differs), and it fires once — the reconciled ledger no longer overlaps. -pub(super) fn mode_takeover_reconciled_detail( - reconciled: &[String], - npmrc_unwound: bool, -) -> String { - let list = reconciled.join(", "); - // Only a run that actually unwound the hosted npm allow-remote - // auto-config says so (with its npm >= 12 caveat). - let npmrc = if npmrc_unwound { - " The hosted redirect's `.npmrc` `allow-remote=all` auto-config was \ - unwound too (a redirect-created file deleted, an appended line \ - removed): the vendored `file:` specs do not need it. If you later \ - restore the hosted lock wiring with `vendor --revert`, npm >=12 \ - refuses it (EALLOWREMOTE) until `allow-remote=all` is back — re-run \ - `scan --mode hosted` afterwards to re-establish it and its ledger \ - record." - } else { - "" - }; + // NEVER offer deleting the `.socket/vendor//` tree here: for cargo + // the leftover `[patch.crates-io]` entry still points at that tree, and + // deleting it hard-fails every cargo invocation ("failed to load source + // for dependency"). Nor `vendor --revert`, which unwinds EVERY vendored + // package including the ones still live in the lockfile — `remove + // ` is the per-package equivalent. format!( - "vendored artifacts superseded the hosted redirect ledger for: {list}; \ - reconciled automatically. Both halves of each superseded entry — the \ - package's `records` entry AND its matching `edits` — were dropped \ - from `.socket/vendor/redirect-state.json` (an emptied ledger is \ - deleted). The lockfile points at the committed `.socket/vendor/` \ - files, and the pre-vendor lock values (including the hosted-spliced \ - fragment) are preserved as the vendor ledger's wiring originals, so \ - `vendor --revert` still restores the hosted lock wiring \ - byte-for-byte.{npmrc} Ledger data for other, still-redirected \ - package(s) was left untouched. No action needed." + "hosted redirect superseded the vendored ledger for: {list}. \ + `.socket/vendor/state.json` still claims these package(s) and their \ + committed artifacts under `.socket/vendor/` are now orphaned — the \ + lockfile points at the hosted patch server, not the vendored files. \ + Clean up per package: run `socket-patch remove ` for each \ + package listed above, so audits and VEX do not read superseded \ + wiring. It drops that package's vendored ledger entry and its own \ + `.socket/vendor///` artifact directory, AND deletes that \ + package's now-superseded `.socket/manifest.json` entry — that entry \ + describes the vendored delivery, while the live hosted patch is \ + recorded in the lockfile itself. In-place file rollback is skipped \ + for vendor-owned package(s), so the installed tree is left as the \ + lockfile wires it; preview with `--dry-run` first. Do not delete the \ + whole `.socket/vendor//` tree and do not run `vendor --revert`: \ + other vendored package(s) may still be live in the lockfile and \ + would break or be mass-reverted." ) } -/// Drop the superseded purls' `records` + `edits` from the redirect ledger -/// and persist it (atomic write; an emptied ledger is deleted). Called ONLY -/// with purls [`classify_overlap_takeover`] proved vendored-live AND -/// hosted-dead: that gate makes the drop lossless (the vendor ledger's -/// wiring `original` embeds the hosted-spliced fragment, so `vendor -/// --revert` needs nothing from these records). `Ok(Some(npmrc))`: -/// reconciled, with the `.npmrc` allow-remote unwind outcome; `Ok(None)`: -/// nothing matched (caller falls back to the manual advisory); `Err`: the -/// ledger could not be read or persisted (fail closed: on-disk ledger -/// untouched or fully pre-drop). -async fn reconcile_superseded_redirect( - cwd: &Path, - purls: &[String], -) -> Result, String> { - let mut state = match socket_patch_core::patch::redirect::load_redirect_state(cwd).await { - Ok(Some(state)) => state, - Ok(None) => return Ok(None), - Err(corrupt) => return Err(corrupt.to_string()), - }; - let mut dropped = false; - for purl in purls { - dropped |= socket_patch_core::patch::redirect::drop_superseded_purl(&mut state, purl); - } - if !dropped { - return Ok(None); - } - // The dropped npm purls may have been the last entries the hosted - // `.npmrc` `allow-remote=all` auto-config served: unwind it before - // persisting, so a hosted→vendored migration leaves no loosened install - // policy behind (vendored `file:` specs are gated by `allow-file`). - let npmrc = - socket_patch_core::patch::redirect::npmrc::unwind_unneeded_npmrc(cwd, &mut state, false) - .await?; - socket_patch_core::patch::redirect::persist_redirect_state(cwd, &state) - .await - .map_err(|e| e.to_string())?; - Ok(Some(npmrc)) -} - /// Record a run-level advisory: stderr `Warning (code): detail` in human /// mode (informational, so muted by `--silent`) and `warnings[]` on the /// envelope for JSON consumers. Shared by the vendored flows here and in @@ -1193,82 +1022,6 @@ pub(super) fn push_run_warning( }); } -/// Cross-mode takeover advisory shared by every VENDORED flow (`vendor`, -/// `scan --mode vendored`): when this ledger and a committed hosted redirect -/// ledger both claim package(s) AND the live lockfile proves vendored won, -/// the redirect ledger records for those package(s) are stale. Warn once at -/// the envelope level (JSON `warnings[]` and stderr) — and, for npm-family -/// package(s) on a non-dry run, reconcile the ledger in place (cargo -/// reverts + drops BEFORE vendoring in `vendor.rs`; npm-family needs no -/// on-disk revert, since vendoring already overwrote the hosted splice and -/// recorded it as the wiring `original`). The reverse direction -/// (`redirect_supersedes_vendored`) is deliberately untouched. -pub(super) async fn note_vendor_supersedes_redirect( - env: &mut crate::json_envelope::Envelope, - cwd: &Path, - common: &GlobalArgs, -) { - // Only the package(s) the LIVE lockfile routes to `.socket/vendor/`. - let superseded = classify_overlap_takeover(common, cwd).await.vendored; - if superseded.is_empty() { - return; - } - // Reconciliation is gated, each fail-closed to the manual advisory: - // never under --dry-run; only npm-family purls (cargo reverts in - // vendor.rs, and other ecosystems' vendor wiring is not verified to - // embed the hosted originals); only purls classified above. - let (reconcilable, manual): (Vec, Vec) = if common.dry_run { - (Vec::new(), superseded) - } else { - superseded - .into_iter() - .partition(|purl| purl.starts_with("pkg:npm/")) - }; - if !manual.is_empty() { - push_run_warning( - env, - common, - VENDOR_SUPERSEDES_REDIRECT, - mode_takeover_detail(&manual, /*current_is_hosted=*/ false), - ); - } - if reconcilable.is_empty() { - return; - } - match reconcile_superseded_redirect(cwd, &reconcilable).await { - Ok(Some(npmrc)) => { - push_run_warning( - env, - common, - VENDOR_SUPERSEDES_REDIRECT, - mode_takeover_reconciled_detail(&reconcilable, npmrc.file_changed), - ); - for (code, detail) in npmrc.warnings { - push_run_warning(env, common, &code, detail); - } - } - // Nothing matched to drop — do not claim a reconciliation that did - // not happen; hand out the manual remediation instead. - Ok(None) => push_run_warning( - env, - common, - VENDOR_SUPERSEDES_REDIRECT, - mode_takeover_detail(&reconcilable, /*current_is_hosted=*/ false), - ), - Err(e) => push_run_warning( - env, - common, - VENDOR_SUPERSEDES_REDIRECT, - format!( - "{} Automatic reconciliation failed ({e}); the ledger was left \ - as it was, so this warning will fire again until the cleanup \ - above succeeds.", - mode_takeover_detail(&reconcilable, /*current_is_hosted=*/ false) - ), - ), - } -} - /// Top-level `warnings[]` JSON for scan's envelope from `(code, detail)` /// pairs (see [`unsupported_layout_warnings`]). Same `{code, detail}` object /// shape as the run-level `warnings[]` on the unified envelope. @@ -1366,7 +1119,7 @@ pub(super) async fn hosted_wiring_retained_purls( } let cwd = &common.cwd; let discovery = crate::commands::discover_wiring(common, cwd).await; - let mut liveness = LedgerLiveness::new(cwd, &discovery, Some(redirect)); + let mut liveness = LedgerLiveness::new(cwd, &discovery, None); let mut out = Vec::new(); for (purl, uuid) in candidates { if liveness.redirect_record(&purl, uuid).await { @@ -1379,28 +1132,20 @@ pub(super) async fn hosted_wiring_retained_purls( } /// Detail for [`HOSTED_WIRING_RETAINED`]. Names the package(s) and the -/// real options — stay hosted, migrate via the vendored flow (which -/// reconciles the superseded ledger entries per package), or unwind via -/// `rollback`. It must never advise hand-deleting the redirect ledger -/// (the only store of the pre-redirect originals plus the records VEX -/// reads). +/// real options — stay hosted, migrate via the vendored flow, or restore +/// the upstream registry entries via `rollback`. pub(super) fn hosted_wiring_retained_detail(retained: &[String]) -> String { let list = retained.join(", "); format!( "agent-mode scan left the hosted redirect wiring live for: {list}. \ The lockfile still resolves these package(s) to the hosted patch \ - server and `.socket/vendor/redirect-state.json` still records the \ - redirect — an agent run patches installed files in place but does \ + server — an agent run patches installed files in place but does \ NOT unwind hosted lockfile wiring, so installs keep fetching \ these package(s) from the patch server. Either keep the project \ in hosted mode (`scan --mode hosted`), migrate to committed \ artifacts with `scan --mode vendored` (which takes these \ - package(s) over in the lockfile and reconciles the superseded \ - redirect ledger entries), or unwind the redirects with \ - `socket-patch rollback`. Do not delete \ - `.socket/vendor/redirect-state.json` by hand: it holds the \ - recorded pre-redirect lockfile originals (the only revert data) \ - and the redirect records VEX reads." + package(s) over in the lockfile), or restore their upstream \ + registry entries with `socket-patch rollback`." ) } @@ -1429,22 +1174,19 @@ pub(super) fn vendored_ownership_retained_detail(purls: &[String]) -> String { } /// Additive top-level `redirectState` block for the scan `--json` envelope: -/// the hosted redirect ledger's records — project STATE, so a descriptive -/// block rather than a warning — plus the scanned purls whose hosted -/// lockfile wiring the live lock still proves. +/// the hosted pins the lockfiles wire — project STATE, so a descriptive +/// block rather than a warning — plus the scanned purls among them. /// -/// `None` (key omitted, additive contract) when the ledger is absent or its -/// `records` are empty — an edits-only ledger asserts no patches. +/// `None` (key omitted, additive contract) when no lockfile pins a hosted +/// patch. /// -/// Shape: `{ mode, ledger, records: [{purl, ledgerKey, uuid}], wiringLive: -/// [purl] }`. `mode` is the constant [`crate::commands::HOSTED_MODE_LABEL`], -/// never the ledger's own `mode` string (older ledgers carry `"redirect"`). -/// Each record's `purl` is canonicalized (qualifiers stripped, -/// percent-decoded) to the spelling `wiringLive` carries; `ledgerKey` is the -/// ledger's verbatim key. `wiring_live` is the caller's -/// [`hosted_wiring_retained_purls`] result, computed once per run. A record -/// with no proof means the wiring was unwound, the lock is unreadable, or -/// the purl was not crawled this run — never "still live". +/// Shape: `{ mode, records: [{purl, uuid}], wiringLive: [purl] }`. `mode` +/// is the constant [`crate::commands::HOSTED_MODE_LABEL`]. Each record's +/// `purl` is canonicalized (qualifiers stripped, percent-decoded) to the +/// spelling `wiringLive` carries. `wiring_live` is the caller's +/// [`hosted_wiring_retained_purls`] result, computed once per run: the pins +/// this run crawled (a pin whose package was not crawled is still wired, +/// just not covered by this run). pub(super) fn redirect_state_json( redirect_state: Option<&socket_patch_core::patch::redirect::RedirectState>, wiring_live: &[String], @@ -1460,14 +1202,12 @@ pub(super) fn redirect_state_json( .map(|(key, record)| { serde_json::json!({ "purl": canon(key), - "ledgerKey": key, "uuid": record.uuid, }) }) .collect(); Some(serde_json::json!({ "mode": crate::commands::HOSTED_MODE_LABEL, - "ledger": socket_patch_core::patch::redirect::REDIRECT_STATE_REL, "records": records, "wiringLive": wiring_live, })) @@ -1881,11 +1621,13 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { // `redirectState` rides the empty-discovery envelope too // (same rule as the ≥1-package path). `wiringLive` is empty // by construction: this run covered zero packages. - let redirect_state = crate::commands::load_redirect_state_lenient( - &args.common.cwd, - args.common.silent, - ) - .await; + let redirect_state = (!args.common.is_global()).then_some( + crate::commands::hosted_state_from_lockfiles( + &args.common, + &args.common.cwd, + ) + .await, + ); if let Some(state) = redirect_state_json(redirect_state.as_ref(), &[]) { result["redirectState"] = state; } @@ -2143,50 +1885,26 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { // Read existing manifest once for update detection. let existing_manifest = read_manifest(&manifest_path).await.ok().flatten(); - // Hosted and vendored modes record their patches ONLY in their ledgers, - // so both ledgers' purl→uuid records are folded into update detection - // (otherwise their `updates[]` would stay empty). A malformed redirect - // ledger is only warned about here (--silent mutes it). A HOSTED run - // does not warn: its engine loads the ledger strictly and reports the - // corruption once as a hard error; the human hosted arm's returns - // BEFORE the engine (empty discovery, nothing downloadable, a - // detail-fetch failure) print it via `warn_unreported_corrupt_ledger`. - let (redirect_state, hosted_corrupt_ledger) = if hosted { - match socket_patch_core::patch::redirect::load_redirect_state(&args.common.cwd).await { - Ok(state) => (state, None), - Err(corrupt) => (None, Some(corrupt.to_string())), - } - } else { - // `load_redirect_state_lenient`, with the scan event flushed before - // its warning (possibly this run's first write since the event). - match socket_patch_core::patch::redirect::load_redirect_state(&args.common.cwd).await { - Ok(state) => (state, None), - Err(corrupt) => { - if !args.common.silent { - telemetry.flush().await; - eprintln!("Warning: {corrupt}"); - } - (None, None) - } - } - }; - // The hosted pins the lockfiles wire count too: the lockfile is the - // record of a hosted redirect even where no ledger was committed. - let hosted_pins: Vec<(String, String)> = + // Hosted mode records its patches ONLY in the lockfiles (v5 keeps no + // hosted ledger) and vendored mode ONLY in its ledger, so the hosted + // pins and the vendor ledger's purl→uuid records are folded into update + // detection (otherwise their `updates[]` would stay empty). + let hosted_pin_list: Vec = if args.common.is_global() { Vec::new() } else { - crate::commands::discover_wiring(&args.common, &args.common.cwd) - .await - .refs - .into_iter() - .filter(|r| r.mode == socket_patch_core::vex::discover::WiringMode::Hosted) - .map(|r| (r.purl, r.uuid)) - .collect() + socket_patch_core::patch::redirect::upstream::HostedPin::all( + &crate::commands::discover_wiring(&args.common, &args.common.cwd).await, + ) }; + let redirect_state = (!args.common.is_global()) + .then(|| crate::commands::hosted_state_from_pins(&hosted_pin_list)); + let hosted_pins: Vec<(String, String)> = hosted_pin_list + .iter() + .map(|pin| (pin.purl.clone(), pin.uuid.clone())) + .collect(); let update_manifest = merge_ledger_records_for_updates( existing_manifest.as_ref(), - redirect_state.as_ref(), vendor_state.as_ref().ok(), &hosted_pins, ); @@ -2477,7 +2195,6 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { if !silent { println!("\nNo patches available for installed packages."); } - warn_unreported_corrupt_ledger(&args.common, hosted_corrupt_ledger.as_deref()); return finish_human(0).await; } @@ -2620,7 +2337,6 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { if !silent { println!("\nNo downloadable patches (paid subscription required)."); } - warn_unreported_corrupt_ledger(&args.common, hosted_corrupt_ledger.as_deref()); return finish_human(0).await; } @@ -2644,7 +2360,6 @@ async fn run_scan(mut args: ScanArgs, telemetry: &mut PendingTelemetry) -> i32 { Ok(s) => s, // `discover_selected` already printed the failure to stderr. Err((code, _)) => { - warn_unreported_corrupt_ledger(&args.common, hosted_corrupt_ledger.as_deref()); return code; } }; diff --git a/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs b/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs index 5457311c1..7dd0566a3 100644 --- a/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs +++ b/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs @@ -46,7 +46,7 @@ use crate::ui::{plural, print_json}; use super::gc::{gc_json, print_gc_vendored_line, run_apply_gc}; use super::{ discover_selected, download_params, embed_vex_into_json, emit_discovery_error_json, - note_vendor_supersedes_redirect, push_run_warning, ScanArgs, + push_run_warning, ScanArgs, }; /// Run-level warning: a `.socket/manifest.json` record for a purl the @@ -190,9 +190,8 @@ async fn run_scan_vendor_step( let manifest_path = common.resolved_manifest_path(); let socket_dir = common.socket_dir(); let timeout = Duration::from_secs(common.lock_timeout.unwrap_or(0)); - // The guard lives to the end of the step so the ledger migration and - // the redirect-ledger reconcile in `note_vendor_supersedes_redirect` - // run under the lock too. + // The guard lives to the end of the step so the ledger migration runs + // under the lock too. let _guard = crate::commands::lock_cli::acquire_with_status(&socket_dir, timeout).map_err(|e| { let (code, message) = lock_failure(&e, timeout); @@ -230,7 +229,6 @@ async fn run_scan_vendor_step( env.mark_partial_failure(); } note_classic_migration_risk(&mut env, &common.cwd, common); - note_vendor_supersedes_redirect(&mut env, &common.cwd, common).await; Ok((has_errors, env)) } diff --git a/crates/socket-patch-cli/src/commands/vendor.rs b/crates/socket-patch-cli/src/commands/vendor.rs index 11293a373..bcbf093b8 100644 --- a/crates/socket-patch-cli/src/commands/vendor.rs +++ b/crates/socket-patch-cli/src/commands/vendor.rs @@ -823,13 +823,9 @@ pub async fn run(args: VendorArgs) -> i32 { } note_classic_migration_risk(&mut env, &args.common.cwd, &args.common); - // Same cross-mode takeover advisory the scan-driven vendored flow emits: - // surface a redirect ledger that this run (or an earlier one) superseded. - super::scan::note_vendor_supersedes_redirect(&mut env, &args.common.cwd, &args.common).await; - // That advisory may persist the redirect ledger, so it ran under the - // lock; everything below is output and telemetry, so release the lock - // before the telemetry round-trip. + // Everything below is output and telemetry, so release the lock before + // the telemetry round-trip. drop(lock); if args.common.json { diff --git a/crates/socket-patch-cli/src/commands/vex.rs b/crates/socket-patch-cli/src/commands/vex.rs index 4ece01fbe..f6d75cfcb 100644 --- a/crates/socket-patch-cli/src/commands/vex.rs +++ b/crates/socket-patch-cli/src/commands/vex.rs @@ -1064,24 +1064,30 @@ async fn generate_vex_from_manifest_path_inner( } }; let had_manifest_file = manifest_file.is_some(); - // Both ledgers are attestation inputs (records, and the entries whose - // wiring liveness gates them), so a MALFORMED one is a hard error: - // attesting with its contents silently dropped would produce a false — - // or silently partial — document. A missing ledger is simply empty. + // The vendor ledger is an attestation input (records, and the entries + // whose wiring liveness gates them), so a MALFORMED one is a hard error + // (below). v5 hosted mode keeps no ledger: hosted references come from + // the lockfiles, their records from the API. A pre-v5 redirect ledger + // is read (never written) only as an extra local record source for the + // pins it still describes, so a malformed one is an advisory: its + // records are simply not consulted. let redirect = match socket_patch_core::patch::redirect::load_redirect_state(&common.cwd).await { Ok(state) => state, Err(corrupt) => { - // Not core's Display: that text ("... so it will not be - // overwritten") is written for the hosted `scan` writer, and - // `vex` only reads the ledger. - let message = format!( - "The redirect ledger {} is malformed ({}); cannot attest redirected patches. \ - Repair its JSON or restore it from version control, then re-run.", - corrupt.path.display(), - corrupt.detail + note_warning( + warnings, + common, + "redirect_ledger_corrupt", + format!( + "the pre-v5 redirect ledger {} is malformed ({}); its records were not \ + consulted. socket-patch v5 no longer uses it: delete it, or restore it \ + from version control.", + corrupt.path.display(), + corrupt.detail + ), ); - return Err(fail(common, "redirect_ledger_corrupt", message).await); + None } }; let vendor = match socket_patch_core::vendor::load_state(&common.cwd).await { diff --git a/crates/socket-patch-cli/tests/in_process_rollback_hosted.rs b/crates/socket-patch-cli/tests/in_process_rollback_hosted.rs index 3fcd5a245..ef47ef479 100644 --- a/crates/socket-patch-cli/tests/in_process_rollback_hosted.rs +++ b/crates/socket-patch-cli/tests/in_process_rollback_hosted.rs @@ -98,6 +98,7 @@ async fn rollback_in_process(cwd: &Path, targets: Vec, preserve_state: b json: true, yes: true, silent: true, + patch_server_url: Some("http://patch.test".to_string()), ..socket_patch_cli::args::GlobalArgs::default() }, one_off: false, @@ -111,6 +112,46 @@ async fn rollback_in_process(cwd: &Path, targets: Vec, preserve_state: b code } +/// Serve the npm registry's version document for the real-flow fixture's +/// package, so the upstream restore can re-resolve its pristine entry. +async fn mock_npm_registry(server: &MockServer) { + Mock::given(method("GET")) + .and(path(format!("/npm-registry/{NAME}/{VERSION}"))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "name": NAME, + "version": VERSION, + "dist": { + "tarball": format!("https://registry.npmjs.org/{NAME}/-/{NAME}-{VERSION}.tgz"), + "integrity": "sha512-UPSTREAMupstream==", + } + }))) + .mount(server) + .await; +} + +/// Bare in-process rollback that may reach the (mocked) npm registry: the +/// upstream restore re-resolves each hosted pin's registry entry. +async fn rollback_online(cwd: &Path, server: &MockServer) -> i32 { + std::env::set_var("SOCKET_NPM_REGISTRY", format!("{}/npm-registry", server.uri())); + let args = RollbackArgs { + targets: Vec::new(), + common: socket_patch_cli::args::GlobalArgs { + cwd: cwd.to_path_buf(), + manifest_path: ".socket/manifest.json".to_string(), + json: true, + yes: true, + silent: true, + patch_server_url: Some("http://patch.test".to_string()), + ..socket_patch_cli::args::GlobalArgs::default() + }, + one_off: false, + preserve_state: false, + }; + let code = rollback_run(args).await; + std::env::remove_var("SOCKET_NPM_REGISTRY"); + code +} + /// A `socket-patch` Command with the ambient `SOCKET_*` env surface scrubbed /// (the `in_process_redirect.rs` seed-then-scrub pattern): hostile seeds /// never reach the child because `env_remove` clears them too, but if a @@ -501,11 +542,12 @@ async fn npm_hosted_round_trip() { ); assert_ne!(wired, pristine, "wiring must actually change the lock"); assert!( - ledger_path(tmp.path()).is_file(), - "scan --mode hosted must write the redirect ledger" + !ledger_path(tmp.path()).exists(), + "scan --mode hosted keeps no redirect ledger: the lockfile is the record" ); - let code = rollback_in_process(tmp.path(), Vec::new(), false).await; + mock_npm_registry(&server).await; + let code = rollback_online(tmp.path(), &server).await; assert_eq!(code, 0, "bare rollback over hosted wiring should exit 0"); let restored = std::fs::read_to_string(tmp.path().join("package-lock.json")).unwrap(); From a01ae92d4bda44a9c24d7ee6ad317525dfb82325 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 22:58:47 +0000 Subject: [PATCH 05/66] Drop the hosted ledger from get, repair, the in-memory engine and the scan fold - get's vendored lock-text gates read the lockfiles' hosted pins instead of the redirect ledger (DownloadParams carries --patch-server-url for it). - repair's hosted-only no-op fires for hosted lockfile pins (or a pre-v5 ledger). - The in-memory hosted engine neither reads nor emits .socket/vendor/redirect-state.json. - Disk hosted scan no longer folds edits into a throwaway ledger; its records feed only the stale-install probes and in-run VEX. - The cargo vendor backend's hosted_redirect_live refusal names rollback / git checkout instead of a ledger. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018tncU7u7FK48tubFNA12UQ --- crates/socket-patch-cli/src/commands/get.rs | 34 +- .../socket-patch-cli/src/commands/repair.rs | 32 +- .../src/commands/scan/hosted.rs | 387 +----------------- .../socket-patch-cli/src/commands/scan/mod.rs | 1 + .../src/hosted_memory/ledger.rs | 187 --------- .../socket-patch-cli/src/hosted_memory/mod.rs | 59 +-- .../src/hosted_memory/select.rs | 17 +- crates/socket-patch-core/src/vendor/cargo.rs | 20 +- 8 files changed, 77 insertions(+), 660 deletions(-) delete mode 100644 crates/socket-patch-cli/src/hosted_memory/ledger.rs diff --git a/crates/socket-patch-cli/src/commands/get.rs b/crates/socket-patch-cli/src/commands/get.rs index 76fe2a8bc..393c7b379 100644 --- a/crates/socket-patch-cli/src/commands/get.rs +++ b/crates/socket-patch-cli/src/commands/get.rs @@ -1222,6 +1222,9 @@ pub struct DownloadParams { /// `false`: their patch content is staged in memory and the committed /// artifact is the patch — nothing should land in `.socket/blobs`. pub persist_blobs: bool, + /// `--patch-server-url`: the extra origin whose URLs count as hosted + /// when lockfile discovery reads the project's hosted pins. + pub patch_server_url: Option, } impl DownloadParams { @@ -1851,10 +1854,9 @@ type LockRefusals = HashMap; /// classic / yarn berry gates and cargo's locked-version gate), over the /// patches the phase would otherwise fetch a view for — past the Bun /// refusal and the ledger's idempotency skip, which take precedence in the -/// fetch loop. A purl the hosted redirect ledger claims is left to the -/// vendor loop: its takeover reverts the hosted lock edits first, and the -/// revert rewrites the very text the gates read. A redirect ledger that -/// cannot be read leaves every purl to the loop. +/// fetch loop. A purl the lockfiles pin hosted is left to the vendor loop: +/// its takeover restores the upstream lock entry first, and the restore +/// rewrites the very text the gates read. /// /// Only a package the vendor loop would hand to its backend is refused /// here (see [`crate::commands::vendor::lock_refusals_reaching_backend`]): @@ -1872,11 +1874,23 @@ async fn lock_text_refusals_for( ) -> LockRefusals { let cwd = params.cwd.as_path(); let claimed: Vec = - match socket_patch_core::patch::redirect::load_redirect_state(cwd).await { - Ok(Some(state)) => state.records.keys().map(|k| canonical_purl(k)).collect(), - Ok(None) => Vec::new(), - Err(_) => return HashMap::new(), - }; + socket_patch_core::patch::redirect::upstream::HostedPin::all( + &socket_patch_core::vex::discover_patched_refs_with( + cwd, + &socket_patch_core::vex::DiscoverOptions { + patch_server_origins: params + .patch_server_url + .iter() + .filter(|url| !url.trim().is_empty()) + .cloned() + .collect(), + }, + ) + .await, + ) + .into_iter() + .map(|pin| canonical_purl(&pin.purl)) + .collect(); let candidates: Vec<(&str, &str)> = selected .iter() .filter(|sr| bun_refusal.filter(|r| r.applies_to(&sr.purl)).is_none()) @@ -3663,6 +3677,7 @@ fn get_download_params(args: &GetArgs, save_only: bool, persist_blobs: bool) -> strict: args.common.strict, ecosystems: args.common.ecosystems.clone(), persist_blobs, + patch_server_url: args.common.patch_server_url.clone(), } } @@ -5223,6 +5238,7 @@ mod tests { strict: false, ecosystems: None, persist_blobs: false, + patch_server_url: None, } } diff --git a/crates/socket-patch-cli/src/commands/repair.rs b/crates/socket-patch-cli/src/commands/repair.rs index 4fb8ed4d0..d85b86c9b 100644 --- a/crates/socket-patch-cli/src/commands/repair.rs +++ b/crates/socket-patch-cli/src/commands/repair.rs @@ -64,18 +64,15 @@ pub async fn run(args: RepairArgs) -> i32 { let mut vendor_references: Option> = None; if tokio::fs::metadata(&manifest_path).await.is_err() { - // Hosted (redirect) mode leaves no local artifacts to repair: the - // lockfiles point at patch.socket.dev URLs, not `.socket/vendor/...`, - // and there is no manifest or vendor ledger. A project whose only - // trace is `redirect-state.json` is therefore a no-op for repair — - // exit success with an informational skip rather than the - // `manifest_not_found` error a bare directory would get. Only cheap - // existence probes (and the read-only lockfile scan) run before the - // lock, so a project with nothing to repair never grows `.socket/`. - let redirect_state = args - .common - .cwd - .join(socket_patch_core::patch::redirect::REDIRECT_STATE_REL); + // Hosted mode leaves no local artifacts to repair: the lockfiles + // point at patch.socket.dev URLs, not `.socket/vendor/...`, and + // there is no manifest or vendor ledger. A project whose only trace + // is its hosted lockfile pins (or a pre-v5 `redirect-state.json`) + // is therefore a no-op for repair — exit success with an + // informational skip rather than the `manifest_not_found` error a + // bare directory would get. Only cheap existence probes (and the + // read-only lockfile scans) run before the lock, so a project with + // nothing to repair never grows `.socket/`. let state_file = args .common .cwd @@ -88,7 +85,16 @@ pub async fn run(args: RepairArgs) -> i32 { vendor_references = Some(refs); } if !has_vendor_traces { - if tokio::fs::metadata(&redirect_state).await.is_ok() { + let legacy_ledger = args + .common + .cwd + .join(socket_patch_core::patch::redirect::REDIRECT_STATE_REL); + let hosted = tokio::fs::metadata(&legacy_ledger).await.is_ok() + || !socket_patch_core::patch::redirect::upstream::HostedPin::all( + &crate::commands::discover_wiring(&args.common, &args.common.cwd).await, + ) + .is_empty(); + if hosted { let msg = HOSTED_ONLY_REASON; if args.common.json { let mut env = Envelope::new(Command::Repair); diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index bf3cbbaf5..75e77f122 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -26,15 +26,6 @@ pub(crate) mod vlt; pub(crate) use vlt::rollback_heal as vlt_rollback_heal; pub(crate) use vlt::takeover_heal as vlt_takeover_heal; -/// Fragment-edit kinds whose lockfile the package manager re-lays in place -/// (keeping the Socket source) — a re-scan REBASES their ledger edits instead -/// of appending; see the ledger merge below. -pub(crate) const REBASE_KINDS: &[&str] = &[ - "redirect_poetry_lock_package", - "redirect_pdm_lock_package", - socket_patch_core::patch::redirect::vlt::KIND, -]; - /// Candidate lockfiles / registry configs the redirect rewriters may touch — /// read from the project when present and handed to `rewrite_registry_redirect`. pub(crate) const REDIRECT_CANDIDATE_FILES: &[&str] = &[ @@ -1213,7 +1204,7 @@ pub(crate) async fn run_redirect_selected( ) -> i32 { use socket_patch_core::manifest::schema::PatchRecord; use socket_patch_core::patch::redirect::{ - rewrite_registry_redirect_withholding_vlt, RedirectState, + rewrite_registry_redirect_withholding_vlt, }; let mut skipped: Vec = Vec::new(); @@ -1425,11 +1416,9 @@ pub(crate) async fn run_redirect_selected( // v5 hosted mode keeps no ledger: the lockfiles are the only record of // a redirect (vex, list, vendor and rollback read the hosted pins from - // them). This run's edits and records still collect here in memory, - // for the edit rebasing below, the stale-install probes and the - // takeover classification. A pre-v5 ledger on disk is left untouched: - // it only goes stale, and replaying a stale edit fails closed. - let mut ledger = RedirectState::new(); + // them). This run's patch records (fetched below) feed only the + // stale-install probes and the in-run VEX attestation. A pre-v5 ledger + // on disk is left untouched: it is read only for migration. // The vendored ledger, loaded ONCE per run (under the same lock, so no // other writer can move the on-disk file under it): the takeover below // mutates it in place per reverted purl (saving after each), and the @@ -2781,8 +2770,8 @@ pub(crate) async fn run_redirect_selected( "code": "record_fetch_failed", "detail": format!( "{purl} redirected, but its patch record could not be fetched; \ - it will be missing from VEX until `socket-patch scan --mode \ - hosted` is re-run" + this run's VEX attestation omits it (`socket-patch vex` \ + fetches it again once the API answers)" ), })); } @@ -2792,109 +2781,6 @@ pub(crate) async fn run_redirect_selected( } if !common.dry_run { - // Fold this run's edits and records into the in-memory ledger. - // New edits append, skipping byte-identical re-plans; records are - // keyed by purl, newest wins. - if !rewrite.edits.is_empty() || !records.is_empty() { - // Older ledgers carry `"mode": "redirect"`; normalize on rewrite - // (the loader accepts either). - ledger.mode = "hosted".to_string(); - // REBASE instead of append for fragment kinds whose file the - // package manager itself rewrites in place: when the ledger already - // holds edits for the same (path, kind, key) and the file no longer - // carried their `new` fragments before this run (Poetry 1.1/1.2 - // `poetry lock --no-update` keeps the Socket source but re-lays the - // unit and drops the inserted `files` line), appending this run's - // edits — recorded against the RELOCKED text — would build a chain - // whose older links match nothing, so rollback and remove refuse - // forever. Keeping the oldest `original` (the pristine - // fragment) and adopting the fresh `new` keeps the chain a single - // invertible link: replay swaps the fragment this run wrote back to - // the fragment the very first run found. - let vlt_merged = rebase_vlt_edits( - &mut ledger.edits, - &rewrite.edits, - files - .get(socket_patch_core::constants::npm_family::VLT_LOCK) - .map(String::as_str), - ); - let mut rebased: Vec = Vec::new(); - for edit in rewrite.edits.iter().filter(|e| { - REBASE_KINDS.contains(&e.kind.as_str()) - && e.kind != socket_patch_core::patch::redirect::vlt::KIND - }) { - let siblings: Vec = ledger - .edits - .iter() - .enumerate() - .filter(|(_, old)| { - old.path == edit.path && old.kind == edit.kind && old.key == edit.key - }) - .map(|(i, _)| i) - .collect(); - let before = files.get(&edit.path).map(String::as_str).unwrap_or(""); - let drifted = !siblings.is_empty() - && siblings.iter().all(|&i| { - ledger.edits[i] - .new - .as_ref() - .and_then(serde_json::Value::as_str) - .is_none_or(|new| !before.contains(new)) - }); - if !drifted { - continue; - } - // Positional pairing: the rewriter emits a key's fragments in a - // fixed order (package unit, then the legacy integrity entry). - let nth = rewrite - .edits - .iter() - .filter(|e| e.path == edit.path && e.kind == edit.kind && e.key == edit.key) - .position(|e| std::ptr::eq(e, edit)) - .unwrap_or(0); - if let Some(&target) = siblings.get(nth) { - if !rebased.contains(&target) { - // `pdm lock` fully un-patches the lock (registry source - // restored) and may reflow line endings (CRLF → LF), so - // the fresh run's `original` IS the correct - // relocked-registry rollback target and the stale - // recorded one would restore a mismatched fragment. - // Poetry's relock instead KEEPS the Socket source (it - // only drops the inserted `files` line), so its oldest - // `original` — the true pre-patch fragment — must - // survive; only its `new` is refreshed. - if edit.kind == "redirect_pdm_lock_package" { - ledger.edits[target].original = edit.original.clone(); - } - ledger.edits[target].new = edit.new.clone(); - ledger.edits[target].action = edit.action.clone(); - rebased.push(target); - } - } - } - // Dedup against the ledger as this run found it, never within - // this run: one run legitimately records identical edits (a - // Cargo.toml declaring the crate with the same line in two - // sections), and each one reverts one occurrence. - let recorded = ledger.edits.len(); - for (i, edit) in rewrite.edits.iter().enumerate() { - if vlt_merged[i] { - continue; - } - let is_rebased = REBASE_KINDS.contains(&edit.kind.as_str()) - && rebased.iter().any(|&t| { - let old = &ledger.edits[t]; - old.path == edit.path - && old.kind == edit.kind - && old.key == edit.key - && old.new == edit.new - }); - if !is_rebased && !ledger.edits[..recorded].contains(edit) { - ledger.edits.push(edit.clone()); - } - } - ledger.records.extend(records); - } for (rel, content) in rewrite .files .iter() @@ -2951,7 +2837,7 @@ pub(crate) async fn run_redirect_selected( common.global, common.global_prefix.clone(), &confirmed, - &ledger.records, + &records, &gem_artifact_shas, ) .await @@ -2963,7 +2849,7 @@ pub(crate) async fn run_redirect_selected( common, &confirmed, &rewrite.confirmed_pipenv_uuids, - &ledger.records, + &records, ) .await }; @@ -2985,7 +2871,7 @@ pub(crate) async fn run_redirect_selected( .or_else(|| files.get(lock_key)) .map(String::as_str), preflight: &vlt_preflight, - records: &ledger.records, + records: &records, confirmed: &confirmed, confirmed_vlt: &rewrite.confirmed_vlt_uuids, foreign: &rewrite.vlt_foreign_uuids, @@ -3594,91 +3480,6 @@ fn format_next_steps(files: &[String], vendored_removed: bool) -> Vec { steps } -/// Merge this run's vlt node edits into the recorded ones. A fresh edit -/// for the same `key` and DepID keeps the oldest recorded `original` (the -/// pristine registry entry), takes the fresh `new` and drops the chain's -/// later links (a server-written ledger appends one per hosted PR). One -/// whose recorded same-key edits all name DepIDs the pre-run lock no longer -/// holds (a re-lock, or a new id grammar after a vlt upgrade) replaces -/// them. So does one for another key of the same `name@version` whose -/// vanished recorded edit's pin vlt carried to the fresh DepID (a new peer -/// context). A replacing edit keeps the recorded pristine slots when vlt -/// carried the pin ([`carried_pin_original`]). Returns, per fresh edit, -/// whether it was merged (anything else is appended as usual). -pub(crate) fn rebase_vlt_edits( - ledger: &mut Vec, - fresh: &[socket_patch_core::patch::redirect::FileEdit], - before_lock: Option<&str>, -) -> Vec { - use socket_patch_core::patch::redirect::vlt::{ - carried_pin_original, edit_dep_id, lock_node_ids, KIND, - }; - use socket_patch_core::patch::redirect::FileEdit; - fn superseding(edit: &FileEdit, old: &FileEdit) -> FileEdit { - let mut next = edit.clone(); - if let Some(original) = carried_pin_original(edit, old) { - next.original = Some(original); - } - next - } - fn key_base(key: &Option) -> Option<&str> { - key.as_deref() - .map(|k| k.split_once('~').map_or(k, |(base, _)| base)) - } - let live = before_lock.and_then(lock_node_ids).unwrap_or_default(); - let mut merged = vec![false; fresh.len()]; - for (i, edit) in fresh.iter().enumerate() { - if edit.kind != KIND { - continue; - } - let id = edit_dep_id(edit); - let same_key: Vec = ledger - .iter() - .enumerate() - .filter(|(_, old)| old.kind == KIND && old.path == edit.path && old.key == edit.key) - .map(|(j, _)| j) - .collect(); - let same_dep: Vec = same_key - .iter() - .copied() - .filter(|&j| id.is_some() && edit_dep_id(&ledger[j]) == id) - .collect(); - if let Some((&first, rest)) = same_dep.split_first() { - ledger[first].new = edit.new.clone(); - ledger[first].action = edit.action.clone(); - for &j in rest.iter().rev() { - ledger.remove(j); - } - merged[i] = true; - continue; - } - let vanished = |j: &usize| edit_dep_id(&ledger[*j]).is_none_or(|old| !live.contains(&old)); - let gone: Vec = same_key.into_iter().filter(vanished).collect(); - if let Some((&first, rest)) = gone.split_first() { - ledger[first] = superseding(edit, &ledger[first]); - for &j in rest.iter().rev() { - ledger.remove(j); - } - merged[i] = true; - continue; - } - let rekeyed = (0..ledger.len()).find(|j| { - let old = &ledger[*j]; - old.kind == KIND - && old.path == edit.path - && old.key != edit.key - && key_base(&old.key) == key_base(&edit.key) - && vanished(j) - && carried_pin_original(edit, old).is_some() - }); - if let Some(j) = rekeyed { - ledger[j] = superseding(edit, &ledger[j]); - merged[i] = true; - } - } - merged -} - /// Transient-frame boxed constructor for [`run_redirect_selected`] — the /// future embeds the whole hosted engine, and callers outside scan (`get /// --mode hosted`) must not materialize it in their own poll frame (Windows @@ -3723,7 +3524,6 @@ mod tests { pnpm_lock_may_need_store_flag, pnpm_trust_rerun_reminder, sentence_case, split_sentences, wrap_tokens, wrap_words, TAKEOVER_INFO_CODES, }; - use super::{rebase_vlt_edits, REBASE_KINDS}; use super::{wheel_metadata_concurrency, WHEEL_METADATA_CONCURRENCY}; use socket_patch_core::constants::npm_family; use socket_patch_core::patch::redirect::{DepOverride, FileEdit}; @@ -5352,175 +5152,6 @@ mod tests { ); } - fn vlt_edit(key: &str, id: &str, slot2: &str, slot3: &str) -> FileEdit { - FileEdit { - path: "vlt-lock.json".into(), - kind: socket_patch_core::patch::redirect::vlt::KIND.into(), - action: "rewritten".into(), - key: Some(key.into()), - original: Some(serde_json::Value::String(format!( - "\"{id}\": [0,\"x\",\"sha512-reg\",null]" - ))), - new: Some(serde_json::Value::String(format!( - "\"{id}\": [0,\"x\",\"{slot2}\",\"{slot3}\"]" - ))), - } - } - - fn vlt_lock_with(ids: &[&str]) -> String { - let nodes: Vec = ids - .iter() - .map(|id| format!(" \"{id}\": [0,\"x\"]")) - .collect(); - format!( - "{{\n \"lockfileVersion\": 1,\n \"nodes\": {{\n{}\n }},\n \"edges\": {{}}\n}}\n", - nodes.join(",\n") - ) - } - - #[test] - fn vlt_rerun_keeps_the_pristine_original_for_the_same_dep_id() { - assert!(REBASE_KINDS.contains(&socket_patch_core::patch::redirect::vlt::KIND)); - let mut ledger = vec![vlt_edit("x@1.0.0", "~npm~x@1.0.0", "sha512-p1", "u1")]; - let mut fresh = vlt_edit("x@1.0.0", "~npm~x@1.0.0", "sha512-p2", "u2"); - fresh.original = ledger[0].new.clone(); - let merged = rebase_vlt_edits( - &mut ledger, - std::slice::from_ref(&fresh), - Some(&vlt_lock_with(&["~npm~x@1.0.0"])), - ); - assert_eq!(merged, [true]); - assert_eq!(ledger.len(), 1); - assert_eq!( - ledger[0].original, - vlt_edit("x@1.0.0", "~npm~x@1.0.0", "", "").original - ); - assert_eq!(ledger[0].new, fresh.new); - } - - #[test] - fn vlt_relocked_dep_id_supersedes_the_recorded_edits() { - let mut ledger = vec![ - vlt_edit("x@1.0.0", "··x@1.0.0", "sha512-p", "u"), - vlt_edit("x@1.0.0", "·npm·x@1.0.0", "sha512-p", "u"), - vlt_edit("y@1.0.0", "·npm·y@1.0.0", "sha512-p", "u"), - ]; - let fresh = vlt_edit("x@1.0.0", "~npm~x@1.0.0", "sha512-p", "u"); - let merged = rebase_vlt_edits( - &mut ledger, - std::slice::from_ref(&fresh), - Some(&vlt_lock_with(&["~npm~x@1.0.0", "·npm·y@1.0.0"])), - ); - assert_eq!(merged, [true]); - assert_eq!( - ledger, - [fresh, vlt_edit("y@1.0.0", "·npm·y@1.0.0", "sha512-p", "u")] - ); - } - - fn vlt_pinned_at(edit: &FileEdit, id: &str) -> Option { - let new = edit.new.as_ref()?.as_str()?; - let (_, tuple) = new.split_once(": ")?; - Some(serde_json::Value::String(format!("\"{id}\": {tuple}"))) - } - - #[test] - fn vlt_rerun_collapses_a_server_appended_chain_into_one_link() { - let first = vlt_edit("x@1.0.0", "~npm~x@1.0.0", "sha512-pa", "ua"); - let mut second = vlt_edit("x@1.0.0", "~npm~x@1.0.0", "sha512-pb", "ub"); - second.original = first.new.clone(); - let mut ledger = vec![first, second.clone()]; - let mut fresh = vlt_edit("x@1.0.0", "~npm~x@1.0.0", "sha512-pc", "uc"); - fresh.original = second.new.clone(); - let merged = rebase_vlt_edits( - &mut ledger, - std::slice::from_ref(&fresh), - Some(&vlt_lock_with(&["~npm~x@1.0.0"])), - ); - assert_eq!(merged, [true]); - assert_eq!( - ledger, - [FileEdit { - original: vlt_edit("x@1.0.0", "~npm~x@1.0.0", "", "").original, - ..fresh - }] - ); - } - - #[test] - fn vlt_rerun_after_a_peer_context_rekey_keeps_the_pristine_slots() { - let old_id = "~npm~x@1.0.0~peer.0df72515a50372ba"; - let new_id = "~npm~x@1.0.0~peer.32643a3290c32d5d"; - let recorded = vlt_edit("x@1.0.0~peer.0df72515a50372ba", old_id, "sha512-p1", "u1"); - let mut ledger = vec![ - vlt_edit("y@1.0.0", "~npm~y@1.0.0", "sha512-p", "u"), - recorded.clone(), - ]; - let mut fresh = vlt_edit("x@1.0.0~peer.32643a3290c32d5d", new_id, "sha512-p2", "u2"); - fresh.original = vlt_pinned_at(&recorded, new_id); - let merged = rebase_vlt_edits( - &mut ledger, - std::slice::from_ref(&fresh), - Some(&vlt_lock_with(&[new_id, "~npm~y@1.0.0"])), - ); - assert_eq!(merged, [true]); - assert_eq!(ledger.len(), 2); - assert_eq!(ledger[1].key, fresh.key); - assert_eq!(ledger[1].new, fresh.new); - assert_eq!( - ledger[1].original, - Some(serde_json::Value::String(format!( - "\"{new_id}\": [0,\"x\",\"sha512-reg\"]" - ))) - ); - - let mut pristine = vec![recorded.clone()]; - let relocked = vlt_edit("x@1.0.0~peer.32643a3290c32d5d", new_id, "sha512-p2", "u2"); - let merged = rebase_vlt_edits( - &mut pristine, - std::slice::from_ref(&relocked), - Some(&vlt_lock_with(&[new_id])), - ); - assert_eq!(merged, [false], "a re-lock that dropped the pin appends"); - assert_eq!(pristine, [recorded]); - } - - #[test] - fn vlt_relocked_dep_id_that_kept_the_pin_keeps_the_pristine_slots() { - let recorded = vlt_edit("x@1.0.0", "·npm·x@1.0.0", "sha512-p1", "u1"); - let mut ledger = vec![recorded.clone()]; - let mut fresh = vlt_edit("x@1.0.0", "~npm~x@1.0.0", "sha512-p2", "u2"); - fresh.original = vlt_pinned_at(&recorded, "~npm~x@1.0.0"); - let merged = rebase_vlt_edits( - &mut ledger, - std::slice::from_ref(&fresh), - Some(&vlt_lock_with(&["~npm~x@1.0.0"])), - ); - assert_eq!(merged, [true]); - assert_eq!( - ledger, - [FileEdit { - original: Some(serde_json::Value::String( - "\"~npm~x@1.0.0\": [0,\"x\",\"sha512-reg\"]".into() - )), - ..fresh - }] - ); - } - - #[test] - fn vlt_edit_of_a_live_sibling_dep_id_is_appended() { - let mut ledger = vec![vlt_edit("x@1.0.0", "··x@1.0.0", "sha512-p", "u")]; - let fresh = vlt_edit("x@1.0.0", "·npm·x@1.0.0", "sha512-p", "u"); - let merged = rebase_vlt_edits( - &mut ledger, - std::slice::from_ref(&fresh), - Some(&vlt_lock_with(&["··x@1.0.0", "·npm·x@1.0.0"])), - ); - assert_eq!(merged, [false]); - assert_eq!(ledger.len(), 1); - } - #[test] fn next_steps_after_a_takeover_name_the_removed_vendored_state() { assert_eq!( diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index 30d15bd87..e2c83a110 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -787,6 +787,7 @@ fn download_params(args: &ScanArgs, save_only: bool, json: bool, silent: bool) - strict: args.common.strict, ecosystems: args.common.ecosystems.clone(), persist_blobs: args.mode != Some(ScanMode::Vendored), + patch_server_url: args.common.patch_server_url.clone(), } } diff --git a/crates/socket-patch-cli/src/hosted_memory/ledger.rs b/crates/socket-patch-cli/src/hosted_memory/ledger.rs deleted file mode 100644 index 2961d107d..000000000 --- a/crates/socket-patch-cli/src/hosted_memory/ledger.rs +++ /dev/null @@ -1,187 +0,0 @@ -//! The redirect ledger (`.socket/vendor/redirect-state.json`) in memory: -//! loaded strictly (a malformed ledger is a project error, never a fresh -//! start), merged exactly like the disk flow (edits appended unless already -//! recorded, `REBASE_KINDS` rebased, records extended newest-wins), and -//! serialized with the disk writer's bytes (`to_vec_pretty` + `\n`). - -use std::collections::BTreeMap; - -use socket_patch_core::manifest::schema::PatchRecord; -use socket_patch_core::patch::redirect::{ - CorruptRedirectState, FileEdit, RedirectState, REDIRECT_STATE_REL, -}; -use socket_patch_core::vendor::lock_inventory::{MemoryEntry, MemoryProject}; - -use crate::commands::scan::hosted::{rebase_vlt_edits, REBASE_KINDS}; - -/// Load the project's ledger: `Ok(None)` when absent, `Err` (the disk -/// message) when present but unreadable or malformed. -pub(crate) fn load(project: &MemoryProject, root: &str) -> Result, String> { - let path = super::roots::join_root(root, REDIRECT_STATE_REL); - let corrupt = |detail: String, unreadable: bool| { - CorruptRedirectState { - path: path.clone().into(), - detail, - quarantined_to: None, - unreadable, - } - .to_string() - }; - let bytes: &[u8] = match project.get(REDIRECT_STATE_REL) { - None => return Ok(None), - Some(MemoryEntry::Text(text)) => text.as_bytes(), - Some(MemoryEntry::Binary(bytes)) => bytes, - Some(MemoryEntry::Present) => { - return Err(corrupt("file content was not provided".into(), true)) - } - Some(MemoryEntry::Symlink) => return Err(corrupt("is a symbolic link".into(), true)), - }; - serde_json::from_slice(bytes) - .map(Some) - .map_err(|e| corrupt(format!("invalid JSON: {e}"), false)) -} - -/// Merge this run's `edits` and `records` into `ledger` (the disk flow's -/// merge, verbatim). `files` are the pre-rewrite candidate contents the -/// rebase drift check reads. -pub(crate) fn merge( - ledger: &mut RedirectState, - edits: &[FileEdit], - records: BTreeMap, - files: &BTreeMap, -) { - ledger.mode = "hosted".to_string(); - let vlt_merged = rebase_vlt_edits( - &mut ledger.edits, - edits, - files - .get(socket_patch_core::constants::npm_family::VLT_LOCK) - .map(String::as_str), - ); - let mut rebased: Vec = Vec::new(); - for edit in edits.iter().filter(|e| { - REBASE_KINDS.contains(&e.kind.as_str()) - && e.kind != socket_patch_core::patch::redirect::vlt::KIND - }) { - let siblings: Vec = ledger - .edits - .iter() - .enumerate() - .filter(|(_, old)| { - old.path == edit.path && old.kind == edit.kind && old.key == edit.key - }) - .map(|(i, _)| i) - .collect(); - let before = files.get(&edit.path).map(String::as_str).unwrap_or(""); - let drifted = !siblings.is_empty() - && siblings.iter().all(|&i| { - ledger.edits[i] - .new - .as_ref() - .and_then(serde_json::Value::as_str) - .is_none_or(|new| !before.contains(new)) - }); - if !drifted { - continue; - } - let nth = edits - .iter() - .filter(|e| e.path == edit.path && e.kind == edit.kind && e.key == edit.key) - .position(|e| std::ptr::eq(e, edit)) - .unwrap_or(0); - if let Some(&target) = siblings.get(nth) { - if !rebased.contains(&target) { - if edit.kind == "redirect_pdm_lock_package" { - ledger.edits[target].original = edit.original.clone(); - } - ledger.edits[target].new = edit.new.clone(); - ledger.edits[target].action = edit.action.clone(); - rebased.push(target); - } - } - } - let recorded = ledger.edits.len(); - for (i, edit) in edits.iter().enumerate() { - if vlt_merged[i] { - continue; - } - let is_rebased = REBASE_KINDS.contains(&edit.kind.as_str()) - && rebased.iter().any(|&t| { - let old = &ledger.edits[t]; - old.path == edit.path - && old.kind == edit.kind - && old.key == edit.key - && old.new == edit.new - }); - if !is_rebased && !ledger.edits[..recorded].contains(edit) { - ledger.edits.push(edit.clone()); - } - } - ledger.records.extend(records); -} - -/// The ledger's on-disk bytes. -pub(crate) fn serialize(ledger: &RedirectState) -> Result { - let mut bytes = serde_json::to_vec_pretty(ledger).map_err(|e| e.to_string())?; - bytes.push(b'\n'); - String::from_utf8(bytes).map_err(|e| e.to_string()) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn edit(kind: &str, new: &str) -> FileEdit { - FileEdit { - path: "poetry.lock".into(), - kind: kind.into(), - action: "replaced".into(), - key: Some("k".into()), - original: Some(serde_json::json!("orig")), - new: Some(serde_json::json!(new)), - } - } - - #[test] - fn corrupt_and_absent_ledgers() { - let mut p = MemoryProject::new(); - assert!(load(&p, "").unwrap().is_none()); - p.insert_text(REDIRECT_STATE_REL, "{not json"); - let err = load(&p, "sub").unwrap_err(); - assert!( - err.contains("sub/.socket/vendor/redirect-state.json"), - "{err}" - ); - assert!(err.contains("malformed"), "{err}"); - p.insert_symlink(REDIRECT_STATE_REL); - assert!(load(&p, "").unwrap_err().contains("cannot be read")); - } - - #[test] - fn merge_appends_new_edits_and_rebases_drifted_fragments() { - let mut ledger = RedirectState::new(); - ledger.edits.push(edit("redirect_npm_lock_entry", "a")); - ledger - .edits - .push(edit("redirect_poetry_lock_package", "old-new")); - let files = BTreeMap::from([("poetry.lock".to_string(), "relocked".to_string())]); - merge( - &mut ledger, - &[ - edit("redirect_npm_lock_entry", "a"), - edit("redirect_npm_lock_entry", "b"), - edit("redirect_poetry_lock_package", "fresh"), - ], - BTreeMap::new(), - &files, - ); - let news: Vec<&str> = ledger - .edits - .iter() - .map(|e| e.new.as_ref().and_then(|v| v.as_str()).unwrap()) - .collect(); - assert_eq!(news, vec!["a", "fresh", "b"]); - let text = serialize(&ledger).unwrap(); - assert!(text.ends_with("}\n")); - } -} diff --git a/crates/socket-patch-cli/src/hosted_memory/mod.rs b/crates/socket-patch-cli/src/hosted_memory/mod.rs index b5f8b0f3d..7d2e9e2c7 100644 --- a/crates/socket-patch-cli/src/hosted_memory/mod.rs +++ b/crates/socket-patch-cli/src/hosted_memory/mod.rs @@ -2,7 +2,8 @@ //! filesystem, no subprocesses, no environment reads, no telemetry. Every //! patch lookup goes through the caller's [`PatchApi`]; the caller hands //! in the repository's candidate files (chosen by [`select_paths`]) and -//! gets back the changed files, ledger included. +//! gets back the changed files (v5 hosted mode keeps no ledger: the +//! rewritten lockfiles are the whole record). //! //! Per project root the result matches `scan --mode hosted --json` over a //! checkout holding the same files (the parity tests hold the two paths to @@ -40,8 +41,6 @@ use std::time::Instant; use socket_patch_core::api::client::PatchApi; use socket_patch_core::api::types::{PatchResponse, PatchSearchResult}; use socket_patch_core::crawlers::Ecosystem; -use socket_patch_core::manifest::schema::PatchRecord; -use socket_patch_core::patch::redirect::{RedirectState, REDIRECT_STATE_REL}; use socket_patch_core::utils::cargo_workspace::member_manifests_in; use socket_patch_core::vendor::lock_inventory::{ inventory_project_diagnosed_in, MemoryEntry, MemoryProject, ProjectView, @@ -49,7 +48,6 @@ use socket_patch_core::vendor::lock_inventory::{ use tokio_util::sync::CancellationToken; pub(crate) mod discover; -pub(crate) mod ledger; pub mod limits; pub(crate) mod redirect; pub(crate) mod roots; @@ -110,7 +108,6 @@ struct RootState { /// (oversize, LFS pointers, presence-only): the disk flow would read /// them, so a rewrite that depends on one is refused. unreadable: BTreeSet, - ledger: Option, purls: Vec, summary: ProjectSummary, packages: Vec, @@ -405,7 +402,6 @@ async fn engine( root: root.clone(), project: Some(project), unreadable, - ledger: None, purls: Vec::new(), summary: ProjectSummary::default(), packages: Vec::new(), @@ -423,13 +419,6 @@ async fn engine( let Some(project) = state.project.as_ref() else { continue; }; - match ledger::load(project, &state.root) { - Ok(loaded) => state.ledger = loaded, - Err(message) => { - state.fail("corrupt_ledger", message); - continue; - } - } let (entries, unsupported) = inventory_project_diagnosed_in(&ProjectView::Memory(project)).await; for (code, detail) in crate::commands::scan::unsupported_layout_warnings(&unsupported) { @@ -711,7 +700,7 @@ async fn engine( }) } -/// Records → ledger merge → the project's result and changed files. +/// Records → the project's result and changed files. fn finish_root( state: &mut RootState, done: Rewritten, @@ -731,22 +720,19 @@ fn finish_root( npm_warnings, } = done; let root = state.root.clone(); - let mut record_map: BTreeMap = BTreeMap::new(); + // No ledger keeps the records; the fetch mirrors the disk flow's, so a + // record the API cannot serve warns the same way. let mut record_warnings: Vec = Vec::new(); if !dry_run { for (purl, uuid) in &confirmed { match records.get(uuid) { - Some(Some(response)) => { - let (rec_purl, record) = - crate::commands::get::record_from_patch_response(response); - record_map.insert(rec_purl, record); - } + Some(Some(_)) => {} _ => record_warnings.push(serde_json::json!({ "code": "record_fetch_failed", "detail": format!( "{purl} redirected, but its patch record could not be fetched; \ - it will be missing from VEX until `socket-patch scan --mode \ - hosted` is re-run" + this run's VEX attestation omits it (`socket-patch vex` \ + fetches it again once the API answers)" ), })), } @@ -754,34 +740,6 @@ fn finish_root( } let mut project_changes: Vec<(String, String)> = Vec::new(); - let mut ledger_error: Option = None; - if !dry_run && (!rewrite.edits.is_empty() || !record_map.is_empty()) { - let mut ledger = state.ledger.take().unwrap_or_default(); - ledger::merge(&mut ledger, &rewrite.edits, record_map, &planned.files); - match ledger::serialize(&ledger) { - Ok(text) => { - if planned.project.text(REDIRECT_STATE_REL) != Some(text.as_str()) { - project_changes.push((REDIRECT_STATE_REL.to_string(), text)); - } - } - Err(message) => { - ledger_error = Some(ProjectError { - code: "ledger_serialize_failed".into(), - message, - }) - } - } - } - if let Some(error) = ledger_error { - return ProjectResult { - root, - redirect: serde_json::json!({ "mode": "hosted" }), - summary: state.summary.clone(), - redirected: Vec::new(), - skipped: planned.skipped, - error: Some(error), - }; - } for (rel, content) in &rewrite.files { if planned.project.text(rel) != Some(content.as_str()) { project_changes.push((rel.clone(), content.clone())); @@ -894,7 +852,6 @@ mod tests { root: root.to_string(), project: Some(project), unreadable: BTreeSet::new(), - ledger: None, purls: Vec::new(), summary: ProjectSummary::default(), packages: Vec::new(), diff --git a/crates/socket-patch-cli/src/hosted_memory/select.rs b/crates/socket-patch-cli/src/hosted_memory/select.rs index 67a4b4b74..b7d939af9 100644 --- a/crates/socket-patch-cli/src/hosted_memory/select.rs +++ b/crates/socket-patch-cli/src/hosted_memory/select.rs @@ -2,7 +2,7 @@ //! per root, the same root-relative candidate set the disk hosted flow //! reads (`REDIRECT_CANDIDATE_FILES`, Python lock / script pairs, Cargo //! member manifests, Rush locks, the install-policy configs, the -//! Plug'n'Play markers and the two `.socket/vendor/` ledgers), plus one +//! Plug'n'Play markers and the vendored ledger), plus one //! presence-only Maven / NuGet marker per ecosystem so a repo holding only //! those still gets its `ecosystem_unsupported_in_memory` warning. @@ -12,7 +12,6 @@ use socket_patch_core::constants::npm_family::{ BUN_LOCKB, PNP_MARKERS, RUSH_COMMON_LOCK_REL, RUSH_SUBSPACES_DIR, }; use socket_patch_core::patch::redirect::npmrc::NPMRC_REL; -use socket_patch_core::patch::redirect::REDIRECT_STATE_REL; use socket_patch_core::utils::python_lock::is_python_lock_name; use super::roots::{ @@ -37,12 +36,7 @@ pub(crate) const VENDOR_STATE_REL: &str = ".socket/vendor/state.json"; pub(crate) const RUSH_REPO_STATE_REL: &str = "common/config/rush/repo-state.json"; /// Root-relative text files read beyond `REDIRECT_CANDIDATE_FILES`. -const EXTRA_TEXT_FILES: [&str; 4] = [ - PNPM_WORKSPACE_REL, - NPMRC_REL, - VENDOR_STATE_REL, - REDIRECT_STATE_REL, -]; +const EXTRA_TEXT_FILES: [&str; 3] = [PNPM_WORKSPACE_REL, NPMRC_REL, VENDOR_STATE_REL]; /// The one directory name the disk Cargo member walk never enters (it /// follows `members`, `exclude`, path dependencies and `[patch]` paths @@ -363,7 +357,6 @@ mod tests { s.fetch_text, vec![ ".npmrc", - ".socket/vendor/redirect-state.json", "package-lock.json", "tool.py", "tool.py.lock", @@ -465,12 +458,14 @@ mod tests { } #[test] - fn candidate_listing_is_sorted_and_names_the_ledgers() { + fn candidate_listing_is_sorted_and_names_the_vendored_ledger_only() { let listed = candidate_files(); let mut sorted = listed.clone(); sorted.sort(); assert_eq!(listed, sorted); - assert!(listed + assert!(listed.iter().any(|f| f == ".socket/vendor/state.json")); + // v5 hosted mode keeps no ledger, so the pre-v5 one is never read. + assert!(!listed .iter() .any(|f| f == ".socket/vendor/redirect-state.json")); assert!(listed.iter().any(|f| f == "package-lock.json")); diff --git a/crates/socket-patch-core/src/vendor/cargo.rs b/crates/socket-patch-core/src/vendor/cargo.rs index 54ea86a63..ef93b9b24 100644 --- a/crates/socket-patch-core/src/vendor/cargo.rs +++ b/crates/socket-patch-core/src/vendor/cargo.rs @@ -784,25 +784,23 @@ async fn cargo_wet_preflight( uuid: &str, ) -> Result { // Cross-mode takeover guard (fail-closed): a LIVE hosted-redirect wiring - // for this crate must be reverted from the redirect ledger BEFORE - // vendoring — the CLI vendored flows do exactly that. Reaching this point - // with the residue still present means the redirect ledger is missing or - // corrupt (no recorded originals to revert with); proceeding would bake + // for this crate must be restored to its crates.io entry BEFORE + // vendoring — the CLI vendored flows do exactly that (the upstream + // restore). Reaching this point with the residue still present means + // that restore did not run or could not undo it; proceeding would bake // the hosted registry values into this entry's lock originals as if they // were pristine, leave Cargo.toml pinned to the hosted registry, and // report success on an unbuildable half-migrated project. Refuse with the // manual remediation instead. Runs after the dry-run branch: a preview - // must not report the wet run's ledger-driven revert as a failure. + // must not report the wet run's restore as a failure. if let Some(residue) = hosted_redirect_residue(project_root, name, version).await { return Err(refused( "hosted_redirect_live", format!( - "{residue}, but no redirect ledger record can revert it \ - (.socket/vendor/redirect-state.json is missing or does not \ - record this package); restore the ledger, or manually remove \ - the `registry = \"socket-patch-…\"` key from Cargo.toml, \ - restore the crates.io source/checksum in Cargo.lock, and drop \ - the `[registries.socket-patch-…]` block, then re-run" + "{residue}, and it was not restored to its crates.io entry; run \ + `socket-patch rollback` for this package first, or restore \ + Cargo.toml and Cargo.lock from version control (`git checkout \ + -- Cargo.toml Cargo.lock`), then re-run" ), )); } From bd0f6d2b7155cf401ee01994f7798f7e69ef4d5e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 23:26:56 +0000 Subject: [PATCH 06/66] vendor ejects a hosted project; vendoring over any hosted pin restores upstream first WS2: standalone `vendor` with no manifest now takes its patch set from the lockfiles' hosted pins (purl + the uuid in each hosted URL), fetches each record from the API, vendors it into .socket/vendor/ through the same step `scan --mode vendored` and `get --mode vendored` use, and rewires the lock from hosted to vendored. Without hosted pins it keeps the no-manifest no-op. The vendor takeover now restores the upstream registry entry before vendoring for every ecosystem, not only cargo/npm/golang, so the vendor ledger always records the upstream entry as its original and `vendor --revert` returns to upstream rather than to hosted. A pin whose upstream entry cannot be restored is refused with the checkout remedy. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018tncU7u7FK48tubFNA12UQ --- crates/socket-patch-cli/src/commands/get.rs | 1 + .../socket-patch-cli/src/commands/vendor.rs | 191 +++++++++++++++++- .../tests/covgap_commands_get.rs | 1 + .../tests/in_process_get_update_count.rs | 1 + 4 files changed, 188 insertions(+), 6 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/get.rs b/crates/socket-patch-cli/src/commands/get.rs index 393c7b379..124015900 100644 --- a/crates/socket-patch-cli/src/commands/get.rs +++ b/crates/socket-patch-cli/src/commands/get.rs @@ -5907,6 +5907,7 @@ mod tests { ecosystems: None, // The vendor-detached posture this fn exists for. persist_blobs: false, + patch_server_url: None, } } diff --git a/crates/socket-patch-cli/src/commands/vendor.rs b/crates/socket-patch-cli/src/commands/vendor.rs index bcbf093b8..ed8394e24 100644 --- a/crates/socket-patch-cli/src/commands/vendor.rs +++ b/crates/socket-patch-cli/src/commands/vendor.rs @@ -26,6 +26,7 @@ use socket_patch_core::crawlers::{CrawlerOptions, Ecosystem}; use socket_patch_core::manifest::operations::{read_manifest, write_manifest}; use socket_patch_core::manifest::schema::{PatchManifest, PatchRecord}; use socket_patch_core::patch::apply::{verify_file_patch, PatchSources}; +use socket_patch_core::patch::redirect::upstream::HostedPin; use socket_patch_core::telemetry::{track_patch_vendor_failed, track_patch_vendored}; use socket_patch_core::utils::concurrent::{ordered_concurrent, registry_concurrency}; use socket_patch_core::utils::group_commit::GroupCommit; @@ -656,6 +657,14 @@ pub async fn run(args: VendorArgs) -> i32 { // vendored` projects have `.socket/` but never a manifest. Nothing is // locked or written on this path. if !args.revert && tokio::fs::metadata(&manifest_path).await.is_err() { + // A hosted project (no manifest, hosted pins in its lockfiles) + // ejects: its patch set is the lockfiles' hosted pins. + if !args.common.is_global() { + let pins = hosted_pins_in_scope(&args.common).await; + if !pins.is_empty() { + return run_eject(&args, pins).await; + } + } // A requested `--vex` still attests what the `.socket/vendor` // ledgers and lockfiles already wire. Same contract as `apply --vex` // with no manifest: nothing referenced anywhere keeps exit 0; any @@ -846,6 +855,180 @@ pub async fn run(args: VendorArgs) -> i32 { exit } +/// The lockfiles' hosted pins whose ecosystem `--ecosystems` selects. +async fn hosted_pins_in_scope(common: &GlobalArgs) -> Vec { + HostedPin::all(&crate::commands::discover_wiring(common, &common.cwd).await) + .into_iter() + .filter(|pin| { + socket_patch_core::utils::purl::purl_parts(&pin.purl) + .is_some_and(|(eco, _, _)| ecosystem_in_scope(common, &eco)) + }) + .collect() +} + +/// Standalone `vendor` in a hosted project — no manifest, hosted pins in the +/// lockfiles: EJECT. The patch set is the pins themselves (purl + the uuid +/// in each hosted URL); each record is fetched from the API, vendored into +/// `.socket/vendor/` exactly like `scan --mode vendored`, and the lock is +/// rewired from hosted to vendored (the engine's takeover restores each +/// pin's upstream registry entry first, so `vendor --revert` later returns +/// the project to upstream, not to hosted). +async fn run_eject(args: &VendorArgs, pins: Vec) -> i32 { + let common = &args.common; + let (client, use_public_proxy) = + get_api_client_with_overrides(common.api_client_overrides()).await; + let (api_token, org_slug) = (client.api_token().cloned(), client.org_slug().cloned()); + if !common.json && !common.silent { + println!( + "{} {} into .socket/vendor/...", + if common.dry_run { "Would eject" } else { "Ejecting" }, + plural(pins.len(), "hosted package", "hosted packages") + ); + } + + // One view per distinct uuid, fetched concurrently and consumed in pin + // order; the views' blobs seed the in-memory staging. + let mut records: HashMap = HashMap::new(); + let mut blobs: HashMap> = HashMap::new(); + let mut fetch_failures: Vec<(String, String)> = Vec::new(); + let mut views = std::pin::pin!(ordered_concurrent( + pins.iter(), + socket_patch_core::utils::concurrent::api_concurrency_for( + client.uses_public_proxy(), + pins.len(), + ), + |pin| { + let client = &client; + async move { client.fetch_patch(&pin.uuid).await } + }, + )); + for pin in &pins { + let Some(view) = views.next().await else { + break; + }; + match view { + Ok(Some(patch)) => { + for info in patch.files.values() { + let (Some(b64), Some(hash)) = (&info.blob_content, &info.after_hash) else { + continue; + }; + if !socket_patch_core::patch::apply::is_valid_blob_hash(hash) + || blobs.contains_key(hash) + { + continue; + } + if let Ok(bytes) = crate::commands::get::base64_decode(b64) { + blobs.insert(hash.clone(), bytes); + } + } + let (_, record) = crate::commands::get::record_from_patch_response(&patch); + records.insert(pin.purl.clone(), record); + } + Ok(None) => fetch_failures.push(( + pin.purl.clone(), + format!("patch {} was not found on the API", pin.uuid), + )), + Err(e) => fetch_failures.push(( + pin.purl.clone(), + format!("could not fetch patch {}: {e}", pin.uuid), + )), + } + } + drop(views); + for (purl, detail) in &fetch_failures { + report_vendor_failure(common, purl, detail); + } + + let step = crate::commands::scan::boxed_scan_vendor_step( + common, + records, + blobs, + client.clone(), + use_public_proxy, + ) + .await; + let (mut exit, mut env) = match step { + Ok((has_errors, env)) => (i32::from(has_errors), env), + Err((code, message, env)) => { + let mut env = env.map(|e| *e).unwrap_or_else(|| { + let mut env = Envelope::new(Command::Vendor); + env.dry_run = common.dry_run; + env + }); + env.mark_error(EnvelopeError::new(code, message.clone())); + if !common.json { + eprintln!( + "{}", + crate::commands::scan::vendor_flow::format_vendor_step_error(code, &message) + ); + } + (1, env) + } + }; + if !fetch_failures.is_empty() { + for (purl, detail) in fetch_failures { + env.record( + PatchEvent::new(PatchAction::Failed, purl).with_error("patch_fetch_failed", detail), + ); + } + env.mark_partial_failure(); + exit = 1; + } + + // Embedded VEX: same contract as the manifest-driven arm — only on + // success, never on a dry run, and a requested-but-failed VEX flips the + // exit code. The ejected project has no manifest. + if exit == 0 { + if let Some(vex_path) = args.vex.vex.as_ref() { + if common.dry_run { + if !common.json && !common.silent { + println!("{}", crate::commands::vex::format_vex_dry_run_skip("vendored")); + } + } else { + let params = args.vex.to_build_params(); + let manifest_path = common.resolved_manifest_path(); + match generate_vex_without_manifest(common, ¶ms, &manifest_path).await { + ManifestlessVex::Written(summary) => { + env.vex = Some(VexSummary { + path: vex_path.display().to_string(), + statements: summary.statements, + format: "openvex-0.2.0".to_string(), + warnings: summary.warnings, + }); + } + ManifestlessVex::NothingToAttest(warnings) => { + env.warnings.extend(warnings); + if !common.json && !common.silent { + println!("{}", crate::commands::vex::format_vex_nothing_to_attest()); + } + } + ManifestlessVex::Failed(e) => { + env.warnings.extend(e.embedded_warnings()); + env.mark_error(EnvelopeError::new(e.code, e.message.clone())); + if !common.json { + e.print_embedded(common); + } + exit = 1; + } + } + } + } + } + + if common.json { + println!("{}", env.to_pretty_json()); + } + track_outcomes_for_vendor( + exit != 0, + &env, + common.dry_run, + api_token.as_deref(), + org_slug.as_deref(), + ) + .await; + exit +} + /// The no-manifest warning when the vendor ledger cannot be read either. fn no_manifest_ledger_unreadable(err: &str) -> String { format!( @@ -1578,9 +1761,7 @@ async fn plan_service_downloads( if bun_refusal.is_some_and(|r| r.applies_to(candidate)) { continue; } - if socket_patch_core::patch::redirect::redirect_revert_supported(candidate) - && takeover_blocked(candidate) - { + if takeover_blocked(candidate) { continue; } // The npm backends re-wire a committed artifact the ledger @@ -2402,9 +2583,7 @@ pub(crate) async fn vendor_records_reusing( // vendor detach the PRISTINE registry entry to record. A purl // whose upstream entry cannot be restored is REFUSED; the cargo // backend's `hosted_redirect_live` guard backstops the rest. - let hosted_pin = hosted_pin_of(candidate) - .filter(|_| socket_patch_core::patch::redirect::redirect_revert_supported(candidate)); - if let Some(pin) = hosted_pin { + if let Some(pin) = hosted_pin_of(candidate) { // The refusal the berry backend would raise after the // restore, raised HERE instead — the same `failed` event, // code and detail, in the dry run and the wet run alike — diff --git a/crates/socket-patch-cli/tests/covgap_commands_get.rs b/crates/socket-patch-cli/tests/covgap_commands_get.rs index ba006e945..da232d432 100644 --- a/crates/socket-patch-cli/tests/covgap_commands_get.rs +++ b/crates/socket-patch-cli/tests/covgap_commands_get.rs @@ -294,6 +294,7 @@ fn engine_params(root: &Path) -> DownloadParams { strict: false, ecosystems: None, persist_blobs: true, + patch_server_url: None, all_releases: true, } } diff --git a/crates/socket-patch-cli/tests/in_process_get_update_count.rs b/crates/socket-patch-cli/tests/in_process_get_update_count.rs index cce2616f1..9ac8bed45 100644 --- a/crates/socket-patch-cli/tests/in_process_get_update_count.rs +++ b/crates/socket-patch-cli/tests/in_process_get_update_count.rs @@ -93,6 +93,7 @@ fn params(root: &Path) -> DownloadParams { strict: false, ecosystems: None, persist_blobs: true, + patch_server_url: None, // Skip release-narrowing; npm has no variants anyway. all_releases: true, } From 65d25c773ea26c0ea2090bd1073b96f2b0ecdd88 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 23:04:33 +0000 Subject: [PATCH 07/66] Rewrite scan/mod.rs takeover unit tests for lockfile-derived hosted state v5 hosted mode keeps no ledger, so the scan/mod.rs unit tests now make a purl "hosted" by writing a lockfile that pins the hosted URL instead of planting .socket/vendor/redirect-state.json: - overlap / classify_overlap_takeover tests use hosted package-lock, yarn (classic + berry), bun and cargo sparse-index pins; the non-default-host test configures --patch-server-url and pins that an unconfigured host is no pin. - New behavior pinned: a pre-v5 ledger on disk is never hosted state; a lock routed to vendored (npm or cargo) yields no pin and no overlap; an edits-only state names no package; a half-migrated project whose locks name both sides stays silent; the redirectState block has no ledger/ledgerKey fields. - hosted_wiring_retained_purls / redirect_state_json tests read the lockfile-derived state, keeping the probe's own liveness gate covered. - Removed tests of retired behavior: note_vendor_supersedes_redirect reconcile (wet/npmrc/dry-run/no-op/persist-failure), the edits-only fallback (degraded ledger, vlt tilde keys) and following the vendored remediation. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018tncU7u7FK48tubFNA12UQ --- .../socket-patch-cli/src/commands/scan/mod.rs | 1052 +++++------------ 1 file changed, 318 insertions(+), 734 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index e2c83a110..f95e9e348 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -2706,20 +2706,18 @@ mod tests { } /// The load-then-derive form of [`overlap_from_states`]: the unit - /// tests' entry point (production classifies over ledgers it already - /// holds via `classify_overlap_takeover_with`). A malformed redirect - /// ledger classifies like a missing one — this path only feeds takeover - /// WARNINGS; the corruption itself is a hard error on every path that - /// would write or attest from the ledger. - async fn overlapping_ledger_purls(cwd: &Path) -> Vec { - let redirect = socket_patch_core::patch::redirect::load_redirect_state(cwd) - .await - .ok() - .flatten(); + /// tests' entry point (production classifies over state it already + /// holds via `classify_overlap_takeover_with`). The hosted side is the + /// lockfiles' hosted pins — never a pre-v5 redirect ledger on disk — and + /// a malformed vendor ledger classifies like a missing one: this path + /// only feeds takeover WARNINGS; the corruption itself is a hard error on + /// every path that would write or attest from the ledger. + async fn overlapping_purls(common: &GlobalArgs, cwd: &Path) -> Vec { + let redirect = crate::commands::hosted_state_from_lockfiles(common, cwd).await; let Ok(vendor) = socket_patch_core::vendor::load_state(cwd).await else { return Vec::new(); }; - overlap_from_states(redirect.as_ref(), &vendor) + overlap_from_states(Some(&redirect), &vendor) } use socket_patch_core::manifest::schema::{PatchManifest, PatchRecord}; use std::collections::HashMap; @@ -2873,7 +2871,7 @@ mod tests { assert_eq!(crawl_scope(true, None), None); } - // ---- cross-mode ledger takeover (hosted ⇄ vendored) -------------------- + // ---- cross-mode takeover (hosted over vendored) ------------------------ const TAKEOVER_UUID: &str = "9f6b2c4e-1d3a-4f6b-8c2d-7e5a9b1c3d5f"; @@ -2889,8 +2887,10 @@ mod tests { } } - /// Write a hosted redirect ledger (`.socket/vendor/redirect-state.json`) - /// recording a redirect for each PURL. + /// Write a PRE-V5 hosted redirect ledger + /// (`.socket/vendor/redirect-state.json`) recording a redirect for each + /// PURL. v5 never writes one and never reads it for hosted state: the + /// tests plant it only to prove it is ignored. async fn write_redirect_ledger(root: &Path, purls: &[&str]) { use socket_patch_core::patch::redirect::RedirectState; let mut state = RedirectState::new(); @@ -2907,6 +2907,18 @@ mod tests { .unwrap(); } + /// An in-memory hosted state holding one [`takeover_record`] per PURL + /// under the given (possibly non-canonical) keys — the shape + /// [`crate::commands::hosted_state_from_pins`] builds, for the block + /// builder and the probe's own liveness gate. + fn pinned_state(purls: &[&str]) -> socket_patch_core::patch::redirect::RedirectState { + let mut state = socket_patch_core::patch::redirect::RedirectState::new(); + for purl in purls { + state.records.insert((*purl).to_string(), takeover_record()); + } + state + } + /// Write a vendored state ledger (`.socket/vendor/state.json`) with one /// entry per PURL, in the committed camelCase wire shape. async fn write_vendor_ledger(root: &Path, purls: &[&str]) { @@ -2939,43 +2951,93 @@ mod tests { } #[tokio::test] - async fn overlapping_ledgers_flag_the_taken_over_package() { - // Both ledgers claim minimist ⇒ one mode took the lockfile over from - // the other and the displaced ledger is stale. The detection names - // exactly the overlapping PURL. + async fn hosted_pin_over_a_vendored_entry_flags_the_taken_over_package() { + // The lock pins minimist to the hosted patch server while the vendored + // ledger still claims it ⇒ one mode took the lockfile over from the + // other. The detection names exactly the overlapping PURL. let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); - write_redirect_ledger(root, &["pkg:npm/minimist@1.2.2"]).await; + write_lock_pointing_at_hosted(root, "minimist", "1.2.2").await; write_vendor_ledger(root, &["pkg:npm/minimist@1.2.2"]).await; - let superseded = overlapping_ledger_purls(root).await; + let superseded = overlapping_purls(&common_at(root), root).await; assert_eq!(superseded, vec!["pkg:npm/minimist@1.2.2".to_string()]); } #[tokio::test] - async fn single_ledger_present_flags_nothing() { - // A first-time redirect (only the redirect ledger, no vendored ledger) - // displaces nothing — no warning. Guards against warning on the FIRST - // scan of a fresh project. + async fn single_side_present_flags_nothing() { + // A first-time redirect (a hosted pin, no vendored ledger) displaces + // nothing — no warning. Guards against warning on the FIRST scan of a + // fresh project. let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); - write_redirect_ledger(root, &["pkg:npm/minimist@1.2.2"]).await; - assert!(overlapping_ledger_purls(root).await.is_empty()); + write_lock_pointing_at_hosted(root, "minimist", "1.2.2").await; + assert!(overlapping_purls(&common_at(root), root).await.is_empty()); - // And a project with no ledgers at all. + // And a project with no lockfile and no ledgers at all. let tmp2 = tempfile::tempdir().unwrap(); - assert!(overlapping_ledger_purls(tmp2.path()).await.is_empty()); + assert!(overlapping_purls(&common_at(tmp2.path()), tmp2.path()) + .await + .is_empty()); } #[tokio::test] - async fn disjoint_ledgers_are_not_a_takeover() { - // A legitimate split — one package redirected, a DIFFERENT one - // vendored — is not a takeover: neither ledger's wiring is stale. + async fn disjoint_states_are_not_a_takeover() { + // A legitimate split — one package pinned hosted, a DIFFERENT one + // vendored — is not a takeover: neither side's wiring is stale. let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); - write_redirect_ledger(root, &["pkg:npm/minimist@1.2.2"]).await; + write_lock_pointing_at_hosted(root, "minimist", "1.2.2").await; write_vendor_ledger(root, &["pkg:npm/lodash@4.17.21"]).await; - assert!(overlapping_ledger_purls(root).await.is_empty()); + assert!(overlapping_purls(&common_at(root), root).await.is_empty()); + } + + #[tokio::test] + async fn legacy_redirect_ledger_is_not_hosted_state() { + // v5 derives hosted state from the lockfiles only: a pre-v5 ledger + // still claiming minimist, with no lockfile pinning it hosted, makes + // no overlap with the vendored ledger — and no directional warning. + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + write_redirect_ledger(root, &["pkg:npm/minimist@1.2.2"]).await; + write_vendor_ledger_wired(root, &["pkg:npm/minimist@1.2.2"]).await; + + assert!(overlapping_purls(&common_at(root), root).await.is_empty()); + assert_eq!( + classify_overlap_takeover(&common_at(root), root).await, + OverlapTakeover::default(), + "a legacy ledger must never be read as hosted state" + ); + } + + /// The overlap keys on hosted RECORDS only: a state carrying edits but no + /// records (the shape a pre-v5 run with failed record fetches persisted) + /// names no package, so nothing overlaps. + #[tokio::test] + async fn edits_only_hosted_state_names_no_package() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + write_vendor_ledger_wired(root, &["pkg:npm/minimist@1.2.2"]).await; + let vendor = socket_patch_core::vendor::load_state(root).await.unwrap(); + + let mut edits_only = socket_patch_core::patch::redirect::RedirectState::new(); + edits_only + .edits + .push(socket_patch_core::patch::redirect::FileEdit { + path: "package-lock.json".to_string(), + kind: "redirect_npm_lock_entry".to_string(), + action: "modified".to_string(), + key: Some("node_modules/minimist".to_string()), + original: None, + new: None, + }); + assert!(overlap_from_states(Some(&edits_only), &vendor).is_empty()); + assert!(overlap_from_states(None, &vendor).is_empty()); + assert_eq!( + overlap_from_states(Some(&pinned_state(&["pkg:npm/minimist@1.2.2"])), &vendor), + vec!["pkg:npm/minimist@1.2.2".to_string()], + "positive control: a record names the package" + ); } #[test] @@ -2993,29 +3055,15 @@ mod tests { } #[test] - fn takeover_detail_names_direction_package_and_remediation() { + fn takeover_detail_names_package_and_remediation() { let purls = vec!["pkg:npm/minimist@1.2.2".to_string()]; - // Vendored displaced a hosted redirect: name the stale ledger, but - // NEVER advise deleting it by hand. - let vendored = mode_takeover_detail(&purls, /*current_is_hosted=*/ false); - assert!(vendored.contains("pkg:npm/minimist@1.2.2")); - assert!(vendored.contains("redirect-state.json")); - assert!( - !vendored.contains("Remove the stale redirect ledger"), - "must not advise deleting the redirect ledger: {vendored}" - ); - assert!( - vendored.contains("Do not delete"), - "must warn against hand-deleting the ledger: {vendored}" - ); - // Hosted displaced a vendored ledger: per-package `remove ` is // the offered remediation; `vendor --revert` is named only as // something NOT to run (it mass-reverts). Deleting the // `.socket/vendor//` tree by hand hard-breaks cargo resolution // while `[patch.crates-io]` still references it. - let hosted = mode_takeover_detail(&purls, /*current_is_hosted=*/ true); + let hosted = mode_takeover_detail(&purls); assert!(hosted.contains("pkg:npm/minimist@1.2.2")); assert!(hosted.contains("state.json")); assert!(hosted.contains("orphaned")); @@ -3025,42 +3073,17 @@ mod tests { "deleting the vendor tree must not be offered as an equal \ alternative: {hosted}" ); - - // The two warning codes are distinct routing tags. - assert_ne!(VENDOR_SUPERSEDES_REDIRECT, REDIRECT_SUPERSEDES_VENDORED); + // v5 keeps no hosted ledger, so the detail must not point at one. + assert!( + !hosted.contains("redirect-state.json"), + "the detail must not name the retired hosted ledger: {hosted}" + ); } // ---- agent-flow hosted-wiring retention (hosted → agent conversion) ---- - /// Redirect ledger with one record per PURL AND a recorded `yarn.lock` - /// edit — the shape a real hosted run leaves behind (the edit is what - /// lets the ledger-file fallback scan the lock). - async fn write_redirect_ledger_with_edit(root: &Path, purls: &[&str]) { - use socket_patch_core::patch::redirect::{FileEdit, RedirectState}; - let mut state = RedirectState::new(); - for purl in purls { - state.records.insert((*purl).to_string(), takeover_record()); - } - state.edits.push(FileEdit { - path: "yarn.lock".to_string(), - kind: "redirect_yarn_entry".to_string(), - action: "rewritten".to_string(), - key: Some("minimist@1.2.2".to_string()), - original: Some(serde_json::Value::String("registry original".to_string())), - new: None, - }); - let dir = root.join(".socket/vendor"); - tokio::fs::create_dir_all(&dir).await.unwrap(); - tokio::fs::write( - dir.join("redirect-state.json"), - serde_json::to_string_pretty(&state).unwrap(), - ) - .await - .unwrap(); - } - /// yarn classic lock whose resolved URL is the hosted artifact (carries - /// the record uuid) — the live-hosted-wiring proof. + /// the patch uuid) — the live-hosted-wiring proof. async fn write_hosted_yarn_lock(root: &Path, uuid: &str) { tokio::fs::write( root.join("yarn.lock"), @@ -3074,10 +3097,25 @@ mod tests { .unwrap(); } - async fn load_ledger(root: &Path) -> Option { - socket_patch_core::patch::redirect::load_redirect_state(root) - .await - .unwrap() + /// yarn classic lock resolving minimist from the public registry — no + /// hosted pin. + async fn write_registry_yarn_lock(root: &Path) { + tokio::fs::write( + root.join("yarn.lock"), + "# yarn lockfile v1\n\n\nminimist@^1.2.2:\n version \"1.2.2\"\n \ + resolved \"https://registry.yarnpkg.com/minimist/-/minimist-1.2.2.tgz#bbbb\"\n \ + integrity sha512-orig==\n", + ) + .await + .unwrap(); + } + + /// The project's hosted state as production derives it: the lockfiles' + /// hosted pins (`Some`, as a non-global scan passes it). + async fn hosted_state( + common: &GlobalArgs, + ) -> Option { + Some(crate::commands::hosted_state_from_lockfiles(common, &common.cwd).await) } /// `GlobalArgs` rooted at `root` (the classifiers read the live @@ -3095,12 +3133,11 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); let purl = "pkg:npm/minimist@1.2.2"; - write_redirect_ledger_with_edit(root, &[purl]).await; write_hosted_yarn_lock(root, TAKEOVER_UUID).await; // Hosted-only wiring (no vendor state.json) is structurally // invisible to the hosted⇄vendored overlap classifier… - assert!(overlapping_ledger_purls(root).await.is_empty()); + assert!(overlapping_purls(&common_at(root), root).await.is_empty()); assert_eq!( classify_overlap_takeover(&common_at(root), root).await, OverlapTakeover::default() @@ -3108,46 +3145,39 @@ mod tests { // …but the agent flow's direct probe sees it for scanned purls. let scanned: HashSet = [purl.to_string()].into_iter().collect(); - let ledger = load_ledger(root).await; + let state = hosted_state(&common_at(root)).await; let retained = - hosted_wiring_retained_purls(&common_at(root), ledger.as_ref(), &scanned).await; + hosted_wiring_retained_purls(&common_at(root), state.as_ref(), &scanned).await; assert_eq!(retained, vec![purl.to_string()]); } #[tokio::test] - async fn hosted_retained_probe_is_silent_without_live_records_or_wiring() { + async fn hosted_retained_probe_is_silent_without_live_pins_or_wiring() { let purl = "pkg:npm/minimist@1.2.2"; let scanned: HashSet = [purl.to_string()].into_iter().collect(); - // (a) Records retired (a hosted→vendored pre-revert drops RECORDS - // while the `edits` remain): silent even with the uuid still in the - // lock text. + // (a) Registry-clean lock beside a pre-v5 ledger still recording the + // redirect: the lockfiles hold no pin, and the legacy ledger is + // never consulted. let tmp = tempfile::tempdir().unwrap(); - write_redirect_ledger_with_edit(tmp.path(), &[]).await; - write_hosted_yarn_lock(tmp.path(), TAKEOVER_UUID).await; - let ledger = load_ledger(tmp.path()).await; + write_redirect_ledger(tmp.path(), &[purl]).await; + write_registry_yarn_lock(tmp.path()).await; + let common = common_at(tmp.path()); + let state = hosted_state(&common).await; + assert!(state.as_ref().is_some_and(|s| s.records.is_empty())); assert!( - hosted_wiring_retained_purls(&common_at(tmp.path()), ledger.as_ref(), &scanned) + hosted_wiring_retained_purls(&common, state.as_ref(), &scanned) .await .is_empty(), - "records gone ⇒ silent (pre-reverted wiring must not re-warn)" + "no pin ⇒ silent (a legacy ledger must not re-warn)" ); - // (b) Registry-clean lock with a live record: the live lock is the - // truth source — never guess from ledger presence alone. - let tmp = tempfile::tempdir().unwrap(); - write_redirect_ledger_with_edit(tmp.path(), &[purl]).await; - tokio::fs::write( - tmp.path().join("yarn.lock"), - "# yarn lockfile v1\n\n\nminimist@^1.2.2:\n version \"1.2.2\"\n \ - resolved \"https://registry.yarnpkg.com/minimist/-/minimist-1.2.2.tgz#bbbb\"\n \ - integrity sha512-orig==\n", - ) - .await - .unwrap(); - let ledger = load_ledger(tmp.path()).await; + // (b) A state record the live lock does not back (the lock was + // re-resolved after the state was taken): the live lock is the truth + // source — never guess from state presence alone. + let stale = pinned_state(&[purl]); assert!( - hosted_wiring_retained_purls(&common_at(tmp.path()), ledger.as_ref(), &scanned) + hosted_wiring_retained_purls(&common, Some(&stale), &scanned) .await .is_empty(), "registry-clean lock ⇒ silent" @@ -3155,25 +3185,23 @@ mod tests { // (c) The purl was not scanned this run. let tmp = tempfile::tempdir().unwrap(); - write_redirect_ledger_with_edit(tmp.path(), &[purl]).await; write_hosted_yarn_lock(tmp.path(), TAKEOVER_UUID).await; + let common = common_at(tmp.path()); let other: HashSet = ["pkg:npm/lodash@4.17.21".to_string()].into_iter().collect(); - let ledger = load_ledger(tmp.path()).await; + let state = hosted_state(&common).await; assert!( - hosted_wiring_retained_purls(&common_at(tmp.path()), ledger.as_ref(), &other) + hosted_wiring_retained_purls(&common, state.as_ref(), &other) .await .is_empty(), "unscanned purl ⇒ silent" ); - // (d) No ledger at all. - let tmp = tempfile::tempdir().unwrap(); - write_hosted_yarn_lock(tmp.path(), TAKEOVER_UUID).await; + // (d) No hosted state at all (a global scan passes `None`). assert!( - hosted_wiring_retained_purls(&common_at(tmp.path()), None, &scanned) + hosted_wiring_retained_purls(&common, None, &scanned) .await .is_empty(), - "no ledger ⇒ silent" + "no state ⇒ silent" ); } @@ -3206,20 +3234,29 @@ mod tests { ), ] { let tmp = tempfile::tempdir().unwrap(); - write_redirect_ledger_with_edit(tmp.path(), &[purl]).await; tokio::fs::write(tmp.path().join("vlt-lock.json"), text) .await .unwrap(); - let ledger = load_ledger(tmp.path()).await; - let retained = - hosted_wiring_retained_purls(&common_at(tmp.path()), ledger.as_ref(), &scanned) - .await; + let common = common_at(tmp.path()); let want = if live { vec![purl.to_string()] } else { Vec::new() }; + // The lockfiles' own pins, as production derives them… + let state = hosted_state(&common).await; + let pinned: Vec = state + .iter() + .flat_map(|s| s.records.keys().cloned()) + .collect(); + assert_eq!(pinned, want, "pins: {what}"); + let retained = hosted_wiring_retained_purls(&common, state.as_ref(), &scanned).await; assert_eq!(retained, want, "{what}"); + // …and the probe's own liveness gate over a record the lock may + // not back. + let retained = + hosted_wiring_retained_purls(&common, Some(&pinned_state(&[purl])), &scanned).await; + assert_eq!(retained, want, "liveness: {what}"); } } @@ -3227,15 +3264,17 @@ mod tests { fn agent_retention_details_name_packages_and_safe_remediation() { let purls = vec!["pkg:npm/minimist@1.2.2".to_string()]; - // hosted_wiring_retained: names the purl and the real options, - // never hand-deleting the ledger. + // hosted_wiring_retained: names the purl and the real options + // (stay hosted, migrate to vendored, or restore upstream via + // rollback), and no longer points at a hosted ledger. let hosted = hosted_wiring_retained_detail(&purls); assert!(hosted.contains("pkg:npm/minimist@1.2.2")); assert!(hosted.contains("scan --mode hosted")); assert!(hosted.contains("scan --mode vendored")); + assert!(hosted.contains("socket-patch rollback")); assert!( - hosted.contains("Do not delete"), - "must warn against hand-deleting the ledger: {hosted}" + !hosted.contains("redirect-state.json"), + "v5 keeps no hosted ledger to name: {hosted}" ); // vendored_ownership_retained: names the purl and the per-package @@ -3250,10 +3289,10 @@ mod tests { ); assert!(vendored.contains("scan --mode agent")); - // Distinct routing tags, also distinct from the takeover family. + // Distinct routing tags, also distinct from the takeover code. assert_ne!(HOSTED_WIRING_RETAINED, VENDORED_OWNERSHIP_RETAINED); assert_ne!(HOSTED_WIRING_RETAINED, REDIRECT_SUPERSEDES_VENDORED); - assert_ne!(VENDORED_OWNERSHIP_RETAINED, VENDOR_SUPERSEDES_REDIRECT); + assert_ne!(VENDORED_OWNERSHIP_RETAINED, REDIRECT_SUPERSEDES_VENDORED); } // ---- redirectState envelope block (read-only cross-mode visibility) ---- @@ -3261,94 +3300,81 @@ mod tests { // hosted/vendored runs don't) is pinned by `tests/scan_invariants.rs`; // these pin the block builder's own gates and shape. - /// Records present ⇒ the block exists with each record's canonicalized - /// purl + verbatim ledger key, the constant mode label, and the - /// caller-supplied wiringLive. Records absent (edits-only ledger, no - /// ledger) ⇒ `None`, so the envelope key stays additive. + /// Pins present ⇒ the block exists with each pin's canonical purl + + /// uuid, the constant mode label, and the caller-supplied wiringLive — + /// and no pre-v5 `ledger` / `ledgerKey` fields. No pin (a + /// registry-clean lock, even beside a legacy ledger; no state) ⇒ `None`, + /// so the envelope key stays additive. #[tokio::test] - async fn redirect_state_block_gates_on_records_and_splits_live_proof() { + async fn redirect_state_block_gates_on_pins_and_splits_live_proof() { let purl = "pkg:npm/minimist@1.2.2"; let scanned: HashSet = [purl.to_string()].into_iter().collect(); - // Records, but no lockfile on disk: listed, with the EMPTY wiringLive - // the probe computes (the ledger's word is never promoted to a - // live-lock proof). + // A hosted pin the run did not crawl: listed, with an EMPTY + // wiringLive (the pin is wired, just not covered by this run). let tmp = tempfile::tempdir().unwrap(); - write_redirect_ledger_with_edit(tmp.path(), &[purl]).await; - let ledger = load_ledger(tmp.path()).await; - let wiring = - hosted_wiring_retained_purls(&common_at(tmp.path()), ledger.as_ref(), &scanned).await; + write_hosted_yarn_lock(tmp.path(), TAKEOVER_UUID).await; + let common = common_at(tmp.path()); + let state = hosted_state(&common).await; + let unscanned: HashSet = HashSet::new(); + let wiring = hosted_wiring_retained_purls(&common, state.as_ref(), &unscanned).await; assert_eq!(wiring, Vec::::new()); let block = - redirect_state_json(ledger.as_ref(), &wiring).expect("records present ⇒ block present"); + redirect_state_json(state.as_ref(), &wiring).expect("pins present ⇒ block present"); assert_eq!(block["mode"], "hosted"); - assert_eq!(block["ledger"], ".socket/vendor/redirect-state.json"); assert_eq!( block["records"], - serde_json::json!([{ "purl": purl, "ledgerKey": purl, "uuid": TAKEOVER_UUID }]) + serde_json::json!([{ "purl": purl, "uuid": TAKEOVER_UUID }]) ); assert_eq!(block["wiringLive"], serde_json::json!([])); + let keys: Vec<&str> = block + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + assert_eq!( + keys, + ["mode", "records", "wiringLive"], + "v5 block carries no `ledger` key: {block}" + ); - // Live lock present too: the same purl graduates into wiringLive - // (a fresh run re-parses the inventory, so re-take it here). - write_hosted_yarn_lock(tmp.path(), TAKEOVER_UUID).await; - let wiring = - hosted_wiring_retained_purls(&common_at(tmp.path()), ledger.as_ref(), &scanned).await; + // Crawled this run: the same purl graduates into wiringLive. + let wiring = hosted_wiring_retained_purls(&common, state.as_ref(), &scanned).await; let block = - redirect_state_json(ledger.as_ref(), &wiring).expect("records present ⇒ block present"); + redirect_state_json(state.as_ref(), &wiring).expect("pins present ⇒ block present"); assert_eq!(block["wiringLive"], serde_json::json!([purl])); - // Edits-only ledger (records retired) ⇒ no block. + // Registry-clean lock beside a legacy ledger that still records the + // redirect ⇒ no pin ⇒ no block. let tmp = tempfile::tempdir().unwrap(); - write_redirect_ledger_with_edit(tmp.path(), &[]).await; - let ledger = load_ledger(tmp.path()).await; + write_redirect_ledger(tmp.path(), &[purl]).await; + write_registry_yarn_lock(tmp.path()).await; + let state = hosted_state(&common_at(tmp.path())).await; assert!( - redirect_state_json(ledger.as_ref(), &[]).is_none(), - "an edits-only ledger asserts no records" + redirect_state_json(state.as_ref(), &[]).is_none(), + "a legacy ledger alone asserts no hosted pin" ); - // No ledger ⇒ no block. + // No state ⇒ no block. assert!(redirect_state_json(None, &[]).is_none()); } /// The records↔wiringLive join is a plain string compare: each record's - /// `purl` is canonicalized to exactly the spelling the probe emits, with - /// the ledger's raw key preserved as `ledgerKey`. Pinned on a - /// percent-encoded scoped npm name and a `?platform=`-qualified gem purl. + /// `purl` is canonicalized to exactly the spelling the probe emits. + /// Pinned on a percent-encoded scoped npm name and a + /// `?platform=`-qualified gem purl. #[tokio::test] async fn redirect_state_records_canonicalize_to_the_wiring_live_spelling() { - use socket_patch_core::patch::redirect::{FileEdit, RedirectState}; - let scoped_key = "pkg:npm/%40scope%2Fpkg@1.0.0"; let scoped_canon = "pkg:npm/@scope/pkg@1.0.0"; let gem_key = "pkg:gem/nokogiri@1.13.3?platform=ruby"; let gem_canon = "pkg:gem/nokogiri@1.13.3"; let tmp = tempfile::tempdir().unwrap(); - let mut state = RedirectState::new(); - state - .records - .insert(scoped_key.to_string(), takeover_record()); - state.records.insert(gem_key.to_string(), takeover_record()); - // A recorded yarn.lock edit + a lock entry resolving the scoped - // package from its hosted artifact — live hosted wiring for the - // scoped purl. - state.edits.push(FileEdit { - path: "yarn.lock".to_string(), - kind: "redirect_yarn_entry".to_string(), - action: "rewritten".to_string(), - key: Some("@scope/pkg@1.0.0".to_string()), - original: Some(serde_json::Value::String("orig".to_string())), - new: None, - }); - let dir = tmp.path().join(".socket/vendor"); - tokio::fs::create_dir_all(&dir).await.unwrap(); - tokio::fs::write( - dir.join("redirect-state.json"), - serde_json::to_string_pretty(&state).unwrap(), - ) - .await - .unwrap(); + let state = pinned_state(&[scoped_key, gem_key]); + // A lock entry resolving the scoped package from its hosted + // artifact — live hosted wiring for the scoped purl. tokio::fs::write( tmp.path().join("yarn.lock"), format!( @@ -3361,25 +3387,23 @@ mod tests { .unwrap(); let scanned: HashSet = [scoped_canon.to_string()].into_iter().collect(); - let ledger = load_ledger(tmp.path()).await; let wiring = - hosted_wiring_retained_purls(&common_at(tmp.path()), ledger.as_ref(), &scanned).await; + hosted_wiring_retained_purls(&common_at(tmp.path()), Some(&state), &scanned).await; assert_eq!( wiring, vec![scoped_canon.to_string()], - "the text proof (uuid in the recorded lock) claims the scoped purl" + "the hosted pin in the lock claims the scoped purl" ); let block = - redirect_state_json(ledger.as_ref(), &wiring).expect("records present ⇒ block present"); + redirect_state_json(Some(&state), &wiring).expect("records present ⇒ block present"); assert_eq!( block["records"], serde_json::json!([ - { "purl": gem_canon, "ledgerKey": gem_key, "uuid": TAKEOVER_UUID }, - { "purl": scoped_canon, "ledgerKey": scoped_key, "uuid": TAKEOVER_UUID }, + { "purl": gem_canon, "uuid": TAKEOVER_UUID }, + { "purl": scoped_canon, "uuid": TAKEOVER_UUID }, ]), - "records carry the canonical purl (wiringLive's spelling) plus \ - the verbatim ledger key; block={block}" + "records carry the canonical purl (wiringLive's spelling); block={block}" ); let live: Vec<&str> = block["wiringLive"] .as_array() @@ -3402,17 +3426,15 @@ mod tests { } } - /// The block's `mode` is the constant label, not the ledger's opaque - /// `mode` string: a ledger carrying `"redirect"` still labels as - /// `"hosted"`. - #[tokio::test] - async fn redirect_state_mode_is_the_constant_label_for_legacy_ledgers() { - let tmp = tempfile::tempdir().unwrap(); - write_redirect_ledger_with_edit(tmp.path(), &["pkg:npm/minimist@1.2.2"]).await; - let mut ledger = load_ledger(tmp.path()).await.unwrap(); - ledger.mode = "redirect".to_string(); + /// The block's `mode` is the constant label, not the state's opaque + /// `mode` string: a state carrying the legacy `"redirect"` still labels + /// as `"hosted"`. + #[test] + fn redirect_state_mode_is_the_constant_label_for_legacy_states() { + let mut state = pinned_state(&["pkg:npm/minimist@1.2.2"]); + state.mode = "redirect".to_string(); let block = - redirect_state_json(Some(&ledger), &[]).expect("records present ⇒ block present"); + redirect_state_json(Some(&state), &[]).expect("records present ⇒ block present"); assert_eq!(block["mode"], "hosted"); } @@ -3513,7 +3535,6 @@ mod tests { // generic wiring scan. let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); - write_redirect_ledger(root, &[CARGO_PURL]).await; write_cargo_vendor_ledger(root).await; write_cargo_hosted_takeover_files(root).await; @@ -3525,15 +3546,16 @@ mod tests { ); assert!( takeover.vendored.is_empty(), - "the INVERSE warning must not fire (pre-fix bug): {takeover:?}" + "the INVERSE direction must not be reported: {takeover:?}" ); } #[tokio::test] - async fn cargo_takeover_classifies_vendored_when_the_lock_is_detached() { + async fn cargo_lock_routed_to_vendored_yields_no_hosted_pin() { // The genuine vendored-live shape: detached lock entry (no source) + - // [patch.crates-io] pointing at the entry's committed copy. The - // redirect ledger is the stale one. + // [patch.crates-io] pointing at the entry's committed copy. The lock + // pins nothing hosted, so there is no hosted state to overlap — even + // with a pre-v5 ledger still claiming the crate. let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); write_redirect_ledger(root, &[CARGO_PURL]).await; @@ -3556,19 +3578,19 @@ mod tests { .await .unwrap(); - let takeover = classify_overlap_takeover(&cargo_common_at(root), root).await; + let common = cargo_common_at(root); + assert!(overlapping_purls(&common, root).await.is_empty()); assert_eq!( - takeover.vendored, - vec![CARGO_PURL.to_string()], - "{takeover:?}" + classify_overlap_takeover(&common, root).await, + OverlapTakeover::default() ); - assert!(takeover.redirect.is_empty(), "{takeover:?}"); } #[tokio::test] async fn cargo_takeover_stays_silent_when_the_lock_points_at_crates_io() { - // Both ledgers claim the purl but a third party re-resolved the lock - // back to crates.io: neither mode is live — no directional warning. + // A third party re-resolved the lock back to crates.io: no hosted pin + // is left (a legacy ledger claiming the crate does not count), so + // no directional warning. let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); write_redirect_ledger(root, &[CARGO_PURL]).await; @@ -3599,6 +3621,7 @@ mod tests { /// (`-.tgz`, what [`write_lock_pointing_at_vendored`] /// wires). async fn write_vendor_ledger_wired(root: &Path, purls: &[&str]) { + use socket_patch_core::utils::purl::purl_name_version; let entries: serde_json::Map = purls .iter() .map(|purl| { @@ -3689,87 +3712,76 @@ mod tests { #[tokio::test] async fn hosted_flow_stays_silent_when_the_lock_still_points_at_vendored() { - // Both ledgers claim minimist, but the LIVE lockfile still resolves it - // to the committed `.socket/vendor/` artifact — vendored is live. A - // hosted dry-run/no-op must NOT emit `redirect_supersedes_vendored`, - // which would point cleanup at the LIVE vendored ledger. + // The vendored ledger (and a pre-v5 redirect ledger) claim minimist, + // but the LIVE lockfile resolves it to the committed + // `.socket/vendor/` artifact — vendored is live and no hosted pin + // exists. A hosted dry-run/no-op must NOT emit + // `redirect_supersedes_vendored`, which would point cleanup at the + // LIVE vendored ledger; nor is there any hosted state to call stale. let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); write_redirect_ledger(root, &["pkg:npm/minimist@1.2.2"]).await; write_vendor_ledger_wired(root, &["pkg:npm/minimist@1.2.2"]).await; write_lock_pointing_at_vendored(root, "minimist", "1.2.2").await; - let takeover = classify_overlap_takeover(&common_at(root), root).await; - // The hosted flow keys its warning off `.redirect` — empty here, so it - // stays silent instead of accusing the live vendored ledger. - assert!( - takeover.redirect.is_empty(), - "hosted flow must not warn when the lock is vendored: {takeover:?}" - ); - // Truthful direction: vendored won ⇒ the redirect ledger is the stale one. + assert!(overlapping_purls(&common_at(root), root).await.is_empty()); assert_eq!( - takeover.vendored, - vec!["pkg:npm/minimist@1.2.2".to_string()] + classify_overlap_takeover(&common_at(root), root).await, + OverlapTakeover::default(), + "a vendored lock leaves no hosted pin to overlap" ); - // The raw overlap is non-empty: only the direction gate keeps it quiet. - assert!(!overlapping_ledger_purls(root).await.is_empty()); } #[tokio::test] - async fn vendored_flow_stays_silent_when_the_lock_still_points_at_hosted() { - // Mirror: both ledgers claim minimist, but the LIVE lockfile resolves it - // to the hosted patch server — hosted is live. A vendored dry-run/no-op - // must NOT emit `vendor_supersedes_redirect` and point cleanup at the - // live redirect ledger. + async fn hosted_lock_classifies_the_vendored_ledger_as_superseded() { + // The vendored ledger claims minimist, but the LIVE lockfile resolves + // it to the hosted patch server — hosted is live, so the vendored + // ledger is the stale one. let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); - write_redirect_ledger(root, &["pkg:npm/minimist@1.2.2"]).await; write_vendor_ledger_wired(root, &["pkg:npm/minimist@1.2.2"]).await; write_lock_pointing_at_hosted(root, "minimist", "1.2.2").await; let takeover = classify_overlap_takeover(&common_at(root), root).await; - assert!( - takeover.vendored.is_empty(), - "vendored flow must not warn when the lock is hosted: {takeover:?}" - ); - // Truthful direction: hosted won ⇒ the vendored ledger is the stale one. assert_eq!( takeover.redirect, vec!["pkg:npm/minimist@1.2.2".to_string()] ); + assert!(takeover.vendored.is_empty(), "{takeover:?}"); } #[tokio::test] - async fn overlap_without_a_lock_to_prove_direction_stays_silent_both_ways() { - // Both ledgers overlap, but no lockfile proves which mode is live. Rather - // than guess the direction from which command is running, both flows stay - // silent — the raw overlap still fires, only the direction is gated. + async fn half_migrated_locks_naming_both_stay_silent() { + // One lockfile pins minimist hosted while another still routes it to + // the committed vendored artifact: the raw overlap fires, but neither + // direction is proven, so the classifier stays silent rather than + // guess from which command is running. let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); - write_redirect_ledger(root, &["pkg:npm/minimist@1.2.2"]).await; write_vendor_ledger_wired(root, &["pkg:npm/minimist@1.2.2"]).await; + write_lock_pointing_at_vendored(root, "minimist", "1.2.2").await; + write_hosted_yarn_lock(root, TAKEOVER_UUID).await; - let takeover = classify_overlap_takeover(&common_at(root), root).await; - assert!( - takeover.redirect.is_empty() && takeover.vendored.is_empty(), - "no lock proof ⇒ no directional warning: {takeover:?}" - ); assert_eq!( - overlapping_ledger_purls(root).await, + overlapping_purls(&common_at(root), root).await, vec!["pkg:npm/minimist@1.2.2".to_string()] ); + assert_eq!( + classify_overlap_takeover(&common_at(root), root).await, + OverlapTakeover::default(), + "both sides live ⇒ no directional warning" + ); } // ---- remediation is per-package and non-destructive --------------------- #[test] fn takeover_detail_remediation_is_per_package_and_non_destructive() { - // Cleanup must be scoped per named package: whole-ledger / whole-tree - // deletion would destroy live data for packages the takeover did not - // touch. + // Cleanup must be scoped per named package: whole-tree deletion would + // destroy live data for packages the takeover did not touch. let purls = vec!["pkg:npm/minimist@1.2.2".to_string()]; - let hosted = mode_takeover_detail(&purls, /*current_is_hosted=*/ true); + let hosted = mode_takeover_detail(&purls); // The sanctioned per-purl cleanup command… assert!( hosted.contains("socket-patch remove "), @@ -3789,21 +3801,6 @@ mod tests { !hosted.contains("vendor --revert` before redirecting"), "hosted remediation must not advise a blanket revert: {hosted}" ); - - let vendored = mode_takeover_detail(&purls, /*current_is_hosted=*/ false); - // Only the named packages' records — never the whole ledger file. - assert!( - vendored.contains("only these package(s)"), - "vendored remediation must be per-package: {vendored}" - ); - assert!( - !vendored.contains("Remove the stale redirect ledger"), - "vendored remediation must not advise deleting the ledger: {vendored}" - ); - assert!( - vendored.contains("Do not delete the ledger file"), - "vendored remediation must warn against file deletion: {vendored}" - ); } #[test] @@ -3811,7 +3808,7 @@ mod tests { // `socket-patch remove ` also deletes the package's // `.socket/manifest.json` entry; the hosted text must say so. let purls = vec!["pkg:npm/minimist@1.2.2".to_string()]; - let hosted = mode_takeover_detail(&purls, /*current_is_hosted=*/ true); + let hosted = mode_takeover_detail(&purls); assert!( !hosted.contains("drops only that entry"), @@ -3821,152 +3818,16 @@ mod tests { hosted.contains("`.socket/manifest.json`"), "hosted remediation must name the manifest entry `remove` deletes: {hosted}" ); - // …and must place the LIVE hosted patch, so "manifest entry deleted" - // does not read as "the hosted patch was dropped too". + // …and must place the LIVE hosted patch (the lockfile pin itself — + // v5 keeps no hosted ledger), so "manifest entry deleted" does not + // read as "the hosted patch was dropped too". assert!( - hosted.contains("redirect-state.json"), + hosted.contains("recorded in the lockfile itself"), "hosted remediation must say where the live hosted patch lives: {hosted}" ); } - // ---- takeover blind spots: degraded ledgers and hosted-proof gaps ------ - - fn redirect_edit(path: &str, key: &str) -> socket_patch_core::patch::redirect::FileEdit { - socket_patch_core::patch::redirect::FileEdit { - path: path.to_string(), - kind: "redirect_npm_lock_entry".to_string(), - action: "modified".to_string(), - key: Some(key.to_string()), - original: None, - new: None, - } - } - - /// Like [`write_redirect_ledger`] but with explicit `edits` (and possibly - /// NO records — the degraded shape a run with failed record fetches - /// persists). - async fn write_redirect_ledger_with_edits( - root: &Path, - purls: &[&str], - edits: Vec, - ) { - use socket_patch_core::patch::redirect::RedirectState; - let mut state = RedirectState::new(); - for purl in purls { - state.records.insert((*purl).to_string(), takeover_record()); - } - state.edits = edits; - let dir = root.join(".socket/vendor"); - tokio::fs::create_dir_all(&dir).await.unwrap(); - tokio::fs::write( - dir.join("redirect-state.json"), - serde_json::to_string_pretty(&state).unwrap(), - ) - .await - .unwrap(); - } - - #[tokio::test] - async fn overlap_detected_when_redirect_ledger_has_edits_but_no_records() { - // A hosted run where every per-uuid record fetch failed persists a - // ledger with edits but an EMPTY records map (`record_fetch_failed`). - // That ledger still asserts stale lock wiring, so a vendored takeover - // of the same package must still be flagged. - let tmp = tempfile::tempdir().unwrap(); - let root = tmp.path(); - write_redirect_ledger_with_edits( - root, - &[], - vec![redirect_edit("package-lock.json", "node_modules/minimist")], - ) - .await; - write_vendor_ledger_wired(root, &["pkg:npm/minimist@1.2.2"]).await; - write_lock_pointing_at_vendored(root, "minimist", "1.2.2").await; - - assert_eq!( - overlapping_ledger_purls(root).await, - vec!["pkg:npm/minimist@1.2.2".to_string()], - "an edits-only redirect ledger must still count as overlapping" - ); - let takeover = classify_overlap_takeover(&common_at(root), root).await; - assert_eq!( - takeover.vendored, - vec!["pkg:npm/minimist@1.2.2".to_string()], - "the vendored takeover of a degraded redirect ledger must be flagged" - ); - assert!(takeover.redirect.is_empty(), "{takeover:?}"); - } - - #[tokio::test] - async fn degraded_ledger_matches_a_vlt_variant_key_at_the_tilde_boundary() { - for (key, overlaps) in [ - ("minimist@1.2.2~peer.2", true), - ("minimist@1.2.2~_croot_s_g_s#a", true), - ("minimist@1.2.20~peer.2", false), - ] { - let tmp = tempfile::tempdir().unwrap(); - let root = tmp.path(); - let mut edit = redirect_edit("vlt-lock.json", key); - edit.kind = socket_patch_core::patch::redirect::vlt::KIND.to_string(); - write_redirect_ledger_with_edits(root, &[], vec![edit]).await; - write_vendor_ledger_wired(root, &["pkg:npm/minimist@1.2.2"]).await; - assert_eq!( - overlapping_ledger_purls(root).await, - if overlaps { - vec!["pkg:npm/minimist@1.2.2".to_string()] - } else { - Vec::new() - }, - "{key}" - ); - } - } - - #[tokio::test] - async fn following_the_vendored_remediation_clears_the_warning() { - // The vendored remediation names the matching `edits` entries as - // well as `records` (leftover edits keep matching through the - // degraded-ledger fallback); carrying it out in full must leave - // nothing to warn about. - let tmp = tempfile::tempdir().unwrap(); - let root = tmp.path(); - write_redirect_ledger_with_edits( - root, - &["pkg:npm/minimist@1.2.2"], - vec![redirect_edit("package-lock.json", "node_modules/minimist")], - ) - .await; - write_vendor_ledger_wired(root, &["pkg:npm/minimist@1.2.2"]).await; - write_lock_pointing_at_vendored(root, "minimist", "1.2.2").await; - - let before = classify_overlap_takeover(&common_at(root), root).await; - assert_eq!( - before.vendored, - vec!["pkg:npm/minimist@1.2.2".to_string()], - "the vendored takeover must be flagged first: {before:?}" - ); - let detail = mode_takeover_detail(&before.vendored, /*current_is_hosted=*/ false); - assert!( - detail.contains("`edits`"), - "the remediation must name the edits entries: {detail}" - ); - - // Exactly what the remediation prescribes for this ledger: the - // package's `records` entry AND its matching `edits` entry gone, the - // ledger file itself left in place. - write_redirect_ledger_with_edits(root, &[], Vec::new()).await; - - let after = classify_overlap_takeover(&common_at(root), root).await; - assert_eq!( - after, - OverlapTakeover::default(), - "following the remediation must clear the warning: {after:?}" - ); - assert!( - overlapping_ledger_purls(root).await.is_empty(), - "no residue may keep the ledgers reading as overlapping" - ); - } + // ---- hosted-proof gaps: other hosts and lock formats ------------------- /// A grant token as it appears between the host and the patch uuid in /// hosted artifact URLs. @@ -3974,13 +3835,13 @@ mod tests { #[tokio::test] async fn hosted_direction_provable_on_non_default_patch_host() { - // Hosted artifact URLs embed the record's patch uuid on ANY host - // (staging / self-hosted `--patch-server-url` deployments), so the - // liveness proof must not be pinned to the `patch.socket.dev` - // hostname. + // Hosted artifact URLs embed the pin's patch uuid on ANY host the + // operator configured (staging / self-hosted `--patch-server-url` + // deployments), so the proof must not be pinned to the + // `patch.socket.dev` hostname — but an unconfigured host is a user's + // own dependency source, never a pin. let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); - write_redirect_ledger(root, &["pkg:npm/minimist@1.2.2"]).await; write_vendor_ledger_wired(root, &["pkg:npm/minimist@1.2.2"]).await; let lock = serde_json::json!({ "name": "app", @@ -4004,28 +3865,31 @@ mod tests { .await .unwrap(); - let takeover = classify_overlap_takeover(&common_at(root), root).await; + let configured = GlobalArgs { + patch_server_url: Some("https://patches.example.com".to_string()), + ..common_at(root) + }; + let takeover = classify_overlap_takeover(&configured, root).await; assert_eq!( takeover.redirect, vec!["pkg:npm/minimist@1.2.2".to_string()], - "a non-default patch host must still prove hosted is live" + "a configured non-default patch host must still prove hosted is live" ); assert!(takeover.vendored.is_empty(), "{takeover:?}"); + + assert_eq!( + classify_overlap_takeover(&common_at(root), root).await, + OverlapTakeover::default(), + "an unconfigured host is not a hosted pin" + ); } #[tokio::test] async fn hosted_direction_provable_for_bun_url_tuple() { - // The bun inventory skips the URL 3-tuples hosted mode writes, so - // hosted liveness must be provable from the redirect-edited lockfile - // text (the record's uuid outside any vendored path). + // bun records the hosted artifact as a URL 3-tuple; the pin must be + // found there. let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); - write_redirect_ledger_with_edits( - root, - &["pkg:npm/minimist@1.2.2"], - vec![redirect_edit("bun.lock", "minimist")], - ) - .await; write_vendor_ledger_wired(root, &["pkg:npm/minimist@1.2.2"]).await; tokio::fs::write( root.join("bun.lock"), @@ -4049,17 +3913,11 @@ mod tests { #[tokio::test] async fn hosted_direction_provable_for_berry_archive_url() { - // The berry inventory always emits `resolved: None`; the hosted URL - // lives percent-encoded in the `::__archiveUrl=` binding. The uuid - // survives encoding verbatim, so the text proof must see it. + // The hosted URL lives percent-encoded in berry's `::__archiveUrl=` + // binding. The uuid survives encoding verbatim, so the pin must be + // found there. let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); - write_redirect_ledger_with_edits( - root, - &["pkg:npm/minimist@1.2.2"], - vec![redirect_edit("yarn.lock", "minimist@1.2.2")], - ) - .await; write_vendor_ledger_wired(root, &["pkg:npm/minimist@1.2.2"]).await; tokio::fs::write( root.join("yarn.lock"), @@ -4082,45 +3940,33 @@ mod tests { } #[tokio::test] - async fn vendored_path_uuid_does_not_prove_hosted() { + async fn vendored_path_uuid_is_not_a_hosted_pin() { // The vendored wiring embeds the SAME patch uuid in its - // `.socket/vendor///` path. When the redirect ledger - // names the same lockfile, those occurrences must NOT read as - // hosted proof — the lock points at the vendored files. + // `.socket/vendor///` path. That occurrence must NOT read + // as a hosted pin — the lock points at the vendored files. let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); - write_redirect_ledger_with_edits( - root, - &["pkg:npm/minimist@1.2.2"], - vec![redirect_edit("package-lock.json", "node_modules/minimist")], - ) - .await; - write_vendor_ledger_wired(root, &["pkg:npm/minimist@1.2.2"]).await; write_lock_pointing_at_vendored(root, "minimist", "1.2.2").await; - let takeover = classify_overlap_takeover(&common_at(root), root).await; + let state = crate::commands::hosted_state_from_lockfiles(&common_at(root), root).await; assert!( - takeover.redirect.is_empty(), - "a vendored-path uuid must not prove hosted: {takeover:?}" - ); - assert_eq!( - takeover.vendored, - vec!["pkg:npm/minimist@1.2.2".to_string()] + state.records.is_empty(), + "a vendored-path uuid must not be a hosted pin: {:?}", + state.records.keys().collect::>() ); } - // ---- takeover detection degradation: corrupt / probe-less ledgers ------ + // ---- takeover detection degradation: corrupt / probe-less state -------- #[tokio::test] async fn corrupt_vendor_state_json_degrades_to_no_overlap() { // A hand-corrupted (or torn mid-write) `.socket/vendor/state.json` // must classify like a missing one: this path only feeds takeover // WARNINGS, and the vendored write paths hard-error on corruption - // themselves. A valid redirect ledger alone must not produce a - // spurious overlap. + // themselves. A hosted pin alone must not produce a spurious overlap. let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); - write_redirect_ledger(root, &["pkg:npm/minimist@1.2.2"]).await; + write_lock_pointing_at_hosted(root, "minimist", "1.2.2").await; let dir = root.join(".socket/vendor"); tokio::fs::create_dir_all(&dir).await.unwrap(); tokio::fs::write(dir.join("state.json"), "not-json {{{") @@ -4128,7 +3974,7 @@ mod tests { .unwrap(); assert!( - overlapping_ledger_purls(root).await.is_empty(), + overlapping_purls(&common_at(root), root).await.is_empty(), "a corrupt vendor ledger must degrade to no-overlap" ); assert_eq!( @@ -4140,24 +3986,22 @@ mod tests { #[tokio::test] async fn cargo_overlap_with_no_lock_to_probe_stays_silent() { - // Both ledgers claim the cargo purl but there is NO Cargo.lock (a - // fresh checkout / deleted lock): discovery finds no cargo wiring - // either way, which proves neither direction — the classifier must - // stay silent rather than guess. + // The vendored ledger (and a pre-v5 redirect ledger) claim the cargo + // purl but there is NO Cargo.lock (a fresh checkout / deleted lock): + // discovery finds no hosted pin, so nothing overlaps and the + // classifier stays silent rather than guess. let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); write_redirect_ledger(root, &[CARGO_PURL]).await; write_cargo_vendor_ledger(root).await; - // The raw overlap fires (both ledgers name the purl)… - assert_eq!( - overlapping_ledger_purls(root).await, - vec![CARGO_PURL.to_string()], - "the overlap itself must be detected" + let common = cargo_common_at(root); + assert!( + overlapping_purls(&common, root).await.is_empty(), + "no lock ⇒ no hosted pin ⇒ no overlap" ); - // …but with no lock to prove a direction, both buckets stay empty. assert_eq!( - classify_overlap_takeover(&common_at(root), root).await, + classify_overlap_takeover(&common, root).await, OverlapTakeover::default(), "no Cargo.lock ⇒ neither direction proven ⇒ silent" ); @@ -4275,266 +4119,6 @@ mod tests { ); } - // ---- note_vendor_supersedes_redirect: warning + npm auto-reconcile ------ - - const NPM_TAKEOVER_PURL: &str = "pkg:npm/minimist@1.2.2"; - - fn vendor_env() -> crate::json_envelope::Envelope { - crate::json_envelope::Envelope::new(crate::json_envelope::Command::Vendor) - } - - /// `GlobalArgs` for the advisory: `json` keeps the stderr print quiet - /// (the envelope `warnings[]` is what the tests read). - fn takeover_common() -> GlobalArgs { - GlobalArgs { - json: true, - ..GlobalArgs::default() - } - } - - /// The WET npm takeover: redirect ledger records the purl (with a - /// version-exact keyed edit `drop_superseded_purl` can claim), the - /// vendored ledger is wired, and the LIVE lock points at the committed - /// vendored artifact. - async fn write_wet_npm_takeover(root: &Path) { - write_redirect_ledger_with_edits( - root, - &[NPM_TAKEOVER_PURL], - vec![redirect_edit("package-lock.json", "minimist@1.2.2")], - ) - .await; - write_vendor_ledger_wired(root, &[NPM_TAKEOVER_PURL]).await; - write_lock_pointing_at_vendored(root, "minimist", "1.2.2").await; - } - - #[tokio::test] - async fn vendored_takeover_wet_npm_run_reconciles_the_ledger_once() { - let tmp = tempfile::tempdir().unwrap(); - let root = tmp.path(); - write_wet_npm_takeover(root).await; - - let mut env = vendor_env(); - note_vendor_supersedes_redirect(&mut env, root, &takeover_common()).await; - - assert_eq!( - env.warnings.len(), - 1, - "exactly one warning: {:?}", - env.warnings - ); - assert_eq!(env.warnings[0].code, VENDOR_SUPERSEDES_REDIRECT); - assert!( - env.warnings[0].detail.contains("reconciled automatically"), - "a wet npm run must report the past-tense reconciled detail: {}", - env.warnings[0].detail - ); - assert!( - env.warnings[0].detail.contains(NPM_TAKEOVER_PURL), - "the warning must name the package: {}", - env.warnings[0].detail - ); - - // Both halves dropped; the emptied ledger is deleted outright. - assert!( - load_ledger(root).await.is_none(), - "an emptied redirect ledger must be deleted" - ); - - // Fires once: the reconciled project no longer overlaps. - let mut env2 = vendor_env(); - note_vendor_supersedes_redirect(&mut env2, root, &takeover_common()).await; - assert!( - env2.warnings.is_empty(), - "a reconciled takeover must not re-warn: {:?}", - env2.warnings - ); - } - - /// The reconcile's `.npmrc` unwind surfaces its own warnings - /// (`redirect_npmrc_allow_remote_modified`), and the detail mentions - /// `.npmrc` with the npm 12 EALLOWREMOTE caveat. - #[tokio::test] - async fn vendored_takeover_reconcile_surfaces_the_npmrc_unwind() { - let tmp = tempfile::tempdir().unwrap(); - let root = tmp.path(); - write_redirect_ledger_with_edits( - root, - &[NPM_TAKEOVER_PURL], - vec![ - redirect_edit("package-lock.json", "minimist@1.2.2"), - socket_patch_core::patch::redirect::FileEdit { - path: ".npmrc".into(), - kind: "redirect_npmrc_allow_remote".into(), - action: "created".into(), - key: Some("allow-remote".into()), - original: None, - new: Some(serde_json::json!("all")), - }, - ], - ) - .await; - write_vendor_ledger_wired(root, &[NPM_TAKEOVER_PURL]).await; - write_lock_pointing_at_vendored(root, "minimist", "1.2.2").await; - // The user added their own setting to the redirect-created file. - tokio::fs::write(root.join(".npmrc"), "allow-remote=all\nfund=false\n") - .await - .unwrap(); - - let mut env = vendor_env(); - note_vendor_supersedes_redirect(&mut env, root, &takeover_common()).await; - - let codes: Vec<&str> = env.warnings.iter().map(|w| w.code.as_str()).collect(); - assert_eq!( - codes, - [ - VENDOR_SUPERSEDES_REDIRECT, - "redirect_npmrc_allow_remote_modified" - ], - "{:?}", - env.warnings - ); - let detail = &env.warnings[0].detail; - assert!(detail.contains("reconciled automatically"), "{detail}"); - assert!(detail.contains("`.npmrc` `allow-remote=all`"), "{detail}"); - assert!(detail.contains("EALLOWREMOTE"), "{detail}"); - assert_eq!( - tokio::fs::read_to_string(root.join(".npmrc")) - .await - .unwrap(), - "fund=false\n", - "only our line removed" - ); - assert!(load_ledger(root).await.is_none(), "emptied ledger deleted"); - - // Without a recorded `.npmrc` edit the detail stays silent on it. - assert!(!mode_takeover_reconciled_detail(&["p".into()], false).contains(".npmrc")); - } - - #[tokio::test] - async fn vendored_takeover_dry_run_warns_manual_and_leaves_the_ledger() { - let tmp = tempfile::tempdir().unwrap(); - let root = tmp.path(); - write_wet_npm_takeover(root).await; - let ledger_path = root.join(".socket/vendor/redirect-state.json"); - let before = tokio::fs::read(&ledger_path).await.unwrap(); - - let mut env = vendor_env(); - let common = GlobalArgs { - dry_run: true, - ..takeover_common() - }; - note_vendor_supersedes_redirect(&mut env, root, &common).await; - - assert_eq!(env.warnings.len(), 1, "{:?}", env.warnings); - assert_eq!(env.warnings[0].code, VENDOR_SUPERSEDES_REDIRECT); - // A dry run hands out the MANUAL remediation (never the past-tense - // reconciled text — nothing was mutated). - assert!( - env.warnings[0].detail.contains("clean up by hand"), - "dry-run must carry the manual advisory: {}", - env.warnings[0].detail - ); - assert!( - !env.warnings[0].detail.contains("reconciled automatically"), - "dry-run must not claim a reconciliation: {}", - env.warnings[0].detail - ); - let after = tokio::fs::read(&ledger_path).await.unwrap(); - assert_eq!( - before, after, - "a dry run must leave the ledger byte-identical" - ); - } - - #[tokio::test] - async fn degraded_ledger_reconcile_matches_nothing_and_falls_back_to_manual() { - // The degraded record-fetch-failed ledger: records EMPTY, one - // version-blind path-keyed edit. The overlap fallback flags it, but - // `drop_superseded_purl` (fail-closed: no record uuid to anchor on, - // key not version-exact) drops nothing — the warning must hand out - // the manual remediation, never claim a reconciliation. - let tmp = tempfile::tempdir().unwrap(); - let root = tmp.path(); - write_redirect_ledger_with_edits( - root, - &[], - vec![redirect_edit("package-lock.json", "node_modules/minimist")], - ) - .await; - write_vendor_ledger_wired(root, &[NPM_TAKEOVER_PURL]).await; - write_lock_pointing_at_vendored(root, "minimist", "1.2.2").await; - let ledger_path = root.join(".socket/vendor/redirect-state.json"); - let before = tokio::fs::read(&ledger_path).await.unwrap(); - - let mut env = vendor_env(); - note_vendor_supersedes_redirect(&mut env, root, &takeover_common()).await; - - assert_eq!(env.warnings.len(), 1, "{:?}", env.warnings); - assert_eq!(env.warnings[0].code, VENDOR_SUPERSEDES_REDIRECT); - assert_eq!( - env.warnings[0].detail, - mode_takeover_detail(&[NPM_TAKEOVER_PURL.to_string()], false), - "an Ok(None) reconcile must fall back to the manual detail verbatim" - ); - let after = tokio::fs::read(&ledger_path).await.unwrap(); - assert_eq!( - before, after, - "a no-op reconcile must leave the degraded ledger byte-identical" - ); - } - - #[cfg(unix)] - #[tokio::test] - async fn reconcile_persist_failure_fails_closed_with_manual_advice() { - use std::os::unix::fs::PermissionsExt; - let tmp = tempfile::tempdir().unwrap(); - let root = tmp.path(); - write_wet_npm_takeover(root).await; - let vendor_dir = root.join(".socket/vendor"); - let ledger_path = vendor_dir.join("redirect-state.json"); - let before = tokio::fs::read(&ledger_path).await.unwrap(); - - std::fs::set_permissions(&vendor_dir, std::fs::Permissions::from_mode(0o555)).unwrap(); - // Root ignores mode bits; skip there (CI containers sometimes run as root). - if std::fs::File::create(vendor_dir.join("probe")).is_ok() { - let _ = std::fs::remove_file(vendor_dir.join("probe")); - let _ = std::fs::set_permissions(&vendor_dir, std::fs::Permissions::from_mode(0o755)); - eprintln!("skipping: running as root, 0555 does not block writes"); - return; - } - - let mut env = vendor_env(); - note_vendor_supersedes_redirect(&mut env, root, &takeover_common()).await; - - // Restore BEFORE asserting so a failure never leaks an undeletable - // tempdir. - std::fs::set_permissions(&vendor_dir, std::fs::Permissions::from_mode(0o755)).unwrap(); - - assert_eq!(env.warnings.len(), 1, "{:?}", env.warnings); - assert_eq!(env.warnings[0].code, VENDOR_SUPERSEDES_REDIRECT); - assert!( - env.warnings[0] - .detail - .contains("Automatic reconciliation failed"), - "the persist failure must be surfaced inside the warning: {}", - env.warnings[0].detail - ); - assert!( - env.warnings[0].detail.starts_with(&mode_takeover_detail( - &[NPM_TAKEOVER_PURL.to_string()], - false - )), - "the failure text must ride on the full manual remediation: {}", - env.warnings[0].detail - ); - // Fail closed: the atomic writer left the ledger fully pre-drop. - let after = tokio::fs::read(&ledger_path).await.unwrap(); - assert_eq!( - before, after, - "a failed persist must leave the ledger untouched" - ); - } - /// A failed embedded VEX's discovery diagnostics reach the scan JSON: /// appended after existing `warnings[]` (layout refusals), or creating /// the array; an empty list leaves the object untouched. From e1bb9a686460d073924dca9df5a8cacc923f83f4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 23:19:34 +0000 Subject: [PATCH 08/66] Restore hosted gem and composer pins to their upstream entries Add the RubyGems and Composer restorers to the v5 ledger-free hosted unwind (`redirect::upstream`). Gem (Gemfile.lock / gems.locked + Gemfile / gems.rb): a converged Socket GEM section is removed and its spec moved back, in name order, into the upstream section (the single remaining one, else the manifest's global source or rubygems.org, else refused as ambiguous); a bundler <= 2.1 merged section loses only the Socket remote; the DEPENDENCIES `!` pin is dropped; CHECKSUMS is re-pinned from the rubygems.org compact index. The Gemfile source block becomes `gem "n", "v"[, opts]` again (the original constraint is not derivable), or is removed with its DEPENDENCIES entry only when provably a transitive append. The pre-2.6 mixed state is undone when a pin is supplied, keeping the untouched lock's own constraint. Composer: dist {type,url,reference,shasum} and the dropped source block are rebuilt from packagist p2 metadata (composer/2.0 minified, expanded), cross-checked against the lock's dist.reference, in the lock's indent, slash style and line endings. Non-packagist entries are refused. UpstreamClient gains cached rubygems and packagist lookups (SOCKET_RUBYGEMS_URL, SOCKET_PACKAGIST_URL). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018tncU7u7FK48tubFNA12UQ --- .../src/patch/redirect/upstream/client.rs | 170 ++++ .../src/patch/redirect/upstream/composer.rs | 484 +++++++++ .../src/patch/redirect/upstream/gem.rs | 959 ++++++++++++++++++ .../src/patch/redirect/upstream/mod.rs | 8 + .../tests/upstream_restore_golden.rs | 567 ++++++++++- 5 files changed, 2185 insertions(+), 3 deletions(-) create mode 100644 crates/socket-patch-core/src/patch/redirect/upstream/composer.rs create mode 100644 crates/socket-patch-core/src/patch/redirect/upstream/gem.rs diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/client.rs b/crates/socket-patch-core/src/patch/redirect/upstream/client.rs index 0d8f9ef08..34eb49e6c 100644 --- a/crates/socket-patch-core/src/patch/redirect/upstream/client.rs +++ b/crates/socket-patch-core/src/patch/redirect/upstream/client.rs @@ -52,6 +52,79 @@ fn crates_index_path(name: &str) -> String { } } +/// The RubyGems compact index root; override with `SOCKET_RUBYGEMS_URL`. +pub(crate) const DEFAULT_RUBYGEMS: &str = "https://rubygems.org"; + +fn rubygems_base() -> String { + std::env::var("SOCKET_RUBYGEMS_URL") + .ok() + .map(|v| v.trim().trim_end_matches('/').to_string()) + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| DEFAULT_RUBYGEMS.to_string()) +} + +/// Packagist's composer v2 metadata repository; override with +/// `SOCKET_PACKAGIST_URL`. +pub(crate) const DEFAULT_PACKAGIST: &str = "https://repo.packagist.org"; + +fn packagist_base() -> String { + std::env::var("SOCKET_PACKAGIST_URL") + .ok() + .map(|v| v.trim().trim_end_matches('/').to_string()) + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| DEFAULT_PACKAGIST.to_string()) +} + +/// The `checksum:` (sha256 hex of the `.gem`) a compact-index `info/` +/// file records for the ruby-platform `version` — the value bundler writes +/// into `CHECKSUMS`. Lines are `[-] |`, the +/// requirement list carrying `checksum:`. +pub(crate) fn compact_index_checksum(info: &str, version: &str) -> Option { + info.lines().find_map(|line| { + let (head, reqs) = line.split_once('|')?; + let token = head.split(' ').next()?; + if token != version { + return None; + } + reqs.split(',') + .find_map(|r| r.trim().strip_prefix("checksum:")) + .and_then(crate::utils::digest::sha256_hex) + }) +} + +/// Expand a packagist `p2` version list. `minified: composer/2.0` documents +/// (composer's `MetadataMinifier`) list each version as the DIFF against the +/// previous expanded one: every key it carries replaces the inherited value, +/// and the string `"__unset"` removes the key. +pub(crate) fn expand_packagist_versions(versions: &[Value], minified: bool) -> Vec { + if !minified { + return versions.to_vec(); + } + let mut out = Vec::with_capacity(versions.len()); + let mut current: Option> = None; + for v in versions { + let Some(obj) = v.as_object() else { + continue; + }; + let next = match current.take() { + None => obj.clone(), + Some(mut prev) => { + for (k, val) in obj { + if val.as_str() == Some("__unset") { + prev.remove(k); + } else { + prev.insert(k.clone(), val.clone()); + } + } + prev + } + }; + out.push(Value::Object(next.clone())); + current = Some(next); + } + out +} + /// The offline refusal every lookup returns under `--offline`. pub(crate) const OFFLINE: &str = "the upstream entry must be re-resolved from the registry, and this run is offline"; @@ -67,6 +140,8 @@ pub(crate) struct UpstreamClient { npm_tarballs: Cache>, cargo: Cache, go: Cache, + rubygems: Cache, + packagist: Cache>, } impl UpstreamClient { @@ -78,6 +153,8 @@ impl UpstreamClient { npm_tarballs: Mutex::default(), cargo: Mutex::default(), go: Mutex::default(), + rubygems: Mutex::default(), + packagist: Mutex::default(), } } @@ -244,6 +321,68 @@ impl UpstreamClient { self.go.lock().await.insert(key, result.clone()); result } + + /// The rubygems.org sha256 of the ruby-platform `name-version.gem`, + /// from the compact index bundler itself reads (`info/`). + pub(crate) async fn rubygems_sha256(&self, name: &str, version: &str) -> Result { + let key = (name.to_string(), version.to_string()); + if let Some(hit) = self.rubygems.lock().await.get(&key) { + return hit.clone(); + } + let result = async { + if self.offline { + return Err(OFFLINE.to_string()); + } + if !crate::vendor::gemfile_lock::is_plain_gem_token(name) { + return Err(format!("{name:?} is not a plain gem name")); + } + let url = format!("{}/info/{name}", rubygems_base()); + let text = self.get_text(&url).await?; + compact_index_checksum(&text, version) + .ok_or_else(|| format!("{url} lists no ruby-platform {version} with a checksum")) + } + .await; + self.rubygems.lock().await.insert(key, result.clone()); + result + } + + /// Every version packagist serves for the composer package `name` + /// (lowercase `vendor/package`), expanded: the stable `p2/.json` + /// list, or the `~dev` one when `dev` (composer splits branches out). + pub(crate) async fn packagist_versions(&self, name: &str, dev: bool) -> Result, String> { + let key = (name.to_string(), if dev { "~dev" } else { "" }.to_string()); + if let Some(hit) = self.packagist.lock().await.get(&key) { + return hit.clone(); + } + let result = async { + if self.offline { + return Err(OFFLINE.to_string()); + } + let safe = name.split('/').count() == 2 + && name.split('/').all(|p| { + !p.is_empty() + && !p.starts_with('.') + && p.bytes().all(|b| { + b.is_ascii_lowercase() || b.is_ascii_digit() || b"._-".contains(&b) + }) + }); + if !safe { + return Err(format!("{name:?} is not a packagist package name")); + } + let url = format!("{}/p2/{name}{}.json", packagist_base(), key.1); + let doc = self.get_json(&url).await?; + let versions = doc + .get("packages") + .and_then(|p| p.get(name)) + .and_then(Value::as_array) + .ok_or_else(|| format!("{url} lists no versions of {name}"))?; + let minified = doc.get("minified").and_then(Value::as_str) == Some("composer/2.0"); + Ok(expand_packagist_versions(versions, minified)) + } + .await; + self.packagist.lock().await.insert(key, result.clone()); + result + } } /// Go's checksum database, `sum.golang.org`; `SOCKET_GOSUMDB_URL` names @@ -298,6 +437,37 @@ mod tests { assert_eq!(crates_index_path("Serde"), "se/rd/serde"); } + #[test] + fn compact_index_checksum_picks_the_ruby_platform_line() { + let sha = "a".repeat(64); + let other = "b".repeat(64); + let info = format!( + "---\n1.0.0 |checksum:{other}\n1.1.0 dep:>= 0|checksum:{sha},ruby:>= 2.7\n\ + 1.1.0-java |checksum:{other}\n" + ); + assert_eq!(compact_index_checksum(&info, "1.1.0"), Some(sha)); + assert_eq!(compact_index_checksum(&info, "2.0.0"), None); + assert_eq!(compact_index_checksum("1.0.0 |ruby:>= 2", "1.0.0"), None); + } + + #[test] + fn packagist_minified_versions_inherit_and_unset() { + let versions = serde_json::json!([ + {"name": "a/b", "version": "2.0.0", "source": {"type": "git"}, "dist": {"url": "x"}}, + {"version": "1.0.0", "source": "__unset"}, + {"version": "0.9.0", "dist": {"url": "y"}} + ]); + let expanded = expand_packagist_versions(versions.as_array().unwrap(), true); + assert_eq!(expanded.len(), 3); + assert_eq!(expanded[1]["name"], "a/b"); + assert!(expanded[1].get("source").is_none()); + assert_eq!(expanded[1]["dist"]["url"], "x"); + assert!(expanded[2].get("source").is_none()); + assert_eq!(expanded[2]["dist"]["url"], "y"); + let raw = expand_packagist_versions(versions.as_array().unwrap(), false); + assert!(raw[1].get("name").is_none()); + } + #[test] fn go_mod_h1_matches_the_x_mod_recipe() { // `module example.com/m\n` — cross-checked with `go mod download diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/composer.rs b/crates/socket-patch-core/src/patch/redirect/upstream/composer.rs new file mode 100644 index 000000000..52e6b8c43 --- /dev/null +++ b/crates/socket-patch-core/src/patch/redirect/upstream/composer.rs @@ -0,0 +1,484 @@ +//! Composer upstream restore: a hosted `composer.lock` entry's `dist` goes +//! back to what packagist serves for the version (`type`, `url`, +//! `reference`, `shasum`), and the `source` block the hosted rewriter +//! deleted (`redirect_composer_dist`) is re-inserted right before it — both +//! re-derived from packagist's composer v2 metadata (`p2/.json`, or +//! `p2/~dev.json` for a branch version), the document composer itself +//! resolved the entry from. +//! +//! The rewrite keeps `dist.reference` (the upstream commit), so it is the +//! cross-check: packagist must still serve that exact reference for the +//! version, or the lock pinned something packagist no longer describes (a +//! moved tag, another repository) and the pin is refused. +//! +//! Only packagist-sourced entries are restored: the entry must carry +//! packagist's `notification-url`, or — composer omits it for hand-trimmed +//! and some older locks — `composer.json` must declare no custom +//! `repositories`. Anything else may have come from a private repository +//! whose metadata this restore cannot read, so it is refused. +//! +//! The lock is edited as text so every other byte stays composer's: the +//! blocks are rebuilt in composer's key order at the indent the entry +//! already uses, with the file's own slash style (`\/` in locks written by +//! composer versions that escaped slashes) and line endings. +//! `content-hash` hashes composer.json, not the lock, and is untouched. + +use std::collections::BTreeMap; + +use serde_json::Value; + +use super::super::{find_composer_entry, json_object_end_from, json_string_field, ComposerEntry}; +use super::{Ctx, FormatResult, HostedPin, View}; +use crate::crawlers::composer_crawler::normalize_version; + +const COMPOSER_LOCK: &str = "composer.lock"; +const DIST_KEY: &str = "\"dist\": {"; +const SOURCE_KEY: &str = "\"source\": {"; +/// The `notification-url` composer records for packagist packages. +const PACKAGIST_NOTIFY: &str = "https://packagist.org/downloads/"; + +/// One hosted entry to restore. +struct Hit { + uuid: String, + /// Lowercase `vendor/package` (the purl name; packagist's key). + name: String, + /// The version as the lock spells it (`v2.0.0`, `dev-main`). + locked_version: String, + /// The purl version, for re-locating the entry. + purl_version: String, + /// The lock's surviving `dist.reference`. + reference: Option, +} + +/// Whether composer resolves `version` from the `~dev` metadata file +/// (branch versions: `dev-` and `.x-dev`). +fn is_dev_version(version: &str) -> bool { + let lower = version.to_ascii_lowercase(); + lower.starts_with("dev-") || lower.ends_with("-dev") +} + +/// The `[start, end]` byte range of the entry's `"dist": {…}` object. +fn dist_range(content: &str, entry: (usize, usize)) -> Option<(usize, usize)> { + let start = entry.0 + content[entry.0..=entry.1].find(DIST_KEY)?; + let end = json_object_end_from(content, start + DIST_KEY.len())?; + Some((start, end)) +} + +/// A JSON string literal in the lock's style: `\/` when the lock escapes +/// slashes (older composer), plain otherwise (composer 2's +/// `JSON_UNESCAPED_SLASHES`). +fn json_str(value: &str, escaped_slashes: bool) -> String { + let lit = Value::String(value.to_string()).to_string(); + if escaped_slashes { + lit.replace('/', "\\/") + } else { + lit + } +} + +/// `"": {` + the string fields of `obj` named in `keys` (in that order, +/// absent ones skipped) at `inner`, closed at `outer` — composer's pretty +/// print. `None` when `obj` lacks a string `type` or `url`. +fn render_block( + key: &str, + obj: &serde_json::Map, + keys: &[&str], + inner: &str, + outer: &str, + eol: &str, + escaped: bool, +) -> Option { + for required in ["type", "url"] { + obj.get(required)?.as_str()?; + } + let fields: Vec = keys + .iter() + .filter_map(|k| { + let v = obj.get(*k)?.as_str()?; + Some(format!("{inner}\"{k}\": {}", json_str(v, escaped))) + }) + .collect(); + Some(format!( + "\"{key}\": {{{eol}{}{eol}{outer}}}", + fields.join(&format!(",{eol}")) + )) +} + +/// Leading whitespace of the line holding byte `at`. +fn indent_at(content: &str, at: usize) -> &str { + let line_start = content[..at].rfind('\n').map_or(0, |i| i + 1); + let line = &content[line_start..]; + &line[..line.len() - line.trim_start_matches([' ', '\t']).len()] +} + +/// Whether `composer.json` points composer at anything but packagist. +/// `Err` when it exists but cannot be read as JSON (the refusal reason). +fn declares_custom_repositories(composer_json: Option<&str>) -> Result { + let Some(text) = composer_json else { + return Ok(false); + }; + let doc: Value = + serde_json::from_str(text).map_err(|e| format!("composer.json is not JSON: {e}"))?; + Ok(match doc.get("repositories") { + None | Some(Value::Null) => false, + Some(Value::Array(a)) => !a.is_empty(), + Some(Value::Object(o)) => !o.is_empty(), + Some(_) => true, + }) +} + +/// The packagist version entry that locked `locked` (exact pretty version +/// first, then composer's leading-`v` normalization). +fn pick_version<'v>(versions: &'v [Value], locked: &str) -> Result<&'v Value, String> { + let version_of = |v: &&Value| v.get("version").and_then(Value::as_str).map(str::to_string); + let exact: Vec<&Value> = versions + .iter() + .filter(|v| version_of(v).as_deref() == Some(locked)) + .collect(); + let candidates = if exact.is_empty() { + versions + .iter() + .filter(|v| { + version_of(v).is_some_and(|x| normalize_version(&x) == normalize_version(locked)) + }) + .collect() + } else { + exact + }; + match candidates.as_slice() { + [one] => Ok(one), + [] => Err(format!("packagist does not list version {locked}")), + _ => Err(format!("packagist lists version {locked} more than once")), + } +} + +pub(crate) async fn restore( + view: &mut View<'_>, + pins: &[&HostedPin], + _files: &[String], + ctx: &Ctx<'_>, +) -> FormatResult { + let mut result = FormatResult::default(); + let content = match view.read(COMPOSER_LOCK).await { + Ok(Some(text)) => text, + Ok(None) => { + for pin in pins { + result.refuse(&pin.uuid, "composer.lock no longer exists"); + } + return result; + } + Err(e) => { + for pin in pins { + result.refuse(&pin.uuid, e.clone()); + } + return result; + } + }; + let composer_json = view.read("composer.json").await.ok().flatten(); + let custom_repos = declares_custom_repositories(composer_json.as_deref()); + + let mut hits: Vec = Vec::new(); + for pin in pins { + let Some((name, version)) = pin.name_version() else { + result.refuse(&pin.uuid, format!("{} is not a composer purl", pin.purl)); + continue; + }; + let ComposerEntry::Found(start, end) = find_composer_entry(&content, &name, &version) + else { + continue; + }; + let entry = &content[start..=end]; + let Some((d_start, d_end)) = dist_range(&content, (start, end)) else { + continue; + }; + let dist = &content[d_start..=d_end]; + let wired = json_string_field(dist, "url") + .and_then(|u| ctx.hosted_uuid(u)) + .is_some_and(|u| u == pin.uuid); + if !wired { + continue; + } + // Packagist gate (module docs). + match json_string_field(entry, "notification-url").map(|u| u.replace("\\/", "/")) { + Some(u) if u == PACKAGIST_NOTIFY => {} + Some(u) => { + result.refuse( + &pin.uuid, + format!("{name} was locked from {u}, not packagist, whose metadata this restore cannot read"), + ); + continue; + } + None => match &custom_repos { + Ok(false) => {} + Ok(true) => { + result.refuse( + &pin.uuid, + format!( + "composer.json declares custom repositories and the {name} lock \ + entry does not record packagist as its origin" + ), + ); + continue; + } + Err(why) => { + result.refuse(&pin.uuid, why.clone()); + continue; + } + }, + } + let Some(locked_version) = json_string_field(entry, "version") else { + continue; + }; + hits.push(Hit { + uuid: pin.uuid.clone(), + name: name.to_ascii_lowercase(), + locked_version: locked_version.to_string(), + purl_version: version, + reference: json_string_field(dist, "reference").map(|r| r.replace("\\/", "/")), + }); + } + if hits.is_empty() { + return result; + } + + let lookups = hits.iter().map(|h| async move { + ( + h.uuid.clone(), + ctx.client + .packagist_versions(&h.name, is_dev_version(&h.locked_version)) + .await, + ) + }); + let metadata: BTreeMap, String>> = + futures_util::future::join_all(lookups) + .await + .into_iter() + .collect(); + + let escaped = content.contains("\\/"); + let eol = if content.contains("\r\n") { + "\r\n" + } else { + "\n" + }; + let mut content = content; + let mut changed = false; + for hit in &hits { + let label = format!("{}@{}", hit.name, hit.locked_version); + let versions = match metadata.get(&hit.uuid) { + Some(Ok(v)) => v, + Some(Err(why)) => { + result.refuse(&hit.uuid, format!("{label}: {why}")); + continue; + } + None => continue, + }; + let upstream = match pick_version(versions, &hit.locked_version) { + Ok(v) => v, + Err(why) => { + result.refuse(&hit.uuid, format!("{label}: {why}")); + continue; + } + }; + let Some(dist) = upstream.get("dist").and_then(Value::as_object) else { + result.refuse( + &hit.uuid, + format!("{label}: packagist serves no dist for it"), + ); + continue; + }; + let upstream_ref = dist.get("reference").and_then(Value::as_str); + if upstream_ref != hit.reference.as_deref() { + result.refuse( + &hit.uuid, + format!( + "{label}: the lock pins dist.reference {:?} but packagist now serves {:?}", + hit.reference.as_deref().unwrap_or(""), + upstream_ref.unwrap_or("") + ), + ); + continue; + } + // Offsets moved with every earlier hit's edit: locate again. + let ComposerEntry::Found(start, end) = + find_composer_entry(&content, &hit.name, &hit.purl_version) + else { + continue; + }; + let Some((d_start, d_end)) = dist_range(&content, (start, end)) else { + continue; + }; + let outer = indent_at(&content, d_start).to_string(); + let block = &content[d_start..=d_end]; + let inner = block + .split('\n') + .nth(1) + .map(|l| &l[..l.len() - l.trim_start_matches([' ', '\t']).len()]) + .filter(|i| !i.is_empty()) + .map(str::to_string) + .unwrap_or_else(|| format!("{outer} ")); + let Some(dist_text) = render_block( + "dist", + dist, + &["type", "url", "reference", "shasum"], + &inner, + &outer, + eol, + escaped, + ) else { + result.refuse( + &hit.uuid, + format!("{label}: packagist's dist has no type or url"), + ); + continue; + }; + // Re-insert the source the rewriter dropped — unless the entry + // still carries one (a lock redirected before the drop, or a + // source it could not remove). + let has_source = content[start..d_start].contains(SOURCE_KEY); + let source_text = match upstream.get("source").and_then(Value::as_object) { + Some(source) if !has_source => { + match render_block( + "source", + source, + &["type", "url", "reference"], + &inner, + &outer, + eol, + escaped, + ) { + Some(text) => format!("{text},{eol}{outer}"), + None => { + result.refuse( + &hit.uuid, + format!("{label}: packagist's source has no type or url"), + ); + continue; + } + } + } + _ => String::new(), + }; + content.replace_range(d_start..=d_end, &format!("{source_text}{dist_text}")); + changed = true; + result.handled.insert(hit.uuid.clone()); + } + if changed { + view.write(COMPOSER_LOCK, content); + } + result +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn dev_versions_use_the_dev_metadata_file() { + assert!(is_dev_version("dev-main")); + assert!(is_dev_version("2.x-dev")); + assert!(is_dev_version("2.1.x-DEV")); + assert!(!is_dev_version("v2.0.0")); + assert!(!is_dev_version("2.0.0-beta1")); + } + + #[test] + fn blocks_render_in_composer_order_and_slash_style() { + let dist = json!({ + "shasum": "", + "url": "https://api.github.com/repos/a/b/zipball/abc", + "type": "zip", + "reference": "abc", + "mirrors": [{"url": "x"}] + }); + let obj = dist.as_object().unwrap(); + let keys = ["type", "url", "reference", "shasum"]; + assert_eq!( + render_block( + "dist", + obj, + &keys, + " ", + " ", + "\n", + false + ) + .unwrap(), + "\"dist\": {\n \"type\": \"zip\",\n \"url\": \ + \"https://api.github.com/repos/a/b/zipball/abc\",\n \ + \"reference\": \"abc\",\n \"shasum\": \"\"\n }" + ); + let escaped = render_block("dist", obj, &keys, " ", "", "\r\n", true).unwrap(); + assert!(escaped.contains("\"https:\\/\\/api.github.com\\/repos\\/a\\/b\\/zipball\\/abc\"")); + assert!(escaped.contains(",\r\n \"reference\"")); + // No shasum upstream: the key stays absent (fixture no-shasum-key). + let bare = json!({"type": "zip", "url": "u", "reference": "r"}); + assert!(!render_block( + "dist", + bare.as_object().unwrap(), + &keys, + " ", + "", + "\n", + false + ) + .unwrap() + .contains("shasum")); + assert!(render_block( + "dist", + json!({"url": "u"}).as_object().unwrap(), + &keys, + "", + "", + "\n", + false + ) + .is_none()); + } + + #[test] + fn custom_repositories_gate() { + assert_eq!(declares_custom_repositories(None), Ok(false)); + assert_eq!(declares_custom_repositories(Some("{}")), Ok(false)); + assert_eq!( + declares_custom_repositories(Some(r#"{"repositories": []}"#)), + Ok(false) + ); + assert_eq!( + declares_custom_repositories(Some( + r#"{"repositories": [{"type": "vcs", "url": "https://git.example/x"}]}"# + )), + Ok(true) + ); + assert_eq!( + declares_custom_repositories(Some(r#"{"repositories": {"packagist.org": false}}"#)), + Ok(true) + ); + assert!(declares_custom_repositories(Some("{")).is_err()); + } + + #[test] + fn version_pick_prefers_the_pretty_spelling() { + let versions = vec![ + json!({"version": "2.0.0"}), + json!({"version": "v2.0.0"}), + json!({"version": "1.0.0"}), + ]; + assert_eq!( + pick_version(&versions, "v2.0.0").unwrap()["version"], + "v2.0.0" + ); + assert_eq!( + pick_version(&versions, "1.0.0").unwrap()["version"], + "1.0.0" + ); + assert_eq!( + pick_version(&[json!({"version": "v3.0.0"})], "3.0.0").unwrap()["version"], + "v3.0.0" + ); + assert!(pick_version(&versions, "9.9.9").is_err()); + let dup = vec![json!({"version": "2.0.0"}), json!({"version": "2.0.0"})]; + assert!(pick_version(&dup, "2.0.0").is_err()); + } +} diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/gem.rs b/crates/socket-patch-core/src/patch/redirect/upstream/gem.rs new file mode 100644 index 000000000..3fb0ff052 --- /dev/null +++ b/crates/socket-patch-core/src/patch/redirect/upstream/gem.rs @@ -0,0 +1,959 @@ +//! RubyGems upstream restore: a hosted gem goes back to the upstream `GEM` +//! section of `Gemfile.lock` / `gems.locked`, and the `source "" do … end` block the hosted rewriter (`rewrite_gem`) put into +//! `Gemfile` / `gems.rb` is undone. Discovery records only the lock; the +//! manifest is its bundler sibling ([`bundler_manifest_for`]). +//! +//! The lock states a hosted gem can be in, and what each restore does: +//! +//! * **Converged** (bundler ≥ 2.2 after the rewrite or an unfrozen install; +//! the rewriter writes it itself in the `CHECKSUMS` era): the gem's spec +//! (+ its dependency sub-lines) sits in a `GEM` section of its own whose +//! single remote is the patch registry. The section is deleted and the +//! spec moves back, in name order, into the upstream `GEM` section: the +//! one remaining non-Socket single-remote section, or — when there are +//! several — the one naming the manifest's global `source`, else the +//! rubygems.org one; anything else is ambiguous and refused. +//! * **Merged** (bundler ≤ 2.1 writes every rubygems source into ONE `GEM` +//! section): the Socket `remote:` line is dropped from it. +//! * **Mixed** (bundler < 2.6: the rewriter edits only the manifest and the +//! lock still records the upstream source; see `redirect_gem_frozen_install`): +//! discovery finds no pin for it (no lock wiring), so it is only unwound +//! when a caller hands in a pin for it — the manifest block alone is then +//! undone, and the untouched lock says exactly how the gem was declared. +//! +//! In every state a `CHECKSUMS` entry is re-pinned to the upstream sha256 +//! from the rubygems.org compact index (`info/`, what bundler itself +//! records; other upstream remotes are refused: their index may need +//! credentials and is not the default registry), and the `DEPENDENCIES` +//! source pin (`name (= v)!`) loses its `!`. +//! +//! What the rewrite discards is NOT derivable, so the restore picks the +//! installable reading and documents it: +//! +//! * the original declaration's version constraint (`"~> 7.0"`, or none), +//! quote style, parenthesized form and comment: the gem comes back as +//! `gem "", ""[, ]` — the exact pin the lock +//! records as `name (= version)`, so the pair stays frozen-installable +//! (the mixed state keeps the lock's own constraint instead); +//! * the blank lines and indentation the rewriter's `^\s*gem` match +//! swallowed before the declaration: see [`declaration_prefix`]; +//! * whether the gem was declared at all. The rewriter APPENDS a block for a +//! transitive gem and adds its `DEPENDENCIES` entry, but a direct gem on +//! the manifest's last line produces the same bytes. The block and entry +//! are removed only when that is provable: an option-less block the +//! rewriter could not have written in place (a blank line before it — +//! the in-place match swallows every blank line before the declaration) +//! that another locked spec depends on. Otherwise the gem is kept as a +//! direct pin: a stray exact pin installs the same bytes, while dropping +//! a real declaration would stop `Bundler.require` loading the gem. +//! * a `CHECKSUMS` entry the rewriter ADDED is only recognizable next to +//! bundler's own bare entry for the gem (then it is dropped); otherwise it +//! is re-pinned in place. + +use std::collections::{BTreeMap, BTreeSet}; + +use regex::Regex; + +use super::{Ctx, FormatResult, HostedPin, View}; +use crate::utils::line_endings::{to_lf, LineEndings}; +use crate::vendor::gem::{gem_declaration_any, quoted_literal}; +use crate::vendor::gemfile_lock::{ + bundler_manifest_for, parse_spec, same_remote, split_checksum_entry, BUNDLER_LOCKS, +}; + +/// The default upstream `GEM` remote. +const RUBYGEMS_REMOTE: &str = "https://rubygems.org/"; + +// ── lock model (LF text, split on `\n`) ───────────────────────────────────── + +fn is_header(line: &str) -> bool { + !line.is_empty() && !line.starts_with(' ') +} + +/// Exactly `n` spaces of indent, then content. +fn indented(line: &str, n: usize) -> Option<&str> { + let rest = line.get(n..)?; + (line[..n].bytes().all(|b| b == b' ') && !rest.starts_with(' ') && !rest.is_empty()) + .then_some(rest) +} + +/// One 4-space spec entry and its 6-space dependency sub-lines. +struct Entry { + line: usize, + /// The last sub-line (== `line` when it has none). + last: usize, + name: String, + version: String, + platform: bool, +} + +/// One `GEM` section: `[start, end)` runs from its header to the next +/// header (its trailing blank separator included). +struct GemSec { + start: usize, + end: usize, + remotes: Vec<(usize, String)>, + specs_line: Option, + entries: Vec, +} + +/// `[start, end)` of every section, with its header. +fn section_ranges<'l>(lines: &[&'l str]) -> Vec<(&'l str, usize, usize)> { + let mut out: Vec<(&str, usize, usize)> = Vec::new(); + for (i, line) in lines.iter().enumerate() { + if is_header(line) { + if let Some(last) = out.last_mut() { + last.2 = i; + } + out.push((line.trim_end(), i, lines.len())); + } + } + out +} + +fn gem_sections(lines: &[&str]) -> Vec { + let mut out = Vec::new(); + for (header, start, end) in section_ranges(lines) { + if header != "GEM" { + continue; + } + let mut sec = GemSec { + start, + end, + remotes: Vec::new(), + specs_line: None, + entries: Vec::new(), + }; + for (k, line) in lines.iter().enumerate().take(end).skip(start + 1) { + if let Some(key) = indented(line, 2) { + if let Some(url) = key.strip_prefix("remote:") { + sec.remotes.push((k, url.trim().to_string())); + } else if key.trim_end() == "specs:" { + sec.specs_line = Some(k); + } + } else if let Some(entry) = indented(line, 4).filter(|_| sec.specs_line.is_some()) { + let spec = parse_spec(entry.trim_end()); + sec.entries.push(Entry { + line: k, + last: k, + name: spec.map(|s| s.name).unwrap_or(entry).to_string(), + version: spec.map(|s| s.version).unwrap_or_default().to_string(), + platform: spec.is_none_or(|s| s.platform.is_some()), + }); + } else if line.starts_with(" ") { + if let Some(e) = sec.entries.last_mut() { + e.last = k; + } + } + } + out.push(sec); + } + out +} + +/// The line range of the column-0 section `header`, header excluded. +fn named_section(lines: &[&str], header: &str) -> Option<(usize, usize)> { + section_ranges(lines) + .into_iter() + .find(|(h, _, _)| *h == header) + .map(|(_, s, e)| (s + 1, e)) +} + +/// The `DEPENDENCIES` entry of `name`: its line and trimmed text. +fn dependency_line<'l>(lines: &[&'l str], name: &str) -> Option<(usize, &'l str)> { + let (s, e) = named_section(lines, "DEPENDENCIES")?; + (s..e).find_map(|k| { + let entry = indented(lines[k], 2)?.trim_end(); + let dep = entry.split([' ', '(']).next()?.trim_end_matches('!'); + (dep == name).then_some((k, entry)) + }) +} + +/// Whether another locked spec depends on `name` (a 6-space sub-line). +fn is_subdependency(lines: &[&str], name: &str) -> bool { + lines + .iter() + .any(|l| indented(l, 6).is_some_and(|d| d.split([' ', '(']).next() == Some(name))) +} + +/// How the lock wires patch `uuid`. +enum Wiring { + /// A `GEM` section of its own (index into the sections). + Own(usize), + /// One `remote:` line (the line index) of a multi-remote section. + Merged(usize, usize), +} + +fn wiring(secs: &[GemSec], uuid: &str, ctx: &Ctx<'_>) -> Result, String> { + let hits: Vec<(usize, usize)> = secs + .iter() + .enumerate() + .flat_map(|(i, s)| { + s.remotes + .iter() + .filter(|(_, r)| ctx.hosted_uuid(r).as_deref() == Some(uuid)) + .map(move |(k, _)| (i, *k)) + }) + .collect(); + match hits.as_slice() { + [] => Ok(None), + [(i, k)] if secs[*i].remotes.len() > 1 => Ok(Some(Wiring::Merged(*i, *k))), + [(i, _)] => Ok(Some(Wiring::Own(*i))), + _ => Err("several GEM remotes name the patch".to_string()), + } +} + +/// The upstream section a spec moves back to (module docs). +fn choose_upstream( + secs: &[GemSec], + own: usize, + globals: &[String], + ctx: &Ctx<'_>, +) -> Result { + let candidates: Vec = (0..secs.len()) + .filter(|&i| { + i != own + && secs[i].remotes.len() == 1 + && ctx.hosted_uuid(&secs[i].remotes[0].1).is_none() + }) + .collect(); + let remote = |i: &usize| secs[*i].remotes[0].1.as_str(); + match candidates.as_slice() { + [] => Err("the lock has no upstream GEM section to move it back to".to_string()), + [one] => Ok(*one), + _ => { + let single = |keep: &dyn Fn(&usize) -> bool| match candidates + .iter() + .filter(|i| keep(i)) + .collect::>() + .as_slice() + { + [one] => Some(**one), + _ => None, + }; + if let Some(one) = single(&|i| globals.iter().any(|g| same_remote(g, remote(i)))) + .or_else(|| single(&|i| same_remote(remote(i), RUBYGEMS_REMOTE))) + { + return Ok(one); + } + Err(format!( + "the lock has {} upstream GEM sections and none is singled out by the \ + manifest's global source or rubygems.org", + candidates.len() + )) + } + } +} + +/// The line after which a spec named `name-version` sorts into `sec` +/// (bundler writes specs sorted by full name). +fn insertion_point(sec: &GemSec, full_name: &str) -> Option { + let pred = sec + .entries + .iter() + .rfind(|e| { + let full = if e.version.is_empty() { + e.name.clone() + } else { + format!("{}-{}", e.name, e.version) + }; + full.as_str() < full_name + }); + pred.map(|e| e.last).or(sec.specs_line) +} + +/// A lock restore step's outcome. +enum LockEdit { + /// The `CHECKSUMS` entry must be re-pinned from `remote` first. + NeedsSha { + remote: Option, + }, + Done(String), +} + +/// One pin's coordinates. +struct Gem<'p> { + uuid: &'p str, + name: String, + version: String, +} + +/// Restore `gem` in the LF lock text (module docs). `transitive`: drop the +/// `DEPENDENCIES` entry instead of unpinning it. +fn lock_edit( + lock: &str, + gem: &Gem<'_>, + globals: &[String], + transitive: bool, + sha: Option<&str>, + ctx: &Ctx<'_>, +) -> Result { + let lines: Vec<&str> = lock.split('\n').collect(); + let secs = gem_sections(&lines); + let wiring = wiring(&secs, gem.uuid, ctx)?; + let mut drop: BTreeSet = BTreeSet::new(); + let mut replace: BTreeMap = BTreeMap::new(); + let mut insert_after: BTreeMap> = BTreeMap::new(); + let is_ours = |e: &Entry| e.name == gem.name && e.version == gem.version && !e.platform; + let checksum_remote = match &wiring { + Some(Wiring::Own(si)) => { + let sec = &secs[*si]; + let entry = match sec.entries.as_slice() { + [e] if is_ours(e) => e, + _ => { + return Err(format!( + "its Socket GEM section does not lock exactly {} ({})", + gem.name, gem.version + )) + } + }; + let up = choose_upstream(&secs, *si, globals, ctx)?; + let at = insertion_point(&secs[up], &format!("{}-{}", gem.name, gem.version)) + .ok_or("the upstream GEM section has no `specs:` list")?; + drop.extend(sec.start..sec.end); + insert_after.insert(at, lines[entry.line..=entry.last].to_vec()); + Some(secs[up].remotes[0].1.clone()) + } + Some(Wiring::Merged(si, line)) => { + let sec = &secs[*si]; + if !sec.entries.iter().any(is_ours) { + return Err(format!( + "its merged GEM section does not lock {} ({})", + gem.name, gem.version + )); + } + drop.insert(*line); + let rest: Vec<&String> = sec + .remotes + .iter() + .filter(|(k, _)| k != line) + .map(|(_, r)| r) + .collect(); + match rest.as_slice() { + [one] => Some((*one).clone()), + _ => None, + } + } + None => { + let holders: Vec<&GemSec> = secs + .iter() + .filter(|s| s.entries.iter().any(is_ours)) + .collect(); + match holders.as_slice() { + [s] if s.remotes.len() == 1 => Some(s.remotes[0].1.clone()), + _ => None, + } + } + }; + if wiring.is_some() { + if let Some((k, entry)) = dependency_line(&lines, &gem.name) { + if transitive { + drop.insert(k); + } else if let Some(unpinned) = entry.strip_suffix('!') { + replace.insert(k, format!(" {unpinned}")); + } + } + } + if let Some((s, e)) = named_section(&lines, "CHECKSUMS") { + let mut with_sha = Vec::new(); + let mut bare = false; + for (k, line) in lines.iter().enumerate().take(e).skip(s) { + let Some(entry) = indented(line, 2) else { + continue; + }; + let Some((name, token, tail)) = split_checksum_entry(entry.trim_end()) else { + continue; + }; + if name != gem.name || token != gem.version { + continue; + } + if tail.contains("sha256=") { + with_sha.push(k); + } else { + bare = true; + } + } + match with_sha.as_slice() { + [] => {} + [k] if bare => { + drop.insert(*k); + } + [k] => { + let Some(sha) = sha else { + return Ok(LockEdit::NeedsSha { + remote: checksum_remote, + }); + }; + let re = Regex::new(r"sha256=[0-9A-Fa-f]*").expect("static sha256 regex is valid"); + replace.insert( + *k, + re.replace(lines[*k], format!("sha256={sha}")).into_owned(), + ); + } + _ => return Err("CHECKSUMS pins it more than once".to_string()), + } + } + let mut out: Vec = Vec::with_capacity(lines.len()); + for (k, line) in lines.iter().enumerate() { + if !drop.contains(&k) { + out.push(replace.get(&k).cloned().unwrap_or_else(|| line.to_string())); + } + if let Some(block) = insert_after.get(&k) { + out.extend(block.iter().map(|l| l.to_string())); + } + } + Ok(LockEdit::Done(out.join("\n"))) +} + +// ── manifest (Gemfile / gems.rb) ──────────────────────────────────────────── + +/// The rewriter's `source "" do\n gem "n", "v"[, opts]\nend` block. +struct Block { + start: usize, + /// Past the `end` line's line break. + end: usize, + opts: Option, + /// Whether a line break followed `end` (the declaration's own). + eol: bool, +} + +fn find_block( + text: &str, + gem: &Gem<'_>, + rel: &str, + ctx: &Ctx<'_>, +) -> Result, String> { + let re = Regex::new(&format!( + r#"(?m)^source "([^"\r\n]*)" do\r?\n gem "{}", "{}"(?:, ([^\r\n]*))?\r?\nend(\r?\n|\z)"#, + regex::escape(&gem.name), + regex::escape(&gem.version) + )) + .expect("source-block regex from escaped coordinates is valid"); + let hits: Vec = re + .captures_iter(text) + .filter(|c| ctx.hosted_uuid(&c[1]).as_deref() == Some(gem.uuid)) + .map(|c| { + let m = c.get(0).expect("group 0 is the whole match"); + Block { + start: m.start(), + end: m.end(), + opts: c.get(2).map(|o| o.as_str().to_string()), + eol: !c[3].is_empty(), + } + }) + .collect(); + let rest_mentions = |b: Option<&Block>| { + let rest = match b { + Some(b) => format!("{}{}", &text[..b.start], &text[b.end..]), + None => text.to_string(), + }; + rest.contains(gem.uuid) + }; + match hits.as_slice() { + [] if rest_mentions(None) => Err(format!( + "{rel} wires it in a shape other than the source block socket-patch writes" + )), + [] => Ok(None), + [_] if rest_mentions(hits.first()) => Err(format!( + "{rel} names the patch outside the source block socket-patch writes" + )), + [_] => Ok(hits.into_iter().next()), + _ => Err(format!("{rel} declares it in several Socket source blocks")), + } +} + +/// The line before byte `at` (without its line break); `None` at the start. +fn line_before(text: &str, at: usize) -> Option<&str> { + let before = text[..at].strip_suffix('\n')?; + let before = before.strip_suffix('\r').unwrap_or(before); + Some(&before[before.rfind('\n').map_or(0, |i| i + 1)..]) +} + +fn indent_of(line: &str) -> &str { + &line[..line.len() - line.trim_start().len()] +} + +/// Whether the rewriter can only have APPENDED `block` (a transitive gem): +/// its in-place rewrite swallows every blank line before the declaration, +/// so a blank line right before the block rules it out. +fn provably_appended(text: &str, block: &Block) -> bool { + line_before(text, block.start).is_some_and(|l| l.trim().is_empty()) +} + +/// The blank line + indent to put before a declaration restored in place of +/// `block`. The rewriter's `^\s*gem` match swallowed whatever whitespace +/// preceded the original line, which no file records, so this re-derives the +/// conventional layout from the neighbors: inside a block (`group … do`) +/// or right after another declaration or a comment, the neighbor's indent +/// and no blank line; after anything else (`source`, `ruby`, `end`, …) one +/// blank line and the indent of the declaration that follows, if any. +fn declaration_prefix(text: &str, block: &Block, eol: &str) -> String { + let Some(prev) = line_before(text, block.start) else { + return String::new(); + }; + let trimmed = prev.trim(); + if trimmed.is_empty() { + return String::new(); + } + let code = trimmed.split('#').next().unwrap_or_default().trim_end(); + if code == "do" || code.ends_with(" do") || (code.ends_with('|') && code.contains(" do |")) { + return format!("{} ", indent_of(prev)); + } + if trimmed.starts_with('#') || gem_declaration_any(trimmed).is_some() { + return indent_of(prev).to_string(); + } + let next = text[block.end..].split('\n').next().unwrap_or_default(); + let indent = if gem_declaration_any(next.trim()).is_some() { + indent_of(next) + } else { + "" + }; + format!("{eol}{indent}") +} + +/// How the gem comes back into the manifest. +enum Decl { + /// Gone: the block was the rewriter's append for a transitive gem. + Transitive, + /// `gem ""[, ]`; `args` is the version constraint list + /// (`, "7.0.0"`). + Direct(String), +} + +fn restore_manifest(text: &str, block: &Block, gem: &Gem<'_>, decl: &Decl) -> String { + let eol = if text.contains("\r\n") { "\r\n" } else { "\n" }; + let replacement = match decl { + Decl::Transitive => String::new(), + Decl::Direct(args) => { + let opts = block + .opts + .as_deref() + .map(|o| format!(", {o}")) + .unwrap_or_default(); + format!( + "{}gem \"{}\"{args}{opts}{}", + if provably_appended(text, block) { + String::new() + } else { + declaration_prefix(text, block, eol) + }, + gem.name, + if block.eol { eol } else { "" } + ) + } + }; + format!( + "{}{replacement}{}", + &text[..block.start], + &text[block.end..] + ) +} + +/// The manifest's global `source ""` declarations (no block). +fn global_sources(manifest: &str) -> Vec { + manifest + .lines() + .filter_map(|l| { + let rest = l.trim().strip_prefix("source")?.trim_start(); + let (rest, paren) = match rest.strip_prefix('(') { + Some(r) => (r.trim_start(), true), + None => (rest, false), + }; + let (_, url, tail) = quoted_literal(rest)?; + let tail = tail.trim_start(); + let tail = if paren { tail.strip_prefix(')')? } else { tail }; + let code = tail.split('#').next().unwrap_or_default().trim(); + code.is_empty().then(|| url.to_string()) + }) + .collect() +} + +/// The version-constraint args of a `DEPENDENCIES` entry (`rails (~> 7.0, +/// >= 7.0.1)` → `, "~> 7.0", ">= 7.0.1"`). +fn constraint_args(entry: &str) -> String { + let entry = entry.trim_end_matches('!'); + let Some((_, rest)) = entry.split_once(" (") else { + return String::new(); + }; + rest.trim_end_matches(')') + .split(", ") + .filter(|c| !c.is_empty()) + .map(|c| format!(", \"{c}\"")) + .collect() +} + +pub(crate) async fn restore( + view: &mut View<'_>, + pins: &[&HostedPin], + files: &[String], + ctx: &Ctx<'_>, +) -> FormatResult { + let mut result = FormatResult::default(); + for lock_rel in BUNDLER_LOCKS { + let manifest_rel = bundler_manifest_for(lock_rel); + if !files.iter().any(|f| f == lock_rel || f == manifest_rel) { + continue; + } + let (lock_raw, manifest_raw) = + match (view.read(lock_rel).await, view.read(manifest_rel).await) { + (Ok(l), Ok(m)) => (l, m), + (Err(e), _) | (_, Err(e)) => { + for pin in pins { + result.refuse(&pin.uuid, e.clone()); + } + continue; + } + }; + let endings = lock_raw.as_deref().map(LineEndings::of); + let mut lock: Option = lock_raw.as_deref().map(|t| to_lf(t).into_owned()); + let mut manifest = manifest_raw.clone(); + let globals = manifest.as_deref().map(global_sources).unwrap_or_default(); + for pin in pins { + if result.refused.contains_key(&pin.uuid) { + continue; + } + let Some((name, version)) = pin.name_version() else { + result.refuse(&pin.uuid, format!("{} is not a gem purl", pin.purl)); + continue; + }; + let gem = Gem { + uuid: &pin.uuid, + name, + version, + }; + match restore_one( + &gem, + lock.as_deref(), + manifest.as_deref(), + manifest_rel, + &globals, + ctx, + ) + .await + { + Ok(None) => {} + Ok(Some((next_lock, next_manifest))) => { + if endings == Some(LineEndings::Mixed) && next_lock != lock { + result.refuse( + &pin.uuid, + format!("{lock_rel} mixes CRLF and LF line endings"), + ); + continue; + } + lock = next_lock; + manifest = next_manifest; + result.handled.insert(pin.uuid.clone()); + } + Err(why) => result.refuse(&pin.uuid, why), + } + } + if let (Some(next), Some(endings)) = (lock, endings) { + let next = endings.restore(&next).into_owned(); + if lock_raw.as_deref() != Some(next.as_str()) { + view.write(lock_rel, next); + } + } + if manifest != manifest_raw { + if let Some(next) = manifest { + view.write(manifest_rel, next); + } + } + } + result +} + +/// Restore one gem in a lock + manifest pair: `Ok(None)` when neither wires +/// it, else the next `(lock, manifest)` texts. +async fn restore_one( + gem: &Gem<'_>, + lock: Option<&str>, + manifest: Option<&str>, + manifest_rel: &str, + globals: &[String], + ctx: &Ctx<'_>, +) -> Result, Option)>, String> { + let block = match manifest { + Some(text) => find_block(text, gem, manifest_rel, ctx)?, + None => None, + }; + let lines: Vec<&str> = lock.map(|l| l.split('\n').collect()).unwrap_or_default(); + let wired = match lock { + Some(_) => wiring(&gem_sections(&lines), gem.uuid, ctx)?.is_some(), + None => false, + }; + if !wired && block.is_none() { + return Ok(None); + } + let decl = if wired { + let transitive = block.as_ref().is_some_and(|b| { + b.opts.is_none() + && manifest.is_some_and(|m| provably_appended(m, b)) + && is_subdependency(&lines, &gem.name) + }); + if transitive { + Decl::Transitive + } else { + Decl::Direct(format!(", \"{}\"", gem.version)) + } + } else if lock.is_some() { + // Mixed state: the lock was never touched, so its DEPENDENCIES say + // how (and whether) the manifest declared the gem. + match dependency_line(&lines, &gem.name) { + Some((_, entry)) => Decl::Direct(constraint_args(entry)), + None => Decl::Transitive, + } + } else { + Decl::Direct(format!(", \"{}\"", gem.version)) + }; + let transitive = matches!(decl, Decl::Transitive); + let next_lock = match lock { + None => None, + Some(text) => Some( + match lock_edit(text, gem, globals, transitive, None, ctx)? { + LockEdit::Done(next) => next, + LockEdit::NeedsSha { remote } => { + let remote = remote.ok_or("its upstream GEM remote is ambiguous")?; + if !same_remote(&remote, RUBYGEMS_REMOTE) { + return Err(format!( + "its upstream GEM remote {remote} is not rubygems.org, so its CHECKSUMS \ + sha256 cannot be re-derived" + )); + } + let sha = ctx + .client + .rubygems_sha256(&gem.name, &gem.version) + .await + .map_err(|why| format!("{} {}: {why}", gem.name, gem.version))?; + match lock_edit(text, gem, globals, transitive, Some(&sha), ctx)? { + LockEdit::Done(next) => next, + LockEdit::NeedsSha { .. } => unreachable!("a sha was supplied"), + } + } + }, + ), + }; + let next_manifest = match (manifest, &block) { + (Some(text), Some(b)) => Some(restore_manifest(text, b, gem, &decl)), + _ => manifest.map(str::to_string), + }; + Ok(Some((next_lock, next_manifest))) +} + +#[cfg(test)] +mod tests { + use super::*; + + const UUID: &str = "77777777-7777-7777-7777-777777777777"; + const IDX: &str = "https://patch.socket.dev/patch-registry/gem/11111111-1111-1111-1111-111111111111/77777777-7777-7777-7777-777777777777/"; + + fn ctx_with<'a>(client: &'a super::super::UpstreamClient) -> Ctx<'a> { + Ctx { + client, + origins: &[], + } + } + + fn gem() -> Gem<'static> { + Gem { + uuid: UUID, + name: "rails".into(), + version: "7.0.0".into(), + } + } + + fn done(e: LockEdit) -> String { + match e { + LockEdit::Done(t) => t, + LockEdit::NeedsSha { .. } => panic!("unexpected sha request"), + } + } + + #[test] + fn converged_section_moves_back_in_name_order() { + let client = super::super::UpstreamClient::new(true); + let ctx = ctx_with(&client); + let hosted = format!( + "GEM\n remote: {IDX}\n specs:\n rails (7.0.0)\n rack (>= 2)\n\nGEM\n \ + remote: https://rubygems.org/\n specs:\n puma (6.0.0)\n rack (3.0.0)\n \ + zeitwerk (2.6.0)\n\nPLATFORMS\n ruby\n\nDEPENDENCIES\n puma\n rails (= 7.0.0)!\n\n\ + BUNDLED WITH\n 2.4.0\n" + ); + let want = "GEM\n remote: https://rubygems.org/\n specs:\n puma (6.0.0)\n rack \ + (3.0.0)\n rails (7.0.0)\n rack (>= 2)\n zeitwerk (2.6.0)\n\nPLATFORMS\n \ + ruby\n\nDEPENDENCIES\n puma\n rails (= 7.0.0)\n\nBUNDLED WITH\n 2.4.0\n"; + // No CHECKSUMS (bundler 2.2–2.5): no registry lookup, offline works. + assert_eq!( + done(lock_edit(&hosted, &gem(), &[], false, None, &ctx).unwrap()), + want + ); + // Transitive: the DEPENDENCIES entry the rewriter added goes. + let out = done(lock_edit(&hosted, &gem(), &[], true, None, &ctx).unwrap()); + assert!(out.contains("DEPENDENCIES\n puma\n\n"), "{out}"); + } + + #[test] + fn checksums_need_the_upstream_sha() { + let client = super::super::UpstreamClient::new(true); + let ctx = ctx_with(&client); + let hosted = format!( + "GEM\n remote: https://rubygems.org/\n specs:\n\nGEM\n remote: {IDX}\n specs:\n \ + rails (7.0.0)\n\nDEPENDENCIES\n rails (= 7.0.0)!\n\nCHECKSUMS\n rails (7.0.0) \ + sha256={}\n", + "d".repeat(64) + ); + match lock_edit(&hosted, &gem(), &[], false, None, &ctx).unwrap() { + LockEdit::NeedsSha { remote } => { + assert_eq!(remote.as_deref(), Some("https://rubygems.org/")) + } + LockEdit::Done(_) => panic!("CHECKSUMS entry left patched"), + } + let sha = "2".repeat(64); + assert_eq!( + done(lock_edit(&hosted, &gem(), &[], false, Some(&sha), &ctx).unwrap()), + format!( + "GEM\n remote: https://rubygems.org/\n specs:\n rails (7.0.0)\n\nDEPENDENCIES\n \ + rails (= 7.0.0)\n\nCHECKSUMS\n rails (7.0.0) sha256={sha}\n" + ) + ); + // The rewriter's ADDED entry next to bundler's bare one is dropped. + let added = hosted.replace("CHECKSUMS\n", "CHECKSUMS\n rails (7.0.0)\n"); + assert!( + done(lock_edit(&added, &gem(), &[], false, None, &ctx).unwrap()) + .ends_with("CHECKSUMS\n rails (7.0.0)\n") + ); + } + + #[test] + fn merged_section_drops_only_the_socket_remote() { + let client = super::super::UpstreamClient::new(true); + let ctx = ctx_with(&client); + let hosted = format!( + "GEM\n remote: https://rubygems.org/\n remote: {IDX}\n specs:\n puma (6.0.0)\n \ + rails (7.0.0)\n\nDEPENDENCIES\n puma\n rails (= 7.0.0)!\n\nBUNDLED WITH\n 2.1.4\n" + ); + assert_eq!( + done(lock_edit(&hosted, &gem(), &[], false, None, &ctx).unwrap()), + "GEM\n remote: https://rubygems.org/\n specs:\n puma (6.0.0)\n rails (7.0.0)\n\n\ + DEPENDENCIES\n puma\n rails (= 7.0.0)\n\nBUNDLED WITH\n 2.1.4\n" + ); + } + + #[test] + fn ambiguous_or_foreign_sections_are_refused() { + let client = super::super::UpstreamClient::new(true); + let ctx = ctx_with(&client); + let two_upstreams = format!( + "GEM\n remote: https://gems.example/\n specs:\n\nGEM\n remote: https://mirror.example/\n \ + specs:\n\nGEM\n remote: {IDX}\n specs:\n rails (7.0.0)\n\nDEPENDENCIES\n rails (= 7.0.0)!\n" + ); + assert!(lock_edit(&two_upstreams, &gem(), &[], false, None, &ctx).is_err()); + // The manifest's global source singles one out. + let globals = vec!["https://mirror.example".to_string()]; + let out = done(lock_edit(&two_upstreams, &gem(), &globals, false, None, &ctx).unwrap()); + assert!( + out.contains("remote: https://mirror.example/\n specs:\n rails (7.0.0)\n"), + "{out}" + ); + // A Socket section locking another gem too is not ours to split. + let extra = format!( + "GEM\n remote: https://rubygems.org/\n specs:\n\nGEM\n remote: {IDX}\n specs:\n \ + rack (3.0.0)\n rails (7.0.0)\n\nDEPENDENCIES\n rails (= 7.0.0)!\n" + ); + assert!(lock_edit(&extra, &gem(), &[], false, None, &ctx).is_err()); + let none = format!( + "GEM\n remote: {IDX}\n specs:\n rails (7.0.0)\n\nDEPENDENCIES\n rails!\n" + ); + assert!(lock_edit(&none, &gem(), &[], false, None, &ctx).is_err()); + } + + #[test] + fn manifest_blocks_restore_in_place() { + let client = super::super::UpstreamClient::new(true); + let ctx = ctx_with(&client); + let block = format!("source \"{IDX}\" do\n gem \"rails\", \"7.0.0\"\nend"); + let direct = Decl::Direct(", \"7.0.0\"".into()); + let run = |text: &str, decl: &Decl| { + let b = find_block(text, &gem(), "Gemfile", &ctx).unwrap().unwrap(); + restore_manifest(text, &b, &gem(), decl) + }; + // After a global source: one blank line comes back. + let t = format!("source \"https://rubygems.org\"\n{block}\ngem \"puma\"\n"); + assert_eq!( + run(&t, &direct), + "source \"https://rubygems.org\"\n\ngem \"rails\", \"7.0.0\"\ngem \"puma\"\n" + ); + // Inside a group: the group's indent + 2, options kept. + let t = format!( + "group :test do\nsource \"{IDX}\" do\n gem \"rails\", \"7.0.0\", require: false\nend\nend\n" + ); + assert_eq!( + run(&t, &direct), + "group :test do\n gem \"rails\", \"7.0.0\", require: false\nend\n" + ); + // CRLF manifest (the rewriter wrote its block in LF). + let t = format!("source \"https://rubygems.org\"\r\ngem \"puma\"\r\n{block}\n"); + assert_eq!( + run(&t, &direct), + "source \"https://rubygems.org\"\r\ngem \"puma\"\r\ngem \"rails\", \"7.0.0\"\r\n" + ); + // Transitive append removed; mixed-state constraint kept. + let t = format!("source \"https://rubygems.org\"\n\ngem \"puma\"\n\n{block}\n"); + assert_eq!( + run(&t, &Decl::Transitive), + "source \"https://rubygems.org\"\n\ngem \"puma\"\n\n" + ); + let t = format!("gem \"puma\"\n{block}\n"); + assert_eq!( + run(&t, &Decl::Direct(constraint_args("rails (>= 6, ~> 7.0)"))), + "gem \"puma\"\ngem \"rails\", \">= 6\", \"~> 7.0\"\n" + ); + } + + #[test] + fn manifest_shapes_it_cannot_unwind_are_refused() { + let client = super::super::UpstreamClient::new(true); + let ctx = ctx_with(&client); + let block = format!("source \"{IDX}\" do\n gem \"rails\", \"7.0.0\"\nend\n"); + assert!(find_block(&format!("{block}{block}"), &gem(), "Gemfile", &ctx).is_err()); + let edited = format!("source \"{IDX}\" do\n gem \"rails\", \"~> 7.0\"\nend\n"); + assert!(find_block(&edited, &gem(), "Gemfile", &ctx).is_err()); + assert!(find_block("gem \"rails\"\n", &gem(), "Gemfile", &ctx) + .unwrap() + .is_none()); + } + + #[test] + fn provably_transitive_needs_a_blank_line_before_the_block() { + let client = super::super::UpstreamClient::new(true); + let ctx = ctx_with(&client); + let block = format!("source \"{IDX}\" do\n gem \"rails\", \"7.0.0\"\nend\n"); + let appended = format!("gem \"puma\"\n\n{block}"); + let b = find_block(&appended, &gem(), "Gemfile", &ctx) + .unwrap() + .unwrap(); + assert!(provably_appended(&appended, &b)); + let ambiguous = format!("gem \"puma\"\n{block}"); + let b = find_block(&ambiguous, &gem(), "Gemfile", &ctx) + .unwrap() + .unwrap(); + assert!(!provably_appended(&ambiguous, &b)); + assert!(is_subdependency( + &[" x (1.0)", " rails (>= 7)"], + "rails" + )); + assert!(!is_subdependency(&[" rails (7.0.0)"], "rails")); + } + + #[test] + fn global_sources_skip_blocks() { + let m = format!("source 'https://rubygems.org'\nsource(\"https://b.example\")\nsource \"{IDX}\" do\nend\n"); + assert_eq!( + global_sources(&m), + vec![ + "https://rubygems.org".to_string(), + "https://b.example".to_string() + ] + ); + } +} diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/mod.rs b/crates/socket-patch-core/src/patch/redirect/upstream/mod.rs index 7c5636d00..c90b83317 100644 --- a/crates/socket-patch-core/src/patch/redirect/upstream/mod.rs +++ b/crates/socket-patch-core/src/patch/redirect/upstream/mod.rs @@ -27,6 +27,8 @@ use crate::vex::discover::{Discovery, PatchedRef, WiringMode}; mod cargo; mod client; +mod composer; +mod gem; mod golang; mod npm; @@ -261,6 +263,8 @@ enum Format { BunLock, Cargo, Golang, + Gem, + Composer, Unsupported, } @@ -273,6 +277,8 @@ fn format_of(rel: &str) -> Format { "bun.lock" => Format::BunLock, "Cargo.toml" | "Cargo.lock" | "config.toml" | "config" => Format::Cargo, "go.mod" | "go.sum" | "go.work" => Format::Golang, + "Gemfile.lock" | "gems.locked" | "Gemfile" | "gems.rb" => Format::Gem, + "composer.lock" => Format::Composer, _ => Format::Unsupported, } } @@ -387,6 +393,8 @@ async fn restore_pass(view: &mut View<'_>, active: &[&HostedPin], ctx: &Ctx<'_>) Format::BunLock => npm::restore_bun_locks(view, &pins, &files, ctx).await, Format::Cargo => cargo::restore(view, &pins, &files, ctx).await, Format::Golang => golang::restore(view, &pins, &files, ctx).await, + Format::Gem => gem::restore(view, &pins, &files, ctx).await, + Format::Composer => composer::restore(view, &pins, &files, ctx).await, Format::Unsupported => { let mut r = FormatResult::default(); for pin in &pins { diff --git a/crates/socket-patch-core/tests/upstream_restore_golden.rs b/crates/socket-patch-core/tests/upstream_restore_golden.rs index c9791785a..0f6d6778c 100644 --- a/crates/socket-patch-core/tests/upstream_restore_golden.rs +++ b/crates/socket-patch-core/tests/upstream_restore_golden.rs @@ -99,15 +99,29 @@ fn load(flavor: &str) -> Vec { } async fn run_case(case: &Case) -> (BTreeMap, Vec<(String, PinStatus)>) { + run_case_with(case, None, &RestoreOptions::default()).await +} + +/// [`run_case`] with explicit pins (instead of discovery's) and options. +async fn run_case_with( + case: &Case, + pins: Option>, + opts: &RestoreOptions, +) -> (BTreeMap, Vec<(String, PinStatus)>) { let tmp = tempfile::tempdir().unwrap(); for (rel, text) in &case.expected { let p = tmp.path().join(rel); fs::create_dir_all(p.parent().unwrap()).unwrap(); fs::write(p, text).unwrap(); } - let discovery = socket_patch_core::vex::discover_patched_refs(tmp.path()).await; - let pins = HostedPin::all(&discovery); - let outcome = restore_upstream(tmp.path(), &pins, &RestoreOptions::default()).await; + let pins = match pins { + Some(p) => p, + None => { + let discovery = socket_patch_core::vex::discover_patched_refs(tmp.path()).await; + HostedPin::all(&discovery) + } + }; + let outcome = restore_upstream(tmp.path(), &pins, opts).await; assert!(outcome.flush_error.is_none(), "{:?}", outcome.flush_error); let statuses = outcome .pins @@ -338,3 +352,550 @@ async fn golang_goldens_round_trip() { } assert!(ran > 0); } + +// ── rubygems ──────────────────────────────────────────────────────────────── + +/// A case built by running the hosted rewriter over `input`, for the edge +/// shapes the shared goldens do not cover. +fn synthetic(label: &str, input: &[(&str, &str)], overrides: serde_json::Value) -> Case { + let input: BTreeMap = input + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + let deps: Vec = + serde_json::from_value(overrides.clone()).unwrap(); + let rewrite = socket_patch_core::patch::redirect::rewrite_registry_redirect(&input, &deps); + assert!( + !rewrite.files.is_empty(), + "{label}: the rewrite changed nothing: {:?}", + rewrite.warnings + ); + let mut expected = input.clone(); + expected.extend(rewrite.files); + Case { + dir: PathBuf::from(format!("synthetic/{label}")), + input, + expected, + overrides: overrides.as_array().unwrap().clone(), + } +} + +impl Case { + fn clone_with(&self, label: &str) -> Case { + Case { + dir: self.dir.with_file_name(label), + input: self.input.clone(), + expected: self.expected.clone(), + overrides: self.overrides.clone(), + } + } + + /// Apply `edit` to the named file of both trees. + fn edit_both(&mut self, rel: &str, edit: impl Fn(&str) -> String) { + for files in [&mut self.input, &mut self.expected] { + let next = edit(&files[rel]); + files.insert(rel.to_string(), next); + } + } +} + +/// Assert the (single) pin was refused naming `want` and the checkout +/// remedy for `lock`, with nothing written. +fn assert_refused( + case: &Case, + after: &BTreeMap, + statuses: &[(String, PinStatus)], + lock: &str, + want: &str, +) { + match &statuses[0].1 { + PinStatus::Refused(why) => { + assert!(why.contains(want), "{}: {why}", case.dir.display()); + assert!(why.contains(&format!("checkout -- {lock}")), "{why}"); + } + other => panic!("{}: expected a refusal, got {other:?}", case.dir.display()), + } + assert_eq!(after, &case.expected, "{}: a refused pin must change nothing", case.dir.display()); +} + +fn offline() -> RestoreOptions { + RestoreOptions { + offline: true, + ..RestoreOptions::default() + } +} + +const GEM_UUID: &str = "77777777-7777-7777-7777-777777777777"; +const GEM_INDEX: &str = "https://patch.socket.dev/patch-registry/gem/11111111-1111-1111-1111-111111111111/77777777-7777-7777-7777-777777777777/"; + +fn gem_override(name: &str, version: &str) -> serde_json::Value { + serde_json::json!([{ + "ecosystem": "gem", + "name": name, + "version": version, + "token": "11111111-1111-1111-1111-111111111111", + "patchUuid": GEM_UUID, + "artifactUrl": format!("{GEM_INDEX}gems/{name}-{version}.gem"), + "registryOverride": { + "kind": "rubygems-compact-index", + "indexUrl": GEM_INDEX, + "identifiers": {"name": name, "version": version, "gemChecksumSha256": "de".repeat(32)} + }, + "integrity": {"sha256": "de".repeat(32)} + }]) +} + +/// Serve a compact-index `info/` for every sha256-pinned gem of the +/// input locks. +async fn gem_mock(case: &Case) -> MockServer { + let server = MockServer::start().await; + let re = regex::Regex::new(r"(?m)^ ([A-Za-z0-9._-]+) \(([^)]+)\) sha256=([0-9a-f]{64})\r?$") + .unwrap(); + let mut by_gem: BTreeMap> = BTreeMap::new(); + for rel in ["Gemfile.lock", "gems.locked"] { + let Some(text) = case.input.get(rel) else { + continue; + }; + for c in re.captures_iter(text) { + by_gem + .entry(c[1].to_string()) + .or_default() + .push(format!("{} dep:>= 0|checksum:{},ruby:>= 2.7", &c[2], &c[3])); + } + } + for (name, lines) in by_gem { + Mock::given(method("GET")) + .and(path(format!("/info/{name}"))) + .respond_with( + ResponseTemplate::new(200).set_body_string(format!("---\n{}\n", lines.join("\n"))), + ) + .mount(&server) + .await; + } + server +} + +async fn gem_run(case: &Case) -> (BTreeMap, Vec<(String, PinStatus)>) { + let server = gem_mock(case).await; + let _env = EnvGuard::set(&[("SOCKET_RUBYGEMS_URL", server.uri())]); + run_case(case).await +} + +#[tokio::test] +#[serial] +async fn gem_goldens_round_trip() { + let mut ran = 0; + for case in load("gem/bundler") { + let (after, statuses) = gem_run(&case).await; + assert_round_trip(&case, &after, &statuses); + ran += 1; + } + assert!(ran > 0); +} + +const GEM_LOCK: &str = "GEM\n remote: https://rubygems.org/\n specs:\n puma (6.0.0)\n nio4r (~> 2.0)\n rails (7.0.0)\n rack (>= 2)\n zeitwerk (2.6.0)\n\nPLATFORMS\n ruby\n\nDEPENDENCIES\n puma\n rails (= 7.0.0)\n\nCHECKSUMS\n puma (6.0.0) sha256=1111111111111111111111111111111111111111111111111111111111111111\n rails (7.0.0) sha256=2222222222222222222222222222222222222222222222222222222222222222\n zeitwerk (2.6.0) sha256=3333333333333333333333333333333333333333333333333333333333333333\n\nBUNDLED WITH\n 2.6.2\n"; + +/// [`GEM_LOCK`] with `zeitwerk` a dependency of puma only (transitive). +fn transitive_lock() -> String { + GEM_LOCK.replace( + " nio4r (~> 2.0)\n", + " nio4r (~> 2.0)\n zeitwerk (~> 2.6)\n", + ) +} + +#[tokio::test] +#[serial] +async fn gem_edge_shapes_round_trip() { + let gemfile = "source \"https://rubygems.org\"\n\ngem \"puma\"\n\ngroup :test do\n gem \"rails\", \"7.0.0\", require: false\nend\n"; + let crlf_gemfile = "source \"https://rubygems.org\"\r\n\r\ngem \"rails\", \"7.0.0\"\r\ngem \"puma\"\r\n"; + let two_sources_gemfile = "source \"https://rubygems.org\"\n\ngem \"rails\", \"7.0.0\"\nsource \"https://gems.example.com\" do\n gem \"private-gem\"\nend\n"; + let two_sources_lock = "GEM\n remote: https://gems.example.com/\n specs:\n private-gem (1.0.0)\n\nGEM\n remote: https://rubygems.org/\n specs:\n rails (7.0.0)\n\nPLATFORMS\n ruby\n\nDEPENDENCIES\n private-gem!\n rails (= 7.0.0)\n\nCHECKSUMS\n private-gem (1.0.0) sha256=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n rails (7.0.0) sha256=2222222222222222222222222222222222222222222222222222222222222222\n\nBUNDLED WITH\n 2.6.2\n"; + // Provably transitive: the rewriter appended after a trailing blank + // line, which its in-place rewrite would have swallowed. + let transitive_gemfile = "source \"https://rubygems.org\"\n\ngem \"puma\"\n\n"; + let transitive = transitive_lock().replace(" rails (= 7.0.0)\n", ""); + let cases = [ + synthetic( + "group-with-options", + &[("Gemfile", gemfile), ("Gemfile.lock", GEM_LOCK)], + gem_override("rails", "7.0.0"), + ), + synthetic( + "gems-rb", + &[("gems.rb", gemfile), ("gems.locked", GEM_LOCK)], + gem_override("rails", "7.0.0"), + ), + synthetic( + "crlf", + &[ + ("Gemfile", crlf_gemfile), + ("Gemfile.lock", &GEM_LOCK.replace('\n', "\r\n")), + ], + gem_override("rails", "7.0.0"), + ), + synthetic( + "multiple-gem-sections", + &[("Gemfile", two_sources_gemfile), ("Gemfile.lock", two_sources_lock)], + gem_override("rails", "7.0.0"), + ), + synthetic( + "transitive-appended", + &[("Gemfile", transitive_gemfile), ("Gemfile.lock", &transitive)], + gem_override("zeitwerk", "2.6.0"), + ), + ]; + for case in &cases { + let (after, statuses) = gem_run(case).await; + assert_round_trip(case, &after, &statuses); + } +} + +#[tokio::test] +#[serial] +async fn gem_pre_checksums_states() { + let gemfile = "source \"https://rubygems.org\"\n\ngem \"rails\", \"~> 7.0\"\ngem \"puma\"\n"; + let checksums = GEM_LOCK + .split("\nCHECKSUMS\n") + .nth(1) + .unwrap() + .split("\n\n") + .next() + .unwrap(); + let lock = GEM_LOCK + .replace(" rails (= 7.0.0)\n", " rails (~> 7.0)\n") + .replace(&format!("\nCHECKSUMS\n{checksums}\n"), "") + .replace("2.6.2", "2.5.22"); + assert!(!lock.contains("CHECKSUMS")); + // Mixed: the rewriter edits only the Gemfile, so discovery finds no pin; + // hand the restore one. The untouched lock keeps the original constraint. + let mut mixed = synthetic( + "mixed", + &[("Gemfile", gemfile), ("Gemfile.lock", &lock)], + gem_override("rails", "7.0.0"), + ); + assert_eq!(mixed.expected["Gemfile.lock"], lock); + let pin = HostedPin { + purl: "pkg:gem/rails@7.0.0".into(), + uuid: GEM_UUID.into(), + files: vec!["Gemfile".into()], + }; + let (after, statuses) = run_case_with(&mixed, Some(vec![pin]), &offline()).await; + assert_round_trip(&mixed, &after, &statuses); + + // Bundler 2.2–2.5 then converges the pair itself (a Socket GEM section, + // a `!` pin, no CHECKSUMS): restorable offline. Not derivable: `~> 7.0` + // comes back as the exact pin the lock records. + let restored_lock = lock.replace(" rails (~> 7.0)\n", " rails (= 7.0.0)\n"); + let restored_gemfile = gemfile.replace("\"~> 7.0\"", "\"7.0.0\""); + let converged = lock + .replace(" rails (7.0.0)\n rack (>= 2)\n", "") + .replace( + "GEM\n remote: https://rubygems.org/", + &format!( + "GEM\n remote: {GEM_INDEX}\n specs:\n rails (7.0.0)\n rack (>= 2)\n\n\ + GEM\n remote: https://rubygems.org/" + ), + ) + .replace(" rails (~> 7.0)\n", " rails (= 7.0.0)!\n"); + mixed.expected.insert("Gemfile.lock".into(), converged); + let (after, statuses) = run_case_with(&mixed, None, &offline()).await; + assert_eq!(statuses, vec![("pkg:gem/rails@7.0.0".to_string(), PinStatus::Restored)]); + assert_eq!(after["Gemfile.lock"], restored_lock); + assert_eq!(after["Gemfile"], restored_gemfile); + + // Bundler ≤ 2.1 merged section: only the Socket remote line goes. + let merged = lock + .replace( + "GEM\n remote: https://rubygems.org/\n", + &format!("GEM\n remote: https://rubygems.org/\n remote: {GEM_INDEX}\n"), + ) + .replace(" rails (~> 7.0)\n", " rails (= 7.0.0)!\n"); + mixed.expected.insert("Gemfile.lock".into(), merged); + let (after, statuses) = run_case_with(&mixed, None, &offline()).await; + assert_eq!(statuses, vec![("pkg:gem/rails@7.0.0".to_string(), PinStatus::Restored)]); + assert_eq!(after["Gemfile.lock"], restored_lock); + assert_eq!(after["Gemfile"], restored_gemfile); +} + +#[tokio::test] +#[serial] +async fn gem_transitive_without_proof_stays_declared() { + // The rewriter's append for a transitive gem is indistinguishable from a + // direct last-line declaration, so it is kept as a direct exact pin. + let gemfile = "source \"https://rubygems.org\"\n\ngem \"puma\"\n"; + let lock = transitive_lock().replace(" rails (= 7.0.0)\n", ""); + let case = synthetic( + "transitive-ambiguous", + &[("Gemfile", gemfile), ("Gemfile.lock", &lock)], + gem_override("zeitwerk", "2.6.0"), + ); + let (after, statuses) = gem_run(&case).await; + assert_eq!(statuses[0].1, PinStatus::Restored); + assert_eq!(after["Gemfile"], format!("{gemfile}gem \"zeitwerk\", \"2.6.0\"\n")); + assert_eq!( + after["Gemfile.lock"], + lock.replace(" puma\n", " puma\n zeitwerk (= 2.6.0)\n") + ); +} + +#[tokio::test] +#[serial] +async fn gem_refusals_leave_everything_hosted() { + let gemfile = "source \"https://rubygems.org\"\n\ngem \"rails\", \"7.0.0\"\ngem \"puma\"\n"; + let case = synthetic( + "refusals", + &[("Gemfile", gemfile), ("Gemfile.lock", GEM_LOCK)], + gem_override("rails", "7.0.0"), + ); + // Offline: the CHECKSUMS sha256 needs the registry. + let (after, statuses) = run_case_with(&case, None, &offline()).await; + assert_refused(&case, &after, &statuses, "Gemfile.lock", "offline"); + // The registry does not know the gem. + { + let server = MockServer::start().await; + let _env = EnvGuard::set(&[("SOCKET_RUBYGEMS_URL", server.uri())]); + let (after, statuses) = run_case(&case).await; + assert_refused(&case, &after, &statuses, "Gemfile.lock", "404"); + } + // The upstream section is another registry's. + let mut foreign = case.clone_with("foreign-upstream"); + foreign.edit_both("Gemfile.lock", |t| { + t.replace("remote: https://rubygems.org/", "remote: https://gems.example.com/") + }); + let (after, statuses) = gem_run(&foreign).await; + assert_refused(&foreign, &after, &statuses, "Gemfile.lock", "not rubygems.org"); + // Two upstream sections, neither singled out. + let mut ambiguous = case.clone_with("ambiguous-upstream"); + ambiguous.edit_both("Gemfile.lock", |t| { + t.replace( + "PLATFORMS", + "GEM\n remote: https://gems.example.com/\n specs:\n other (1.0.0)\n\nPLATFORMS", + ) + }); + ambiguous.edit_both("Gemfile", |t| t.replace("source \"https://rubygems.org\"\n", "")); + ambiguous.edit_both("Gemfile.lock", |t| t.replace("remote: https://rubygems.org/", "remote: https://mirror.example.com/")); + let (after, statuses) = gem_run(&ambiguous).await; + assert_refused(&ambiguous, &after, &statuses, "Gemfile.lock", "upstream GEM sections"); + // The Gemfile block was hand-edited. + let mut edited = case.clone_with("edited-block"); + edited.expected.insert( + "Gemfile".into(), + edited.expected["Gemfile"].replace(" gem \"rails\", \"7.0.0\"", " gem \"rails\", \"~> 7.0\""), + ); + let (after, statuses) = gem_run(&edited).await; + assert_refused(&edited, &after, &statuses, "Gemfile.lock", "shape other than the source block"); +} + +// ── composer ──────────────────────────────────────────────────────────────── + +/// Serve packagist `p2` metadata for every entry of the input lock, in +/// composer/2.0 minified form behind a decoy version (so the expansion — +/// inherited keys, `__unset` — is exercised). `tweak` edits each served +/// version's diff. +async fn composer_mock(case: &Case, tweak: impl Fn(&mut serde_json::Value)) -> MockServer { + let server = MockServer::start().await; + let lock: serde_json::Value = serde_json::from_str(&case.input["composer.lock"]).unwrap(); + let mut by_file: BTreeMap)> = BTreeMap::new(); + for section in ["packages", "packages-dev"] { + for entry in lock[section].as_array().cloned().unwrap_or_default() { + let name = entry["name"].as_str().unwrap().to_ascii_lowercase(); + let version = entry["version"].as_str().unwrap(); + let dev = version.starts_with("dev-") || version.ends_with("-dev"); + let file = format!("{name}{}", if dev { "~dev" } else { "" }); + let mut diff = serde_json::json!({ + "version": version, + "dist": entry["dist"], + "source": entry.get("source").cloned().unwrap_or_else(|| "__unset".into()), + }); + tweak(&mut diff); + let (_, versions) = by_file.entry(file).or_insert_with(|| { + ( + name.clone(), + vec![serde_json::json!({ + "name": name, + "version": "99.0.0", + "type": "library", + "source": {"type": "git", "url": "https://decoy.example/x.git", "reference": "decoy"}, + "dist": {"type": "zip", "url": "https://decoy.example/x.zip", "reference": "decoy", "shasum": "decoy"} + })], + ) + }); + versions.push(diff); + } + } + for (file, (name, versions)) in by_file { + Mock::given(method("GET")) + .and(path(format!("/p2/{file}.json"))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "packages": { name: versions }, + "minified": "composer/2.0" + }))) + .mount(&server) + .await; + } + server +} + +async fn composer_run( + case: &Case, + tweak: impl Fn(&mut serde_json::Value), +) -> (BTreeMap, Vec<(String, PinStatus)>) { + let server = composer_mock(case, tweak).await; + let _env = EnvGuard::set(&[("SOCKET_PACKAGIST_URL", server.uri())]); + run_case(case).await +} + +#[tokio::test] +#[serial] +async fn composer_goldens_round_trip() { + // Every composer golden with a rewrite is invertible. + let mut ran = 0; + for case in load("composer/composer-lock") { + let (after, statuses) = composer_run(&case, |_| {}).await; + assert_round_trip(&case, &after, &statuses); + ran += 1; + } + assert!(ran >= 5, "{ran}"); +} + +fn composer_override(name: &str, version: &str) -> serde_json::Value { + let (ns, leaf) = name.split_once('/').unwrap(); + serde_json::json!([{ + "ecosystem": "composer", + "name": leaf, + "namespace": ns, + "version": version, + "token": "11111111-1111-1111-1111-111111111111", + "patchUuid": "44444444-4444-4444-4444-444444444444", + "artifactUrl": format!( + "https://patch.socket.dev/patch/composer/{name}/{version}/11111111-1111-1111-1111-111111111111/44444444-4444-4444-4444-444444444444/{leaf}.zip" + ), + "integrity": {"sha1": "abcdef0123456789abcdef0123456789abcdef01"} + }]) +} + +const COMPOSER_LOCK: &str = r#"{ + "_readme": [ + "This file locks the dependencies of your project to a known state" + ], + "content-hash": "abc123", + "packages": [ + { + "name": "psr/log", + "version": "1.1.4", + "source": { + "type": "git", + "url": "https://github.com/php-fig/log.git", + "reference": "d49695b909c3b7628b6289db5479a1c204601f11" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/log/zipball/d49695b909c3b7628b6289db5479a1c204601f11", + "reference": "d49695b909c3b7628b6289db5479a1c204601f11", + "shasum": "" + }, + "type": "library", + "notification-url": "https://packagist.org/downloads/" + } + ], + "packages-dev": [ + { + "name": "acme/tool", + "version": "dev-main", + "source": { + "type": "git", + "url": "https://github.com/acme/tool.git", + "reference": "0123456789abcdef0123456789abcdef01234567" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/acme/tool/zipball/0123456789abcdef0123456789abcdef01234567", + "reference": "0123456789abcdef0123456789abcdef01234567", + "shasum": "" + }, + "type": "library", + "notification-url": "https://packagist.org/downloads/" + } + ] +} +"#; + +#[tokio::test] +#[serial] +async fn composer_edge_shapes_round_trip() { + // Older composer escaped every slash; the rewriter matches names as + // written, so only the URL values are escaped here. + let escaped = regex::Regex::new(r#""(url|notification-url)": "([^"]*)""#) + .unwrap() + .replace_all(COMPOSER_LOCK, |c: ®ex::Captures| { + format!("\"{}\": \"{}\"", &c[1], c[2].replace('/', "\\/")) + }) + .into_owned(); + let crlf = COMPOSER_LOCK.replace('\n', "\r\n"); + let cases = [ + synthetic( + "dev-version-packages-dev", + &[("composer.lock", COMPOSER_LOCK)], + composer_override("acme/tool", "dev-main"), + ), + synthetic( + "prod-entry", + &[("composer.lock", COMPOSER_LOCK)], + composer_override("psr/log", "1.1.4"), + ), + synthetic( + "escaped-slashes", + &[("composer.lock", &escaped)], + composer_override("acme/tool", "dev-main"), + ), + synthetic("crlf", &[("composer.lock", &crlf)], composer_override("psr/log", "1.1.4")), + ]; + for case in &cases { + let (after, statuses) = composer_run(case, |_| {}).await; + assert_round_trip(case, &after, &statuses); + } +} + +#[tokio::test] +#[serial] +async fn composer_refusals_leave_everything_hosted() { + let case = synthetic( + "refusals", + &[("composer.lock", COMPOSER_LOCK)], + composer_override("psr/log", "1.1.4"), + ); + // Packagist now serves another commit for the version. + let (after, statuses) = + composer_run(&case, |d| d["dist"]["reference"] = "feedface".into()).await; + assert_refused(&case, &after, &statuses, "composer.lock", "packagist now serves"); + // Packagist does not list the version. + let (after, statuses) = composer_run(&case, |d| d["version"] = "0.0.1".into()).await; + assert_refused(&case, &after, &statuses, "composer.lock", "does not list version 1.1.4"); + // Offline. + let (after, statuses) = run_case_with(&case, None, &offline()).await; + assert_refused(&case, &after, &statuses, "composer.lock", "offline"); + // Locked from another repository. + let mut foreign = case.clone_with("foreign"); + foreign.edit_both("composer.lock", |t| { + t.replacen("https://packagist.org/downloads/", "https://repo.example.com/downloads/", 1) + }); + let (after, statuses) = composer_run(&foreign, |_| {}).await; + assert_refused(&foreign, &after, &statuses, "composer.lock", "not packagist"); + // No notification-url, and composer.json names custom repositories. + let mut custom = case.clone_with("custom-repos"); + custom.edit_both("composer.lock", |t| { + t.replacen( + ",\n \"notification-url\": \"https://packagist.org/downloads/\"", + "", + 1, + ) + }); + for files in [&mut custom.input, &mut custom.expected] { + files.insert( + "composer.json".into(), + r#"{"repositories": [{"type": "composer", "url": "https://repo.example.com"}]}"#.into(), + ); + } + let (after, statuses) = composer_run(&custom, |_| {}).await; + assert_refused(&custom, &after, &statuses, "composer.lock", "custom repositories"); +} From 37e96e25d42294b4a27e7a70de55e8c4c6441017 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 23:28:47 +0000 Subject: [PATCH 09/66] Fix clippy findings in the eject and takeover paths Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018tncU7u7FK48tubFNA12UQ --- crates/socket-patch-cli/src/commands/scan/hosted.rs | 2 +- crates/socket-patch-cli/src/commands/vendor.rs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 75e77f122..23dbd3f96 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -3526,7 +3526,7 @@ mod tests { }; use super::{wheel_metadata_concurrency, WHEEL_METADATA_CONCURRENCY}; use socket_patch_core::constants::npm_family; - use socket_patch_core::patch::redirect::{DepOverride, FileEdit}; + use socket_patch_core::patch::redirect::DepOverride; use socket_patch_core::utils::concurrent::API_CONCURRENCY_ENV; /// The wheel window is a patch-API window, so the documented escape diff --git a/crates/socket-patch-cli/src/commands/vendor.rs b/crates/socket-patch-cli/src/commands/vendor.rs index ed8394e24..c0605077a 100644 --- a/crates/socket-patch-cli/src/commands/vendor.rs +++ b/crates/socket-patch-cli/src/commands/vendor.rs @@ -934,7 +934,6 @@ async fn run_eject(args: &VendorArgs, pins: Vec) -> i32 { )), } } - drop(views); for (purl, detail) in &fetch_failures { report_vendor_failure(common, purl, detail); } @@ -2653,7 +2652,7 @@ pub(crate) async fn vendor_records_reusing( continue; } for (code, detail) in &restore.warnings { - record_warning(env, candidate, &VendorWarning::new(*code, detail.clone()), common); + record_warning(env, candidate, &VendorWarning::new(code, detail.clone()), common); } if common.dry_run { record_warning( From 12578699ca6534f293e37878ef0c82cee0bec8e1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 23:27:03 +0000 Subject: [PATCH 10/66] Restore hosted PyPI pins to their upstream registry entries Add the Python family to the v5 hosted -> upstream restore: Pipfile.lock, requirements.txt, Hatch direct references (pyproject.toml / hatch.toml), poetry.lock, pdm.lock, and uv.lock / PEP 723 script locks / PEP 751 pylock files with their paired pyproject / script metadata. Each restorer rewrites only entries whose reference is a hosted URL for an in-scope patch uuid and re-derives what the hosted rewrite overwrote from PyPI's JSON API (UpstreamClient::pypi_files, base overridable with SOCKET_PYPI_JSON_API, cached like the other lookups). Where a field is not derivable the pin is refused instead of guessed: requirements hash-checking mode that no other line settles, a Pipfile.lock index that is not PyPI, PDM locks without cross_platform (or uv locks) when the release ships platform- or interpreter-specific wheels, uv locks with no sibling registry package to show the artifact shape, several or non-PyPI registries, exclude-newer / no-binary / no-build filtering, multi-clause uv specifiers with no spelling evidence, uv 0.2 [[distribution]] locks, offline runs and registry failures. The golden harness round-trips the native Poetry (1.0-2.4), PDM (every supported lock_version) and Pipenv fixtures plus synthetic requirements/Hatch/uv/pylock projects through the real hosted rewriter; the shared requirements golden restores modulo the grant's name casing and the uv golden (no registry sibling) is refused. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018tncU7u7FK48tubFNA12UQ --- .../src/patch/redirect/pipenv.rs | 12 +- .../src/patch/redirect/requirements.rs | 16 +- .../src/patch/redirect/upstream/client.rs | 110 +- .../src/patch/redirect/upstream/mod.rs | 25 + .../src/patch/redirect/upstream/pypi.rs | 1015 ++++++++++++++ .../src/patch/redirect/upstream/pypi_locks.rs | 380 ++++++ .../src/patch/redirect/upstream/uv.rs | 1197 +++++++++++++++++ .../tests/upstream_restore_golden.rs | 695 ++++++++++ 8 files changed, 3435 insertions(+), 15 deletions(-) create mode 100644 crates/socket-patch-core/src/patch/redirect/upstream/pypi.rs create mode 100644 crates/socket-patch-core/src/patch/redirect/upstream/pypi_locks.rs create mode 100644 crates/socket-patch-core/src/patch/redirect/upstream/uv.rs diff --git a/crates/socket-patch-core/src/patch/redirect/pipenv.rs b/crates/socket-patch-core/src/patch/redirect/pipenv.rs index 7725c9e62..9f42fd0ca 100644 --- a/crates/socket-patch-core/src/patch/redirect/pipenv.rs +++ b/crates/socket-patch-core/src/patch/redirect/pipenv.rs @@ -7,10 +7,10 @@ use serde_json::{json, Value}; use super::{DepOverride, FileEdit, RewriteResult, RewriteWarning}; use crate::crawlers::python_crawler::canonicalize_pypi_name; -struct Property { - name: String, - range: Range, - value: Value, +pub(super) struct Property { + pub(super) name: String, + pub(super) range: Range, + pub(super) value: Value, } fn properties(text: &str, offset: usize) -> Result, String> { @@ -82,7 +82,7 @@ fn properties(text: &str, offset: usize) -> Result, String> { } } -fn entries(text: &str) -> Result, String> { +pub(super) fn entries(text: &str) -> Result, String> { // A UTF-8 BOM (Windows editors) is not JSON; parse past it. Offsets // below come from `text.find('{')`, so they stay byte-accurate. let value = @@ -111,7 +111,7 @@ fn entries(text: &str) -> Result, String> { Ok(result) } -fn format_entry(value: &Value, text: &str, start: usize) -> Result { +pub(super) fn format_entry(value: &Value, text: &str, start: usize) -> Result { let mut bytes = Vec::new(); let formatter = serde_json::ser::PrettyFormatter::with_indent(b" "); value diff --git a/crates/socket-patch-core/src/patch/redirect/requirements.rs b/crates/socket-patch-core/src/patch/redirect/requirements.rs index 5e0b921c0..d6d64c3c0 100644 --- a/crates/socket-patch-core/src/patch/redirect/requirements.rs +++ b/crates/socket-patch-core/src/patch/redirect/requirements.rs @@ -7,14 +7,14 @@ use super::{DepOverride, FileEdit, RewriteResult, RewriteWarning}; use crate::crawlers::python_crawler::canonicalize_pypi_name; use crate::utils::purl::percent_decode_purl_component; -struct LogicalRequirement { - original: String, - text: String, - ending: String, - unterminated: bool, +pub(super) struct LogicalRequirement { + pub(super) original: String, + pub(super) text: String, + pub(super) ending: String, + pub(super) unterminated: bool, } -fn logical_requirements(content: &str) -> Vec { +pub(super) fn logical_requirements(content: &str) -> Vec { let physical: Vec<&str> = content.split_inclusive('\n').collect(); let mut requirements = Vec::new(); let mut index = 0; @@ -62,7 +62,7 @@ fn logical_requirements(content: &str) -> Vec { requirements } -fn unquoted_index(text: &str, target: char, after_whitespace: bool) -> Option { +pub(super) fn unquoted_index(text: &str, target: char, after_whitespace: bool) -> Option { let mut quote = None; let mut escaped = false; let mut previous = None; @@ -87,7 +87,7 @@ fn unquoted_index(text: &str, target: char, after_whitespace: bool) -> Option Vec<&str> { +pub(super) fn requirement_tokens(text: &str) -> Vec<&str> { let mut tokens = Vec::new(); let mut start = None; let mut quote = None; diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/client.rs b/crates/socket-patch-core/src/patch/redirect/upstream/client.rs index 34eb49e6c..62671093e 100644 --- a/crates/socket-patch-core/src/patch/redirect/upstream/client.rs +++ b/crates/socket-patch-core/src/patch/redirect/upstream/client.rs @@ -8,7 +8,9 @@ use std::collections::HashMap; use serde_json::Value; use tokio::sync::Mutex; -use crate::vendor::registry_fetch::{build_registry_client, npm_registry_base, RegistryClient}; +use crate::vendor::registry_fetch::{ + build_registry_client, npm_registry_base, pypi_json_api_base, RegistryClient, +}; /// An npm version's `dist` block. #[derive(Debug, Clone, PartialEq, Eq)] @@ -30,6 +32,23 @@ pub(crate) struct GoSums { pub mod_h1: String, } +/// One release file of a PyPI version, from the JSON API's `urls[]`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PypiFile { + /// The distribution filename (`--.whl`, `.tar.gz`, …). + pub filename: String, + /// The file's download URL (files.pythonhosted.org on PyPI). + pub url: String, + /// Lowercase hex sha256 (`digests.sha256`). + pub sha256: String, + /// Size in bytes. + pub size: Option, + /// `upload_time_iso_8601` (microsecond precision, `Z`): the same instant + /// the PEP 691 simple API reports as `upload-time`, which is what uv + /// records. + pub upload_time: Option, +} + /// The sparse crates.io index; override with `SOCKET_CRATES_INDEX`. pub(crate) const DEFAULT_CRATES_INDEX: &str = "https://index.crates.io"; @@ -142,6 +161,7 @@ pub(crate) struct UpstreamClient { go: Cache, rubygems: Cache, packagist: Cache>, + pypi: Cache>, } impl UpstreamClient { @@ -155,6 +175,7 @@ impl UpstreamClient { go: Mutex::default(), rubygems: Mutex::default(), packagist: Mutex::default(), + pypi: Mutex::default(), } } @@ -272,6 +293,38 @@ impl UpstreamClient { result } + /// Every release file of `name@version` from PyPI's JSON API (`GET + /// ///json`, base overridable with + /// `SOCKET_PYPI_JSON_API`), sorted by filename — the order Poetry, PDM, + /// Pipenv and pip-compile record them in. `name` is PEP 503 + /// canonicalized first (PyPI redirects every other spelling to it). + pub(crate) async fn pypi_files( + &self, + name: &str, + version: &str, + ) -> Result, String> { + let name = crate::crawlers::python_crawler::canonicalize_pypi_name(name); + let key = (name.clone(), version.to_string()); + if let Some(hit) = self.pypi.lock().await.get(&key) { + return hit.clone(); + } + let result = async { + if self.offline { + return Err(OFFLINE.to_string()); + } + let url = format!( + "{}/{name}/{}/json", + pypi_json_api_base(), + crate::utils::uri::encode_uri_component(version) + ); + let doc = self.get_json(&url).await?; + pypi_release_files(&doc).map_err(|why| format!("{url} {why}")) + } + .await; + self.pypi.lock().await.insert(key, result.clone()); + result + } + /// The go.sum hashes of `module@version`, computed from the module /// proxy's `.zip` and `.mod` the way `go` computes them. pub(crate) async fn go_sums(&self, module: &str, version: &str) -> Result { @@ -385,6 +438,42 @@ impl UpstreamClient { } } +/// The release files of a PyPI JSON API version document, sorted by +/// filename. +fn pypi_release_files(doc: &Value) -> Result, String> { + let urls = doc + .get("urls") + .and_then(Value::as_array) + .ok_or("carries no `urls` list")?; + let mut files = Vec::with_capacity(urls.len()); + for file in urls { + let str_field = |k: &str| file.get(k).and_then(Value::as_str).map(str::to_string); + let filename = str_field("filename") + .filter(|f| !f.is_empty() && !f.contains(['/', '\\'])) + .ok_or("lists a file without a plain filename")?; + let url = str_field("url").ok_or_else(|| format!("lists {filename} without a url"))?; + let sha256 = file + .get("digests") + .and_then(|d| d.get("sha256")) + .and_then(Value::as_str) + .map(str::to_ascii_lowercase) + .filter(|h| crate::utils::digest::is_hex64_lower(h)) + .ok_or_else(|| format!("lists {filename} without a sha256 digest"))?; + files.push(PypiFile { + filename, + url, + sha256, + size: file.get("size").and_then(Value::as_u64), + upload_time: str_field("upload_time_iso_8601"), + }); + } + if files.is_empty() { + return Err("lists no release files".to_string()); + } + files.sort_by(|a, b| a.filename.cmp(&b.filename)); + Ok(files) +} + /// Go's checksum database, `sum.golang.org`; `SOCKET_GOSUMDB_URL` names /// another (tests, mirrors). pub(crate) const DEFAULT_GOSUMDB: &str = "https://sum.golang.org"; @@ -468,6 +557,25 @@ mod tests { assert!(raw[1].get("name").is_none()); } + #[test] + fn pypi_release_files_are_validated_and_sorted() { + let doc = serde_json::json!({ "urls": [ + { "filename": "x-1.tar.gz", "url": "https://f/x-1.tar.gz", + "digests": { "sha256": "B".repeat(64) }, "size": 3, + "upload_time_iso_8601": "2023-01-01T00:00:00.123456Z" }, + { "filename": "x-1-py3-none-any.whl", "url": "https://f/x.whl", + "digests": { "sha256": "a".repeat(64) } }, + ]}); + let files = pypi_release_files(&doc).unwrap(); + assert_eq!(files[0].filename, "x-1-py3-none-any.whl"); + assert_eq!(files[1].sha256, "b".repeat(64)); + assert_eq!(files[1].size, Some(3)); + assert!(pypi_release_files(&serde_json::json!({ "urls": [] })).is_err()); + let bad = serde_json::json!({ "urls": [{ "filename": "x.whl", "url": "u", + "digests": { "sha256": "zz" } }] }); + assert!(pypi_release_files(&bad).unwrap_err().contains("sha256")); + } + #[test] fn go_mod_h1_matches_the_x_mod_recipe() { // `module example.com/m\n` — cross-checked with `go mod download diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/mod.rs b/crates/socket-patch-core/src/patch/redirect/upstream/mod.rs index c90b83317..528bf766b 100644 --- a/crates/socket-patch-core/src/patch/redirect/upstream/mod.rs +++ b/crates/socket-patch-core/src/patch/redirect/upstream/mod.rs @@ -31,6 +31,9 @@ mod composer; mod gem; mod golang; mod npm; +mod pypi; +mod pypi_locks; +mod uv; pub(crate) use client::UpstreamClient; @@ -265,6 +268,15 @@ enum Format { Golang, Gem, Composer, + PipfileLock, + PoetryLock, + PdmLock, + Requirements, + /// Hatch direct references (`pyproject.toml`, `hatch.toml`). + Hatch, + /// uv.lock, PEP 723 script locks and PEP 751 pylock files (the uv + /// restorer also edits their paired `pyproject.toml` / script). + PythonLock, Unsupported, } @@ -279,6 +291,13 @@ fn format_of(rel: &str) -> Format { "go.mod" | "go.sum" | "go.work" => Format::Golang, "Gemfile.lock" | "gems.locked" | "Gemfile" | "gems.rb" => Format::Gem, "composer.lock" => Format::Composer, + "Pipfile.lock" => Format::PipfileLock, + "poetry.lock" => Format::PoetryLock, + "pdm.lock" => Format::PdmLock, + "pyproject.toml" | "hatch.toml" => Format::Hatch, + leaf if crate::utils::python_lock::is_python_lock_name(leaf) => Format::PythonLock, + // The root requirements.txt and the `-r` includes discovery walks. + leaf if leaf.ends_with(".txt") => Format::Requirements, _ => Format::Unsupported, } } @@ -395,6 +414,12 @@ async fn restore_pass(view: &mut View<'_>, active: &[&HostedPin], ctx: &Ctx<'_>) Format::Golang => golang::restore(view, &pins, &files, ctx).await, Format::Gem => gem::restore(view, &pins, &files, ctx).await, Format::Composer => composer::restore(view, &pins, &files, ctx).await, + Format::PipfileLock => pypi::restore_pipfile_lock(view, &pins, &files, ctx).await, + Format::PoetryLock => pypi_locks::restore_poetry(view, &pins, &files, ctx).await, + Format::PdmLock => pypi_locks::restore_pdm(view, &pins, &files, ctx).await, + Format::Requirements => pypi::restore_requirements(view, &pins, &files, ctx).await, + Format::Hatch => pypi::restore_hatch(view, &pins, &files, ctx).await, + Format::PythonLock => uv::restore(view, &pins, &files, ctx).await, Format::Unsupported => { let mut r = FormatResult::default(); for pin in &pins { diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/pypi.rs b/crates/socket-patch-core/src/patch/redirect/upstream/pypi.rs new file mode 100644 index 000000000..6fed12fa5 --- /dev/null +++ b/crates/socket-patch-core/src/patch/redirect/upstream/pypi.rs @@ -0,0 +1,1015 @@ +//! PyPI upstream restores for the formats whose hosted rewrite is a single +//! reference swap: `Pipfile.lock` entries, `requirements.txt` lines and +//! Hatch's PEP 508 direct references (`pyproject.toml` / `hatch.toml`). +//! The TOML locks live in [`super::pypi_locks`] (Poetry, PDM) and +//! [`super::uv`] (uv, PEP 723 script locks, PEP 751 pylock). +//! +//! Every restorer rewrites ONLY the entries whose reference is a hosted URL +//! naming an in-scope patch uuid; the version is the pin's (the hosted +//! rewriters drop every `==` pin, and discovery reads it back from the +//! url). Hashes are re-resolved from PyPI's JSON API +//! ([`super::client::UpstreamClient::pypi_files`]): every release file, +//! sorted by filename — what Pipenv and pip-compile record. + +use std::collections::{BTreeMap, BTreeSet}; + +use regex::Regex; +use serde_json::{json, Value}; +use toml_edit::{DocumentMut, Item}; + +use super::client::PypiFile; +use super::{Ctx, FormatResult, HostedPin, View}; +use crate::crawlers::python_crawler::canonicalize_pypi_name; +use crate::utils::python_lock::preserve_line_endings; +use crate::vendor::common::pep508_name; + +/// The pins by uuid. +pub(super) fn by_uuid<'p>(pins: &[&'p HostedPin]) -> BTreeMap<&'p str, &'p HostedPin> { + pins.iter().map(|p| (p.uuid.as_str(), *p)).collect() +} + +/// The in-scope pin a hosted `location` names. +pub(super) fn pin_of<'p>( + location: &str, + pins: &BTreeMap<&str, &'p HostedPin>, + ctx: &Ctx<'_>, +) -> Option<&'p HostedPin> { + let uuid = ctx.hosted_uuid(location)?; + pins.get(uuid.as_str()).copied() +} + +/// `(canonical name, version)` of a pin, or its refusal. +pub(super) fn pin_coords(pin: &HostedPin, result: &mut FormatResult) -> Option<(String, String)> { + match pin.name_version() { + Some((name, version)) => Some((canonicalize_pypi_name(&name), version)), + None => { + result.refuse(&pin.uuid, format!("{} is not a pypi purl", pin.purl)); + None + } + } +} + +/// Read `rel` through the view; a missing or unreadable file refuses every +/// pin discovery found in it. +pub(super) async fn read_or_refuse( + view: &mut View<'_>, + rel: &str, + pins: &BTreeMap<&str, &HostedPin>, + result: &mut FormatResult, +) -> Option { + match view.read(rel).await { + Ok(Some(text)) => Some(text), + Ok(None) => { + refuse_all_in(pins, rel, result, format!("{rel} no longer exists")); + None + } + Err(e) => { + refuse_all_in(pins, rel, result, e); + None + } + } +} + +pub(super) fn refuse_all_in( + pins: &BTreeMap<&str, &HostedPin>, + rel: &str, + result: &mut FormatResult, + why: String, +) { + for pin in pins.values() { + if pin.files.iter().any(|f| f == rel) { + result.refuse(&pin.uuid, why.clone()); + } + } +} + +/// The release files of every `(uuid, name, version)` wanted, fetched +/// concurrently and keyed by `(name, version)`. A failed lookup refuses its +/// pin. +pub(super) async fn fetch_release_files( + wanted: &BTreeSet<(String, String, String)>, + ctx: &Ctx<'_>, + result: &mut FormatResult, +) -> BTreeMap<(String, String), Vec> { + let lookups = wanted.iter().map(|(uuid, name, version)| async move { + ( + uuid.clone(), + name.clone(), + version.clone(), + ctx.client.pypi_files(name, version).await, + ) + }); + let mut out = BTreeMap::new(); + for (uuid, name, version, files) in futures_util::future::join_all(lookups).await { + match files { + Ok(files) => { + out.insert((name, version), files); + } + Err(why) => result.refuse(&uuid, format!("{name}=={version}: {why}")), + } + } + out +} + +/// Whether every wheel of a release installs on every platform and every +/// Python 3 (`py3` in its python tag, `none` ABI, `any` platform), so a +/// lock that keeps only the files its targets can install — PDM without +/// `cross_platform`, uv's `requires-python` / environment filtering — still +/// records all of them. Sdists always qualify. +pub(super) fn universal_release(files: &[PypiFile]) -> bool { + files.iter().all(|f| { + let Some(stem) = f.filename.strip_suffix(".whl") else { + return true; + }; + let tags: Vec<&str> = stem.rsplitn(4, '-').collect(); + matches!(tags.as_slice(), [platform, abi, python, _] + if *platform == "any" && *abi == "none" && python.split('.').any(|t| t == "py3")) + }) +} + +/// A TOML basic string. +pub(super) fn toml_quote(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 2); + out.push('"'); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + c if c.is_control() => out.push_str(&format!("\\u{:04X}", c as u32)), + c => out.push(c), + } + } + out.push('"'); + out +} + +/// A TOML value parsed from `text`, keeping its own layout; its outer +/// decor is cleared so it renders after `key = ` like any other value. +pub(super) fn toml_value(text: &str) -> Option { + let doc: DocumentMut = format!("v = {text}\n").parse().ok()?; + let mut value = doc.get("v")?.as_value()?.clone(); + value.decor_mut().clear(); + Some(value) +} + +/// `entries` as the multi-line array Poetry and PDM write (4-space indent, +/// trailing comma; `[]` when empty). +pub(super) fn multiline_toml_array(entries: &[String]) -> String { + if entries.is_empty() { + return "[]".to_string(); + } + let mut out = String::from("[\n"); + for entry in entries { + out.push_str(" "); + out.push_str(entry); + out.push_str(",\n"); + } + out.push(']'); + out +} + +// ── Pipfile.lock ───────────────────────────────────────────────────────────── + +/// Whether `url` is PyPI's simple index (the only upstream the restore can +/// re-derive hashes for). +pub(super) fn is_pypi_simple(url: &str) -> bool { + matches!( + url.trim() + .trim_end_matches('/') + .to_ascii_lowercase() + .as_str(), + "https://pypi.org/simple" | "https://pypi.python.org/simple" + ) +} + +/// The index name a restored entry records: the `_meta.sources` entry that +/// is PyPI. `Ok(None)` when the lock's registry entries carry no `index` +/// at all (Pipenv < 2018). +fn pipenv_index( + doc: &Value, + registry_indexes: &[Option<&str>], + pipfile: Option<&str>, + name: &str, +) -> Result, String> { + if !registry_indexes.is_empty() && registry_indexes.iter().all(Option::is_none) { + return Ok(None); + } + let sources: Vec<(&str, &str)> = doc + .pointer("/_meta/sources") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|s| { + Some(( + s.get("name").and_then(Value::as_str)?, + s.get("url").and_then(Value::as_str)?, + )) + }) + .collect(); + let pypi: Vec<&str> = sources + .iter() + .filter(|(_, url)| is_pypi_simple(url)) + .map(|(n, _)| *n) + .collect(); + let chosen = match pypi.as_slice() { + [one] => *one, + [] => { + return Err(format!( + "no package index in Pipfile.lock `_meta.sources` is PyPI ({}), so the \ + upstream hashes cannot be re-derived", + sources + .iter() + .map(|(_, u)| *u) + .collect::>() + .join(", ") + )) + } + _ => { + return Err( + "Pipfile.lock `_meta.sources` names PyPI more than once; the entry's index is \ + ambiguous" + .to_string(), + ) + } + }; + if let Some(explicit) = pipfile.and_then(|p| pipfile_explicit_index(p, name)) { + if explicit != chosen { + return Err(format!( + "the Pipfile installs {name} from index {explicit:?}, not PyPI" + )); + } + } + Ok(Some(chosen.to_string())) +} + +/// The `index = "…"` a Pipfile declares for `name` in any package category. +fn pipfile_explicit_index(pipfile: &str, name: &str) -> Option { + let doc: DocumentMut = pipfile.trim_start_matches('\u{feff}').parse().ok()?; + let canon = canonicalize_pypi_name(name); + for (category, table) in doc.iter() { + if matches!(category, "source" | "requires" | "pipenv" | "scripts") { + continue; + } + let Some(table) = table.as_table_like() else { + continue; + }; + for (key, item) in table.iter() { + if canonicalize_pypi_name(key) != canon { + continue; + } + if let Some(index) = item + .as_table_like() + .and_then(|t| t.get("index")) + .and_then(Item::as_str) + { + return Some(index.to_string()); + } + } + } + None +} + +/// One hosted `Pipfile.lock` entry. +struct PipenvHit { + /// Index into the lock's `pipenv::entries`. + entry: usize, + uuid: String, + name: String, + version: String, +} + +pub(crate) async fn restore_pipfile_lock( + view: &mut View<'_>, + pins: &[&HostedPin], + files: &[String], + ctx: &Ctx<'_>, +) -> FormatResult { + let mut result = FormatResult::default(); + let pins = by_uuid(pins); + for rel in files { + let Some(text) = read_or_refuse(view, rel, &pins, &mut result).await else { + continue; + }; + let (entries, doc) = match ( + super::super::pipenv::entries(&text), + crate::vendor::lock_inventory::pypi::parse_pipfile_lock(&text), + ) { + (Ok(entries), Ok(doc)) => (entries, doc), + (Err(e), _) => { + refuse_all_in(&pins, rel, &mut result, format!("{rel}: {e}")); + continue; + } + (_, Err(e)) => { + refuse_all_in( + &pins, + rel, + &mut result, + format!("{rel} is not valid JSON: {e}"), + ); + continue; + } + }; + let mut hits: Vec = Vec::new(); + // The `index` of every registry entry the rewrite did not touch. + let mut registry_indexes: Vec> = Vec::new(); + for (i, (_, entry)) in entries.iter().enumerate() { + let Some(object) = entry.value.as_object() else { + continue; + }; + let reference = object + .get("file") + .or_else(|| object.get("path")) + .and_then(Value::as_str); + let Some(pin) = reference.and_then(|r| pin_of(r, &pins, ctx)) else { + if reference.is_none() && object.contains_key("version") { + registry_indexes.push(object.get("index").and_then(Value::as_str)); + } + continue; + }; + let Some((_, version)) = pin_coords(pin, &mut result) else { + continue; + }; + if object.contains_key("file") && object.contains_key("path") { + result.refuse( + &pin.uuid, + format!("{rel}: {} carries both `file` and `path`", entry.name), + ); + continue; + } + if let Some(pinned) = object.get("version").and_then(Value::as_str) { + if pinned != format!("=={version}") { + result.refuse( + &pin.uuid, + format!( + "{rel}: {} pins {pinned} beside the hosted {version} reference", + entry.name + ), + ); + continue; + } + } + hits.push(PipenvHit { + entry: i, + uuid: pin.uuid.clone(), + name: entry.name.clone(), + version, + }); + } + if hits.is_empty() { + continue; + } + let pipfile_rel = match rel.rsplit_once('/') { + Some((dir, _)) => format!("{dir}/Pipfile"), + None => "Pipfile".to_string(), + }; + let pipfile = view.read(&pipfile_rel).await.ok().flatten(); + let wanted = hits + .iter() + .filter(|h| !result.refused.contains_key(&h.uuid)) + .map(|h| { + ( + h.uuid.clone(), + canonicalize_pypi_name(&h.name), + h.version.clone(), + ) + }) + .collect(); + let released = fetch_release_files(&wanted, ctx, &mut result).await; + let mut splices: Vec<(std::ops::Range, String, String)> = Vec::new(); + for hit in &hits { + if result.refused.contains_key(&hit.uuid) { + continue; + } + let index = match pipenv_index(&doc, ®istry_indexes, pipfile.as_deref(), &hit.name) { + Ok(index) => index, + Err(why) => { + result.refuse(&hit.uuid, format!("{rel}: {why}")); + continue; + } + }; + let Some(release) = + released.get(&(canonicalize_pypi_name(&hit.name), hit.version.clone())) + else { + continue; + }; + let (_, entry) = &entries[hit.entry]; + let mut object = entry.value.as_object().cloned().unwrap_or_default(); + object.remove("file"); + object.remove("path"); + object.insert("version".into(), json!(format!("=={}", hit.version))); + match index { + Some(index) => object.insert("index".into(), json!(index)), + None => object.remove("index"), + }; + let mut hashes: Vec = release + .iter() + .map(|f| format!("sha256:{}", f.sha256)) + .collect(); + hashes.sort(); + hashes.dedup(); + object.insert("hashes".into(), json!(hashes)); + let mut value = Value::Object(object); + value.sort_all_objects(); + match super::super::pipenv::format_entry(&value, &text, entry.range.start) { + Ok(rendered) => splices.push((entry.range.clone(), rendered, hit.uuid.clone())), + Err(e) => result.refuse(&hit.uuid, format!("{rel}: {e}")), + } + } + let mut next = text.clone(); + splices.sort_by_key(|(range, _, _)| std::cmp::Reverse(range.start)); + let mut changed = false; + for (range, rendered, uuid) in splices { + // A pin refused in another entry of this lock stays hosted in all. + if result.refused.contains_key(&uuid) { + continue; + } + next.replace_range(range, &rendered); + result.handled.insert(uuid); + changed = true; + } + if changed { + view.write(rel, next); + } + } + result +} + +// ── requirements.txt ───────────────────────────────────────────────────────── + +static HOSTED_LINE_RE: std::sync::LazyLock = std::sync::LazyLock::new(|| { + Regex::new(r"^([A-Za-z0-9][A-Za-z0-9._-]*)(\s*\[[^\]\r\n]*\])?\s*@\s*(\S+)(.*)$") + .expect("static hosted requirement regex is valid") +}); + +/// Whether a requirements line carries a `--hash` option. +fn has_hash_option(tokens: &[&str]) -> bool { + tokens + .iter() + .any(|t| *t == "--hash" || t.starts_with("--hash=")) +} + +/// A hosted requirement line, cut into what its registry spelling keeps. +struct HostedLine { + uuid: String, + version: String, + /// BOM + indentation before the name. + prefix: String, + name: String, + extras: String, + marker: String, + options: String, + comment: String, +} + +/// The in-scope hosted line `requirement` is, if any: `Err((uuid, why))` +/// when it is one that cannot be restored. +fn hosted_line( + requirement: &super::super::requirements::LogicalRequirement, + pins: &BTreeMap<&str, &HostedPin>, + ctx: &Ctx<'_>, +) -> Option> { + use super::super::requirements::{requirement_tokens, unquoted_index}; + let text = requirement.text.trim(); + let caps = HOSTED_LINE_RE.captures(text)?; + let pin = pin_of(&caps[3], pins, ctx)?; + let version = match pin.name_version() { + Some((name, version)) + if canonicalize_pypi_name(&caps[1]) == canonicalize_pypi_name(&name) => + { + version + } + _ => { + return Some(Err(( + pin.uuid.clone(), + format!( + "{:?} is wired to the hosted artifact of another package", + &caps[1] + ), + ))) + } + }; + let rest = caps.get(4).map_or("", |m| m.as_str()); + let (body, comment) = + unquoted_index(rest, '#', true).map_or((rest, ""), |i| (&rest[..i], &rest[i..])); + let tokens = requirement_tokens(body); + let marker_len = tokens.iter().take_while(|t| !t.starts_with("--")).count(); + let marker = tokens[..marker_len].join(" "); + let mut options = Vec::new(); + let mut rest_tokens = tokens[marker_len..].iter(); + while let Some(token) = rest_tokens.next() { + if *token == "--hash" { + rest_tokens.next(); + } else if !token.starts_with("--hash=") { + options.push(*token); + } + } + let unprefixed = requirement + .original + .strip_prefix('\u{feff}') + .unwrap_or(&requirement.original); + let indent = &unprefixed[..unprefixed.len() - unprefixed.trim_start_matches([' ', '\t']).len()]; + let bom = if requirement.original.starts_with('\u{feff}') { + "\u{feff}" + } else { + "" + }; + Some(Ok(HostedLine { + uuid: pin.uuid.clone(), + version, + prefix: format!("{bom}{indent}"), + name: caps[1].to_string(), + extras: caps.get(2).map_or("", |m| m.as_str().trim()).to_string(), + marker, + options: options.join(" "), + comment: comment.trim().to_string(), + })) +} + +pub(crate) async fn restore_requirements( + view: &mut View<'_>, + pins: &[&HostedPin], + files: &[String], + ctx: &Ctx<'_>, +) -> FormatResult { + use super::super::requirements::{logical_requirements, requirement_tokens}; + let mut result = FormatResult::default(); + let pins = by_uuid(pins); + for rel in files { + let Some(text) = read_or_refuse(view, rel, &pins, &mut result).await else { + continue; + }; + let mut requirements = logical_requirements(&text); + let mut hits: Vec<(usize, HostedLine)> = Vec::new(); + // The file's hash-checking mode, read off every line the hosted + // rewrite did not write. + let (mut require_hashes, mut hashed, mut unhashed) = (false, 0usize, 0usize); + // The continuation indent of a hashed line pip-compile style + // (`name==v \` then one indented `--hash=…` per line). + let mut continuation: Option = None; + for (i, requirement) in requirements.iter().enumerate() { + match hosted_line(requirement, &pins, ctx) { + Some(Ok(line)) => { + hits.push((i, line)); + continue; + } + Some(Err((uuid, why))) => { + result.refuse(&uuid, format!("{rel}: {why}")); + continue; + } + None => {} + } + let (code, _) = crate::utils::requirements::split_comment(&requirement.text); + let code = code.trim(); + if code.is_empty() { + continue; + } + let tokens = requirement_tokens(code); + if tokens.contains(&"--require-hashes") { + require_hashes = true; + } + if code.starts_with('-') { + continue; + } + // Another pin's hosted line (always hashed) says nothing about + // the original mode. + if tokens + .iter() + .any(|t| ctx.hosted_uuid(t.trim_end_matches(';')).is_some()) + { + continue; + } + if has_hash_option(&tokens) { + hashed += 1; + if continuation.is_none() { + continuation = requirement + .original + .split('\n') + .nth(1) + .filter(|l| l.trim_start().starts_with("--hash")) + .map(|l| l[..l.len() - l.trim_start().len()].to_string()); + } + } else { + unhashed += 1; + } + } + if hits.is_empty() { + continue; + } + let hash_mode = match (require_hashes || hashed > 0, unhashed > 0) { + (true, false) => Ok(true), + (false, true) => Ok(false), + (true, true) => Err(format!( + "{rel} mixes hashed and unhashed requirements, so whether the original line \ + carried `--hash` options is not derivable" + )), + (false, false) => Err(format!( + "every requirement in {rel} is a hosted pin, so whether the original used pip's \ + hash-checking mode (`--hash`) is not derivable" + )), + }; + let hash_mode = match hash_mode { + Ok(mode) => mode, + Err(why) => { + for (_, line) in &hits { + result.refuse(&line.uuid, why.clone()); + } + continue; + } + }; + let released = if hash_mode { + let wanted = hits + .iter() + .map(|(_, l)| { + ( + l.uuid.clone(), + canonicalize_pypi_name(&l.name), + l.version.clone(), + ) + }) + .collect(); + fetch_release_files(&wanted, ctx, &mut result).await + } else { + BTreeMap::new() + }; + let eol = if text.contains("\r\n") { "\r\n" } else { "\n" }; + let mut rewritten: Vec<(usize, String, String)> = Vec::new(); + for (i, line) in &hits { + if result.refused.contains_key(&line.uuid) { + continue; + } + let mut out = format!( + "{}{}{}=={}", + line.prefix, line.name, line.extras, line.version + ); + for suffix in [&line.marker, &line.options] { + if !suffix.is_empty() { + out.push(' '); + out.push_str(suffix); + } + } + if hash_mode { + let Some(release) = + released.get(&(canonicalize_pypi_name(&line.name), line.version.clone())) + else { + continue; + }; + let mut hashes: Vec<&str> = release.iter().map(|f| f.sha256.as_str()).collect(); + hashes.sort(); + hashes.dedup(); + for hash in hashes { + match &continuation { + Some(indent) => { + out.push_str(&format!(" \\{eol}{indent}--hash=sha256:{hash}")) + } + None => out.push_str(&format!(" --hash=sha256:{hash}")), + } + } + } + if !line.comment.is_empty() { + out.push(' '); + out.push_str(&line.comment); + } + rewritten.push((*i, out, line.uuid.clone())); + } + let mut changed = false; + for (i, out, uuid) in rewritten { + if result.refused.contains_key(&uuid) { + continue; + } + requirements[i].original = out; + result.handled.insert(uuid); + changed = true; + } + if changed { + let next: String = requirements + .into_iter() + .map(|r| r.original + &r.ending) + .collect(); + view.write(rel, next); + } + } + result +} + +// ── Hatch: pyproject.toml / hatch.toml ─────────────────────────────────────── + +/// `declared[extras] @ [ ; marker]` → `(declared, extras, +/// location, marker)`. +fn direct_reference(spec: &str) -> Option<(&str, &str, &str, &str)> { + let spec = spec.trim(); + let declared = pep508_name(spec); + if declared.is_empty() { + return None; + } + let mut rest = spec[declared.len()..].trim_start(); + let mut extras = ""; + if rest.starts_with('[') { + let end = rest.find(']')?; + extras = &rest[..=end]; + rest = rest[end + 1..].trim_start(); + } + let rest = rest.strip_prefix('@')?.trim_start(); + let end = rest.find(char::is_whitespace).unwrap_or(rest.len()); + let location = rest[..end].trim_end_matches(';'); + let after = &rest[location.len()..]; + let marker = after.trim().strip_prefix(';').map_or("", str::trim); + Some((declared, extras, location, marker)) +} + +/// Restore every hosted direct reference in one dependency array; the uuids +/// restored are added to `restored`, a mismatched one refuses. +fn restore_hatch_array( + item: &mut Item, + pins: &BTreeMap<&str, &HostedPin>, + ctx: &Ctx<'_>, + result: &mut FormatResult, + restored: &mut BTreeSet, +) -> usize { + let Some(array) = item.as_array_mut() else { + return 0; + }; + let mut count = 0; + for entry in array.iter_mut() { + let Some(spec) = entry.as_str() else { + continue; + }; + let Some((declared, extras, location, marker)) = direct_reference(spec) else { + continue; + }; + let Some(pin) = pin_of(location, pins, ctx) else { + continue; + }; + let Some((name, version)) = pin_coords(pin, result) else { + continue; + }; + if canonicalize_pypi_name(declared) != name { + result.refuse( + &pin.uuid, + format!("{declared:?} is wired to the hosted artifact of another package"), + ); + continue; + } + let mut next = format!("{declared}{extras}=={version}"); + if !marker.is_empty() { + next.push_str(" ; "); + next.push_str(marker); + } + let decor = entry.decor().clone(); + *entry = toml_edit::Value::from(next); + *entry.decor_mut() = decor; + restored.insert(pin.uuid.clone()); + count += 1; + } + count +} + +/// Every `dependencies` / `extra-dependencies` array of an `envs` table. +fn restore_hatch_envs( + envs: Option<&mut Item>, + pins: &BTreeMap<&str, &HostedPin>, + ctx: &Ctx<'_>, + result: &mut FormatResult, + restored: &mut BTreeSet, +) { + let Some(envs) = envs.and_then(Item::as_table_like_mut) else { + return; + }; + for (_, env) in envs.iter_mut() { + let Some(env) = env.as_table_like_mut() else { + continue; + }; + for key in ["dependencies", "extra-dependencies"] { + if let Some(item) = env.get_mut(key) { + restore_hatch_array(item, pins, ctx, result, restored); + } + } + } +} + +/// Remove `keys`' last table's `allow-direct-references = true` (the Hatch +/// permission the hosted rewrite set), then every table on the path that is +/// left empty. +fn drop_direct_reference_permission(doc: &mut DocumentMut, keys: &[&str]) -> bool { + fn walk(table: &mut dyn toml_edit::TableLike, keys: &[&str]) -> bool { + let Some((first, rest)) = keys.split_first() else { + return table.get("allow-direct-references").and_then(Item::as_bool) == Some(true) + && table.remove("allow-direct-references").is_some(); + }; + let Some(child) = table.get_mut(first).and_then(Item::as_table_like_mut) else { + return false; + }; + let removed = walk(child, rest); + if removed && child.is_empty() { + table.remove(first); + } + removed + } + walk(doc.as_table_mut(), keys) +} + +pub(crate) async fn restore_hatch( + view: &mut View<'_>, + pins: &[&HostedPin], + files: &[String], + ctx: &Ctx<'_>, +) -> FormatResult { + use crate::utils::hatch::HATCH_FILES; + let mut result = FormatResult::default(); + let pins = by_uuid(pins); + let mut docs: BTreeMap<&str, (String, DocumentMut)> = BTreeMap::new(); + for rel in HATCH_FILES { + let text = if files.iter().any(|f| f == rel) { + match read_or_refuse(view, rel, &pins, &mut result).await { + Some(text) => text, + None => continue, + } + } else { + match view.read(rel).await { + Ok(Some(text)) => text, + _ => continue, + } + }; + match text.trim_start_matches('\u{feff}').parse::() { + Ok(doc) => { + docs.insert(rel, (text, doc)); + } + Err(e) => refuse_all_in(&pins, rel, &mut result, format!("{rel}: {e}")), + } + } + let mut restored: BTreeSet = BTreeSet::new(); + let mut project_restored = 0; + if let Some((_, doc)) = docs.get_mut("pyproject.toml") { + if let Some(project) = doc.get_mut("project").and_then(Item::as_table_like_mut) { + if let Some(item) = project.get_mut("dependencies") { + project_restored += + restore_hatch_array(item, &pins, ctx, &mut result, &mut restored); + } + if let Some(groups) = project + .get_mut("optional-dependencies") + .and_then(Item::as_table_like_mut) + { + for (_, item) in groups.iter_mut() { + project_restored += + restore_hatch_array(item, &pins, ctx, &mut result, &mut restored); + } + } + } + if let Some(groups) = doc + .get_mut("dependency-groups") + .and_then(Item::as_table_like_mut) + { + for (_, item) in groups.iter_mut() { + project_restored += + restore_hatch_array(item, &pins, ctx, &mut result, &mut restored); + } + } + let hatch = doc + .get_mut("tool") + .and_then(Item::as_table_like_mut) + .and_then(|t| t.get_mut("hatch")); + restore_hatch_envs( + hatch.and_then(|h| h.get_mut("envs")), + &pins, + ctx, + &mut result, + &mut restored, + ); + } + if let Some((_, doc)) = docs.get_mut("hatch.toml") { + restore_hatch_envs(doc.get_mut("envs"), &pins, ctx, &mut result, &mut restored); + } + // The permission the rewrite set once a PROJECT table got a direct + // reference: dropped when none is left there (whatever its prior value, + // it then governs nothing). + let project_direct = docs.get("pyproject.toml").is_some_and(|(_, doc)| { + crate::vendor::common::pyproject_dependency_specs(doc) + .into_iter() + .any(|(_, spec)| spec.split(';').next().is_some_and(|r| r.contains('@'))) + }); + if project_restored > 0 && !project_direct { + let external = docs + .get("hatch.toml") + .is_some_and(|(_, doc)| doc.contains_key("metadata")); + let (file, keys): (&str, &[&str]) = if external { + ("hatch.toml", &["metadata"]) + } else { + ("pyproject.toml", &["tool", "hatch", "metadata"]) + }; + if let Some((_, doc)) = docs.get_mut(file) { + drop_direct_reference_permission(doc, keys); + } + } + let restored: BTreeSet = restored + .into_iter() + .filter(|u| !result.refused.contains_key(u)) + .collect(); + if !restored.is_empty() { + for (rel, (text, doc)) in docs { + let bom = if text.starts_with('\u{feff}') { + "\u{feff}" + } else { + "" + }; + let next = preserve_line_endings(&text, format!("{bom}{doc}")); + if next != text { + view.write(rel, next); + } + } + } + result.handled.extend(restored); + result +} + +#[cfg(test)] +mod tests { + use super::*; + + fn file(name: &str) -> PypiFile { + PypiFile { + filename: name.into(), + url: format!("https://files.example/{name}"), + sha256: "a".repeat(64), + size: None, + upload_time: None, + } + } + + #[test] + fn universal_release_accepts_only_pure_python3_wheels() { + assert!(universal_release(&[ + file("x-1.tar.gz"), + file("x-1-py3-none-any.whl") + ])); + assert!(universal_release(&[file("x-1-py2.py3-none-any.whl")])); + assert!(universal_release(&[file("x-1-2-py3-none-any.whl")])); + assert!(!universal_release(&[file("x-1-py27-none-any.whl")])); + assert!(!universal_release(&[file( + "x-1-cp311-cp311-manylinux_2_17_x86_64.whl" + )])); + assert!(!universal_release(&[file("x-1-py3-abi3-any.whl")])); + } + + #[test] + fn direct_references_split_like_the_hatch_rewriter_writes_them() { + assert_eq!( + direct_reference("Urllib3[socks] @ https://h/u.whl#sha256=ab ; python_version >= '3'"), + Some(( + "Urllib3", + "[socks]", + "https://h/u.whl#sha256=ab", + "python_version >= '3'" + )) + ); + assert_eq!( + direct_reference("urllib3 @ https://h/u.whl"), + Some(("urllib3", "", "https://h/u.whl", "")) + ); + assert_eq!(direct_reference("urllib3==1.0"), None); + } + + #[test] + fn pipenv_index_follows_sources_siblings_and_the_pipfile() { + let doc = json!({"_meta": {"sources": [ + {"name": "private", "url": "https://mirror.example/simple"}, + {"name": "pypi", "url": "https://pypi.org/simple/"}, + ]}}); + assert_eq!( + pipenv_index(&doc, &[Some("private")], None, "x").unwrap(), + Some("pypi".into()) + ); + // Old Pipenv: registry siblings record no index. + assert_eq!(pipenv_index(&doc, &[None, None], None, "x").unwrap(), None); + let pipfile = "[packages]\nX = { version = \"==1\", index = \"private\" }\n"; + assert!(pipenv_index(&doc, &[], Some(pipfile), "x") + .unwrap_err() + .contains("not PyPI")); + let mirror = json!({"_meta": {"sources": [{"name": "m", "url": "https://m/simple"}]}}); + assert!(pipenv_index(&mirror, &[], None, "x") + .unwrap_err() + .contains("is PyPI")); + let twice = json!({"_meta": {"sources": [ + {"name": "a", "url": "https://pypi.org/simple"}, + {"name": "b", "url": "https://pypi.python.org/simple"}, + ]}}); + assert!(pipenv_index(&twice, &[], None, "x") + .unwrap_err() + .contains("ambiguous")); + } + + #[test] + fn toml_values_keep_their_layout() { + let rendered = multiline_toml_array(&[format!( + "{{file = {}, hash = \"sha256:ab\"}}", + toml_quote("a.whl") + )]); + let value = toml_value(&rendered).unwrap(); + let mut doc: DocumentMut = "files = []\n".parse().unwrap(); + doc["files"] = Item::Value(value); + assert_eq!( + doc.to_string(), + "files = [\n {file = \"a.whl\", hash = \"sha256:ab\"},\n]\n" + ); + assert_eq!(multiline_toml_array(&[]), "[]"); + assert_eq!(toml_quote("a\"b\\"), "\"a\\\"b\\\\\""); + } +} diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/pypi_locks.rs b/crates/socket-patch-core/src/patch/redirect/upstream/pypi_locks.rs new file mode 100644 index 000000000..c883461aa --- /dev/null +++ b/crates/socket-patch-core/src/patch/redirect/upstream/pypi_locks.rs @@ -0,0 +1,380 @@ +//! Poetry and PDM upstream restores. +//! +//! * `poetry.lock` (`utils::poetry_lock::rewrite_poetry_lock`): the hosted +//! rewrite adds `[package.source] type = "url"` (lock 1.0 also a +//! `reference = ""` and a `#sha256=…&` url fragment) and replaces the +//! package's files with the patched wheel — `files = [...]` on lock 2.x, +//! `[metadata.files].` on 1.0/1.1, where it ALSO adds a package-level +//! `files` no Poetry 1.x lock carries. The restore drops the source table +//! (a package without one is a PyPI package — the rewriter refuses every +//! pre-existing source) and re-derives every release file from PyPI. +//! * `pdm.lock` (`utils::pdm_lock::rewrite_pdm_lock`): the rewrite adds a +//! package `url` and replaces its files, inline or in the lock_version 2 +//! `[metadata.files]."[extras] "` table, in every extras +//! variant. A lock without `cross_platform` keeps only the files its +//! targets install; which ones is re-derivable only when every wheel is +//! pure Python 3 ([`super::pypi::universal_release`]), otherwise the pin +//! is refused. +//! +//! Both edit a parsed `toml_edit` document, so every byte outside the +//! restored package entries is the file's own. + +use std::collections::BTreeSet; + +use toml_edit::{DocumentMut, Item}; + +use super::client::PypiFile; +use super::pypi::{ + by_uuid, fetch_release_files, multiline_toml_array, pin_of, read_or_refuse, refuse_all_in, + toml_quote, toml_value, universal_release, +}; +use super::{Ctx, FormatResult, HostedPin, View}; +use crate::crawlers::python_crawler::canonicalize_pypi_name; +use crate::utils::python_lock::preserve_line_endings; + +/// One hosted package entry of a TOML lock. +struct LockHit { + /// Index into the lock's `[[package]]` array. + index: usize, + uuid: String, + name: String, + version: String, +} + +/// The `files` array value Poetry / PDM write for `release`: one +/// `{ = …, hash = "sha256:…"}` per file. +fn files_value(release: &[PypiFile], by_url: bool) -> Option { + let entries: Vec = release + .iter() + .map(|f| { + let (key, location) = if by_url { + ("url", &f.url) + } else { + ("file", &f.filename) + }; + format!( + "{{{key} = {}, hash = {}}}", + toml_quote(location), + toml_quote(&format!("sha256:{}", f.sha256)) + ) + }) + .collect(); + toml_value(&multiline_toml_array(&entries)) +} + +/// Replace `key`'s value in place (keeping the key and its position), or +/// insert it. +fn set_value(table: &mut dyn toml_edit::TableLike, key: &str, value: toml_edit::Value) { + match table.get_mut(key) { + Some(item) => *item = Item::Value(value), + None => { + table.insert(key, Item::Value(value)); + } + } +} + +/// Parse `rel`'s lock; a malformed one refuses every pin wired in it. +async fn parse_lock( + view: &mut View<'_>, + rel: &str, + pins: &std::collections::BTreeMap<&str, &HostedPin>, + result: &mut FormatResult, +) -> Option<(String, DocumentMut)> { + let text = read_or_refuse(view, rel, pins, result).await?; + match text.parse::() { + Ok(doc) => Some((text, doc)), + Err(e) => { + refuse_all_in(pins, rel, result, format!("{rel} is not valid TOML: {e}")); + None + } + } +} + +/// The hosted entries of a lock's `[[package]]` array: `location` reads a +/// package's install location (Poetry's `[package.source]` url, PDM's +/// `url`). An entry naming another package or version than its pin +/// refuses it. +fn lock_hits( + doc: &DocumentMut, + rel: &str, + pins: &std::collections::BTreeMap<&str, &HostedPin>, + ctx: &Ctx<'_>, + location: impl Fn(&toml_edit::Table) -> Option<&str>, + result: &mut FormatResult, +) -> Vec { + let mut hits = Vec::new(); + let Some(packages) = doc.get("package").and_then(Item::as_array_of_tables) else { + return hits; + }; + for (index, package) in packages.iter().enumerate() { + let Some(pin) = location(package).and_then(|l| pin_of(l, pins, ctx)) else { + continue; + }; + let name = package.get("name").and_then(Item::as_str).unwrap_or(""); + let version = package.get("version").and_then(Item::as_str).unwrap_or(""); + let agrees = pin.name_version().is_some_and(|(n, v)| { + canonicalize_pypi_name(&n) == canonicalize_pypi_name(name) && v == version + }); + if !agrees { + result.refuse( + &pin.uuid, + format!( + "{rel}: the entry wiring it names {name:?} {version:?}, not {}", + pin.purl + ), + ); + continue; + } + hits.push(LockHit { + index, + uuid: pin.uuid.clone(), + name: name.to_string(), + version: version.to_string(), + }); + } + hits +} + +fn wanted(hits: &[LockHit], result: &FormatResult) -> BTreeSet<(String, String, String)> { + hits.iter() + .filter(|h| !result.refused.contains_key(&h.uuid)) + .map(|h| { + ( + h.uuid.clone(), + canonicalize_pypi_name(&h.name), + h.version.clone(), + ) + }) + .collect() +} + +/// Write `doc` back when any hit survived, marking the survivors handled. +fn finish( + view: &mut View<'_>, + rel: &str, + text: &str, + doc: &DocumentMut, + restored: BTreeSet, + result: &mut FormatResult, +) { + if restored.iter().all(|u| result.refused.contains_key(u)) { + return; + } + view.write(rel, preserve_line_endings(text, doc.to_string())); + result.handled.extend(restored); +} + +// ── poetry.lock ────────────────────────────────────────────────────────────── + +pub(crate) async fn restore_poetry( + view: &mut View<'_>, + pins: &[&HostedPin], + files: &[String], + ctx: &Ctx<'_>, +) -> FormatResult { + let mut result = FormatResult::default(); + let pins = by_uuid(pins); + for rel in files { + let Some((text, mut doc)) = parse_lock(view, rel, &pins, &mut result).await else { + continue; + }; + let hits = lock_hits( + &doc, + rel, + &pins, + ctx, + |package| { + let source = package.get("source")?; + (source.get("type").and_then(Item::as_str) == Some("url")) + .then(|| source.get("url").and_then(Item::as_str)) + .flatten() + }, + &mut result, + ); + if hits.is_empty() { + continue; + } + let format = match crate::utils::poetry_lock::lock_version(&doc) { + Ok("0") => Err("Poetry 0.12 locks (format \"0\") ignore url sources".to_string()), + Ok(format) => Ok(format.to_string()), + Err(e) => Err(e), + }; + let format = match format { + Ok(format) => format, + Err(why) => { + for hit in &hits { + result.refuse(&hit.uuid, format!("{rel}: {why}")); + } + continue; + } + }; + let released = fetch_release_files(&wanted(&hits, &result), ctx, &mut result).await; + let legacy = !format.starts_with('2'); + // Poetry 1.x locks keep files in `[metadata.files]` only; the + // package-level copy is the rewriter's unless a package it never + // touched (no source table) has one too. + let siblings_carry_files = doc + .get("package") + .and_then(Item::as_array_of_tables) + .is_some_and(|packages| { + packages + .iter() + .any(|p| !p.contains_key("source") && p.contains_key("files")) + }); + let mut restored = BTreeSet::new(); + for hit in &hits { + if result.refused.contains_key(&hit.uuid) { + continue; + } + let Some(release) = + released.get(&(canonicalize_pypi_name(&hit.name), hit.version.clone())) + else { + continue; + }; + let Some(value) = files_value(release, false) else { + result.refuse(&hit.uuid, "the release file list does not render as TOML"); + continue; + }; + if legacy { + let canon = canonicalize_pypi_name(&hit.name); + let table = doc + .get_mut("metadata") + .and_then(Item::as_table_like_mut) + .and_then(|m| m.get_mut("files")) + .and_then(Item::as_table_like_mut); + let key = table.as_ref().and_then(|t| { + t.iter() + .find(|(k, _)| canonicalize_pypi_name(k) == canon) + .map(|(k, _)| k.to_string()) + }); + let (Some(table), Some(key)) = (table, key) else { + result.refuse( + &hit.uuid, + format!("{rel} has no [metadata.files] entry for {}", hit.name), + ); + continue; + }; + set_value(table, &key, value.clone()); + } + let Some(package) = doc + .get_mut("package") + .and_then(Item::as_array_of_tables_mut) + .and_then(|p| p.get_mut(hit.index)) + else { + continue; + }; + package.remove("source"); + if legacy && !siblings_carry_files { + package.remove("files"); + } else { + set_value(package, "files", value); + } + restored.insert(hit.uuid.clone()); + } + finish(view, rel, &text, &doc, restored, &mut result); + } + result +} + +// ── pdm.lock ───────────────────────────────────────────────────────────────── + +pub(crate) async fn restore_pdm( + view: &mut View<'_>, + pins: &[&HostedPin], + files: &[String], + ctx: &Ctx<'_>, +) -> FormatResult { + use crate::utils::pdm_lock::{legacy_files_key, lock_version, validate_strategy}; + let mut result = FormatResult::default(); + let pins = by_uuid(pins); + for rel in files { + let Some((text, mut doc)) = parse_lock(view, rel, &pins, &mut result).await else { + continue; + }; + let hits = lock_hits( + &doc, + rel, + &pins, + ctx, + |package| package.get("url").and_then(Item::as_str), + &mut result, + ); + if hits.is_empty() { + continue; + } + let shape = lock_version(doc.as_table()).and_then(|version| { + let flags = validate_strategy(doc.as_table())?; + Ok((version.to_string(), flags)) + }); + let (version, flags) = match shape { + Ok(shape) => shape, + Err(why) => { + for hit in &hits { + result.refuse(&hit.uuid, format!("{rel}: {why}")); + } + continue; + } + }; + let static_urls = flags.iter().any(|f| f == "static_urls"); + // lock_version 2 (PDM 0.x/1.x) always recorded every release file. + let cross_platform = version == "2" || flags.iter().any(|f| f == "cross_platform"); + let released = fetch_release_files(&wanted(&hits, &result), ctx, &mut result).await; + let mut restored = BTreeSet::new(); + for hit in &hits { + if result.refused.contains_key(&hit.uuid) { + continue; + } + let Some(release) = + released.get(&(canonicalize_pypi_name(&hit.name), hit.version.clone())) + else { + continue; + }; + if !cross_platform && !universal_release(release) { + result.refuse( + &hit.uuid, + format!( + "{rel} (lock_version {version}, no cross_platform strategy) records only \ + the files its lock targets install, and {}=={} ships platform- or \ + interpreter-specific wheels, so which ones it kept is not derivable", + hit.name, hit.version + ), + ); + continue; + } + let Some(value) = files_value(release, static_urls) else { + result.refuse(&hit.uuid, "the release file list does not render as TOML"); + continue; + }; + let Some(package) = doc + .get_mut("package") + .and_then(Item::as_array_of_tables_mut) + .and_then(|p| p.get_mut(hit.index)) + else { + continue; + }; + if package.contains_key("files") { + package.remove("url"); + set_value(package, "files", value); + } else { + let key = legacy_files_key(package); + package.remove("url"); + let table = doc + .get_mut("metadata") + .and_then(Item::as_table_like_mut) + .and_then(|m| m.get_mut("files")) + .and_then(Item::as_table_like_mut); + let (Some(table), Some(key)) = (table, key) else { + result.refuse( + &hit.uuid, + format!("{rel} has no [metadata.files] entry for {}", hit.name), + ); + continue; + }; + set_value(table, &key, value); + } + restored.insert(hit.uuid.clone()); + } + finish(view, rel, &text, &doc, restored, &mut result); + } + result +} diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/uv.rs b/crates/socket-patch-core/src/patch/redirect/upstream/uv.rs new file mode 100644 index 000000000..321b01de7 --- /dev/null +++ b/crates/socket-patch-core/src/patch/redirect/upstream/uv.rs @@ -0,0 +1,1197 @@ +//! uv upstream restore: `uv.lock`, PEP 723 script locks (`