From 34f9a092e6768e1afa4b408490777d6d533eefa7 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Wed, 7 Oct 2026 12:19:01 -0400 Subject: [PATCH 01/20] Read lockfile discovery through any project view Discovery read the disk directly in ten places (the requirements -r walk, the Python lock listing, cargo's config resolution, Rush subspace listing, nuget's spelling and feed probes, maven's vendored jars, sbt's resolution evidence and hashes, vlt's bundled-store walk), so it could only run over a real checkout. Route the content reads and listings through ProjectView, keep the probes that need an installed tree behind DiscoverCtx::disk_root() (they see nothing in memory), and add discover_patched_refs_view() so the in-memory hosted engine can ask the same "is this pinned" question the disk commands ask. The view-based Python lock listing replaces the hosted engine's private copy, the requirements include walk and cargo's effective-config probe take a view, and the discovery future is boxed as Send (the in-memory engine's future must be Send; vlt's bundled-store pairs are collected first for the same reason). Behavior on disk is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/socket-patch-core/src/hosted/engine.rs | 20 +------ .../src/vendor/cargo_config.rs | 15 +++-- .../src/vendor/lock_inventory/pnpm.rs | 23 +++---- .../src/vendor/lock_inventory/view.rs | 29 ++++++++- .../src/vendor/pypi_requirements.rs | 36 ++++++----- .../src/vex/discover/cargo.rs | 8 +-- .../src/vex/discover/maven.rs | 8 ++- .../socket-patch-core/src/vex/discover/mod.rs | 60 +++++++++++++++---- .../socket-patch-core/src/vex/discover/npm.rs | 2 +- .../src/vex/discover/nuget.rs | 13 +++- .../src/vex/discover/pypi_locks.rs | 2 +- .../src/vex/discover/pypi_other.rs | 29 ++++----- .../socket-patch-core/src/vex/discover/sbt.rs | 11 +++- .../socket-patch-core/src/vex/discover/vlt.rs | 17 ++++-- 14 files changed, 178 insertions(+), 95 deletions(-) diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index f0c1440b7..e5e259d85 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -361,24 +361,6 @@ impl CandidateFiles { } } -/// The root-level Python lock names (sorted). -async fn python_lock_paths(view: &ProjectView<'_>) -> Vec { - match view { - ProjectView::Disk(cwd) - | ProjectView::Snapshot(crate::vendor::lock_inventory::DiskSnapshot { - root: cwd, .. - }) => crate::utils::python_lock::python_lock_paths(cwd).unwrap_or_default(), - ProjectView::Memory(project) => project - .children("") - .into_iter() - .filter(|(name, is_dir)| { - !is_dir && crate::utils::python_lock::is_python_lock_name(name) - }) - .map(|(name, _)| name) - .collect(), - } -} - /// Whether the project is a Rush monorepo (disk: `rush.json` is a file). fn rush_repo(view: &ProjectView<'_>) -> bool { match view { @@ -552,7 +534,7 @@ pub async fn read_candidate_files( } } - for path in python_lock_paths(view).await { + for path in view.python_lock_paths() { if let Some(script) = crate::utils::python_lock::script_of_lock(&path) { out.read(view, unreadable, script).await; } diff --git a/crates/socket-patch-core/src/vendor/cargo_config.rs b/crates/socket-patch-core/src/vendor/cargo_config.rs index 6740038f7..448f866ae 100644 --- a/crates/socket-patch-core/src/vendor/cargo_config.rs +++ b/crates/socket-patch-core/src/vendor/cargo_config.rs @@ -321,10 +321,17 @@ pub(crate) fn patch_entries(doc: &DocumentMut) -> Vec> { /// extension (and warns) — else [`CONFIG_TOML`] (which may not exist yet). /// `metadata`, not lstat: cargo's own existence probe follows symlinks. pub(crate) async fn effective_config_rel(project_root: &Path) -> &'static str { - if fs::metadata(project_root.join(".cargo").join("config")) - .await - .is_ok() - { + effective_config_rel_in(crate::vendor::lock_inventory::ProjectView::Disk( + project_root, + )) + .await +} + +/// [`effective_config_rel`] over any project view. +pub(crate) async fn effective_config_rel_in( + view: crate::vendor::lock_inventory::ProjectView<'_>, +) -> &'static str { + if view.exists(CONFIG_LEGACY).await { CONFIG_LEGACY } else { CONFIG_TOML diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/pnpm.rs b/crates/socket-patch-core/src/vendor/lock_inventory/pnpm.rs index f477d9cb6..c89228a69 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/pnpm.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/pnpm.rs @@ -21,20 +21,15 @@ use super::LockfileEntry; /// directories only (a symlinked subspace dir could point the read outside /// the project) with traversal-safe UTF-8 names. Stat / list only; whether /// the project IS a Rush monorepo (`rush.json`) is the caller's check. -pub(crate) async fn rush_lock_rels(root: &Path) -> Vec { - let mut names = Vec::new(); - if let Ok(mut dir) = tokio::fs::read_dir(root.join(RUSH_SUBSPACES_DIR)).await { - while let Ok(Some(entry)) = dir.next_entry().await { - if !entry.file_type().await.is_ok_and(|t| t.is_dir()) { - continue; - } - if let Some(name) = entry.file_name().to_str() { - if is_safe_single_segment(name) { - names.push(name.to_string()); - } - } - } - } +pub(crate) async fn rush_lock_rels(view: ProjectView<'_>) -> Vec { + let mut names: Vec = view + .list_dir(RUSH_SUBSPACES_DIR) + .await + .unwrap_or_default() + .into_iter() + .filter(|entry| entry.is_dir && is_safe_single_segment(&entry.name)) + .map(|entry| entry.name) + .collect(); names.sort(); let mut rels = vec![RUSH_COMMON_LOCK_REL.to_string()]; rels.extend( diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/view.rs b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs index acd48d721..2a356a70d 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/view.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs @@ -248,7 +248,34 @@ pub enum ProjectView<'a> { Snapshot(&'a DiskSnapshot<'a>), } -impl ProjectView<'_> { +impl<'a> ProjectView<'a> { + /// The project root on disk; `None` in memory. + pub fn disk_root(&self) -> Option<&'a Path> { + match *self { + ProjectView::Disk(root) => Some(root), + ProjectView::Snapshot(snap) => Some(snap.root), + ProjectView::Memory(_) => None, + } + } + + /// The root-level Python lock names (sorted; see + /// [`crate::utils::python_lock::python_lock_paths`]). + pub fn python_lock_paths(&self) -> Vec { + match self { + ProjectView::Disk(root) | ProjectView::Snapshot(DiskSnapshot { root, .. }) => { + crate::utils::python_lock::python_lock_paths(root).unwrap_or_default() + } + ProjectView::Memory(project) => project + .children("") + .into_iter() + .filter(|(name, is_dir)| { + !is_dir && crate::utils::python_lock::is_python_lock_name(name) + }) + .map(|(name, _)| name) + .collect(), + } + } + /// FIFO-safe regular-file text read. pub async fn read_text(&self, rel: &str) -> io::Result { match self { diff --git a/crates/socket-patch-core/src/vendor/pypi_requirements.rs b/crates/socket-patch-core/src/vendor/pypi_requirements.rs index 7733f6154..c70ed6ca4 100644 --- a/crates/socket-patch-core/src/vendor/pypi_requirements.rs +++ b/crates/socket-patch-core/src/vendor/pypi_requirements.rs @@ -21,7 +21,7 @@ //! newline style is preserved. use std::collections::HashSet; -use std::path::{Path, PathBuf}; +use std::path::Path; use crate::crawlers::python_crawler::canonicalize_pypi_name; use crate::utils::fs::{atomic_write_bytes_preserving_mode, read_regular_to_string}; @@ -835,7 +835,8 @@ pub(in crate::vendor) fn vendor_line( /// must never edit them. The root file is always element 0. async fn collect_requirements_files(root: &Path) -> Result, (&'static str, String)> { let mut out: Vec = Vec::new(); - walk_requirements_tree(root, |rel, path, read| match read { + let view = crate::vendor::lock_inventory::ProjectView::Disk(root); + walk_requirements_tree(view, |rel, read| match read { Ok(content) => { // Out-of-root (`../`) and absolute includes resolve outside any // committable root — readable so a pin inside can refuse, never @@ -851,7 +852,7 @@ async fn collect_requirements_files(root: &Path) -> Result, (&'stat } Err(_) if out.is_empty() => Err(( "pypi_no_requirements", - format!("cannot read {}", path.display()), + format!("cannot read {}", root.join(rel).display()), )), // A broken include is pip's error to report; vendor just can't see // inside it. Skip. @@ -877,8 +878,16 @@ async fn collect_requirements_files(root: &Path) -> Result, (&'stat /// and absolute includes are never editable, so they are neither named nor /// followed. pub async fn requirements_include_names(root: &Path) -> std::io::Result> { + requirements_include_names_in(crate::vendor::lock_inventory::ProjectView::Disk(root)).await +} + +/// [`requirements_include_names`] over any project view (the disk, a +/// snapshot of it, or an in-memory project). +pub(crate) async fn requirements_include_names_in( + view: crate::vendor::lock_inventory::ProjectView<'_>, +) -> std::io::Result> { let mut names: Vec = Vec::new(); - walk_requirements_tree(root, |rel, _path, read| { + walk_requirements_tree(view, |rel, read| { if !is_in_root_rel(rel) { return Ok(false); } @@ -939,29 +948,24 @@ pub(crate) fn is_in_root_rel(rel: &str) -> bool { /// result and answers whether to descend into its includes (`Ok(true)`), or /// aborts the walk with its own error. async fn walk_requirements_tree( - root: &Path, - mut visit: impl FnMut(&str, &Path, std::io::Result) -> Result, + view: crate::vendor::lock_inventory::ProjectView<'_>, + mut visit: impl FnMut(&str, std::io::Result) -> Result, ) -> Result<(), E> { let mut visited: HashSet = HashSet::new(); - let mut stack: Vec<(String, PathBuf)> = vec![( - "requirements.txt".to_string(), - root.join("requirements.txt"), - )]; - while let Some((rel, path)) = stack.pop() { + let mut stack: Vec = vec!["requirements.txt".to_string()]; + while let Some(rel) = stack.pop() { if !visited.insert(rel.clone()) { continue; } - let read = read_regular_to_string(&path).await; + let read = view.read_text(&rel).await; // Parse the includes BEFORE handing the content over (the visitor // takes it by value); nothing is pushed unless it asks to descend. let includes: Vec = match &read { Ok(content) => requirements_includes(&rel, content), Err(_) => Vec::new(), }; - if visit(&rel, &path, read)? { - for normalized in includes { - stack.push((normalized.clone(), root.join(&normalized))); - } + if visit(&rel, read)? { + stack.extend(includes); } } Ok(()) diff --git a/crates/socket-patch-core/src/vex/discover/cargo.rs b/crates/socket-patch-core/src/vex/discover/cargo.rs index 86aab22de..dfd22f64d 100644 --- a/crates/socket-patch-core/src/vex/discover/cargo.rs +++ b/crates/socket-patch-core/src/vex/discover/cargo.rs @@ -121,7 +121,7 @@ use super::{ use crate::formats::cargo::{CargoLock, CopyClaim, LockedPackage}; use crate::utils::digest::is_hex64_lower; use crate::vendor::cargo_config::{ - effective_config_rel, patch_entries, registry_definitions, CargoPatchEntry, CONFIG_LEGACY, + effective_config_rel_in, patch_entries, registry_definitions, CargoPatchEntry, CONFIG_LEGACY, CONFIG_TOML, SOCKET_REGISTRY_PREFIX, }; use crate::vendor::cargo_manifest::{crates_io_url_alias_tables, is_crates_io_source}; @@ -312,7 +312,7 @@ fn parse_toml(file: &str, text: &str, out: &mut Discovery) -> Option, out: &mut Discovery, ) -> Option<(&'static str, DocumentMut)> { - if effective_config_rel(ctx.root).await == CONFIG_TOML { + if effective_config_rel_in(ctx.view).await == CONFIG_TOML { return read_toml(ctx, CONFIG_TOML, out) .await .map(|doc| (CONFIG_TOML, doc)); @@ -666,7 +666,7 @@ async fn vendored_from_patches( file: &str, doc: &DocumentMut, lock: &Lock, - shadowed: &dyn Fn(&CargoPatchEntry<'_>) -> Option, + shadowed: &(dyn Fn(&CargoPatchEntry<'_>) -> Option + Sync), out: &mut Discovery, ) { for entry in patch_entries(doc) { diff --git a/crates/socket-patch-core/src/vex/discover/maven.rs b/crates/socket-patch-core/src/vex/discover/maven.rs index c73e1f978..e5fab50e8 100644 --- a/crates/socket-patch-core/src/vex/discover/maven.rs +++ b/crates/socket-patch-core/src/vex/discover/maven.rs @@ -369,7 +369,11 @@ async fn extract_vendored( vendored: &BTreeSet, out: &mut Discovery, ) { - let swept: BTreeMap> = sweep_vendor_dirs(ctx.root) + // The vendored repository is a tree of jars: only a disk has one. + let Some(root) = ctx.disk_root() else { + return; + }; + let swept: BTreeMap> = sweep_vendor_dirs(root) .await .into_iter() .filter(|d| d.eco == "maven") @@ -399,7 +403,7 @@ async fn extract_vendored( "{dir}/{}", local_repo_artifact_path(group, artifact, version, "jar") ); - let jar_ok = tokio::fs::symlink_metadata(ctx.root.join(&rel)) + let jar_ok = tokio::fs::symlink_metadata(root.join(&rel)) .await .is_ok_and(|m| m.is_file()); let (Some(vref), Some(purl), true) = ( diff --git a/crates/socket-patch-core/src/vex/discover/mod.rs b/crates/socket-patch-core/src/vex/discover/mod.rs index ba855e20b..fec2ffb37 100644 --- a/crates/socket-patch-core/src/vex/discover/mod.rs +++ b/crates/socket-patch-core/src/vex/discover/mod.rs @@ -887,9 +887,34 @@ pub async fn discover_patched_refs_in( snapshot: &crate::vendor::lock_inventory::DiskSnapshot<'_>, opts: &DiscoverOptions, ) -> Discovery { - let mut ctx = DiscoverCtx::with_origins(snapshot.root, &opts.patch_server_origins); - ctx.view = crate::vendor::lock_inventory::ProjectView::Snapshot(snapshot); - discover_with_ctx(ctx).await + discover_patched_refs_view( + crate::vendor::lock_inventory::ProjectView::Snapshot(snapshot), + opts, + ) + .await +} + +/// [`discover_patched_refs_with`] over any [`ProjectView`]: the disk, a +/// per-run snapshot of it (possibly overlaid with a rewrite's output, see +/// [`DiskSnapshot::overlay`]), or an in-memory project. Over memory the +/// disk-only probes (installed trees, vendored artifact dirs, the sbt +/// resolution evidence) see nothing, exactly as an in-memory hosted scan +/// sees no installed tree. +/// +/// [`ProjectView`]: crate::vendor::lock_inventory::ProjectView +/// [`DiskSnapshot::overlay`]: crate::vendor::lock_inventory::DiskSnapshot::overlay +pub async fn discover_patched_refs_view( + view: crate::vendor::lock_inventory::ProjectView<'_>, + opts: &DiscoverOptions, +) -> Discovery { + // Boxed as `Send` here, where the extractors' futures are concrete: the + // in-memory hosted engine's future must be `Send`, and proving it + // through every extractor from inside the engine's loops trips rustc's + // higher-ranked auto-trait check. + let run: std::pin::Pin + Send + '_>> = Box::pin( + discover_with_ctx(DiscoverCtx::over(view, &opts.patch_server_origins)), + ); + run.await } async fn discover_with_ctx(ctx: DiscoverCtx<'_>) -> Discovery { @@ -921,13 +946,12 @@ fn is_script_lock(file: &Path) -> bool { crate::utils::python_lock::is_script_lock_name(&file.to_string_lossy()) } -/// What every extractor receives: the project root plus the hosted-origin +/// What every extractor receives: the project view plus the hosted-origin /// allowlist, with the guarded-read and identity helpers bolted on. pub(crate) struct DiscoverCtx<'a> { - pub(crate) root: &'a Path, - /// Where the guarded reads read from: `root` on disk, or a per-run - /// snapshot of it. - view: crate::vendor::lock_inventory::ProjectView<'a>, + /// Where the guarded reads read from: the root on disk, a per-run + /// snapshot of it, or an in-memory project. + pub(crate) view: crate::vendor::lock_inventory::ProjectView<'a>, patch_server_origins: &'a [String], /// What the guarded reads have recognized so far (rule 11) — collected /// here, not in the extractor's `&mut Discovery`, so a read into a @@ -938,14 +962,30 @@ pub(crate) struct DiscoverCtx<'a> { impl<'a> DiscoverCtx<'a> { pub(crate) fn with_origins(root: &'a Path, patch_server_origins: &'a [String]) -> Self { + Self::over( + crate::vendor::lock_inventory::ProjectView::Disk(root), + patch_server_origins, + ) + } + + pub(crate) fn over( + view: crate::vendor::lock_inventory::ProjectView<'a>, + patch_server_origins: &'a [String], + ) -> Self { DiscoverCtx { - root, - view: crate::vendor::lock_inventory::ProjectView::Disk(root), + view, patch_server_origins, recognized: Mutex::new(BTreeSet::new()), } } + /// The project root on disk, for the probes only a real tree answers + /// (installed packages, vendored artifact dirs, build evidence); `None` + /// over an in-memory project, where those probes find nothing. + pub(crate) fn disk_root(&self) -> Option<&'a Path> { + self.view.disk_root() + } + /// Record every Socket identity `text` (the content of root-relative /// `rel`) mentions — see [`socket_identities`]. fn recognize_text(&self, rel: &str, text: &str) { diff --git a/crates/socket-patch-core/src/vex/discover/npm.rs b/crates/socket-patch-core/src/vex/discover/npm.rs index c130efa95..234148c0b 100644 --- a/crates/socket-patch-core/src/vex/discover/npm.rs +++ b/crates/socket-patch-core/src/vex/discover/npm.rs @@ -552,7 +552,7 @@ async fn extract_pnpm(ctx: &DiscoverCtx<'_>, out: &mut Discovery) { if ctx.exists("rush.json").await { // The common lock, then each subspace's, sorted for deterministic // diagnostics (stat / list only — the reads below stay on the ctx). - for rel in rush_lock_rels(ctx.root).await { + for rel in rush_lock_rels(ctx.view).await { extract_pnpm_lock(ctx, &rel, out).await; } } diff --git a/crates/socket-patch-core/src/vex/discover/nuget.rs b/crates/socket-patch-core/src/vex/discover/nuget.rs index d7f3cd95d..1d2d87084 100644 --- a/crates/socket-patch-core/src/vex/discover/nuget.rs +++ b/crates/socket-patch-core/src/vex/discover/nuget.rs @@ -98,7 +98,12 @@ pub(crate) async fn extract(ctx: &DiscoverCtx<'_>, out: &mut Discovery) { // case-insensitive filesystem the "others" are the file read below // itself — skipped, so it is not reported under three names. for name in CONFIG_NAMES.iter().filter(|name| **name != cfg_rel) { - if !same_file(&ctx.root.join(name), &ctx.root.join(cfg_rel)).await { + // In memory every spelling is its own entry. + let same = match ctx.disk_root() { + Some(root) => same_file(&root.join(name), &root.join(cfg_rel)).await, + None => false, + }; + if !same { ctx.recognize_ignored(name).await; } } @@ -544,7 +549,11 @@ async fn emit_vendored( async fn feed_leaves(ctx: &DiscoverCtx<'_>, uuid: &str, id_lower: &str) -> Vec<(String, String)> { // `uuid` passed the canonical grammar (socket_patch_name_uuid), so the // join cannot escape `.socket/vendor/nuget/`. - let dir = ctx.root.join(VENDOR_DIR).join("nuget").join(uuid); + // The vendored feed is a dir of packages: only a disk has one. + let Some(root) = ctx.disk_root() else { + return Vec::new(); + }; + let dir = root.join(VENDOR_DIR).join("nuget").join(uuid); let Ok(mut entries) = tokio::fs::read_dir(&dir).await else { return Vec::new(); }; diff --git a/crates/socket-patch-core/src/vex/discover/pypi_locks.rs b/crates/socket-patch-core/src/vex/discover/pypi_locks.rs index 59f1cc7dd..c87fce1d6 100644 --- a/crates/socket-patch-core/src/vex/discover/pypi_locks.rs +++ b/crates/socket-patch-core/src/vex/discover/pypi_locks.rs @@ -124,7 +124,7 @@ pub(crate) async fn extract(ctx: &DiscoverCtx<'_>, out: &mut Discovery) { // `uv.lock` is tried explicitly so a non-regular file squatting the name // (which the directory listing skips) still diagnoses as unreadable. let mut python_locks = vec![UV_LOCK.to_string()]; - for path in crate::utils::python_lock::python_lock_paths(ctx.root).unwrap_or_default() { + for path in ctx.view.python_lock_paths() { if !python_locks.contains(&path) { python_locks.push(path); } diff --git a/crates/socket-patch-core/src/vex/discover/pypi_other.rs b/crates/socket-patch-core/src/vex/discover/pypi_other.rs index 4372d887e..a7d5e1e05 100644 --- a/crates/socket-patch-core/src/vex/discover/pypi_other.rs +++ b/crates/socket-patch-core/src/vex/discover/pypi_other.rs @@ -219,20 +219,21 @@ async fn extract_pipfile_lock(ctx: &DiscoverCtx<'_>, out: &mut Discovery) { // ── requirements files ─────────────────────────────────────────────────── async fn extract_requirements(ctx: &DiscoverCtx<'_>, out: &mut Discovery) { - let files = match crate::vendor::requirements_include_names(ctx.root).await { - Ok(files) => files, - Err(e) => { - // A reached include exists but cannot be read: the tree is - // unknowable past it. Still read the root file (its own read - // diagnoses if IT is the unreadable one). - out.diag( - DIAG_LOCKFILE_UNREADABLE, - ROOT_REQUIREMENTS, - format!("cannot read the -r include tree of {ROOT_REQUIREMENTS}: {e}"), - ); - vec![ROOT_REQUIREMENTS.to_string()] - } - }; + let files = + match crate::vendor::pypi_requirements::requirements_include_names_in(ctx.view).await { + Ok(files) => files, + Err(e) => { + // A reached include exists but cannot be read: the tree is + // unknowable past it. Still read the root file (its own read + // diagnoses if IT is the unreadable one). + out.diag( + DIAG_LOCKFILE_UNREADABLE, + ROOT_REQUIREMENTS, + format!("cannot read the -r include tree of {ROOT_REQUIREMENTS}: {e}"), + ); + vec![ROOT_REQUIREMENTS.to_string()] + } + }; for file in files.iter().take(MAX_REQUIREMENTS_FILES) { let Some(text) = ctx.read_text(file, out).await else { continue; diff --git a/crates/socket-patch-core/src/vex/discover/sbt.rs b/crates/socket-patch-core/src/vex/discover/sbt.rs index fb46ac399..f77b5f81e 100644 --- a/crates/socket-patch-core/src/vex/discover/sbt.rs +++ b/crates/socket-patch-core/src/vex/discover/sbt.rs @@ -51,7 +51,11 @@ pub(crate) async fn extract(ctx: &DiscoverCtx<'_>, out: &mut Discovery) { if hosted.is_none() && vendored.is_none() { return; } - let (res, blocker) = evidence(ctx.root).await; + // The resolution evidence is sbt's own output tree: only a disk has one. + let (res, blocker) = match ctx.disk_root() { + Some(root) => evidence(root).await, + None => (None, None), + }; let blocker = blocker.as_deref(); let mut hashes = Hashes::default(); if let Some(text) = hosted { @@ -324,7 +328,10 @@ async fn vendored_check( ) { for (ext, want) in [("pom", &pin.pom_sha256), ("jar", &pin.jar_sha256)] { let rel = format!("{VENDORED_REPO_REL}/{}", pin.repo_path(ext)); - let got = hashes.of(&ctx.root.join(&rel)).await; + let got = match ctx.disk_root() { + Some(root) => hashes.of(&root.join(&rel)).await, + None => None, + }; if got.as_deref() != Some(want.as_str()) { out.diag( DIAG_VENDORED_TREE_MISSING, diff --git a/crates/socket-patch-core/src/vex/discover/vlt.rs b/crates/socket-patch-core/src/vex/discover/vlt.rs index 9211f7d78..ac109f02e 100644 --- a/crates/socket-patch-core/src/vex/discover/vlt.rs +++ b/crates/socket-patch-core/src/vex/discover/vlt.rs @@ -246,11 +246,18 @@ pub(crate) async fn extract(ctx: &DiscoverCtx<'_>, out: &mut Discovery) { /// resolved elsewhere (another lock's wiring of the same version is /// contested, as for npm's `inBundle` copies). async fn contest_bundled_copies(ctx: &DiscoverCtx<'_>, nodes: &[VltLockNode], out: &mut Discovery) { - let copies = store_bundled_copies( - ctx.root, - nodes.iter().map(|n| (n.key.as_str(), n.name.as_str())), - ) - .await; + // The store is an installed tree: only a disk has one. + let Some(root) = ctx.disk_root() else { + return; + }; + // Collected first: a closure-mapped iterator held across the walk's + // awaits would keep the future from being `Send` (the in-memory engine + // awaits discovery). + let pairs: Vec<(&str, &str)> = nodes + .iter() + .map(|n| (n.key.as_str(), n.name.as_str())) + .collect(); + let copies = store_bundled_copies(root, pairs).await; if copies.is_empty() { return; } From b2e5e3924d4df25bfcb027be91f14bbffeb7579c Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Wed, 7 Oct 2026 12:38:22 -0400 Subject: [PATCH 02/20] Decide recorded hosted pins through lockfile discovery The rollout had three answers to "is this patch already pinned": the disk scan's discovery pins, plus a second pass (mark_pinned) that counted any uuid-shaped token in any candidate text, and the in-memory engine's memory_recorded, which counted any mention of an offered uuid in any file. A stale hosted URL in a package.json field, an inactive pdm.lock or a comment therefore read as ALREADY, and the row rode past the --max-new-patches cap. Delete the mention scan (mentioned_uuids) and decide both engines' recorded view with HostedPin::discover, the discovery the management commands read. A pin on a patch server that is not configured is recognized once the run's references name it: both engines re-run discovery with those origins (foreign_dep_origins). The disk scan now makes that check before it decides whether to take the apply lock, so a run that writes a row found pinned this way locks before it reads what it writes. The late lock taken after an unlocked read is gone (audit B58). Lockless NuGet / Cargo pins (an exclusive Socket source without packages.lock.json, a registry pin without Cargo.lock) are contested wiring that nothing can attribute to a version. Their refusal no longer tells the user to re-run the hosted scan, which only writes the same pin again: it names the lockfile to create, and the pin's own grant token no longer shows up as a second contested patch (audit B13; whether such pins should be written at all stays with the E45 decision). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/socket-patch-cli/src/commands/mod.rs | 7 +- .../src/commands/scan/hosted.rs | 35 ++-- .../src/commands/scan/rollout.rs | 28 +--- .../tests/hosted_memory_rollout.rs | 35 ++++ .../src/hosted/memory/mod.rs | 83 +++++----- .../src/patch/redirect/upstream/mod.rs | 150 ++++++++++++++++-- crates/socket-patch-core/src/rollout/stage.rs | 58 +++---- .../src/vendor/lock_inventory/view.rs | 1 + .../src/vex/discover/cargo.rs | 1 + .../socket-patch-core/src/vex/discover/mod.rs | 4 + .../src/vex/discover/nuget.rs | 1 + .../src/vex/discover/testing/golden.rs | 1 + .../tests/hosted_inventory.rs | 46 ++++++ 13 files changed, 317 insertions(+), 133 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/mod.rs b/crates/socket-patch-cli/src/commands/mod.rs index ea45e6915..a1e52739e 100644 --- a/crates/socket-patch-cli/src/commands/mod.rs +++ b/crates/socket-patch-cli/src/commands/mod.rs @@ -106,12 +106,7 @@ pub(crate) async fn discover_wiring_in( fn discover_options(common: &crate::args::GlobalArgs) -> socket_patch_core::vex::DiscoverOptions { socket_patch_core::vex::DiscoverOptions { - patch_server_origins: common - .patch_server_url - .iter() - .filter(|url| !url.trim().is_empty()) - .cloned() - .collect(), + patch_server_origins: rollback::patch_server_origins(common), } } diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 27ab8dc18..96b94c049 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -778,6 +778,28 @@ pub(crate) async fn run_redirect_selected( &vlt_preflight.withheld_everywhere, &mut skipped, ); + // A NEW row the lockfiles already pin on the patch server these + // references name (not a configured one, so the recorded view's + // discovery could not see it) is ALREADY: decided here, before the lock + // decision below, so a run that will write such a row locks before it + // reads anything it writes. + if let Some(gate) = rollout.as_mut() { + if super::rollout::any_new(&gate.rows) { + let configured = crate::commands::rollback::patch_server_origins(common); + let foreign = socket_patch_core::patch::redirect::upstream::foreign_dep_origins( + candidates.iter().map(|c| &c.dep), + &configured, + ); + if !foreign.is_empty() { + let origins: Vec = configured.into_iter().chain(foreign).collect(); + let pins = socket_patch_core::patch::redirect::upstream::HostedPin::discover( + view, &origins, + ) + .await; + super::rollout::mark_pinned(&mut gate.rows, &pins); + } + } + } // The apply lock (see `acquire_hosted_lock`), taken only by a WET run // that holds at least one granted reference — the only runs that can @@ -795,7 +817,7 @@ pub(crate) async fn run_redirect_selected( !new.contains(&(sel_purl(c), c.dep.patch_uuid.clone())) || gate.may_admit(&sel_purl(c)) }) }); - let mut lock: Option = if !common.dry_run && !candidates.is_empty() && may_write { + let lock: Option = if !common.dry_run && !candidates.is_empty() && may_write { match acquire_hosted_lock(common, &mut scan_result) { Ok(guard) => Some(guard), Err(code) => return code, @@ -1030,8 +1052,6 @@ pub(crate) async fn run_redirect_selected( (purl.to_string(), uuid.clone()) }) .collect(); - let texts: Vec<&str> = done.files.values().map(String::as_str).collect(); - super::rollout::mark_pinned(&mut gate.rows, &texts); let unknown = gate.stage.reference_failed.is_some(); gate.stage.plan(&gate.rows, |row| { unknown || eligible.contains(&(row.writer.purl.clone(), row.writer.uuid.clone())) @@ -1052,15 +1072,6 @@ pub(crate) async fn run_redirect_selected( ) .await; } - // A row that turned out to be pinned already (`mark_pinned`: a pin - // discovery did not recognize) still gets written: take the lock - // skipped above. Only NEW rows ran without it, so no takeover did. - if lock.is_none() && !common.dry_run && !candidates.is_empty() { - match acquire_hosted_lock(common, &mut scan_result) { - Ok(guard) => lock = Some(guard), - Err(code) => return code, - } - } } // Held to the end of the function. let _lock = lock; diff --git a/crates/socket-patch-cli/src/commands/scan/rollout.rs b/crates/socket-patch-cli/src/commands/scan/rollout.rs index 82ef99e17..21d60de2c 100644 --- a/crates/socket-patch-cli/src/commands/scan/rollout.rs +++ b/crates/socket-patch-cli/src/commands/scan/rollout.rs @@ -420,27 +420,7 @@ mod tests { } #[test] - fn mentioned_uuids_finds_every_canonical_shape_once() { - let mut out = HashSet::new(); - mentioned_uuids( - "https://h/p/22222222-2222-4222-8222-222222222222/AAAAAAAA-1111-4111-8111-00000000000A/x.tgz \ - not-a-uuid 1234 socket-patch-bbbbbbbb-1111-4111-8111-00000000000b", - &mut out, - ); - let mut got: Vec<&str> = out.iter().map(String::as_str).collect(); - got.sort(); - assert_eq!( - got, - [ - "22222222-2222-4222-8222-222222222222", - "aaaaaaaa-1111-4111-8111-00000000000a", - "bbbbbbbb-1111-4111-8111-00000000000b", - ] - ); - } - - #[test] - fn a_lock_naming_the_selected_uuid_marks_the_row_already() { + fn a_discovered_pin_of_the_selected_uuid_marks_the_row_already() { let results = vec![ offer( "pkg:npm/a@1", @@ -459,7 +439,11 @@ mod tests { let mut rows = classify(&offers, &RecordedIndex::default(), ""); mark_pinned( &mut rows, - &["resolved: https://x/AAAAAAAA-1111-4111-8111-00000000000A/a.tgz"], + &[socket_patch_core::patch::redirect::upstream::HostedPin { + purl: "pkg:npm/a@1".into(), + uuid: "aaaaaaaa-1111-4111-8111-00000000000a".into(), + files: vec!["package-lock.json".into()], + }], ); assert_eq!(rows[0].candidate.recorded, Recorded::Same); assert_eq!(rows[1].candidate.recorded, Recorded::None); diff --git a/crates/socket-patch-cli/tests/hosted_memory_rollout.rs b/crates/socket-patch-cli/tests/hosted_memory_rollout.rs index ccaf92cc4..2742929ec 100644 --- a/crates/socket-patch-cli/tests/hosted_memory_rollout.rs +++ b/crates/socket-patch-cli/tests/hosted_memory_rollout.rs @@ -363,6 +363,41 @@ async fn one_root_disk_and_memory_admit_and_defer_the_same_rows_until_converged( ); } +/// A patch uuid a file merely MENTIONS — here a stale hosted URL left in +/// a `package.json` field no installer reads — is not a pin: the row stays +/// NEW and costs its slot, on disk and in memory alike. Both engines used +/// to count any mention as ALREADY, which let the row ride past the cap. +#[tokio::test] +async fn a_uuid_mentioned_outside_a_pin_is_new_not_already() { + let server = MockServer::start().await; + mount(&server).await; + let mut files = BTreeMap::new(); + let names: Vec<&str> = PACKAGES.iter().map(|(n, _)| *n).collect(); + lock(&mut files, "", &names); + let mut manifest: Value = serde_json::from_slice(&files["package.json"]).unwrap(); + manifest["description"] = json!(format!("was pinned to {}", url("mem-e"))); + files.insert( + "package.json".to_string(), + serde_json::to_vec(&manifest).unwrap(), + ); + + let want = json!({ "new": 1, "deferred": 4, "upgrade": 0, "already": 0 }); + let mem = memory(&server, &files, options(Some(1))).await; + assert!( + mem.projects[0].error.is_none(), + "{:?}", + mem.projects[0].error + ); + assert_eq!(mem.rollout["counts"], want); + assert_eq!(pinned(&apply(&files, &mem), "package-lock.json"), ["mem-e"]); + + let disk = run_disk_with(&server, &files, false, &["--max-new-patches", "1"]); + assert_eq!(disk.envelope["rollout"]["counts"], want, "{}", disk.stderr); + let mut disk_files = files.clone(); + disk_files.extend(disk.changed); + assert_eq!(pinned(&disk_files, "package-lock.json"), ["mem-e"]); +} + #[tokio::test] async fn two_roots_spend_one_budget_in_memory_and_one_per_directory_on_disk() { let server = MockServer::start().await; diff --git a/crates/socket-patch-core/src/hosted/memory/mod.rs b/crates/socket-patch-core/src/hosted/memory/mod.rs index 00520854f..3b871c9d8 100644 --- a/crates/socket-patch-core/src/hosted/memory/mod.rs +++ b/crates/socket-patch-core/src/hosted/memory/mod.rs @@ -59,7 +59,7 @@ pub use select::{candidate_files, safe_repo_path, select_paths}; pub use types::*; use crate::rollout::stage::{ - classify, lookup_incomplete, mentioned_uuids, offers_from_results, Offers, RecordedIndex, Row, + classify, lookup_incomplete, mark_pinned, offers_from_results, Offers, RecordedIndex, Row, Stage, ROLLOUT_DEFERRED, }; @@ -378,47 +378,26 @@ fn unrooted_unsupported_warnings<'a>( } /// One root's recorded view (§5.1) in memory: its `.socket/manifest.json`, -/// the hosted pins its lockfiles name, and its vendor ledger — the disk -/// merge's precedence. A pin is a mention of an offered uuid for the purl -/// in one of the root's own files (a nested root's files are its own); a -/// pin to a patch the API no longer offers reads as NEW, which costs one -/// slot once instead of stalling. -fn memory_recorded( - project: &MemoryProject, - root: &str, - roots: &[String], - offers: &Offers, -) -> RecordedIndex { +/// the hosted pins its lockfiles carry, and its vendor ledger — the disk +/// merge's precedence. The pins are discovery's over the root's files +/// ([`HostedPin::discover`], the one "is this pinned" answer the disk scan +/// uses too), so a uuid a stale or inactive file merely mentions pins +/// nothing. Pins on a patch server other than Socket's are recognized once +/// the run's references name it ([`stage::mark_pinned`]). +/// +/// [`HostedPin::discover`]: crate::patch::redirect::upstream::HostedPin::discover +/// [`stage::mark_pinned`]: crate::rollout::stage::mark_pinned +async fn memory_recorded(project: &MemoryProject) -> RecordedIndex { let manifest = project .text(select::MANIFEST_REL) .and_then(|text| serde_json::from_str(text).ok()); let vendor = stages::vendored_entries(project); - let nested: Vec = roots - .iter() - .filter(|other| other.as_str() != root) - .filter_map(|other| roots::strip_root(root, other).map(|rel| format!("{rel}/"))) - .filter(|rel| rel != "/") - .collect(); - let mut mentioned = std::collections::HashSet::new(); - for (path, entry) in project.entries() { - if path.starts_with(".socket/") || nested.iter().any(|n| path.starts_with(n.as_str())) { - continue; - } - if let MemoryEntry::Text(text) = entry { - mentioned_uuids(text, &mut mentioned); - } - } - let pins: Vec<(String, String)> = offers - .selected - .iter() - .filter_map(|(purl, selected)| { - let offered = offers.unfiltered.get(purl)?; - std::iter::once(selected) - .chain(offered.iter()) - .find(|p| mentioned.contains(&p.uuid.to_ascii_lowercase())) - .map(|p| (purl.clone(), p.uuid.clone())) - }) - .collect(); + let pins: Vec<(String, String)> = + crate::patch::redirect::upstream::HostedPin::discover(ProjectView::Memory(project), &[]) + .await + .into_iter() + .map(|pin| (pin.purl, pin.uuid)) + .collect(); let merged = crate::ledgers::merge_ledger_records_for_updates( manifest.as_ref(), vendor.as_ref(), @@ -728,12 +707,14 @@ async fn engine( stage.incomplete |= states .iter() .any(|s| s.error.as_ref().is_some_and(|e| e.code == "patch_lookup_failed")); - let roots_by_path: Vec = states.iter().map(|s| s.root.clone()).collect(); - for state in states.iter_mut().filter(|s| s.error.is_none()) { + for state in states.iter_mut() { + if state.error.is_some() { + continue; + } let Some(project) = state.project.as_ref() else { continue; }; - let recorded = memory_recorded(project, &state.root, &roots_by_path, &state.offers); + let recorded = memory_recorded(project).await; stage.incomplete |= lookup_incomplete( &recorded, &state.failed_details, @@ -802,6 +783,26 @@ async fn engine( Err(refusal) => state.error = Some(ProjectError::from(refusal)), } } + // A pin on the patch server these references name, when that is not + // Socket's own, is recognized only now (see `mark_pinned`). + for (index, plan) in &planned { + if !crate::rollout::stage::any_new(&states[*index].rows) { + continue; + } + let origins = crate::patch::redirect::upstream::foreign_dep_origins( + plan.candidates.iter().map(|c| &c.dep), + &[], + ); + if origins.is_empty() { + continue; + } + let pins = crate::patch::redirect::upstream::HostedPin::discover( + ProjectView::Memory(&plan.project), + &origins, + ) + .await; + mark_pinned(&mut states[*index].rows, &pins); + } let wheels: BTreeSet<(String, String)> = planned .iter() .flat_map(|(_, p)| p.wheels.iter().cloned()) diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/mod.rs b/crates/socket-patch-core/src/patch/redirect/upstream/mod.rs index 434d1a03b..9f5f0afe4 100644 --- a/crates/socket-patch-core/src/patch/redirect/upstream/mod.rs +++ b/crates/socket-patch-core/src/patch/redirect/upstream/mod.rs @@ -87,6 +87,24 @@ impl HostedPin { Self::from_refs(&discovery.refs) } + /// THE "is this patch pinned" answer: the attributable hosted pins + /// lockfile discovery reads through `view` (the disk, a snapshot of it + /// overlaid with a pending rewrite, or an in-memory project), with + /// `origins` counting as patch servers besides Socket's own. The + /// forward rewrite's confirmation, the rollout's recorded view (disk + /// and in memory) and the management commands' [`HostedInventory`] all + /// read pins through discovery, so none of them can call a uuid pinned + /// that another one calls unpinned or contested. + pub async fn discover( + view: crate::vendor::lock_inventory::ProjectView<'_>, + origins: &[String], + ) -> Vec { + let opts = crate::vex::DiscoverOptions { + patch_server_origins: origins.to_vec(), + }; + Self::all(&crate::vex::discover::discover_patched_refs_view(view, &opts).await) + } + /// `(name, version)` of the purl, percent-decoded. pub(crate) fn name_version(&self) -> Option<(String, String)> { let (_, name, version) = crate::utils::purl::purl_parts(&self.purl)?; @@ -108,6 +126,11 @@ pub struct ContestedWiring { pub files: Vec, /// Discovery's own findings for those files (`code: detail`), if any. pub details: Vec, + /// The ecosystems (`cargo`, `nuget`) of a LOCKLESS pin among this + /// wiring: a registry pin no lockfile records a version for. Nothing + /// the lockfiles hold can attribute it, so its remedy is to create the + /// lockfile, not to reconcile one. + pub lockless: BTreeSet, } /// The project's hosted state as raw wiring: the attributable pins (what @@ -154,16 +177,22 @@ impl HostedInventory { // is excused in a file that also names the pin's own patch uuid. let pin_tokens: BTreeMap> = { let mut tokens: BTreeMap> = BTreeMap::new(); - for r in &discovery.refs { - if r.mode != WiringMode::Hosted { - continue; - } - let Some(url) = r.url.as_deref() else { - continue; - }; + // A lockless pin's index url carries its token the same way. + let urls = discovery + .refs + .iter() + .filter(|r| r.mode == WiringMode::Hosted) + .filter_map(|r| Some((r.url.as_deref()?, r.uuid.as_str()))) + .chain( + discovery + .unlocked_pins + .iter() + .filter_map(|p| Some((p.index_url.as_deref()?, p.uuid.as_str()))), + ); + for (url, uuid) in urls { for token in url_uuid_segments(url) { - if token != r.uuid { - tokens.entry(token).or_default().insert(r.uuid.as_str()); + if token != uuid { + tokens.entry(token).or_default().insert(uuid); } } } @@ -183,6 +212,7 @@ impl HostedInventory { }) }; let mut contested: BTreeMap> = BTreeMap::new(); + let mut lockless: BTreeMap> = BTreeMap::new(); for r in &discovery.recognized { let file = norm(&r.file); if r.mode == WiringMode::Hosted @@ -202,6 +232,10 @@ impl HostedInventory { .entry(pin.uuid.clone()) .or_default() .insert(norm(&pin.file)); + lockless + .entry(pin.uuid.clone()) + .or_default() + .insert(pin.ecosystem.clone()); } } let contested = contested @@ -216,6 +250,7 @@ impl HostedInventory { .into_iter() .collect(); ContestedWiring { + lockless: lockless.remove(&uuid).unwrap_or_default(), uuid, files: files.into_iter().collect(), details, @@ -249,26 +284,111 @@ impl HostedInventory { .collect::>() .into_iter() .collect(); + let lockless: BTreeSet<&str> = self + .contested + .iter() + .flat_map(|c| c.lockless.iter().map(String::as_str)) + .collect(); + let all_lockless = self.contested.iter().all(|c| !c.lockless.is_empty()); // Files, not uuids: a hosted URL also carries its grant token as a // uuid-shaped segment, so the recognized set over-names patches. let mut msg = format!( "{} wire(s) Socket-hosted patches that cannot be attributed to one package \ - version (the lockfiles disagree, or the reference is malformed), so socket-patch \ - cannot manage them safely", - files.join(", ") + version ({}), so socket-patch cannot manage them safely", + files.join(", "), + if all_lockless { + "no lockfile records which version the pin resolves" + } else if lockless.is_empty() { + "the lockfiles disagree, or the reference is malformed" + } else { + "the lockfiles disagree, the reference is malformed, or no lockfile records \ + the pinned version" + } ); if !details.is_empty() { msg.push_str(&format!(" ({})", details.join("; "))); } - msg.push_str(&format!( - "; reconcile the lockfiles (re-run `socket-patch scan --mode hosted`) or restore \ - them from version control (`git checkout -- {}`)", + // A lockless pin is attributed once its lockfile exists: re-running + // the hosted scan alone would only write the same pin again. + let mut remedies: Vec = Vec::new(); + if !all_lockless { + remedies.push( + "reconcile the lockfiles (re-run `socket-patch scan --mode hosted`)".to_string(), + ); + } + for eco in &lockless { + match *eco { + "nuget" => remedies.push( + "create packages.lock.json (`dotnet restore --use-lock-file`)".to_string(), + ), + "cargo" => { + remedies.push("create Cargo.lock (`cargo generate-lockfile`)".to_string()) + } + _ => {} + } + } + remedies.push(format!( + "restore them from version control (`git checkout -- {}`)", files.join(" ") )); + msg.push_str(&format!("; {}", remedies.join(" or "))); Some(msg) } } +/// The origins (`scheme://host[:port]`) of the patch servers `deps` are +/// served from — their artifact and registry index urls, a `sparse+` / +/// `registry+` kind prefix dropped — sorted and deduplicated. Passed as +/// discovery's extra origins, they let it recognize the pins a run writes +/// for them when the server is not the configured one. +pub fn dep_origins<'a>(deps: impl IntoIterator) -> Vec { + let mut out = BTreeSet::new(); + for dep in deps { + let index = dep.registry_override.as_ref().map(|r| r.index_url.as_str()); + for url in std::iter::once(dep.artifact_url.as_str()).chain(index) { + out.extend(dep_origins_of_url(url)); + } + } + out.into_iter().collect() +} + +/// `scheme://host[:port]` of `url` (a `kind+` scheme prefix dropped, the +/// default port omitted), or `None` when it does not parse. +fn dep_origins_of_url(url: &str) -> Option { + let url = url.trim(); + let (scheme, _) = url.split_once("://")?; + let text = match scheme.rsplit_once('+') { + Some((kind, _)) => &url[kind.len() + 1..], + None => url, + }; + let parsed = reqwest::Url::parse(text).ok()?; + let host = parsed.host_str()?; + Some(match parsed.port() { + Some(port) => format!("{}://{host}:{port}", parsed.scheme()), + None => format!("{}://{host}", parsed.scheme()), + }) +} + +/// [`dep_origins`] less the ones discovery already counts: Socket's own +/// patch server and `configured` (the operator's `--patch-server-url`). An +/// empty answer means a discovery over `configured` already recognizes +/// every pin these deps could have. +pub fn foreign_dep_origins<'a>( + deps: impl IntoIterator, + configured: &[String], +) -> Vec { + dep_origins(deps) + .into_iter() + .filter(|origin| { + let known = std::iter::once(format!("https://{}", super::SOCKET_PATCH_SERVER_HOST)) + .chain(configured.iter().cloned()); + !known + .into_iter() + .any(|k| dep_origins_of_url(&k).as_deref() == Some(origin.as_str())) + }) + .collect() +} + /// The canonical-uuid path segments of a hosted URL discovery already /// accepted (`\/` unescaped; a wholly percent-encoded URL decoded first; /// each segment percent-decoded after splitting). diff --git a/crates/socket-patch-core/src/rollout/stage.rs b/crates/socket-patch-core/src/rollout/stage.rs index 9ebd7e7ac..bf7c56c03 100644 --- a/crates/socket-patch-core/src/rollout/stage.rs +++ b/crates/socket-patch-core/src/rollout/stage.rs @@ -238,49 +238,33 @@ pub fn lookup_incomplete( .any(|purl| !recorded.records_package(purl)) } -/// Every canonical-shaped uuid (`8-4-4-4-12` hex) `text` mentions, -/// lowercased, in one linear pass. -pub fn mentioned_uuids(text: &str, out: &mut HashSet) { - let bytes = text.as_bytes(); - if bytes.len() < 36 { - return; - } - let mut i = 0; - while i + 36 <= bytes.len() { - let window = &bytes[i..i + 36]; - let shaped = window.iter().enumerate().all(|(k, b)| match k { - 8 | 13 | 18 | 23 => *b == b'-', - _ => b.is_ascii_hexdigit(), - }); - if shaped { - out.insert(String::from_utf8_lossy(window).to_ascii_lowercase()); - i += 36; - } else { - i += 1; - } - } -} - -/// Mark NEW rows whose selected uuid the project's lockfile texts already -/// mention as ALREADY. A hosted pin on a patch server discovery does not -/// recognize (an origin missing from `--patch-server-url`) would otherwise -/// read as NEW on every run and hold its slot forever; patch uuids are -/// unique, so a mention is a pin. -pub fn mark_pinned(rows: &mut [Row], texts: &[&str]) { - if !rows.iter().any(|r| r.candidate.recorded.is_new()) { - return; - } - let mut mentioned = HashSet::new(); - for text in texts { - mentioned_uuids(text, &mut mentioned); - } +/// Mark NEW rows whose selected uuid discovery already finds pinned +/// ([`HostedPin::discover`]) as ALREADY. The recorded view is discovery +/// over the configured patch servers; a pin on the server THIS run's +/// references name (an origin missing from `--patch-server-url`) is only +/// recognized once those references are known, so the caller re-runs +/// discovery with their origins ([`dep_origins`]) and hands the pins here. +/// Without it such a pin would read as NEW on every run and hold its slot +/// forever. Only discovery's attributable pins count: a uuid a stale or +/// inactive file merely mentions (an unused `pdm.lock`, a `package.json` +/// `resolutions` leftover, a comment) pins nothing and stays NEW. +/// +/// [`HostedPin::discover`]: crate::patch::redirect::upstream::HostedPin::discover +/// [`dep_origins`]: crate::patch::redirect::upstream::dep_origins +pub fn mark_pinned(rows: &mut [Row], pins: &[crate::patch::redirect::upstream::HostedPin]) { + let pinned: HashSet = pins.iter().map(|p| p.uuid.to_ascii_lowercase()).collect(); for row in rows.iter_mut().filter(|r| r.candidate.recorded.is_new()) { - if mentioned.contains(&row.candidate.uuid.to_ascii_lowercase()) { + if pinned.contains(&row.candidate.uuid.to_ascii_lowercase()) { row.candidate.recorded = Recorded::Same; } } } +/// Whether any row is NEW (only then can [`mark_pinned`] change anything). +pub fn any_new(rows: &[Row]) -> bool { + rows.iter().any(|r| r.candidate.recorded.is_new()) +} + /// One directory's budget and the outcome of its plan. #[derive(Debug, Clone)] pub struct Stage { diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/view.rs b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs index 2a356a70d..49acc569e 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/view.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs @@ -84,6 +84,7 @@ impl MemoryProject { matches!(self.entries.get(rel), Some(MemoryEntry::Symlink)) } + #[cfg(test)] pub(crate) fn entries(&self) -> impl Iterator { self.entries.iter().map(|(k, v)| (k.as_str(), v)) } diff --git a/crates/socket-patch-core/src/vex/discover/cargo.rs b/crates/socket-patch-core/src/vex/discover/cargo.rs index dfd22f64d..7dc60b3a1 100644 --- a/crates/socket-patch-core/src/vex/discover/cargo.rs +++ b/crates/socket-patch-core/src/vex/discover/cargo.rs @@ -599,6 +599,7 @@ fn unresolved_manifest_pins( uuid: uuid.clone(), file: CARGO_TOML.into(), version_reqs, + index_url: definitions.get(reg.as_str()).cloned(), }); } format!("there is no {CARGO_LOCK} to fix its version") diff --git a/crates/socket-patch-core/src/vex/discover/mod.rs b/crates/socket-patch-core/src/vex/discover/mod.rs index fec2ffb37..2a47965cc 100644 --- a/crates/socket-patch-core/src/vex/discover/mod.rs +++ b/crates/socket-patch-core/src/vex/discover/mod.rs @@ -369,6 +369,10 @@ pub struct UnlockedPin { /// `version = "…"`); a ledger version must satisfy every one. Empty = /// none to check (nuget: the csproj `Version` is only a minimum). pub version_reqs: Vec, + /// The Socket index url the pin routes to (the nuget source value, the + /// cargo registry's `index`): its grant-token segment is part of this + /// pin, not other hosted wiring. + pub index_url: Option, } impl UnlockedPin { diff --git a/crates/socket-patch-core/src/vex/discover/nuget.rs b/crates/socket-patch-core/src/vex/discover/nuget.rs index 1d2d87084..3f0ea666f 100644 --- a/crates/socket-patch-core/src/vex/discover/nuget.rs +++ b/crates/socket-patch-core/src/vex/discover/nuget.rs @@ -439,6 +439,7 @@ fn emit_hosted( uuid: src.uuid.clone(), file: cfg_rel.into(), version_reqs: Vec::new(), + index_url: Some(src.value.clone()), }); } None diff --git a/crates/socket-patch-core/src/vex/discover/testing/golden.rs b/crates/socket-patch-core/src/vex/discover/testing/golden.rs index 36b15632e..e3305d728 100644 --- a/crates/socket-patch-core/src/vex/discover/testing/golden.rs +++ b/crates/socket-patch-core/src/vex/discover/testing/golden.rs @@ -181,6 +181,7 @@ fn render(out: &Discovery, root: &Path) -> Value { uuid, file, version_reqs, + index_url: _, } = p; json!({ "ecosystem": ecosystem, diff --git a/crates/socket-patch-core/tests/hosted_inventory.rs b/crates/socket-patch-core/tests/hosted_inventory.rs index 1bb3772d2..82a9567b8 100644 --- a/crates/socket-patch-core/tests/hosted_inventory.rs +++ b/crates/socket-patch-core/tests/hosted_inventory.rs @@ -206,3 +206,49 @@ async fn half_reverted_uv_pair_is_still_contested() { assert!(inv.pins.is_empty(), "{inv:?}"); assert!(inv.contested_refusal().is_some(), "{inv:?}"); } + +/// A lockless NuGet pin (an exclusive Socket source mapping, no +/// `packages.lock.json`: `rewrite_nuget`'s output for most projects) is +/// contested wiring nothing can attribute to a version. Its refusal names +/// the remedy that makes it attributable — create the lockfile — never a +/// hosted re-scan, which would only write the same pin again (B13). +#[tokio::test] +async fn lockless_nuget_pin_refusal_names_the_lockfile_remedy() { + let tmp = tempfile::tempdir().unwrap(); + let key = format!("socket-patch-{PATCH}"); + let index = format!("https://patch.socket.dev/patch-registry/nuget/{GRANT}/{PATCH}/index.json"); + std::fs::write( + tmp.path().join("nuget.config"), + format!( + "\n\n \n \ + \n \n \ + \n \n \ + \n \n \ + \n\n" + ), + ) + .unwrap(); + let inv = inventory(tmp.path()).await; + assert!(inv.pins.is_empty(), "{inv:?}"); + let lockless: Vec<_> = inv + .contested + .iter() + .filter(|c| !c.lockless.is_empty()) + .collect(); + assert_eq!(lockless.len(), 1, "{inv:?}"); + assert!(lockless[0].lockless.contains("nuget"), "{inv:?}"); + let refusal = inv.contested_refusal().expect("a refusal"); + assert!(refusal.contains("no lockfile records"), "{refusal}"); + assert!( + refusal.contains("dotnet restore --use-lock-file"), + "{refusal}" + ); + assert!( + refusal.contains("git checkout -- nuget.config"), + "{refusal}" + ); + assert!( + !refusal.contains("re-run `socket-patch scan --mode hosted`"), + "a hosted re-scan cannot attribute a lockless pin: {refusal}" + ); +} From 6d6ac148175321d24c87f16dcd4453b7f94beb38 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Wed, 7 Oct 2026 12:38:23 -0400 Subject: [PATCH 03/20] Never write hosted pins lockfile discovery would contest A hosted run confirmed a redirect when the rewriters' own report or a substring probe said its URL landed, while vex, list, rollback, remove and vendor read the same files through lockfile discovery. When the two disagreed the run reported success and wrote wiring every later command refused as contested, with a remedy (re-run the scan) that changed nothing: a Pipfile.lock rewired beside a requirements -r include that still pins the registry release (#567), a Maven pin inside a (#260). engine::rewrite now runs discovery over the project as the rewrite would leave it (a DiskSnapshot overlaid with the pending writes, or the in-memory project plus them) and reads it as the management commands do (HostedInventory). A confirmed candidate whose pin would be contested wiring is dropped and the rest rewritten without it; it is reported in redirect.skipped[] as redirect_unattributable, nothing is written for it, and the exit code is unchanged. Both the disk scan and the in-memory engine go through this one gate. Kept as they were, and documented: a pin in a file discovery does not read (a pre-2.6 bundler Gemfile, locked by the next bundle install), a deliberate partial redirect the run already reports (a bundled or bun-patched copy left on the registry, a dep withheld from the vlt rewrite while a sibling lock takes it; which unreachable copies should block a redirect is the audit's copy-source policy, B16), and lockless NuGet / Cargo pins, which are written with a new redirect_pin_lockless warning naming the lockfile to create (E45 decides whether they should be written at all). The engine's unit fixtures now use real uuids in their hosted urls, so discovery recognizes the pins they land. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/socket-patch-cli/CLI_CONTRACT.md | 4 + .../src/commands/scan/hosted.rs | 3 + .../tests/in_process_get_hosted_ecosystems.rs | 80 ++++++ .../tests/in_process_redirect_pipenv.rs | 34 +++ crates/socket-patch-core/src/hosted/engine.rs | 253 ++++++++++++++++-- .../src/hosted/memory/mod.rs | 1 + .../src/hosted/memory/stages.rs | 1 + .../src/vendor/lock_inventory/view.rs | 25 ++ .../src/vendor/vlt_bundled.rs | 13 +- 9 files changed, 394 insertions(+), 20 deletions(-) diff --git a/crates/socket-patch-cli/CLI_CONTRACT.md b/crates/socket-patch-cli/CLI_CONTRACT.md index b02fa44fa..ba869bb2b 100644 --- a/crates/socket-patch-cli/CLI_CONTRACT.md +++ b/crates/socket-patch-cli/CLI_CONTRACT.md @@ -163,6 +163,8 @@ For a **9.0 root lock**, the CLI ensures `pnpm-workspace.yaml` carries `trustLoc `scan --mode hosted` swaps the in-place apply for the registry-redirect pipeline: discover → resolve hosted-patch references (grant token + integrity + per-dep registry override) → rewrite ONLY the patched dependencies' lockfile / registry-config entries to point at the hosted packages. A dep counts as **redirected** only when its hosted-artifact URL (or per-dep registry index URL) actually landed in a project file — a granted reference whose rewriter found nothing to edit is neither counted nor attested. **No ledger (v5.0)**: hosted mode writes ONLY the lockfile / registry-config edits — `.socket/vendor/redirect-state.json` is never written (on success or failure), and a pre-v5 one on disk is ignored (never read for planning, never quarantined, left byte-identical). The lockfiles are the only record of a hosted patch: `list`, `vex`, `rollback`, `remove`, `vendor` and `repair` all discover the hosted pins from them (a hosted URL counts only on `https://patch.socket.dev` or the `--patch-server-url` / `SOCKET_PATCH_SERVER_URL` origin), and commit-ready output is just the lockfile / config changes. Cargo and golang are confirmed only by their rewriter's own report (`confirmed_cargo_uuids` / `confirmed_golang_uuids`): a golang dep counts only when its go.mod `replace M V => patch.socket.dev/gopatch/ ` and both go.sum lines are in place, never because the patch-server origin or leftover go.sum lines appear somewhere. Gradle is confirmed the same way (`confirmed_gradle_uuids`): only when the final files hold the owned script, the index row, the live apply line in every build's settings file and the suffixed version in every lock entry of the GA (see [Gradle builds](#gradle-builds-v50)). A golang module that go.mod does not require and go.sum does not list at the patched version is outside the build graph and is refused with `redirect_golang_not_in_module_graph` (nothing written). Only the exact module `patch.socket.dev/gopatch/` is socket-owned; any other module path is refused with `redirect_golang_untrusted_module_path`. A vendored golang module is taken over like cargo and the npm family: its vendor wiring, committed copy and ledger entry are reverted first (`redirect_takeover_reverted_vendored`). A vendored PyPI package (requirements.txt, Poetry, Pipenv, uv, Hatch, PDM, pylock) is taken over the same way: its vendored wiring is restored to the recorded registry entry, its ledger entry and wheel are removed, and only then is it redirected. The Python rewriters treat any non-registry source as user-authored, so without the revert they refused socket-patch's own vendored source and left the project vendored. A takeover revert that leaves vendored wiring in place is refused with `redirect_vendored_revert_failed`. That covers a drift-skipped record (`vendor_lock_entry_drifted`) and a reverted file that still references the artifact (`vendor_revert_residual_reference`). The ledger entry and artifact are kept, and the package stays vendored and skipped. `--dry-run` predicts the same refusal from the same signals instead of previewing `redirect_would_revert_vendored`. The hosted requirements.txt rewriter only rewrites an existing pin in the root `requirements.txt`, so a vendored requirements.txt package whose wiring is a pin in a `-r` include or a `(transitive)` line vendored mode appended is refused BEFORE its revert, wet and `--dry-run` alike, with `redirect_requirements_takeover_unreachable` (`redirect.warnings[]`, and `redirect.skipped[].reason`). Its wiring, ledger entry and wheel are kept, so it stays vendored and patched (exit 0). A taken-over package whose wiring was reverted but that was then not pinned to hosted now installs the unpatched registry release in both modes. Causes include a refused lock, unavailable hosted wheel metadata, or a vendored ledger update that failed after the revert (refused with `redirect_vendored_revert_failed`). It is reported as `redirect_takeover_unpatched` with `status: "partial_failure"` and exit 1, never as success. That warning also prints under `--silent`. Human output prints no `Migrated …` progress line for the package and no "keep the hosted patches" next steps. Re-runs over already-rewritten output plan from the current lock text and are idempotent (exit 0, lock unchanged). **Lock (v5.0)**: the hosted engine acquires `<.socket>/apply.lock` around its first wet write (the takeover pre-reverts) — not on `--dry-run`, and not when the run would write nothing (zero redirects, all skipped) — so previews and no-op runs never create `.socket/`; contention is `lock_held` and a lock-file I/O fault (a read-only project root, a file squatting on `.socket/`) is `lock_io` — both exit 1, refused BEFORE any project file is written, and rendered like every other lock holder: human `Error (): ` on stderr (+ the `--lock-timeout` hint for a live holder); JSON keeps the hosted shape — top-level `status: "error"`, `errorCode: "lock_held" | "lock_io"`, a string `error`, and `redirect: {mode: "hosted"}` retained (NOT the vendored `error: {code, message}` object). **Takeover symlink pre-check (v5.0)**: a vendored→hosted takeover whose recorded wiring file is a symlink is refused up front with `redirect_symlinked_file_unsupported` — wet and `--dry-run` alike, before any revert — so "nothing was written" holds. **Human mode (v5.0)**: hosted `scan` prints the results table and update detection like the other modes, then rewrites without a prompt (scan never prompts); `--dry-run` previews through the engine, and a detail fetch that leaves nothing to redirect enters the engine as a no-op (`Redirected 0 packages; rewrote 0 files.`, no lock, no `.socket/`). The detail fetch prints the same progress counter and per-package `Warning: could not fetch details for …` lines as the agent arm. An EMPTY hosted discovery prints `No patches available for installed packages.` and exits 0 without entering the engine; a discovery whose every offer is paid-tier for an org without paid access prints the table's paid nudge, then `No downloadable patches (paid subscription required).`, and exits 0 without entering the engine (parity with the agent/vendored arms). JSON output gains a `redirect` sub-object: `{ mode: "hosted", redirected, rewrittenFiles, skipped, warnings, dryRun }` (`mode` is additive so consumers can dispatch without inferring it). Rewriter warnings carry stable `redirect_*` codes (e.g. `redirect_npm_no_lockfile`, `redirect_gradle_manual_snippet`, `redirect_golang_unsupported`); new codes are additive (MINOR). v5.0 additive codes: `redirect_composer_no_lockfile` / `redirect_gem_no_gemfile` (composer / gem: neither manifest nor lock present — once per run, after the intake gates), `redirect_gem_bundle_gemfile_unsupported` (gem: `BUNDLE_GEMFILE` — `BUNDLE_GEMFILE:` in the bundler app config, which outranks the environment variable as in `Bundler::Settings`, else the environment variable — names a manifest other than the project's `Gemfile` / `gems.rb`, so no gem is redirected or attested; a value naming one of those two selects that pair even when the other spelling is present), `redirect_gem_mirror_overrides_source` (gem: Bundler's all-source, exact patch-source or patch-hostname mirror can route the per-dep `source` block to an unpatched upstream gem. Intake reads the app config (`BUNDLE_APP_CONFIG`, where a set-but-empty value selects `/config`, honoring `BUNDLE_IGNORE_CONFIG`) and all `BUNDLE_MIRROR__...` variables visible to the scan; app config overrides the environment per encoded key, then `mirror.all` takes precedence over exact source, which takes precedence over hostname. URI matching follows Bundler's whole-URI case folding, default-port/trailing-slash normalization and single slash key alias, not URL prefixes. An exact-source fallback-timeout key without a mirror URL shadows the hostname mirror and fetches that source directly; a configured URL is conservatively refused even if a timeout could bypass an unreachable mirror at install time. Like `redirect_gem_bundle_gemfile_unsupported`, the gate leaves the Gemfile pair byte-identical and confirms no gem redirect. On an embedded `scan --vex`, rediscovered older hosted gem pins may attest only from verified installed bytes: a missing tree is not excused by the lockfile, and `--vex-no-verify` omits those hosted gems with `mirror_overrides_source` rather than trusting their intercepted source. Agent/vendored evidence, unrelated ecosystems and standalone VEX behavior are unchanged. Details identify the setting form and its app/environment origin without printing mirror values or source URLs, which may contain credentials. Remove the applicable all/source/hostname setting (including any slash alias) from that origin and reuse its existing mirror URL under `mirror.https://rubygems.org` to clear the refusal; an environment setting must be unset in the scan/install environment. User-global Bundler config and mirrors set only in a later install environment are not inspected; keep those mirrors scoped to the upstream source too), `redirect_maven_no_pom` (no `pom.xml` and no Gradle build), `redirect_nuget_lock_unparseable` (a present-but-corrupt `packages.lock.json` — warned once, nothing mutated; an absent lock still proceeds), `redirect_cargo_lock_pkg_ambiguous` (several same-name+version `[[package]]` blocks and none carries the index `source` — transactional skip). Also v5.0: a registry override of the wrong kind (or none at all) warns the arm's missing-override code for nuget/gem/golang. Refusals stay fail-closed with a diagnosis that names the actual cause: a yarn-berry lock entry resolving through a non-`npm:` protocol keeps `redirect_yarn_berry_unsupported_protocol` with the entry's ACTUAL protocol in the detail — except socket-patch's OWN vendored wiring (a `file:` range into `.socket/vendor/`), which gets the distinct `redirect_yarn_berry_vendored_entry` code whose detail names the retirement path (`remove ` per package, or `vendor --revert` which unwinds every vendored package, then re-run `scan --mode hosted`). Both leave the entry byte-identical; neither changes exit code or status. **yarn berry line endings (v5.0)**: yarn writes a NEW `yarn.lock` with the OS line ending (`os.EOL` — CRLF on Windows) and keeps an existing lock's majority ending on every later write, and a `core.autocrlf` checkout turns an LF lock CRLF on any OS — so a uniformly CRLF lock is rewritten in its own ending: every untouched byte (a leading BOM included) round-trips (and `rollback`'s upstream restore keeps the lock's own ending). A lock that MIXES CRLF and LF (or holds a bare CR) has no single ending to keep — yarn's own `--immutable` check rejects it too (YN0028) — so it is refused untouched with `redirect_yarn_berry_mixed_line_endings` (the detail names `yarn install`, which normalizes it). The root `package.json`, which the rewrite re-renders to add `resolutions`, gets the same gate: a mixed one is refused untouched with the same code — the decision vendored mode takes with `vendor_yarn_berry_mixed_line_endings`, from the same shared berry gate set. This replaces v4's `redirect_yarn_berry_crlf_unsupported`, which refused every CRLF lock and is no longer emitted. A vendored→hosted takeover runs these berry gates (mixed line endings, unsupported `cacheKey`, a non-zero `.yarnrc.yml` `compressionLevel`) BEFORE reverting a vendored berry purl — wet and `--dry-run` alike — so a refused purl keeps its vendored wiring, ledger entry and artifact byte-identical and is skipped with the gate's code (never announced as `redirect_takeover_reverted_vendored` and then left unpatched in both modes). +**Attribution gate (v5.0).** A hosted run never leaves wiring that lockfile discovery calls contested: before anything is written, the same discovery `vex`, `list`, `rollback`, `remove` and `vendor` read runs over the project as the rewrite would leave it. A candidate whose pin discovery reads but cannot attribute to one package version (a requirements `-r` include resolving the same version from the registry beside a rewired `Pipfile.lock`, #567; a Maven pin in a ``, #260) is left out of the rewrite and reported in `redirect.skipped[]` as `redirect_unattributable` (nothing written for it; exit code unchanged). Unchanged: a pin in a file discovery does not read (a pre-2.6 bundler `Gemfile`, locked by the next `bundle install`) keeps the rewriter's verdict, and so does a deliberate partial redirect the run already reports (a bundled or `bun patch`-ed copy left on the registry, a dep withheld from the vlt rewrite while a sibling lock takes it). A lockless NuGet / Cargo pin (an exclusive Socket source mapping without `packages.lock.json`, a Cargo registry pin without `Cargo.lock`) is still written, with a `redirect_pin_lockless` warning: no lockfile records its version, so `vex` cannot attest it and `rollback` / `remove` / `vendor` refuse it until the lockfile exists (whether such pins should be written at all is an open decision). The rollout's recorded view uses the same discovery: a uuid a file merely mentions (a stale `package.json` field, an inactive `pdm.lock`, a comment) is not a pin and does not count as already patched. + The rewriter reads a fixed set of candidate files from the project root: the npm-family locks (`package-lock.json`, `npm-shrinkwrap.json`, `pnpm-lock.yaml`, `shrinkwrap.yaml`, `yarn.lock`, plus `.yarnrc.yml` for the berry cache-config gate, `bun.lock` / `bun.lockb`, and `vlt-lock.json` with `vlt.json` and `node_modules/.vlt-lock.json` read only), `requirements.txt` / `uv.lock` / `Pipfile.lock` (pipfile-spec 6; see the Pipenv section below) / `poetry.lock` (every Poetry lock generation from 1.0 on — the 0.12 `[metadata.hashes]` layout is refused because that installer ignores URL sources; a Poetry < 1.4 writer additionally gets `redirect_poetry_stale_install_risk`, see `docs/testing/poetry-compatibility.md`) / `pdm.lock` (PDM lock formats `2` and `4.3`–`4.5.1`; the identity-losing `3.1` / `4.0`–`4.2` formats and unknown future formats are refused with `redirect_pdm_refused`, and a lock-format-`2` writer additionally gets `redirect_pdm_legacy_sync_required`, see `docs/testing/pdm-compatibility.md`; when `uv.lock` or `poetry.lock` sits beside it they drive and `pdm.lock` is left alone), `Cargo.toml` / `Cargo.lock` / `.cargo/config.toml` (plus the legacy extensionless `.cargo/config` — cargo reads that spelling in preference when both exist, so the managed `[registries.…]` block is written into whichever one is present; **cargo also reads every workspace-member manifest** — the `[workspace] members` globs minus `exclude` — and every in-root path-dependency manifest, recursively, reached without crossing a symbolic link and never under `.socket/`, and pins the crate in each one that declares it, so those `/Cargo.toml` files can appear in `rewrittenFiles`. A crate is redirected only when every declaration pins and every other `Cargo.lock` package depending on it is a planned member: one a registry or git crate — or a path package outside the root or behind a link — also depends on is refused `redirect_cargo_transitive_dependents` (a pin reaches only the declarations it sits on), a crate no manifest declares keeps `redirect_cargo_toml_dep_not_found` with a transitive-only detail naming `--mode vendored`, a crate every declaration of which requires another version (no requirement accepts the patched version) is refused `redirect_cargo_toml_dep_unrewritable`, and so is a requirement that also matches another locked version of the crate — each a transactional skip, never recorded or attested. With NO `Cargo.lock` there is no resolved graph to ask, so the dependents question is answered from the manifests instead: a crate declared beside any other dependency — anything but a path dependency on a manifest this run also pins, or a `workspace = true` inheritor of a table it scans — or beside a workspace member this run did not read (a `members` glob, or a member outside the project or behind a symbolic link, which member discovery drops) is refused `redirect_cargo_lockless_dependents`, whose detail names the remedies (commit a lockfile, or `--mode vendored`); a project whose only dependency is the patched crate has nothing that could pull it in and still redirects. All-CRLF manifests, locks and configs are rewritten with CRLF kept (mixed endings keep refusing where the grammar does not match), and `remove` / rollback match the recorded fragments across a later CRLF↔LF checkout conversion), `composer.lock`, `nuget.config` / `packages.lock.json`, `Gemfile` / `Gemfile.lock`, `pom.xml` (+ `.mvn/maven.config` / `.mvn/checksums/checksums.sha256` for maven Trusted Checksums merge, and, for a Gradle build, every settings, build, `buildSrc`, included-build, applied and plugin-source script, version catalog and lock file the script graph reaches, plus `gradle/verification-metadata.xml`, `gradle/wrapper/gradle-wrapper.properties` and the owned `.socket/gradle/` files), and the sbt build files (`socket-patch.sbt`, `socket-patch-vendor.sbt`, `build.sbt`, `project/build.properties`, `.sbtopts`, `.jvmopts`; `build.sbt.lock` and the Mill / scala-cli build files `build.mill`, `build.mill.yaml`, `build.sc`, `.mill-version`, `project.scala` for their presence only) — read, never edited; `socket-patch.sbt` is the only sbt file hosted mode writes (see **Hosted sbt** below). **npm-family flavor coverage**: package-lock / npm-shrinkwrap, pnpm (root OR any nested `*/pnpm-lock.yaml`), yarn classic (a yarn 2+ install migrates a v1 `yarn.lock` and drops its pins, so a run whose v1 lock carries a hosted pin warns `redirect_yarn_classic_berry_migration_risk` — the hosted twin of the vendored `yarn_classic_berry_migration_risk` — unless the root `package.json`, read as advisory input, declares `"packageManager": "yarn@1…"`), **yarn berry** (the pin yarn writes for a root `resolutions` entry: the root `package.json` — edited only beside a berry `yarn.lock` — gains one `"@npm:": ""` selector per locked range (`redirect_yarn_berry_resolution` edits), and only that `yarn.lock` entry is re-keyed `"@"` with the same `resolution:` + `yarnBerry10c0` checksum (`redirect_yarn_berry_entry`), moved to yarn's key order; never an `npm:` locator, whose fetcher sends npm registry auth to the patch host, nor a tarball locator under an `npm:` key, which hardened mode rejects (YN0078). An older release's `npm:::__archiveUrl=` pin is still recognized and is re-pinned on the next run; rollback rebuilds the key from the selectors and drops them. Refused, nothing written: a user-authored `resolutions` entry for the package `redirect_yarn_berry_resolutions_conflict`, no root manifest `redirect_yarn_berry_manifest_missing`, a builtin `patch:` entry wrapping the same descriptor `redirect_yarn_berry_shared_descriptor`, an artifact URL yarn cannot fetch as a tarball `redirect_yarn_berry_artifact_url_unsupported`; cacheKey `10c0` and `.yarnrc.yml compressionLevel 0` gated by `redirect_yarn_berry_cache_unsupported`), and **bun** (text `bun.lock` lockfileVersion 0, 1 or 2 — 0 is the `--save-text-lockfile` opt-in lock of Bun 1.1.39–1.1.45, 1 the 1.2–1.3 default, 2 the 1.4+ default; all three emit one `packages` grammar, so the registry 4-tuple → URL 3-tuple rewrite is version-independent and the lock's own version line is kept. Any other or missing version, or a `packages` section outside bun's single-line grammar, is refused `redirect_bun_lock_unsupported` — the detail is the shared version gate's text (a newer version: update socket-patch, re-locking would reproduce it; no integer: re-lock with Bun ≥ 1.2), identical to the vendored refusal. A version-0 lock holding `workspace:` packages is refused `redirect_bun_workspace_unsupported` (its 2-tuple workspace grammar cannot keep the hosted tuple through a frozen install); the remedy is to delete `bun.lock` and re-run `bun install` with Bun ≥ 1.2, which writes lockfileVersion 1 (accepted). A plain in-place `bun install` bumps the version only when a workspace depends on another workspace (e.g. root → member — the shape the matrix measured); otherwise Bun 1.2.0 keeps version 0 and Bun 1.2.23+ fail to resolve, so the in-place bump is not the documented remedy. Bun lock version, grammar and workspace compatibility are checked before a vendored takeover, including during dry-run: these refusals preserve the existing lock, artifact and vendor ledger. Version-1 and version-2 workspace locks are rewritten, nested versions included. A granted dep with no rewritable entry warns `redirect_bun_entry_not_found`, a grant without a sha512 `redirect_bun_missing_sha512`; a CRLF lock keeps `\r\n` on the rewritten line, and a hosted URL left by an earlier grant of the same `name@version` is re-pinned in place. **Digest-less re-saves (Bun 1.1.39–1.3.9)**: every text-lock Bun below 1.3.10 re-saves a URL tuple WITHOUT its `sha512` whenever the lock is re-saved for another reason (`bun add`, `bun install` after a package.json or workspace change), leaving the 2-tuple `["name@", {meta}]` — the spec Bun installs from is intact. The CLI treats that spelling as its own wiring: a repeat hosted run counts the dep as redirected (no `redirect_bun_entry_not_found`) and HEALS the line back to the 3-tuple with the current `sha512`, recording the heal as a further `redirect_bun_lock_package` edit whose `original` is the 2-tuple (a stale URL is re-pinned from either spelling); `rollback`, scoped `rollback ` / `remove ` and the vendored takeover accept the digest-less spelling of a recorded `new` line (same key, spec and meta, only the trailing `"sha512-…"` missing) and restore the recorded original over it, so the chain always unwinds to the pristine registry line. Anything else — another uuid/token, another version, a re-laid meta object — is still drift. **Native `bun.lockb`**: when no text `bun.lock` exists, binary format versions 1, 2 and 3 are read and rewritten directly. Socket Patch does not invoke Bun or convert the project to a text lockfile. Exact matching package records are rewritten to hosted tarballs with the granted integrity, preserving dependency resolution IDs, workspace/dependency topology and unrelated package metadata; binary pointers and the package metadata hash are updated. Per-package `redirect_bun_lockb_package` snapshots support scoped rollback, repeat runs, superseding grants and hosted ↔ vendored takeover. A regular binary lock is discoverable even with no Bun runtime or `node_modules`; a dry run previews the same binary edits without writing them. A malformed, unreadable, unsupported or unverified binary structure is `redirect_bun_lockb_invalid` (exit 0, `redirected: 0`), and it refuses the npm rewrite before any takeover or sibling npm-family lock mutation. A symlinked binary write target is `redirect_symlinked_file_unsupported` (exit 1, including dry-run). `bun.lock` wins when both spellings exist. Binary-only projects do not receive `redirect_npm_no_lockfile`. Measured boundaries and the real-Bun matrix: `docs/testing/bun-compatibility.md`), and **vlt** (`vlt-lock.json` without `lockfileVersion`, `0` or `1`; see the vlt hosted-mode contract below). **Rush monorepos**: when `rush.json` is present the rewriter also reads `common/config/rush/pnpm-lock.yaml` and each `common/config/subspaces//pnpm-lock.yaml` (sorted for determinism) under their repo-relative keys and repoints them in place; editing them emits `redirect_rush_repo_state_stale` when `common/config/rush/repo-state.json` exists (the `pnpmShrinkwrapHash` desync is refreshed by `rush update`, which the redirect survives). **maven** is fail-closed via version suffixing: a `mavenSuffixedVersion` + `mavenPomSha256` override pins the Socket-only `-socket.` by rewriting the literal `` (`redirect_maven_dep_version`) or adding a `` entry (`redirect_maven_dep_management_added`), plus optional Trusted Checksums (`redirect_maven_trusted_checksums`, conflicts as `redirect_maven_trusted_checksums_conflict`; when `.mvn/wrapper/maven-wrapper.properties` pins a Maven older than 3.9.4, which ignores those files, the additive warning `redirect_maven_trusted_checksums_unenforced`); a `${property}` version is refused (`redirect_maven_dep_unpinned`), a non-matching literal skipped (`redirect_maven_dep_version_mismatch`), and an override without a suffixed version falls back to same-GAV repository injection (`redirect_maven_same_gav_fallback`, NOT fail-closed). **gradle** (v5.0) is automated wiring, no longer a pasted snippet: the owned settings script `.socket/gradle/socket-patch.hosted.settings.gradle` with its index `.socket/gradle/hosted-index.tsv`, one apply line per build's settings file, every lock entry of the GA moved to the suffixed version, and the suffixed component in an existing `gradle/verification-metadata.xml`. A refused dep writes nothing and keeps `redirect_gradle_manual_snippet` as its fallback; same-GAV grants are refused (`redirect_gradle_same_gav_unsupported`). Rules, refusals and codes: [Gradle builds](#gradle-builds-v50). **Hosted sbt (v5.0, additive)**: an sbt build root (`project/build.properties` naming an `sbt.version`, 0.13.18 or later) is wired through ONE generated root file, `socket-patch.sbt` — no user file is edited. It pins every granted Maven patch build-wide (a `ThisBuild` `dependencyOverrides +=` of the Socket-only `-socket.` version plus a `file:` resolver over `.socket/sbt-hosted/maven2/`, moved ahead of the default repositories on sbt 0.13 / 1.x so an unreachable one never blocks it offline), downloads the pinned pom and jar there on the first sbt load (sha256-checked, gitignored by the file itself), and installs a load-time verifier that fails `update` when any project resolves another version or a pinned artifact whose bytes are not pinned. Edits: `redirect_sbt_pin` (added), `redirect_sbt_pin_updated` (an existing row replaced: same GA and base under a new uuid, or the same uuid with new served values; `original` names the previous uuid and version), `redirect_sbt_pin_rechecked` (an existing row re-verified after the build's dependencies changed: its dependency digest is recorded anew, `original`/`new` are `{deps}`). The load-time verifier also fails `update` when a project declares a pinned GA at a version newer than the pin's base (the build-wide override would otherwise force it back down). A new pin is gated on sbt's own resolution records under `target/` (never the machine-wide cache): run-level stops wire nothing, warn once and exit 0 — `redirect_sbt_no_resolution_evidence` (none; run `sbt update` first; always the in-memory engine's answer), `redirect_sbt_resolution_incomplete` (a declared project left no evidence, or the project definitions cannot be read statically), `redirect_sbt_resolution_stale` (a build source is newer than some project's evidence: each project is dated by its own newest record, so a partial `sbt /update` does not vouch for the others). Per-patch refusals (never confirmed): `redirect_sbt_missing_override` (no `maven2` override or no suffixed version), `redirect_sbt_integrity_missing` (jar or pom sha256 missing), `redirect_sbt_unsafe_value` (a value unsafe in a Scala literal, or an index URL not naming the uuid), `redirect_sbt_version_conflict` (some project resolves another version, or a build source declares the GA newer than the patch's base), `redirect_sbt_override_conflict` (two patches for one GA in a run, or another base already pinned), `redirect_sbt_vendored_conflict` (the GA is pinned by `socket-patch-vendor.sbt`, or that file cannot be parsed — then every Maven patch), `redirect_sbt_owned_file_modified` / `redirect_sbt_owned_file_foreign` (`socket-patch.sbt` edited, or not socket-patch's — every Maven patch), `redirect_sbt_owned_file_unreadable` (a whole-run refusal: `socket-patch.sbt` is on disk but cannot be read as UTF-8 text, so writing it would replace it; nothing is written), `redirect_sbt_unsupported_version`, `redirect_sbt_build_root_unknown` (sbt files but no versioned build root — every Maven patch), `redirect_sbt_overrides_assignment` / `redirect_sbt_resolvers_assignment` (a build source reassigns `dependencyOverrides` / `resolvers` with `:=`, `~=` or `--=`), `redirect_sbt_dependency_lock_present` (a `build.sbt.lock`), `redirect_sbt_scala_runtime_unsupported` (`org.scala-lang`), `redirect_sbt_classifier_unsupported`; a GA no library configuration resolves is skipped silently (`redirect_sbt_meta_build_only` when only the meta-build resolves it). Advisories: `redirect_sbt_version_untested` (sbt 2.1+, still wired), `redirect_sbt_override_build_repos` (`sbt.override.build.repos=true`), `redirect_maven_pom_ignored_sbt_build` (a `pom.xml` beside the sbt build, which sbt never reads; the Maven rewriter still edits it for the Maven build). A re-run keeps an existing row and re-checks it. When the build's dependency digest changed since the pin, evidence resolved after the change (fresh, newer than the generated file) re-verifies it and the row's digest is refreshed (`redirect_sbt_pin_rechecked`); the uuid is NOT confirmed on `redirect_sbt_pin_declared_newer` (a build source now declares the GA newer than the pin's base; the row stays, sbt's load-time verifier fails the build, and the remedy is `socket-patch rollback` or declaring the base again), `redirect_sbt_pin_unverifiable` (the digest changed and the evidence predates the change, or the digest cannot be computed: run `sbt update`, then re-run socket-patch), `redirect_sbt_override_shadowed` (the evidence still resolves the base version) or `redirect_sbt_resolved_elsewhere` (the pinned version resolves from outside the pin repository from a file whose sha256 is not the pinned jar's; a copy holding the pinned bytes, such as the Ivy cache a second checkout reads, is fine — at most 64 pinned artifact files of up to 256 MiB are hashed, anything else counts as elsewhere), and also when a build source now reassigns `dependencyOverrides` / `resolvers` or a `build.sbt.lock` appeared (the same `redirect_sbt_overrides_assignment` / `redirect_sbt_resolvers_assignment` / `redirect_sbt_dependency_lock_present` codes; the row stays and sbt's load-time verifier fails the build). For a pure sbt root (no `pom.xml` / Gradle script beside it), maven confirmation is decided only by the sbt rewriter's report; on a mixed root a uuid the sbt rewriter refused is still confirmed by the Maven rewriter's own `pom.xml` pin (the generated sbt files never prove a pin by substring). **Mill and scala-cli** are guidance only: per Maven patch `redirect_mill_manual_snippet` / `redirect_scala_cli_manual_snippet` carry a paste-able snippet (repository + forced suffixed version), nothing is written or confirmed, and a pure Mill / scala-cli root gets no `redirect_maven_no_pom`; there, a Maven patch the server sent without a `maven2` registry override gets `redirect_maven_missing_override` instead of a snippet (with a `pom.xml` beside the Mill / scala-cli files the pom rewriter reports it). `rollback` / `remove` restore `socket-patch.sbt` offline (the rows removed, the file deleted with its last pin; the gitignored downloads are left). Manifest-less VEX reads every strictly parsed pin as a hosted reference but grants it the lockfile basis only when the local evidence shows every recorded version of the GA is the pinned one and every recorded artifact hashes to a pinned sha256 (else `sbt_resolution_unverified`). @@ -1308,6 +1310,8 @@ Every `--json` invocation emits a single JSON object that follows the **unified | `vendor_uuid_mismatch` | `skipped` | repair: the manifest's patch uuid moved past the vendored artifact — a re-vendor (`vendor` / `scan --vendor`) is pending; repair does not cross patch generations. | | `content_mismatch_overwritten` | `skipped` (warning) | apply (default policy): a file matched NEITHER beforeHash nor afterHash and was overwritten with the full verified patched content. `--strict` turns this case into a `failed` event instead. | | `vendor_lock_checksums_unsupported` / `vendor_stale_lock_checksum` | `failed` | vendor (gem): an ambiguous/platform CHECKSUMS entry, or a v1-wired lock whose stale token blocks the hot path (run `vendor --revert` + re-vendor). | +| `redirect_unattributable` | `redirect.skipped[].reason` | scan/get `--mode hosted`: the rewriters would pin the patch, but lockfile discovery over the result reads that pin as contested (another lock or requirements file resolves the same version elsewhere, or the pin is not one the package manager consumes), so `vex`, `rollback`, `remove` and `vendor` would refuse it. The candidate is left out of the rewrite, so nothing is written for it; the detail carries discovery's findings. Exit code unchanged. | +| `redirect_pin_lockless` | `redirect.warnings[]` (warning) | scan/get `--mode hosted` (nuget, cargo): the pin was written without a lockfile that records its version, so `vex` cannot attest it and `rollback` / `remove` / `vendor` refuse it as unattributable. The detail names the lockfile to create (`dotnet restore --use-lock-file`, `cargo generate-lockfile`) before re-running the hosted scan. | | `redirect_pypi_stale_install` | `redirect.warnings[]` (warning) | Hosted Python redirect: readable installed files differ from patched hashes. Read-only, repeated on re-scan, and excludes the package from same-run VEX. See the "Python stale-install guard" section. | | `redirect_gem_stale_install` | `redirect.warnings[]` (warning) | scan `--mode hosted` (gem): a stale UNPATCHED materialization (installed gem, or committed archive in bundler's cache dir — `vendor/cache` unless `cache_path` moves it) that `bundle install` will reuse instead of fetching the redirected patch; the detail carries the verified remedy. Full rules and flavors: the "Gem stale-install guard" section. | | `redirect_pipenv_refused` | `redirect.warnings[]` (warning) | scan `--mode hosted` (pipenv): the Pipfile.lock pins another version or a non-registry / foreign source for the package — refused atomically across categories, and the patch is vetoed from the sibling Python rewriters (see the "Pipenv hosted redirect" section). | diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 96b94c049..070f3f7aa 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -1073,6 +1073,9 @@ pub(crate) async fn run_redirect_selected( .await; } } + // Candidates the rewrite left out because discovery would not attribute + // their pin (`engine::rewrite`). + skipped.extend(done.unattributed.iter().cloned()); // Held to the end of the function. let _lock = lock; // Dry-run mode-takeover previews were withheld from the rewriters (their diff --git a/crates/socket-patch-cli/tests/in_process_get_hosted_ecosystems.rs b/crates/socket-patch-cli/tests/in_process_get_hosted_ecosystems.rs index db2aab79f..1eafaf897 100644 --- a/crates/socket-patch-cli/tests/in_process_get_hosted_ecosystems.rs +++ b/crates/socket-patch-cli/tests/in_process_get_hosted_ecosystems.rs @@ -450,6 +450,86 @@ async fn maven_pom_hosted_pins_suffixed_version_fail_closed() { .expect("manifest-less vex leg"); } +/// #260: a pom whose only `commons-lang3` dependency sits in a `` +/// (read by Maven only when that profile is active). Lockfile discovery — +/// what `vex`, `list` and `rollback` read — never takes a profile-scoped +/// pin as wiring, so the run must not claim the redirect; it used to be +/// confirmed by substring once any Socket text landed in the pom. Nothing +/// may be written for it. +#[tokio::test] +#[serial] +async fn maven_profile_scoped_dependency_is_not_redirected() { + const UUID: &str = "b3b3b3b3-b3b3-4b3b-8b3b-b3b3b3b3b3b3"; + const PURL: &str = "pkg:maven/org.apache.commons/commons-lang3@3.12.0"; + const SUFFIXED: &str = "3.12.0-socket.b3b3b3b3"; + let index_url = format!("http://patch.test/patch-registry/maven/{TOKEN}/{UUID}/maven2"); + let url = format!( + "http://patch.test/patch/maven/org.apache.commons/commons-lang3/3.12.0/{TOKEN}/{UUID}/commons-lang3-{SUFFIXED}.jar" + ); + let server = MockServer::start().await; + mock_view(&server, UUID, PURL).await; + mock_reference( + &server, + UUID, + PURL, + &url, + serde_json::json!({ "sha256": "c".repeat(64) }), + serde_json::json!({ + "kind": "maven2", + "indexUrl": index_url, + "identifiers": { + "name": "org.apache.commons/commons-lang3", + "version": "3.12.0", + "mavenGroupId": "org.apache.commons", + "mavenArtifactId": "commons-lang3", + "mavenSuffixedVersion": SUFFIXED, + "mavenPomSha256": "d".repeat(64), + } + }), + ) + .await; + + let tmp = tempfile::tempdir().unwrap(); + let pom = r#" + + 4.0.0 + dev.socket.test + consumer + 1.0.0 + + + extra + + + org.apache.commons + commons-lang3 + 3.12.0 + + + + + +"#; + std::fs::write(tmp.path().join("pom.xml"), pom).unwrap(); + + socket_patch_cli::commands::get::run(get_hosted_args(UUID, tmp.path(), server.uri())).await; + assert_eq!( + reference_bodies(&server).await.len(), + 1, + "the grant flow ran (anti-vacuity)" + ); + assert_eq!( + std::fs::read_to_string(tmp.path().join("pom.xml")).unwrap(), + pom, + "a pin discovery never attributes is not written" + ); + assert!( + !tmp.path().join(".mvn").exists(), + "no trusted-checksums wiring either" + ); + assert_no_manifest_no_blobs(tmp.path()); +} + /// Manifest-less VEX over what `get --mode hosted` committed for a /// maven pom (nothing installed: the fail-closed suffixed pin is the /// evidence). v5 `get` writes no ledger, so: attested from the pom wiring + diff --git a/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs b/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs index ceeb2f111..1c5fd45a5 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs @@ -407,6 +407,40 @@ async fn lock_only_pipenv_project_redirects_attests_rescans_and_rolls_back() { assert_eq!(read(&lock_path), LOCK, "rollback must restore the pristine lock byte for byte"); } +/// #567: the project's requirements tree pins the package through an `-r` +/// include the hosted requirements rewriter does not reach. Rewiring only +/// `Pipfile.lock` would leave a lock pair lockfile discovery reads as +/// contested — `vex`, `rollback` and `vendor` refuse it, and re-running the +/// scan changes nothing — so the run leaves the patch out (skipped as +/// `redirect_unattributable`) and writes nothing. +#[tokio::test] +#[serial] +async fn an_unreached_requirements_include_vetoes_the_pipfile_lock_redirect() { + let _major = MajorGuard::set("2026"); + let server = MockServer::start().await; + mock_api(&server).await; + let tmp = tempfile::tempdir().unwrap(); + write_project(tmp.path()); + std::fs::write(tmp.path().join("requirements.txt"), "-r req/base.txt\n").unwrap(); + std::fs::create_dir_all(tmp.path().join("req")).unwrap(); + std::fs::write(tmp.path().join("req/base.txt"), "urllib3==1.26.18\n").unwrap(); + + run(hosted_args(tmp.path(), server.uri(), None)).await; + assert_eq!( + read(&tmp.path().join("Pipfile.lock")), + LOCK, + "a pin discovery would contest is never written" + ); + assert_eq!(read(&tmp.path().join("req/base.txt")), "urllib3==1.26.18\n"); + let inventory = socket_patch_core::patch::redirect::upstream::HostedInventory::of( + &socket_patch_core::vex::discover_patched_refs(tmp.path()).await, + ); + assert!( + inventory.is_empty(), + "no hosted wiring is left for rollback to refuse: {inventory:?}" + ); +} + #[tokio::test] #[serial] async fn legacy_installer_major_selects_path_references() { diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index e5e259d85..97a86eb21 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -885,6 +885,7 @@ pub struct TakeoverPreview { } /// The host-dependent inputs of [`rewrite`]. +#[derive(Clone)] pub struct RewriteOptions<'a> { pub dry_run: bool, /// Whether a pypi candidate targets `Pipfile.lock` @@ -923,6 +924,10 @@ pub struct Rewritten { /// `(purl, uuid)` of each candidate whose redirect is pinned by the /// project's final files, in candidate order. pub confirmed: Vec<(String, String)>, + /// Candidates left out of the rewrite because lockfile discovery could + /// not attribute the pin they would land (see [`rewrite`]): reported as + /// skipped, written nowhere. + pub unattributed: Vec, /// A `bun.lockb` without a text `bun.lock` drives npm. pub binary_bun: bool, pub rush_warnings: Vec, @@ -1058,6 +1063,209 @@ pub async fn rewrite( withheld_from_vlt: &BTreeSet, takeover_previews: &[TakeoverPreview], options: RewriteOptions<'_>, +) -> Rewritten { + // The run must never leave wiring that lockfile discovery — what `vex`, + // `list`, `rollback`, `remove` and `vendor` read — calls contested. A + // candidate the rewriters confirm but discovery cannot attribute to one + // package version (another lock resolving the same version elsewhere, a + // pin Maven never consumes) would be refused by every later command, so + // it is dropped and the rest rewritten without it. Each pass drops at + // least one candidate, so this ends. + let mut kept: Vec = candidates.to_vec(); + let mut unattributed: Vec = Vec::new(); + loop { + let mut done = rewrite_once( + view, + read.clone(), + &kept, + python_metadata.clone(), + withheld_from_vlt, + takeover_previews, + options.clone(), + ) + .await; + let (vetoed, lockless) = unattributed_pins(view, &done, &kept, withheld_from_vlt).await; + if vetoed.is_empty() { + done.unattributed = unattributed; + done.rewrite.warnings.extend(lockless); + return done; + } + kept.retain(|c| !vetoed.iter().any(|skip| skip.uuid == c.dep.patch_uuid)); + unattributed.extend(vetoed); + } +} + +/// Lockfile discovery over the project as `done` would leave it, read as +/// the management commands read it ([`HostedInventory`]): the skips for +/// the confirmed candidates whose pin would be contested wiring, and a +/// [`REDIRECT_PIN_LOCKLESS`] warning per lockless pin. A lockless NuGet / +/// Cargo pin ([`UnlockedPin`]) is written as before — whether such a pin +/// may be written at all is the open hosted-rollback decision (E45) — but +/// the run says that nothing can manage it until a lockfile exists. +/// +/// A deliberate partial redirect keeps its behavior too: a dep whose +/// bundled or user-patched copy the rewriters knowingly left on the +/// registry (`bundled_skipped_uuids`, or a bundled copy in vlt's store; +/// both are warned and kept out of the in-run VEX), or one withheld from +/// the vlt rewrite while a sibling lock takes it (`withheld_from_vlt`). Which unreachable copies should block a +/// redirect is the copy-source policy (audit B16), not decided here. +/// +/// [`HostedInventory`]: crate::patch::redirect::upstream::HostedInventory +/// [`UnlockedPin`]: crate::vex::discover::UnlockedPin +async fn unattributed_pins( + view: &ProjectView<'_>, + done: &Rewritten, + candidates: &[Candidate], + withheld_from_vlt: &BTreeSet, +) -> (Vec, Vec) { + if done.confirmed.is_empty() { + return (Vec::new(), Vec::new()); + } + let opts = crate::vex::DiscoverOptions { + patch_server_origins: crate::patch::redirect::upstream::dep_origins( + candidates.iter().map(|c| &c.dep), + ), + }; + let mut written: Vec<(&str, &[u8])> = Vec::new(); + for (rel, text) in &done.rewrite.files { + if !crate::patch::redirect::sbt::is_synthetic_key(rel) { + written.push((rel.as_str(), text.as_bytes())); + } + } + for (rel, bytes) in &done.rewrite.binary_files { + written.push((rel.as_str(), bytes.as_slice())); + } + let discovery = match view.disk_root() { + None => { + let ProjectView::Memory(project) = *view else { + unreachable!("only a memory view has no disk root") + }; + let mut after = project.clone(); + for (rel, bytes) in written { + let entry = match std::str::from_utf8(bytes) { + Ok(text) => crate::vendor::lock_inventory::MemoryEntry::Text(text.into()), + Err(_) => crate::vendor::lock_inventory::MemoryEntry::Binary(bytes.into()), + }; + after.insert(rel, entry); + } + crate::vex::discover::discover_patched_refs_view(ProjectView::Memory(&after), &opts) + .await + } + Some(root) => { + let after = crate::vendor::lock_inventory::DiskSnapshot::new(root); + for (rel, bytes) in written { + after.overlay(rel, bytes); + } + crate::vex::discover::discover_patched_refs_view(ProjectView::Snapshot(&after), &opts) + .await + } + }; + // The management commands' own view of the result: an attributable + // pin, or contested wiring they would refuse around. + let inventory = crate::patch::redirect::upstream::HostedInventory::of(&discovery); + let attributed: BTreeSet<&str> = inventory.pins.iter().map(|p| p.uuid.as_str()).collect(); + let lockless: Vec = discovery + .unlocked_pins + .iter() + .filter(|pin| { + !attributed.contains(pin.uuid.as_str()) + && done.confirmed.iter().any(|(_, uuid)| *uuid == pin.uuid) + }) + .map(|pin| { + let create = match pin.ecosystem.as_str() { + "nuget" => "create packages.lock.json (`dotnet restore --use-lock-file`)", + "cargo" => "create Cargo.lock (`cargo generate-lockfile`)", + _ => "create the lockfile", + }; + warning( + REDIRECT_PIN_LOCKLESS, + format!( + "{}: {} is pinned to patch {} without a lockfile that records its version, \ + so `vex` cannot attest it and `rollback`, `remove` and `vendor` refuse it \ + as unattributable; {create} and re-run `socket-patch scan --mode hosted` \ + to make it manageable", + pin.file.display(), + pin.name, + pin.uuid + ), + ) + }) + .collect(); + // Contested wiring (not a lockless pin, see above) is what the run must + // never leave behind. A pin discovery does not see at all (a file it + // does not read, such as a pre-2.6 bundler Gemfile the next `bundle + // install` locks) is no such wiring and keeps the rewriters' verdict. + let contested: BTreeSet<&str> = inventory + .contested + .iter() + .filter(|c| c.lockless.is_empty()) + .map(|c| c.uuid.as_str()) + .collect(); + // vlt's bundled copies live only in its installed store, which the + // rewriters never read (the scan warns about them after the writes). + let vlt_bundled: BTreeSet = match view.disk_root() { + Some(root) if !contested.is_empty() => crate::vendor::vlt_bundled::bundled_copies(root) + .await + .into_keys() + .collect(), + _ => BTreeSet::new(), + }; + let vetoed = done + .confirmed + .iter() + .filter(|(purl, uuid)| { + !attributed.contains(uuid.as_str()) + && contested.contains(uuid.as_str()) + && !done.rewrite.bundled_skipped_uuids.contains(uuid) + && !withheld_from_vlt.contains(uuid) + && !vlt_bundled.contains(&crate::vex::discover::canonical_base_purl(purl)) + }) + .map(|(purl, uuid)| { + let findings: Vec<&str> = discovery + .diagnostics + .iter() + .filter(|d| d.detail.contains(uuid.as_str()) || d.detail.contains(purl.as_str())) + .map(|d| d.detail.as_str()) + .collect::>() + .into_iter() + .collect(); + let why = if findings.is_empty() { + String::new() + } else { + format!(" ({})", findings.join("; ")) + }; + SkippedPatch { + purl: purl.clone(), + uuid: uuid.clone(), + reason: REDIRECT_UNATTRIBUTABLE.to_string(), + detail: Some(format!( + "the rewrite would wire patch {uuid} for {purl}, but lockfile discovery (what \ + `vex`, `list`, `rollback` and `vendor` read) cannot attribute that pin to \ + one package version{why}, so nothing was changed for it; reconcile the \ + project's lockfiles and re-run" + )), + } + }) + .collect(); + (vetoed, lockless) +} + +/// Warning: a confirmed pin no lockfile records a version for (a lockless +/// NuGet / Cargo redirect), which no later command can attribute. +pub const REDIRECT_PIN_LOCKLESS: &str = "redirect_pin_lockless"; + +/// `skipped[].reason` of a candidate whose pin lockfile discovery would not +/// attribute (see [`rewrite`]). +pub const REDIRECT_UNATTRIBUTABLE: &str = "redirect_unattributable"; + +async fn rewrite_once( + view: &ProjectView<'_>, + read: CandidateFiles, + candidates: &[Candidate], + python_metadata: BTreeMap, + withheld_from_vlt: &BTreeSet, + takeover_previews: &[TakeoverPreview], + options: RewriteOptions<'_>, ) -> Rewritten { let CandidateFiles { files, @@ -1244,6 +1452,7 @@ pub async fn rewrite( rewrite, rewritten, confirmed, + unattributed: Vec::new(), binary_bun, rush_warnings, pnpm_warnings, @@ -2215,6 +2424,16 @@ mod tests { assert!(confirmed.is_empty(), "{confirmed:?}"); } + /// The grant token and patch uuid of the engine fixtures' hosted urls: + /// real uuids, so lockfile discovery recognizes the pins the rewrite + /// lands (the engine keeps only the ones it attributes). + const FIXTURE_TOKEN: &str = "11111111-1111-4111-8111-111111111111"; + const FIXTURE_UUID: &str = "77777777-7777-4777-8777-777777777777"; + + fn left_pad_url() -> String { + format!("https://patch.test/{FIXTURE_TOKEN}/{FIXTURE_UUID}/left-pad-1.3.0.tgz") + } + fn left_pad_candidate() -> Candidate { use crate::patch::redirect::Integrity; Candidate { @@ -2224,9 +2443,9 @@ mod tests { name: "left-pad".into(), namespace: None, version: "1.3.0".into(), - token: "tok".into(), - patch_uuid: "uuid".into(), - artifact_url: "https://patch.test/left-pad-1.3.0.tgz".into(), + token: FIXTURE_TOKEN.into(), + patch_uuid: FIXTURE_UUID.into(), + artifact_url: left_pad_url(), registry_override: None, integrity: Integrity { sha512: Some("sha512-PATCHED==".into()), @@ -2294,7 +2513,7 @@ mod tests { done.rewrite .files .get("package-lock.json") - .is_some_and(|lock| lock.contains("https://patch.test/left-pad-1.3.0.tgz")) + .is_some_and(|lock| lock.contains(&left_pad_url())) }; // In memory. let mut p = MemoryProject::new(); @@ -2352,9 +2571,11 @@ mod tests { name: "is-number".into(), namespace: None, version: "7.0.0".into(), - token: "tok".into(), - patch_uuid: "uuid".into(), - artifact_url: "https://patch.test/is-number-7.0.0.tgz".into(), + token: FIXTURE_TOKEN.into(), + patch_uuid: FIXTURE_UUID.into(), + artifact_url: format!( + "https://patch.test/{FIXTURE_TOKEN}/{FIXTURE_UUID}/is-number-7.0.0.tgz" + ), registry_override: None, integrity: Integrity { sha512: Some(format!("sha512-{}==", "A".repeat(86))), @@ -2420,7 +2641,7 @@ mod tests { "{}", skipped.detail ); - assert!(done.rewrite.bundled_skipped_uuids.contains("uuid")); + assert!(done.rewrite.bundled_skipped_uuids.contains(FIXTURE_UUID)); } else { assert!( done.rewrite.binary_files.contains_key("bun.lockb"), @@ -2492,12 +2713,14 @@ mod tests { name: "rails".into(), namespace: None, version: "7.0.0".into(), - token: "tok".into(), - patch_uuid: "uuid".into(), - artifact_url: "https://patch.test/rails-7.0.0.gem".into(), + token: FIXTURE_TOKEN.into(), + patch_uuid: FIXTURE_UUID.into(), + artifact_url: format!( + "https://patch.test/gem/{FIXTURE_TOKEN}/{FIXTURE_UUID}/gems/rails-7.0.0.gem" + ), registry_override: Some(RegistryOverride { kind: "rubygems-compact-index".into(), - index_url: "https://patch.test/gem/tok/uuid/".into(), + index_url: format!("https://patch.test/gem/{FIXTURE_TOKEN}/{FIXTURE_UUID}/"), identifiers: RegistryOverrideIdentifiers { name: "rails".into(), version: "7.0.0".into(), @@ -2897,9 +3120,9 @@ mod tests { #[tokio::test] async fn bundler_mirror_for_the_patch_source_redirects_nothing() { for key in [ - "BUNDLE_MIRROR__HTTPS://PATCH__TEST/GEM/TOK/UUID/", - "BUNDLE_MIRROR__PATCH__TEST", - "BUNDLE_MIRROR__PATCH__TEST/", + format!("BUNDLE_MIRROR__HTTPS://PATCH__TEST/GEM/{FIXTURE_TOKEN}/{FIXTURE_UUID}/"), + "BUNDLE_MIRROR__PATCH__TEST".to_string(), + "BUNDLE_MIRROR__PATCH__TEST/".to_string(), ] { let mut p = MemoryProject::new(); p.insert_text("Gemfile", GEMFILE); diff --git a/crates/socket-patch-core/src/hosted/memory/mod.rs b/crates/socket-patch-core/src/hosted/memory/mod.rs index 3b871c9d8..ec3378b07 100644 --- a/crates/socket-patch-core/src/hosted/memory/mod.rs +++ b/crates/socket-patch-core/src/hosted/memory/mod.rs @@ -1341,6 +1341,7 @@ mod tests { rewrite, rewritten: files.iter().map(|(rel, _)| (*rel).to_string()).collect(), confirmed: vec![("pkg:cargo/serde@1.0.190".into(), "u".into())], + unattributed: Vec::new(), binary_bun: false, rush_warnings: Vec::new(), pnpm_warnings: Vec::new(), diff --git a/crates/socket-patch-core/src/hosted/memory/stages.rs b/crates/socket-patch-core/src/hosted/memory/stages.rs index 97c1dd97a..ed0430df1 100644 --- a/crates/socket-patch-core/src/hosted/memory/stages.rs +++ b/crates/socket-patch-core/src/hosted/memory/stages.rs @@ -339,6 +339,7 @@ pub(crate) async fn rewrite( skipped: skipped_before, }); } + skipped.extend(done.unattributed.iter().cloned()); Ok(Rewritten { project, skipped, diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/view.rs b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs index 49acc569e..9627f98f5 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/view.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs @@ -195,6 +195,9 @@ type ReadCache = std::collections::HashMap, (io::ErrorK pub struct DiskSnapshot<'a> { pub root: &'a Path, reads: std::sync::Mutex, + /// Paths [`Self::overlay`] put in place of the disk content (they exist + /// in this view even when the disk has no such file yet). + overlaid: std::sync::Mutex>, } impl<'a> DiskSnapshot<'a> { @@ -202,9 +205,28 @@ impl<'a> DiskSnapshot<'a> { Self { root, reads: std::sync::Mutex::new(std::collections::HashMap::new()), + overlaid: std::sync::Mutex::new(std::collections::BTreeSet::new()), } } + /// Read `rel` as `content` instead of the disk's: the project as a + /// pending write would leave it. Content reads and existence probes see + /// the overlay; directory listings still list the disk. + pub fn overlay(&self, rel: &str, content: &[u8]) { + self.remember(rel, &Ok(content.to_vec())); + self.overlaid + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(rel.to_string()); + } + + fn is_overlaid(&self, rel: &str) -> bool { + self.overlaid + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .contains(rel) + } + fn lock(&self) -> std::sync::MutexGuard<'_, ReadCache> { self.reads .lock() @@ -311,6 +333,7 @@ impl<'a> ProjectView<'a> { /// `metadata` (follows links) succeeds. pub async fn exists(&self, rel: &str) -> bool { match self { + ProjectView::Snapshot(snap) if snap.is_overlaid(rel) => true, ProjectView::Disk(root) | ProjectView::Snapshot(DiskSnapshot { root, .. }) => { tokio::fs::metadata(root.join(rel)).await.is_ok() } @@ -321,6 +344,7 @@ impl<'a> ProjectView<'a> { /// `symlink_metadata` (does not follow links) succeeds. pub async fn exists_no_follow(&self, rel: &str) -> bool { match self { + ProjectView::Snapshot(snap) if snap.is_overlaid(rel) => true, ProjectView::Disk(root) | ProjectView::Snapshot(DiskSnapshot { root, .. }) => { tokio::fs::symlink_metadata(root.join(rel)).await.is_ok() } @@ -331,6 +355,7 @@ impl<'a> ProjectView<'a> { /// A regular file (following links on disk). pub fn is_file(&self, rel: &str) -> bool { match self { + ProjectView::Snapshot(snap) if snap.is_overlaid(rel) => true, ProjectView::Disk(root) | ProjectView::Snapshot(DiskSnapshot { root, .. }) => { root.join(rel).is_file() } diff --git a/crates/socket-patch-core/src/vendor/vlt_bundled.rs b/crates/socket-patch-core/src/vendor/vlt_bundled.rs index 9f4c3277d..22c8d0311 100644 --- a/crates/socket-patch-core/src/vendor/vlt_bundled.rs +++ b/crates/socket-patch-core/src/vendor/vlt_bundled.rs @@ -30,11 +30,14 @@ pub async fn bundled_copies(root: &Path) -> BTreeMap { let Ok(lock) = vlt_lock_model(&text) else { return BTreeMap::new(); }; - store_bundled_copies( - root, - lock.nodes.iter().map(|n| (n.key.as_str(), n.name.as_str())), - ) - .await + // Collected first: a closure-mapped iterator held across the walk's + // awaits would keep the caller's future from being `Send`. + let pairs: Vec<(&str, &str)> = lock + .nodes + .iter() + .map(|n| (n.key.as_str(), n.name.as_str())) + .collect(); + store_bundled_copies(root, pairs).await } /// The scan warning detail for a bundled copy at `location` that a hosted From c225773ef75e7907dafef11e6cd866fa38a814e4 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Wed, 7 Oct 2026 13:19:29 -0400 Subject: [PATCH 04/20] Keep vendored takeovers out of the attribution veto A wet `scan --mode hosted` reverts a purl's vendored wiring and saves the ledger before the rewrite runs. The attribution gate then dropped such a purl when discovery contested its pin (the #567 shape: an unreached requirements -r include beside Pipfile.lock), so the run left the package on the unpatched registry release and exited 1, while the dry run, whose takeover previews never reach the gate, reported success. A takeover's uuid is now exempt from the veto (RewriteOptions:: takeover_uuids) and keeps the rewriters' verdict, as before the gate. The dry run and the wet run agree again. CLI_CONTRACT no longer claims the gate runs before anything is written. Also: - describe_skip_reason has human text for redirect_unattributable instead of blaming the server. - New tests: the takeover is never stranded (failed before this fix), the --json skip contract (reason, detail, redirected 0, exit 0), the redirect_pin_lockless warning for a lockless NuGet pin, and exit-code assertions on the #567 / #260 tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/socket-patch-cli/CLI_CONTRACT.md | 2 +- .../src/commands/scan/hosted.rs | 27 ++++ .../tests/in_process_get_hosted_ecosystems.rs | 4 +- .../tests/in_process_redirect_pipenv.rs | 3 +- .../tests/mode_migration_pypi.rs | 87 +++++++++++++ crates/socket-patch-core/src/hosted/engine.rs | 118 +++++++++++++++++- .../src/hosted/memory/stages.rs | 1 + 7 files changed, 233 insertions(+), 9 deletions(-) diff --git a/crates/socket-patch-cli/CLI_CONTRACT.md b/crates/socket-patch-cli/CLI_CONTRACT.md index ba869bb2b..dfadfbdd2 100644 --- a/crates/socket-patch-cli/CLI_CONTRACT.md +++ b/crates/socket-patch-cli/CLI_CONTRACT.md @@ -163,7 +163,7 @@ For a **9.0 root lock**, the CLI ensures `pnpm-workspace.yaml` carries `trustLoc `scan --mode hosted` swaps the in-place apply for the registry-redirect pipeline: discover → resolve hosted-patch references (grant token + integrity + per-dep registry override) → rewrite ONLY the patched dependencies' lockfile / registry-config entries to point at the hosted packages. A dep counts as **redirected** only when its hosted-artifact URL (or per-dep registry index URL) actually landed in a project file — a granted reference whose rewriter found nothing to edit is neither counted nor attested. **No ledger (v5.0)**: hosted mode writes ONLY the lockfile / registry-config edits — `.socket/vendor/redirect-state.json` is never written (on success or failure), and a pre-v5 one on disk is ignored (never read for planning, never quarantined, left byte-identical). The lockfiles are the only record of a hosted patch: `list`, `vex`, `rollback`, `remove`, `vendor` and `repair` all discover the hosted pins from them (a hosted URL counts only on `https://patch.socket.dev` or the `--patch-server-url` / `SOCKET_PATCH_SERVER_URL` origin), and commit-ready output is just the lockfile / config changes. Cargo and golang are confirmed only by their rewriter's own report (`confirmed_cargo_uuids` / `confirmed_golang_uuids`): a golang dep counts only when its go.mod `replace M V => patch.socket.dev/gopatch/ ` and both go.sum lines are in place, never because the patch-server origin or leftover go.sum lines appear somewhere. Gradle is confirmed the same way (`confirmed_gradle_uuids`): only when the final files hold the owned script, the index row, the live apply line in every build's settings file and the suffixed version in every lock entry of the GA (see [Gradle builds](#gradle-builds-v50)). A golang module that go.mod does not require and go.sum does not list at the patched version is outside the build graph and is refused with `redirect_golang_not_in_module_graph` (nothing written). Only the exact module `patch.socket.dev/gopatch/` is socket-owned; any other module path is refused with `redirect_golang_untrusted_module_path`. A vendored golang module is taken over like cargo and the npm family: its vendor wiring, committed copy and ledger entry are reverted first (`redirect_takeover_reverted_vendored`). A vendored PyPI package (requirements.txt, Poetry, Pipenv, uv, Hatch, PDM, pylock) is taken over the same way: its vendored wiring is restored to the recorded registry entry, its ledger entry and wheel are removed, and only then is it redirected. The Python rewriters treat any non-registry source as user-authored, so without the revert they refused socket-patch's own vendored source and left the project vendored. A takeover revert that leaves vendored wiring in place is refused with `redirect_vendored_revert_failed`. That covers a drift-skipped record (`vendor_lock_entry_drifted`) and a reverted file that still references the artifact (`vendor_revert_residual_reference`). The ledger entry and artifact are kept, and the package stays vendored and skipped. `--dry-run` predicts the same refusal from the same signals instead of previewing `redirect_would_revert_vendored`. The hosted requirements.txt rewriter only rewrites an existing pin in the root `requirements.txt`, so a vendored requirements.txt package whose wiring is a pin in a `-r` include or a `(transitive)` line vendored mode appended is refused BEFORE its revert, wet and `--dry-run` alike, with `redirect_requirements_takeover_unreachable` (`redirect.warnings[]`, and `redirect.skipped[].reason`). Its wiring, ledger entry and wheel are kept, so it stays vendored and patched (exit 0). A taken-over package whose wiring was reverted but that was then not pinned to hosted now installs the unpatched registry release in both modes. Causes include a refused lock, unavailable hosted wheel metadata, or a vendored ledger update that failed after the revert (refused with `redirect_vendored_revert_failed`). It is reported as `redirect_takeover_unpatched` with `status: "partial_failure"` and exit 1, never as success. That warning also prints under `--silent`. Human output prints no `Migrated …` progress line for the package and no "keep the hosted patches" next steps. Re-runs over already-rewritten output plan from the current lock text and are idempotent (exit 0, lock unchanged). **Lock (v5.0)**: the hosted engine acquires `<.socket>/apply.lock` around its first wet write (the takeover pre-reverts) — not on `--dry-run`, and not when the run would write nothing (zero redirects, all skipped) — so previews and no-op runs never create `.socket/`; contention is `lock_held` and a lock-file I/O fault (a read-only project root, a file squatting on `.socket/`) is `lock_io` — both exit 1, refused BEFORE any project file is written, and rendered like every other lock holder: human `Error (): ` on stderr (+ the `--lock-timeout` hint for a live holder); JSON keeps the hosted shape — top-level `status: "error"`, `errorCode: "lock_held" | "lock_io"`, a string `error`, and `redirect: {mode: "hosted"}` retained (NOT the vendored `error: {code, message}` object). **Takeover symlink pre-check (v5.0)**: a vendored→hosted takeover whose recorded wiring file is a symlink is refused up front with `redirect_symlinked_file_unsupported` — wet and `--dry-run` alike, before any revert — so "nothing was written" holds. **Human mode (v5.0)**: hosted `scan` prints the results table and update detection like the other modes, then rewrites without a prompt (scan never prompts); `--dry-run` previews through the engine, and a detail fetch that leaves nothing to redirect enters the engine as a no-op (`Redirected 0 packages; rewrote 0 files.`, no lock, no `.socket/`). The detail fetch prints the same progress counter and per-package `Warning: could not fetch details for …` lines as the agent arm. An EMPTY hosted discovery prints `No patches available for installed packages.` and exits 0 without entering the engine; a discovery whose every offer is paid-tier for an org without paid access prints the table's paid nudge, then `No downloadable patches (paid subscription required).`, and exits 0 without entering the engine (parity with the agent/vendored arms). JSON output gains a `redirect` sub-object: `{ mode: "hosted", redirected, rewrittenFiles, skipped, warnings, dryRun }` (`mode` is additive so consumers can dispatch without inferring it). Rewriter warnings carry stable `redirect_*` codes (e.g. `redirect_npm_no_lockfile`, `redirect_gradle_manual_snippet`, `redirect_golang_unsupported`); new codes are additive (MINOR). v5.0 additive codes: `redirect_composer_no_lockfile` / `redirect_gem_no_gemfile` (composer / gem: neither manifest nor lock present — once per run, after the intake gates), `redirect_gem_bundle_gemfile_unsupported` (gem: `BUNDLE_GEMFILE` — `BUNDLE_GEMFILE:` in the bundler app config, which outranks the environment variable as in `Bundler::Settings`, else the environment variable — names a manifest other than the project's `Gemfile` / `gems.rb`, so no gem is redirected or attested; a value naming one of those two selects that pair even when the other spelling is present), `redirect_gem_mirror_overrides_source` (gem: Bundler's all-source, exact patch-source or patch-hostname mirror can route the per-dep `source` block to an unpatched upstream gem. Intake reads the app config (`BUNDLE_APP_CONFIG`, where a set-but-empty value selects `/config`, honoring `BUNDLE_IGNORE_CONFIG`) and all `BUNDLE_MIRROR__...` variables visible to the scan; app config overrides the environment per encoded key, then `mirror.all` takes precedence over exact source, which takes precedence over hostname. URI matching follows Bundler's whole-URI case folding, default-port/trailing-slash normalization and single slash key alias, not URL prefixes. An exact-source fallback-timeout key without a mirror URL shadows the hostname mirror and fetches that source directly; a configured URL is conservatively refused even if a timeout could bypass an unreachable mirror at install time. Like `redirect_gem_bundle_gemfile_unsupported`, the gate leaves the Gemfile pair byte-identical and confirms no gem redirect. On an embedded `scan --vex`, rediscovered older hosted gem pins may attest only from verified installed bytes: a missing tree is not excused by the lockfile, and `--vex-no-verify` omits those hosted gems with `mirror_overrides_source` rather than trusting their intercepted source. Agent/vendored evidence, unrelated ecosystems and standalone VEX behavior are unchanged. Details identify the setting form and its app/environment origin without printing mirror values or source URLs, which may contain credentials. Remove the applicable all/source/hostname setting (including any slash alias) from that origin and reuse its existing mirror URL under `mirror.https://rubygems.org` to clear the refusal; an environment setting must be unset in the scan/install environment. User-global Bundler config and mirrors set only in a later install environment are not inspected; keep those mirrors scoped to the upstream source too), `redirect_maven_no_pom` (no `pom.xml` and no Gradle build), `redirect_nuget_lock_unparseable` (a present-but-corrupt `packages.lock.json` — warned once, nothing mutated; an absent lock still proceeds), `redirect_cargo_lock_pkg_ambiguous` (several same-name+version `[[package]]` blocks and none carries the index `source` — transactional skip). Also v5.0: a registry override of the wrong kind (or none at all) warns the arm's missing-override code for nuget/gem/golang. Refusals stay fail-closed with a diagnosis that names the actual cause: a yarn-berry lock entry resolving through a non-`npm:` protocol keeps `redirect_yarn_berry_unsupported_protocol` with the entry's ACTUAL protocol in the detail — except socket-patch's OWN vendored wiring (a `file:` range into `.socket/vendor/`), which gets the distinct `redirect_yarn_berry_vendored_entry` code whose detail names the retirement path (`remove ` per package, or `vendor --revert` which unwinds every vendored package, then re-run `scan --mode hosted`). Both leave the entry byte-identical; neither changes exit code or status. **yarn berry line endings (v5.0)**: yarn writes a NEW `yarn.lock` with the OS line ending (`os.EOL` — CRLF on Windows) and keeps an existing lock's majority ending on every later write, and a `core.autocrlf` checkout turns an LF lock CRLF on any OS — so a uniformly CRLF lock is rewritten in its own ending: every untouched byte (a leading BOM included) round-trips (and `rollback`'s upstream restore keeps the lock's own ending). A lock that MIXES CRLF and LF (or holds a bare CR) has no single ending to keep — yarn's own `--immutable` check rejects it too (YN0028) — so it is refused untouched with `redirect_yarn_berry_mixed_line_endings` (the detail names `yarn install`, which normalizes it). The root `package.json`, which the rewrite re-renders to add `resolutions`, gets the same gate: a mixed one is refused untouched with the same code — the decision vendored mode takes with `vendor_yarn_berry_mixed_line_endings`, from the same shared berry gate set. This replaces v4's `redirect_yarn_berry_crlf_unsupported`, which refused every CRLF lock and is no longer emitted. A vendored→hosted takeover runs these berry gates (mixed line endings, unsupported `cacheKey`, a non-zero `.yarnrc.yml` `compressionLevel`) BEFORE reverting a vendored berry purl — wet and `--dry-run` alike — so a refused purl keeps its vendored wiring, ledger entry and artifact byte-identical and is skipped with the gate's code (never announced as `redirect_takeover_reverted_vendored` and then left unpatched in both modes). -**Attribution gate (v5.0).** A hosted run never leaves wiring that lockfile discovery calls contested: before anything is written, the same discovery `vex`, `list`, `rollback`, `remove` and `vendor` read runs over the project as the rewrite would leave it. A candidate whose pin discovery reads but cannot attribute to one package version (a requirements `-r` include resolving the same version from the registry beside a rewired `Pipfile.lock`, #567; a Maven pin in a ``, #260) is left out of the rewrite and reported in `redirect.skipped[]` as `redirect_unattributable` (nothing written for it; exit code unchanged). Unchanged: a pin in a file discovery does not read (a pre-2.6 bundler `Gemfile`, locked by the next `bundle install`) keeps the rewriter's verdict, and so does a deliberate partial redirect the run already reports (a bundled or `bun patch`-ed copy left on the registry, a dep withheld from the vlt rewrite while a sibling lock takes it). A lockless NuGet / Cargo pin (an exclusive Socket source mapping without `packages.lock.json`, a Cargo registry pin without `Cargo.lock`) is still written, with a `redirect_pin_lockless` warning: no lockfile records its version, so `vex` cannot attest it and `rollback` / `remove` / `vendor` refuse it until the lockfile exists (whether such pins should be written at all is an open decision). The rollout's recorded view uses the same discovery: a uuid a file merely mentions (a stale `package.json` field, an inactive `pdm.lock`, a comment) is not a pin and does not count as already patched. +**Attribution gate (v5.0).** A hosted run never leaves wiring that lockfile discovery calls contested: before the rewrite writes any lockfile, the same discovery `vex`, `list`, `rollback`, `remove` and `vendor` read runs over the project as the rewrite would leave it. A candidate whose pin discovery reads but cannot attribute to one package version (a requirements `-r` include resolving the same version from the registry beside a rewired `Pipfile.lock`, #567; a Maven pin in a ``, #260) is left out of the rewrite and reported in `redirect.skipped[]` as `redirect_unattributable` (nothing written for it; exit code unchanged). Unchanged: a pin in a file discovery does not read (a pre-2.6 bundler `Gemfile`, locked by the next `bundle install`) keeps the rewriter's verdict, and so does a deliberate partial redirect the run already reports (a bundled or `bun patch`-ed copy left on the registry, a dep withheld from the vlt rewrite while a sibling lock takes it), and so does a vendored→hosted takeover: its vendored wiring is reverted (and the vendored ledger saved) before the rewrite runs, so dropping it would leave the package on the unpatched registry release; it is redirected as before the gate, and its `--dry-run` preview reports the same outcome. A lockless NuGet / Cargo pin (an exclusive Socket source mapping without `packages.lock.json`, a Cargo registry pin without `Cargo.lock`) is still written, with a `redirect_pin_lockless` warning: no lockfile records its version, so `vex` cannot attest it and `rollback` / `remove` / `vendor` refuse it until the lockfile exists (whether such pins should be written at all is an open decision). The rollout's recorded view uses the same discovery: a uuid a file merely mentions (a stale `package.json` field, an inactive `pdm.lock`, a comment) is not a pin and does not count as already patched. The rewriter reads a fixed set of candidate files from the project root: the npm-family locks (`package-lock.json`, `npm-shrinkwrap.json`, `pnpm-lock.yaml`, `shrinkwrap.yaml`, `yarn.lock`, plus `.yarnrc.yml` for the berry cache-config gate, `bun.lock` / `bun.lockb`, and `vlt-lock.json` with `vlt.json` and `node_modules/.vlt-lock.json` read only), `requirements.txt` / `uv.lock` / `Pipfile.lock` (pipfile-spec 6; see the Pipenv section below) / `poetry.lock` (every Poetry lock generation from 1.0 on — the 0.12 `[metadata.hashes]` layout is refused because that installer ignores URL sources; a Poetry < 1.4 writer additionally gets `redirect_poetry_stale_install_risk`, see `docs/testing/poetry-compatibility.md`) / `pdm.lock` (PDM lock formats `2` and `4.3`–`4.5.1`; the identity-losing `3.1` / `4.0`–`4.2` formats and unknown future formats are refused with `redirect_pdm_refused`, and a lock-format-`2` writer additionally gets `redirect_pdm_legacy_sync_required`, see `docs/testing/pdm-compatibility.md`; when `uv.lock` or `poetry.lock` sits beside it they drive and `pdm.lock` is left alone), `Cargo.toml` / `Cargo.lock` / `.cargo/config.toml` (plus the legacy extensionless `.cargo/config` — cargo reads that spelling in preference when both exist, so the managed `[registries.…]` block is written into whichever one is present; **cargo also reads every workspace-member manifest** — the `[workspace] members` globs minus `exclude` — and every in-root path-dependency manifest, recursively, reached without crossing a symbolic link and never under `.socket/`, and pins the crate in each one that declares it, so those `/Cargo.toml` files can appear in `rewrittenFiles`. A crate is redirected only when every declaration pins and every other `Cargo.lock` package depending on it is a planned member: one a registry or git crate — or a path package outside the root or behind a link — also depends on is refused `redirect_cargo_transitive_dependents` (a pin reaches only the declarations it sits on), a crate no manifest declares keeps `redirect_cargo_toml_dep_not_found` with a transitive-only detail naming `--mode vendored`, a crate every declaration of which requires another version (no requirement accepts the patched version) is refused `redirect_cargo_toml_dep_unrewritable`, and so is a requirement that also matches another locked version of the crate — each a transactional skip, never recorded or attested. With NO `Cargo.lock` there is no resolved graph to ask, so the dependents question is answered from the manifests instead: a crate declared beside any other dependency — anything but a path dependency on a manifest this run also pins, or a `workspace = true` inheritor of a table it scans — or beside a workspace member this run did not read (a `members` glob, or a member outside the project or behind a symbolic link, which member discovery drops) is refused `redirect_cargo_lockless_dependents`, whose detail names the remedies (commit a lockfile, or `--mode vendored`); a project whose only dependency is the patched crate has nothing that could pull it in and still redirects. All-CRLF manifests, locks and configs are rewritten with CRLF kept (mixed endings keep refusing where the grammar does not match), and `remove` / rollback match the recorded fragments across a later CRLF↔LF checkout conversion), `composer.lock`, `nuget.config` / `packages.lock.json`, `Gemfile` / `Gemfile.lock`, `pom.xml` (+ `.mvn/maven.config` / `.mvn/checksums/checksums.sha256` for maven Trusted Checksums merge, and, for a Gradle build, every settings, build, `buildSrc`, included-build, applied and plugin-source script, version catalog and lock file the script graph reaches, plus `gradle/verification-metadata.xml`, `gradle/wrapper/gradle-wrapper.properties` and the owned `.socket/gradle/` files), and the sbt build files (`socket-patch.sbt`, `socket-patch-vendor.sbt`, `build.sbt`, `project/build.properties`, `.sbtopts`, `.jvmopts`; `build.sbt.lock` and the Mill / scala-cli build files `build.mill`, `build.mill.yaml`, `build.sc`, `.mill-version`, `project.scala` for their presence only) — read, never edited; `socket-patch.sbt` is the only sbt file hosted mode writes (see **Hosted sbt** below). **npm-family flavor coverage**: package-lock / npm-shrinkwrap, pnpm (root OR any nested `*/pnpm-lock.yaml`), yarn classic (a yarn 2+ install migrates a v1 `yarn.lock` and drops its pins, so a run whose v1 lock carries a hosted pin warns `redirect_yarn_classic_berry_migration_risk` — the hosted twin of the vendored `yarn_classic_berry_migration_risk` — unless the root `package.json`, read as advisory input, declares `"packageManager": "yarn@1…"`), **yarn berry** (the pin yarn writes for a root `resolutions` entry: the root `package.json` — edited only beside a berry `yarn.lock` — gains one `"@npm:": ""` selector per locked range (`redirect_yarn_berry_resolution` edits), and only that `yarn.lock` entry is re-keyed `"@"` with the same `resolution:` + `yarnBerry10c0` checksum (`redirect_yarn_berry_entry`), moved to yarn's key order; never an `npm:` locator, whose fetcher sends npm registry auth to the patch host, nor a tarball locator under an `npm:` key, which hardened mode rejects (YN0078). An older release's `npm:::__archiveUrl=` pin is still recognized and is re-pinned on the next run; rollback rebuilds the key from the selectors and drops them. Refused, nothing written: a user-authored `resolutions` entry for the package `redirect_yarn_berry_resolutions_conflict`, no root manifest `redirect_yarn_berry_manifest_missing`, a builtin `patch:` entry wrapping the same descriptor `redirect_yarn_berry_shared_descriptor`, an artifact URL yarn cannot fetch as a tarball `redirect_yarn_berry_artifact_url_unsupported`; cacheKey `10c0` and `.yarnrc.yml compressionLevel 0` gated by `redirect_yarn_berry_cache_unsupported`), and **bun** (text `bun.lock` lockfileVersion 0, 1 or 2 — 0 is the `--save-text-lockfile` opt-in lock of Bun 1.1.39–1.1.45, 1 the 1.2–1.3 default, 2 the 1.4+ default; all three emit one `packages` grammar, so the registry 4-tuple → URL 3-tuple rewrite is version-independent and the lock's own version line is kept. Any other or missing version, or a `packages` section outside bun's single-line grammar, is refused `redirect_bun_lock_unsupported` — the detail is the shared version gate's text (a newer version: update socket-patch, re-locking would reproduce it; no integer: re-lock with Bun ≥ 1.2), identical to the vendored refusal. A version-0 lock holding `workspace:` packages is refused `redirect_bun_workspace_unsupported` (its 2-tuple workspace grammar cannot keep the hosted tuple through a frozen install); the remedy is to delete `bun.lock` and re-run `bun install` with Bun ≥ 1.2, which writes lockfileVersion 1 (accepted). A plain in-place `bun install` bumps the version only when a workspace depends on another workspace (e.g. root → member — the shape the matrix measured); otherwise Bun 1.2.0 keeps version 0 and Bun 1.2.23+ fail to resolve, so the in-place bump is not the documented remedy. Bun lock version, grammar and workspace compatibility are checked before a vendored takeover, including during dry-run: these refusals preserve the existing lock, artifact and vendor ledger. Version-1 and version-2 workspace locks are rewritten, nested versions included. A granted dep with no rewritable entry warns `redirect_bun_entry_not_found`, a grant without a sha512 `redirect_bun_missing_sha512`; a CRLF lock keeps `\r\n` on the rewritten line, and a hosted URL left by an earlier grant of the same `name@version` is re-pinned in place. **Digest-less re-saves (Bun 1.1.39–1.3.9)**: every text-lock Bun below 1.3.10 re-saves a URL tuple WITHOUT its `sha512` whenever the lock is re-saved for another reason (`bun add`, `bun install` after a package.json or workspace change), leaving the 2-tuple `["name@", {meta}]` — the spec Bun installs from is intact. The CLI treats that spelling as its own wiring: a repeat hosted run counts the dep as redirected (no `redirect_bun_entry_not_found`) and HEALS the line back to the 3-tuple with the current `sha512`, recording the heal as a further `redirect_bun_lock_package` edit whose `original` is the 2-tuple (a stale URL is re-pinned from either spelling); `rollback`, scoped `rollback ` / `remove ` and the vendored takeover accept the digest-less spelling of a recorded `new` line (same key, spec and meta, only the trailing `"sha512-…"` missing) and restore the recorded original over it, so the chain always unwinds to the pristine registry line. Anything else — another uuid/token, another version, a re-laid meta object — is still drift. **Native `bun.lockb`**: when no text `bun.lock` exists, binary format versions 1, 2 and 3 are read and rewritten directly. Socket Patch does not invoke Bun or convert the project to a text lockfile. Exact matching package records are rewritten to hosted tarballs with the granted integrity, preserving dependency resolution IDs, workspace/dependency topology and unrelated package metadata; binary pointers and the package metadata hash are updated. Per-package `redirect_bun_lockb_package` snapshots support scoped rollback, repeat runs, superseding grants and hosted ↔ vendored takeover. A regular binary lock is discoverable even with no Bun runtime or `node_modules`; a dry run previews the same binary edits without writing them. A malformed, unreadable, unsupported or unverified binary structure is `redirect_bun_lockb_invalid` (exit 0, `redirected: 0`), and it refuses the npm rewrite before any takeover or sibling npm-family lock mutation. A symlinked binary write target is `redirect_symlinked_file_unsupported` (exit 1, including dry-run). `bun.lock` wins when both spellings exist. Binary-only projects do not receive `redirect_npm_no_lockfile`. Measured boundaries and the real-Bun matrix: `docs/testing/bun-compatibility.md`), and **vlt** (`vlt-lock.json` without `lockfileVersion`, `0` or `1`; see the vlt hosted-mode contract below). **Rush monorepos**: when `rush.json` is present the rewriter also reads `common/config/rush/pnpm-lock.yaml` and each `common/config/subspaces//pnpm-lock.yaml` (sorted for determinism) under their repo-relative keys and repoints them in place; editing them emits `redirect_rush_repo_state_stale` when `common/config/rush/repo-state.json` exists (the `pnpmShrinkwrapHash` desync is refreshed by `rush update`, which the redirect survives). **maven** is fail-closed via version suffixing: a `mavenSuffixedVersion` + `mavenPomSha256` override pins the Socket-only `-socket.` by rewriting the literal `` (`redirect_maven_dep_version`) or adding a `` entry (`redirect_maven_dep_management_added`), plus optional Trusted Checksums (`redirect_maven_trusted_checksums`, conflicts as `redirect_maven_trusted_checksums_conflict`; when `.mvn/wrapper/maven-wrapper.properties` pins a Maven older than 3.9.4, which ignores those files, the additive warning `redirect_maven_trusted_checksums_unenforced`); a `${property}` version is refused (`redirect_maven_dep_unpinned`), a non-matching literal skipped (`redirect_maven_dep_version_mismatch`), and an override without a suffixed version falls back to same-GAV repository injection (`redirect_maven_same_gav_fallback`, NOT fail-closed). **gradle** (v5.0) is automated wiring, no longer a pasted snippet: the owned settings script `.socket/gradle/socket-patch.hosted.settings.gradle` with its index `.socket/gradle/hosted-index.tsv`, one apply line per build's settings file, every lock entry of the GA moved to the suffixed version, and the suffixed component in an existing `gradle/verification-metadata.xml`. A refused dep writes nothing and keeps `redirect_gradle_manual_snippet` as its fallback; same-GAV grants are refused (`redirect_gradle_same_gav_unsupported`). Rules, refusals and codes: [Gradle builds](#gradle-builds-v50). diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 070f3f7aa..b78c6e75d 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -1009,6 +1009,20 @@ pub(crate) async fn run_redirect_selected( socket_patch_core::utils::fs::read_regular_to_string_sync(path).ok() }) }; + // The wet takeovers' uuids: their vendored wiring is already reverted, + // so the rewrite's attribution gate must not drop them (dry-run previews + // left `candidates`, so this holds only wet takeovers). + let takeover_uuids: std::collections::BTreeSet = { + use socket_patch_core::utils::purl::{canonical_purl, strip_purl_qualifiers}; + let key = |purl: &str| canonical_purl(strip_purl_qualifiers(purl)); + let migrated: std::collections::HashSet = + takeover_migrated.iter().map(|p| key(p)).collect(); + candidates + .iter() + .filter(|c| migrated.contains(&key(&c.purl))) + .map(|c| c.dep.patch_uuid.clone()) + .collect() + }; let rewrite_options = || RewriteOptions { dry_run: common.dry_run, targets_pipenv_lock, @@ -1021,6 +1035,7 @@ pub(crate) async fn run_redirect_selected( npm_allow_remote_config: !common.no_npm_allow_remote_config, npm_outer: &npm_outer, blocking: true, + takeover_uuids: takeover_uuids.clone(), }; // The rollout gate plans again without its deferred rows: keep what // the second pass needs. @@ -2451,6 +2466,12 @@ fn describe_skip_reason(reason: &str) -> String { "redirect_vlt_artifact_unverifiable" => { "vlt could not verify the hosted artifact (see the warning)".into() } + "redirect_unattributable" => { + "lockfile discovery could not attribute its pin to one package version, so \ + nothing was written for it (reconcile the project's lockfiles; --json has the \ + detail)" + .into() + } other => format!("server status `{other}`"), } } @@ -4299,6 +4320,12 @@ mod tests { "hosted mode cannot pin it where vendored mode wired it, so it stays vendored \ (see the warning)" ); + assert_eq!( + describe_skip_reason(socket_patch_core::hosted::engine::REDIRECT_UNATTRIBUTABLE), + "lockfile discovery could not attribute its pin to one package version, so \ + nothing was written for it (reconcile the project's lockfiles; --json has the \ + detail)" + ); assert_eq!(describe_skip_reason("mystery"), "server status `mystery`"); for code in [ "not_found", diff --git a/crates/socket-patch-cli/tests/in_process_get_hosted_ecosystems.rs b/crates/socket-patch-cli/tests/in_process_get_hosted_ecosystems.rs index 1eafaf897..b65363f77 100644 --- a/crates/socket-patch-cli/tests/in_process_get_hosted_ecosystems.rs +++ b/crates/socket-patch-cli/tests/in_process_get_hosted_ecosystems.rs @@ -512,7 +512,9 @@ async fn maven_profile_scoped_dependency_is_not_redirected() { "#; std::fs::write(tmp.path().join("pom.xml"), pom).unwrap(); - socket_patch_cli::commands::get::run(get_hosted_args(UUID, tmp.path(), server.uri())).await; + let code = + socket_patch_cli::commands::get::run(get_hosted_args(UUID, tmp.path(), server.uri())).await; + assert_eq!(code, 0, "an unattributable pin is a skip, not a failure"); assert_eq!( reference_bodies(&server).await.len(), 1, diff --git a/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs b/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs index 1c5fd45a5..bf467082c 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs @@ -425,7 +425,8 @@ async fn an_unreached_requirements_include_vetoes_the_pipfile_lock_redirect() { std::fs::create_dir_all(tmp.path().join("req")).unwrap(); std::fs::write(tmp.path().join("req/base.txt"), "urllib3==1.26.18\n").unwrap(); - run(hosted_args(tmp.path(), server.uri(), None)).await; + let code = run(hosted_args(tmp.path(), server.uri(), None)).await; + assert_eq!(code, 0, "an unattributable pin is a skip, not a failure"); assert_eq!( read(&tmp.path().join("Pipfile.lock")), LOCK, diff --git a/crates/socket-patch-cli/tests/mode_migration_pypi.rs b/crates/socket-patch-cli/tests/mode_migration_pypi.rs index 706ac5b0c..edfcb97b2 100644 --- a/crates/socket-patch-cli/tests/mode_migration_pypi.rs +++ b/crates/socket-patch-cli/tests/mode_migration_pypi.rs @@ -521,6 +521,93 @@ async fn pipenv_vendored_to_hosted() { assert_vendored_to_hosted(&root, files).await; } +/// A vendored Pipenv project beside a requirements `-r` include the hosted +/// rewriter does not reach (the #567 shape). The attribution gate would +/// veto the Pipfile.lock pin as contested, but a wet takeover has already +/// reverted the vendored wiring by then: dropping it would strand the +/// package on the unpatched registry release (exit 1) while the dry run +/// reported success. A takeover keeps the rewriters' verdict, so the dry +/// run and the wet run agree and the package is never left unpatched. +#[tokio::test] +async fn pipenv_takeover_beside_an_unreached_include_is_never_stranded() { + let (_tmp, root) = project(); + let files = stage_pipenv(&root); + vendor_project(&root, files); + std::fs::write(root.join("requirements.txt"), "-r req/base.txt\n").unwrap(); + std::fs::create_dir_all(root.join("req")).unwrap(); + std::fs::write(root.join("req/base.txt"), "six==1.16.0\n").unwrap(); + let vendored_lock = std::fs::read_to_string(root.join("Pipfile.lock")).unwrap(); + let server = MockServer::start().await; + let hosted_url = mount_hosted_api(&server, true).await; + let uri = server.uri(); + + let mut dry = hosted_scan_args(&uri); + dry.push("--dry-run"); + let (dry_code, dry_env) = run_cli(&root, &dry, &[]); + assert_eq!( + std::fs::read_to_string(root.join("Pipfile.lock")).unwrap(), + vendored_lock, + "a dry run writes nothing" + ); + + let (code, env) = hosted_scan(&root, &server); + assert_eq!(code, 0, "the takeover is not stranded: {env:#}"); + assert!( + !env.to_string().contains("redirect_takeover_unpatched"), + "{env:#}" + ); + assert_eq!(env["redirect"]["redirected"], 1, "{env:#}"); + let lock = std::fs::read_to_string(root.join("Pipfile.lock")).unwrap(); + assert!( + lock.contains(&hosted_url), + "six is pinned to the patch:\n{lock}" + ); + assert!(!lock.contains(".socket/vendor/"), "{lock}"); + assert_eq!( + (dry_code, &dry_env["redirect"]["redirected"]), + (code, &env["redirect"]["redirected"]), + "the dry run predicts the wet run: {dry_env:#}" + ); +} + +/// #567 without a takeover: the Pipfile.lock pin the hosted rewriter would +/// land is contested by an `-r` include it does not reach, so the patch is +/// left out — reported in `redirect.skipped[]` as `redirect_unattributable` +/// with discovery's finding — nothing is written and the exit code is 0. +#[tokio::test] +async fn pipenv_redirect_beside_an_unreached_include_is_skipped_unattributable() { + let (_tmp, root) = project(); + stage_pipenv(&root); + std::fs::write(root.join("requirements.txt"), "-r req/base.txt\n").unwrap(); + std::fs::create_dir_all(root.join("req")).unwrap(); + std::fs::write(root.join("req/base.txt"), "six==1.16.0\n").unwrap(); + let pristine = std::fs::read_to_string(root.join("Pipfile.lock")).unwrap(); + let server = MockServer::start().await; + mount_hosted_api(&server, true).await; + + let (code, env) = hosted_scan(&root, &server); + assert_eq!( + code, 0, + "an unattributable pin is a skip, not a failure: {env:#}" + ); + assert_eq!(env["redirect"]["redirected"], 0, "{env:#}"); + let skipped = env["redirect"]["skipped"].as_array().expect("skipped[]"); + assert_eq!(skipped.len(), 1, "{env:#}"); + assert_eq!(skipped[0]["purl"], PURL, "{env:#}"); + assert_eq!(skipped[0]["reason"], "redirect_unattributable", "{env:#}"); + assert!( + skipped[0]["detail"] + .as_str() + .is_some_and(|d| d.contains("req/base.txt")), + "the detail names the contesting file: {env:#}" + ); + assert_eq!( + std::fs::read_to_string(root.join("Pipfile.lock")).unwrap(), + pristine, + "nothing is written for it" + ); +} + /// A superseding patch for the same release (a fixed patch, or one /// covering more CVEs). const UUID_B: &str = "6d4f2b3c-8e5a-4f7b-9c9d-2e3f4a5b6c7d"; diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index 97a86eb21..dc30a91cf 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -906,6 +906,13 @@ pub struct RewriteOptions<'a> { /// Run the rewriters on the blocking pool (the disk flow: pure CPU over /// every lock text). pub blocking: bool, + /// Uuids of the wet run's vendored→hosted takeovers: the caller already + /// reverted their vendored wiring (and saved the ledger) before the + /// rewrite, so the attribution gate never drops them — that would leave + /// the package on the unpatched registry release — and their pin keeps + /// the rewriters' verdict, as a dry run's withheld takeover preview + /// does. Empty for the in-memory engine, which takes nothing over. + pub takeover_uuids: BTreeSet, } /// One project's rewrite, ready for the guard, the record fetch and the @@ -1070,7 +1077,13 @@ pub async fn rewrite( // package version (another lock resolving the same version elsewhere, a // pin Maven never consumes) would be refused by every later command, so // it is dropped and the rest rewritten without it. Each pass drops at - // least one candidate, so this ends. + // least one candidate, so this ends. A wet takeover is never dropped + // (see [`RewriteOptions::takeover_uuids`]): its vendored wiring is + // already gone, and its pin keeps the rewriters' verdict. + let exempt: BTreeSet = withheld_from_vlt + .union(&options.takeover_uuids) + .cloned() + .collect(); let mut kept: Vec = candidates.to_vec(); let mut unattributed: Vec = Vec::new(); loop { @@ -1084,7 +1097,7 @@ pub async fn rewrite( options.clone(), ) .await; - let (vetoed, lockless) = unattributed_pins(view, &done, &kept, withheld_from_vlt).await; + let (vetoed, lockless) = unattributed_pins(view, &done, &kept, &exempt).await; if vetoed.is_empty() { done.unattributed = unattributed; done.rewrite.warnings.extend(lockless); @@ -1107,8 +1120,10 @@ pub async fn rewrite( /// bundled or user-patched copy the rewriters knowingly left on the /// registry (`bundled_skipped_uuids`, or a bundled copy in vlt's store; /// both are warned and kept out of the in-run VEX), or one withheld from -/// the vlt rewrite while a sibling lock takes it (`withheld_from_vlt`). Which unreachable copies should block a -/// redirect is the copy-source policy (audit B16), not decided here. +/// the vlt rewrite while a sibling lock takes it, and a wet vendored→hosted +/// takeover whose vendored wiring the caller already reverted (both in +/// `exempt`). Which unreachable copies should block a redirect is the +/// copy-source policy (audit B16), not decided here. /// /// [`HostedInventory`]: crate::patch::redirect::upstream::HostedInventory /// [`UnlockedPin`]: crate::vex::discover::UnlockedPin @@ -1116,7 +1131,7 @@ async fn unattributed_pins( view: &ProjectView<'_>, done: &Rewritten, candidates: &[Candidate], - withheld_from_vlt: &BTreeSet, + exempt: &BTreeSet, ) -> (Vec, Vec) { if done.confirmed.is_empty() { return (Vec::new(), Vec::new()); @@ -1217,7 +1232,7 @@ async fn unattributed_pins( !attributed.contains(uuid.as_str()) && contested.contains(uuid.as_str()) && !done.rewrite.bundled_skipped_uuids.contains(uuid) - && !withheld_from_vlt.contains(uuid) + && !exempt.contains(uuid) && !vlt_bundled.contains(&crate::vex::discover::canonical_base_purl(purl)) }) .map(|(purl, uuid)| { @@ -2260,6 +2275,7 @@ mod tests { npm_allow_remote_config: true, npm_outer: &outer, blocking: false, + takeover_uuids: Default::default(), }; let mut skipped = Vec::new(); let candidates = build_candidates(&selected, &refs, &mut skipped); @@ -2491,6 +2507,7 @@ mod tests { npm_allow_remote_config: true, npm_outer: &outer, blocking: false, + takeover_uuids: Default::default(), }; let candidates = vec![left_pad_candidate()]; let read = read_candidate_files(view, unreadable, &candidates).await; @@ -2612,6 +2629,7 @@ mod tests { npm_allow_remote_config: true, npm_outer: &outer, blocking: false, + takeover_uuids: Default::default(), }; let read = read_candidate_files(&view, &BTreeSet::new(), &candidates).await; assert!(read.files.contains_key("package.json")); @@ -2788,6 +2806,7 @@ mod tests { npm_allow_remote_config: true, npm_outer: &outer, blocking: false, + takeover_uuids: Default::default(), }; let candidates = vec![gradle_candidate()]; let read = read_candidate_files(view, &BTreeSet::new(), &candidates).await; @@ -2999,6 +3018,92 @@ mod tests { assert!(done.confirmed.is_empty(), "{:?}", done.confirmed); } + /// A lockless NuGet pin (a Socket source mapping, no + /// `packages.lock.json`) is still written, but the run says no later + /// command can manage it and names the lockfile that fixes that. + #[tokio::test] + async fn a_lockless_nuget_pin_is_written_with_the_lockless_warning() { + use crate::patch::redirect::{Integrity, RegistryOverride, RegistryOverrideIdentifiers}; + let base = + format!("https://patch.test/patch-registry/nuget/{FIXTURE_TOKEN}/{FIXTURE_UUID}"); + let candidate = Candidate { + purl: "pkg:nuget/Newtonsoft.Json@13.0.3".into(), + dep: DepOverride { + ecosystem: "nuget".into(), + name: "Newtonsoft.Json".into(), + namespace: None, + version: "13.0.3".into(), + token: FIXTURE_TOKEN.into(), + patch_uuid: FIXTURE_UUID.into(), + artifact_url: format!( + "{base}/flat/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg" + ), + registry_override: Some(RegistryOverride { + kind: "nuget-v3".into(), + index_url: format!("{base}/index.json"), + identifiers: RegistryOverrideIdentifiers { + name: "Newtonsoft.Json".into(), + version: "13.0.3".into(), + nuget_id_lower: Some("newtonsoft.json".into()), + nuget_version_norm: Some("13.0.3".into()), + ..Default::default() + }, + }), + integrity: Integrity { + sha512: Some("sha512-NUGETPATCHED==".into()), + ..Default::default() + }, + }, + }; + let mut p = MemoryProject::new(); + p.insert_text( + "nuget.config", + "\n\n \n \ + \n \ + \n\n", + ); + let outer = OuterAllowRemote::default; + let options = RewriteOptions { + dry_run: false, + targets_pipenv_lock: false, + pipenv_major: None, + pipenv_unknown_detail: String::new(), + trust_lockfile_config: true, + npm_allow_remote_config: true, + npm_outer: &outer, + blocking: false, + takeover_uuids: Default::default(), + }; + let candidates = vec![candidate]; + let view = ProjectView::Memory(&p); + let read = read_candidate_files(&view, &BTreeSet::new(), &candidates).await; + let done = rewrite( + &view, + read, + &candidates, + BTreeMap::new(), + &BTreeSet::new(), + &[], + options, + ) + .await; + assert_eq!(done.confirmed.len(), 1, "{:?}", done.rewrite.warnings); + assert!(done.unattributed.is_empty(), "{:?}", done.unattributed); + let lockless: Vec<&RewriteWarning> = done + .rewrite + .warnings + .iter() + .filter(|w| w.code == REDIRECT_PIN_LOCKLESS) + .collect(); + assert_eq!(lockless.len(), 1, "{:?}", done.rewrite.warnings); + let detail = &lockless[0].detail; + assert!(detail.contains("Newtonsoft.Json"), "{detail}"); + assert!( + detail.contains("dotnet restore --use-lock-file"), + "{detail}" + ); + } + const GEMFILE: &str = "source \"https://rubygems.org\"\n\ngem \"rails\", \"7.0.0\"\n"; const GEM_LOCK: &str = "GEM\n remote: https://rubygems.org/\n specs:\n rails (7.0.0)\n\n\ PLATFORMS\n ruby\n\nDEPENDENCIES\n rails (= 7.0.0)\n\nBUNDLED WITH\n 2.5.22\n"; @@ -3014,6 +3119,7 @@ mod tests { npm_allow_remote_config: true, npm_outer: &outer, blocking: false, + takeover_uuids: Default::default(), }; let candidates = vec![gem_candidate()]; let view = ProjectView::Memory(p); diff --git a/crates/socket-patch-core/src/hosted/memory/stages.rs b/crates/socket-patch-core/src/hosted/memory/stages.rs index ed0430df1..5d078fcb7 100644 --- a/crates/socket-patch-core/src/hosted/memory/stages.rs +++ b/crates/socket-patch-core/src/hosted/memory/stages.rs @@ -330,6 +330,7 @@ pub(crate) async fn rewrite( npm_allow_remote_config: options.npm_allow_remote_config, npm_outer: &npm_outer, blocking: false, + takeover_uuids: Default::default(), }, ) .await; From e25c17dacab277eb68c4801119de73c90ef6c301 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Wed, 7 Oct 2026 13:19:31 -0400 Subject: [PATCH 05/20] Keep nested roots' pins out of a root's in-memory recorded view memory_recorded now reads discovery's pins, which can include files under a nested root that discovery reaches through a requirements include or a rush subspace. Those pins are that root's own, as they were under the mention scan this replaced, so they are filtered out again. Pins to patches the API no longer offers now count (as on disk); the doc comment says so. Also document that DiskSnapshot::overlay does not change directory listings, and rename the vlt heal's private origin list to vlt_heal_origins so it is not confused with rollback's patch_server_origins (it also counts --api-url). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commands/scan/hosted/vlt.rs | 10 ++++++++-- .../src/hosted/memory/mod.rs | 19 +++++++++++++++++-- .../src/vendor/lock_inventory/view.rs | 8 +++++++- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/hosted/vlt.rs b/crates/socket-patch-cli/src/commands/scan/hosted/vlt.rs index 79fed157d..518c8449b 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted/vlt.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted/vlt.rs @@ -68,7 +68,13 @@ pub(super) async fn artifact_preflight( hosted_vlt::judge(&plan, deps, probes) } -fn patch_server_origins(common: &crate::args::GlobalArgs) -> Vec { +/// The origins the vlt heal treats as Socket-owned in the final lock: the +/// patch server plus `--api-url`. Unlike discovery's +/// [`crate::commands::rollback::patch_server_origins`], it also counts the +/// `--api-url` origin (as it has since vlt support landed, #269), so a +/// setup serving the hosted tarballs from the API origin is healed too. +/// Discovery-facing code must use the rollback helper, not this one. +fn vlt_heal_origins(common: &crate::args::GlobalArgs) -> Vec { common .patch_server_url .iter() @@ -322,7 +328,7 @@ pub(super) async fn heal_after_rewrite( let mut out = StaleInstallOutcome::default(); let owned: Vec = inputs .final_lock - .map(|lock| vlt_heal::socket_owned_instances(lock, &patch_server_origins(common))) + .map(|lock| vlt_heal::socket_owned_instances(lock, &vlt_heal_origins(common))) .unwrap_or_default() .into_iter() .filter(|i| inputs.preflight.passed.contains(&i.patch_uuid)) diff --git a/crates/socket-patch-core/src/hosted/memory/mod.rs b/crates/socket-patch-core/src/hosted/memory/mod.rs index ec3378b07..73eca2877 100644 --- a/crates/socket-patch-core/src/hosted/memory/mod.rs +++ b/crates/socket-patch-core/src/hosted/memory/mod.rs @@ -385,17 +385,31 @@ fn unrooted_unsupported_warnings<'a>( /// nothing. Pins on a patch server other than Socket's are recognized once /// the run's references name it ([`stage::mark_pinned`]). /// +/// Unlike the mention scan this replaced, a pin counts whether or not the +/// API still offers its patch (as on disk): a re-scan re-confirms it as +/// ALREADY instead of spending a NEW slot. A pin wired only by files under +/// a nested root (one discovery reaches through a requirements include or +/// a rush subspace) is that root's own and does not count here. +/// /// [`HostedPin::discover`]: crate::patch::redirect::upstream::HostedPin::discover /// [`stage::mark_pinned`]: crate::rollout::stage::mark_pinned -async fn memory_recorded(project: &MemoryProject) -> RecordedIndex { +async fn memory_recorded(project: &MemoryProject, root: &str, roots: &[String]) -> RecordedIndex { let manifest = project .text(select::MANIFEST_REL) .and_then(|text| serde_json::from_str(text).ok()); let vendor = stages::vendored_entries(project); + let nested: Vec = roots + .iter() + .filter(|other| other.as_str() != root) + .filter_map(|other| roots::strip_root(root, other).map(|rel| format!("{rel}/"))) + .filter(|rel| rel != "/") + .collect(); + let own = |file: &String| !nested.iter().any(|n| file.starts_with(n.as_str())); let pins: Vec<(String, String)> = crate::patch::redirect::upstream::HostedPin::discover(ProjectView::Memory(project), &[]) .await .into_iter() + .filter(|pin| pin.files.iter().any(own)) .map(|pin| (pin.purl, pin.uuid)) .collect(); let merged = crate::ledgers::merge_ledger_records_for_updates( @@ -707,6 +721,7 @@ async fn engine( stage.incomplete |= states .iter() .any(|s| s.error.as_ref().is_some_and(|e| e.code == "patch_lookup_failed")); + let roots_by_path: Vec = states.iter().map(|s| s.root.clone()).collect(); for state in states.iter_mut() { if state.error.is_some() { continue; @@ -714,7 +729,7 @@ async fn engine( let Some(project) = state.project.as_ref() else { continue; }; - let recorded = memory_recorded(project).await; + let recorded = memory_recorded(project, &state.root, &roots_by_path).await; stage.incomplete |= lookup_incomplete( &recorded, &state.failed_details, diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/view.rs b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs index 9627f98f5..8c3ab7b5c 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/view.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs @@ -211,7 +211,13 @@ impl<'a> DiskSnapshot<'a> { /// Read `rel` as `content` instead of the disk's: the project as a /// pending write would leave it. Content reads and existence probes see - /// the overlay; directory listings still list the disk. + /// the overlay; directory listings (`list_dir`, `python_lock_paths`, + /// rush subspace locks) still list the disk. So a file the write would + /// CREATE that discovery only finds by listing (a new `pylock.*.toml`, a + /// new rush subspace lock) is invisible to a discovery over the overlay, + /// and the hosted attribution gate keeps the rewriters' verdict for its + /// pin. No hosted rewriter creates such a file today; one that does must + /// merge its outputs into the listings first. pub fn overlay(&self, rel: &str, content: &[u8]) { self.remember(rel, &Ok(content.to_vec())); self.overlaid From ddd285ab7036111fdecb427db7d9c4ea70fa6d4e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 18:32:19 +0000 Subject: [PATCH 06/20] Tell NuGet config spellings apart on Windows by file identity same_file returned false on every non-Unix platform, so on Windows' case-insensitive filesystem the one nuget.config was also recognized as NuGet.config and NuGet.Config. A contested-wiring refusal then named the file three times, and its `git checkout --` remedy listed three paths for one file, which lockless_nuget_pin_refusal_names_the_lockfile_remedy caught on windows-latest. Compare volume serial + file index on Windows through the same-file crate the core already depends on. Unix keeps its stat-only dev + inode check so a FIFO under a config name is never opened. Co-Authored-By: Claude --- .../src/vendor/nuget_config.rs | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/crates/socket-patch-core/src/vendor/nuget_config.rs b/crates/socket-patch-core/src/vendor/nuget_config.rs index 21f58ae16..3481a13e8 100644 --- a/crates/socket-patch-core/src/vendor/nuget_config.rs +++ b/crates/socket-patch-core/src/vendor/nuget_config.rs @@ -8,8 +8,10 @@ pub(crate) const CONFIG_NAMES: [&str; 3] = ["nuget.config", "NuGet.config", "NuGet.Config"]; /// Whether `a` and `b` are one file (a case-insensitive filesystem's two -/// spellings of it). Unix compares device + inode; elsewhere `false` (the -/// worst case is a duplicate file name in recognition, never a lost one). +/// spellings of it). Unix compares device + inode (stat only: never opens +/// a FIFO planted under a config name); Windows compares volume serial + +/// file index; elsewhere `false` (the worst case is a duplicate file name +/// in recognition, never a lost one). pub(crate) async fn same_file(a: &std::path::Path, b: &std::path::Path) -> bool { #[cfg(unix)] { @@ -17,8 +19,18 @@ pub(crate) async fn same_file(a: &std::path::Path, b: &std::path::Path) -> bool if let (Ok(x), Ok(y)) = (tokio::fs::metadata(a).await, tokio::fs::metadata(b).await) { return x.dev() == y.dev() && x.ino() == y.ino(); } + false + } + #[cfg(windows)] + { + let (a, b) = (a.to_path_buf(), b.to_path_buf()); + tokio::task::spawn_blocking(move || same_file::is_same_file(a, b).unwrap_or(false)) + .await + .unwrap_or(false) + } + #[cfg(not(any(unix, windows)))] + { + let _ = (a, b); + false } - #[cfg(not(unix))] - let _ = (a, b); - false } From 8da8a1aac8770453654e8f6bfde74d82005b9d95 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Wed, 7 Oct 2026 14:49:37 -0400 Subject: [PATCH 07/20] Count configured patch servers in the attribution gate and foreign upgrades The attribution gate discovered with only the hosts this run's grants name, not the operator's --patch-server-url allowlist that vex, list, rollback, remove and vendor read. An existing pin on a configured server was invisible to the gate whenever this run's grants lived on another host, so a write those commands would refuse as contested could land. RewriteOptions now carries the configured origins and the gate discovers over them plus the grants' hosts. The second rollout pass (mark_pinned) only flipped a NEW row to ALREADY when the selected uuid was pinned on a server only this run's references name. An older patch pinned there stayed NEW, so an upgrade spent a NEW cap slot (in both engines; in memory always, since it has no configured origins). It now marks such a row Superseded. Addresses Bugbot findings on #1058. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commands/scan/hosted.rs | 2 + .../src/commands/scan/rollout.rs | 29 +++++++++++++ crates/socket-patch-core/src/hosted/engine.rs | 28 +++++++++++-- .../src/hosted/memory/stages.rs | 1 + crates/socket-patch-core/src/rollout/stage.rs | 42 +++++++++++++------ 5 files changed, 86 insertions(+), 16 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index b78c6e75d..695dd916a 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -1023,6 +1023,7 @@ pub(crate) async fn run_redirect_selected( .map(|c| c.dep.patch_uuid.clone()) .collect() }; + let patch_server_origins = crate::commands::rollback::patch_server_origins(common); let rewrite_options = || RewriteOptions { dry_run: common.dry_run, targets_pipenv_lock, @@ -1036,6 +1037,7 @@ pub(crate) async fn run_redirect_selected( npm_outer: &npm_outer, blocking: true, takeover_uuids: takeover_uuids.clone(), + patch_server_origins: patch_server_origins.clone(), }; // The rollout gate plans again without its deferred rows: keep what // the second pass needs. diff --git a/crates/socket-patch-cli/src/commands/scan/rollout.rs b/crates/socket-patch-cli/src/commands/scan/rollout.rs index 21d60de2c..fc25416f3 100644 --- a/crates/socket-patch-cli/src/commands/scan/rollout.rs +++ b/crates/socket-patch-cli/src/commands/scan/rollout.rs @@ -449,6 +449,35 @@ mod tests { assert_eq!(rows[1].candidate.recorded, Recorded::None); } + #[test] + fn a_discovered_pin_of_another_uuid_marks_the_row_an_upgrade() { + // A pin on a patch server only this run's references name is found + // by the second discovery pass: an older patch pinned there is an + // UPGRADE, not a NEW row spending a cap slot. + let results = vec![offer( + "pkg:npm/a@1", + "aaaaaaaa-1111-4111-8111-00000000000a", + "", + &["high"], + )]; + let offers = offers_from_results(&results, true); + let mut rows = classify(&offers, &RecordedIndex::default(), ""); + mark_pinned( + &mut rows, + &[socket_patch_core::patch::redirect::upstream::HostedPin { + purl: "pkg:npm/a@1".into(), + uuid: "AAAAAAAA-2222-4222-8222-00000000000A".into(), + files: vec!["package-lock.json".into()], + }], + ); + assert_eq!( + rows[0].candidate.recorded, + Recorded::Superseded { + old_uuid: "aaaaaaaa-2222-4222-8222-00000000000a".into() + } + ); + } + #[test] fn case_folded_pins_match_the_selection_spelling() { // Discovery keys a nuget pin by the lowercased name; the API and diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index dc30a91cf..015ffc3aa 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -913,6 +913,12 @@ pub struct RewriteOptions<'a> { /// the rewriters' verdict, as a dry run's withheld takeover preview /// does. Empty for the in-memory engine, which takes nothing over. pub takeover_uuids: BTreeSet, + /// The operator's extra patch-server origins (`--patch-server-url`): + /// the allowlist `vex`, `list`, `rollback`, `remove` and `vendor` + /// discover with, so the attribution gate sees an existing pin on a + /// configured server even when this run's grants live on another host. + /// Empty for the in-memory engine, which has no such knob. + pub patch_server_origins: Vec, } /// One project's rewrite, ready for the guard, the record fetch and the @@ -1097,7 +1103,8 @@ pub async fn rewrite( options.clone(), ) .await; - let (vetoed, lockless) = unattributed_pins(view, &done, &kept, &exempt).await; + let (vetoed, lockless) = + unattributed_pins(view, &done, &kept, &exempt, &options.patch_server_origins).await; if vetoed.is_empty() { done.unattributed = unattributed; done.rewrite.warnings.extend(lockless); @@ -1132,14 +1139,21 @@ async fn unattributed_pins( done: &Rewritten, candidates: &[Candidate], exempt: &BTreeSet, + configured: &[String], ) -> (Vec, Vec) { if done.confirmed.is_empty() { return (Vec::new(), Vec::new()); } + // The management commands' allowlist plus the hosts this run's grants + // name: a pin on either counts, as it will for them. + let mut origins = configured.to_vec(); + for origin in crate::patch::redirect::upstream::dep_origins(candidates.iter().map(|c| &c.dep)) { + if !origins.contains(&origin) { + origins.push(origin); + } + } let opts = crate::vex::DiscoverOptions { - patch_server_origins: crate::patch::redirect::upstream::dep_origins( - candidates.iter().map(|c| &c.dep), - ), + patch_server_origins: origins, }; let mut written: Vec<(&str, &[u8])> = Vec::new(); for (rel, text) in &done.rewrite.files { @@ -2276,6 +2290,7 @@ mod tests { npm_outer: &outer, blocking: false, takeover_uuids: Default::default(), + patch_server_origins: Vec::new(), }; let mut skipped = Vec::new(); let candidates = build_candidates(&selected, &refs, &mut skipped); @@ -2508,6 +2523,7 @@ mod tests { npm_outer: &outer, blocking: false, takeover_uuids: Default::default(), + patch_server_origins: Vec::new(), }; let candidates = vec![left_pad_candidate()]; let read = read_candidate_files(view, unreadable, &candidates).await; @@ -2630,6 +2646,7 @@ mod tests { npm_outer: &outer, blocking: false, takeover_uuids: Default::default(), + patch_server_origins: Vec::new(), }; let read = read_candidate_files(&view, &BTreeSet::new(), &candidates).await; assert!(read.files.contains_key("package.json")); @@ -2807,6 +2824,7 @@ mod tests { npm_outer: &outer, blocking: false, takeover_uuids: Default::default(), + patch_server_origins: Vec::new(), }; let candidates = vec![gradle_candidate()]; let read = read_candidate_files(view, &BTreeSet::new(), &candidates).await; @@ -3073,6 +3091,7 @@ mod tests { npm_outer: &outer, blocking: false, takeover_uuids: Default::default(), + patch_server_origins: Vec::new(), }; let candidates = vec![candidate]; let view = ProjectView::Memory(&p); @@ -3120,6 +3139,7 @@ mod tests { npm_outer: &outer, blocking: false, takeover_uuids: Default::default(), + patch_server_origins: Vec::new(), }; let candidates = vec![gem_candidate()]; let view = ProjectView::Memory(p); diff --git a/crates/socket-patch-core/src/hosted/memory/stages.rs b/crates/socket-patch-core/src/hosted/memory/stages.rs index 5d078fcb7..cf88cac57 100644 --- a/crates/socket-patch-core/src/hosted/memory/stages.rs +++ b/crates/socket-patch-core/src/hosted/memory/stages.rs @@ -331,6 +331,7 @@ pub(crate) async fn rewrite( npm_outer: &npm_outer, blocking: false, takeover_uuids: Default::default(), + patch_server_origins: Vec::new(), }, ) .await; diff --git a/crates/socket-patch-core/src/rollout/stage.rs b/crates/socket-patch-core/src/rollout/stage.rs index bf7c56c03..af46768d6 100644 --- a/crates/socket-patch-core/src/rollout/stage.rs +++ b/crates/socket-patch-core/src/rollout/stage.rs @@ -238,24 +238,42 @@ pub fn lookup_incomplete( .any(|purl| !recorded.records_package(purl)) } -/// Mark NEW rows whose selected uuid discovery already finds pinned -/// ([`HostedPin::discover`]) as ALREADY. The recorded view is discovery -/// over the configured patch servers; a pin on the server THIS run's -/// references name (an origin missing from `--patch-server-url`) is only -/// recognized once those references are known, so the caller re-runs -/// discovery with their origins ([`dep_origins`]) and hands the pins here. -/// Without it such a pin would read as NEW on every run and hold its slot -/// forever. Only discovery's attributable pins count: a uuid a stale or -/// inactive file merely mentions (an unused `pdm.lock`, a `package.json` -/// `resolutions` leftover, a comment) pins nothing and stays NEW. +/// Re-classify NEW rows against the pins discovery finds +/// ([`HostedPin::discover`]): the selected uuid pinned is ALREADY, another +/// uuid pinned for the same package is an UPGRADE +/// ([`Recorded::Superseded`]). The recorded view is discovery over the +/// configured patch servers; a pin on the server THIS run's references name +/// (an origin missing from `--patch-server-url`) is only recognized once +/// those references are known, so the caller re-runs discovery with their +/// origins ([`dep_origins`]) and hands the pins here. Without it such a pin +/// would read as NEW on every run and hold its slot forever, and an upgrade +/// on that server would spend a NEW slot. Only discovery's attributable pins +/// count: a uuid a stale or inactive file merely mentions (an unused +/// `pdm.lock`, a `package.json` `resolutions` leftover, a comment) pins +/// nothing and stays NEW. +/// +/// The writers were already chosen from the selection, so a pinned uuid +/// that the selection does not supersede is reported as an UPGRADE rather +/// than kept ([`Recorded::Kept`]): what is written does not change, only +/// that it spends no NEW slot. /// /// [`HostedPin::discover`]: crate::patch::redirect::upstream::HostedPin::discover /// [`dep_origins`]: crate::patch::redirect::upstream::dep_origins pub fn mark_pinned(rows: &mut [Row], pins: &[crate::patch::redirect::upstream::HostedPin]) { - let pinned: HashSet = pins.iter().map(|p| p.uuid.to_ascii_lowercase()).collect(); + let pairs: Vec<(String, String)> = pins + .iter() + .map(|p| (p.purl.clone(), p.uuid.to_ascii_lowercase())) + .collect(); + let index = RecordedIndex::new(None, &pairs); for row in rows.iter_mut().filter(|r| r.candidate.recorded.is_new()) { - if pinned.contains(&row.candidate.uuid.to_ascii_lowercase()) { + let uuids = index.uuids(&row.candidate.purl); + let selected = row.candidate.uuid.to_ascii_lowercase(); + if uuids.contains(&selected) { row.candidate.recorded = Recorded::Same; + } else if let Some(old) = uuids.first() { + row.candidate.recorded = Recorded::Superseded { + old_uuid: old.clone(), + }; } } } From f27cf57a5be275ab0bf5e866a7a6189a568ec79a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 22:43:01 +0000 Subject: [PATCH 08/20] WIP: reuse scan's discovery in the unattributed-pins gate when nothing was written Rescan benches recover (npm/rescan +25% -> +5%, pnpm/rescan +13% -> +2.5%) but npm/hosted is still ~26% slower than main because a writing rewrite still runs a full fresh discovery over the overlay. Not validated: clippy and tests have not been re-run after the last edit. Parked for #1058. Co-Authored-By: Claude --- .../src/commands/scan/hosted.rs | 14 ++- .../socket-patch-cli/src/commands/scan/mod.rs | 10 +- .../src/commands/scan/rollout.rs | 19 ++- crates/socket-patch-core/src/hosted/engine.rs | 112 ++++++++++++++---- .../src/hosted/memory/stages.rs | 1 + 5 files changed, 130 insertions(+), 26 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 695dd916a..aff18e10c 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -561,6 +561,8 @@ pub(super) async fn run_redirect( recorded: &super::rollout::RecordedState<'_>, batch_failed: bool, stage: &mut super::rollout::Stage, + // Scan's pre-redirect lockfile discovery (see `rollout::Gate::prior`). + prior: Option<&socket_patch_core::vex::discover::Discovery>, ) -> i32 { // Same discovery/selection as `--apply`/`--vendor`. let discovered = match discover_selected( @@ -618,7 +620,7 @@ pub(super) async fn run_redirect( &pairs, scan_result, npm_prior, - Some(super::rollout::Gate::new(stage, rows)), + Some(super::rollout::Gate::new(stage, rows).with_prior(prior)), ) .await } @@ -1024,6 +1026,15 @@ pub(crate) async fn run_redirect_selected( .collect() }; let patch_server_origins = crate::commands::rollback::patch_server_origins(common); + // Scan's discovery predates this run's writes, and only the takeover's + // revert above changes the project before the rewrite (scan writes + // nothing between its discovery and this call): it still describes the + // project the rewrite reads unless a takeover touched files. It was + // made with `patch_server_origins` (`discover_wiring`). + let prior_discovery = rollout + .as_ref() + .and_then(|gate| gate.prior) + .filter(|_| takeover_files.is_empty() && takeover_migrated.is_empty()); let rewrite_options = || RewriteOptions { dry_run: common.dry_run, targets_pipenv_lock, @@ -1038,6 +1049,7 @@ pub(crate) async fn run_redirect_selected( blocking: true, takeover_uuids: takeover_uuids.clone(), patch_server_origins: patch_server_origins.clone(), + prior_discovery, }; // The rollout gate plans again without its deferred rows: keep what // the second pass needs. diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index 66df3be83..c2c7a43bb 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -1885,6 +1885,13 @@ async fn run_scan( } else { socket_patch_core::patch::redirect::upstream::HostedPin::all(ctx.discovery().await) }; + // The same discovery, handed to the hosted redirect's attribution gate + // (nothing below writes before it; see `rollout::Gate::prior`). + let prior_discovery = if args.common.is_global() { + None + } else { + Some(ctx.discovery().await) + }; let hosted_state = (!args.common.is_global()) .then(|| crate::commands::hosted_state_from_pins(&hosted_pin_list)); let redirect_state = hosted_state.as_ref(); @@ -2424,6 +2431,7 @@ async fn run_scan( &recorded, batch_error_count > 0, &mut stage, + prior_discovery, ) .await; } @@ -2905,7 +2913,7 @@ async fn run_scan( &pairs, None, npm_crawl.as_ref(), - Some(rollout::Gate::new(&mut stage, rows)), + Some(rollout::Gate::new(&mut stage, rows).with_prior(prior_discovery)), ) .await; } diff --git a/crates/socket-patch-cli/src/commands/scan/rollout.rs b/crates/socket-patch-cli/src/commands/scan/rollout.rs index fc25416f3..6a0833eea 100644 --- a/crates/socket-patch-cli/src/commands/scan/rollout.rs +++ b/crates/socket-patch-cli/src/commands/scan/rollout.rs @@ -42,11 +42,28 @@ pub(super) fn upgrades(rows: &[Row], package_purls: &[String]) -> Vec { pub(crate) stage: &'a mut Stage, pub(crate) rows: Vec, + /// Scan's lockfile discovery of `--cwd`, made before the redirect + /// (with the configured patch-server origins): the rewrite's + /// attribution gate reuses it when nothing changed the project since. + pub(crate) prior: Option<&'a socket_patch_core::vex::discover::Discovery>, } impl<'a> Gate<'a> { pub(crate) fn new(stage: &'a mut Stage, rows: Vec) -> Self { - Gate { stage, rows } + Gate { + stage, + rows, + prior: None, + } + } + + /// This gate carrying scan's pre-redirect discovery (see [`Self::prior`]). + pub(crate) fn with_prior( + mut self, + prior: Option<&'a socket_patch_core::vex::discover::Discovery>, + ) -> Self { + self.prior = prior; + self } /// `(purl, uuid)` of every NEW row, for O(1) [`Self::is_new`] checks. diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index 015ffc3aa..9ae370f3d 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -919,6 +919,15 @@ pub struct RewriteOptions<'a> { /// configured server even when this run's grants live on another host. /// Empty for the in-memory engine, which has no such knob. pub patch_server_origins: Vec, + /// Lockfile discovery of the project exactly as this rewrite reads it, + /// made with exactly `patch_server_origins` (the caller's pre-rewrite + /// discovery, `None` when nothing was discovered or the project may + /// have changed since). The attribution gate reuses it instead of + /// discovering again when a pass writes nothing and this run's grants + /// name no origin beyond `patch_server_origins` (see + /// [`reusable_prior`]): the project it would discover is then the + /// same, read the same way. + pub prior_discovery: Option<&'a crate::vex::discover::Discovery>, } /// One project's rewrite, ready for the guard, the record fetch and the @@ -1103,8 +1112,15 @@ pub async fn rewrite( options.clone(), ) .await; - let (vetoed, lockless) = - unattributed_pins(view, &done, &kept, &exempt, &options.patch_server_origins).await; + let (vetoed, lockless) = unattributed_pins( + view, + &done, + &kept, + &exempt, + &options.patch_server_origins, + options.prior_discovery, + ) + .await; if vetoed.is_empty() { done.unattributed = unattributed; done.rewrite.warnings.extend(lockless); @@ -1140,6 +1156,7 @@ async fn unattributed_pins( candidates: &[Candidate], exempt: &BTreeSet, configured: &[String], + prior: Option<&crate::vex::discover::Discovery>, ) -> (Vec, Vec) { if done.confirmed.is_empty() { return (Vec::new(), Vec::new()); @@ -1164,34 +1181,48 @@ async fn unattributed_pins( for (rel, bytes) in &done.rewrite.binary_files { written.push((rel.as_str(), bytes.as_slice())); } - let discovery = match view.disk_root() { - None => { - let ProjectView::Memory(project) = *view else { - unreachable!("only a memory view has no disk root") - }; - let mut after = project.clone(); - for (rel, bytes) in written { - let entry = match std::str::from_utf8(bytes) { - Ok(text) => crate::vendor::lock_inventory::MemoryEntry::Text(text.into()), - Err(_) => crate::vendor::lock_inventory::MemoryEntry::Binary(bytes.into()), + let fresh; + let discovery = if let Some(prior) = reusable_prior( + prior, + written.is_empty(), + &opts.patch_server_origins, + configured, + ) { + prior + } else { + fresh = match view.disk_root() { + None => { + let ProjectView::Memory(project) = *view else { + unreachable!("only a memory view has no disk root") }; - after.insert(rel, entry); + let mut after = project.clone(); + for (rel, bytes) in written { + let entry = match std::str::from_utf8(bytes) { + Ok(text) => crate::vendor::lock_inventory::MemoryEntry::Text(text.into()), + Err(_) => crate::vendor::lock_inventory::MemoryEntry::Binary(bytes.into()), + }; + after.insert(rel, entry); + } + crate::vex::discover::discover_patched_refs_view(ProjectView::Memory(&after), &opts) + .await } - crate::vex::discover::discover_patched_refs_view(ProjectView::Memory(&after), &opts) + Some(root) => { + let after = crate::vendor::lock_inventory::DiskSnapshot::new(root); + for (rel, bytes) in written { + after.overlay(rel, bytes); + } + crate::vex::discover::discover_patched_refs_view( + ProjectView::Snapshot(&after), + &opts, + ) .await - } - Some(root) => { - let after = crate::vendor::lock_inventory::DiskSnapshot::new(root); - for (rel, bytes) in written { - after.overlay(rel, bytes); } - crate::vex::discover::discover_patched_refs_view(ProjectView::Snapshot(&after), &opts) - .await - } + }; + &fresh }; // The management commands' own view of the result: an attributable // pin, or contested wiring they would refuse around. - let inventory = crate::patch::redirect::upstream::HostedInventory::of(&discovery); + let inventory = crate::patch::redirect::upstream::HostedInventory::of(discovery); let attributed: BTreeSet<&str> = inventory.pins.iter().map(|p| p.uuid.as_str()).collect(); let lockless: Vec = discovery .unlocked_pins @@ -1279,6 +1310,19 @@ async fn unattributed_pins( (vetoed, lockless) } +/// The caller's pre-rewrite discovery, when it is exactly what the gate +/// would discover: the pass writes nothing (so the project is the one the +/// caller discovered) and the gate's origins (`origins`: `configured` plus +/// the grants' hosts) are the ones the caller discovered with. +fn reusable_prior<'d>( + prior: Option<&'d crate::vex::discover::Discovery>, + nothing_written: bool, + origins: &[String], + configured: &[String], +) -> Option<&'d crate::vex::discover::Discovery> { + prior.filter(|_| nothing_written && origins == configured) +} + /// Warning: a confirmed pin no lockfile records a version for (a lockless /// NuGet / Cargo redirect), which no later command can attribute. pub const REDIRECT_PIN_LOCKLESS: &str = "redirect_pin_lockless"; @@ -2291,6 +2335,7 @@ mod tests { blocking: false, takeover_uuids: Default::default(), patch_server_origins: Vec::new(), + prior_discovery: None, }; let mut skipped = Vec::new(); let candidates = build_candidates(&selected, &refs, &mut skipped); @@ -2524,6 +2569,7 @@ mod tests { blocking: false, takeover_uuids: Default::default(), patch_server_origins: Vec::new(), + prior_discovery: None, }; let candidates = vec![left_pad_candidate()]; let read = read_candidate_files(view, unreadable, &candidates).await; @@ -2647,6 +2693,7 @@ mod tests { blocking: false, takeover_uuids: Default::default(), patch_server_origins: Vec::new(), + prior_discovery: None, }; let read = read_candidate_files(&view, &BTreeSet::new(), &candidates).await; assert!(read.files.contains_key("package.json")); @@ -2825,6 +2872,7 @@ mod tests { blocking: false, takeover_uuids: Default::default(), patch_server_origins: Vec::new(), + prior_discovery: None, }; let candidates = vec![gradle_candidate()]; let read = read_candidate_files(view, &BTreeSet::new(), &candidates).await; @@ -3036,6 +3084,22 @@ mod tests { assert!(done.confirmed.is_empty(), "{:?}", done.confirmed); } + /// The gate reuses the caller's discovery only for a pass that writes + /// nothing and whose origins are exactly the ones it was made with. + #[test] + fn the_prior_discovery_is_reused_only_unwritten_with_the_same_origins() { + let prior = crate::vex::discover::Discovery::default(); + let configured = vec!["https://patch.test".to_string()]; + let extra = vec![ + "https://patch.test".to_string(), + "https://other.test".to_string(), + ]; + assert!(reusable_prior(Some(&prior), true, &configured, &configured).is_some()); + assert!(reusable_prior(Some(&prior), false, &configured, &configured).is_none()); + assert!(reusable_prior(Some(&prior), true, &extra, &configured).is_none()); + assert!(reusable_prior(None, true, &configured, &configured).is_none()); + } + /// A lockless NuGet pin (a Socket source mapping, no /// `packages.lock.json`) is still written, but the run says no later /// command can manage it and names the lockfile that fixes that. @@ -3092,6 +3156,7 @@ mod tests { blocking: false, takeover_uuids: Default::default(), patch_server_origins: Vec::new(), + prior_discovery: None, }; let candidates = vec![candidate]; let view = ProjectView::Memory(&p); @@ -3140,6 +3205,7 @@ mod tests { blocking: false, takeover_uuids: Default::default(), patch_server_origins: Vec::new(), + prior_discovery: None, }; let candidates = vec![gem_candidate()]; let view = ProjectView::Memory(p); diff --git a/crates/socket-patch-core/src/hosted/memory/stages.rs b/crates/socket-patch-core/src/hosted/memory/stages.rs index cf88cac57..fcd86f65c 100644 --- a/crates/socket-patch-core/src/hosted/memory/stages.rs +++ b/crates/socket-patch-core/src/hosted/memory/stages.rs @@ -332,6 +332,7 @@ pub(crate) async fn rewrite( blocking: false, takeover_uuids: Default::default(), patch_server_origins: Vec::new(), + prior_discovery: None, }, ) .await; From caf07990d1a08732201ca70cbcfc0a757ca187ac Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Wed, 7 Oct 2026 20:18:29 -0400 Subject: [PATCH 09/20] Reuse the attribution gate's discovery after the hosted writes The hosted flow discovered the project's lockfiles three times per scan: scan's own discovery, the attribution gate's discovery inside `engine::rewrite`, and `hosted_state_from_lockfiles` after the writes (plus a fourth inside `classify_overlap_takeover_with` when a vendored ledger overlaps). This finishes the WIP reuse of scan's discovery in the gate and adds the post-write reuse. Gate (engine): - The gate counts the same pins as the caller's discovery when no grant names an origin outside Socket's server and the configured ones (`foreign_dep_origins` is empty), instead of comparing origin lists as strings. - `Rewritten::final_discovery` hands back the final pass's discovery when it was made with exactly the configured origins: `Prior` (the pass wrote nothing and reused the caller's) or `Overlaid` (the overlay of the pass's writes). `None` when nothing was confirmed or a foreign origin was counted. CLI (`run_redirect_selected`, `discovery_after_writes`): - The vlt heal touched the installed tree: discover again. - The rewrite planned nothing: scan's discovery (None when a takeover wrote first), else the gate's of the unwritten project. - Dry run: scan's discovery only (the overlay describes the preview). - Files landed: the gate's overlaid discovery, when every written path was a regular file before the write or is a root `.npmrc` / `pnpm-workspace.yaml` (no discovery lists a directory for them; the overlay hides a created file only from listings). - Otherwise a fresh `discover_wiring`. The overlap classifier now takes that discovery instead of making its own. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/socket-patch-cli/src/commands/mod.rs | 1 + .../src/commands/scan/hosted.rs | 175 +++++++++- .../src/commands/scan/hosted/vlt.rs | 1 + .../socket-patch-cli/src/commands/scan/mod.rs | 18 +- crates/socket-patch-core/src/hosted/engine.rs | 325 ++++++++++++++++-- .../src/hosted/memory/mod.rs | 1 + 6 files changed, 475 insertions(+), 46 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/mod.rs b/crates/socket-patch-cli/src/commands/mod.rs index a1e52739e..bfa2bbee7 100644 --- a/crates/socket-patch-cli/src/commands/mod.rs +++ b/crates/socket-patch-cli/src/commands/mod.rs @@ -132,6 +132,7 @@ pub(crate) async fn hosted_inventory( /// edits) — it is never persisted. /// /// [`RedirectState`]: socket_patch_core::patch::redirect::RedirectState +#[cfg(test)] pub(crate) async fn hosted_state_from_lockfiles( common: &crate::args::GlobalArgs, root: &Path, diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index aff18e10c..f74751e57 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -181,6 +181,10 @@ fn acquire_hosted_lock( struct StaleInstallOutcome { warnings: Vec, stale_purls: std::collections::BTreeSet, + /// The probe may have changed the installed tree (the vlt heal + /// invalidates store copies, which lockfile discovery reads for + /// bundled copies); the read-only probes never set it. + touched_install: bool, } /// The `redirect_gem_stale_install` warning for one stale installed @@ -625,6 +629,62 @@ pub(super) async fn run_redirect( .await } +/// What the hosted run wrote after its rewrite read the project, for +/// [`discovery_after_writes`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Written<'a> { + /// The rewrite planned no file write (a no-op or already-redirected run). + Nothing, + /// A dry run: the rewrite planned writes, none landed. + Previewed, + /// The rewrite's files landed; these paths were not a regular file + /// before the write. + Landed { created: &'a [&'a str] }, +} + +/// An already-made lockfile discovery (made with the configured patch-server +/// origins, as [`crate::commands::discover_wiring`] makes it) that equals a +/// fresh discovery of the project as the hosted run left it, or `None` when +/// none provably does and the caller must discover again. +/// +/// - Anything that touched the installed tree after the rewrite +/// (`touched_install`: the vlt heal) may change what discovery reads, so +/// nothing is reused. +/// - Nothing written, or a dry run: the project is as scan's pre-redirect +/// discovery (`prior`) saw it; `prior` is `None` when a takeover changed +/// it first. Failing that, when the rewrite planned nothing, the gate's +/// own discovery of the unwritten project. +/// - Files written: the gate's discovery over exactly those writes +/// ([`FinalDiscovery::Overlaid`]), when every written path existed before +/// or is one no discovery finds by listing a directory +/// ([`engine::overlay_creation_is_invisible`]). +/// +/// [`FinalDiscovery::Overlaid`]: socket_patch_core::hosted::engine::FinalDiscovery::Overlaid +/// [`engine::overlay_creation_is_invisible`]: socket_patch_core::hosted::engine::overlay_creation_is_invisible +fn discovery_after_writes<'d>( + prior: Option<&'d socket_patch_core::vex::discover::Discovery>, + gate: Option<&'d socket_patch_core::hosted::engine::FinalDiscovery>, + written: Written<'_>, + touched_install: bool, +) -> Option<&'d socket_patch_core::vex::discover::Discovery> { + use socket_patch_core::hosted::engine::{overlay_creation_is_invisible, FinalDiscovery}; + if touched_install { + return None; + } + let overlaid = match gate { + Some(FinalDiscovery::Overlaid(discovery)) => Some(&**discovery), + // The gate read `prior` itself (see `engine::rewrite`). + Some(FinalDiscovery::Prior) | None => None, + }; + match written { + Written::Nothing => prior.or(overlaid), + Written::Previewed => prior, + Written::Landed { created } => { + overlaid.filter(|_| created.iter().all(|rel| overlay_creation_is_invisible(rel))) + } + } +} + /// The hosted-redirect flow over an ALREADY-SELECTED `(purl, uuid)` set, /// on disk. The plan → rewrite → edits core is the shared hosted engine /// ([`socket_patch_core::hosted::engine`], over a @@ -1186,6 +1246,9 @@ pub(crate) async fn run_redirect_selected( } let rewrite = &done.rewrite; + // Written paths that were not a regular file before the write (see + // `discovery_after_writes`). + let mut created: Vec<&str> = Vec::new(); if !common.dry_run { for (rel, content) in rewrite .files @@ -1194,6 +1257,12 @@ pub(crate) async fn run_redirect_selected( .chain(rewrite.binary_files.iter().map(|(p, b)| (p, b.as_slice()))) { let path = common.cwd.join(rel); + if !tokio::fs::symlink_metadata(&path) + .await + .is_ok_and(|m| m.is_file()) + { + created.push(rel); + } if let Some(parent) = path.parent() { let _ = std::fs::create_dir_all(parent); } @@ -1306,12 +1375,41 @@ pub(crate) async fn run_redirect_selected( // Classified over the lockfiles as this run left them and the vendored // ledger as the takeover left it. let mut takeover_warnings: Vec = Vec::new(); - let hosted_now = crate::commands::hosted_state_from_lockfiles(common, &common.cwd).await; + // The lockfiles as this run left them: the gate's (or scan's) discovery + // when it provably describes them, else a fresh one. + let written = if !rewrite + .files + .keys() + .chain(rewrite.binary_files.keys()) + .any(|rel| !socket_patch_core::patch::redirect::sbt::is_synthetic_key(rel)) + { + Written::Nothing + } else if common.dry_run { + Written::Previewed + } else { + Written::Landed { created: &created } + }; + let fresh_now; + let discovery_now = match discovery_after_writes( + prior_discovery, + done.final_discovery.as_ref(), + written, + vlt_stale.touched_install, + ) { + Some(discovery) => discovery, + None => { + fresh_now = crate::commands::discover_wiring(common, &common.cwd).await; + &fresh_now + } + }; + let hosted_now = crate::commands::hosted_state_from_pins( + &socket_patch_core::patch::redirect::upstream::HostedPin::all(discovery_now), + ); let superseded = super::classify_overlap_takeover_with( - common, &common.cwd, Some(&hosted_now), vendor_state.as_ref().ok(), + discovery_now, ) .await .redirect; @@ -2732,10 +2830,83 @@ mod tests { wrap_tokens, wrap_words, TAKEOVER_INFO_CODES, }; use super::{wheel_metadata_concurrency, WHEEL_METADATA_CONCURRENCY}; + use socket_patch_core::hosted::engine::REDIRECT_CANDIDATE_FILES; use socket_patch_core::patch::redirect::DepOverride; use socket_patch_core::utils::concurrent::API_CONCURRENCY_ENV; + /// The post-write classification reuses an already-made discovery only + /// when it provably describes the project the run left behind, and + /// discovers again otherwise. + #[test] + fn the_post_write_discovery_is_reused_only_when_it_describes_the_written_project() { + use super::{discovery_after_writes, Written}; + use socket_patch_core::hosted::engine::FinalDiscovery; + use socket_patch_core::vex::discover::Discovery; + let prior = Discovery::default(); + let overlaid = FinalDiscovery::Overlaid(Box::default()); + let Some(FinalDiscovery::Overlaid(gate)) = Some(&overlaid) else { + unreachable!() + }; + let same = |got: Option<&Discovery>, want: &Discovery| { + got.is_some_and(|got| std::ptr::eq(got, want)) + }; + let landed = Written::Landed { created: &[] }; + + // Files landed: the gate's discovery over exactly those writes. + assert!(same( + discovery_after_writes(Some(&prior), Some(&overlaid), landed, false), + gate + )); + // ...also when it created only a root config file no listing finds. + for created in [&[".npmrc"][..], &["pnpm-workspace.yaml", ".npmrc"]] { + let written = Written::Landed { created }; + assert!(same( + discovery_after_writes(None, Some(&overlaid), written, false), + gate + )); + } + // A created file a directory listing finds (a new pylock, a rush + // subspace lock): the overlay never listed it, so discover again. + let written = Written::Landed { + created: &[".npmrc", "pylock.toml"], + }; + assert!(discovery_after_writes(Some(&prior), Some(&overlaid), written, false).is_none()); + // Files landed, but the gate counted another origin or discovered + // nothing: scan's pre-write discovery is stale, so discover again. + assert!(discovery_after_writes(Some(&prior), None, landed, false).is_none()); + // The vlt heal touched the installed tree discovery reads. + assert!(discovery_after_writes(Some(&prior), Some(&overlaid), landed, true).is_none()); + + // Nothing written: scan's discovery, else the gate's of the same + // unwritten project. + let reused = discovery_after_writes( + Some(&prior), + Some(&FinalDiscovery::Prior), + Written::Nothing, + false, + ); + assert!(same(reused, &prior)); + assert!(same( + discovery_after_writes(Some(&prior), Some(&overlaid), Written::Nothing, false), + &prior + )); + assert!(same( + discovery_after_writes(None, Some(&overlaid), Written::Nothing, false), + gate + )); + assert!(discovery_after_writes(None, None, Written::Nothing, false).is_none()); + assert!(discovery_after_writes(Some(&prior), None, Written::Nothing, true).is_none()); + + // A dry run left the disk as scan saw it; the gate's discovery + // describes the preview, not the disk. + assert!(same( + discovery_after_writes(Some(&prior), Some(&overlaid), Written::Previewed, false), + &prior + )); + assert!(discovery_after_writes(None, Some(&overlaid), Written::Previewed, false).is_none()); + } + /// The wheel window is a patch-API window, so the documented escape /// hatch has to reach it: an operator behind something that caps /// in-flight requests per client sets `SOCKET_API_CONCURRENCY=1` and diff --git a/crates/socket-patch-cli/src/commands/scan/hosted/vlt.rs b/crates/socket-patch-cli/src/commands/scan/hosted/vlt.rs index 518c8449b..9bb8d9bfe 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted/vlt.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted/vlt.rs @@ -354,6 +354,7 @@ pub(super) async fn heal_after_rewrite( }) .collect(); tally = heal_targets(common, &targets, Expected::Patched).await; + out.touched_install = true; out.warnings.push(serde_json::json!({ "code": REINSTALL_REQUIRED, "detail": reinstall_detail(&tally), diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index c2c7a43bb..c3a79e776 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -951,19 +951,24 @@ pub(super) async fn classify_overlap_takeover(common: &GlobalArgs, cwd: &Path) - // A malformed vendor ledger classifies like a missing one (this path // only feeds takeover warnings; corruption is a hard error on the // write/attest paths). - let redirect = crate::commands::hosted_state_from_lockfiles(common, cwd).await; + let discovery = crate::commands::discover_wiring(common, cwd).await; + let redirect = crate::commands::hosted_state_from_pins( + &socket_patch_core::patch::redirect::upstream::HostedPin::all(&discovery), + ); let vendor = socket_patch_core::vendor::load_state(cwd).await.ok(); - classify_overlap_takeover_with(common, cwd, Some(&redirect), vendor.as_ref()).await + classify_overlap_takeover_with(cwd, Some(&redirect), vendor.as_ref(), &discovery).await } /// [`classify_overlap_takeover`] over already-loaded state (the hosted -/// engine classifies against its post-takeover vendor ledger); still reads -/// the LIVE lockfiles in `cwd`. `None` for either yields no overlap. +/// engine classifies against its post-takeover vendor ledger) and +/// `discovery`, the lockfile discovery of `cwd` as it is now +/// ([`crate::commands::discover_wiring`]). `None` for either state yields +/// no overlap. pub(super) async fn classify_overlap_takeover_with( - common: &GlobalArgs, cwd: &Path, redirect: Option<&socket_patch_core::patch::redirect::RedirectState>, vendor: Option<&VendorState>, + discovery: &socket_patch_core::vex::discover::Discovery, ) -> OverlapTakeover { let mut out = OverlapTakeover::default(); let Some(vendor) = vendor else { @@ -995,8 +1000,7 @@ pub(super) async fn classify_overlap_takeover_with( .entry(canon(key)) .or_insert(record.uuid.as_str()); } - let discovery = crate::commands::discover_wiring(common, cwd).await; - let mut liveness = LedgerLiveness::new(cwd, &discovery, None); + let mut liveness = LedgerLiveness::new(cwd, discovery, None); for purl in overlap { let hosted_live = match redirect_uuid_by_purl.get(&purl) { Some(uuid) => liveness.redirect_record(&purl, uuid).await, diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index 9ae370f3d..583bc50e8 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -924,8 +924,8 @@ pub struct RewriteOptions<'a> { /// discovery, `None` when nothing was discovered or the project may /// have changed since). The attribution gate reuses it instead of /// discovering again when a pass writes nothing and this run's grants - /// name no origin beyond `patch_server_origins` (see - /// [`reusable_prior`]): the project it would discover is then the + /// name no origin that `patch_server_origins` does not already count + /// (see [`reusable_prior`]): the project it would discover is then the /// same, read the same way. pub prior_discovery: Option<&'a crate::vex::discover::Discovery>, } @@ -962,6 +962,40 @@ pub struct Rewritten { /// In memory only: the trust auto-config would write through a /// symlinked `pnpm-workspace.yaml`. pub(crate) workspace_symlinked: bool, + /// The attribution gate's discovery of the project as this rewrite + /// leaves it, when it equals a discovery made with exactly + /// [`RewriteOptions::patch_server_origins`] (this run's grants name no + /// other origin). `None` when the gate discovered nothing (no + /// confirmed candidate) or counted another origin. A caller that + /// writes exactly [`RewriteResult::files`] and + /// [`RewriteResult::binary_files`] and nothing else may use it in place + /// of discovering the written project again (see [`FinalDiscovery`]). + pub final_discovery: Option, +} + +/// Where [`Rewritten::final_discovery`] lives. +#[derive(Debug)] +pub enum FinalDiscovery { + /// The pass wrote nothing, and the gate reused the caller's + /// [`RewriteOptions::prior_discovery`]: still the caller's to read. + Prior, + /// A discovery of the project with the pass's writes overlaid (the + /// project read when the gate ran). A directory listing does not see an + /// overlaid file the disk lacks (see + /// [`DiskSnapshot::overlay`](crate::vendor::lock_inventory::DiskSnapshot::overlay)), + /// so it equals a discovery of the written disk only when every written + /// file already existed or is one no discovery lists for + /// ([`overlay_creation_is_invisible`]). + Overlaid(Box), +} + +/// Whether CREATING `rel` (a write over no existing regular file) leaves a +/// discovery over the overlaid project equal to one over the written disk: +/// the root config files the install-policy auto-configs create +/// ([`NPMRC_REL`], [`PNPM_WORKSPACE_REL`]), which discovery reads, if at +/// all, by path and never finds by listing a directory. +pub fn overlay_creation_is_invisible(rel: &str) -> bool { + rel == NPMRC_REL || rel == PNPM_WORKSPACE_REL } /// The pnpm-workspace.yaml read, classified for the trust auto-config @@ -1112,7 +1146,11 @@ pub async fn rewrite( options.clone(), ) .await; - let (vetoed, lockless) = unattributed_pins( + let Gated { + vetoed, + lockless, + discovery, + } = unattributed_pins( view, &done, &kept, @@ -1124,6 +1162,7 @@ pub async fn rewrite( if vetoed.is_empty() { done.unattributed = unattributed; done.rewrite.warnings.extend(lockless); + done.final_discovery = discovery; return done; } kept.retain(|c| !vetoed.iter().any(|skip| skip.uuid == c.dep.patch_uuid)); @@ -1157,14 +1196,22 @@ async fn unattributed_pins( exempt: &BTreeSet, configured: &[String], prior: Option<&crate::vex::discover::Discovery>, -) -> (Vec, Vec) { +) -> Gated { if done.confirmed.is_empty() { - return (Vec::new(), Vec::new()); + return Gated::default(); } // The management commands' allowlist plus the hosts this run's grants - // name: a pin on either counts, as it will for them. + // name: a pin on either counts, as it will for them. A grant on + // Socket's own server or a configured one adds nothing discovery does + // not already count, so the discovery is then the one `configured` + // alone makes. + let foreign = crate::patch::redirect::upstream::foreign_dep_origins( + candidates.iter().map(|c| &c.dep), + configured, + ); + let same_origins = foreign.is_empty(); let mut origins = configured.to_vec(); - for origin in crate::patch::redirect::upstream::dep_origins(candidates.iter().map(|c| &c.dep)) { + for origin in foreign { if !origins.contains(&origin) { origins.push(origin); } @@ -1181,16 +1228,11 @@ async fn unattributed_pins( for (rel, bytes) in &done.rewrite.binary_files { written.push((rel.as_str(), bytes.as_slice())); } - let fresh; - let discovery = if let Some(prior) = reusable_prior( - prior, - written.is_empty(), - &opts.patch_server_origins, - configured, - ) { - prior + let reused = reusable_prior(prior, written.is_empty(), same_origins); + let fresh = if reused.is_some() { + None } else { - fresh = match view.disk_root() { + Some(match view.disk_root() { None => { let ProjectView::Memory(project) = *view else { unreachable!("only a memory view has no disk root") @@ -1217,8 +1259,12 @@ async fn unattributed_pins( ) .await } - }; - &fresh + }) + }; + let discovery = match (reused, &fresh) { + (Some(prior), _) => prior, + (None, Some(fresh)) => fresh, + (None, None) => unreachable!("a pass reuses the prior discovery or discovers afresh"), }; // The management commands' own view of the result: an attributable // pin, or contested wiring they would refuse around. @@ -1307,20 +1353,43 @@ async fn unattributed_pins( } }) .collect(); - (vetoed, lockless) + let discovery = match (same_origins, fresh) { + (false, _) => None, + (true, None) => Some(FinalDiscovery::Prior), + (true, Some(fresh)) => Some(FinalDiscovery::Overlaid(Box::new(fresh))), + }; + Gated { + vetoed, + lockless, + discovery, + } +} + +/// What [`unattributed_pins`] decided for one pass. +#[derive(Default)] +struct Gated { + /// The confirmed candidates whose pin would be contested wiring. + vetoed: Vec, + /// A [`REDIRECT_PIN_LOCKLESS`] warning per lockless pin. + lockless: Vec, + /// The discovery the verdict read, for [`Rewritten::final_discovery`]. + discovery: Option, } /// The caller's pre-rewrite discovery, when it is exactly what the gate /// would discover: the pass writes nothing (so the project is the one the -/// caller discovered) and the gate's origins (`origins`: `configured` plus -/// the grants' hosts) are the ones the caller discovered with. -fn reusable_prior<'d>( - prior: Option<&'d crate::vex::discover::Discovery>, +/// caller discovered) and the gate's origins (`configured` plus the +/// grants' hosts) count exactly the pins the caller's (`configured` alone) +/// did: `same_origins`, no grant on a host outside Socket's own server and +/// `configured` ([`foreign_dep_origins`]). +/// +/// [`foreign_dep_origins`]: crate::patch::redirect::upstream::foreign_dep_origins +fn reusable_prior( + prior: Option<&crate::vex::discover::Discovery>, nothing_written: bool, - origins: &[String], - configured: &[String], -) -> Option<&'d crate::vex::discover::Discovery> { - prior.filter(|_| nothing_written && origins == configured) + same_origins: bool, +) -> Option<&crate::vex::discover::Discovery> { + prior.filter(|_| nothing_written && same_origins) } /// Warning: a confirmed pin no lockfile records a version for (a lockless @@ -1532,6 +1601,7 @@ async fn rewrite_once( npm_warnings, pnpm_rerun_only, workspace_symlinked, + final_discovery: None, } } @@ -3085,19 +3155,200 @@ mod tests { } /// The gate reuses the caller's discovery only for a pass that writes - /// nothing and whose origins are exactly the ones it was made with. + /// nothing and counts exactly the origins it was made with. #[test] fn the_prior_discovery_is_reused_only_unwritten_with_the_same_origins() { let prior = crate::vex::discover::Discovery::default(); - let configured = vec!["https://patch.test".to_string()]; - let extra = vec![ - "https://patch.test".to_string(), - "https://other.test".to_string(), - ]; - assert!(reusable_prior(Some(&prior), true, &configured, &configured).is_some()); - assert!(reusable_prior(Some(&prior), false, &configured, &configured).is_none()); - assert!(reusable_prior(Some(&prior), true, &extra, &configured).is_none()); - assert!(reusable_prior(None, true, &configured, &configured).is_none()); + assert!(reusable_prior(Some(&prior), true, true).is_some()); + assert!(reusable_prior(Some(&prior), false, true).is_none()); + assert!(reusable_prior(Some(&prior), true, false).is_none()); + assert!(reusable_prior(None, true, true).is_none()); + } + + /// A registry-resolved left-pad: the hosted rewrite redirects it. + const LEFT_PAD_LOCK: &str = r#"{ + "name": "app", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { "name": "app", "dependencies": { "left-pad": "1.3.0" } }, + "node_modules/left-pad": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz", + "integrity": "sha512-UPSTREAM==" + } + } +} +"#; + + /// The left-pad rewrite of the project on disk at `root`, counting + /// `configured` patch servers and handed `prior` as the caller's + /// pre-rewrite discovery. + async fn gated_left_pad_rewrite( + root: &std::path::Path, + configured: &[&str], + prior: Option<&crate::vex::discover::Discovery>, + ) -> Rewritten { + let outer = OuterAllowRemote::default; + let options = RewriteOptions { + dry_run: false, + targets_pipenv_lock: false, + pipenv_major: None, + pipenv_unknown_detail: String::new(), + trust_lockfile_config: true, + npm_allow_remote_config: true, + npm_outer: &outer, + blocking: false, + takeover_uuids: Default::default(), + patch_server_origins: configured.iter().map(|o| o.to_string()).collect(), + prior_discovery: prior, + }; + let view = ProjectView::Disk(root); + let candidates = vec![left_pad_candidate()]; + let read = read_candidate_files(&view, &BTreeSet::new(), &candidates).await; + rewrite( + &view, + read, + &candidates, + BTreeMap::new(), + &BTreeSet::new(), + &[], + options, + ) + .await + } + + /// Lockfile discovery of `root` as the management commands make it. + async fn discover_configured( + root: &std::path::Path, + configured: &[&str], + ) -> crate::vex::discover::Discovery { + crate::vex::discover_patched_refs_with( + root, + &crate::vex::DiscoverOptions { + patch_server_origins: configured.iter().map(|o| o.to_string()).collect(), + }, + ) + .await + } + + /// Write `done`'s files under `root`, as the disk flow does. + fn write_rewrite(root: &std::path::Path, done: &Rewritten) { + for (rel, text) in &done.rewrite.files { + std::fs::write(root.join(rel), text).unwrap(); + } + for (rel, bytes) in &done.rewrite.binary_files { + std::fs::write(root.join(rel), bytes).unwrap(); + } + } + + /// A writing pass hands back the gate's discovery over its overlaid + /// writes, and it is the discovery of the written disk; the caller's + /// prior discovery is never reused for a pass that writes. + #[tokio::test] + async fn a_writing_pass_hands_back_the_discovery_of_its_writes() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("package-lock.json"), LEFT_PAD_LOCK).unwrap(); + let configured = ["https://patch.test"]; + let before = discover_configured(tmp.path(), &configured).await; + let done = gated_left_pad_rewrite(tmp.path(), &configured, Some(&before)).await; + assert!(done.rewrite.files.contains_key("package-lock.json")); + assert_eq!(done.confirmed.len(), 1, "{:?}", done.rewrite.warnings); + let Some(FinalDiscovery::Overlaid(overlaid)) = &done.final_discovery else { + panic!( + "expected the overlaid discovery: {:?}", + done.final_discovery + ); + }; + write_rewrite(tmp.path(), &done); + let after = discover_configured(tmp.path(), &configured).await; + assert_eq!(format!("{overlaid:?}"), format!("{after:?}")); + assert_ne!(format!("{before:?}"), format!("{after:?}")); + assert_eq!( + crate::patch::redirect::upstream::HostedPin::all(overlaid).len(), + 1 + ); + } + + /// A pass that writes nothing (an already-redirected project) reuses + /// the caller's discovery and says so. + #[tokio::test] + async fn an_unwritten_pass_hands_back_the_prior_discovery() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("package-lock.json"), LEFT_PAD_LOCK).unwrap(); + let configured = ["https://patch.test"]; + let first = gated_left_pad_rewrite(tmp.path(), &configured, None).await; + write_rewrite(tmp.path(), &first); + let prior = discover_configured(tmp.path(), &configured).await; + let done = gated_left_pad_rewrite(tmp.path(), &configured, Some(&prior)).await; + assert!( + done.rewrite.files.is_empty(), + "{:?}", + done.rewrite.files.keys() + ); + assert_eq!(done.confirmed.len(), 1); + assert!(matches!(done.final_discovery, Some(FinalDiscovery::Prior))); + // Without a prior discovery, the gate discovers the (unwritten) + // project itself and hands that back. + let done = gated_left_pad_rewrite(tmp.path(), &configured, None).await; + let Some(FinalDiscovery::Overlaid(fresh)) = &done.final_discovery else { + panic!("expected a fresh discovery: {:?}", done.final_discovery); + }; + assert_eq!(format!("{fresh:?}"), format!("{prior:?}")); + } + + /// A grant on a server the caller did not configure makes the gate count + /// another origin: its discovery is not the caller's, so the gate never + /// reuses the prior one and hands back none. + #[tokio::test] + async fn a_foreign_grant_origin_hands_back_no_discovery() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("package-lock.json"), LEFT_PAD_LOCK).unwrap(); + let done = gated_left_pad_rewrite(tmp.path(), &[], None).await; + assert_eq!(done.confirmed.len(), 1, "{:?}", done.rewrite.warnings); + assert!(done.final_discovery.is_none()); + write_rewrite(tmp.path(), &done); + // Unwritten now, with a prior discovery made without the grant's + // origin: still not reused. + let prior = discover_configured(tmp.path(), &[]).await; + let done = gated_left_pad_rewrite(tmp.path(), &[], Some(&prior)).await; + assert!(done.rewrite.files.is_empty()); + assert_eq!(done.confirmed.len(), 1); + assert!(done.final_discovery.is_none()); + } + + /// Nothing confirmed: the gate discovers nothing and hands back none. + #[tokio::test] + async fn an_unconfirmed_rewrite_hands_back_no_discovery() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::write( + tmp.path().join("package-lock.json"), + LEFT_PAD_LOCK.replace("left-pad", "right-pad"), + ) + .unwrap(); + let prior = discover_configured(tmp.path(), &["https://patch.test"]).await; + let done = gated_left_pad_rewrite(tmp.path(), &["https://patch.test"], Some(&prior)).await; + assert!(done.confirmed.is_empty()); + assert!(done.final_discovery.is_none()); + } + + /// Only the install-policy auto-configs' root config files may be + /// created under an overlaid discovery. + #[test] + fn only_root_config_files_are_invisible_overlay_creations() { + assert!(overlay_creation_is_invisible(".npmrc")); + assert!(overlay_creation_is_invisible("pnpm-workspace.yaml")); + for rel in [ + "package-lock.json", + "pylock.toml", + "pylock.dev.toml", + "packages/a/.npmrc", + "common/config/subspaces/a/pnpm-lock.yaml", + "settings.gradle", + "NuGet.Config", + ] { + assert!(!overlay_creation_is_invisible(rel), "{rel}"); + } } /// A lockless NuGet pin (a Socket source mapping, no diff --git a/crates/socket-patch-core/src/hosted/memory/mod.rs b/crates/socket-patch-core/src/hosted/memory/mod.rs index 73eca2877..3e56aaf07 100644 --- a/crates/socket-patch-core/src/hosted/memory/mod.rs +++ b/crates/socket-patch-core/src/hosted/memory/mod.rs @@ -1363,6 +1363,7 @@ mod tests { npm_warnings: Vec::new(), pnpm_rerun_only: false, workspace_symlinked: false, + final_discovery: None, }, } } From 6e8f7f1fd514807394969b9f2336489d70b30ab8 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Wed, 7 Oct 2026 21:21:19 -0400 Subject: [PATCH 10/20] Reuse the post-write discovery across the vlt heal when the store holds The vlt heal can invalidate store entries, and lockfile discovery reads the installed store for one thing only: the bundled copies of the lock's nodes (#471). Falling back to a fresh discovery after every heal kept vlt/hosted ~17% slower than main, and comparing the copies before and after the heal cost a full store walk, as much as the discovery saved. Discovery now records the copies its vlt extractor read (`Discovery::vlt_bundled_copies`, `None` when it did not look), and the heal hands back the copies after it (`healed_store`, which it already walked for its bundled-copy warning). `discovery_after_writes` reuses a discovery after a heal only when the two maps are equal; a discovery that never looked at the store (`None`) is never reused after a heal. The discovery golden renderer skips the field: every copy it lists is already a contest and a diagnostic there. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commands/scan/hosted.rs | 77 ++++++++++++------- .../src/commands/scan/hosted/vlt.rs | 12 ++- .../socket-patch-core/src/vex/discover/mod.rs | 8 ++ .../src/vex/discover/testing/golden.rs | 4 + .../socket-patch-core/src/vex/discover/vlt.rs | 8 ++ 5 files changed, 81 insertions(+), 28 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index f74751e57..6ae4abba3 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -181,10 +181,11 @@ fn acquire_hosted_lock( struct StaleInstallOutcome { warnings: Vec, stale_purls: std::collections::BTreeSet, - /// The probe may have changed the installed tree (the vlt heal - /// invalidates store copies, which lockfile discovery reads for - /// bundled copies); the read-only probes never set it. - touched_install: bool, + /// Set when the vlt heal ran (it may invalidate store entries): the + /// store's bundled copies after it, discovery's only read of the + /// installed tree (`Discovery::vlt_bundled_copies`). The read-only + /// probes never set it. + healed_store: Option>, } /// The `redirect_gem_stale_install` warning for one stale installed @@ -647,9 +648,10 @@ enum Written<'a> { /// fresh discovery of the project as the hosted run left it, or `None` when /// none provably does and the caller must discover again. /// -/// - Anything that touched the installed tree after the rewrite -/// (`touched_install`: the vlt heal) may change what discovery reads, so -/// nothing is reused. +/// - After a vlt heal (`healed_store`: the store's bundled copies after +/// it), only a discovery that saw exactly those copies +/// (`Discovery::vlt_bundled_copies`) is reused: the heal may have removed +/// store entries, and the bundled copies are all discovery reads there. /// - Nothing written, or a dry run: the project is as scan's pre-redirect /// discovery (`prior`) saw it; `prior` is `None` when a takeover changed /// it first. Failing that, when the rewrite planned nothing, the gate's @@ -665,24 +667,24 @@ fn discovery_after_writes<'d>( prior: Option<&'d socket_patch_core::vex::discover::Discovery>, gate: Option<&'d socket_patch_core::hosted::engine::FinalDiscovery>, written: Written<'_>, - touched_install: bool, + healed_store: Option<&std::collections::BTreeMap>, ) -> Option<&'d socket_patch_core::vex::discover::Discovery> { use socket_patch_core::hosted::engine::{overlay_creation_is_invisible, FinalDiscovery}; - if touched_install { - return None; - } let overlaid = match gate { Some(FinalDiscovery::Overlaid(discovery)) => Some(&**discovery), // The gate read `prior` itself (see `engine::rewrite`). Some(FinalDiscovery::Prior) | None => None, }; - match written { + let candidate = match written { Written::Nothing => prior.or(overlaid), Written::Previewed => prior, Written::Landed { created } => { overlaid.filter(|_| created.iter().all(|rel| overlay_creation_is_invisible(rel))) } - } + }; + candidate.filter(|discovery| { + healed_store.is_none_or(|after| discovery.vlt_bundled_copies.as_ref() == Some(after)) + }) } /// The hosted-redirect flow over an ALREADY-SELECTED `(purl, uuid)` set, @@ -1394,7 +1396,7 @@ pub(crate) async fn run_redirect_selected( prior_discovery, done.final_discovery.as_ref(), written, - vlt_stale.touched_install, + vlt_stale.healed_store.as_ref(), ) { Some(discovery) => discovery, None => { @@ -2855,14 +2857,14 @@ mod tests { // Files landed: the gate's discovery over exactly those writes. assert!(same( - discovery_after_writes(Some(&prior), Some(&overlaid), landed, false), + discovery_after_writes(Some(&prior), Some(&overlaid), landed, None), gate )); // ...also when it created only a root config file no listing finds. for created in [&[".npmrc"][..], &["pnpm-workspace.yaml", ".npmrc"]] { let written = Written::Landed { created }; assert!(same( - discovery_after_writes(None, Some(&overlaid), written, false), + discovery_after_writes(None, Some(&overlaid), written, None), gate )); } @@ -2871,12 +2873,31 @@ mod tests { let written = Written::Landed { created: &[".npmrc", "pylock.toml"], }; - assert!(discovery_after_writes(Some(&prior), Some(&overlaid), written, false).is_none()); + assert!(discovery_after_writes(Some(&prior), Some(&overlaid), written, None).is_none()); // Files landed, but the gate counted another origin or discovered // nothing: scan's pre-write discovery is stale, so discover again. - assert!(discovery_after_writes(Some(&prior), None, landed, false).is_none()); - // The vlt heal touched the installed tree discovery reads. - assert!(discovery_after_writes(Some(&prior), Some(&overlaid), landed, true).is_none()); + assert!(discovery_after_writes(Some(&prior), None, landed, None).is_none()); + // After a vlt heal, only a discovery that saw the store's bundled + // copies as the heal left them. + let after: std::collections::BTreeMap = + [("pkg:npm/b@1.0.0".to_string(), "node_modules/.vlt/x".to_string())].into(); + assert!(discovery_after_writes(Some(&prior), Some(&overlaid), landed, Some(&after)).is_none()); + let saw = |copies: &std::collections::BTreeMap| { + FinalDiscovery::Overlaid(Box::new(Discovery { + vlt_bundled_copies: Some(copies.clone()), + ..Discovery::default() + })) + }; + let current = saw(&after); + let Some(FinalDiscovery::Overlaid(current_gate)) = Some(¤t) else { + unreachable!() + }; + assert!(same( + discovery_after_writes(None, Some(¤t), landed, Some(&after)), + current_gate + )); + let stale = saw(&Default::default()); + assert!(discovery_after_writes(None, Some(&stale), landed, Some(&after)).is_none()); // Nothing written: scan's discovery, else the gate's of the same // unwritten project. @@ -2884,27 +2905,29 @@ mod tests { Some(&prior), Some(&FinalDiscovery::Prior), Written::Nothing, - false, + None, ); assert!(same(reused, &prior)); assert!(same( - discovery_after_writes(Some(&prior), Some(&overlaid), Written::Nothing, false), + discovery_after_writes(Some(&prior), Some(&overlaid), Written::Nothing, None), &prior )); assert!(same( - discovery_after_writes(None, Some(&overlaid), Written::Nothing, false), + discovery_after_writes(None, Some(&overlaid), Written::Nothing, None), gate )); - assert!(discovery_after_writes(None, None, Written::Nothing, false).is_none()); - assert!(discovery_after_writes(Some(&prior), None, Written::Nothing, true).is_none()); + assert!(discovery_after_writes(None, None, Written::Nothing, None).is_none()); + assert!( + discovery_after_writes(Some(&prior), None, Written::Nothing, Some(&after)).is_none() + ); // A dry run left the disk as scan saw it; the gate's discovery // describes the preview, not the disk. assert!(same( - discovery_after_writes(Some(&prior), Some(&overlaid), Written::Previewed, false), + discovery_after_writes(Some(&prior), Some(&overlaid), Written::Previewed, None), &prior )); - assert!(discovery_after_writes(None, Some(&overlaid), Written::Previewed, false).is_none()); + assert!(discovery_after_writes(None, Some(&overlaid), Written::Previewed, None).is_none()); } /// The wheel window is a patch-API window, so the documented escape diff --git a/crates/socket-patch-cli/src/commands/scan/hosted/vlt.rs b/crates/socket-patch-cli/src/commands/scan/hosted/vlt.rs index 9bb8d9bfe..102f01442 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted/vlt.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted/vlt.rs @@ -335,6 +335,7 @@ pub(super) async fn heal_after_rewrite( .collect(); let targeted: BTreeSet<&str> = owned.iter().map(|i| i.patch_uuid.as_str()).collect(); let mut tally = HealTally::default(); + let mut healed = false; if !owned.is_empty() { let targets: Vec<(Target<'_>, &str)> = owned .iter() @@ -354,7 +355,7 @@ pub(super) async fn heal_after_rewrite( }) .collect(); tally = heal_targets(common, &targets, Expected::Patched).await; - out.touched_install = true; + healed = true; out.warnings.push(serde_json::json!({ "code": REINSTALL_REQUIRED, "detail": reinstall_detail(&tally), @@ -382,6 +383,9 @@ pub(super) async fn heal_after_rewrite( // in-run attestation (lockfile discovery contests it the same way). if inputs.final_lock.is_some() { let copies = socket_patch_core::vendor::vlt_bundled::bundled_copies(&common.cwd).await; + if healed { + out.healed_store = Some(copies.clone()); + } for purl in inputs.records.keys() { let base = socket_patch_core::utils::purl::strip_purl_qualifiers(purl); let Some(location) = copies.get(base) else { @@ -403,6 +407,12 @@ pub(super) async fn heal_after_rewrite( out.stale_purls.insert(purl.clone()); } } + // A heal implies a final lock, so the block above recorded the store; + // never report a heal without it. + if healed && out.healed_store.is_none() { + out.healed_store = + Some(socket_patch_core::vendor::vlt_bundled::bundled_copies(&common.cwd).await); + } out } diff --git a/crates/socket-patch-core/src/vex/discover/mod.rs b/crates/socket-patch-core/src/vex/discover/mod.rs index 2a47965cc..61d0eed58 100644 --- a/crates/socket-patch-core/src/vex/discover/mod.rs +++ b/crates/socket-patch-core/src/vex/discover/mod.rs @@ -485,6 +485,14 @@ pub struct Discovery { pub unattested: Vec, /// Refs dropped because another lock contests them ([`ContestedRef`]). pub contested: Vec, + /// The bundled copies (purl → root-relative directory) the vlt + /// extractor found in the installed store for the lock's nodes, exactly + /// as [`crate::vendor::vlt_bundled::bundled_copies`] reports them: the + /// only installed-tree input that discovery reads for vlt. `None` when it + /// did not look (no readable `vlt-lock.json`, or no disk). A caller + /// that changed the store since (the vlt heal) can compare this with + /// the store's copies now to tell whether this discovery still holds. + pub vlt_bundled_copies: Option>, } impl Discovery { diff --git a/crates/socket-patch-core/src/vex/discover/testing/golden.rs b/crates/socket-patch-core/src/vex/discover/testing/golden.rs index e3305d728..731d3e2b7 100644 --- a/crates/socket-patch-core/src/vex/discover/testing/golden.rs +++ b/crates/socket-patch-core/src/vex/discover/testing/golden.rs @@ -123,6 +123,10 @@ fn render(out: &Discovery, root: &Path) -> Value { unpatched_copies, unattested, contested, + // Bookkeeping of what the vlt extractor read from the store, not a + // finding: every copy it found already shows as a contest and a + // diagnostic above. + vlt_bundled_copies: _, } = out; let refs: Vec = refs .iter() diff --git a/crates/socket-patch-core/src/vex/discover/vlt.rs b/crates/socket-patch-core/src/vex/discover/vlt.rs index ac109f02e..660bd35c1 100644 --- a/crates/socket-patch-core/src/vex/discover/vlt.rs +++ b/crates/socket-patch-core/src/vex/discover/vlt.rs @@ -258,6 +258,7 @@ async fn contest_bundled_copies(ctx: &DiscoverCtx<'_>, nodes: &[VltLockNode], ou .map(|n| (n.key.as_str(), n.name.as_str())) .collect(); let copies = store_bundled_copies(root, pairs).await; + out.vlt_bundled_copies = Some(copies.clone()); if copies.is_empty() { return; } @@ -1030,6 +1031,13 @@ mod tests { format!(r#"{{"name":"left-pad","version":"{bundled_version}"}}"#), ); let out = p.run(|c, o| Box::pin(super::extract(c, o))).await; + // The store copies discovery read are recorded exactly as + // the vlt heal's store probe reports them. + assert_eq!( + out.vlt_bundled_copies.as_ref(), + Some(&crate::vendor::vlt_bundled::bundled_copies(p.root()).await) + ); + assert_eq!(out.vlt_bundled_copies.as_ref().map(|c| c.len()), Some(1)); if contested { assert_refs(&out, &[]); assert_eq!( From 83dbebea228fb1f1a3c9f2ca2878905fda3772b4 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Wed, 7 Oct 2026 21:49:05 -0400 Subject: [PATCH 11/20] Make each lockfile discovery cheaper and memoize the identity sweep A hosted scan now runs three full discoveries (scan's own, the attribution gate's over the pending rewrite, and the post-write hosted state). Profiling npm/hosted showed the time going to per-discovery CPU, not parsing: the rule-11 identity sweep over every read file (~30%), a quadratic dedup in Discovery::resolved_elsewhere (~20%), and a full url parse of every lock entry's registry url in DiscoverCtx::hosted_uuid. - resolved_elsewhere no longer scans the list on every push; finalize already sorts and dedups it, and the first match contest_across_locks finds is the same either way. - socket_identities is memoized process-wide by (SHA-256 of the swept text, sorted origin set), never by path, so an overlaid or rewritten lock is swept afresh. Texts under 16 KiB skip the memo; 64 entries max. - The sweep finds all of its anchors in one Aho-Corasick pass (none of them overlaps itself, so each pattern's hits equal its match_indices), matches hosts case-insensitively instead of lowercasing a copy of the file, and skips the backslash folds when the text has no backslash. - hosted_uuid answers a plain http(s) url on a host no accepted origin names without parsing it; anything not plainly a DNS name (IPs, %, \, userinfo, xn--, ...) still takes the full parse. A test checks the shortcut against hosted_patch_uuid over tricky spellings. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../socket-patch-core/src/vex/discover/mod.rs | 392 ++++++++++++++++-- 1 file changed, 349 insertions(+), 43 deletions(-) diff --git a/crates/socket-patch-core/src/vex/discover/mod.rs b/crates/socket-patch-core/src/vex/discover/mod.rs index 2a47965cc..a2fa5f623 100644 --- a/crates/socket-patch-core/src/vex/discover/mod.rs +++ b/crates/socket-patch-core/src/vex/discover/mod.rs @@ -617,9 +617,10 @@ impl Discovery { purl: canonical_base_purl(&purl), file: PathBuf::from(file), }; - if !self.elsewhere.contains(&entry) { - self.elsewhere.push(entry); - } + // Deduplicated once, in `finalize` (a per-push scan is quadratic in + // the lock's size); an earlier duplicate is identical, so the first + // match `contest_across_locks` finds is the same either way. + self.elsewhere.push(entry); } /// Record that lock `file`'s entry `key` installs its own copy of `purl` @@ -962,6 +963,8 @@ pub(crate) struct DiscoverCtx<'a> { /// scratch `Discovery` (a file parsed only to explain it) still counts. /// A `Mutex` keeps the ctx `Sync` across the extractors' `.await`s. recognized: Mutex>, + /// The hosts a Socket-hosted url can name (see [`DiscoverCtx::hosted_uuid`]). + hosted_hosts: std::sync::OnceLock>, } impl<'a> DiscoverCtx<'a> { @@ -980,6 +983,7 @@ impl<'a> DiscoverCtx<'a> { view, patch_server_origins, recognized: Mutex::new(BTreeSet::new()), + hosted_hosts: std::sync::OnceLock::new(), } } @@ -1057,7 +1061,27 @@ impl<'a> DiscoverCtx<'a> { /// The patch uuid of a Socket-HOSTED url, or `None` for anything else /// (see [`crate::patch::redirect::hosted_patch_uuid`] for the accepted /// spellings and the host allowlist). + /// + /// A plain url ([`plain_url_domain`]) on any other host is answered + /// without parsing it: every lock entry's registry url comes through + /// here. pub(crate) fn hosted_uuid(&self, url: &str) -> Option { + if let Some(host) = plain_url_domain(url) { + let hosts = self.hosted_hosts.get_or_init(|| { + let mut hosts = + BTreeSet::from([crate::patch::redirect::SOCKET_PATCH_SERVER_HOST.to_string()]); + hosts.extend(self.patch_server_origins.iter().filter_map(|o| { + reqwest::Url::parse(o.trim()) + .ok()? + .host_str() + .map(str::to_string) + })); + hosts + }); + if !hosts.contains(&host) { + return None; + } + } crate::patch::redirect::hosted_patch_uuid(url, self.patch_server_origins) } @@ -1129,6 +1153,53 @@ impl<'a> DiscoverCtx<'a> { // ── identity helpers ───────────────────────────────────────────────────── +/// The host of `url` when it is a PLAIN `http(s)` url — printable ASCII +/// only, no `%`, `\\` or userinfo, a DNS name of letters, digits, `.` and +/// `-` whose last label starts with a letter — lowercased, which is then +/// exactly what [`crate::patch::redirect::hosted_patch_url_uuids`]'s url +/// parse would call its host (no decoding, IDNA mapping or IP-address +/// reading applies to such a name). `None` for anything else, which takes +/// the full parse. +fn plain_url_domain(url: &str) -> Option { + let url = url.trim(); + if !url + .bytes() + .all(|b| b.is_ascii_graphic() && b != b'%' && b != b'\\') + { + return None; + } + let (scheme, rest) = url.split_once("://")?; + // `sparse+https://…` / `registry+https://…`, as the parse strips it. + let scheme = match scheme.rsplit_once('+') { + Some((kind, scheme)) + if !kind.is_empty() && kind.bytes().all(|b| b.is_ascii_alphabetic()) => + { + scheme + } + _ => scheme, + }; + if !scheme.eq_ignore_ascii_case("https") && !scheme.eq_ignore_ascii_case("http") { + return None; + } + let authority = rest.split(['/', '?', '#']).next()?; + let host = match authority.rsplit_once(':') { + Some((host, port)) if port.bytes().all(|b| b.is_ascii_digit()) => host, + Some(_) => return None, + None => authority, + }; + let last = host.strip_suffix('.').unwrap_or(host).rsplit('.').next()?; + let plain = !host.is_empty() + && !rest.starts_with('/') + && host + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'.' || b == b'-') + && last.starts_with(|c: char| c.is_ascii_alphabetic()) + && !host + .split('.') + .any(|label| label.len() >= 4 && label[..4].eq_ignore_ascii_case("xn--")); + plain.then(|| host.to_ascii_lowercase()) +} + /// The uuid of a Socket-owned registry / repository / source NAME: /// `socket-patch-` (cargo `[registries.*]` + Cargo.toml `registry =`, /// maven ``, nuget ``) or, with @@ -1162,8 +1233,66 @@ pub(crate) fn socket_patch_name_uuid(name: &str, vendored: bool) -> Option BTreeSet<(String, WiringMode)> { - socket_identities_inner(text, origins) + IdentityMemo::sweep(text, origins) +} + +/// The process-wide memo behind [`socket_identities`]: the sweep's result +/// keyed by the SHA-256 of the swept text and the (sorted, deduplicated) +/// origin allowlist — never by path, so a pending rewrite overlaid on a +/// path, or the file rewritten on disk, is swept afresh. The sweep is a +/// pure function of exactly that key (the origins are only ever tested +/// with `any`). Small texts are swept directly (hashing them saves +/// nothing), and only the most recent [`IdentityMemo::CAPACITY`] results +/// are kept. +struct IdentityMemo; + +type IdentitySet = BTreeSet<(String, WiringMode)>; +type IdentityKey = ([u8; 32], Vec); + +impl IdentityMemo { + const CAPACITY: usize = 64; + /// Below this many bytes the sweep is cheaper than the hash. + const MIN_LEN: usize = 16 * 1024; + + fn entries() -> &'static Mutex> { + static ENTRIES: std::sync::OnceLock< + Mutex>, + > = std::sync::OnceLock::new(); + ENTRIES.get_or_init(|| Mutex::new(std::collections::VecDeque::new())) + } + + fn sweep(text: &str, origins: &[String]) -> IdentitySet { + if text.len() < Self::MIN_LEN { + return socket_identities_inner(text, origins); + } + use sha2::Digest as _; + let mut origin_key = origins.to_vec(); + origin_key.sort(); + origin_key.dedup(); + let key: IdentityKey = (sha2::Sha256::digest(text.as_bytes()).into(), origin_key); + let lock = || { + Self::entries() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + }; + if let Some((_, found)) = lock().iter().find(|(k, _)| *k == key) { + return found.clone(); + } + let found = socket_identities_inner(text, origins); + let mut entries = lock(); + if !entries.iter().any(|(k, _)| *k == key) { + if entries.len() == Self::CAPACITY { + entries.pop_front(); + } + entries.push_back((key, found.clone())); + } + found + } } /// The hosted patch uuids `text` mentions, under the same recognition @@ -1178,41 +1307,66 @@ pub(crate) fn hosted_uuids_in_text(text: &str, origins: &[String]) -> BTreeSet BTreeSet<(String, WiringMode)> { + use aho_corasick::AhoCorasick; + use std::borrow::Cow; + let mut found = BTreeSet::new(); - let norm = decode_escapes(text) - .replace("\\/", "/") - .replace("\\\\", "/") - .replace('\\', "/"); + let decoded = decode_escapes(text); + // The folds only ever touch a backslash. + let norm: Cow<'_, str> = if decoded.contains('\\') { + Cow::Owned( + decoded + .replace("\\/", "/") + .replace("\\\\", "/") + .replace('\\', "/"), + ) + } else { + decoded + }; let bytes = norm.as_bytes(); - for sep in ['/', '+', '§'] { - let anchor = format!("{VENDOR_DIR}/").replace('/', &sep.to_string()); - for (at, _) in norm.match_indices(anchor.as_str()) { - let rest = &norm[at + anchor.len()..]; - for eco in ECOSYSTEM_DIRS { - let uuid = rest - .strip_prefix(eco) - .and_then(|r| r.strip_prefix(sep)) - .and_then(|r| r.get(..36)) - .filter(|u| is_canonical_uuid(u)); - if let Some(uuid) = uuid { - found.insert((uuid.to_string(), WiringMode::Vendored)); + // Every anchor in one pass. None of them overlaps itself, so each + // pattern's occurrences here are exactly its `match_indices`. + const VENDOR_SEPS: [char; 3] = ['/', '+', '§']; + const GO: usize = VENDOR_SEPS.len(); + static ANCHORS: std::sync::OnceLock<(Vec, AhoCorasick)> = std::sync::OnceLock::new(); + let (anchors, anchor_finder) = ANCHORS.get_or_init(|| { + let mut anchors: Vec = VENDOR_SEPS + .iter() + .map(|sep| format!("{VENDOR_DIR}/").replace('/', &sep.to_string())) + .collect(); + anchors.push(HOSTED_GO_MODULE_PREFIX.to_string()); + let finder = AhoCorasick::new(&anchors).expect("literal anchor patterns"); + (anchors, finder) + }); + for m in anchor_finder.find_overlapping_iter(bytes) { + let (pattern, at) = (m.pattern().as_usize(), m.start()); + if pattern == GO { + let start = m.end(); + let end = token_end(bytes, start, b"@"); + for segment in norm[start..end].split('/') { + if is_canonical_uuid(segment) { + found.insert((segment.to_string(), WiringMode::Hosted)); } } + continue; } - } - - for (at, _) in norm.match_indices(HOSTED_GO_MODULE_PREFIX) { - let start = at + HOSTED_GO_MODULE_PREFIX.len(); - let end = token_end(bytes, start, b"@"); - for segment in norm[start..end].split('/') { - if is_canonical_uuid(segment) { - found.insert((segment.to_string(), WiringMode::Hosted)); + let sep = VENDOR_SEPS[pattern]; + let rest = &norm[at + anchors[pattern].len()..]; + for eco in ECOSYSTEM_DIRS { + let uuid = rest + .strip_prefix(eco) + .and_then(|r| r.strip_prefix(sep)) + .and_then(|r| r.get(..36)) + .filter(|u| is_canonical_uuid(u)); + if let Some(uuid) = uuid { + found.insert((uuid.to_string(), WiringMode::Vendored)); } } } - // Cheap pre-filter: only a token naming an accepted host can be ours. + // Cheap pre-filter: only a token naming an accepted host (any ASCII + // case) can be ours. let mut hosts = vec![crate::patch::redirect::SOCKET_PATCH_SERVER_HOST.to_string()]; hosts.extend(origins.iter().filter_map(|o| { reqwest::Url::parse(o.trim()) @@ -1220,20 +1374,28 @@ fn socket_identities_inner(text: &str, origins: &[String]) -> BTreeSet<(String, .host_str() .map(str::to_ascii_lowercase) })); - let lower = norm.to_ascii_lowercase(); - for separator in ["://", "%3a%2f%2f"] { - for (at, _) in lower.match_indices(separator) { - let Some(start) = scheme_start(bytes, at) else { - continue; - }; - let end = token_end(bytes, at + separator.len(), b""); - if !hosts.iter().any(|h| lower[start..end].contains(h.as_str())) { - continue; - } - let uuids = crate::patch::redirect::hosted_patch_url_uuids(&norm[start..end], origins); - for uuid in uuids.into_iter().flatten() { - found.insert((uuid, WiringMode::Hosted)); - } + let host_finder = AhoCorasick::builder() + .ascii_case_insensitive(true) + .build(&hosts) + .expect("literal host patterns"); + static SEPARATORS: std::sync::OnceLock = std::sync::OnceLock::new(); + let separators = SEPARATORS.get_or_init(|| { + AhoCorasick::builder() + .ascii_case_insensitive(true) + .build(["://", "%3a%2f%2f"]) + .expect("literal separator patterns") + }); + for m in separators.find_overlapping_iter(bytes) { + let Some(start) = scheme_start(bytes, m.start()) else { + continue; + }; + let end = token_end(bytes, m.end(), b""); + if !host_finder.is_match(&bytes[start..end]) { + continue; + } + let uuids = crate::patch::redirect::hosted_patch_url_uuids(&norm[start..end], origins); + for uuid in uuids.into_iter().flatten() { + found.insert((uuid, WiringMode::Hosted)); } } found @@ -3593,6 +3755,150 @@ mod tests { ); } + /// The sweep's memo is keyed by content and origins, never by path or + /// by call: a large text swept twice answers the same, and the same + /// text with one byte changed (a pending rewrite overlaid on the same + /// path) or under other origins is swept afresh. + #[test] + fn socket_identities_memo_is_keyed_by_content_and_origins() { + let a = UUID_A; + let b = UUID_B; + let pad = "x".repeat(IdentityMemo::MIN_LEN); + let lock = |uuid: &str| { + format!( + "{pad}\n\"resolved\": \"http://127.0.0.1:4545/patch/npm/x/1/{TOKEN}/{uuid}/x.tgz\"\n\ + \"file:.socket/vendor/npm/{uuid}/x-1.0.0.tgz\"\n" + ) + }; + let staging = ["http://127.0.0.1:4545".to_string()]; + let uuids = |text: &str, origins: &[String]| { + socket_identities(text, origins) + .into_iter() + .map(|(u, m)| (u, m == WiringMode::Hosted)) + .collect::>() + }; + let all = |uuid: &str| { + BTreeSet::from([ + (uuid.to_string(), false), + (uuid.to_string(), true), + (TOKEN.to_string(), true), + ]) + }; + for _ in 0..2 { + assert_eq!(uuids(&lock(a), &staging), all(a)); + assert_eq!( + uuids(&lock(a), &[]), + BTreeSet::from([(a.to_string(), false)]) + ); + assert_eq!(uuids(&lock(b), &staging), all(b)); + } + // Origins are a set: order and duplicates do not change the key. + let twice = [ + staging[0].clone(), + "https://patch.socket.dev".into(), + staging[0].clone(), + ]; + assert_eq!(uuids(&lock(a), &twice), uuids(&lock(a), &staging)); + assert_eq!( + socket_identities(&lock(a), &staging), + socket_identities_inner(&lock(a), &staging) + ); + } + + /// [`DiscoverCtx::hosted_uuid`]'s plain-url shortcut answers exactly + /// what the full parse does, for plain urls on other hosts and for + /// every spelling it must leave to the parse. + #[test] + fn hosted_uuid_shortcut_agrees_with_the_full_parse() { + let (a, t) = (UUID_A, TOKEN); + let path = format!("/patch/npm/x/1.0.0/{t}/{a}/x-1.0.0.tgz"); + let hosts = [ + "patch.socket.dev", + "PATCH.Socket.DEV", + "patch.socket.dev.", + "patch.socket.dev:443", + "patch.socket.dev:8443", + "patch.socket.dev:", + "registry.npmjs.org", + "registry.npmjs.org:443", + "127.0.0.1:4545", + "127.0.0.1", + "0x7f.0.0.1:4545", + "127.1:4545", + "[::1]:4545", + "staging.example.com:8443", + "Staging.Example.COM:8443", + "staging.example.com", + "xn--stging-bua.example.com", + "user@patch.socket.dev", + "user:pw@registry.npmjs.org", + "patch%2esocket.dev", + "patch.soc\tket.dev", + "patch.socket.dev\\", + "foo.123", + "foo.0x1f", + "", + ]; + let urls: Vec = hosts + .iter() + .flat_map(|host| { + [ + format!("https://{host}{path}"), + format!("http://{host}{path}"), + format!("HTTPS://{host}{path}"), + format!("sparse+https://{host}{path}"), + format!("git+https://{host}{path}"), + format!("https:///{host}{path}"), + format!("https://{host}?q={path}"), + format!(" https://{host}{path} "), + format!("https://{host}"), + ] + }) + .chain([ + format!("https:\\/\\/patch.socket.dev{}", path.replace('/', "\\/")), + format!("https%3A%2F%2Fpatch.socket.dev{}", path.replace('/', "%2F")), + "file:x.tgz".to_string(), + "npm:x@1.0.0".to_string(), + "https://".to_string(), + ]) + .collect(); + for origins in [ + vec![], + vec!["http://127.0.0.1:4545".to_string()], + vec![ + "https://staging.example.com:8443".to_string(), + "http://[::1]:4545".to_string(), + "https://xn--stging-bua.example.com".to_string(), + ], + ] { + let ctx = DiscoverCtx::with_origins(Path::new("/nonexistent"), &origins); + for url in &urls { + assert_eq!( + ctx.hosted_uuid(url), + crate::patch::redirect::hosted_patch_uuid(url, &origins), + "{url} under {origins:?}" + ); + } + assert_eq!( + ctx.hosted_uuid(&format!("https://patch.socket.dev{path}")) + .as_deref(), + Some(a) + ); + } + assert_eq!( + plain_url_domain("https://Registry.NPMJS.org/x"), + Some("registry.npmjs.org".into()) + ); + for not_plain in [ + "https://127.0.0.1/x", + "https://a%2eb.com/x", + "https://u@a.com/x", + "https://xn--a.com/x", + ] { + assert_eq!(plain_url_domain(not_plain), None, "{not_plain}"); + } + } + /// The claim API: a recognized uuid is decided by the refs alone (live /// only for the package — and, vendored, the artifact — a ref wires), /// while an unrecognized one is left to the caller (`None`). From 2f9837f6be7f96e23a0a7f9e5406e0b424dde77c Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Wed, 7 Oct 2026 21:49:05 -0400 Subject: [PATCH 12/20] Walk vlt's store for bundled copies in one blocking task vlt discovery walks node_modules/.vlt for bundled copies on every discovery, and the attribution gate's extra discovery doubled that cost (vlt/hosted +14% in the bench). Each lock node paid a canonicalize (realpath: one getattrlist per path component, root included) plus several tokio::fs round-trips to the blocking pool. The walk now runs as one spawn_blocking with std::fs, and checks that a package directory resolves inside the project by lstat-ing only the components under the root (shared prefixes once): all real directories means the canonical path is the canonical root joined with it; a missing or non-directory component is rejected as before; a symlinked component still goes through canonicalize. vlt/hosted is now ~36% faster than main. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/vendor/vlt_bundled.rs | 174 ++++++++++++++++-- 1 file changed, 159 insertions(+), 15 deletions(-) diff --git a/crates/socket-patch-core/src/vendor/vlt_bundled.rs b/crates/socket-patch-core/src/vendor/vlt_bundled.rs index 22c8d0311..328e69905 100644 --- a/crates/socket-patch-core/src/vendor/vlt_bundled.rs +++ b/crates/socket-patch-core/src/vendor/vlt_bundled.rs @@ -56,15 +56,31 @@ const BUNDLED_WALK_LIMIT: usize = 20_000; /// purl → root-relative directory of the first bundled copy of it found in /// the store entries of `nodes` (`(DepID key, package name)` pairs of -/// `vlt-lock.json`). +/// `vlt-lock.json`). The walk is one blocking task of plain syscalls (a +/// store holds one entry per lock node, each probed several times). pub(crate) async fn store_bundled_copies<'n>( root: &Path, nodes: impl IntoIterator, ) -> BTreeMap { + let root = root.to_path_buf(); + let nodes: Vec<(String, String)> = nodes + .into_iter() + .map(|(key, name)| (key.to_string(), name.to_string())) + .collect(); + tokio::task::spawn_blocking(move || store_bundled_copies_sync(&root, &nodes)) + .await + .unwrap_or_else(|e| match e.try_into_panic() { + Ok(payload) => std::panic::resume_unwind(payload), + Err(e) => panic!("vlt store walk failed: {e}"), + }) +} + +fn store_bundled_copies_sync(root: &Path, nodes: &[(String, String)]) -> BTreeMap { let mut copies = BTreeMap::new(); - let Ok(canonical_root) = tokio::fs::canonicalize(root).await else { + let Ok(canonical_root) = std::fs::canonicalize(root) else { return copies; }; + let mut real_dirs = RealDirs::default(); let mut budget = BUNDLED_WALK_LIMIT; for (key, name) in nodes { // Both come from the lock: never let them climb out of the store. @@ -73,14 +89,18 @@ pub(crate) async fn store_bundled_copies<'n>( } let package = format!("{VLT_STORE_DIR}/{key}/node_modules/{name}"); // The package itself must be a real directory inside the project. - match tokio::fs::canonicalize(root.join(&package)).await { - Ok(real) if real.starts_with(&canonical_root) && real.is_dir() => {} - _ => continue, + match real_dirs.check(root, &package) { + Some(true) => {} + Some(false) => continue, + None => match std::fs::canonicalize(root.join(&package)) { + Ok(real) if real.starts_with(&canonical_root) && real.is_dir() => {} + _ => continue, + }, } let mut pending = vec![format!("{package}/node_modules")]; while let Some(dir) = pending.pop() { - for child in real_package_dirs(root, &dir, &mut budget).await { - if let Some(purl) = installed_purl(root, &child).await { + for child in real_package_dirs(root, &dir, &mut budget) { + if let Some(purl) = installed_purl(root, &child) { copies.entry(purl).or_insert_with(|| child.clone()); } pending.push(format!("{child}/node_modules")); @@ -90,16 +110,60 @@ pub(crate) async fn store_bundled_copies<'n>( copies } +/// The `lstat` answer to "does root-relative `rel` canonicalize to a +/// directory inside the (canonical) root": `Some(true)` when every +/// component is a real directory (the canonical path is then the canonical +/// root joined with `rel`), `Some(false)` when a component is missing or a +/// non-directory (canonicalizing fails, or ends at a non-directory), and +/// `None` when a component is a symbolic link, for the caller's +/// `canonicalize` to follow. Shared prefixes (`node_modules/.vlt`) are +/// probed once. +#[derive(Default)] +struct RealDirs { + seen: std::collections::HashMap>, +} + +impl RealDirs { + fn check(&mut self, root: &Path, rel: &str) -> Option { + let mut prefix = String::with_capacity(rel.len()); + for segment in rel.split('/') { + if !prefix.is_empty() { + prefix.push('/'); + } + prefix.push_str(segment); + let verdict = match self.seen.get(&prefix) { + Some(verdict) => *verdict, + None => { + let verdict = match std::fs::symlink_metadata(root.join(&prefix)) { + Ok(meta) if meta.file_type().is_symlink() => None, + Ok(meta) => Some(meta.is_dir()), + Err(_) => Some(false), + }; + self.seen.insert(prefix.clone(), verdict); + verdict + } + }; + if verdict != Some(true) { + return verdict; + } + } + Some(true) + } +} + /// Root-relative paths of the real (not symlinked) package directories in /// the `node_modules` dir `dir`, one `@scope` level deep. -async fn real_package_dirs(root: &Path, dir: &str, budget: &mut usize) -> Vec { +fn real_package_dirs(root: &Path, dir: &str, budget: &mut usize) -> Vec { let mut found = Vec::new(); let mut pending = vec![(dir.to_string(), true)]; while let Some((dir, scopes)) = pending.pop() { - let Ok(mut entries) = tokio::fs::read_dir(root.join(&dir)).await else { + let Ok(entries) = std::fs::read_dir(root.join(&dir)) else { continue; }; - while let Ok(Some(entry)) = entries.next_entry().await { + for entry in entries { + let Ok(entry) = entry else { + break; + }; if *budget == 0 { return found; } @@ -108,7 +172,7 @@ async fn real_package_dirs(root: &Path, dir: &str, budget: &mut usize) -> Vec Vec Option { - let text = crate::utils::fs::read_regular_to_string(&root.join(dir).join("package.json")) - .await - .ok()?; +fn installed_purl(root: &Path, dir: &str) -> Option { + let text = + crate::utils::fs::read_regular_to_string_sync(&root.join(dir).join("package.json")).ok()?; let manifest: serde_json::Value = serde_json::from_str(&text).ok()?; npm_purl( manifest.get("name")?.as_str()?, @@ -149,3 +212,84 @@ fn is_package_name(name: &str) -> bool { None => is_plain_segment(name), } } + +#[cfg(test)] +mod tests { + use super::*; + + fn write(root: &Path, rel: &str, text: &str) { + let path = root.join(rel); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, text).unwrap(); + } + + /// A bundled copy at `/node_modules/left-pad` of the store + /// entry `key`. + fn store_entry(root: &Path, key: &str) -> String { + let package = format!("{VLT_STORE_DIR}/{key}/node_modules/bund"); + write( + root, + &format!("{package}/package.json"), + r#"{"name":"bund","version":"1.0.0"}"#, + ); + write( + root, + &format!("{package}/node_modules/left-pad/package.json"), + r#"{"name":"left-pad","version":"1.3.0"}"#, + ); + package + } + + async fn copies(root: &Path, key: &str) -> Vec { + store_bundled_copies(root, [(key, "bund")]) + .await + .into_keys() + .collect() + } + + /// The lstat shortcut answers what `canonicalize` did: real store + /// directories are walked; a missing entry, a file where the package + /// directory should be, and anything resolving outside the project are + /// not; a symlinked component that stays inside the project is followed. + #[tokio::test] + async fn the_store_walk_keeps_inside_the_project() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("project"); + store_entry(&root, "real"); + assert_eq!(copies(&root, "real").await, ["pkg:npm/left-pad@1.3.0"]); + assert!(copies(&root, "missing").await.is_empty()); + write( + &root, + &format!("{VLT_STORE_DIR}/file/node_modules/bund"), + "not a dir", + ); + assert!(copies(&root, "file").await.is_empty()); + + #[cfg(unix)] + { + use std::os::unix::fs::symlink; + let outside = tmp.path().join("outside"); + store_entry(&outside, "away"); + // A store entry symlinked inside the project is followed ... + symlink( + root.join(VLT_STORE_DIR).join("real"), + root.join(VLT_STORE_DIR).join("linked"), + ) + .unwrap(); + assert_eq!(copies(&root, "linked").await, ["pkg:npm/left-pad@1.3.0"]); + // ... one resolving outside it is not. + symlink( + outside.join(VLT_STORE_DIR).join("away"), + root.join(VLT_STORE_DIR).join("away"), + ) + .unwrap(); + assert!(copies(&root, "away").await.is_empty()); + // Nor is any entry of a store that is itself a link outside. + let other = tmp.path().join("other"); + store_entry(&other, "real"); + std::fs::create_dir_all(other.join("x")).unwrap(); + symlink(other.join("node_modules"), other.join("x/node_modules")).unwrap(); + assert!(copies(&other.join("x"), "real").await.is_empty()); + } + } +} From 9b0ca610735eb277b0fe6c87d1d0c12771a7a242 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Wed, 7 Oct 2026 21:49:06 -0400 Subject: [PATCH 13/20] Check Gradle's build-level refusals once per discovery, not per pin Gradle discovery re-ran the planner's project refusal (wrapper version, Android/KMP and custom-lockFile lexes of every script) and the lock-path listing for every pinned row, though neither depends on the row, and compiled the wrapper-version regex on each call. PinnedRowChecks works the build-level half out once, on the first row that asks, and the regex is compiled once per process. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/socket-patch-core/src/gradle/graph.rs | 5 +- .../src/patch/redirect/gradle.rs | 59 +++++++++++++------ .../src/vex/discover/gradle.rs | 7 ++- 3 files changed, 48 insertions(+), 23 deletions(-) diff --git a/crates/socket-patch-core/src/gradle/graph.rs b/crates/socket-patch-core/src/gradle/graph.rs index 8b74e690e..f44d6022a 100644 --- a/crates/socket-patch-core/src/gradle/graph.rs +++ b/crates/socket-patch-core/src/gradle/graph.rs @@ -1463,7 +1463,10 @@ pub fn wrapper_version(read: TextReadFn<'_>, root: &str) -> Option<(u32, u32, u3 .unwrap_or(rest); Some(rest.trim().replace('\\', "")) })?; - let re = regex::Regex::new(r"gradle-(\d+)\.(\d+)(?:\.(\d+))?").ok()?; + static RE: std::sync::OnceLock = std::sync::OnceLock::new(); + let re = RE.get_or_init(|| { + regex::Regex::new(r"gradle-(\d+)\.(\d+)(?:\.(\d+))?").expect("valid wrapper regex") + }); let caps = re.captures(&url)?; Some(( caps[1].parse().ok()?, diff --git a/crates/socket-patch-core/src/patch/redirect/gradle.rs b/crates/socket-patch-core/src/patch/redirect/gradle.rs index 6ae1f877f..cbb9e424e 100644 --- a/crates/socket-patch-core/src/patch/redirect/gradle.rs +++ b/crates/socket-patch-core/src/patch/redirect/gradle.rs @@ -655,6 +655,7 @@ pub fn lockfile_paths(graph: &ScriptGraph, files: &BTreeMap) -> // ── the planner ────────────────────────────────────────────────────────── /// A refusal: nothing is written for the dep. +#[derive(Clone)] struct Refusal { code: &'static str, detail: String, @@ -1015,28 +1016,48 @@ fn ga_refusal( None } -/// Why the hosted wiring of `row` no longer holds in the build `files` +/// Why the hosted wiring of a row no longer holds in the build `files` /// holds, by the planner's own build- and GA-level refusals (see /// [`ga_refusal`]): `(code, detail)`. Discovery's re-check of a pin made -/// before the build changed. -pub(crate) fn pinned_row_refusal( - files: &BTreeMap, - graph: &ScriptGraph, - row: &HostedRow, -) -> Option<(&'static str, String)> { - let lock_paths = lockfile_paths(graph, files); - project_refusal(files, graph, &Ok(Vec::new())) - .or_else(|| { - ga_refusal( - files, - graph, - &lock_paths, - &row.group, - &row.artifact, - &row.base, +/// before the build changed. The build-level half (the project refusal and +/// the lock paths) depends on no row, so it is worked out once, on the +/// first row that asks. +pub(crate) struct PinnedRowChecks<'a> { + files: &'a BTreeMap, + graph: &'a ScriptGraph, + build: std::sync::OnceLock<(Vec, Option)>, +} + +impl<'a> PinnedRowChecks<'a> { + pub(crate) fn new(files: &'a BTreeMap, graph: &'a ScriptGraph) -> Self { + Self { + files, + graph, + build: std::sync::OnceLock::new(), + } + } + + pub(crate) fn refusal(&self, row: &HostedRow) -> Option<(&'static str, String)> { + let (lock_paths, project) = self.build.get_or_init(|| { + ( + lockfile_paths(self.graph, self.files), + project_refusal(self.files, self.graph, &Ok(Vec::new())), ) - }) - .map(|r| (r.code, r.detail)) + }); + project + .clone() + .or_else(|| { + ga_refusal( + self.files, + self.graph, + lock_paths, + &row.group, + &row.artifact, + &row.base, + ) + }) + .map(|r| (r.code, r.detail)) + } } /// Whether `version` orders above `base` (Gradle's ordering): a newer diff --git a/crates/socket-patch-core/src/vex/discover/gradle.rs b/crates/socket-patch-core/src/vex/discover/gradle.rs index 95fe73d82..e90a86f76 100644 --- a/crates/socket-patch-core/src/vex/discover/gradle.rs +++ b/crates/socket-patch-core/src/vex/discover/gradle.rs @@ -22,7 +22,7 @@ //! - no build script sets a custom lock-file location (`lockFile`), which //! would hide a lock from the check above; //! - none of the hosted planner's build- or GA-level refusals holds now -//! (`pinned_row_refusal`: a settings-classpath declaration, a +//! (`PinnedRowChecks`: a settings-classpath declaration, a //! non-literal `includeBuild`, an Android / KMP plugin, a classifier //! request, a user `exclusiveContent` claiming the group, …): a build //! changed after the scan in a way the pin cannot reach stops attesting. @@ -56,7 +56,7 @@ use crate::gradle::eol::eol_eq; use crate::gradle::locks; use crate::patch::redirect::gradle::{ apply_line_digest, graph_of, index_digest, is_settings_lock, lockfile_paths, parse_index, - pinned_row_refusal, settings_targets, GradleFiles, HOSTED_INDEX_REL, HOSTED_SCRIPT, + settings_targets, GradleFiles, PinnedRowChecks, HOSTED_INDEX_REL, HOSTED_SCRIPT, HOSTED_SCRIPT_REL, MAX_ROUNDS, }; @@ -111,6 +111,7 @@ pub(crate) async fn extract(ctx: &DiscoverCtx<'_>, out: &mut Discovery) { Some((rel, state)) }) .collect(); + let pinned_checks = PinnedRowChecks::new(&files, &graph); for row in rows { let ga = row.ga(); let problem = wiring.clone().or_else(|| match ctx.hosted_uuid(&row.url) { @@ -168,7 +169,7 @@ pub(crate) async fn extract(ctx: &DiscoverCtx<'_>, out: &mut Discovery) { // `includeBuild` added after the scan resolves the upstream jar // where the pin never reaches. let problem = problem.or_else(|| { - pinned_row_refusal(&files, &graph, &row).map(|(code, detail)| { + pinned_checks.refusal(&row).map(|(code, detail)| { format!("the hosted planner would refuse it now ({code}): {detail}") }) }); From 8bf77dceef280c3bd9c641eeb378d15eb4c67c00 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Wed, 7 Oct 2026 22:17:38 -0400 Subject: [PATCH 14/20] Reuse scan's pre-lock discovery only while what it read is unchanged Scan's discovery is taken before the apply lock, so reusing it inside the attribution gate partly undid #1058's B58 fix: a concurrent run that held the lock and rewrote a lockfile between scan's read and ours left the gate deciding on stale wiring. The context's DiskSnapshot is now tracked: before it first touches a path it fingerprints it (lstat, plus stat of a symlink's target: kind, length, mtime, and dev/ino/ctime on Unix or the creation time elsewhere; a directory keeps only kind and identity), and a directory it lists also records its sorted entry names. `ProjectContext::recorded_discovery` records the paths discovery touched (`DiskSnapshot::begin_recording` / `end_recording`, a `ReadSet`). Under the apply lock, the hosted flow reuses the prior discovery only when `ReadSet::unchanged()` re-stats every path the same (`rollout::Prior::still_current`); otherwise the gate discovers afresh under the lock. Stats and one readdir per listed directory only, no file reads: ~37 paths, ~50 us on the bench fixtures. Soundness is enforced at compile time: `DiskSnapshot::root` is private, and every read that bypasses the view now goes through `root()`, which marks an open recording unusable (`end_recording` returns `None`), or `root_reading(paths)`, which declares exactly what it reads. The bundler manifest probe (every project) declares its app and global config files, and NuGet's same-file probe declares the two config spellings. The vlt store walk and the sbt / Maven / NuGet feed probes still use `root()`, so those projects never reuse the prior discovery and keep main's discovery count. The listing leaves out `.socket/` at the project root: taking the apply lock creates it, no listing consumer selects it, and any read inside it is fingerprinted on its own. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../socket-patch-cli/src/commands/context.rs | 21 +- .../src/commands/scan/hosted.rs | 27 +- .../socket-patch-cli/src/commands/scan/mod.rs | 6 +- .../src/commands/scan/rollout.rs | 74 ++- .../src/crawlers/ruby_crawler.rs | 18 +- crates/socket-patch-core/src/hosted/engine.rs | 58 +-- .../src/hosted/governing_root.rs | 2 +- .../socket-patch-core/src/hosted/sbt_reads.rs | 2 +- crates/socket-patch-core/src/hosted/vlt.rs | 14 +- .../src/utils/cargo_workspace.rs | 4 +- .../src/vendor/lock_inventory/cargo.rs | 7 +- .../src/vendor/lock_inventory/mod.rs | 2 +- .../src/vendor/lock_inventory/pnpm.rs | 9 +- .../src/vendor/lock_inventory/pypi.rs | 4 +- .../src/vendor/lock_inventory/view.rs | 437 +++++++++++++++++- .../socket-patch-core/src/vex/discover/mod.rs | 9 + .../src/vex/discover/nuget.rs | 4 +- 17 files changed, 617 insertions(+), 81 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/context.rs b/crates/socket-patch-cli/src/commands/context.rs index 3ae3b0050..418a58623 100644 --- a/crates/socket-patch-cli/src/commands/context.rs +++ b/crates/socket-patch-cli/src/commands/context.rs @@ -18,7 +18,7 @@ use std::path::PathBuf; use socket_patch_core::ledgers::{Ledgers, LoadedLedgers}; use socket_patch_core::vendor::lock_inventory::{ - DiskSnapshot, LockfileEntry, ProjectView, UnsupportedNpmLayout, + DiskSnapshot, LockfileEntry, ProjectView, ReadSet, UnsupportedNpmLayout, }; use socket_patch_core::vex::discover::Discovery; use tokio::sync::OnceCell; @@ -39,7 +39,10 @@ pub(crate) struct ProjectContext<'a> { snapshot: DiskSnapshot<'a>, ledgers: OnceCell, locks: OnceCell, - discovery: OnceCell, + /// The discovery, with the paths it read and their fingerprints + /// (`None` when they cannot cover what it read; see + /// [`DiskSnapshot::end_recording`]). + discovery: OnceCell<(Discovery, Option)>, } impl<'a> ProjectContext<'a> { @@ -53,7 +56,7 @@ impl<'a> ProjectContext<'a> { Self { common, root, - snapshot: DiskSnapshot::new(&common.cwd), + snapshot: DiskSnapshot::tracked(&common.cwd), ledgers: OnceCell::new(), locks: OnceCell::new(), discovery: OnceCell::new(), @@ -93,8 +96,18 @@ impl<'a> ProjectContext<'a> { /// The lockfile wiring discovery of `--cwd` ([`super::discover_wiring`]). pub(crate) async fn discovery(&self) -> &Discovery { + &self.recorded_discovery().await.0 + } + + /// [`Self::discovery`] with the read set that tells whether it still + /// describes the project (stats only; see [`ReadSet::unchanged`]). + pub(crate) async fn recorded_discovery(&self) -> &(Discovery, Option) { self.discovery - .get_or_init(|| super::discover_wiring_in(self.common, &self.snapshot)) + .get_or_init(|| async { + self.snapshot.begin_recording(); + let discovery = super::discover_wiring_in(self.common, &self.snapshot).await; + (discovery, self.snapshot.end_recording()) + }) .await } } diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 6ae4abba3..499734f94 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -567,7 +567,7 @@ pub(super) async fn run_redirect( batch_failed: bool, stage: &mut super::rollout::Stage, // Scan's pre-redirect lockfile discovery (see `rollout::Gate::prior`). - prior: Option<&socket_patch_core::vex::discover::Discovery>, + prior: Option>, ) -> i32 { // Same discovery/selection as `--apply`/`--vendor`. let discovered = match discover_selected( @@ -1088,15 +1088,17 @@ pub(crate) async fn run_redirect_selected( .collect() }; let patch_server_origins = crate::commands::rollback::patch_server_origins(common); - // Scan's discovery predates this run's writes, and only the takeover's - // revert above changes the project before the rewrite (scan writes - // nothing between its discovery and this call): it still describes the - // project the rewrite reads unless a takeover touched files. It was - // made with `patch_server_origins` (`discover_wiring`). + // Scan's discovery predates this run's writes and the apply lock, so it + // is reused only when nothing changed the project since: no takeover + // reverted files above, and every path it read re-stats the same now, + // under the lock (a concurrent writer that finished before the lock was + // taken shows up here). It was made with `patch_server_origins` + // (`discover_wiring`). let prior_discovery = rollout .as_ref() .and_then(|gate| gate.prior) - .filter(|_| takeover_files.is_empty() && takeover_migrated.is_empty()); + .filter(|_| takeover_files.is_empty() && takeover_migrated.is_empty()) + .and_then(|prior| prior.still_current()); let rewrite_options = || RewriteOptions { dry_run: common.dry_run, targets_pipenv_lock, @@ -2879,9 +2881,14 @@ mod tests { assert!(discovery_after_writes(Some(&prior), None, landed, None).is_none()); // After a vlt heal, only a discovery that saw the store's bundled // copies as the heal left them. - let after: std::collections::BTreeMap = - [("pkg:npm/b@1.0.0".to_string(), "node_modules/.vlt/x".to_string())].into(); - assert!(discovery_after_writes(Some(&prior), Some(&overlaid), landed, Some(&after)).is_none()); + let after: std::collections::BTreeMap = [( + "pkg:npm/b@1.0.0".to_string(), + "node_modules/.vlt/x".to_string(), + )] + .into(); + assert!( + discovery_after_writes(Some(&prior), Some(&overlaid), landed, Some(&after)).is_none() + ); let saw = |copies: &std::collections::BTreeMap| { FinalDiscovery::Overlaid(Box::new(Discovery { vlt_bundled_copies: Some(copies.clone()), diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index c3a79e776..505b89f17 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -1894,7 +1894,11 @@ async fn run_scan( let prior_discovery = if args.common.is_global() { None } else { - Some(ctx.discovery().await) + let (discovery, read_set) = ctx.recorded_discovery().await; + Some(rollout::Prior { + discovery, + read_set: read_set.as_ref(), + }) }; let hosted_state = (!args.common.is_global()) .then(|| crate::commands::hosted_state_from_pins(&hosted_pin_list)); diff --git a/crates/socket-patch-cli/src/commands/scan/rollout.rs b/crates/socket-patch-cli/src/commands/scan/rollout.rs index 6a0833eea..b3751e2be 100644 --- a/crates/socket-patch-cli/src/commands/scan/rollout.rs +++ b/crates/socket-patch-cli/src/commands/scan/rollout.rs @@ -45,7 +45,28 @@ pub(crate) struct Gate<'a> { /// Scan's lockfile discovery of `--cwd`, made before the redirect /// (with the configured patch-server origins): the rewrite's /// attribution gate reuses it when nothing changed the project since. - pub(crate) prior: Option<&'a socket_patch_core::vex::discover::Discovery>, + pub(crate) prior: Option>, +} + +/// Scan's discovery, made BEFORE the apply lock, with the paths it read. +#[derive(Clone, Copy)] +pub(crate) struct Prior<'a> { + pub(crate) discovery: &'a socket_patch_core::vex::discover::Discovery, + /// `None` when the read set cannot cover what discovery read (it read + /// the disk around the snapshot): never reusable. + pub(crate) read_set: Option<&'a socket_patch_core::vendor::lock_inventory::ReadSet>, +} + +impl<'a> Prior<'a> { + /// The discovery, when every path it read still has the fingerprint it + /// had then (stats only). Called under the apply lock, so nothing that + /// takes it can change the project between this check and the gate; a + /// change since the unlocked read sends the gate to a fresh discovery. + pub(crate) fn still_current(&self) -> Option<&'a socket_patch_core::vex::discover::Discovery> { + self.read_set + .filter(|read| read.unchanged()) + .map(|_| self.discovery) + } } impl<'a> Gate<'a> { @@ -58,10 +79,7 @@ impl<'a> Gate<'a> { } /// This gate carrying scan's pre-redirect discovery (see [`Self::prior`]). - pub(crate) fn with_prior( - mut self, - prior: Option<&'a socket_patch_core::vex::discover::Discovery>, - ) -> Self { + pub(crate) fn with_prior(mut self, prior: Option>) -> Self { self.prior = prior; self } @@ -223,6 +241,52 @@ pub(crate) fn human_lines( #[cfg(test)] mod tests { + /// Scan's discovery is taken before the apply lock: the gate reuses it + /// only while every path it read is unchanged, so a lockfile written + /// between scan's discovery and the gate (a concurrent run that held the + /// lock first) sends the gate to a fresh discovery. + #[tokio::test] + async fn the_prior_discovery_is_reused_only_while_the_project_is_unchanged() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + let lock = r#"{"name":"app","lockfileVersion":3,"requires":true,"packages":{"":{"name":"app","dependencies":{"left-pad":"1.3.0"}},"node_modules/left-pad":{"version":"1.3.0","resolved":"https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz","integrity":"sha512-UPSTREAM=="}}}"#; + std::fs::write(root.join("package-lock.json"), lock).unwrap(); + std::fs::write(root.join("package.json"), r#"{"name":"app"}"#).unwrap(); + let common = crate::args::GlobalArgs { + cwd: root.to_path_buf(), + json: true, + ..crate::args::GlobalArgs::default() + }; + let ctx = crate::commands::context::ProjectContext::new(&common); + let (discovery, read_set) = ctx.recorded_discovery().await; + let prior = super::Prior { + discovery, + read_set: read_set.as_ref(), + }; + let read = read_set + .as_ref() + .expect("an npm project's discovery is recorded"); + assert!(!read.is_empty()); + // Unchanged (taking the apply lock creates `.socket/`): reused. + std::fs::create_dir_all(root.join(".socket")).unwrap(); + std::fs::write(root.join(".socket/apply.lock"), "").unwrap(); + assert!(std::ptr::eq(prior.still_current().unwrap(), discovery)); + // A concurrent writer rewrote the lockfile: never reused. + std::fs::write( + root.join("package-lock.json"), + lock.replace("1.3.0.tgz", "1.3.0.tgz?x"), + ) + .unwrap(); + assert!(prior.still_current().is_none()); + // Without a read set (discovery read the disk around the snapshot): + // never reused either. + let unrecorded = super::Prior { + discovery, + read_set: None, + }; + assert!(unrecorded.still_current().is_none()); + } + use super::*; use socket_patch_core::api::types::PatchSearchResult; use socket_patch_core::manifest::schema::PatchManifest; diff --git a/crates/socket-patch-core/src/crawlers/ruby_crawler.rs b/crates/socket-patch-core/src/crawlers/ruby_crawler.rs index e44f8de45..9a3dfa7e2 100644 --- a/crates/socket-patch-core/src/crawlers/ruby_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/ruby_crawler.rs @@ -8,7 +8,7 @@ use crate::utils::fs::{ entry_is_dir, home_dir, is_dir, is_file, list_dir_entries, normalize_lexically, run_blocking, }; use crate::utils::process::{CommandRunner, SystemCommandRunner}; -use crate::vendor::lock_inventory::{DiskSnapshot, ProjectView}; +use crate::vendor::lock_inventory::ProjectView; /// Ruby/RubyGems ecosystem crawler for discovering gems in Bundler vendor /// directories or global gem installation paths. @@ -1254,6 +1254,16 @@ fn expand_tilde(value: &Path, home: Option<&Path>) -> PathBuf { /// [`crate::formats::gem::manifest::classify`] for `root` on disk: the /// manifest bundler loads, reading the ambient `BUNDLE_GEMFILE` / /// `BUNDLE_APP_CONFIG` and the app config file. +/// The config files [`bundler_loaded_manifest`] reads for `root`: the app +/// config and, when there is one, the global config. +fn bundler_config_files(root: &Path) -> Vec { + let app = bundler_app_config_dir(root, std::env::var_os("BUNDLE_APP_CONFIG").as_deref()) + .join("config"); + std::iter::once(app) + .chain(ambient_bundler_global_config_file(root)) + .collect() +} + pub async fn bundler_loaded_manifest(root: &Path) -> crate::formats::gem::manifest::LoadedManifest { bundler_loaded_manifest_with_env( root, @@ -1274,7 +1284,11 @@ pub(crate) async fn bundler_loaded_manifest_in( ) -> crate::formats::gem::manifest::LoadedManifest { use crate::formats::gem::manifest; match view { - ProjectView::Disk(root) | ProjectView::Snapshot(DiskSnapshot { root, .. }) => { + ProjectView::Disk(root) => bundler_loaded_manifest(root).await, + ProjectView::Snapshot(snap) => { + // The only files the probe reads: the app config and the global + // config (the environment it also reads is fixed for the run). + let root = snap.root_reading(bundler_config_files(snap.root_reading::<&Path>([]))); bundler_loaded_manifest(root).await } ProjectView::Memory(_) => { diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index 583bc50e8..2240ea86e 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -256,10 +256,10 @@ pub fn build_candidates( /// Bun's precedence when both lock spellings are present. pub fn bun_lock_present(view: &ProjectView<'_>) -> bool { match view { - ProjectView::Disk(cwd) - | ProjectView::Snapshot(crate::vendor::lock_inventory::DiskSnapshot { - root: cwd, .. - }) => cwd.join("bun.lock").exists(), + ProjectView::Disk(_) | ProjectView::Snapshot(_) => { + let cwd = view.disk_root().expect("a disk view has a root"); + cwd.join("bun.lock").exists() + } ProjectView::Memory(project) => project.contains("bun.lock"), } } @@ -364,10 +364,10 @@ impl CandidateFiles { /// Whether the project is a Rush monorepo (disk: `rush.json` is a file). fn rush_repo(view: &ProjectView<'_>) -> bool { match view { - ProjectView::Disk(cwd) - | ProjectView::Snapshot(crate::vendor::lock_inventory::DiskSnapshot { - root: cwd, .. - }) => cwd.join("rush.json").is_file(), + ProjectView::Disk(_) | ProjectView::Snapshot(_) => { + let cwd = view.disk_root().expect("a disk view has a root"); + cwd.join("rush.json").is_file() + } ProjectView::Memory(project) => project.contains("rush.json"), } } @@ -708,8 +708,8 @@ async fn keep_bundler_loaded_gem_files( .collect(); let loaded = crate::crawlers::ruby_crawler::bundler_loaded_manifest_in(view).await; let mirror = match view { - ProjectView::Disk(root) - | ProjectView::Snapshot(crate::vendor::lock_inventory::DiskSnapshot { root, .. }) => { + ProjectView::Disk(_) | ProjectView::Snapshot(_) => { + let root = view.disk_root().expect("a disk view has a root"); crate::crawlers::ruby_crawler::bundler_source_mirror(root, &sources).await } ProjectView::Memory(_) => { @@ -1003,13 +1003,13 @@ pub fn overlay_creation_is_invisible(rel: &str) -> bool { /// symbolic link (absent to the planner, refused by [`guard`]). fn read_workspace(view: &ProjectView<'_>) -> (std::io::Result>, bool) { match view { - ProjectView::Disk(cwd) - | ProjectView::Snapshot(crate::vendor::lock_inventory::DiskSnapshot { - root: cwd, .. - }) => ( - read_workspace_for_trust(&cwd.join(PNPM_WORKSPACE_REL)), - false, - ), + ProjectView::Disk(_) | ProjectView::Snapshot(_) => { + let cwd = view.disk_root().expect("a disk view has a root"); + ( + read_workspace_for_trust(&cwd.join(PNPM_WORKSPACE_REL)), + false, + ) + } ProjectView::Memory(project) => match project.get(PNPM_WORKSPACE_REL) { None => (Ok(None), false), Some(MemoryEntry::Text(text)) => (Ok(Some(text.to_string())), false), @@ -1036,10 +1036,10 @@ fn read_workspace(view: &ProjectView<'_>) -> (std::io::Result>, b /// [`read_npmrc_for_allow_remote`]). fn read_npmrc(view: &ProjectView<'_>) -> Result, String> { match view { - ProjectView::Disk(cwd) - | ProjectView::Snapshot(crate::vendor::lock_inventory::DiskSnapshot { - root: cwd, .. - }) => read_npmrc_for_allow_remote(&cwd.join(NPMRC_REL)), + ProjectView::Disk(_) | ProjectView::Snapshot(_) => { + let cwd = view.disk_root().expect("a disk view has a root"); + read_npmrc_for_allow_remote(&cwd.join(NPMRC_REL)) + } ProjectView::Memory(project) => match project.get(NPMRC_REL) { None => Ok(None), Some(MemoryEntry::Symlink) => { @@ -1059,10 +1059,10 @@ fn read_npmrc(view: &ProjectView<'_>) -> Result, String> { /// Whether Rush's repo-state file is present (disk: a regular file). fn rush_repo_state_present(view: &ProjectView<'_>) -> bool { match view { - ProjectView::Disk(cwd) - | ProjectView::Snapshot(crate::vendor::lock_inventory::DiskSnapshot { - root: cwd, .. - }) => cwd.join(RUSH_REPO_STATE_REL).is_file(), + ProjectView::Disk(_) | ProjectView::Snapshot(_) => { + let cwd = view.disk_root().expect("a disk view has a root"); + cwd.join(RUSH_REPO_STATE_REL).is_file() + } ProjectView::Memory(project) => project.contains(RUSH_REPO_STATE_REL), } } @@ -1867,10 +1867,10 @@ fn pnpm_trust_user_set_detail(server: &str, file: &str, value: &str) -> String { /// no ancestors. fn governing_workspace(view: &ProjectView<'_>) -> Option { match view { - ProjectView::Disk(cwd) - | ProjectView::Snapshot(crate::vendor::lock_inventory::DiskSnapshot { - root: cwd, .. - }) => governing_workspace_file(cwd), + ProjectView::Disk(_) | ProjectView::Snapshot(_) => { + let cwd = view.disk_root().expect("a disk view has a root"); + governing_workspace_file(cwd) + } ProjectView::Memory(_) => None, } } diff --git a/crates/socket-patch-core/src/hosted/governing_root.rs b/crates/socket-patch-core/src/hosted/governing_root.rs index c646adc4c..b4ab63033 100644 --- a/crates/socket-patch-core/src/hosted/governing_root.rs +++ b/crates/socket-patch-core/src/hosted/governing_root.rs @@ -78,7 +78,7 @@ pub async fn refusal( ) -> Option { let root: &Path = match view { ProjectView::Disk(root) => root, - ProjectView::Snapshot(snap) => snap.root, + ProjectView::Snapshot(snap) => snap.root(), ProjectView::Memory(_) => return None, }; if candidates.iter().any(|c| c.dep.ecosystem == "cargo") { diff --git a/crates/socket-patch-core/src/hosted/sbt_reads.rs b/crates/socket-patch-core/src/hosted/sbt_reads.rs index 68ec2add9..1e4ae1255 100644 --- a/crates/socket-patch-core/src/hosted/sbt_reads.rs +++ b/crates/socket-patch-core/src/hosted/sbt_reads.rs @@ -31,7 +31,7 @@ use crate::vendor::lock_inventory::ProjectView; pub async fn extra_resolution(view: &ProjectView<'_>) -> Option { let root: PathBuf = match view { ProjectView::Disk(root) => root.to_path_buf(), - ProjectView::Snapshot(snap) => snap.root.to_path_buf(), + ProjectView::Snapshot(snap) => snap.root().to_path_buf(), ProjectView::Memory(_) => return None, }; let doc = tokio::task::spawn_blocking(move || sbt_evidence::distill(&root)) diff --git a/crates/socket-patch-core/src/hosted/vlt.rs b/crates/socket-patch-core/src/hosted/vlt.rs index 85f02cf99..554a153fa 100644 --- a/crates/socket-patch-core/src/hosted/vlt.rs +++ b/crates/socket-patch-core/src/hosted/vlt.rs @@ -25,10 +25,8 @@ pub const WITHHELD_REASON: &str = ARTIFACT_UNVERIFIABLE; /// megabytes and is never read into the rewriter's input. pub fn install_state_present(view: &ProjectView<'_>) -> bool { match view { - ProjectView::Disk(cwd) - | ProjectView::Snapshot(crate::vendor::lock_inventory::DiskSnapshot { - root: cwd, .. - }) => { + ProjectView::Disk(_) | ProjectView::Snapshot(_) => { + let cwd = view.disk_root().expect("a disk view has a root"); let is = |rel: &str, dir: bool| { std::fs::symlink_metadata(cwd.join(rel)).is_ok_and(|m| { if dir { @@ -52,10 +50,10 @@ pub fn install_state_present(view: &ProjectView<'_>) -> bool { /// Whether `bun.lockb` is present (disk: `exists`, which follows links). pub(crate) fn bun_lockb_present(view: &ProjectView<'_>) -> bool { match view { - ProjectView::Disk(cwd) - | ProjectView::Snapshot(crate::vendor::lock_inventory::DiskSnapshot { - root: cwd, .. - }) => cwd.join(BUN_LOCKB).exists(), + ProjectView::Disk(_) | ProjectView::Snapshot(_) => { + let cwd = view.disk_root().expect("a disk view has a root"); + cwd.join(BUN_LOCKB).exists() + } ProjectView::Memory(project) => project.contains(BUN_LOCKB), } } diff --git a/crates/socket-patch-core/src/utils/cargo_workspace.rs b/crates/socket-patch-core/src/utils/cargo_workspace.rs index 38749c36a..dc48c7780 100644 --- a/crates/socket-patch-core/src/utils/cargo_workspace.rs +++ b/crates/socket-patch-core/src/utils/cargo_workspace.rs @@ -37,8 +37,8 @@ pub fn member_manifests(root: &Path) -> Vec { /// under it; symbolic links are never directories). pub fn member_manifests_in(view: &ProjectView<'_>) -> Vec { match view { - ProjectView::Disk(root) - | ProjectView::Snapshot(crate::vendor::lock_inventory::DiskSnapshot { root, .. }) => { + ProjectView::Disk(_) | ProjectView::Snapshot(_) => { + let root = view.disk_root().expect("a disk view has a root"); member_manifests(root) } ProjectView::Memory(project) => member_manifests_with(&MemoryTree(project)), diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/cargo.rs b/crates/socket-patch-core/src/vendor/lock_inventory/cargo.rs index 4f4095b5b..464eb889a 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/cargo.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/cargo.rs @@ -41,11 +41,8 @@ pub(super) async fn inventory_cargo_lock_raw_in( view: &ProjectView<'_>, ) -> Option> { let doc: std::sync::Arc = match view { - ProjectView::Disk(project_root) - | ProjectView::Snapshot(crate::vendor::lock_inventory::DiskSnapshot { - root: project_root, - .. - }) => { + ProjectView::Disk(_) | ProjectView::Snapshot(_) => { + let project_root = view.disk_root().expect("a disk view has a root"); crate::vendor::cargo_lock::read_lock(project_root) .await .ok()? diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/mod.rs b/crates/socket-patch-core/src/vendor/lock_inventory/mod.rs index 622a3d3e8..58853484e 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/mod.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/mod.rs @@ -78,7 +78,7 @@ pub(crate) use self::npm::{ pub(crate) use self::npm_family::inventory_npm_lock; pub(crate) use self::pypi::pipfile_lock_entries; pub use self::recover::recover_lock_entry; -pub use self::view::{DiskSnapshot, MemoryEntry, MemoryProject, ProjectView}; +pub use self::view::{DiskSnapshot, MemoryEntry, MemoryProject, ProjectView, ReadSet}; pub use self::wired::wired_vendor_integrity; // The per-format views `inventory_project_diagnosed` unions (and the test diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/pnpm.rs b/crates/socket-patch-core/src/vendor/lock_inventory/pnpm.rs index c89228a69..adb01c4bb 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/pnpm.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/pnpm.rs @@ -85,11 +85,10 @@ fn pnpm_lock_text_inventory(text: &str) -> Option> { /// it comes back empty. pub(super) async fn inventory_rush_pnpm_locks_in(view: &ProjectView<'_>) -> Vec { let project = match view { - ProjectView::Disk(project_root) - | ProjectView::Snapshot(crate::vendor::lock_inventory::DiskSnapshot { - root: project_root, - .. - }) => return inventory_rush_pnpm_locks(project_root).await, + ProjectView::Disk(_) | ProjectView::Snapshot(_) => { + let project_root = view.disk_root().expect("a disk view has a root"); + return inventory_rush_pnpm_locks(project_root).await; + } ProjectView::Memory(project) => *project, }; if !project.contains("rush.json") { diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/pypi.rs b/crates/socket-patch-core/src/vendor/lock_inventory/pypi.rs index b1d154c9a..7503768db 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/pypi.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/pypi.rs @@ -227,8 +227,8 @@ pub(super) async fn inventory_pypi_locks_in(view: &ProjectView<'_>) -> Option) -> std::io::Result> { match view { - ProjectView::Disk(root) - | ProjectView::Snapshot(crate::vendor::lock_inventory::DiskSnapshot { root, .. }) => { + ProjectView::Disk(_) | ProjectView::Snapshot(_) => { + let root = view.disk_root().expect("a disk view has a root"); crate::utils::python_lock::python_lock_paths(root) } ProjectView::Memory(project) => Ok(project diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/view.rs b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs index 8c3ab7b5c..f82eb064b 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/view.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs @@ -193,11 +193,183 @@ type ReadCache = std::collections::HashMap, (io::ErrorK /// directory listings, the disk-only probes) goes to the disk directly. #[derive(Debug)] pub struct DiskSnapshot<'a> { - pub root: &'a Path, + /// Private so that every raw use of the root goes through + /// [`Self::root`], which a [`ReadSet`] recording counts as an + /// unrecordable disk access. + root: &'a Path, reads: std::sync::Mutex, /// Paths [`Self::overlay`] put in place of the disk content (they exist /// in this view even when the disk has no such file yet). overlaid: std::sync::Mutex>, + /// `Some` for a [`Self::tracked`] snapshot. + tracking: Option>, +} + +/// What a [`DiskSnapshot::tracked`] snapshot has seen. +#[derive(Debug, Default)] +struct Tracking { + /// Each root-relative path any access touched, with its fingerprint + /// taken before that first access. + seen: std::collections::HashMap, + /// The open recording window ([`DiskSnapshot::begin_recording`]): the + /// paths touched in it, and whether something read the disk around the + /// view (its fingerprints then cannot cover what was read). + window: Option<(BTreeSet, bool)>, +} + +/// One filesystem entry's identity and version as stats report it: the +/// entry itself (`lstat`) and, for a symbolic link, its target (`stat`). +/// `None` for an entry that does not exist. A directory's own stat is only +/// its kind and identity (its times move whenever any entry in it changes, +/// including socket-patch's own `.socket/` lock); a directory something +/// LISTED also carries its entry names ([`Self::listing`]). +#[derive(Debug, Clone, PartialEq, Eq)] +struct Fingerprint { + entry: Option, + target: Option, + /// The sorted `(name, is_dir)` entries of a listed directory. + listing: Option>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Stat { + kind: u8, + len: u64, + modified: Option, + /// Unix: `(dev, ino, ctime)`, which any write, rename over or + /// metadata change moves; elsewhere the creation time. A directory + /// keeps only `(dev, ino)`. + identity: (u64, u64, i64, i64), +} + +impl Stat { + fn of(m: &std::fs::Metadata) -> Self { + let kind = if m.file_type().is_symlink() { + 2 + } else if m.is_dir() { + 1 + } else { + 0 + }; + #[cfg(unix)] + let identity = { + use std::os::unix::fs::MetadataExt; + (m.dev(), m.ino(), m.ctime(), m.ctime_nsec()) + }; + #[cfg(not(unix))] + let identity = { + let created = m + .created() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .unwrap_or_default(); + ( + 0, + 0, + created.as_secs() as i64, + i64::from(created.subsec_nanos()), + ) + }; + if kind == 1 { + return Stat { + kind, + len: 0, + modified: None, + identity: (identity.0, identity.1, 0, 0), + }; + } + Stat { + kind, + len: m.len(), + modified: m.modified().ok(), + identity, + } + } +} + +/// How an access uses a path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Access { + /// Content read or existence / type probe. + Probe, + /// Directory listing. + List, +} + +impl Fingerprint { + fn of(root: &Path, rel: &str, access: Access) -> Self { + let path = root.join(rel); + let entry = std::fs::symlink_metadata(&path).ok(); + let target = entry + .as_ref() + .filter(|m| m.file_type().is_symlink()) + .and_then(|_| std::fs::metadata(&path).ok()); + Fingerprint { + entry: entry.as_ref().map(Stat::of), + target: target.as_ref().map(Stat::of), + listing: (access == Access::List).then(|| listing(&path, rel.is_empty())), + } + } + + /// Whether `rel` still has this fingerprint. + fn holds(&self, root: &Path, rel: &str) -> bool { + let access = if self.listing.is_some() { + Access::List + } else { + Access::Probe + }; + Fingerprint::of(root, rel, access) == *self + } +} + +/// The sorted `(name, is_dir)` entries of `dir` (empty when unreadable), +/// leaving out socket-patch's own state directory at the project root: +/// taking the apply lock creates it, no listing consumer selects it, and +/// any read inside it is fingerprinted on its own. +fn listing(dir: &Path, at_root: bool) -> Vec<(String, bool)> { + let Ok(entries) = std::fs::read_dir(dir) else { + return Vec::new(); + }; + let mut out: Vec<(String, bool)> = entries + .filter_map(Result::ok) + .map(|e| { + let is_dir = e.file_type().is_ok_and(|t| t.is_dir()); + (e.file_name().to_string_lossy().into_owned(), is_dir) + }) + .filter(|(name, _)| !(at_root && name == crate::constants::SOCKET_DIR)) + .collect(); + out.sort(); + out +} + +/// Every path a [`DiskSnapshot::tracked`] snapshot touched while recording +/// (see [`DiskSnapshot::begin_recording`]), with the fingerprint each had +/// before it was first read. Stats only: checking it never reads a file. +#[derive(Debug, Clone)] +pub struct ReadSet { + root: std::path::PathBuf, + paths: BTreeMap, +} + +impl ReadSet { + /// Whether every recorded path still has the fingerprint it had when it + /// was first read: no file was written, replaced, created or removed and + /// no listed directory gained or lost an entry since. + pub fn unchanged(&self) -> bool { + self.paths + .iter() + .all(|(rel, before)| before.holds(&self.root, rel)) + } + + /// How many paths [`Self::unchanged`] re-stats. + pub fn len(&self) -> usize { + self.paths.len() + } + + /// No path recorded. + pub fn is_empty(&self) -> bool { + self.paths.is_empty() + } } impl<'a> DiskSnapshot<'a> { @@ -206,6 +378,94 @@ impl<'a> DiskSnapshot<'a> { root, reads: std::sync::Mutex::new(std::collections::HashMap::new()), overlaid: std::sync::Mutex::new(std::collections::BTreeSet::new()), + tracking: None, + } + } + + /// A snapshot that fingerprints every path before it first touches it, + /// so a [`ReadSet`] can later tell whether what it read still holds. + pub fn tracked(root: &'a Path) -> Self { + Self { + tracking: Some(std::sync::Mutex::new(Tracking::default())), + ..Self::new(root) + } + } + + /// The project root, for a read the view does not mediate. While a + /// recording is open this makes it unusable ([`Self::end_recording`] + /// returns `None`): the fingerprints cannot cover what such a read sees. + pub fn root(&self) -> &'a Path { + if let Some(tracking) = &self.tracking { + if let Some((_, raw)) = &mut lock_tracking(tracking).window { + *raw = true; + } + } + self.root + } + + /// The project root, for a read the view does not mediate that reads + /// exactly `paths` (root-relative, or absolute for a file outside the + /// project) and nothing else: a recording fingerprints them like the + /// view's own reads, instead of giving up as [`Self::root`] does. + pub fn root_reading>(&self, paths: impl IntoIterator) -> &'a Path { + for path in paths { + self.touch(&path.as_ref().to_string_lossy()); + } + self.root + } + + /// Start recording the paths this (tracked) snapshot's reads touch. + pub fn begin_recording(&self) { + if let Some(tracking) = &self.tracking { + lock_tracking(tracking).window = Some((BTreeSet::new(), false)); + } + } + + /// Stop recording: the paths touched since [`Self::begin_recording`], + /// or `None` when the snapshot is untracked, nothing was recording, or + /// something read the disk around the view meanwhile. + pub fn end_recording(&self) -> Option { + let tracking = self.tracking.as_ref()?; + let mut tracking = lock_tracking(tracking); + let (touched, raw) = tracking.window.take()?; + if raw { + return None; + } + let paths = touched + .into_iter() + .filter_map(|rel| Some((rel.clone(), tracking.seen.get(&rel)?.clone()))) + .collect(); + Some(ReadSet { + root: self.root.to_path_buf(), + paths, + }) + } + + /// Note that `rel` is about to be read or probed (see [`Self::tracked`]). + fn touch(&self, rel: &str) { + self.touch_as(rel, Access::Probe); + } + + /// Note that directory `rel` is about to be listed. + fn touch_listing(&self, rel: &str) { + self.touch_as(rel, Access::List); + } + + fn touch_as(&self, rel: &str, access: Access) { + let Some(tracking) = &self.tracking else { + return; + }; + let mut tracking = lock_tracking(tracking); + let known = tracking + .seen + .get(rel) + .is_some_and(|print| access == Access::Probe || print.listing.is_some()); + if !known { + let print = Fingerprint::of(self.root, rel, access); + tracking.seen.insert(rel.to_string(), print); + } + if let Some((touched, _)) = &mut tracking.window { + touched.insert(rel.to_string()); } } @@ -255,6 +515,7 @@ impl<'a> DiskSnapshot<'a> { } async fn read_bytes(&self, rel: &str) -> io::Result> { + self.touch(rel); if let Some(hit) = self.cached(rel) { return hit.map(|b| b.to_vec()); } @@ -264,6 +525,12 @@ impl<'a> DiskSnapshot<'a> { } } +fn lock_tracking(tracking: &std::sync::Mutex) -> std::sync::MutexGuard<'_, Tracking> { + tracking + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + fn utf8(bytes: Vec) -> io::Result { String::from_utf8(bytes).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) } @@ -282,7 +549,20 @@ impl<'a> ProjectView<'a> { pub fn disk_root(&self) -> Option<&'a Path> { match *self { ProjectView::Disk(root) => Some(root), - ProjectView::Snapshot(snap) => Some(snap.root), + ProjectView::Snapshot(snap) => Some(snap.root()), + ProjectView::Memory(_) => None, + } + } + + /// [`Self::disk_root`] for a read the view does not mediate that reads + /// exactly `paths` (see [`DiskSnapshot::root_reading`]). + pub fn disk_root_reading>( + &self, + paths: impl IntoIterator, + ) -> Option<&'a Path> { + match *self { + ProjectView::Disk(root) => Some(root), + ProjectView::Snapshot(snap) => Some(snap.root_reading(paths)), ProjectView::Memory(_) => None, } } @@ -290,6 +570,10 @@ impl<'a> ProjectView<'a> { /// The root-level Python lock names (sorted; see /// [`crate::utils::python_lock::python_lock_paths`]). pub fn python_lock_paths(&self) -> Vec { + if let ProjectView::Snapshot(snap) = self { + // A listing of the root (names only). + snap.touch_listing(""); + } match self { ProjectView::Disk(root) | ProjectView::Snapshot(DiskSnapshot { root, .. }) => { crate::utils::python_lock::python_lock_paths(root).unwrap_or_default() @@ -307,6 +591,9 @@ impl<'a> ProjectView<'a> { /// FIFO-safe regular-file text read. pub async fn read_text(&self, rel: &str) -> io::Result { + if let ProjectView::Snapshot(snap) = self { + snap.touch(rel); + } match self { ProjectView::Disk(root) => read_regular_to_string(&root.join(rel)).await, ProjectView::Memory(project) => project.read_text(rel), @@ -338,6 +625,9 @@ impl<'a> ProjectView<'a> { /// `metadata` (follows links) succeeds. pub async fn exists(&self, rel: &str) -> bool { + if let ProjectView::Snapshot(snap) = self { + snap.touch(rel); + } match self { ProjectView::Snapshot(snap) if snap.is_overlaid(rel) => true, ProjectView::Disk(root) | ProjectView::Snapshot(DiskSnapshot { root, .. }) => { @@ -349,6 +639,9 @@ impl<'a> ProjectView<'a> { /// `symlink_metadata` (does not follow links) succeeds. pub async fn exists_no_follow(&self, rel: &str) -> bool { + if let ProjectView::Snapshot(snap) = self { + snap.touch(rel); + } match self { ProjectView::Snapshot(snap) if snap.is_overlaid(rel) => true, ProjectView::Disk(root) | ProjectView::Snapshot(DiskSnapshot { root, .. }) => { @@ -360,6 +653,9 @@ impl<'a> ProjectView<'a> { /// A regular file (following links on disk). pub fn is_file(&self, rel: &str) -> bool { + if let ProjectView::Snapshot(snap) = self { + snap.touch(rel); + } match self { ProjectView::Snapshot(snap) if snap.is_overlaid(rel) => true, ProjectView::Disk(root) | ProjectView::Snapshot(DiskSnapshot { root, .. }) => { @@ -374,6 +670,9 @@ impl<'a> ProjectView<'a> { /// The path itself is a symbolic link. pub fn is_symlink(&self, rel: &str) -> bool { + if let ProjectView::Snapshot(snap) = self { + snap.touch(rel); + } match self { ProjectView::Disk(root) | ProjectView::Snapshot(DiskSnapshot { root, .. }) => { std::fs::symlink_metadata(root.join(rel)).is_ok_and(|m| m.file_type().is_symlink()) @@ -384,6 +683,9 @@ impl<'a> ProjectView<'a> { /// The UTF-8-named entries of directory `rel`, sorted by name. pub async fn list_dir(&self, rel: &str) -> io::Result> { + if let ProjectView::Snapshot(snap) = self { + snap.touch_listing(rel); + } match self { ProjectView::Disk(root) | ProjectView::Snapshot(DiskSnapshot { root, .. }) => { let mut dir = tokio::fs::read_dir(root.join(rel)).await?; @@ -420,9 +722,12 @@ pub(crate) async fn detect_npm_lock_flavor_in( view: &ProjectView<'_>, ) -> Result<(NpmLockFlavor, Vec), (&'static str, String)> { let project = match view { - ProjectView::Disk(root) | ProjectView::Snapshot(DiskSnapshot { root, .. }) => { + ProjectView::Disk(root) => { return crate::vendor::npm_flavor::detect_npm_lock_flavor(root).await } + ProjectView::Snapshot(snap) => { + return crate::vendor::npm_flavor::detect_npm_lock_flavor(snap.root()).await + } ProjectView::Memory(project) => *project, }; let exists = |name: &str| project.contains(name); @@ -630,6 +935,132 @@ mod tests { ); } + /// Record what `read` touches through a tracked snapshot of `root`. + async fn recorded(root: &Path, read: F) -> Option + where + F: for<'v> FnOnce( + ProjectView<'v>, + ) -> std::pin::Pin + 'v>>, + { + let snap = DiskSnapshot::tracked(root); + snap.begin_recording(); + read(ProjectView::Snapshot(&snap)).await; + snap.end_recording() + } + + /// The read set holds while nothing changes, and breaks when a file it + /// read is rewritten (same length, in place or replaced), created after + /// a miss, or removed. + #[tokio::test] + async fn a_read_set_notices_a_changed_file() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + std::fs::write(root.join("a.lock"), "one").unwrap(); + fn read( + view: ProjectView<'_>, + ) -> std::pin::Pin + '_>> { + Box::pin(async move { + view.read_text("a.lock").await.unwrap(); + assert!(view.read_text("b.lock").await.is_err()); + }) + } + let set = recorded(root, read).await.expect("recorded"); + assert_eq!(set.len(), 2); + assert!(set.unchanged()); + + // Rewritten in place with the same length. + std::fs::write(root.join("a.lock"), "two").unwrap(); + assert!(!set.unchanged(), "an in-place rewrite"); + // Replaced (rename over) with the same bytes. + let set = recorded(root, read).await.unwrap(); + std::fs::write(root.join("a.tmp"), "two").unwrap(); + std::fs::rename(root.join("a.tmp"), root.join("a.lock")).unwrap(); + assert!(!set.unchanged(), "a replacement"); + // A file that was missing appears. + let set = recorded(root, read).await.unwrap(); + std::fs::write(root.join("b.lock"), "late").unwrap(); + assert!(!set.unchanged(), "a created file"); + // A file it read is removed. + let set = recorded(root, |view| { + Box::pin(async move { + view.read_text("a.lock").await.unwrap(); + }) + }) + .await + .unwrap(); + std::fs::remove_file(root.join("a.lock")).unwrap(); + assert!(!set.unchanged(), "a removed file"); + } + + /// A listed directory breaks the read set when it gains or loses an + /// entry, but not when socket-patch creates its own `.socket/` (the + /// apply lock) at the root; an unlisted directory is only probed. + #[tokio::test] + async fn a_read_set_notices_a_changed_listing() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + std::fs::create_dir(root.join("sub")).unwrap(); + fn list( + view: ProjectView<'_>, + ) -> std::pin::Pin + '_>> { + Box::pin(async move { + view.python_lock_paths(); + view.list_dir("sub").await.unwrap(); + }) + } + let set = recorded(root, list).await.unwrap(); + std::fs::create_dir_all(root.join(".socket")).unwrap(); + std::fs::write(root.join(".socket/apply.lock"), "").unwrap(); + assert!(set.unchanged(), "socket-patch's own state dir"); + std::fs::write(root.join("pylock.toml"), "").unwrap(); + assert!(!set.unchanged(), "a new root entry"); + let set = recorded(root, list).await.unwrap(); + std::fs::write(root.join("sub/x"), "").unwrap(); + assert!(!set.unchanged(), "a new entry in a listed dir"); + // Probed, not listed: a new entry inside does not matter. + let set = recorded(root, |view| { + Box::pin(async move { + assert!(view.exists("sub").await); + }) + }) + .await + .unwrap(); + std::fs::write(root.join("sub/y"), "").unwrap(); + assert!(set.unchanged()); + } + + /// A raw use of the root while recording makes the read set unusable; + /// a declared raw read is fingerprinted instead, and nothing outside + /// the window counts. + #[tokio::test] + async fn a_raw_disk_read_makes_the_read_set_unusable() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + std::fs::write(root.join("a.lock"), "one").unwrap(); + let snap = DiskSnapshot::tracked(root); + let _ = snap.root(); + snap.begin_recording(); + ProjectView::Snapshot(&snap) + .read_text("a.lock") + .await + .unwrap(); + assert!(ProjectView::Snapshot(&snap).disk_root().is_some()); + assert!(snap.end_recording().is_none(), "a raw read in the window"); + + snap.begin_recording(); + let config = root.join("outside.cfg"); + let _ = snap.root_reading([&config]); + let set = snap.end_recording().expect("a declared read"); + assert_eq!(set.len(), 1); + std::fs::write(&config, "x").unwrap(); + assert!(!set.unchanged(), "the declared file appeared"); + + // An untracked snapshot records nothing. + let plain = DiskSnapshot::new(root); + plain.begin_recording(); + assert!(plain.end_recording().is_none()); + } + #[tokio::test] async fn a_snapshot_reads_each_file_once() { let tmp = tempfile::tempdir().unwrap(); diff --git a/crates/socket-patch-core/src/vex/discover/mod.rs b/crates/socket-patch-core/src/vex/discover/mod.rs index 61d0eed58..02c68e21c 100644 --- a/crates/socket-patch-core/src/vex/discover/mod.rs +++ b/crates/socket-patch-core/src/vex/discover/mod.rs @@ -998,6 +998,15 @@ impl<'a> DiscoverCtx<'a> { self.view.disk_root() } + /// [`Self::disk_root`] for a probe that reads exactly `paths` (see + /// [`crate::vendor::lock_inventory::DiskSnapshot::root_reading`]). + pub(crate) fn disk_root_reading>( + &self, + paths: impl IntoIterator, + ) -> Option<&'a Path> { + self.view.disk_root_reading(paths) + } + /// Record every Socket identity `text` (the content of root-relative /// `rel`) mentions — see [`socket_identities`]. fn recognize_text(&self, rel: &str, text: &str) { diff --git a/crates/socket-patch-core/src/vex/discover/nuget.rs b/crates/socket-patch-core/src/vex/discover/nuget.rs index 3f0ea666f..225e97f36 100644 --- a/crates/socket-patch-core/src/vex/discover/nuget.rs +++ b/crates/socket-patch-core/src/vex/discover/nuget.rs @@ -73,8 +73,8 @@ use super::{ Discovery, PatchedRef, UnlockedPin, WiringMode, DIAG_LOCKFILE_UNPARSEABLE, DIAG_REF_INVALID, DIAG_REF_UNATTRIBUTABLE, }; -use crate::vendor::lock_inventory::LockIntegrity; use crate::formats::nuget::{parse_config, NugetConfig}; +use crate::vendor::lock_inventory::LockIntegrity; use crate::vendor::nuget_config::{same_file, CONFIG_NAMES}; use crate::vendor::nuget_feed::{is_plain_nuget_token, nuget_lock_entries, nupkg_leaf}; use crate::vendor::path::VENDOR_DIR; @@ -99,7 +99,7 @@ pub(crate) async fn extract(ctx: &DiscoverCtx<'_>, out: &mut Discovery) { // itself — skipped, so it is not reported under three names. for name in CONFIG_NAMES.iter().filter(|name| **name != cfg_rel) { // In memory every spelling is its own entry. - let same = match ctx.disk_root() { + let same = match ctx.disk_root_reading([*name, cfg_rel]) { Some(root) => same_file(&root.join(name), &root.join(cfg_rel)).await, None => false, }; From 699086022a92875ff2385f5ce8a0c6fc24fe03f1 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Wed, 7 Oct 2026 22:31:25 -0400 Subject: [PATCH 15/20] Show overlaid creations to view listings; reuse the gate's discovery across them gradle/hosted creates three files under `.socket/gradle/`, so the post-write reuse fell back to a fresh discovery and the scenario sat at the gate's edge (+8-9% wall, +11% CPU vs main). The overlay snapshot now shows a file it would CREATE to every read the view mediates: `list_dir` merges the overlaid children (also of a directory the disk lacks), `python_lock_paths` adds overlaid root Python locks, and `exists` / `exists_no_follow` answer for the directories an overlaid file implies. This also lets the attribution gate see a created file that discovery finds by listing, which it previously could not. The gate's overlay snapshot is tracked, and `FinalDiscovery::Overlaid` carries `view_only`: discovery made no read around the view (no `DiskSnapshot::root()`). Such a discovery equals a discovery of the written disk whatever the write created, so `discovery_after_writes` reuses it for any created file. A discovery that did read around the view keeps the `.npmrc` / `pnpm-workspace.yaml` allowlist. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commands/scan/hosted.rs | 63 ++++++-- crates/socket-patch-core/src/hosted/engine.rs | 55 +++++-- .../src/vendor/lock_inventory/view.rs | 153 +++++++++++++++--- 3 files changed, 219 insertions(+), 52 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 499734f94..1ffe39a17 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -657,9 +657,10 @@ enum Written<'a> { /// it first. Failing that, when the rewrite planned nothing, the gate's /// own discovery of the unwritten project. /// - Files written: the gate's discovery over exactly those writes -/// ([`FinalDiscovery::Overlaid`]), when every written path existed before -/// or is one no discovery finds by listing a directory -/// ([`engine::overlay_creation_is_invisible`]). +/// ([`FinalDiscovery::Overlaid`]), when every written path existed +/// before, or discovery read only through the overlaid view (which shows +/// created files too), or each created path is one no read around the +/// view sees ([`engine::overlay_creation_is_invisible`]). /// /// [`FinalDiscovery::Overlaid`]: socket_patch_core::hosted::engine::FinalDiscovery::Overlaid /// [`engine::overlay_creation_is_invisible`]: socket_patch_core::hosted::engine::overlay_creation_is_invisible @@ -670,17 +671,19 @@ fn discovery_after_writes<'d>( healed_store: Option<&std::collections::BTreeMap>, ) -> Option<&'d socket_patch_core::vex::discover::Discovery> { use socket_patch_core::hosted::engine::{overlay_creation_is_invisible, FinalDiscovery}; - let overlaid = match gate { - Some(FinalDiscovery::Overlaid(discovery)) => Some(&**discovery), + let (overlaid, view_only) = match gate { + Some(FinalDiscovery::Overlaid { + discovery, + view_only, + }) => (Some(&**discovery), *view_only), // The gate read `prior` itself (see `engine::rewrite`). - Some(FinalDiscovery::Prior) | None => None, + Some(FinalDiscovery::Prior) | None => (None, false), }; let candidate = match written { Written::Nothing => prior.or(overlaid), Written::Previewed => prior, - Written::Landed { created } => { - overlaid.filter(|_| created.iter().all(|rel| overlay_creation_is_invisible(rel))) - } + Written::Landed { created } => overlaid + .filter(|_| view_only || created.iter().all(|rel| overlay_creation_is_invisible(rel))), }; candidate.filter(|discovery| { healed_store.is_none_or(|after| discovery.vlt_bundled_copies.as_ref() == Some(after)) @@ -2848,8 +2851,14 @@ mod tests { use socket_patch_core::hosted::engine::FinalDiscovery; use socket_patch_core::vex::discover::Discovery; let prior = Discovery::default(); - let overlaid = FinalDiscovery::Overlaid(Box::default()); - let Some(FinalDiscovery::Overlaid(gate)) = Some(&overlaid) else { + let overlaid = FinalDiscovery::Overlaid { + discovery: Box::default(), + view_only: false, + }; + let Some(FinalDiscovery::Overlaid { + discovery: gate, .. + }) = Some(&overlaid) + else { unreachable!() }; let same = |got: Option<&Discovery>, want: &Discovery| { @@ -2870,12 +2879,27 @@ mod tests { gate )); } - // A created file a directory listing finds (a new pylock, a rush - // subspace lock): the overlay never listed it, so discover again. + // Any other created file, when discovery also read around the + // overlaid view (which alone shows created files): discover again. let written = Written::Landed { created: &[".npmrc", "pylock.toml"], }; assert!(discovery_after_writes(Some(&prior), Some(&overlaid), written, None).is_none()); + // ...but a discovery that read only through the view saw it. + let view_only = FinalDiscovery::Overlaid { + discovery: Box::default(), + view_only: true, + }; + let Some(FinalDiscovery::Overlaid { + discovery: seen, .. + }) = Some(&view_only) + else { + unreachable!() + }; + assert!(same( + discovery_after_writes(Some(&prior), Some(&view_only), written, None), + seen + )); // Files landed, but the gate counted another origin or discovered // nothing: scan's pre-write discovery is stale, so discover again. assert!(discovery_after_writes(Some(&prior), None, landed, None).is_none()); @@ -2889,14 +2913,19 @@ mod tests { assert!( discovery_after_writes(Some(&prior), Some(&overlaid), landed, Some(&after)).is_none() ); - let saw = |copies: &std::collections::BTreeMap| { - FinalDiscovery::Overlaid(Box::new(Discovery { + let saw = |copies: &std::collections::BTreeMap| FinalDiscovery::Overlaid { + discovery: Box::new(Discovery { vlt_bundled_copies: Some(copies.clone()), ..Discovery::default() - })) + }), + view_only: false, }; let current = saw(&after); - let Some(FinalDiscovery::Overlaid(current_gate)) = Some(¤t) else { + let Some(FinalDiscovery::Overlaid { + discovery: current_gate, + .. + }) = Some(¤t) + else { unreachable!() }; assert!(same( diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index 2240ea86e..9181a5adb 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -980,13 +980,19 @@ pub enum FinalDiscovery { /// [`RewriteOptions::prior_discovery`]: still the caller's to read. Prior, /// A discovery of the project with the pass's writes overlaid (the - /// project read when the gate ran). A directory listing does not see an - /// overlaid file the disk lacks (see + /// project read when the gate ran). Every read the view mediates sees + /// the overlay, created files included (see /// [`DiskSnapshot::overlay`](crate::vendor::lock_inventory::DiskSnapshot::overlay)), - /// so it equals a discovery of the written disk only when every written - /// file already existed or is one no discovery lists for - /// ([`overlay_creation_is_invisible`]). - Overlaid(Box), + /// so it equals a discovery of the written disk when every written file + /// already existed, or when discovery read nothing around the view + /// (`view_only`), or when each created file is one no such read can + /// see ([`overlay_creation_is_invisible`]). + Overlaid { + discovery: Box, + /// Discovery read the project only through the overlaid view (no + /// raw disk read: the vlt store, sbt evidence, a vendored feed). + view_only: bool, + }, } /// Whether CREATING `rel` (a write over no existing regular file) leaves a @@ -1245,25 +1251,33 @@ async fn unattributed_pins( }; after.insert(rel, entry); } - crate::vex::discover::discover_patched_refs_view(ProjectView::Memory(&after), &opts) - .await + let discovery = crate::vex::discover::discover_patched_refs_view( + ProjectView::Memory(&after), + &opts, + ) + .await; + (discovery, true) } Some(root) => { - let after = crate::vendor::lock_inventory::DiskSnapshot::new(root); + // Tracked only to learn whether discovery read around the + // overlay (see `FinalDiscovery::Overlaid::view_only`). + let after = crate::vendor::lock_inventory::DiskSnapshot::tracked(root); for (rel, bytes) in written { after.overlay(rel, bytes); } - crate::vex::discover::discover_patched_refs_view( + after.begin_recording(); + let discovery = crate::vex::discover::discover_patched_refs_view( ProjectView::Snapshot(&after), &opts, ) - .await + .await; + (discovery, after.end_recording().is_some()) } }) }; let discovery = match (reused, &fresh) { (Some(prior), _) => prior, - (None, Some(fresh)) => fresh, + (None, Some((fresh, _))) => fresh, (None, None) => unreachable!("a pass reuses the prior discovery or discovers afresh"), }; // The management commands' own view of the result: an attributable @@ -1356,7 +1370,10 @@ async fn unattributed_pins( let discovery = match (same_origins, fresh) { (false, _) => None, (true, None) => Some(FinalDiscovery::Prior), - (true, Some(fresh)) => Some(FinalDiscovery::Overlaid(Box::new(fresh))), + (true, Some((fresh, view_only))) => Some(FinalDiscovery::Overlaid { + discovery: Box::new(fresh), + view_only, + }), }; Gated { vetoed, @@ -3254,12 +3271,17 @@ mod tests { let done = gated_left_pad_rewrite(tmp.path(), &configured, Some(&before)).await; assert!(done.rewrite.files.contains_key("package-lock.json")); assert_eq!(done.confirmed.len(), 1, "{:?}", done.rewrite.warnings); - let Some(FinalDiscovery::Overlaid(overlaid)) = &done.final_discovery else { + let Some(FinalDiscovery::Overlaid { + discovery: overlaid, + view_only, + }) = &done.final_discovery + else { panic!( "expected the overlaid discovery: {:?}", done.final_discovery ); }; + assert!(*view_only, "npm discovery reads only through the view"); write_rewrite(tmp.path(), &done); let after = discover_configured(tmp.path(), &configured).await; assert_eq!(format!("{overlaid:?}"), format!("{after:?}")); @@ -3291,7 +3313,10 @@ mod tests { // Without a prior discovery, the gate discovers the (unwritten) // project itself and hands that back. let done = gated_left_pad_rewrite(tmp.path(), &configured, None).await; - let Some(FinalDiscovery::Overlaid(fresh)) = &done.final_discovery else { + let Some(FinalDiscovery::Overlaid { + discovery: fresh, .. + }) = &done.final_discovery + else { panic!("expected a fresh discovery: {:?}", done.final_discovery); }; assert_eq!(format!("{fresh:?}"), format!("{prior:?}")); diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/view.rs b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs index f82eb064b..a357a92ee 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/view.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs @@ -470,14 +470,13 @@ impl<'a> DiskSnapshot<'a> { } /// Read `rel` as `content` instead of the disk's: the project as a - /// pending write would leave it. Content reads and existence probes see - /// the overlay; directory listings (`list_dir`, `python_lock_paths`, - /// rush subspace locks) still list the disk. So a file the write would - /// CREATE that discovery only finds by listing (a new `pylock.*.toml`, a - /// new rush subspace lock) is invisible to a discovery over the overlay, - /// and the hosted attribution gate keeps the rewriters' verdict for its - /// pin. No hosted rewriter creates such a file today; one that does must - /// merge its outputs into the listings first. + /// pending write would leave it. Content reads, existence probes (of + /// the file and of the directories it implies) and the view's + /// directory listings (`list_dir`, `python_lock_paths`) see the + /// overlay, so a file the write would CREATE is there for every read + /// the view mediates. A read around the view ([`Self::root`]) still + /// sees the disk alone; a [`Self::tracked`] recording tells whether + /// one happened. pub fn overlay(&self, rel: &str, content: &[u8]) { self.remember(rel, &Ok(content.to_vec())); self.overlaid @@ -493,6 +492,45 @@ impl<'a> DiskSnapshot<'a> { .contains(rel) } + /// The direct children of directory `dir` (`""` for the root) that the + /// overlaid files imply, as `name -> is_dir`. + fn overlaid_children(&self, dir: &str) -> BTreeMap { + let overlaid = self + .overlaid + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut out = BTreeMap::new(); + for path in overlaid.iter() { + let rest = if dir.is_empty() { + Some(path.as_str()) + } else { + path.strip_prefix(dir).and_then(|r| r.strip_prefix('/')) + }; + let Some(rest) = rest.filter(|r| !r.is_empty()) else { + continue; + }; + match rest.split_once('/') { + Some((name, _)) => { + out.insert(name.to_string(), true); + } + None => { + out.entry(rest.to_string()).or_insert(false); + } + } + } + out + } + + /// `rel` is a directory an overlaid file lives under. + fn is_overlaid_dir(&self, rel: &str) -> bool { + let prefix = format!("{rel}/"); + self.overlaid + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .any(|path| path.starts_with(&prefix)) + } + fn lock(&self) -> std::sync::MutexGuard<'_, ReadCache> { self.reads .lock() @@ -525,6 +563,21 @@ impl<'a> DiskSnapshot<'a> { } } +/// The UTF-8-named entries of directory `dir` on disk, sorted by name. +async fn list_disk_dir(dir: &Path) -> io::Result> { + let mut read = tokio::fs::read_dir(dir).await?; + let mut out = Vec::new(); + while let Ok(Some(entry)) = read.next_entry().await { + let Some(name) = entry.file_name().to_str().map(str::to_string) else { + continue; + }; + let is_dir = entry.file_type().await.is_ok_and(|t| t.is_dir()); + out.push(DirEntryInfo { name, is_dir }); + } + out.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(out) +} + fn lock_tracking(tracking: &std::sync::Mutex) -> std::sync::MutexGuard<'_, Tracking> { tracking .lock() @@ -575,9 +628,24 @@ impl<'a> ProjectView<'a> { snap.touch_listing(""); } match self { - ProjectView::Disk(root) | ProjectView::Snapshot(DiskSnapshot { root, .. }) => { + ProjectView::Disk(root) => { crate::utils::python_lock::python_lock_paths(root).unwrap_or_default() } + ProjectView::Snapshot(snap) => { + let mut names = + crate::utils::python_lock::python_lock_paths(snap.root).unwrap_or_default(); + names.extend( + snap.overlaid_children("") + .into_iter() + .filter(|(name, is_dir)| { + !is_dir && crate::utils::python_lock::is_python_lock_name(name) + }) + .map(|(name, _)| name), + ); + names.sort(); + names.dedup(); + names + } ProjectView::Memory(project) => project .children("") .into_iter() @@ -629,7 +697,9 @@ impl<'a> ProjectView<'a> { snap.touch(rel); } match self { - ProjectView::Snapshot(snap) if snap.is_overlaid(rel) => true, + ProjectView::Snapshot(snap) if snap.is_overlaid(rel) || snap.is_overlaid_dir(rel) => { + true + } ProjectView::Disk(root) | ProjectView::Snapshot(DiskSnapshot { root, .. }) => { tokio::fs::metadata(root.join(rel)).await.is_ok() } @@ -643,7 +713,9 @@ impl<'a> ProjectView<'a> { snap.touch(rel); } match self { - ProjectView::Snapshot(snap) if snap.is_overlaid(rel) => true, + ProjectView::Snapshot(snap) if snap.is_overlaid(rel) || snap.is_overlaid_dir(rel) => { + true + } ProjectView::Disk(root) | ProjectView::Snapshot(DiskSnapshot { root, .. }) => { tokio::fs::symlink_metadata(root.join(rel)).await.is_ok() } @@ -687,15 +759,20 @@ impl<'a> ProjectView<'a> { snap.touch_listing(rel); } match self { - ProjectView::Disk(root) | ProjectView::Snapshot(DiskSnapshot { root, .. }) => { - let mut dir = tokio::fs::read_dir(root.join(rel)).await?; - let mut out = Vec::new(); - while let Ok(Some(entry)) = dir.next_entry().await { - let Some(name) = entry.file_name().to_str().map(str::to_string) else { - continue; - }; - let is_dir = entry.file_type().await.is_ok_and(|t| t.is_dir()); - out.push(DirEntryInfo { name, is_dir }); + ProjectView::Disk(root) => list_disk_dir(&root.join(rel)).await, + ProjectView::Snapshot(snap) => { + let created = snap.overlaid_children(rel); + let mut out = match list_disk_dir(&snap.root.join(rel)).await { + Ok(out) => out, + Err(e) if e.kind() == io::ErrorKind::NotFound && !created.is_empty() => { + Vec::new() + } + Err(e) => return Err(e), + }; + for (name, is_dir) in created { + if !out.iter().any(|e| e.name == name) { + out.push(DirEntryInfo { name, is_dir }); + } } out.sort_by(|a, b| a.name.cmp(&b.name)); Ok(out) @@ -1061,6 +1138,42 @@ mod tests { assert!(plain.end_recording().is_none()); } + /// A file an overlay CREATES is there for every read the view + /// mediates: listings (also of a directory the disk lacks), directory + /// probes and the root Python lock names. + #[tokio::test] + async fn an_overlaid_creation_is_listed() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + std::fs::create_dir(root.join("sub")).unwrap(); + std::fs::write(root.join("sub/old"), "").unwrap(); + let snap = DiskSnapshot::new(root); + snap.overlay("sub/new", b"x"); + snap.overlay("sub/old", b"y"); + snap.overlay("gone/deep/file", b"z"); + snap.overlay("pylock.toml", b""); + let view = ProjectView::Snapshot(&snap); + let names = |entries: Vec| -> Vec<(String, bool)> { + entries.into_iter().map(|e| (e.name, e.is_dir)).collect() + }; + assert_eq!( + names(view.list_dir("sub").await.unwrap()), + [("new".to_string(), false), ("old".to_string(), false)] + ); + assert_eq!( + names(view.list_dir("gone").await.unwrap()), + [("deep".to_string(), true)] + ); + assert!(view.list_dir("absent").await.is_err()); + assert!(view.exists("gone/deep").await); + assert!(view.exists_no_follow("gone").await); + assert!(!view.exists("gon").await); + assert_eq!(view.python_lock_paths(), ["pylock.toml"]); + let root_names = names(view.list_dir("").await.unwrap()); + assert!(root_names.contains(&("gone".to_string(), true))); + assert!(root_names.contains(&("pylock.toml".to_string(), false))); + } + #[tokio::test] async fn a_snapshot_reads_each_file_once() { let tmp = tempfile::tempdir().unwrap(); From 08eb0373cb8b11bf8a0f6af79f9ff60bb2de3d13 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Wed, 7 Oct 2026 22:38:56 -0400 Subject: [PATCH 16/20] Never open non-regular NuGet configs on Windows The Windows same-file check opens both config spellings to compare volume serial and file index. A pipe, device or link planted under nuget.config / NuGet.Config could block that open indefinitely and hang discovery and the attribution gate. lstat both paths first and compare only two regular, non-reparse-point files; anything else is not the same file, which at worst recognizes one more spelling as ignored. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/vendor/nuget_config.rs | 56 +++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/crates/socket-patch-core/src/vendor/nuget_config.rs b/crates/socket-patch-core/src/vendor/nuget_config.rs index 3481a13e8..460530741 100644 --- a/crates/socket-patch-core/src/vendor/nuget_config.rs +++ b/crates/socket-patch-core/src/vendor/nuget_config.rs @@ -10,8 +10,11 @@ pub(crate) const CONFIG_NAMES: [&str; 3] = ["nuget.config", "NuGet.config", "NuG /// Whether `a` and `b` are one file (a case-insensitive filesystem's two /// spellings of it). Unix compares device + inode (stat only: never opens /// a FIFO planted under a config name); Windows compares volume serial + -/// file index; elsewhere `false` (the worst case is a duplicate file name -/// in recognition, never a lost one). +/// file index, which needs a handle, so it first refuses (answers `false` +/// for) anything but two regular, non-reparse-point files: opening a +/// device, pipe or link planted under a config name could block discovery +/// indefinitely. Elsewhere `false`. The worst case of a `false` is a +/// duplicate file name in recognition, never a lost one. pub(crate) async fn same_file(a: &std::path::Path, b: &std::path::Path) -> bool { #[cfg(unix)] { @@ -24,9 +27,11 @@ pub(crate) async fn same_file(a: &std::path::Path, b: &std::path::Path) -> bool #[cfg(windows)] { let (a, b) = (a.to_path_buf(), b.to_path_buf()); - tokio::task::spawn_blocking(move || same_file::is_same_file(a, b).unwrap_or(false)) - .await - .unwrap_or(false) + tokio::task::spawn_blocking(move || { + regular_file(&a) && regular_file(&b) && same_file::is_same_file(a, b).unwrap_or(false) + }) + .await + .unwrap_or(false) } #[cfg(not(any(unix, windows)))] { @@ -34,3 +39,44 @@ pub(crate) async fn same_file(a: &std::path::Path, b: &std::path::Path) -> bool false } } + +/// `path` is a regular file itself (`lstat`: a symlink or junction to one +/// is not), so opening it cannot reach a pipe or device. +#[cfg(windows)] +fn regular_file(path: &std::path::Path) -> bool { + std::fs::symlink_metadata(path).is_ok_and(|meta| meta.file_type().is_file()) +} + +#[cfg(test)] +mod tests { + use super::same_file; + + /// Two names of one regular file are one file; distinct files are not. + #[tokio::test] + async fn two_names_of_one_regular_file_are_the_same_file() { + let tmp = tempfile::tempdir().unwrap(); + let (a, b, c) = ( + tmp.path().join("nuget.config"), + tmp.path().join("NuGet.Config.link"), + tmp.path().join("other.config"), + ); + std::fs::write(&a, "").unwrap(); + std::fs::hard_link(&a, &b).unwrap(); + std::fs::write(&c, "").unwrap(); + assert_eq!(same_file(&a, &b).await, cfg!(any(unix, windows))); + assert!(!same_file(&a, &c).await); + assert!(!same_file(&a, &tmp.path().join("missing")).await); + } + + /// Windows opens a handle to compare identities, so it compares only + /// regular files: a directory (like a pipe, a device or a link planted + /// under a config name) is never opened and never the same file. Unix + /// compares by `stat` alone and needs no such guard. + #[tokio::test] + async fn windows_never_opens_a_non_regular_file() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("nuget.config"); + std::fs::create_dir(&dir).unwrap(); + assert_eq!(same_file(&dir, &dir).await, cfg!(unix)); + } +} From 0bf0c3f057f7029e499d71a1f719f6fa199d62ac Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 8 Oct 2026 02:48:28 -0400 Subject: [PATCH 17/20] Look up vlt bundled copies by canonical purl in the scan warning The vlt store's bundled copies are keyed by decoded purls (pkg:npm/@scope/x@v). The attribution gate looks them up with canonical_base_purl, but the scan's bundled-instance warning used only strip_purl_qualifiers, so a record spelled pkg:npm/%40scope/x@v missed its warning and stayed attestable. Use canonical_base_purl there too, and test that every spelling maps to the store's key. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commands/scan/hosted/vlt.rs | 6 ++-- .../src/vendor/vlt_bundled.rs | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/hosted/vlt.rs b/crates/socket-patch-cli/src/commands/scan/hosted/vlt.rs index 102f01442..d2ce28232 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted/vlt.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted/vlt.rs @@ -387,8 +387,10 @@ pub(super) async fn heal_after_rewrite( out.healed_store = Some(copies.clone()); } for purl in inputs.records.keys() { - let base = socket_patch_core::utils::purl::strip_purl_qualifiers(purl); - let Some(location) = copies.get(base) else { + // The store's keys are decoded purls: look a `%40scope` record + // up the way the attribution gate does. + let base = socket_patch_core::vex::discover::canonical_base_purl(purl); + let Some(location) = copies.get(&base) else { continue; }; let Some((name, version)) = base diff --git a/crates/socket-patch-core/src/vendor/vlt_bundled.rs b/crates/socket-patch-core/src/vendor/vlt_bundled.rs index 328e69905..23482959e 100644 --- a/crates/socket-patch-core/src/vendor/vlt_bundled.rs +++ b/crates/socket-patch-core/src/vendor/vlt_bundled.rs @@ -240,6 +240,35 @@ mod tests { package } + /// The store's keys are what `canonical_base_purl` makes of any + /// spelling of the purl (the attribution gate and the scan's bundled + /// warning both look them up that way). + #[tokio::test] + async fn store_keys_are_canonical_base_purls() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + let package = format!("{VLT_STORE_DIR}/key/node_modules/bund"); + write( + root, + &format!("{package}/package.json"), + r#"{"name":"bund","version":"1.0.0"}"#, + ); + write( + root, + &format!("{package}/node_modules/@scope/x/package.json"), + r#"{"name":"@scope/x","version":"2.0.0"}"#, + ); + let keys = store_bundled_copies(root, [("key", "bund")]).await; + for spelling in [ + "pkg:npm/@scope/x@2.0.0", + "pkg:npm/%40scope/x@2.0.0", + "pkg:npm/%40scope/x@2.0.0?vcs_url=x", + ] { + let key = crate::vex::discover::canonical_base_purl(spelling); + assert!(keys.contains_key(&key), "{spelling} -> {key}: {keys:?}"); + } + } + async fn copies(root: &Path, key: &str) -> Vec { store_bundled_copies(root, [(key, "bund")]) .await From 5c8f29d5f32465b589d72c93d5f1a12f62262cde Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 8 Oct 2026 03:24:00 -0400 Subject: [PATCH 18/20] Compare racily-current files by content in the read set Filesystems stamp times in ticks (about 16 ms on Windows, jiffies on Linux, a second on HFS+), so a same-length rewrite inside one tick of the write before it keeps every stat. A file written just before scan's discovery could then be rewritten before the apply lock without the read set noticing, and the attribution gate would reuse a stale discovery. The Windows CI leg caught it in the read-set test. As git does for racily clean files, a file modified within two seconds of its fingerprint now also holds only while its content is still what the view read (from the snapshot's read cache). A racy file the view only probed has no content to compare and never holds. Older files are still judged by stats alone. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/vendor/lock_inventory/view.rs | 116 +++++++++++++++++- 1 file changed, 111 insertions(+), 5 deletions(-) diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/view.rs b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs index 4529a901f..5d150ebdb 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/view.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs @@ -214,6 +214,8 @@ struct Tracking { /// Each root-relative path any access touched, with its fingerprint /// taken before that first access. seen: std::collections::HashMap, + /// When each fingerprint in [`Self::seen`] was taken. + taken: std::collections::HashMap, /// The open recording window ([`DiskSnapshot::begin_recording`]): the /// paths touched in it, and whether something read the disk around the /// view (its fingerprints then cannot cover what was read). @@ -347,11 +349,34 @@ fn listing(dir: &Path, at_root: bool) -> Vec<(String, bool)> { /// Every path a [`DiskSnapshot::tracked`] snapshot touched while recording /// (see [`DiskSnapshot::begin_recording`]), with the fingerprint each had -/// before it was first read. Stats only: checking it never reads a file. +/// before it was first read. Stats only, except for a racily-current file +/// (see [`RACY_WINDOW`]), whose content is compared too. #[derive(Debug, Clone)] pub struct ReadSet { root: std::path::PathBuf, paths: BTreeMap, + /// The files modified within [`RACY_WINDOW`] of their fingerprint: a + /// later write in the same timestamp tick leaves their stats unchanged, + /// so they hold only while their content is still the content the view + /// read (`None`: no content the view read to compare, never holds). + racy: BTreeMap>>, +} + +/// How close to its fingerprint a file's modification time may be before +/// the stats alone cannot vouch for it (git's "racily clean" files). +/// Filesystems stamp times in ticks: about 16 ms on Windows, jiffies on +/// Linux, a second on HFS+ and two on FAT, so a rewrite of the same length +/// inside one tick of the write before it keeps every stat. +const RACY_WINDOW: std::time::Duration = std::time::Duration::from_secs(2); + +/// Whether a file stamped `modified` may be rewritten unseen after a +/// fingerprint taken at `taken`. +fn racy(modified: Option, taken: std::time::SystemTime) -> bool { + modified.is_none_or(|modified| { + taken + .checked_sub(RACY_WINDOW) + .is_none_or(|horizon| modified >= horizon) + }) } impl ReadSet { @@ -359,9 +384,15 @@ impl ReadSet { /// was first read: no file was written, replaced, created or removed and /// no listed directory gained or lost an entry since. pub fn unchanged(&self) -> bool { - self.paths - .iter() - .all(|(rel, before)| before.holds(&self.root, rel)) + self.paths.iter().all(|(rel, before)| { + before.holds(&self.root, rel) + && self.racy.get(rel).is_none_or(|read| { + read.as_ref().is_some_and(|read| { + crate::utils::fs::read_regular_to_bytes_sync(&self.root.join(rel)) + .is_ok_and(|now| now[..] == read[..]) + }) + }) + }) } /// How many paths [`Self::unchanged`] re-stats. @@ -434,13 +465,38 @@ impl<'a> DiskSnapshot<'a> { if raw { return None; } - let paths = touched + let paths: BTreeMap = touched .into_iter() .filter_map(|rel| Some((rel.clone(), tracking.seen.get(&rel)?.clone()))) .collect(); + let racy_paths: Vec = paths + .iter() + .filter(|(rel, print)| { + let taken = tracking.taken.get(*rel).copied(); + [&print.entry, &print.target] + .into_iter() + .flatten() + .filter(|stat| stat.kind == 0) + .any(|stat| taken.is_none_or(|taken| racy(stat.modified, taken))) + }) + .map(|(rel, _)| rel.clone()) + .collect(); + drop(tracking); + let reads = self.lock(); + let racy = racy_paths + .into_iter() + .map(|rel| { + let read = match reads.get(&rel) { + Some(Ok(bytes)) if !self.is_overlaid(&rel) => Some(Arc::clone(bytes)), + _ => None, + }; + (rel, read) + }) + .collect(); Some(ReadSet { root: self.root.to_path_buf(), paths, + racy, }) } @@ -464,8 +520,10 @@ impl<'a> DiskSnapshot<'a> { .get(rel) .is_some_and(|print| access == Access::Probe || print.listing.is_some()); if !known { + let taken = std::time::SystemTime::now(); let print = Fingerprint::of(self.root, rel, access); tracking.seen.insert(rel.to_string(), print); + tracking.taken.insert(rel.to_string(), taken); } if let Some((touched, _)) = &mut tracking.window { touched.insert(rel.to_string()); @@ -1138,6 +1196,54 @@ mod tests { assert!(!set.unchanged(), "a removed file"); } + /// A file written just before its fingerprint is racily current: its + /// stats may survive a same-length rewrite in the same timestamp tick, + /// so its content is compared too: what the view read must still be + /// there, and a racy file the view only probed (no content to compare) + /// never holds. An old file is judged by stats alone. + #[tokio::test] + async fn a_racily_current_file_is_compared_by_content() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + std::fs::write(root.join("a.lock"), "one").unwrap(); + fn read( + view: ProjectView<'_>, + ) -> std::pin::Pin + '_>> { + Box::pin(async move { + view.read_text("a.lock").await.unwrap(); + }) + } + let set = recorded(root, read).await.unwrap(); + assert_eq!(set.racy.len(), 1, "written just now"); + // The content compared is what the view read. + assert_eq!(set.racy["a.lock"].as_deref(), Some(&b"one"[..])); + assert!(set.unchanged()); + let set = recorded(root, read).await.unwrap(); + std::fs::write(root.join("a.lock"), "two").unwrap(); + assert!(!set.unchanged(), "a same-length rewrite"); + + let probed = recorded(root, |view| { + Box::pin(async move { + assert!(view.exists("a.lock").await); + }) + }) + .await + .unwrap(); + assert!(!probed.unchanged(), "nothing read to compare"); + + let old = std::time::SystemTime::now() - std::time::Duration::from_secs(3600); + std::fs::File::options() + .write(true) + .open(root.join("a.lock")) + .unwrap() + .set_modified(old) + .unwrap(); + let set = recorded(root, read).await.unwrap(); + assert!(set.racy.is_empty(), "an old file"); + assert!(set.unchanged()); + assert!(racy(None, std::time::SystemTime::now()), "no time known"); + } + /// A listed directory breaks the read set when it gains or loses an /// entry, but not when socket-patch creates its own `.socket/` (the /// apply lock) at the root; an unlisted directory is only probed. From 9c49ebf2269215ea72b878350d4825b10e95069d Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 8 Oct 2026 03:29:41 -0400 Subject: [PATCH 19/20] Fix the build after merging main's UTF-16 requirements refusal #724 landed against the old requirements-walk callback, which named the file `path`; on this branch the walk hands over the root-relative `rel`. Name the file from `root.join(rel)`. Its new engine test also needs this branch's RewriteOptions fields. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/socket-patch-core/src/hosted/engine.rs | 3 +++ crates/socket-patch-core/src/vendor/pypi_requirements.rs | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index b153a17b8..3ba3ea092 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -2620,6 +2620,9 @@ mod tests { npm_outer: &outer, yarn_classic_outer: &OuterYarnMirror::default, blocking: false, + takeover_uuids: Default::default(), + patch_server_origins: Vec::new(), + prior_discovery: None, }; let tmp = tempfile::tempdir().unwrap(); diff --git a/crates/socket-patch-core/src/vendor/pypi_requirements.rs b/crates/socket-patch-core/src/vendor/pypi_requirements.rs index 95386b90b..b4848a2cf 100644 --- a/crates/socket-patch-core/src/vendor/pypi_requirements.rs +++ b/crates/socket-patch-core/src/vendor/pypi_requirements.rs @@ -866,7 +866,7 @@ async fn collect_requirements_files(root: &Path) -> Result, (&'stat format!( "{} is not UTF-8 text (for example UTF-16, which Windows PowerShell 5.1 \ writes for `pip freeze > requirements.txt`); re-save it as UTF-8 and re-run", - path.display() + root.join(rel).display() ), )), Err(_) if out.is_empty() => Err(( From da274996b95470c9799cf425879e37c9f7734aab Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 8 Oct 2026 04:48:04 -0400 Subject: [PATCH 20/20] Label ci.yml's setup-php pin with the release it resolves to zizmor's ref-version-mismatch audit now fails on the `# v2` comment: upstream's moving v2 tag no longer points at f3e473d1. That hash is 2.37.2, as composer-compatibility.yml already labels it. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4f8067ea..cce660612 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1397,7 +1397,7 @@ jobs: # The composer capstones shell out to a real composer; `composer:` # pins the release line (1, 2.2 LTS, 2) so the composer.lock grammar # the edits assert stays stable across runners. - uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # v2 + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 with: php-version: '8.2' tools: composer:${{ matrix.composer }}