diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4f8067ea..cce660612 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1397,7 +1397,7 @@ jobs: # The composer capstones shell out to a real composer; `composer:` # pins the release line (1, 2.2 LTS, 2) so the composer.lock grammar # the edits assert stays stable across runners. - uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # v2 + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 with: php-version: '8.2' tools: composer:${{ matrix.composer }} diff --git a/crates/socket-patch-cli/src/commands/apply.rs b/crates/socket-patch-cli/src/commands/apply.rs index c5e8116cf..dcb36632d 100644 --- a/crates/socket-patch-cli/src/commands/apply.rs +++ b/crates/socket-patch-cli/src/commands/apply.rs @@ -15,7 +15,8 @@ use socket_patch_core::patch::redirect::golang_local::{ use socket_patch_core::patch::sidecars::{maven as maven_sidecars, SidecarAdvisoryCode}; use socket_patch_core::telemetry::{track_patch_applied, track_patch_apply_failed}; use socket_patch_core::utils::purl::parse_golang_purl; -use socket_patch_core::utils::purl::{normalize_purl, purl_eq, strip_purl_qualifiers}; +use socket_patch_core::utils::purl::{normalize_purl, strip_purl_qualifiers}; +use socket_patch_core::utils::purl_key::PurlKey; use socket_patch_core::vendor::purl_keys_cover; use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::{Path, PathBuf}; @@ -103,7 +104,7 @@ async fn ensure_blobs_for_mismatches( args: &ApplyArgs, manifest: &PatchManifest, all_packages: &HashMap>, - vendored_purls: &HashSet, + vendored_purls: &HashSet, staged: &mut StagedSources, client: &ApiClient, ) { @@ -224,7 +225,7 @@ fn format_mismatch_fetch_result(downloaded: usize, needed: usize) -> String { async fn mismatch_blob_gaps( manifest: &PatchManifest, all_packages: &HashMap>, - vendored_purls: &HashSet, + vendored_purls: &HashSet, blobs_path: &Path, force: bool, ) -> HashSet { @@ -247,10 +248,11 @@ async fn mismatch_blob_gaps( }; let variant_eco = Ecosystem::from_purl(purl).is_some_and(|e| e.supports_release_variants()); let stripped = strip_purl_qualifiers(purl); + let identity = PurlKey::new(purl); let records: Vec<(&String, &PatchRecord)> = manifest .patches .iter() - .filter(|(key, _)| *key == purl || strip_purl_qualifiers(key) == stripped) + .filter(|(key, _)| *key == purl || PurlKey::new(key) == identity) .collect(); if purl_keys_cover(vendored_purls, purl) || records @@ -1729,7 +1731,7 @@ async fn report_apply_failure( /// main-thread stack in debug builds. fn synthesize_vendor_owned_results( target_manifest_purls: &HashSet, - vendored_purls: &HashSet, + vendored_purls: &HashSet, ) -> (Vec, HashSet, HashSet) { let is_vendored = |p: &str| purl_keys_cover(vendored_purls, p); let mut results: Vec = Vec::new(); @@ -3264,18 +3266,10 @@ async fn lockfile_resolved(common: &GlobalArgs, unmatched: &[String]) -> HashSet } let ctx = crate::commands::context::ProjectContext::new(common); let entries = &ctx.locks().await.entries; - let lock_purls: HashSet = entries - .iter() - .map(|e| normalize_purl(strip_purl_qualifiers(&e.purl)).into_owned()) - .collect(); + let lock_purls: HashSet = entries.iter().map(|e| PurlKey::new(&e.purl)).collect(); unmatched .iter() - .filter(|p| { - let base = strip_purl_qualifiers(p); - lock_purls.contains(normalize_purl(base).as_ref()) - || (base.starts_with("pkg:composer/") - && entries.iter().any(|e| purl_eq(&e.purl, base))) - }) + .filter(|p| lock_purls.contains(&PurlKey::new(p))) .cloned() .collect() } @@ -4001,7 +3995,7 @@ mod tests { "without a vendor claim the drifted singleton must queue (fixture sanity)" ); - let vendored = HashSet::from(["pkg:gem/foo@1.0.0".to_string()]); + let vendored = HashSet::from([PurlKey::new("pkg:gem/foo@1.0.0")]); let needed = mismatch_blob_gaps(&manifest, &all_packages, &vendored, &blobs, false).await; assert!( needed.is_empty(), diff --git a/crates/socket-patch-cli/src/commands/get.rs b/crates/socket-patch-cli/src/commands/get.rs index 2bf5edfeb..8b488b39a 100644 --- a/crates/socket-patch-cli/src/commands/get.rs +++ b/crates/socket-patch-cli/src/commands/get.rs @@ -23,6 +23,7 @@ use socket_patch_core::utils::concurrent::{api_concurrency_for, ordered_concurre use socket_patch_core::utils::purl::{ canonical_purl, is_purl, normalize_purl, strip_purl_qualifiers, }; +use socket_patch_core::utils::purl_key::PurlKey; use socket_patch_core::vendor::{load_state, lookup_entry, VendorEntry, VendorState}; use std::collections::HashMap; use std::fmt; @@ -1430,9 +1431,8 @@ struct InstalledNarrowing { /// runs — the coarse layer above [`filter_to_installed_releases`]'s /// per-release variant narrowing (which still runs later, unchanged). /// -/// Presence evidence per result purl (compared on -/// `normalize_purl(strip_purl_qualifiers(..))` — API purls are -/// percent-encoded/qualified, crawler purls literal): +/// Presence evidence per result purl (compared by [`PurlKey`] — API purls +/// are percent-encoded/qualified/mixed-case, crawler purls literal): /// * installed on disk — `find_packages_for_rollback` over the deduped base /// purls (the qualified-aware resolver; memory invariant); /// * already tracked in the manifest — the user opted this purl in earlier, @@ -1465,8 +1465,6 @@ async fn filter_to_installed_purls( use socket_patch_core::vendor::lock_inventory; use std::collections::HashSet; - let canon = canonical_purl; - // Deduped base purls, probed against the installed tree. The resolver // keys its result by the purls we pass, so canonicalize the found keys // the same way as the membership probes below. @@ -1480,14 +1478,14 @@ async fn filter_to_installed_purls( }; let partitioned = partition_purls(&bases, None); let found = find_packages_for_rollback(&partitioned, &common.crawler_options(), true).await; - let mut present: HashSet = found.keys().map(|k| canon(k)).collect(); + let mut present: HashSet = found.keys().map(|k| PurlKey::new(k)).collect(); let ctx = super::context::ProjectContext::rooted(common, common.cwd.clone()); // Manifest membership counts as presence (read-only probe: a corrupt // manifest degrades to "no extension" here — the download path's // fail-closed read still guards every write). if let Some(manifest) = ctx.ledgers().await.manifest { - present.extend(manifest.patches.keys().map(|k| canon(k))); + present.extend(manifest.patches.keys().map(|k| PurlKey::new(k))); } // scan's lockfile + vendored-ledger discovery supplements (and their @@ -1498,11 +1496,11 @@ async fn filter_to_installed_purls( let supplement = super::scan::project_lockfile_supplement(&ctx, &[], None).await; pnp_diags = supplement.unsupported; if mode != super::scan::ScanMode::Agent { - present.extend(supplement.entries.iter().map(|e| canon(&e.purl))); + present.extend(supplement.entries.iter().map(|e| PurlKey::new(&e.purl))); let vendored = super::scan::project_vendored_supplement(common, &[], &ctx.loaded().await.vendor) .await; - present.extend(vendored.packages.iter().map(|p| canon(&p.purl))); + present.extend(vendored.packages.iter().map(|p| PurlKey::new(&p.purl))); } } @@ -1536,7 +1534,7 @@ async fn filter_to_installed_purls( warnings, }; for result in accessible { - if present.contains(&canon(&result.purl)) { + if present.contains(&PurlKey::new(&result.purl)) { out.kept.push(result.clone()); continue; } @@ -1565,7 +1563,7 @@ async fn filter_to_installed_purls( // nothing — so it carries the same `package_not_installed` code // a non-PnP pnpm project would get; only an UNREADABLE lock // (no judgment possible) keeps the layout-refusal code. - let decoded = canon(&result.purl); + let decoded = canonical_purl(&result.purl); let coord = decoded.strip_prefix("pkg:npm/").unwrap_or(&decoded); if mode == super::scan::ScanMode::Hosted { match (&pnpm_pnp_lock, coord.rsplit_once('@')) { @@ -1780,7 +1778,8 @@ async fn lock_text_refusals_for( ) .await, ); - let claimed: Vec = pins.iter().map(|pin| canonical_purl(&pin.purl)).collect(); + let claimed: std::collections::HashSet = + pins.iter().map(|pin| PurlKey::new(&pin.purl)).collect(); let fetchable: Vec<&PatchSearchResult> = selected .iter() .filter(|sr| bun_refusal.filter(|r| r.applies_to(&sr.purl)).is_none()) @@ -1791,7 +1790,7 @@ async fn lock_text_refusals_for( .collect(); let candidates: Vec<(&str, &str)> = fetchable .iter() - .filter(|sr| !claimed.contains(&canonical_purl(&sr.purl))) + .filter(|sr| !claimed.contains(&PurlKey::new(&sr.purl))) .map(|sr| (sr.purl.as_str(), sr.uuid.as_str())) .collect(); let refused = socket_patch_core::vendor::lock_text_refusals(cwd, &candidates).await; @@ -1815,7 +1814,7 @@ async fn lock_text_refusals_for( cwd, fetchable .iter() - .filter(|sr| claimed.contains(&canonical_purl(&sr.purl))) + .filter(|sr| claimed.contains(&PurlKey::new(&sr.purl))) .map(|sr| sr.purl.as_str()), &pins, false, @@ -2389,8 +2388,8 @@ async fn run_nested_apply( /// qualified record (apply keys a release-variant base by its base purl). /// A qualified key never covers a sibling variant. fn apply_key_covers(key: &str, record: &str) -> bool { - let (key, record) = (normalize_purl(key), normalize_purl(record)); - key == record || (!key.contains(['?', '#']) && record.split(['?', '#']).next() == Some(&*key)) + PurlKey::qualified(key) == PurlKey::qualified(record) + || (!key.trim().contains(['?', '#']) && PurlKey::same(key, record)) } /// Fold a failed nested apply into a `get` / `scan --mode agent` JSON @@ -2433,7 +2432,8 @@ fn fold_apply_failures( } } let appended = patches[selected..].iter().any(|r| { - normalize_purl(r["purl"].as_str().unwrap_or_default()) == normalize_purl(&failure.purl) + PurlKey::qualified(r["purl"].as_str().unwrap_or_default()) + == PurlKey::qualified(&failure.purl) }); if !hit && !appended { let mut rec = serde_json::json!({ diff --git a/crates/socket-patch-cli/src/commands/list.rs b/crates/socket-patch-cli/src/commands/list.rs index 44c719038..35f55e429 100644 --- a/crates/socket-patch-cli/src/commands/list.rs +++ b/crates/socket-patch-cli/src/commands/list.rs @@ -70,19 +70,14 @@ impl HostedListing { /// One listing per hosted pin in `pins`, detailed from `legacy` where /// it records the same purl and uuid. pub(crate) fn from_pins(pins: &[HostedPin], legacy: Option<&RedirectState>) -> Vec { - let canon = |p: &str| { - socket_patch_core::utils::purl::normalize_purl( - socket_patch_core::utils::purl::strip_purl_qualifiers(p), - ) - .into_owned() - }; + use socket_patch_core::utils::purl_key::PurlKey; pins.iter() .map(|pin| { let record = legacy .and_then(|l| { l.records .iter() - .find(|(k, r)| canon(k) == canon(&pin.purl) && r.uuid == pin.uuid) + .find(|(k, r)| PurlKey::same(k, &pin.purl) && r.uuid == pin.uuid) .map(|(_, r)| r.clone()) }) .unwrap_or_else(|| PatchRecord { diff --git a/crates/socket-patch-cli/src/commands/remove.rs b/crates/socket-patch-cli/src/commands/remove.rs index 6bbb80ed6..d4d611d54 100644 --- a/crates/socket-patch-cli/src/commands/remove.rs +++ b/crates/socket-patch-cli/src/commands/remove.rs @@ -7,6 +7,7 @@ use socket_patch_core::manifest::schema::PatchManifest; use socket_patch_core::patch::redirect::upstream::HostedPin; use socket_patch_core::telemetry::{track_patch_remove_failed, track_patch_removed}; use socket_patch_core::utils::purl::patch_matches; +use socket_patch_core::utils::purl_key::PurlKey; use socket_patch_core::vendor::{ load_state, RevertOpts, VendorEntry, VendorState, VENDOR_STATE_REL, }; @@ -562,7 +563,7 @@ pub async fn run(args: RemoveArgs) -> i32 { // Vendor-owned purls are excluded from the in-place restore (the // vendored leg below reverts them); an unreadable ledger degrades to // "nothing vendored" here and fails closed at that leg. - let vendored_keys: HashSet = vendor_state_result + let vendored_keys: HashSet = vendor_state_result .as_ref() .map(socket_patch_core::vendor::VendorState::purl_keys) .unwrap_or_default(); diff --git a/crates/socket-patch-cli/src/commands/rollback.rs b/crates/socket-patch-cli/src/commands/rollback.rs index b51a058f4..9397c9220 100644 --- a/crates/socket-patch-cli/src/commands/rollback.rs +++ b/crates/socket-patch-cli/src/commands/rollback.rs @@ -15,8 +15,8 @@ use socket_patch_core::patch::rollback::{ }; use socket_patch_core::telemetry::{track_patch_rollback_failed, track_patch_rolled_back}; use socket_patch_core::utils::purl::{patch_matches, strip_purl_qualifiers}; +use socket_patch_core::utils::purl_key::PurlKey; use socket_patch_core::vendor::{purl_keys_cover, RevertOpts, VendorState}; -use socket_patch_core::vex::discover::{canonical_base_purl, same_release}; use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -1168,7 +1168,7 @@ pub async fn run(args: RollbackArgs) -> i32 { // runs, and the ledger does not own the global copies, so the in-place // leg restores them. let loaded_vendor_state = socket_patch_core::vendor::load_state(&cwd).await; - let project_vendored_keys: HashSet = loaded_vendor_state + let project_vendored_keys: HashSet = loaded_vendor_state .as_ref() .map(VendorState::purl_keys) .unwrap_or_default(); @@ -1181,7 +1181,7 @@ pub async fn run(args: RollbackArgs) -> i32 { let vendor_corrupt = vendor_state_result.is_err(); // An unreadable ledger degrades to "nothing vendored" for the in-place // leg (its own containment is the `vendor_state_unreadable` exit below). - let vendored_keys: HashSet = vendor_state_result + let vendored_keys: HashSet = vendor_state_result .as_ref() .map(VendorState::purl_keys) .unwrap_or_default(); @@ -2020,7 +2020,7 @@ pub(crate) async fn rollback_patches_inner( common: &GlobalArgs, socket_dir: &Path, manifest: &PatchManifest, - vendored_keys: &HashSet, + vendored_keys: &HashSet, selection: InnerSelection<'_>, // Manifest purl -> the hosted uuid a live lockfile pin superseded its // record with ([`superseded_by_hosted`]); empty when no hosted pin @@ -2811,10 +2811,10 @@ pub(crate) fn superseded_by_hosted( .patches .iter() .filter_map(|(purl, record)| { - let pkg = canonical_base_purl(purl); + let pkg = PurlKey::new(purl); let same: Vec<&HostedPin> = pins .iter() - .filter(|pin| same_release(&pin.purl, &pkg)) + .filter(|pin| PurlKey::new(&pin.purl) == pkg) .collect(); if same.iter().any(|pin| pin.uuid == record.uuid) { return None; diff --git a/crates/socket-patch-cli/src/commands/scan/discovery.rs b/crates/socket-patch-cli/src/commands/scan/discovery.rs index 79ca66737..c46e8457c 100644 --- a/crates/socket-patch-cli/src/commands/scan/discovery.rs +++ b/crates/socket-patch-cli/src/commands/scan/discovery.rs @@ -9,9 +9,9 @@ use socket_patch_core::api::types::{ BatchPackagePatches, BatchPatchInfo, PatchResponse, PatchSearchResult, }; use socket_patch_core::manifest::schema::{PatchManifest, PatchRecord}; -use socket_patch_core::utils::composer_version::{composer_purl_identity, purl_identity_key}; use socket_patch_core::utils::concurrent::{api_concurrency_for, ordered_concurrent}; -use socket_patch_core::utils::purl::{normalize_purl, purl_eq, strip_purl_qualifiers}; +use socket_patch_core::utils::purl::{normalize_purl, strip_purl_qualifiers}; +use socket_patch_core::utils::purl_key::PurlKey; use socket_patch_core::vendor::lock_inventory::LockfileEntry; use socket_patch_core::vendor::VendorState; use std::collections::{HashMap, HashSet}; @@ -33,8 +33,9 @@ pub(super) struct UpdateInfo { #[derive(Default)] pub(crate) struct LockfileSupplement { pub(crate) packages: Vec, - /// Literal crawler-form purls, for fast membership tests. - pub(crate) purls: HashSet, + /// The lockfile-only packages' identities ([`PurlKey`]), keyed once so + /// [`lockfile_only_contains`] is a single hash lookup. + pub(crate) purls: HashSet, /// The FULL lockfile inventory the supplement was derived from (installed /// packages included), kept so the hosted-wiring probes reuse it instead /// of re-parsing every project lockfile. Empty for global scans. @@ -74,7 +75,29 @@ pub(crate) async fn lockfile_supplement( if entries.is_empty() { return out; } - let crawled_purls: HashSet<&str> = crawled.iter().map(|p| p.purl.as_str()).collect(); + (out.packages, out.purls) = lockfile_only_packages(entries, crawled, only, &common.cwd); + out.entries = entries.clone(); + out +} + +/// The lockfile entries with no crawled counterpart, fabricated as crawl +/// entries, plus their [`PurlKey`]s. "Crawled" is by [`PurlKey`], the same +/// relation [`lockfile_only_contains`] answers by: a lock spelling that +/// differs from the installed crawl's only in encoding, NuGet case, PEP 503 +/// form or composer padding is the installed package, not a lockfile-only +/// one — keyed in, its every spelling would read as not installed. +fn lockfile_only_packages( + entries: &[LockfileEntry], + crawled: &[socket_patch_core::crawlers::types::CrawledPackage], + only: Option<&[String]>, + cwd: &std::path::Path, +) -> ( + Vec, + HashSet, +) { + let mut packages = Vec::new(); + let mut purls = HashSet::new(); + let crawled_keys: HashSet = crawled.iter().map(|p| PurlKey::new(&p.purl)).collect(); let in_scope = |purl: &str| { only.is_none_or(|list| { socket_patch_core::crawlers::Ecosystem::from_purl(purl) @@ -82,30 +105,28 @@ pub(crate) async fn lockfile_supplement( }) }; for entry in entries { - if crawled_purls.contains(entry.purl.as_str()) || !in_scope(&entry.purl) { + let key = PurlKey::new(&entry.purl); + if crawled_keys.contains(&key) || !in_scope(&entry.purl) { continue; } - let Some(pkg) = crawled_from_purl(&entry.purl, &common.cwd) else { + let Some(pkg) = crawled_from_purl(&entry.purl, cwd) else { continue; }; - out.purls.insert(entry.purl.clone()); - out.packages.push(pkg); + purls.insert(key); + packages.push(pkg); } - out.entries = entries.clone(); - out + (packages, purls) } /// Whether an API-spelled purl (percent-encoded, possibly qualified) names -/// a lockfile-only package: `purls` holds the crawler's literal spelling, so -/// the comparison bridges the two via `normalize_purl`. The ONE predicate -/// behind the `notInstalled` flag, the `[NOT INSTALLED]` marker, the +/// a lockfile-only package: `purls` holds the crawler spellings' keys, so +/// the comparison bridges the two by [`PurlKey`] (encoding, qualifiers, +/// PyPI/NuGet name folding, composer release identity: the API may serve +/// the padded `@3.0.2.0` for a lock's `3.0.2`). The ONE predicate behind the +/// `notInstalled` flag, the `[NOT INSTALLED]` marker, the /// `package_not_installed` skip partition and the vendor baseline pre-check. -/// A composer purl also matches its lock spelling of the same release (the -/// API may serve the padded `@3.0.2.0` for a lock's `3.0.2`). -pub(super) fn lockfile_only_contains(purls: &HashSet, api_purl: &str) -> bool { - let base = strip_purl_qualifiers(api_purl); - purls.contains(normalize_purl(base).as_ref()) - || (base.starts_with("pkg:composer/") && purls.iter().any(|p| purl_eq(p, base))) +pub(super) fn lockfile_only_contains(purls: &HashSet, api_purl: &str) -> bool { + purls.contains(&PurlKey::new(api_purl)) } /// A displayable crawl entry fabricated from a purl (decoded form). The @@ -194,13 +215,13 @@ pub(crate) async fn vendored_ledger_supplement( .map(|base| (base.clone(), base, None)) .collect(), }; - // Composer by release identity: a ledger `@3.0.2.0` is the crawled - // `@3.0.2`, not a second package to supplement. - let key = |p: &str| composer_purl_identity(p).unwrap_or_else(|| normalize_purl(p).into_owned()); - let crawled_norm: HashSet = crawled.iter().map(|p| key(&p.purl)).collect(); - let mut seen: HashSet = HashSet::new(); + // By release identity: a ledger `@3.0.2.0` is the crawled composer + // `@3.0.2`, a ledger `Newtonsoft.Json` the crawled `newtonsoft.json` — + // not a second package to supplement. + let crawled_norm: HashSet = crawled.iter().map(|p| PurlKey::new(&p.purl)).collect(); + let mut seen: HashSet = HashSet::new(); for (ledger_key, base, entry) in &candidates { - let norm = key(base); + let norm = PurlKey::new(base); if crawled_norm.contains(&norm) || seen.contains(&norm) { continue; } @@ -268,7 +289,7 @@ pub(super) async fn preverify_vendor_baselines( api_client: &socket_patch_core::api::client::ApiClient, selected: &[PatchSearchResult], crawled: &[socket_patch_core::crawlers::types::CrawledPackage], - lockfile_only: &HashSet, + lockfile_only: &HashSet, vendor: Option<&HashMap>, status: &mut crate::ui::StatusLine, ) -> (HashSet, HashMap) { @@ -291,7 +312,7 @@ pub(super) async fn preverify_vendor_baselines( .iter() .map(|patch| { // API purls come percent-encoded, crawler purls literal — - // purl_eq bridges the two spellings. + // PurlKey bridges the two spellings. let base = strip_purl_qualifiers(&patch.purl); // Lockfile-only packages have no installed bytes to compare // — the vendor engine fetches them pristine (nothing to @@ -299,7 +320,7 @@ pub(super) async fn preverify_vendor_baselines( if lockfile_only_contains(lockfile_only, base) { return None; } - let pkg = crawled.iter().find(|c| purl_eq(&c.purl, base))?; + let pkg = crawled.iter().find(|c| PurlKey::same(&c.purl, base))?; // The same predicate as the download phase's ledger // idempotency skip: its no-fetch set and this one must be // the same set. @@ -427,19 +448,20 @@ pub(super) fn detect_updates( // artifact-pinned ecosystems, qualified (`?artifact_id=...`); the // batch *package* purl is the crawler's literal spelling. Bridge // both divergences like the lockfile-only partition does: exact hit - // first, then a normalized qualifier-stripped comparison (composer - // by release identity: a `@3.0.2.0` key is the crawler's `@3.0.2`). + // first, then by [`PurlKey`] (a composer `@3.0.2.0` key is the + // crawler's `@3.0.2`, a NuGet `Newtonsoft.Json` key its lowercase + // global-cache spelling). // // Qualifier TWINS (e.g. a pypi wheel + sdist pair) all match the // stripped form: any stale twin means an update, so prefer the // first (sorted-key order, for stability) whose uuid differs. let existing = manifest.patches.get(&pkg.purl).or_else(|| { - let want = purl_identity_key(&pkg.purl); + let want = PurlKey::new(&pkg.purl); let mut twins: Vec<(&String, &socket_patch_core::manifest::schema::PatchRecord)> = manifest .patches .iter() - .filter(|(k, _)| purl_identity_key(k) == want) + .filter(|(k, _)| PurlKey::new(k) == want) .collect(); twins.sort_by(|a, b| a.0.cmp(b.0)); twins @@ -554,6 +576,56 @@ mod tests { use crate::commands::scan::tests::manifest_with; + // ---- lockfile_only_packages -------------------------------------------- + + fn lock_entry(ecosystem: &'static str, purl: &str) -> LockfileEntry { + use socket_patch_core::vendor::lock_inventory::{LockIntegrity, SourceKind}; + LockfileEntry { + ecosystem, + name: String::new(), + version: String::new(), + purl: purl.to_string(), + resolved: None, + integrity: LockIntegrity::None, + source_kind: SourceKind::Unspecified, + } + } + + fn crawled_from(purl: &str) -> socket_patch_core::crawlers::types::CrawledPackage { + crawled_from_purl(purl, std::path::Path::new("/p")).unwrap() + } + + /// An installed package whose lock spelling differs from the crawl's + /// (NuGet case, PEP 503 form, composer padding) is NOT lockfile-only: + /// keyed in, [`lockfile_only_contains`] would mark every spelling of the + /// live install `package_not_installed`. A truly absent one still is. + #[test] + fn lockfile_only_packages_excludes_crawled_spelling_variants() { + let entries = vec![ + lock_entry("nuget", "pkg:nuget/Newtonsoft.Json@13.0.1"), + lock_entry("pypi", "pkg:pypi/typing_extensions@4.12.2"), + lock_entry("composer", "pkg:composer/psr/log@3.0.2"), + lock_entry("npm", "pkg:npm/lockonly@1.0.0"), + ]; + let crawled = vec![ + crawled_from("pkg:nuget/newtonsoft.json@13.0.1"), + crawled_from("pkg:pypi/typing-extensions@4.12.2"), + crawled_from("pkg:composer/psr/log@3.0.2.0"), + ]; + let (packages, purls) = + lockfile_only_packages(&entries, &crawled, None, std::path::Path::new("/p")); + let got: Vec<&str> = packages.iter().map(|p| p.purl.as_str()).collect(); + assert_eq!(got, vec!["pkg:npm/lockonly@1.0.0"]); + assert!(lockfile_only_contains(&purls, "pkg:npm/lockonly@1.0.0")); + for api in [ + "pkg:nuget/Newtonsoft.Json@13.0.1", + "pkg:pypi/typing-extensions@4.12.2", + "pkg:composer/psr/log@3.0.2.0", + ] { + assert!(!lockfile_only_contains(&purls, api), "{api}"); + } + } + // ---- severity_order ---------------------------------------------------- #[test] @@ -1713,8 +1785,8 @@ mod tests { std::path::PathBuf::from("/nonexistent"), ), ]; - let lockfile_only: HashSet = - std::iter::once("pkg:npm/@scope/lockonly@1.0.0".to_string()).collect(); + let lockfile_only: HashSet = + std::iter::once(PurlKey::new("pkg:npm/@scope/lockonly@1.0.0")).collect(); // A live status line: every step is shown, and the line is gone // once the check returns (nothing left over for the preview). @@ -2182,7 +2254,7 @@ mod tests { &api_client_for(&mock.uri()), &selected, &crawled, - &std::iter::once("pkg:npm/lockonly@1.0.0".to_string()).collect(), + &std::iter::once(PurlKey::new("pkg:npm/lockonly@1.0.0")).collect(), Some(&ledger), &mut crate::ui::StatusLine::new(Vec::new(), false, false, 80), ) diff --git a/crates/socket-patch-cli/src/commands/scan/gc.rs b/crates/socket-patch-cli/src/commands/scan/gc.rs index cd0bf9db5..e9c285f76 100644 --- a/crates/socket-patch-cli/src/commands/scan/gc.rs +++ b/crates/socket-patch-cli/src/commands/scan/gc.rs @@ -5,8 +5,7 @@ use socket_patch_core::manifest::cleanup_blobs::{ArtifactReferences, CleanupResult}; use socket_patch_core::manifest::operations::{read_manifest, write_manifest}; use socket_patch_core::manifest::schema::PatchManifest; -use socket_patch_core::utils::composer_version::purl_identity_key; -use socket_patch_core::utils::purl::strip_purl_qualifiers; +use socket_patch_core::utils::purl_key::PurlKey; use socket_patch_core::vendor::{purl_keys_cover, VENDOR_STATE_REL}; use std::collections::HashSet; use std::path::Path; @@ -191,7 +190,7 @@ pub(super) async fn run_apply_gc( manifest_path: &Path, socket_dir: &Path, scanned_purls: &HashSet, - vendored: &HashSet, + vendored: &HashSet, ) -> GcSummary { // Existence gate BEFORE the lock: `acquire` creates `.socket/`, and a // pristine checkout with neither a manifest nor a ledger must not gain @@ -303,7 +302,7 @@ async fn preview_apply_gc( manifest_path: &Path, socket_dir: &Path, scanned_purls: &HashSet, - vendored: &HashSet, + vendored: &HashSet, ) -> GcSummary { // Read-only preview of the vendored-state GC (lists, never reverts). let vendor_gc = run_vendor_gc(common, manifest_path, /*dry_run=*/ true).await; @@ -314,11 +313,23 @@ async fn preview_apply_gc( }; // Mirror the wet pass, which drops an unused vendored entry's manifest // keys before the blob sweep, or the preview under-reports orphans. - for purl in &vendor_gc.unused_reverted { - let base = strip_purl_qualifiers(purl).to_string(); - manifest - .patches - .retain(|k, _| k != purl && strip_purl_qualifiers(k) != base); + // The dry pass reverted nothing, so the ledger still holds each entry + // (its base purl is part of the relation: a `!x`-encoded golang key). + if !vendor_gc.unused_reverted.is_empty() { + if let Ok(state) = socket_patch_core::vendor::load_state(&common.cwd).await { + for purl in &vendor_gc.unused_reverted { + let Some(entry) = state.entries.get(purl) else { + continue; + }; + for k in crate::commands::vendor::unused_vendored_manifest_keys( + &manifest.patches, + purl, + entry, + ) { + manifest.patches.remove(&k); + } + } + } } let prunable = detect_prunable(&manifest, scanned_purls, vendored); // Likewise drop the prunable entries in memory before the sweep: the @@ -339,7 +350,7 @@ pub(super) async fn gc_json( manifest_path: &Path, socket_dir: &Path, scanned_purls: &HashSet, - vendored: &HashSet, + vendored: &HashSet, dry_run: bool, ) -> serde_json::Value { if dry_run { @@ -453,7 +464,7 @@ pub(super) async fn run_human_gc( manifest_path: &Path, socket_dir: &Path, scanned_purls: &HashSet, - vendored: &HashSet, + vendored: &HashSet, ) { let preview = common.dry_run; let gc = if preview { @@ -488,11 +499,11 @@ pub(super) fn print_human_gc(gc: &GcSummary, preview: bool) { /// installed (or no longer reachable to the crawler). Pure / no I/O so /// it's unit-testable. /// -/// Comparison is on the canonical **base** PURL (qualifiers stripped, -/// percent-decoded) on both sides: a manifest may hold several qualified -/// release variants of one installed package, and API keys are encoded -/// (`pkg:npm/%40scope/x@1`) where crawler purls are literal. Otherwise -/// `--prune`/`--sync` would GC the very patches it just downloaded. +/// Comparison is by [`PurlKey`] on both sides: a manifest may hold several +/// qualified release variants of one installed package, API keys are +/// encoded (`pkg:npm/%40scope/x@1`) and mixed-case (`pkg:nuget/Newtonsoft.Json`) +/// where crawler purls are literal (the NuGet global cache is lowercase). +/// Otherwise `--prune`/`--sync` would GC the very patches it just downloaded. /// /// `vendored` (the ledger's purl-key set) is always exempt: a vendored /// package is consumed from the committed `.socket/vendor/` artifact, so @@ -504,15 +515,14 @@ pub(super) fn print_human_gc(gc: &GcSummary, preview: bool) { fn detect_prunable( manifest: &PatchManifest, scanned_purls: &HashSet, - vendored: &HashSet, + vendored: &HashSet, ) -> Vec { - let scanned_bases: HashSet = - scanned_purls.iter().map(|p| purl_identity_key(p)).collect(); + let scanned_bases: HashSet = scanned_purls.iter().map(|p| PurlKey::new(p)).collect(); manifest .patches .keys() .filter(|p| { - !scanned_bases.contains(&purl_identity_key(p)) + !scanned_bases.contains(&PurlKey::new(p)) && !purl_keys_cover(vendored, p) && crate::ecosystem_dispatch::crawl_covers_purl(p.as_str()) }) @@ -591,7 +601,7 @@ mod tests { } /// The "nothing vendored" set most prune tests run with. - fn no_vendored() -> HashSet { + fn no_vendored() -> HashSet { HashSet::new() } @@ -704,7 +714,7 @@ mod tests { // A vendored package is consumed from the committed artifact — // the crawler not seeing an installed copy is its normal state. let m = manifest_with(&[("pkg:npm/foo@1.0", "uuid-a"), ("pkg:npm/bar@2.0", "uuid-b")]); - let vendored: HashSet = ["pkg:npm/foo@1.0".to_string()].into_iter().collect(); + let vendored: HashSet = [PurlKey::new("pkg:npm/foo@1.0")].into_iter().collect(); let out = detect_prunable(&m, &scanned(&[]), &vendored); assert_eq!( out, @@ -771,13 +781,35 @@ mod tests { ); } + /// B20: the NuGet global-cache crawl spells the purl lowercase + /// (`newtonsoft.json`) while the manifest key is the API's mixed-case + /// `Newtonsoft.Json`; NuGet names and versions are case-insensitive, so + /// the installed package keeps its entry. Same for a PEP 503 spelling. + #[test] + fn detect_prunable_keeps_case_and_pep503_spellings_of_installed_packages() { + let m = manifest_with(&[ + ("pkg:nuget/Newtonsoft.Json@13.0.3", "uuid-a"), + ("pkg:pypi/typing_extensions@4.12.2", "uuid-b"), + ("pkg:nuget/Gone.Package@1.0.0", "uuid-c"), + ]); + let s = scanned(&[ + "pkg:nuget/newtonsoft.json@13.0.3", + "pkg:pypi/typing-extensions@4.12.2", + ]); + assert_eq!( + detect_prunable(&m, &s, &no_vendored()), + vec!["pkg:nuget/Gone.Package@1.0.0".to_string()] + ); + } + #[test] fn detect_prunable_exempts_qualified_variant_of_vendored_base() { // The ledger key set carries qualifier-stripped bases, so a // qualified manifest variant of a vendored package is exempt via // its base purl. let m = manifest_with(&[("pkg:pypi/six@1.16.0?artifact_id=wheel-a", "uuid-a")]); - let vendored: HashSet = ["pkg:pypi/six@1.16.0".to_string()].into_iter().collect(); + let vendored: HashSet = + [PurlKey::new("pkg:pypi/six@1.16.0")].into_iter().collect(); let out = detect_prunable(&m, &scanned(&[]), &vendored); assert!( out.is_empty(), @@ -1304,7 +1336,7 @@ mod tests { .unwrap(); let state_before = std::fs::read(tmp.path().join(".socket/vendor/state.json")).unwrap(); - let vendored: HashSet = [PURL.to_string()].into_iter().collect(); + let vendored: HashSet = [PurlKey::new(PURL)].into_iter().collect(); let gc = preview_apply_gc( &gc_common(tmp.path()), &manifest_path, @@ -1438,7 +1470,7 @@ mod tests { .await .unwrap(); - let vendored: HashSet = [PURL.to_string()].into_iter().collect(); + let vendored: HashSet = [PurlKey::new(PURL)].into_iter().collect(); let gc = run_apply_gc( &gc_common(tmp.path()), &manifest_path, diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 3047056c3..99557af0f 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -1612,16 +1612,17 @@ fn stranded_takeovers( confirmed: &[(String, String)], dry_run: bool, ) -> Vec { - use socket_patch_core::utils::purl::{canonical_purl, strip_purl_qualifiers}; + use socket_patch_core::utils::purl_key::PurlKey; if dry_run { return Vec::new(); } - let key = |purl: &str| canonical_purl(strip_purl_qualifiers(purl)); - let pinned: std::collections::HashSet = - confirmed.iter().map(|(purl, _)| key(purl)).collect(); + let pinned: std::collections::HashSet = confirmed + .iter() + .map(|(purl, _)| PurlKey::new(purl)) + .collect(); migrated .iter() - .filter(|purl| !pinned.contains(&key(purl))) + .filter(|purl| !pinned.contains(&PurlKey::new(purl))) .cloned() .collect() } @@ -1687,7 +1688,8 @@ async fn vendored_takeover( // No takeover-capable candidates — nothing to reconcile. return Ok(out); } - use socket_patch_core::utils::purl::{canonical_purl as canon, strip_purl_qualifiers}; + use socket_patch_core::utils::purl::strip_purl_qualifiers; + use socket_patch_core::utils::purl_key::PurlKey; // Each takeover-capable candidate with its vendored ledger entry, if // any (cloned out so the loop can mutate the state). let takeover: Vec<(&Candidate, Option)> = candidates @@ -2087,7 +2089,7 @@ async fn vendored_takeover( .expect("a vendored ledger entry was looked up in this state, so it loaded"); state .entries - .retain(|k, e| canon(k) != canon(purl) && canon(&e.base_purl) != canon(purl)); + .retain(|k, e| !PurlKey::same(k, purl) && !PurlKey::same(&e.base_purl, purl)); if let Err(e) = socket_patch_core::vendor::save_state(&common.cwd, state).await { // The wiring is reverted but the ledger still claims it; // redirecting now would leave a ledger asserting wiring diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index 3fe50564c..7f13d319d 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -19,9 +19,9 @@ use socket_patch_core::manifest::schema::PatchManifest; use socket_patch_core::telemetry::{ spawn_patch_scan_failed, spawn_patch_scanned, PendingTelemetry, }; -use socket_patch_core::utils::composer_version::purl_identity_key; use socket_patch_core::utils::concurrent::{api_concurrency_for, ordered_concurrent}; -use socket_patch_core::utils::purl::{normalize_purl, strip_purl_qualifiers}; +use socket_patch_core::utils::purl::{canonical_purl, normalize_purl}; +use socket_patch_core::utils::purl_key::{canonical_base_purl, PurlKey}; use socket_patch_core::vendor::{purl_keys_cover, VendorState}; use socket_patch_core::vex::discover::{LedgerLiveness, WiringMode}; use std::collections::{HashMap, HashSet}; @@ -791,7 +791,7 @@ struct AgentSelection { fn partition_agent_selection( selected: Vec, - vendored: &HashSet, + vendored: &HashSet, lockfile_only: &LockfileSupplement, ) -> AgentSelection { let (kept, vendored_records) = @@ -975,34 +975,33 @@ pub(super) async fn classify_overlap_takeover_with( } // Each overlapping vendored entry's uuid + the lockfiles it wired // (revert reads the same set). - let canon = |p: &str| normalize_purl(strip_purl_qualifiers(p)).into_owned(); let mut vendor_by_purl: std::collections::HashMap< - String, + PurlKey, &socket_patch_core::vendor::VendorEntry, > = std::collections::HashMap::new(); for (key, entry) in &vendor.entries { - vendor_by_purl.entry(canon(key)).or_insert(entry); + vendor_by_purl.entry(PurlKey::new(key)).or_insert(entry); vendor_by_purl - .entry(canon(&entry.base_purl)) + .entry(PurlKey::new(&entry.base_purl)) .or_insert(entry); } // Each hosted pin's patch uuid (embedded in every hosted artifact URL, // whatever the host). A non-empty overlap proves `redirect` is `Some`. - let mut redirect_uuid_by_purl: std::collections::HashMap = + let mut redirect_uuid_by_purl: std::collections::HashMap = std::collections::HashMap::new(); for (key, record) in redirect.iter().flat_map(|r| &r.records) { redirect_uuid_by_purl - .entry(canon(key)) + .entry(PurlKey::new(key)) .or_insert(record.uuid.as_str()); } let discovery = crate::commands::discover_wiring(common, cwd).await; let mut liveness = LedgerLiveness::new(cwd, &discovery, None); for purl in overlap { - let hosted_live = match redirect_uuid_by_purl.get(&purl) { + let hosted_live = match redirect_uuid_by_purl.get(&PurlKey::new(&purl)) { Some(uuid) => liveness.redirect_record(&purl, uuid).await, None => discovery.wires_package(&purl, WiringMode::Hosted), }; - let vendored_live = match vendor_by_purl.get(&purl) { + let vendored_live = match vendor_by_purl.get(&PurlKey::new(&purl)) { Some(entry) => liveness.vendor_entry(entry).await, None => false, }; @@ -1154,7 +1153,10 @@ pub(super) const GRADLE_USER_HOME_DIFFERS: &str = "gradle_user_home_differs"; struct GradleScan { /// `(code, detail)` run-level warnings. notes: Vec<(String, String)>, - /// Base purls (normalized) of the packages crawled from a Gradle cache. + /// Base purls ([`canonical_base_purl`]) of the packages crawled from a + /// Gradle cache. Maven coordinates are case-sensitive, so the canonical + /// spelling is already the identity; these stay strings because the + /// lock-file GAVs they are checked against are built as strings. gradle_purls: HashSet, /// Base purls the build's lock files name; `None` when they were not /// read (no Gradle build at the cwd, or no Gradle-cached package to @@ -1204,7 +1206,7 @@ async fn gradle_scan( gradle_purls: crawled .iter() .filter(|p| gradle_cache::is_gradle_version_dir(&p.path)) - .map(|p| normalize_purl(strip_purl_qualifiers(&p.purl)).into_owned()) + .map(|p| canonical_base_purl(&p.purl)) .collect(), ..GradleScan::default() }; @@ -1220,7 +1222,7 @@ async fn gradle_scan( m.patches .keys() .filter(|k| k.starts_with("pkg:maven/")) - .map(|k| normalize_purl(strip_purl_qualifiers(k)).into_owned()) + .map(|k| canonical_base_purl(k)) .collect() }) .unwrap_or_default(); @@ -1342,20 +1344,19 @@ pub(super) async fn hosted_wiring_retained_purls( if redirect.records.is_empty() { return Vec::new(); } - let canon = |p: &str| normalize_purl(strip_purl_qualifiers(p)).into_owned(); // By release identity: a record keyed `@3.0.2.0` names the scanned - // composer `@3.0.2`. - let scanned: std::collections::BTreeSet = scanned_purls + // composer `@3.0.2`, a `Newtonsoft.Json` record the lowercase NuGet crawl. + let scanned: std::collections::BTreeSet = scanned_purls .into_iter() - .map(|p| purl_identity_key(p.as_ref())) + .map(|p| PurlKey::new(p.as_ref())) .collect(); // Cheap no-I/O gate: skip the lockfile proofs when no record names a // scanned purl. let candidates: Vec<(String, &str)> = redirect .records .iter() - .filter(|(key, _)| scanned.contains(&purl_identity_key(key))) - .map(|(key, record)| (canon(key), record.uuid.as_str())) + .filter(|(key, _)| scanned.contains(&PurlKey::new(key))) + .map(|(key, record)| (canonical_purl(key), record.uuid.as_str())) .collect(); if candidates.is_empty() { return Vec::new(); @@ -1438,13 +1439,12 @@ pub(super) fn redirect_state_json( if redirect.records.is_empty() { return None; } - let canon = |p: &str| normalize_purl(strip_purl_qualifiers(p)).into_owned(); let records: Vec = redirect .records .iter() .map(|(key, record)| { serde_json::json!({ - "purl": canon(key), + "purl": canonical_purl(key), "uuid": record.uuid, }) }) @@ -1859,7 +1859,7 @@ async fn run_scan( // Vendor-ledger purl keys, shared by the prune exemption (a vendored // package's normal state is absent from the crawl) and the // vendored-skip in the apply path. A corrupt ledger yields the empty set. - let vendored_purls: HashSet = vendor_state + let vendored_purls: HashSet = vendor_state .as_ref() .map(VendorState::purl_keys) .unwrap_or_default(); @@ -1867,7 +1867,7 @@ async fn run_scan( // scan's agent leg patches the global copy even when the cwd project // vendors the same purl (see `project_state_in_scope`). let project_state = crate::commands::project_state_in_scope(&args.common); - let vendor_owned_purls: HashSet = if project_state { + let vendor_owned_purls: HashSet = if project_state { vendored_purls.clone() } else { HashSet::new() @@ -2384,16 +2384,13 @@ async fn run_scan( push_scan_json_warning(&mut result, API_BATCH_FAILED, detail); } policy.fold_into_json(&mut result); - // Flag lockfile-only packages (additive; absent means installed). - // `normalize_purl` bridges the API's percent-encoded spelling to the - // supplement's literal form. + // Flag lockfile-only packages (additive; absent means installed), + // with the same predicate as the `[NOT INSTALLED]` marker. if let Some(packages) = result["packages"].as_array_mut() { for pkg in packages { - let is_lockfile_only = pkg["purl"].as_str().is_some_and(|p| { - lockfile_only - .purls - .contains(normalize_purl(strip_purl_qualifiers(p)).as_ref()) - }); + let is_lockfile_only = pkg["purl"] + .as_str() + .is_some_and(|p| lockfile_only_contains(&lockfile_only.purls, p)); if is_lockfile_only { pkg["notInstalled"] = serde_json::json!(true); } @@ -2401,7 +2398,7 @@ async fn run_scan( // name them (additive; an annotation, never a filter). if let Some(base) = pkg["purl"] .as_str() - .map(|p| normalize_purl(strip_purl_qualifiers(p)).into_owned()) + .map(canonical_base_purl) .filter(|base| gradle.gradle_purls.contains(base)) { if let Some(locked) = &gradle.locked { diff --git a/crates/socket-patch-cli/src/commands/scan/policy.rs b/crates/socket-patch-cli/src/commands/scan/policy.rs index 483ffad9c..adf404294 100644 --- a/crates/socket-patch-cli/src/commands/scan/policy.rs +++ b/crates/socket-patch-cli/src/commands/scan/policy.rs @@ -11,11 +11,12 @@ use socket_patch_core::api::ranking::cmp_search_results; use socket_patch_core::api::types::PatchSearchResult; use socket_patch_core::manifest::schema::PatchManifest; use socket_patch_core::policy::{ - canon, find_repo_root_with_warnings, patch_severity_order, policy_block, repo_relative_checked, + find_repo_root_with_warnings, patch_severity_order, policy_block, repo_relative_checked, sanitize, severity_name, DiskPolicyFs, FilterReason, FilteredEntry, Offers, PolicyError, PolicySource, PolicyWarning, RetainedEntry, Root, SelectionPolicy, PATCHES_DISABLED, }; use socket_patch_core::utils::purl::normalize_purl; +use socket_patch_core::utils::purl_key::PurlKey; use super::ScanArgs; use crate::hosted_memory::roots::{marker_ecosystem, UNSUPPORTED_MARKERS}; @@ -225,14 +226,14 @@ impl ScanPolicy { .map(|m| { m.patches .iter() - .map(|(purl, record)| (canon(purl), record.uuid.clone())) + .map(|(purl, record)| (PurlKey::new(purl).into_string(), record.uuid.clone())) .collect() }) .unwrap_or_default(); } fn recorded_uuid(&self, purl: &str) -> Option<&str> { - self.recorded.get(&canon(purl)).map(String::as_str) + self.recorded.get(&PurlKey::new(purl).into_string()).map(String::as_str) } /// Step 3: the root, ecosystem and package filters. Returns whether the @@ -250,7 +251,7 @@ impl ScanPolicy { }; let mut report = self.report(); if let Some(uuid) = self.recorded_uuid(purl) { - let key = canon(purl); + let key = PurlKey::new(purl).into_string(); if report.retained_purls.insert(key.clone()) { report.retained.push(RetainedEntry { purl: key, @@ -264,9 +265,9 @@ impl ScanPolicy { } if self.root_verdict.is_err() { // Already reported as the root's one entry. - } else if report.filtered_purls.insert(canon(purl)) { + } else if report.filtered_purls.insert(PurlKey::new(purl).into_string()) { report.filtered.push(FilteredEntry { - purl: Some(canon(purl)), + purl: Some(PurlKey::new(purl).into_string()), uuid: None, project: self.project.clone(), reason, @@ -279,7 +280,7 @@ impl ScanPolicy { /// Record the purls with a newer patch (`updates[]`), for /// `retained[].upgradeAvailable`. pub(crate) fn set_update_purls<'a>(&self, purls: impl IntoIterator) { - self.report().update_purls = purls.into_iter().map(canon).collect(); + self.report().update_purls = purls.into_iter().map(|p| PurlKey::new(p).into_string()).collect(); } /// Steps 5-6: group the tier-accessible offers, keep retained packages @@ -293,7 +294,7 @@ impl ScanPolicy { { let report = self.report(); for offer in accessible { - if report.retained_purls.contains(&canon(&offer.purl)) { + if report.retained_purls.contains(&PurlKey::new(&offer.purl).into_string()) { continue; } grouped.entry(offer.purl.clone()).or_default().push(offer); @@ -313,7 +314,7 @@ impl ScanPolicy { let reason = FilterReason::Disabled; match recorded { Some(uuid) => { - let key = canon(&purl); + let key = PurlKey::new(&purl).into_string(); if report.retained_purls.insert(key.clone()) { report.retained.push(RetainedEntry { purl: key, @@ -325,7 +326,7 @@ impl ScanPolicy { } } None => report.filtered.push(FilteredEntry { - purl: Some(canon(&purl)), + purl: Some(PurlKey::new(&purl).into_string()), uuid: Some(group[0].uuid.clone()), project: self.project.clone(), severity: Some(patch_severity_order(&group[0])), @@ -357,7 +358,7 @@ impl ScanPolicy { chosen.is_some() && chosen == recorded_at && recorded_at != Some(0); if chosen.is_none() || upgrade_withheld { report.filtered.push(FilteredEntry { - purl: Some(canon(&purl)), + purl: Some(PurlKey::new(&purl).into_string()), uuid: Some(group[0].uuid.clone()), project: self.project.clone(), severity: Some(patch_severity_order(&group[0])), diff --git a/crates/socket-patch-cli/src/commands/scan/rollout.rs b/crates/socket-patch-cli/src/commands/scan/rollout.rs index fe7470a83..e77ddd7ca 100644 --- a/crates/socket-patch-cli/src/commands/scan/rollout.rs +++ b/crates/socket-patch-cli/src/commands/scan/rollout.rs @@ -5,9 +5,8 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; pub(crate) use socket_patch_core::rollout::stage::*; -use socket_patch_core::rollout::{ - canonical_base_purl, severity_label, MaxNew, MaxNewSource, Recorded, RolloutPlan, -}; +use socket_patch_core::rollout::{severity_label, MaxNew, MaxNewSource, Recorded, RolloutPlan}; +use socket_patch_core::utils::purl_key::PurlKey; use super::discovery::UpdateInfo; @@ -16,7 +15,7 @@ use super::discovery::UpdateInfo; pub(super) fn upgrades(rows: &[Row], package_purls: &[String]) -> Vec { let by_base: BTreeMap = package_purls .iter() - .map(|p| (canonical_base_purl(p), p)) + .map(|p| (PurlKey::new(p).into_string(), p)) .collect(); let mut seen: HashSet = HashSet::new(); let mut out = Vec::new(); @@ -68,7 +67,7 @@ impl<'a> Gate<'a> { && self .stage .already_admitted - .contains(&canonical_base_purl(purl))) + .contains(&PurlKey::new(purl).into_string())) } } @@ -84,13 +83,13 @@ pub(super) fn merge_updates( let offered: BTreeSet = offers .unfiltered .keys() - .map(|p| canonical_base_purl(p)) + .map(|p| PurlKey::new(p).into_string()) .collect(); let mut out = upgrades(rows, package_purls); out.extend( batch .into_iter() - .filter(|u| !offered.contains(&canonical_base_purl(&u.purl))), + .filter(|u| !offered.contains(&PurlKey::new(&u.purl).into_string())), ); out.sort_by(|a, b| a.purl.cmp(&b.purl)); out.dedup_by(|a, b| a.purl == b.purl); diff --git a/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs b/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs index 7240cd1f1..3c085ecfb 100644 --- a/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs +++ b/crates/socket-patch-cli/src/commands/scan/vendor_flow.rs @@ -21,8 +21,7 @@ use socket_patch_core::api::types::{BatchPackagePatches, PatchResponse, PatchSea use socket_patch_core::manifest::operations::{read_manifest, write_manifest}; use socket_patch_core::manifest::schema::PatchRecord; use socket_patch_core::telemetry::{track_patch_vendor_failed, PendingTelemetry}; -use socket_patch_core::utils::composer_version::composer_purls_equivalent; -use socket_patch_core::utils::purl::strip_purl_qualifiers; +use socket_patch_core::utils::purl_key::PurlKey; use socket_patch_core::vendor::{load_state, lookup_entry, save_state, VendorState}; use std::collections::{HashMap, HashSet}; use std::path::Path; @@ -481,16 +480,11 @@ async fn migrate_legacy_manifest_records( if !(entry.detached && entry.record.is_some() && entry.uuid == record.uuid) { continue; } - let base = strip_purl_qualifiers(&entry.base_purl); + let base = PurlKey::new(&entry.base_purl); let keys: Vec = manifest .patches .keys() - .filter(|k| { - *k == &key - || *k == purl - || strip_purl_qualifiers(k) == base - || composer_purls_equivalent(k, base) - }) + .filter(|k| *k == &key || *k == purl || PurlKey::new(k) == base) .cloned() .collect(); for k in keys { @@ -550,7 +544,7 @@ async fn run_vendor_json_path( manifest_path: &Path, socket_dir: &Path, scanned_purls: &HashSet, - vendored_purls: &HashSet, + vendored_purls: &HashSet, prune: bool, telemetry_token: Option<&str>, telemetry_org: Option<&str>, @@ -730,7 +724,7 @@ async fn run_vendor_interactive_path( manifest_path: &Path, socket_dir: &Path, scanned_purls: &HashSet, - vendored_purls: &HashSet, + vendored_purls: &HashSet, prune: bool, telemetry_token: Option<&str>, telemetry_org: Option<&str>, @@ -915,7 +909,7 @@ pub(super) fn boxed_vendor_json_path<'a>( manifest_path: &'a Path, socket_dir: &'a Path, scanned_purls: &'a HashSet, - vendored_purls: &'a HashSet, + vendored_purls: &'a HashSet, prune: bool, telemetry_token: Option<&'a str>, telemetry_org: Option<&'a str>, @@ -958,7 +952,7 @@ pub(super) fn boxed_vendor_interactive_path<'a>( manifest_path: &'a Path, socket_dir: &'a Path, scanned_purls: &'a HashSet, - vendored_purls: &'a HashSet, + vendored_purls: &'a HashSet, prune: bool, telemetry_token: Option<&'a str>, telemetry_org: Option<&'a str>, diff --git a/crates/socket-patch-cli/src/commands/vendor.rs b/crates/socket-patch-cli/src/commands/vendor.rs index 5413d6327..4641794b9 100644 --- a/crates/socket-patch-cli/src/commands/vendor.rs +++ b/crates/socket-patch-cli/src/commands/vendor.rs @@ -28,10 +28,10 @@ use socket_patch_core::manifest::schema::{PatchManifest, PatchRecord}; use socket_patch_core::patch::apply::{verify_file_patch, PatchSources}; use socket_patch_core::patch::redirect::upstream::HostedPin; use socket_patch_core::telemetry::{track_patch_vendor_failed, track_patch_vendored}; -use socket_patch_core::utils::composer_version::composer_purls_equivalent; use socket_patch_core::utils::concurrent::ordered_concurrent; use socket_patch_core::utils::group_commit::{CommittedFile, GroupCommit}; -use socket_patch_core::utils::purl::{canonical_purl, normalize_purl, strip_purl_qualifiers}; +use socket_patch_core::utils::purl::{normalize_purl, strip_purl_qualifiers}; +use socket_patch_core::utils::purl_key::PurlKey; use socket_patch_core::utils::socket_dir::remove_tree_and_prune; use socket_patch_core::vendor::{ self, ecosystem_dir_for_purl, load_state, lock_inventory, lookup_entry, lookup_entry_kv, @@ -1349,10 +1349,7 @@ pub(crate) async fn gem_takeover_refusals_for<'a>( bun_lockb: true, }; for purl in purls.filter(|p| p.starts_with("pkg:gem/")) { - let Some(pin) = pins - .iter() - .find(|pin| canonical_purl(&pin.purl) == canonical_purl(purl)) - else { + let Some(pin) = pins.iter().find(|pin| PurlKey::same(&pin.purl, purl)) else { continue; }; if let Some(refusal) = gem_takeover_refusal(cwd, purl, pin, &restore_opts).await { @@ -2737,7 +2734,7 @@ pub(crate) async fn vendor_records_reusing( let hosted_pin_of = |purl: &str| { hosted_pins .iter() - .find(|pin| canonical_purl(&pin.purl) == canonical_purl(purl)) + .find(|pin| PurlKey::same(&pin.purl, purl)) }; // Yarn berry / npm package-lock takeover preflight (see @@ -3536,11 +3533,8 @@ pub(crate) async fn vendor_records_reusing( .collect(); unmatched.sort(); // A base that vendored one variant accounts for its qualified siblings. - let vendored_bases: HashSet = matched - .iter() - .map(|p| strip_purl_qualifiers(p).to_string()) - .collect(); - unmatched.retain(|p| !vendored_bases.contains(strip_purl_qualifiers(p))); + let vendored_bases: HashSet = matched.iter().map(|p| PurlKey::new(p)).collect(); + unmatched.retain(|p| !vendored_bases.contains(&PurlKey::new(p))); has_errors |= !fetch_failed.is_empty(); if !unmatched.is_empty() { has_errors = true; @@ -4007,16 +4001,16 @@ async fn run_revert(args: &VendorArgs, env: &mut Envelope) -> i32 { // to their upstream registry entries too (a wet run only — a dry revert // wrote nothing to inspect). if !common.dry_run { - let reverted: HashSet = env + let reverted: HashSet = env .events .iter() .filter(|e| e.action == PatchAction::Removed) - .filter_map(|e| e.purl.as_deref().map(canonical_purl)) + .filter_map(|e| e.purl.as_deref().map(PurlKey::new)) .collect(); let rehosted: Vec = HostedPin::all(&crate::commands::discover_wiring(common, &common.cwd).await) .into_iter() - .filter(|pin| reverted.contains(&canonical_purl(&pin.purl))) + .filter(|pin| reverted.contains(&PurlKey::new(&pin.purl))) .collect(); if !rehosted.is_empty() { let leg = crate::commands::rollback::run_hosted_leg(common, &rehosted).await; @@ -4153,6 +4147,26 @@ pub(crate) struct VendorGcSummary { /// happened on disk; the stale record is what the caller must report. pub write_failures: Vec<(&'static str, String)>, } +/// The manifest keys an unused vendored `entry`, stored under ledger key +/// `purl`, owns: every key with the same [`PurlKey`] as the ledger key OR +/// the entry's base purl ([`VendorEntry::covers_purl`]: any qualifier set, +/// encoding, NuGet case, PEP 503 spelling or composer release padding). The +/// base purl matters for golang, whose ledger key may keep the module +/// proxy's `!x` case encoding (`!burnt!sushi`) while the manifest holds the +/// decoded `BurntSushi` spelling. The ONE relation behind the wet vendor +/// GC's manifest drop and `scan --prune --dry-run`'s preview of it, so the +/// two never report different prune sets. +pub(crate) fn unused_vendored_manifest_keys( + patches: &std::collections::HashMap, + purl: &str, + entry: &VendorEntry, +) -> Vec { + patches + .keys() + .filter(|k| k.as_str() == purl || entry.covers_purl(purl, k)) + .cloned() + .collect() +} /// The vendored-state GC behind `scan --prune`: /// @@ -4268,18 +4282,7 @@ pub(crate) async fn run_vendor_gc( state.entries.remove(&purl); ledger_dirty = true; if let Some(m) = manifest.as_mut() { - let base = strip_purl_qualifiers(&entry.base_purl).to_string(); - let dropped: Vec = m - .patches - .keys() - .filter(|k| { - *k == &purl - || strip_purl_qualifiers(k) == base - || composer_purls_equivalent(k, &base) - }) - .cloned() - .collect(); - for k in dropped { + for k in unused_vendored_manifest_keys(&m.patches, &purl, &entry) { m.patches.remove(&k); manifest_dirty = true; } @@ -6878,3 +6881,100 @@ mod eject_snapshot_tests { assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); } } + +#[cfg(test)] +mod unused_vendored_manifest_keys_tests { + use super::unused_vendored_manifest_keys; + use socket_patch_core::vendor::state::{VendorArtifact, VendorEntry}; + use std::collections::HashMap; + + /// A ledger entry whose base purl is `base_purl`; only the purl + /// matters to the manifest-key relation. + fn entry(ecosystem: &str, base_purl: &str) -> VendorEntry { + VendorEntry { + ecosystem: ecosystem.into(), + base_purl: base_purl.into(), + uuid: "11111111-1111-4111-8111-111111111111".into(), + artifact: VendorArtifact { + yarn_berry10c0: None, + path: String::new(), + sha256: String::new(), + size: None, + platform_locked: None, + file_inventory: None, + }, + wiring: Vec::new(), + lock: None, + took_over_go_patches: false, + detached: false, + record: None, + flavor: None, + uv: None, + pnpm: None, + poetry: None, + pdm: None, + pipenv: None, + } + } + + /// The keys an unused ledger entry `key` (base purl = the key) owns. + fn keys_for(patches: &HashMap, eco: &str, key: &str) -> Vec { + let mut out = unused_vendored_manifest_keys(patches, key, &entry(eco, key)); + out.sort(); + out + } + + /// A golang ledger key may keep the module proxy's `!x` case encoding + /// while the entry's base purl and the manifest key are the decoded + /// spelling; [`PurlKey`](socket_patch_core::utils::purl_key::PurlKey) + /// does not decode `!x`, so the base purl must be matched too or the + /// manifest entry (and its blobs) survive the revert. + #[test] + fn covers_the_decoded_golang_base_purl_of_a_bang_encoded_key() { + let patches: HashMap = [ + "pkg:golang/github.com/BurntSushi/toml@v1.0.0", + "pkg:golang/github.com/BurntSushi/toml@v1.1.0", + ] + .into_iter() + .map(|k| (k.to_string(), ())) + .collect(); + let key = "pkg:golang/github.com/!burnt!sushi/toml@v1.0.0"; + let e = entry("golang", "pkg:golang/github.com/BurntSushi/toml@v1.0.0"); + assert_eq!( + unused_vendored_manifest_keys(&patches, key, &e), + vec!["pkg:golang/github.com/BurntSushi/toml@v1.0.0".to_string()] + ); + } + + /// The wet vendor GC and `scan --prune --dry-run`'s preview both drop + /// these keys, so they must cover every spelling of the release and + /// nothing else. + #[test] + fn covers_every_spelling_of_the_release() { + let patches: HashMap = [ + "pkg:nuget/Newtonsoft.Json@13.0.1", + "pkg:nuget/newtonsoft.json@13.0.1?x=1", + "pkg:nuget/newtonsoft.json@13.0.2", + "pkg:pypi/typing-extensions@4.12.2", + "pkg:composer/psr/log@3.0.2", + ] + .into_iter() + .map(|k| (k.to_string(), ())) + .collect(); + assert_eq!( + keys_for(&patches, "nuget", "pkg:nuget/newtonsoft.json@13.0.1"), + vec![ + "pkg:nuget/Newtonsoft.Json@13.0.1".to_string(), + "pkg:nuget/newtonsoft.json@13.0.1?x=1".to_string(), + ] + ); + assert_eq!( + keys_for(&patches, "pypi", "pkg:pypi/typing_extensions@4.12.2"), + vec!["pkg:pypi/typing-extensions@4.12.2".to_string()] + ); + assert_eq!( + keys_for(&patches, "composer", "pkg:composer/psr/log@3.0.2.0"), + vec!["pkg:composer/psr/log@3.0.2".to_string()] + ); + } +} diff --git a/crates/socket-patch-cli/src/commands/vendored_backend/repair.rs b/crates/socket-patch-cli/src/commands/vendored_backend/repair.rs index 464feec93..8cd3790c9 100644 --- a/crates/socket-patch-cli/src/commands/vendored_backend/repair.rs +++ b/crates/socket-patch-cli/src/commands/vendored_backend/repair.rs @@ -343,8 +343,7 @@ impl VendoredBackend<'_> { ); continue; } - if socket_patch_core::utils::purl::canonical_purl(purl) - != socket_patch_core::utils::purl::canonical_purl(&entry.base_purl) + if !socket_patch_core::utils::purl_key::PurlKey::same(purl, &entry.base_purl) || !vendor::is_vendorable(&entry.base_purl) { fail( diff --git a/crates/socket-patch-cli/src/commands/vex_sources.rs b/crates/socket-patch-cli/src/commands/vex_sources.rs index 12b317da8..36a64ddf5 100644 --- a/crates/socket-patch-cli/src/commands/vex_sources.rs +++ b/crates/socket-patch-cli/src/commands/vex_sources.rs @@ -79,10 +79,10 @@ use socket_patch_core::manifest::schema::{PatchManifest, PatchRecord}; use socket_patch_core::patch::redirect::RedirectState; use socket_patch_core::utils::concurrent::{api_concurrency, ordered_concurrent}; use socket_patch_core::utils::purl::strip_purl_qualifiers; +use socket_patch_core::utils::purl_key::{canonical_base_purl, PurlKey}; use socket_patch_core::vendor::state::{VendorArtifact, VendorEntry, VendorState}; use socket_patch_core::vex::discover::{ - canonical_base_purl, same_release, vendor_ref, Discovery, LedgerLiveness, PatchedRef, - WiringMode, + vendor_ref, Discovery, LedgerLiveness, PatchedRef, WiringMode, }; use socket_patch_core::vex::{FailedPatch, UnattestedKind}; @@ -281,7 +281,7 @@ impl Cand { let vendor_entry = vendor .entries .values() - .find(|e| e.uuid == r.uuid && same_release(&canonical_base_purl(&e.base_purl), &r.purl)) + .find(|e| e.uuid == r.uuid && PurlKey::same(&e.base_purl, &r.purl)) .cloned(); Cand { key: r.purl.clone(), @@ -329,7 +329,7 @@ pub(crate) async fn plan(common: &GlobalArgs, sources: Sources, assume_live: &[S let mut conflicted_keys: BTreeMap<&str, Vec> = BTreeMap::new(); cands.retain(|c| { let pkg = canonical_base_purl(&c.key); - match conflicts.iter().find(|(k, _)| same_release(k, &pkg)) { + match conflicts.iter().find(|(k, _)| PurlKey::same(k, &pkg)) { Some((k, _)) => { conflicted_keys .entry(k.as_str()) @@ -370,7 +370,7 @@ pub(crate) async fn plan(common: &GlobalArgs, sources: Sources, assume_live: &[S let Some(u) = discovery .unattested .iter() - .find(|u| u.uuid == c.uuid && same_release(&u.purl, &pkg)) + .find(|u| u.uuid == c.uuid && PurlKey::same(&u.purl, &pkg)) else { return true; }; @@ -498,9 +498,7 @@ pub(crate) async fn plan(common: &GlobalArgs, sources: Sources, assume_live: &[S } let expected_pkg = expected_package(cand); match local_record_by_uuid(&cand.uuid, &manifest, &redirect_records, &vendor) { - Some((found_key, record)) - if same_release(&canonical_base_purl(&found_key), &expected_pkg) => - { + Some((found_key, record)) if PurlKey::same(&found_key, &expected_pkg) => { if cand.lockfile_only { cand.key = found_key; } @@ -524,8 +522,7 @@ pub(crate) async fn plan(common: &GlobalArgs, sources: Sources, assume_live: &[S let cand = &mut based[i].0; match fetched.get(&cand.uuid) { Some((api_purl, record)) - if record.uuid == cand.uuid - && same_release(&canonical_base_purl(api_purl), &expected_package(cand)) => + if record.uuid == cand.uuid && PurlKey::same(api_purl, &expected_package(cand)) => { if cand.lockfile_only { cand.key = api_purl.clone(); @@ -715,7 +712,7 @@ fn attach_discovered( let mut superseded = Vec::new(); for (pkg, refs) in groups { let idxs: Vec = (0..cands.len()) - .filter(|&i| same_release(&canonical_base_purl(&cands[i].key), pkg)) + .filter(|&i| PurlKey::same(&cands[i].key, pkg)) .collect(); // Pass 1: the candidate already attests the wired uuid. let mut unmatched: Vec<&PatchedRef> = Vec::new(); diff --git a/crates/socket-patch-cli/src/ecosystem_dispatch.rs b/crates/socket-patch-cli/src/ecosystem_dispatch.rs index 938270666..c0fa50176 100644 --- a/crates/socket-patch-cli/src/ecosystem_dispatch.rs +++ b/crates/socket-patch-cli/src/ecosystem_dispatch.rs @@ -1,7 +1,8 @@ use socket_patch_core::crawlers::{ CrawledPackage, CrawlerOptions, Ecosystem, NpmCrawler, PythonCrawler, RubyCrawler, }; -use socket_patch_core::utils::purl::{canonical_purl, normalize_purl, strip_purl_qualifiers}; +use socket_patch_core::utils::purl::strip_purl_qualifiers; +use socket_patch_core::utils::purl_key::PurlKey; use std::collections::{HashMap, HashSet}; use std::path::PathBuf; @@ -576,10 +577,10 @@ pub(crate) fn npm_paths_by_identity_in( ) -> HashMap> { let mut out = HashMap::new(); for purl in purls { - let want = canonical_purl(purl); + let want = PurlKey::new(purl); let paths: Vec = installed .iter() - .filter(|pkg| normalize_purl(&pkg.purl) == want) + .filter(|pkg| PurlKey::new(&pkg.purl) == want) .map(|pkg| pkg.path.clone()) .collect(); if !paths.is_empty() { diff --git a/crates/socket-patch-cli/tests/docker_e2e_nuget.rs b/crates/socket-patch-cli/tests/docker_e2e_nuget.rs index 9ea858d11..cca26cc7d 100644 --- a/crates/socket-patch-cli/tests/docker_e2e_nuget.rs +++ b/crates/socket-patch-cli/tests/docker_e2e_nuget.rs @@ -24,11 +24,12 @@ use wiremock::{Mock, MockServer, ResponseTemplate}; const ORG: &str = "test-org"; // The nuget crawler reports installed packages with the lowercased -// directory name (because ~/.nuget/packages stores them as lowercase -// dirs). The wiremock fixture must return the same casing so scan's -// GC pass doesn't prune the freshly-saved manifest entry as -// "not-in-scanned-purls". +// directory name (~/.nuget/packages stores them as lowercase dirs), and the +// batch endpoint echoes the queried purl back; the patch itself carries the +// API's own mixed-case spelling, which lands in the manifest verbatim. NuGet +// ids are case-insensitive, so scan's GC pass must keep that entry (B20). const PURL: &str = "pkg:nuget/newtonsoft.json@13.0.3"; +const API_PURL: &str = "pkg:nuget/Newtonsoft.Json@13.0.3"; const UUID: &str = "18181818-1818-4181-8181-181818181818"; /// The vulnerability the staged manifest carries so the agent-mode VEX leg /// has something to attest (plain agent provenance — no vendored/redirected @@ -87,7 +88,7 @@ async fn make_mock_server(after_hash: &str) -> MockServer { "packages": [{ "purl": PURL, "patches": [{ - "uuid": UUID, "purl": PURL, + "uuid": UUID, "purl": API_PURL, "tier": "free", "cveIds": [], "ghsaIds": [], "severity": "medium", "title": "nuget e2e fixture" }] @@ -103,7 +104,7 @@ async fn make_mock_server(after_hash: &str) -> MockServer { ))) .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "patches": [{ - "uuid": UUID, "purl": PURL, + "uuid": UUID, "purl": API_PURL, "publishedAt": "2024-01-01T00:00:00Z", "description": "nuget e2e fixture", "license": "MIT", "tier": "free", @@ -119,7 +120,7 @@ async fn make_mock_server(after_hash: &str) -> MockServer { .and(path(format!("/v0/orgs/{ORG}/patches/view/{UUID}"))) .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "uuid": UUID, - "purl": PURL, + "purl": API_PURL, "publishedAt": "2024-01-01T00:00:00Z", "files": { // nuget uses `package/`; apply strips and joins @@ -592,7 +593,8 @@ async fn nuget_local_install_full_apply_chain() { stderr.contains("===VEX VERIFIED==="), "agent-mode VEX leg did not run/pass (===VEX VERIFIED=== missing).\nstderr=\n{stderr}" ); - assert_vex_agent_attested(&stdout, PURL); + // Agent VEX names the manifest key: the API's spelling. + assert_vex_agent_attested(&stdout, API_PURL); assert!( stderr.contains("===MANIFESTLESS VEX VERIFIED==="), "manifest-less agent-mode VEX leg did not run/pass.\nstderr=\n{stderr}" diff --git a/crates/socket-patch-cli/tests/remove/covgap_commands_remove.rs b/crates/socket-patch-cli/tests/remove/covgap_commands_remove.rs index 1f5776505..3e60c9fa7 100644 --- a/crates/socket-patch-cli/tests/remove/covgap_commands_remove.rs +++ b/crates/socket-patch-cli/tests/remove/covgap_commands_remove.rs @@ -1336,7 +1336,7 @@ fn remove_drift_keep_excludes_manifest_entry_by_base_purl() { /// A golang ledger key carries the module path case-ENCODED /// (`!burnt!sushi`) while `basePurl` holds the decoded form users type; -/// `purl_eq` does not decode `!x` escaping, so only the base_purl arm can +/// `PurlKey` does not decode `!x` escaping, so only the base_purl arm can /// match. The dry-run preview proves the match end-to-end without needing /// a working golang revert. #[test] diff --git a/crates/socket-patch-core/src/hosted/memory/discover.rs b/crates/socket-patch-core/src/hosted/memory/discover.rs index 9469b50ba..83f2bc039 100644 --- a/crates/socket-patch-core/src/hosted/memory/discover.rs +++ b/crates/socket-patch-core/src/hosted/memory/discover.rs @@ -16,6 +16,7 @@ use crate::api::client::{ApiError, ApiFuture, PatchApi}; use crate::api::ranking::cmp_search_results; use crate::api::types::{BatchPackagePatches, PackageVendorResult, PatchResponse, SearchResponse}; use crate::utils::purl::{normalize_purl, strip_purl_qualifiers}; +use crate::utils::purl_key::PurlKey; use super::types::MAX_REFERENCE_BATCH; @@ -200,6 +201,19 @@ pub(crate) async fn batch_search( owners.entry(purl.as_str()).or_default().push(root.as_str()); } } + // Response packages are matched to their askers by identity, not by + // spelling: the API may answer `Newtonsoft.Json` for a crawled + // `newtonsoft.json`, `typing_extensions` for `typing-extensions`, or a + // composer `@3.0.2.0` for `@3.0.2`. + let mut owners_by_key: HashMap> = HashMap::new(); + for (purl, roots) in &owners { + let list = owners_by_key.entry(PurlKey::new(purl)).or_default(); + for root in roots { + if !list.contains(root) { + list.push(root); + } + } + } let union: Vec = owners.keys().map(|p| p.to_string()).collect(); let chunks: Vec> = union .chunks(batch_size.max(1)) @@ -229,8 +243,7 @@ pub(crate) async fn batch_search( if pkg.patches.is_empty() { continue; } - let key = normalize_purl(strip_purl_qualifiers(&pkg.purl)).into_owned(); - let targets: Vec<&str> = match owners.get(key.as_str()) { + let targets: Vec<&str> = match owners_by_key.get(&PurlKey::new(&pkg.purl)) { Some(roots) => roots.clone(), None => chunk_roots.iter().copied().collect(), }; @@ -468,4 +481,77 @@ mod tests { ); assert_eq!(pairs(true)[0].1, "a-paid"); } + + /// Answers every batch with the mixed-case NuGet spelling of + /// `pkg:nuget/newtonsoft.json@13.0.1`, as the API does. + struct CasedNuget; + + impl PatchApi for CasedNuget { + fn uses_public_proxy(&self) -> bool { + false + } + fn search_patches_batch<'a>( + &'a self, + _purls: &'a [String], + ) -> ApiFuture<'a, crate::api::types::BatchSearchResponse> { + Box::pin(async { + Ok(crate::api::types::BatchSearchResponse { + packages: vec![crate::api::types::BatchPackagePatches { + purl: "pkg:nuget/Newtonsoft.Json@13.0.1".to_string(), + patches: vec![crate::api::types::BatchPatchInfo { + uuid: "u-1".to_string(), + purl: "pkg:nuget/Newtonsoft.Json@13.0.1".to_string(), + tier: "free".to_string(), + cve_ids: Vec::new(), + ghsa_ids: Vec::new(), + severity: None, + title: String::new(), + published_at: None, + }], + }], + can_access_paid_patches: false, + }) + }) + } + fn search_patches_by_package<'a>( + &'a self, + _purl: &'a str, + ) -> ApiFuture<'a, crate::api::types::SearchResponse> { + Box::pin(async { Err(ApiError::Other("unused".into())) }) + } + fn fetch_registry_references<'a>( + &'a self, + _uuids: &'a [String], + ) -> ApiFuture<'a, HashMap> { + Box::pin(async { Err(ApiError::Other("unused".into())) }) + } + fn fetch_patch<'a>( + &'a self, + _uuid: &'a str, + ) -> ApiFuture<'a, Option> { + Box::pin(async { Err(ApiError::Other("unused".into())) }) + } + fn download_artifact<'a>(&'a self, _url: &'a str, _max: u64) -> ApiFuture<'a, Vec> { + Box::pin(async { Err(ApiError::Other("unused".into())) }) + } + } + + #[tokio::test] + async fn batch_search_credits_a_respelled_response_only_to_its_asker() { + let provider = Provider::new(Arc::new(CasedNuget), Duration::from_secs(5), 2); + let mut roots: BTreeMap> = BTreeMap::new(); + roots.insert( + "a".to_string(), + vec!["pkg:nuget/newtonsoft.json@13.0.1".to_string()], + ); + roots.insert("b".to_string(), vec!["pkg:npm/left-pad@1.3.0".to_string()]); + // One chunk holds both roots' purls: a spelling-keyed owner lookup + // misses and falls back to crediting every root in the chunk. + let outcome = batch_search(&provider, &roots, 10).await; + assert_eq!(outcome.roots["a"].packages.len(), 1); + assert!( + outcome.roots["b"].packages.is_empty(), + "a NuGet package must not be credited to a root that never asked for it" + ); + } } diff --git a/crates/socket-patch-core/src/hosted/memory/limits.rs b/crates/socket-patch-core/src/hosted/memory/limits.rs index da4dd695d..f7cec2339 100644 --- a/crates/socket-patch-core/src/hosted/memory/limits.rs +++ b/crates/socket-patch-core/src/hosted/memory/limits.rs @@ -149,7 +149,7 @@ pub(crate) fn resolve_options(options: &HostedScanOptions) -> Result+"`; the sha comes from the /// `SOCKET_PATCH_GIT_SHA` build-time variable. @@ -613,7 +613,7 @@ async fn engine( match policy.admits_purl(&purl) { Ok(()) => admitted.push(purl), Err(reason) => policy_filtered.push(FilteredEntry { - purl: Some(canon(&purl)), + purl: Some(PurlKey::new(&purl).into_string()), uuid: None, project: state.root.clone(), reason, @@ -1175,7 +1175,7 @@ fn select_with_policy( detail: Some(reason.detail()), }); filtered.push(FilteredEntry { - purl: Some(canon(&purl)), + purl: Some(PurlKey::new(&purl).into_string()), uuid: Some(winner.uuid.clone()), project: root.to_string(), severity: Some(patch_severity_order(&winner)), diff --git a/crates/socket-patch-core/src/ledgers.rs b/crates/socket-patch-core/src/ledgers.rs index 00a86caa5..52d7c1b1a 100644 --- a/crates/socket-patch-core/src/ledgers.rs +++ b/crates/socket-patch-core/src/ledgers.rs @@ -29,9 +29,9 @@ use std::path::Path; use crate::manifest::schema::{PatchManifest, PatchRecord}; use crate::patch::redirect::upstream::HostedPin; use crate::patch::redirect::{CorruptRedirectState, RedirectState}; -use crate::utils::purl::{normalize_purl, patch_matches, strip_purl_qualifiers}; +use crate::utils::purl::{canonical_purl, patch_matches}; +use crate::utils::purl_key::PurlKey; use crate::vendor::{VendorEntry, VendorState}; -use crate::vex::discover::{canonical_base_purl, same_release}; /// A patch store, in owner-precedence order (a lower store wins a key). #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -150,7 +150,7 @@ pub fn hosted_pins_matching( patch_matches(&pin.purl, &pin.uuid, identifier) || manifest_keys .iter() - .any(|key| same_release(&canonical_base_purl(key), &pin.purl)) + .any(|key| PurlKey::same(key, &pin.purl)) }) .cloned() .collect(); @@ -372,9 +372,10 @@ impl<'a> Ledgers<'a> { } } - /// The purls both the hosted records and the vendored ledger claim, - /// canonical (qualifiers dropped, percent-decoded), sorted: each is - /// stale in exactly one store, which only the live lockfile can tell. + /// The purls both the hosted records and the vendored ledger claim, in + /// the records' display spelling ([`canonical_purl`]: qualifiers + /// dropped, percent-decoded), sorted and deduplicated: each is stale in + /// exactly one store, which only the live lockfile can tell. pub fn hosted_vendored_overlap(&self) -> Vec { let (Some(redirect), Some(vendor)) = (self.redirect, self.vendor) else { return Vec::new(); @@ -382,22 +383,32 @@ impl<'a> Ledgers<'a> { if vendor.entries.is_empty() { return Vec::new(); } - // Canonicalize both sides (drop qualifiers, percent-decode) so the API - // purl form the redirect records carry matches the vendor entry's base - // purl — mirrors `vendored_ledger_supplement`. - let canon = |p: &str| normalize_purl(strip_purl_qualifiers(p)).into_owned(); - let mut vendor_purls: std::collections::BTreeSet = - std::collections::BTreeSet::new(); - for (key, entry) in &vendor.entries { - vendor_purls.insert(canon(key)); - vendor_purls.insert(canon(&entry.base_purl)); + // Match by `PurlKey`, so the API purl form the redirect records carry + // matches the vendor entry's key or base purl in any spelling. + // Two spellings of one release (composer `@3.0.2` and + // `@3.0.2.0`, NuGet case twins) are ONE overlap: deduplicate by + // `PurlKey`, reporting the smallest display spelling. + let vendor_purls = vendor.purl_keys(); + let mut overlap: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + for purl in redirect + .records + .keys() + .filter(|p| crate::vendor::purl_keys_cover(&vendor_purls, p)) + { + let display = canonical_purl(purl); + overlap + .entry(PurlKey::new(purl)) + .and_modify(|kept| { + if display < *kept { + *kept = display.clone(); + } + }) + .or_insert(display); } - let redirect_purls: std::collections::BTreeSet = - redirect.records.keys().map(|p| canon(p)).collect(); - redirect_purls - .intersection(&vendor_purls) - .cloned() - .collect() + let mut out: Vec = overlap.into_values().collect(); + out.sort(); + out } } @@ -619,6 +630,39 @@ mod tests { assert!(l.matching("nope").is_empty()); assert_eq!(l.matching("hb").hosted, vec!["pkg:npm/b@1"]); } + + #[test] + fn overlap_reports_one_entry_per_release_across_spellings() { + let v = vendor(vec![ + ( + "pkg:nuget/newtonsoft.json@13.0.1", + entry("v", "pkg:nuget/newtonsoft.json@13.0.1", true, None), + ), + ( + "pkg:composer/acme/lib@3.0.2", + entry("c", "pkg:composer/acme/lib@3.0.2", true, None), + ), + ]); + let r = redirect(&[ + ("pkg:nuget/Newtonsoft.Json@13.0.1", "h1"), + ("pkg:nuget/newtonsoft.json@13.0.1", "h2"), + ("pkg:composer/acme/lib@3.0.2", "h3"), + ("pkg:composer/acme/lib@3.0.2.0", "h4"), + ]); + let l = Ledgers { + manifest: None, + vendor: Some(&v), + redirect: Some(&r), + }; + assert_eq!( + l.hosted_vendored_overlap(), + vec![ + "pkg:composer/acme/lib@3.0.2".to_string(), + "pkg:nuget/Newtonsoft.Json@13.0.1".to_string(), + ] + ); + } + /// #999: a uuid identifier that matches the manifest's generation also /// selects the vendored entry the matched key claims, even when the /// ledger recorded an older generation; an unclaimed entry for another diff --git a/crates/socket-patch-core/src/patch/redirect/golang_local.rs b/crates/socket-patch-core/src/patch/redirect/golang_local.rs index 415dd4903..2c5868ec2 100644 --- a/crates/socket-patch-core/src/patch/redirect/golang_local.rs +++ b/crates/socket-patch-core/src/patch/redirect/golang_local.rs @@ -30,7 +30,7 @@ use crate::patch::apply::{ MismatchPolicy, PatchSources, }; use crate::patch::file_hash::compute_file_git_sha256; -use crate::utils::purl::{build_golang_purl, canonical_purl, parse_golang_purl}; +use crate::utils::purl::{build_golang_purl, parse_golang_purl}; use crate::vendor::common::{ already_patched_result, copy_matches_after_hashes, synthesized_result, }; @@ -393,14 +393,18 @@ pub async fn reconcile_go_redirects( // (b) Orphan copy dirs not referenced by a desired PURL (catches copies left // behind by a hand-deleted directive or a version bump). A desired manifest // key may carry `?qualifiers`/`#subpath` (raw API PURL), while the PURL - // reconstructed from the copy dir is the canonical base — compare bases, or - // a qualified key's freshly applied copy is pruned as an orphan. - let desired_bases: HashSet = desired.iter().map(|p| canonical_purl(p)).collect(); + // reconstructed from the copy dir is the canonical base — compare by + // `PurlKey`, or a qualified (or `%2B`-encoded) key's freshly applied copy + // is pruned as an orphan. + let desired_bases: HashSet = desired + .iter() + .map(|p| crate::utils::purl_key::PurlKey::new(p)) + .collect(); // Re-read after (a)'s drops so the dangling-directive probe below sees the // current file. let entries = read_replace_entries(project_root).await; for (purl, dir) in collect_copy_modules(&project_root.join(GO_PATCHES_DIR)).await { - if !desired_bases.contains(&purl) { + if !desired_bases.contains(&crate::utils::purl_key::PurlKey::new(&purl)) { // A go-patches directive still targeting THIS copy dangles once the // copy is pruned — loop (a) keeps it whenever the module is desired // at ANOTHER version (a bump whose apply hasn't succeeded), and a @@ -1927,7 +1931,7 @@ mod tests { .await; assert!(result.success, "apply failed: {:?}", result.error); - // Reconcile with the same encoded manifest key — canonical_purl + // Reconcile with the same encoded manifest key — PurlKey // decodes both sides, so they match. let desired: HashSet = [encoded_purl.to_string()].into_iter().collect(); let removed = reconcile_go_redirects(root, &desired, false).await; diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/gradle.rs b/crates/socket-patch-core/src/patch/redirect/upstream/gradle.rs index ea03dfa5b..4497e5d9c 100644 --- a/crates/socket-patch-core/src/patch/redirect/upstream/gradle.rs +++ b/crates/socket-patch-core/src/patch/redirect/upstream/gradle.rs @@ -85,7 +85,7 @@ pub(crate) async fn restore( ); continue; }; - if crate::vex::discover::canonical_base_purl(&pin.purl) != row.purl() { + if !crate::utils::purl_key::PurlKey::same(&pin.purl, &row.purl()) { result.refuse( &pin.uuid, format!( diff --git a/crates/socket-patch-core/src/patch/redirect/vlt_heal.rs b/crates/socket-patch-core/src/patch/redirect/vlt_heal.rs index caf0f4759..ed6b07cf6 100644 --- a/crates/socket-patch-core/src/patch/redirect/vlt_heal.rs +++ b/crates/socket-patch-core/src/patch/redirect/vlt_heal.rs @@ -20,7 +20,8 @@ use crate::manifest::schema::PatchRecord; use crate::patch::apply::{is_safe_relative_subpath, normalize_file_path}; use crate::patch::file_hash::compute_file_git_sha256; use crate::patch::package::read_archive_bytes_to_map_strict; -use crate::utils::purl::{canonical_purl, purl_parts}; +use crate::utils::purl::purl_parts; +use crate::utils::purl_key::PurlKey; use crate::vendor::vlt_lock_text::{ is_default_registry, is_registry_package_name, parse_node_entry_text, sniff_lock, split_dep_id, DepIdKind, LockSniff, @@ -465,13 +466,13 @@ pub async fn invalidate(root: &Path, state: &InstallState, stale: &[String]) -> /// targets. No patch record rides along (v5 keeps no hosted ledger), so the /// heal judges each installed copy against the lock's own pins. pub fn lock_targets(lock: &str, origins: &[String], purls: &[String]) -> Vec { - let wanted: Vec = purls.iter().map(|p| canonical_purl(p)).collect(); + let wanted: Vec = purls.iter().map(|p| PurlKey::new(p)).collect(); socket_owned_instances(lock, origins) .into_iter() .filter_map(|instance| { let purl = crate::utils::purl::npm_purl(&instance.name, &instance.version)?; - let canon = canonical_purl(&purl); - let at = wanted.iter().position(|w| *w == canon)?; + let key = PurlKey::new(&purl); + let at = wanted.iter().position(|w| *w == key)?; Some(LedgerTarget { purl: purls[at].clone(), dep_id: instance.dep_id, @@ -500,11 +501,10 @@ pub fn ledger_targets( if ecosystem != "npm" { continue; } - let canon = canonical_purl(purl); let record = state .records .iter() - .find(|(key, _)| canonical_purl(key) == canon) + .find(|(key, _)| PurlKey::same(key, purl)) .map(|(_, record)| record.clone()); for edit in &state.edits { if edit.kind != vlt::KIND diff --git a/crates/socket-patch-core/src/policy/mod.rs b/crates/socket-patch-core/src/policy/mod.rs index e50e186bb..070599590 100644 --- a/crates/socket-patch-core/src/policy/mod.rs +++ b/crates/socket-patch-core/src/policy/mod.rs @@ -19,12 +19,12 @@ use crate::api::ranking::max_severity_order; use crate::api::types::PatchSearchResult; use crate::crawlers::python_crawler::canonicalize_pypi_name; use crate::crawlers::Ecosystem; -use crate::utils::purl::{normalize_purl, strip_purl_qualifiers}; +use crate::utils::purl_key::PurlKey; use self::paths::{PathHit, PathMatcher}; use self::socket_yml::{parse_file, ParsedFile, PatchesBlock}; -pub use self::report::{canon, policy_block, FilteredEntry, RetainedEntry}; +pub use self::report::{policy_block, FilteredEntry, RetainedEntry}; pub use self::socket_yml::MAX_FILE_BYTES; /// Root file names, in the order they are read. @@ -746,17 +746,16 @@ pub struct Offers { /// form, so `typing_extensions` and `typing.extensions` name the project /// whose purl is `pkg:pypi/typing-extensions`. pub fn package_spec_matches(spec: &str, purl: &str) -> bool { - // Versioned Composer specs name a release, including its pretty/padded - // spellings. Compare before lowercasing: dev branch names retain case. - if crate::utils::composer_version::composer_purl_identity(spec.trim()).is_some() { - return crate::utils::composer_version::composer_purls_equivalent(spec.trim(), purl); - } - let decoded = canonical_pypi_purl(normalize_purl(strip_purl_qualifiers(purl)).to_lowercase()); - let spec = spec.trim().to_lowercase(); + let spec = spec.trim(); if spec.is_empty() { return false; } - let Some(rest) = decoded.strip_prefix("pkg:") else { + let key = PurlKey::new(purl); + // A filter spec is matched leniently: the package's identity + // (decoded, PyPI/NuGet/Composer names folded), then compared + // case-insensitively. + let folded = key.as_str().to_lowercase(); + let Some(rest) = folded.strip_prefix("pkg:") else { return false; }; let Some((eco, name_version)) = rest.split_once('/') else { @@ -766,22 +765,28 @@ pub fn package_spec_matches(spec: &str, purl: &str) -> bool { Some(at) => &name_version[..at], None => name_version, }; - if let Some(spec_rest) = spec.strip_prefix("pkg:") { - let spec_purl = canonical_pypi_purl( - normalize_purl(strip_purl_qualifiers(&format!("pkg:{spec_rest}"))).to_lowercase(), - ); - let spec_rest = &spec_purl[4..]; - let has_version = spec_rest + if spec + .get(..4) + .is_some_and(|p| p.eq_ignore_ascii_case("pkg:")) + { + let spec_key = PurlKey::new(&format!("pkg:{}", &spec[4..])); + let spec_folded = spec_key.as_str().to_lowercase(); + let has_version = spec_folded[4..] .split_once('/') .is_some_and(|(_, nv)| nv.rfind('@').is_some_and(|i| i > 0)); - return if has_version { - decoded == spec_purl - } else { - decoded - .strip_prefix(&spec_purl) + return if !has_version { + folded + .strip_prefix(&spec_folded) .is_some_and(|tail| tail.starts_with('@')) + } else if eco == "composer" { + // A versioned Composer spec names a release, including its + // pretty/padded spellings; dev branch names retain case. + spec_key == key + } else { + spec_folded == folded }; } + let spec = spec.to_lowercase(); if eco == "pypi" { return name == canonicalize_pypi_name(&spec); } @@ -789,19 +794,6 @@ pub fn package_spec_matches(spec: &str, purl: &str) -> bool { name == spec || name.rsplit('/').next() == Some(spec.as_str()) } -/// Rewrite the name of a lowercased `pkg:pypi/[@]` purl to -/// its PEP 503 canonical form; any other purl is returned unchanged. -fn canonical_pypi_purl(purl: String) -> String { - let Some(name_version) = purl.strip_prefix("pkg:pypi/") else { - return purl; - }; - let (name, version) = match name_version.rfind('@').filter(|&i| i > 0) { - Some(at) => name_version.split_at(at), - None => (name_version, ""), - }; - format!("pkg:pypi/{}{version}", canonicalize_pypi_name(name)) -} - /// The repo root for `cwd` (4.5) with the lookup's warnings: the checkout /// [`crate::utils::repo_root::find_git_repo`] finds (nearest `.git` /// directory or file, not past `GIT_CEILING_DIRECTORIES` or into the home diff --git a/crates/socket-patch-core/src/policy/report.rs b/crates/socket-patch-core/src/policy/report.rs index 0d095c7dc..cd1d8ad28 100644 --- a/crates/socket-patch-core/src/policy/report.rs +++ b/crates/socket-patch-core/src/policy/report.rs @@ -2,16 +2,6 @@ //! in-memory engine build the same entries and render them here. use super::{severity_name, FilterReason, PolicySource, SelectionPolicy}; -use crate::utils::purl::{normalize_purl, strip_purl_qualifiers}; - -/// The canonical spelling filters and the recorded view compare on. -pub fn canon(purl: &str) -> String { - crate::utils::composer_version::composer_purl_identity(purl) - // This spelling is also emitted in policy reports; keep internal - // invalid-version sentinels out of the user-facing purl. - .filter(|key| !key.contains('\u{1}')) - .unwrap_or_else(|| normalize_purl(strip_purl_qualifiers(purl)).into_owned()) -} /// One `policy.filtered[]` entry. #[derive(Debug, Clone)] diff --git a/crates/socket-patch-core/src/rollout.rs b/crates/socket-patch-core/src/rollout.rs index f3439057b..0e06b6f24 100644 --- a/crates/socket-patch-core/src/rollout.rs +++ b/crates/socket-patch-core/src/rollout.rs @@ -44,7 +44,10 @@ pub struct Candidate { /// Repo-relative project root; `""` is the repo root. pub project: String, pub purl: String, - /// [`canonical_base_purl`] of `purl`: the budget unit. + /// [`crate::utils::purl_key::PurlKey`] of `purl`: the budget unit, so qualifier twins (a wheel + /// and its sdist, gem platforms), the API's encoded spelling and a + /// lockfile's `Newtonsoft.Json` vs a pin's `newtonsoft.json` are one + /// package. pub base_purl: String, /// The selected uuid. pub uuid: String, @@ -134,20 +137,6 @@ pub fn resolve_max_new( } } -/// The budget unit: ecosystem + name + version, qualifiers stripped, -/// percent-decoded and case-folded where the ecosystem is case-insensitive -/// (discovery's [`crate::vex::discover::canonical_base_purl`], the key -/// hosted pins carry), so qualifier twins (a wheel and its sdist, gem -/// platforms), the API's encoded spelling and a lockfile's `Newtonsoft.Json` -/// vs a pin's `newtonsoft.json` are one package. -pub fn canonical_base_purl(purl: &str) -> String { - crate::utils::composer_version::composer_purl_identity(purl) - // Invalid-version identity keys contain an internal sentinel, which - // must not appear in the deferred purls reported to callers. - .filter(|key| !key.contains('\u{1}')) - .unwrap_or_else(|| crate::vex::discover::canonical_base_purl(purl)) -} - /// Rollout order, most urgent first: in-flight, severity, advisory count /// (descending), ecosystem, base purl, uuid. Total, and free of /// time-dependent keys. @@ -296,12 +285,13 @@ pub fn severity_label(order: u8) -> &'static str { #[cfg(test)] mod tests { use super::*; + use crate::utils::purl_key::PurlKey; fn row(project: &str, purl: &str, uuid: &str, severity: u8, advisories: usize) -> Candidate { Candidate { project: project.to_string(), purl: purl.to_string(), - base_purl: canonical_base_purl(purl), + base_purl: PurlKey::new(purl).into_string(), uuid: uuid.to_string(), ecosystem: purl .strip_prefix("pkg:") @@ -583,38 +573,35 @@ mod tests { assert_eq!(next.admitted.len(), 1); assert!(next.deferred.is_empty()); assert_eq!( - crate::policy::canon("pkg:composer/psr/log@3.0.2.0"), - crate::policy::canon("pkg:composer/psr/log@v3.0.2") + PurlKey::new("pkg:composer/psr/log@3.0.2.0").into_string(), + PurlKey::new("pkg:composer/psr/log@v3.0.2").into_string() ); - for key in [ - canonical_base_purl("pkg:composer/psr/log@not-a-version"), - crate::policy::canon("pkg:composer/psr/log@not-a-version"), - ] { - assert!(!key.contains('\u{1}')); - } + assert!(!PurlKey::new("pkg:composer/psr/log@not-a-version") + .as_str() + .contains('\u{1}')); assert_ne!( - canonical_base_purl("pkg:composer/psr/log@dev-Feature"), - canonical_base_purl("pkg:composer/psr/log@dev-feature") + PurlKey::new("pkg:composer/psr/log@dev-Feature").into_string(), + PurlKey::new("pkg:composer/psr/log@dev-feature").into_string() ); } #[test] fn qualifier_twins_share_a_base_purl_and_a_rank() { assert_eq!( - canonical_base_purl("pkg:pypi/foo@1.0?artifact_id=abc"), - canonical_base_purl("pkg:pypi/foo@1.0?artifact_id=def") + PurlKey::new("pkg:pypi/foo@1.0?artifact_id=abc").into_string(), + PurlKey::new("pkg:pypi/foo@1.0?artifact_id=def").into_string() ); assert_eq!( - canonical_base_purl("pkg:npm/%40scope/x@1.0.0"), + PurlKey::new("pkg:npm/%40scope/x@1.0.0").into_string(), "pkg:npm/@scope/x@1.0.0" ); assert_eq!( - canonical_base_purl("pkg:nuget/Newtonsoft.Json@13.0.3"), - canonical_base_purl("pkg:nuget/newtonsoft.json@13.0.3") + PurlKey::new("pkg:nuget/Newtonsoft.Json@13.0.3").into_string(), + PurlKey::new("pkg:nuget/newtonsoft.json@13.0.3").into_string() ); assert_eq!( - canonical_base_purl("pkg:pypi/Foo_Bar@1.0"), - canonical_base_purl("pkg:pypi/foo-bar@1.0") + PurlKey::new("pkg:pypi/Foo_Bar@1.0").into_string(), + PurlKey::new("pkg:pypi/foo-bar@1.0").into_string() ); let rows = vec![ row("", "pkg:pypi/foo@1.0?artifact_id=whl", "u2", 1, 1), diff --git a/crates/socket-patch-core/src/rollout/stage.rs b/crates/socket-patch-core/src/rollout/stage.rs index 7c24ebadf..65ca664b8 100644 --- a/crates/socket-patch-core/src/rollout/stage.rs +++ b/crates/socket-patch-core/src/rollout/stage.rs @@ -13,11 +13,9 @@ use crate::api::types::PatchSearchResult; use crate::crawlers::Ecosystem; use crate::manifest::schema::PatchManifest; pub use crate::policy::Offers; +use crate::utils::purl_key::PurlKey; -use super::{ - canonical_base_purl, plan_rollout, severity_label, Candidate, MaxNew, MaxNewSource, Recorded, - RolloutPlan, -}; +use super::{plan_rollout, severity_label, Candidate, MaxNew, MaxNewSource, Recorded, RolloutPlan}; /// The env binding of `scan --max-new-patches`. pub const MAX_NEW_PATCHES_ENV: &str = "SOCKET_MAX_NEW_PATCHES"; @@ -103,9 +101,9 @@ pub struct Row { #[derive(Debug, Default)] pub struct RecordedIndex { exact: HashMap>, - /// Discovery's folded base purl plus the raw qualifier suffix. - qualified: HashMap>, - by_base: HashMap>, + /// [`PurlKey::qualified`]: one release variant in any spelling. + qualified: HashMap>, + by_base: HashMap>, } /// The recorded view one project root classifies against: the merged @@ -115,13 +113,6 @@ pub struct RecordedState<'a> { pub index: RecordedIndex, } -/// `purl`'s folded base plus its qualifiers: equal for two spellings of the -/// same qualified purl (percent-encoding, case where it does not matter). -pub fn qualified_key(purl: &str) -> String { - let suffix = purl.find(['?', '#']).map_or("", |i| &purl[i..]); - format!("{}{suffix}", canonical_base_purl(purl)) -} - impl RecordedIndex { pub fn new(manifest: Option<&PatchManifest>, pins: &[(String, String)]) -> Self { let mut index = RecordedIndex::default(); @@ -137,12 +128,12 @@ impl RecordedIndex { .push(uuid.to_string()); index .qualified - .entry(qualified_key(key)) + .entry(PurlKey::qualified(key)) .or_default() .push(uuid.to_string()); index .by_base - .entry(canonical_base_purl(key)) + .entry(PurlKey::new(key)) .or_default() .push(uuid.to_string()); } @@ -163,14 +154,14 @@ impl RecordedIndex { pub fn uuids(&self, purl: &str) -> &[String] { self.exact .get(purl) - .or_else(|| self.qualified.get(&qualified_key(purl))) - .or_else(|| self.by_base.get(&canonical_base_purl(purl))) + .or_else(|| self.qualified.get(&PurlKey::qualified(purl))) + .or_else(|| self.by_base.get(&PurlKey::new(purl))) .map_or(&[], Vec::as_slice) } /// Whether any patch is recorded for `purl`'s base purl. pub fn records_package(&self, purl: &str) -> bool { - self.by_base.contains_key(&canonical_base_purl(purl)) + self.by_base.contains_key(&PurlKey::new(purl)) } } @@ -204,7 +195,7 @@ pub fn classify(offers: &Offers, recorded: &RecordedIndex, project: &str) -> Vec candidate: Candidate { project: project.to_string(), purl: purl.clone(), - base_purl: canonical_base_purl(purl), + base_purl: PurlKey::new(purl).into_string(), uuid: selected.uuid.clone(), ecosystem: Ecosystem::from_purl(purl).map_or("", |e| e.cli_name()), severity_order: max_severity_order( diff --git a/crates/socket-patch-core/src/utils/composer_version.rs b/crates/socket-patch-core/src/utils/composer_version.rs index 5d8f6b1d2..b35b5d15a 100644 --- a/crates/socket-patch-core/src/utils/composer_version.rs +++ b/crates/socket-patch-core/src/utils/composer_version.rs @@ -19,10 +19,16 @@ //! parts too, since that padding erases whether the lock said `X`, `X.0` or //! `X.0.0`. Any other spelling Composer rejects keys as itself minus one //! leading `v`, and is equivalent only to other rejected spellings with the -//! same key. +//! same key. [`composer_version_identity_key`] keeps that space apart with a +//! `\u{1}` sentinel; [`composer_version_key`] (what `PurlKey` uses, so its +//! string stays printable in reports) relies instead on every accepted key +//! being accepted itself, bare and behind a `v`, which +//! `sentinel_free_key_keeps_rejected_spellings_apart` pins over the vectors. //! -//! Only comparisons go through this module. Stored spellings (manifest keys, -//! vendored leaf directories, ledger keys, crawler purls) are unchanged. +//! Only comparisons go through this module, and purl comparisons reach it +//! only through [`crate::utils::purl_key::PurlKey`]. Stored spellings +//! (manifest keys, vendored leaf directories, ledger keys, crawler purls) +//! are unchanged. //! //! Composer's normalize is not idempotent for a few forms (`2010-01-02` → //! `2010.01.02` → `2010.01.02.0`; `1.0.0-STABLE` keeps `-stable`), so key raw @@ -32,8 +38,6 @@ use std::sync::LazyLock; use regex::Regex; -use crate::utils::purl::{canonical_purl, normalize_purl, strip_purl_qualifiers}; - /// PCRE `$` also matches before one final `\n`; PHP `\s` includes `\v`. const MODIFIER: &str = r"[._-]?(?:((?i-u:stable|beta|b|RC|alpha|a|patch|pl|p))((?:[.-]?[0-9]+)*)?)?((?i-u:[.-]?dev))?"; @@ -263,61 +267,6 @@ pub fn composer_versions_equivalent(a: &str, b: &str) -> bool { composer_version_identity_key(a) == composer_version_identity_key(b) } -/// `(lowercased vendor/name, version)` of an already-decoded, -/// qualifier-free `pkg:composer//@` base. -fn composer_base_parts(base: &str) -> Option<(String, &str)> { - let rest = base - .get(..13) - .filter(|prefix| prefix.eq_ignore_ascii_case("pkg:composer/")) - .map(|_| &base[13..])?; - let (name, version) = rest.rsplit_once('@')?; - let (vendor, package) = name.split_once('/')?; - if vendor.is_empty() || package.is_empty() || package.contains('/') || version.is_empty() { - return None; - } - Some((name.to_lowercase(), version)) -} - -/// `pkg:composer//@` for a composer purl in any spelling -/// (qualifiers and subpath stripped, percent-decoded, name lowercased, -/// version through [`composer_version_identity_key`]); `None` for anything -/// else. -pub fn composer_purl_identity(purl: &str) -> Option { - let base = canonical_purl(purl); - let (name, version) = composer_base_parts(&base)?; - Some(format!( - "pkg:composer/{name}@{}", - composer_version_identity_key(version) - )) -} - -/// The key ledger and prune bookkeeping compare purls by: the composer -/// identity for composer purls, [`canonical_purl`] for every other type. -pub fn purl_identity_key(purl: &str) -> String { - composer_purl_identity(purl).unwrap_or_else(|| canonical_purl(purl)) -} - -/// Whether two already-decoded, qualifier-free composer bases name the same -/// package release. `false` unless both are composer purls. -pub(crate) fn composer_bases_equivalent(a: &str, b: &str) -> bool { - match (composer_base_parts(a), composer_base_parts(b)) { - (Some((left_name, left_version)), Some((right_name, right_version))) => { - left_name == right_name && composer_versions_equivalent(left_version, right_version) - } - _ => false, - } -} - -/// Whether two composer purls, in any spelling, name the same package -/// release (qualifiers and subpath ignored). `false` unless both are -/// composer purls. -pub fn composer_purls_equivalent(a: &str, b: &str) -> bool { - composer_bases_equivalent( - &normalize_purl(strip_purl_qualifiers(a)), - &normalize_purl(strip_purl_qualifiers(b)), - ) -} - #[cfg(test)] mod tests { use super::*; @@ -407,6 +356,53 @@ mod tests { assert!(failures.is_empty(), "{}", failures.join("\n")); } + /// [`PurlKey`](crate::utils::purl_key::PurlKey) keys on the sentinel-free + /// [`composer_version_key`], which is only sound while a rejected + /// spelling's key can never equal an accepted one's: every accepted key + /// `N` must itself be accepted, bare and behind a `v` (the only text a + /// rejected spelling `N` / `vN` would key as; a `v` only before a digit). + #[test] + fn sentinel_free_key_keeps_rejected_spellings_apart() { + let v = vectors(); + let mut inputs: Vec<&str> = v["vectors"] + .as_array() + .unwrap() + .iter() + .map(|case| case["input"].as_str().unwrap()) + .collect(); + for case in v["equivalence"].as_array().unwrap() { + inputs.push(case["left"].as_str().unwrap()); + inputs.push(case["right"].as_str().unwrap()); + } + let mut failures = Vec::new(); + for input in &inputs { + let Some(key) = identity_normalize(input) else { + continue; + }; + // `strip_leading_v` only strips a `v` before a digit. + let mut spellings = vec![key.clone()]; + if key.starts_with(|c: char| c.is_ascii_digit()) { + spellings.push(format!("v{key}")); + } + for spelling in spellings { + if identity_normalize(&spelling).is_none() { + failures.push(format!( + "{input:?} keys as {key:?}, but {spelling:?} is rejected" + )); + } + } + } + for a in &inputs { + for b in &inputs { + let plain = composer_version_key(a) == composer_version_key(b); + if plain != composer_versions_equivalent(a, b) { + failures.push(format!("{a:?} vs {b:?}: plain key {plain}")); + } + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); + } + #[test] fn sbom_padded_date_versions_match_their_lock_spelling() { assert_eq!(composer_version_key("20231001.0.0.0"), "20231001"); @@ -415,77 +411,5 @@ mod tests { assert!(composer_versions_equivalent("20231001.0", "20231001.0.0.0")); assert!(composer_versions_equivalent("202301.1", "202301.1.0")); assert!(!composer_versions_equivalent("1.2.3.4.5", "1.2.3.4.5.0")); - assert_eq!( - composer_purl_identity("pkg:composer/acme/dated@20231001.0.0.0"), - composer_purl_identity("pkg:composer/acme/dated@20231001"), - ); - } - - #[test] - fn purl_identity_bridges_padding_prefix_case_and_encoding() { - let want = Some("pkg:composer/psr/log@3.0.2.0".to_string()); - for purl in [ - "pkg:composer/psr/log@3.0.2", - "pkg:composer/psr/log@v3.0.2", - "pkg:composer/psr/log@3.0.2.0", - "pkg:composer/Psr/Log@3.0.2", - "pkg:composer/psr/log@3.0.2.0?repository_url=https://repo.packagist.org", - "pkg:composer/psr/log@3.0.2#src", - "pkg:composer/psr/log@3.0.2%2Bbuild.5", - ] { - assert_eq!(composer_purl_identity(purl), want, "{purl}"); - } - assert_eq!( - composer_purl_identity("pkg:composer/symfony/http-kernel@v8.1.0-rc.1").as_deref(), - Some("pkg:composer/symfony/http-kernel@8.1.0.0-RC1") - ); - assert_eq!(composer_purl_identity("pkg:npm/left-pad@1.3.0"), None); - assert_eq!(composer_purl_identity("pkg:composer/log@1.0.0"), None); - assert_eq!(composer_purl_identity("pkg:composer/a/b/c@1.0.0"), None); - assert_eq!(composer_purl_identity("pkg:composer/psr/log@"), None); - } - - #[test] - fn purl_equivalence_is_composer_only_and_version_exact() { - assert!(composer_purls_equivalent( - "pkg:composer/psr/log@3.0.2", - "pkg:composer/psr/log@3.0.2.0" - )); - assert!(composer_purls_equivalent( - "pkg:composer/psr/log@v3.0.2", - "pkg:composer/PSR/LOG@3.0.2.0?x=y" - )); - assert!(composer_purls_equivalent( - "pkg:composer/psr/log@1.0", - "pkg:composer/psr/log@1.0.0.0" - )); - assert!(!composer_purls_equivalent( - "pkg:composer/psr/log@3.0.2", - "pkg:composer/psr/log@3.0.20" - )); - assert!(!composer_purls_equivalent( - "pkg:composer/psr/log@3.0.2", - "pkg:composer/psr/cache@3.0.2" - )); - assert!(!composer_purls_equivalent( - "pkg:npm/left-pad@1.3.0", - "pkg:npm/left-pad@1.3.0" - )); - assert!(!composer_purls_equivalent( - "pkg:composer/psr/log@1.0.0-RC1", - "pkg:composer/psr/log@1.0.0" - )); - } - - #[test] - fn identity_key_falls_back_to_the_canonical_purl() { - assert_eq!( - purl_identity_key("pkg:composer/psr/log@v3.0.2?x=1"), - "pkg:composer/psr/log@3.0.2.0" - ); - assert_eq!( - purl_identity_key("pkg:npm/%40scope/x@1.0.0?y=2"), - "pkg:npm/@scope/x@1.0.0" - ); } } diff --git a/crates/socket-patch-core/src/utils/mod.rs b/crates/socket-patch-core/src/utils/mod.rs index 5ed233e60..cdeba888a 100644 --- a/crates/socket-patch-core/src/utils/mod.rs +++ b/crates/socket-patch-core/src/utils/mod.rs @@ -19,6 +19,7 @@ pub mod pnpm_workspace; pub mod poetry_lock; pub mod process; pub mod purl; +pub mod purl_key; pub mod python_lock; pub mod python_script; pub(crate) mod relpath; diff --git a/crates/socket-patch-core/src/utils/purl.rs b/crates/socket-patch-core/src/utils/purl.rs index 37e2bf478..6826ef2b0 100644 --- a/crates/socket-patch-core/src/utils/purl.rs +++ b/crates/socket-patch-core/src/utils/purl.rs @@ -2,6 +2,7 @@ use std::borrow::Cow; use crate::crawlers::python_crawler::canonicalize_pypi_name; use crate::patch::path_safety::{is_safe_multi_segment, is_safe_single_segment}; +use crate::utils::purl_key::PurlKey; /// Strip the trailing `?qualifiers` and `#subpath` components from a PURL, /// leaving the canonical `pkg:type/namespace/name@version` base. @@ -81,13 +82,12 @@ pub fn percent_decode_purl_component(component: &str) -> Cow<'_, str> { } } -/// Canonical string form for purl-to-purl comparison and display: -/// percent-decode each `/`-separated component of the -/// `pkg:type/...@version` base; qualifiers/subpath are appended verbatim. +/// Display form of a purl: percent-decode each `/`-separated component of +/// the `pkg:type/...@version` base; qualifiers/subpath are appended verbatim. /// -/// Used ONLY for string equality (`purl_eq`) and human output — never to -/// build filesystem paths (a `%2f` decoding into a name can at worst make -/// two distinct purls compare equal, not change a write location). +/// For human output and for recovering a literal coordinate — never to +/// build filesystem paths, and not an equality relation: compare purls with +/// [`crate::utils::purl_key::PurlKey`]. pub fn normalize_purl(purl: &str) -> Cow<'_, str> { if !purl.contains('%') { return Cow::Borrowed(purl); @@ -105,41 +105,6 @@ pub fn normalize_purl(purl: &str) -> Cow<'_, str> { Cow::Owned(out) } -/// Purl equality up to percent-encoding of the base components -/// (`pkg:npm/%40scope/x@1` ≡ `pkg:npm/@scope/x@1`) and, for composer, up to -/// the version spelling of one release (`@3.0.2` ≡ `@v3.0.2` ≡ `@3.0.2.0`, -/// name case-insensitive; see [`crate::utils::composer_version`]) and, for -/// pypi, up to the PEP 503 spelling of the name (`typing_extensions` ≡ -/// `Typing-Extensions` ≡ `typing-extensions`). -/// Qualifiers and subpath must still match exactly. -pub fn purl_eq(a: &str, b: &str) -> bool { - let (a, b) = (normalize_purl(a), normalize_purl(b)); - if a == b { - return true; - } - let split = |p: &str| p.find(['?', '#']).unwrap_or(p.len()); - let (base_a, suffix_a) = a.split_at(split(&a)); - let (base_b, suffix_b) = b.split_at(split(&b)); - suffix_a == suffix_b - && (crate::utils::composer_version::composer_bases_equivalent(base_a, base_b) - || pypi_bases_equivalent(base_a, base_b)) -} - -/// Whether two decoded `pkg:pypi/@` bases name the same -/// release once both names are in PEP 503 canonical form. `false` unless -/// both are pypi bases. -fn pypi_bases_equivalent(a: &str, b: &str) -> bool { - fn canonical(base: &str) -> Option { - let rest = base.strip_prefix("pkg:pypi/")?; - let (name, version) = match rest.rfind('@').filter(|&i| i > 0) { - Some(at) => rest.split_at(at), - None => (rest, ""), - }; - Some(format!("{}{version}", canonicalize_pypi_name(name))) - } - matches!((canonical(a), canonical(b)), (Some(x), Some(y)) if x == y) -} - /// Extract the value of a single PURL qualifier (`?key=value&…`), if present. /// /// The PURL grammar places qualifiers after the base as `?k1=v1&k2=v2`, @@ -161,12 +126,13 @@ pub fn purl_qualifier<'a>(purl: &'a str, key: &str) -> Option<&'a str> { }) } -/// The ledger / lookup spelling of a purl: `?qualifiers` and `#subpath` -/// stripped, then percent-decoded per component ([`normalize_purl`]). Two -/// purls naming the same package version compare equal after this, whatever -/// URL escaping or `?artifact_id=` decoration they arrived with — the one -/// composition every ledger key match (redirect takeover, vendor GC, the -/// hosted→vendored reconciliation) goes through. +/// The display spelling of a purl's release: `?qualifiers` and `#subpath` +/// stripped, then percent-decoded per component ([`normalize_purl`]). +/// +/// It keeps the name's case and spelling (and composer's version +/// spelling), so it is NOT an identity: `pkg:nuget/Newtonsoft.Json@13.0.3` +/// and `pkg:nuget/newtonsoft.json@13.0.3` stay distinct here. Compare purls +/// with [`crate::utils::purl_key::PurlKey`]. pub fn canonical_purl(purl: &str) -> String { normalize_purl(strip_purl_qualifiers(purl)).into_owned() } @@ -413,21 +379,17 @@ pub fn is_purl(s: &str) -> bool { /// /// For keys without a qualifier this reduces to plain equality. /// -/// Comparison is encoding-tolerant (`purl_eq`): manifest keys come from -/// the API in percent-encoded form (`pkg:npm/%40scope/x@1`) while users -/// type the literal form — both spellings must match either way around. +/// Comparison is by [`PurlKey`]: manifest keys come from the API in +/// percent-encoded, mixed-case form (`pkg:npm/%40scope/x@1`, +/// `pkg:nuget/Newtonsoft.Json@13.0.3`) while users type the literal form or +/// another PEP 503 spelling — every spelling of the release matches. pub fn purl_matches_identifier(manifest_key: &str, identifier: &str) -> bool { if identifier.contains('?') { - purl_eq(manifest_key, identifier) + PurlKey::qualified(manifest_key) == PurlKey::qualified(identifier) } else { - // Base identifier: compare bases. Strip both sides so a subpath - // (`#...`) carried by either the key or the identifier doesn't - // defeat the match — `strip_purl_qualifiers(identifier)` is a no-op - // for a plain base PURL, so existing behaviour is unchanged. - purl_eq( - strip_purl_qualifiers(manifest_key), - strip_purl_qualifiers(identifier), - ) + // Base identifier: compare bases (a subpath `#...` carried by + // either side doesn't defeat the match). + PurlKey::same(manifest_key, identifier) } } @@ -1268,20 +1230,21 @@ mod tests { } #[test] - fn test_normalize_purl_and_purl_eq() { + fn test_normalize_purl_and_qualified_purl_key() { + let qualified_eq = |a: &str, b: &str| PurlKey::qualified(a) == PurlKey::qualified(b); assert_eq!( normalize_purl("pkg:npm/%40modelcontextprotocol/sdk@1.12.0"), "pkg:npm/@modelcontextprotocol/sdk@1.12.0" ); - assert!(purl_eq( + assert!(qualified_eq( "pkg:npm/%40scope/x@1.0.0", "pkg:npm/@scope/x@1.0.0" )); - assert!(purl_eq( + assert!(qualified_eq( "pkg:npm/@scope/x@1.0.0", "pkg:npm/%40scope/x@1.0.0" )); - assert!(!purl_eq( + assert!(!qualified_eq( "pkg:npm/%40scope/x@1.0.0", "pkg:npm/@scope/x@2.0.0" )); @@ -1292,24 +1255,25 @@ mod tests { "pkg:composer/Psr/Log@3.0.2", ] { assert!( - purl_eq("pkg:composer/psr/log@3.0.2", spelling), + qualified_eq("pkg:composer/psr/log@3.0.2", spelling), "{spelling}" ); assert!( - purl_eq(spelling, "pkg:composer/psr/log@3.0.2"), + qualified_eq(spelling, "pkg:composer/psr/log@3.0.2"), "{spelling}" ); } - assert!(!purl_eq( + assert!(!qualified_eq( "pkg:composer/psr/log@3.0.2", "pkg:composer/psr/log@3.0.20" )); - assert!(!purl_eq( + assert!(!qualified_eq( "pkg:composer/psr/log@3.0.2?a=1", "pkg:composer/psr/log@3.0.2.0?a=2" )); - // Only composer: other ecosystems stay spelling-exact. - assert!(!purl_eq("pkg:npm/x@1.0", "pkg:npm/x@1.0.0.0")); + // Only composer gets release identity: other ecosystems stay + // version-exact. + assert!(!qualified_eq("pkg:npm/x@1.0", "pkg:npm/x@1.0.0.0")); // Qualifiers/subpath are preserved verbatim (not decoded). assert_eq!( normalize_purl("pkg:npm/%40s/x@1?artifact_id=a%2Fb"), @@ -1322,6 +1286,42 @@ mod tests { )); } + /// B73: remove/rollback identifiers fold what the ecosystem folds: a + /// PEP 503 spelling or a NuGet case variant names the recorded patch. + #[test] + fn test_purl_matches_identifier_folds_pep503_and_nuget_case() { + assert!(patch_matches( + "pkg:pypi/typing-extensions@4.12.2", + "uuid", + "pkg:pypi/typing_extensions@4.12.2" + )); + assert!(purl_matches_identifier( + "pkg:pypi/typing-extensions@4.12.2?artifact_id=whl", + "pkg:pypi/Typing.Extensions@4.12.2" + )); + assert!(purl_matches_identifier( + "pkg:nuget/Newtonsoft.Json@13.0.3", + "pkg:nuget/newtonsoft.json@13.0.3" + )); + assert!(purl_matches_identifier( + "pkg:pypi/typing-extensions@4.12.2?artifact_id=whl", + "pkg:pypi/typing_extensions@4.12.2?artifact_id=whl" + )); + assert!(!purl_matches_identifier( + "pkg:pypi/typing-extensions@4.12.2?artifact_id=whl", + "pkg:pypi/typing_extensions@4.12.2?artifact_id=sdist" + )); + assert!(!purl_matches_identifier( + "pkg:nuget/Newtonsoft.Json@13.0.3", + "pkg:nuget/newtonsoft.json@13.0.4" + )); + // Case-sensitive ecosystems stay exact. + assert!(!purl_matches_identifier( + "pkg:maven/Org.Foo/bar@1.0", + "pkg:maven/org.foo/bar@1.0" + )); + } + #[test] fn test_purl_matches_identifier_decodes_encoded_key() { // Encoded manifest key vs literal identifier — and vice versa. @@ -1344,6 +1344,10 @@ mod tests { // #1024: patch keys are PEP 503 canonical, but users type the // name as the project declares it. Every spelling must select the // patch, for base and qualified identifiers alike. + let purl_eq = |a: &str, b: &str| { + crate::utils::purl_key::PurlKey::qualified(a) + == crate::utils::purl_key::PurlKey::qualified(b) + }; let key = "pkg:pypi/typing-extensions@4.7.1"; let qualified = "pkg:pypi/typing-extensions@4.7.1?artifact_id=abc"; for spelling in [ diff --git a/crates/socket-patch-core/src/utils/purl_key.rs b/crates/socket-patch-core/src/utils/purl_key.rs new file mode 100644 index 000000000..d3f4f4685 --- /dev/null +++ b/crates/socket-patch-core/src/utils/purl_key.rs @@ -0,0 +1,404 @@ +//! One purl identity: when two purl spellings name the same package release. +//! +//! The same release reaches the CLI under several spellings: the patches API +//! percent-encodes (`pkg:npm/%40scope/x@1`, `pkg:golang/…@v1.0.0%2Bincompatible`) +//! and decorates (`?artifact_id=…`, `#subpath`), crawlers and lockfiles use +//! the literal on-disk names (`pkg:nuget/newtonsoft.json` from the NuGet +//! global cache, `pkg:nuget/Newtonsoft.Json` from the API), PyPI names +//! arrive in any PEP 503 spelling (`typing_extensions` / `typing-extensions`), +//! and a composer release has a pretty tag, a padded SBOM form and a `v` +//! prefix (`3.0.2` / `3.0.2.0` / `v3.0.2`). +//! +//! Every "is this the same package" comparison goes through this module: +//! +//! - [`canonical_base_purl`] is the canonical *spelling*: safe to show and +//! to carry in reports (VEX product purls), but composer versions keep +//! their spelling. +//! - [`PurlKey`] is the *identity*: [`canonical_base_purl`] plus the composer +//! release identity, so two purls are the same package release exactly +//! when their keys are equal. Use it for every map key, set membership and +//! equality test; [`PurlKey::qualified`] when a release variant +//! (`?artifact_id=`, `?platform=`) must still be told apart. +//! +//! Neither form is ever used to build a filesystem path or a download URL: +//! a `%2f` decoding into a name can at worst make two distinct purls compare +//! equal, never change where something is written. + +use std::fmt; + +use crate::crawlers::python_crawler::canonicalize_pypi_name; +use crate::utils::composer_version::composer_version_key; +use crate::utils::purl::{normalize_purl, strip_purl_qualifiers}; + +/// The canonical spelling of a purl's package release: surrounding +/// whitespace trimmed, qualifiers and subpath stripped, components +/// percent-decoded, the type lowercased, and the name folded where the +/// ecosystem's own resolution is insensitive: +/// +/// - `pypi`: PEP 503 (case, and runs of `-`/`_`/`.` become one `-`); +/// - `nuget`: case (name and version — NuGet compares both +/// case-insensitively); +/// - `composer`: name case (`vendor/name`); the version keeps its spelling, +/// since a `dev-` branch name is case-sensitive. +/// +/// Every other ecosystem keeps its spelling: npm forbids uppercase, and +/// Maven groups and Go module paths are case-sensitive. +/// +/// Composer release spellings (`3.0.2` vs `3.0.2.0`) still differ here; +/// compare with [`PurlKey`], which folds them. +pub fn canonical_base_purl(purl: &str) -> String { + let base = normalize_purl(strip_purl_qualifiers(purl.trim())).into_owned(); + let Some(rest) = base.strip_prefix("pkg:") else { + return base; + }; + let Some((ty, tail)) = rest.split_once('/') else { + return base; + }; + let ty = ty.to_ascii_lowercase(); + let (name, version) = match tail.rsplit_once('@').filter(|(name, _)| !name.is_empty()) { + Some((name, version)) => (name, Some(version)), + None => (tail, None), + }; + let (name, version) = match ty.as_str() { + "pypi" => (canonicalize_pypi_name(name), version.map(str::to_string)), + "nuget" => (name.to_lowercase(), version.map(str::to_lowercase)), + "composer" => (name.to_lowercase(), version.map(str::to_string)), + _ => (name.to_string(), version.map(str::to_string)), + }; + match version { + Some(version) => format!("pkg:{ty}/{name}@{version}"), + None => format!("pkg:{ty}/{name}"), + } +} + +/// The identity of a purl's package release; equal for exactly the +/// spellings that name the same release. See the [module docs](self). +/// +/// The string form is [`canonical_base_purl`], with a composer +/// `pkg:composer//@` version replaced by its release +/// identity ([`composer_version_key`]: `v3.0.2` → `3.0.2.0`). It contains no +/// internal sentinels, so rollout and policy reports may show it. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct PurlKey(String); + +impl PurlKey { + /// The key of `purl`'s package release; qualifiers and subpath are + /// ignored, so every release variant of one `name@version` shares it. + pub fn new(purl: &str) -> Self { + let canonical = canonical_base_purl(purl); + PurlKey(composer_identity(&canonical).unwrap_or(canonical)) + } + + /// [`PurlKey::new`] followed by `purl`'s verbatim `?qualifiers` / + /// `#subpath`: one release *variant* (a wheel vs its sdist, a gem + /// platform). Two spellings of the same qualified purl share it; two + /// variants of one release do not. + pub fn qualified(purl: &str) -> Self { + let purl = purl.trim(); + let suffix = purl.find(['?', '#']).map_or("", |i| &purl[i..]); + let mut key = Self::new(purl).0; + key.push_str(suffix); + PurlKey(key) + } + + /// Whether `a` and `b` name the same package release. + pub fn same(a: &str, b: &str) -> bool { + Self::new(a) == Self::new(b) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn into_string(self) -> String { + self.0 + } +} + +impl fmt::Display for PurlKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl AsRef for PurlKey { + fn as_ref(&self) -> &str { + &self.0 + } +} + +/// `pkg:composer//@` for a canonical +/// composer base with a `vendor/name` coordinate and a version; `None` for +/// anything else (which then keys as its canonical spelling). +fn composer_identity(canonical: &str) -> Option { + let rest = canonical.strip_prefix("pkg:composer/")?; + let (name, version) = rest.rsplit_once('@')?; + let (vendor, package) = name.split_once('/')?; + if vendor.is_empty() || package.is_empty() || package.contains('/') || version.is_empty() { + return None; + } + Some(format!( + "pkg:composer/{name}@{}", + composer_version_key(version) + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::utils::composer_version::composer_versions_equivalent; + + #[test] + fn canonical_base_purl_folds_only_insensitive_ecosystems() { + assert_eq!( + canonical_base_purl("pkg:pypi/Python_Dateutil@2.8.2?artifact_id=py3-none-any-whl"), + "pkg:pypi/python-dateutil@2.8.2" + ); + assert_eq!( + canonical_base_purl("pkg:npm/%40scope/Name@1.0.0"), + "pkg:npm/@scope/Name@1.0.0" + ); + assert_eq!( + canonical_base_purl("pkg:nuget/Newtonsoft.Json@13.0.1-Beta"), + "pkg:nuget/newtonsoft.json@13.0.1-beta" + ); + assert_eq!( + canonical_base_purl("pkg:composer/Monolog/Monolog@dev-Main"), + "pkg:composer/monolog/monolog@dev-Main" + ); + assert_eq!( + canonical_base_purl("pkg:gem/nokogiri@1.16.5?platform=java"), + "pkg:gem/nokogiri@1.16.5" + ); + assert_eq!( + canonical_base_purl("pkg:golang/github.com/Foo/bar@v1.0.0#sub/dir"), + "pkg:golang/github.com/Foo/bar@v1.0.0" + ); + assert_eq!( + canonical_base_purl("pkg:Maven/Org.Foo/Bar@1.0"), + "pkg:maven/Org.Foo/Bar@1.0" + ); + assert_eq!(canonical_base_purl("pkg:pypi/Foo_Bar"), "pkg:pypi/foo-bar"); + assert_eq!(canonical_base_purl(" not-a-purl "), "not-a-purl"); + } + + #[test] + fn composer_identity_bridges_padding_prefix_case_and_encoding() { + for purl in [ + "pkg:composer/psr/log@3.0.2", + "pkg:composer/psr/log@v3.0.2", + "pkg:composer/psr/log@3.0.2.0", + "pkg:composer/Psr/Log@3.0.2", + "pkg:composer/psr/log@3.0.2.0?repository_url=https://repo.packagist.org", + "pkg:composer/psr/log@3.0.2#src", + "pkg:composer/psr/log@3.0.2%2Bbuild.5", + ] { + assert_eq!( + PurlKey::new(purl).as_str(), + "pkg:composer/psr/log@3.0.2.0", + "{purl}" + ); + } + assert_eq!( + PurlKey::new("pkg:composer/symfony/http-kernel@v8.1.0-rc.1").as_str(), + "pkg:composer/symfony/http-kernel@8.1.0.0-RC1" + ); + assert!(PurlKey::same( + "pkg:composer/acme/dated@20231001.0.0.0", + "pkg:composer/acme/dated@20231001" + )); + assert!(!PurlKey::same( + "pkg:composer/psr/log@3.0.2", + "pkg:composer/psr/log@3.0.20" + )); + assert!(!PurlKey::same( + "pkg:composer/psr/log@3.0.2", + "pkg:composer/psr/cache@3.0.2" + )); + assert!(!PurlKey::same( + "pkg:composer/psr/log@1.0.0-RC1", + "pkg:composer/psr/log@1.0.0" + )); + // A dev branch name is case-sensitive. + assert!(!PurlKey::same( + "pkg:composer/psr/log@dev-Feature", + "pkg:composer/psr/log@dev-feature" + )); + // Rejected versions key without the internal sentinel. + assert!(!PurlKey::new("pkg:composer/psr/log@not-a-version") + .as_str() + .contains('\u{1}')); + // Only composer gets release identity; other types stay version-exact. + assert!(!PurlKey::same("pkg:npm/x@1.0", "pkg:npm/x@1.0.0.0")); + // Malformed composer coordinates key as their canonical spelling. + assert_eq!( + PurlKey::new("pkg:composer/log@1.0.0").as_str(), + "pkg:composer/log@1.0.0" + ); + assert_eq!( + PurlKey::new("pkg:composer/a/b/c@1.0.0").as_str(), + "pkg:composer/a/b/c@1.0.0" + ); + } + + /// B20 / #553: the NuGet global-cache crawl spells the purl lowercase, + /// the API mixed-case; B73: a PEP 503 or NuGet case variant names the + /// same release. + #[test] + fn nuget_case_and_pep503_spellings_share_a_key() { + assert!(PurlKey::same( + "pkg:nuget/Newtonsoft.Json@13.0.3", + "pkg:nuget/newtonsoft.json@13.0.3" + )); + assert!(PurlKey::same( + "pkg:pypi/typing_extensions@4.12.2", + "pkg:pypi/typing-extensions@4.12.2" + )); + assert!(PurlKey::same( + "pkg:pypi/Typing.Extensions@4.12.2", + "pkg:pypi/typing-extensions@4.12.2" + )); + assert!(!PurlKey::same( + "pkg:nuget/Newtonsoft.Json@13.0.3", + "pkg:nuget/Newtonsoft.Json@13.0.4" + )); + // Case-sensitive ecosystems are not folded. + assert!(!PurlKey::same( + "pkg:maven/Org.Foo/bar@1.0", + "pkg:maven/org.foo/bar@1.0" + )); + assert!(!PurlKey::same( + "pkg:golang/github.com/Foo/bar@v1.0.0", + "pkg:golang/github.com/foo/bar@v1.0.0" + )); + } + + #[test] + fn qualified_keys_keep_variants_apart() { + assert_eq!( + PurlKey::qualified("pkg:pypi/Foo_Bar@1.0?artifact_id=abc"), + PurlKey::qualified("pkg:pypi/foo-bar@1.0?artifact_id=abc") + ); + assert_ne!( + PurlKey::qualified("pkg:pypi/foo@1.0?artifact_id=abc"), + PurlKey::qualified("pkg:pypi/foo@1.0?artifact_id=def") + ); + assert_eq!( + PurlKey::new("pkg:pypi/foo@1.0?artifact_id=abc"), + PurlKey::new("pkg:pypi/foo@1.0?artifact_id=def") + ); + assert_eq!( + PurlKey::qualified("pkg:npm/%40s/x@1#sub").as_str(), + "pkg:npm/@s/x@1#sub" + ); + assert_eq!(PurlKey::qualified("pkg:npm/x@1").as_str(), "pkg:npm/x@1"); + } + + /// Every spelling variant of one release maps to one key, and changing + /// the release (another name or version) never does: the property every + /// former equality relation (ledger keys, prune, update detection, + /// redirect matching, VEX, rollout, policy, remove/rollback identifiers) + /// now inherits from this one type. + #[test] + fn every_spelling_variant_shares_one_key_and_no_other_release_does() { + // (release, spellings of that same release) + let cases: &[(&str, &[&str])] = &[ + ( + "pkg:npm/@scope/pkg@1.0.0", + &[ + "pkg:npm/%40scope/pkg@1.0.0", + "pkg:NPM/@scope/pkg@1.0.0", + "pkg:npm/@scope/pkg@1.0.0?artifact_id=x", + "pkg:npm/%40scope/pkg@1.0.0#lib", + " pkg:npm/@scope/pkg@1.0.0 ", + ], + ), + ( + "pkg:pypi/typing-extensions@4.12.2", + &[ + "pkg:pypi/typing_extensions@4.12.2", + "pkg:pypi/Typing.Extensions@4.12.2", + "pkg:pypi/typing__extensions@4.12.2?artifact_id=whl", + "pkg:PyPI/TYPING-EXTENSIONS@4.12.2", + ], + ), + ( + "pkg:nuget/Newtonsoft.Json@13.0.3", + &[ + "pkg:nuget/newtonsoft.json@13.0.3", + "pkg:nuget/NEWTONSOFT.JSON@13.0.3", + "pkg:nuget/Newtonsoft.Json@13.0.3?repository_url=x", + ], + ), + ( + "pkg:composer/psr/log@3.0.2", + &[ + "pkg:composer/psr/log@v3.0.2", + "pkg:composer/PSR/Log@3.0.2.0", + "pkg:composer/psr%2Flog@3.0.2", + ], + ), + ( + "pkg:golang/github.com/foo/bar@v1.0.0+incompatible", + &["pkg:golang/github.com/foo/bar@v1.0.0%2Bincompatible"], + ), + ( + "pkg:gem/nokogiri@1.16.5", + &["pkg:gem/nokogiri@1.16.5?platform=java"], + ), + ]; + for (release, spellings) in cases { + let key = PurlKey::new(release); + for spelling in *spellings { + assert_eq!(PurlKey::new(spelling), key, "{spelling} vs {release}"); + } + for (other, _) in cases.iter().filter(|(other, _)| other != release) { + assert_ne!(PurlKey::new(other), key, "{other} vs {release}"); + } + let (name, version) = release.rsplit_once('@').unwrap(); + for bumped in [format!("{name}@{version}9"), format!("{name}x@{version}")] { + assert_ne!(PurlKey::new(&bumped), key, "{bumped}"); + } + } + } + + /// The composer half of [`PurlKey`] is exactly composer release + /// equivalence, over the whole shared vector file (pairs of accepted + /// AND rejected spellings), even though the key drops the internal + /// rejected-version sentinel. + #[test] + fn composer_key_equality_is_release_equivalence_over_every_vector_pair() { + let v: serde_json::Value = serde_json::from_str(include_str!( + "../../tests/fixtures/composer-version-vectors.json" + )) + .unwrap(); + let mut inputs: Vec<&str> = v["vectors"] + .as_array() + .unwrap() + .iter() + .map(|case| case["input"].as_str().unwrap()) + .collect(); + for case in v["equivalence"].as_array().unwrap() { + inputs.push(case["left"].as_str().unwrap()); + inputs.push(case["right"].as_str().unwrap()); + } + inputs.retain(|version| { + !version.is_empty() + && version.trim() == *version + && !version.contains(['?', '#', '%', '@', '/']) + }); + let mut failures = Vec::new(); + for a in &inputs { + for b in &inputs { + let keyed = PurlKey::same( + &format!("pkg:composer/psr/log@{a}"), + &format!("pkg:composer/psr/log@{b}"), + ); + if keyed != composer_versions_equivalent(a, b) { + failures.push(format!("{a:?} vs {b:?}: keyed {keyed}")); + } + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); + } +} diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/mod.rs b/crates/socket-patch-core/src/vendor/lock_inventory/mod.rs index 622a3d3e8..b7d42529e 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/mod.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/mod.rs @@ -52,8 +52,7 @@ use std::collections::HashMap; use std::path::Path; -use crate::crawlers::python_crawler::canonicalize_pypi_name; -use crate::utils::purl::strip_purl_qualifiers; +use crate::utils::purl_key::PurlKey; pub(crate) mod bun; pub(crate) mod cargo; @@ -240,30 +239,12 @@ pub fn unsupported_layout_warnings(unsupported: &[UnsupportedNpmLayout]) -> Vec< } /// Match a manifest/API purl (possibly percent-encoded, possibly carrying -/// qualifiers) against the inventory: components decode via -/// [`crate::utils::purl::normalize_purl`], so `pkg:npm/%40scope/x@1` -/// matches the literal entry. +/// qualifiers) against the inventory by [`PurlKey`]: `pkg:npm/%40scope/x@1` +/// matches the literal entry, a PyPI name in any PEP 503 spelling matches, +/// and a composer API purl's padded `@3.0.2.0` matches the lock's `3.0.2`. pub fn lookup<'a>(entries: &'a [LockfileEntry], purl: &str) -> Option<&'a LockfileEntry> { - let decoded = crate::utils::purl::normalize_purl(strip_purl_qualifiers(purl)).into_owned(); - let rest = decoded.strip_prefix("pkg:")?; - let (purl_type, rest) = rest.split_once('/')?; - // purl types double as the vendor-ecosystem tags (same set the - // dispatcher recognizes). - let eco = match purl_type { - "npm" | "cargo" | "golang" | "pypi" | "gem" | "composer" => purl_type, - _ => return None, - }; - let at = rest.rfind('@').filter(|&i| i > 0)?; - let (name, version) = (&rest[..at], &rest[at + 1..]); - // pypi names compare in PEP 503 normalized form. - let name = if eco == "pypi" { - canonicalize_pypi_name(name) - } else { - name.to_string() - }; - entries - .iter() - .find(|e| e.ecosystem == eco && e.name == name && e.version == version) + let key = PurlKey::new(purl); + entries.iter().find(|e| PurlKey::new(&e.purl) == key) } /// Everything every recognized lockfile in the project resolves — the diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/tests.rs b/crates/socket-patch-core/src/vendor/lock_inventory/tests.rs index 84c2f00ff..70e2e6b9c 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/tests.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/tests.rs @@ -3560,3 +3560,25 @@ async fn requirements_index_option_in_an_include_spans_the_tree() { LockIntegrity::Sha256AnyOf(vec![sha.clone()]) ); } + +#[tokio::test] +async fn lookup_matches_by_purl_identity() { + let entry = |ecosystem: &'static str, name: &str, version: &str| LockfileEntry { + ecosystem, + source_kind: SourceKind::Unspecified, + name: name.into(), + version: version.into(), + purl: format!("pkg:{ecosystem}/{name}@{version}"), + resolved: None, + integrity: LockIntegrity::None, + }; + let entries = vec![ + entry("composer", "psr/log", "3.0.2"), + entry("pypi", "typing-extensions", "4.12.2"), + ]; + // The API pads composer releases and may spell a PyPI name either way. + assert!(lookup(&entries, "pkg:composer/psr/log@3.0.2.0").is_some()); + assert!(lookup(&entries, "pkg:composer/Psr/Log@v3.0.2").is_some()); + assert!(lookup(&entries, "pkg:composer/psr/log@3.0.3").is_none()); + assert!(lookup(&entries, "pkg:pypi/typing_extensions@4.12.2").is_some()); +} diff --git a/crates/socket-patch-core/src/vendor/mod.rs b/crates/socket-patch-core/src/vendor/mod.rs index 9129348d3..0e9477a15 100644 --- a/crates/socket-patch-core/src/vendor/mod.rs +++ b/crates/socket-patch-core/src/vendor/mod.rs @@ -139,7 +139,6 @@ use std::path::Path; use crate::manifest::schema::PatchRecord; use crate::patch::apply::{is_safe_relative_subpath, normalize_file_path, ApplyResult}; use crate::utils::fs::read_regular_to_string_sync; -use crate::utils::purl::strip_purl_qualifiers; /// A non-fatal advisory surfaced as a warning event (`code` is a stable /// reason tag from the CLI contract; `detail` is human text). @@ -382,9 +381,10 @@ pub async fn harvest_artifact_blobs_from( continue; } let Some(entry) = entries.get(purl).or_else(|| { + let key = crate::utils::purl_key::PurlKey::new(purl); entries .values() - .find(|e| e.base_purl == strip_purl_qualifiers(purl)) + .find(|e| crate::utils::purl_key::PurlKey::new(&e.base_purl) == key) }) else { continue; }; @@ -866,7 +866,7 @@ pub async fn lock_text_refusals( /// (apply / rollback / scan prune). An unreadable ledger degrades to the /// empty set (fail-open); mutating callers that need fail-closed semantics /// use [`load_state`] directly. -pub async fn vendored_purl_keys(project_root: &Path) -> HashSet { +pub async fn vendored_purl_keys(project_root: &Path) -> HashSet { load_state(project_root) .await .map(|state| state.purl_keys()) @@ -1823,9 +1823,9 @@ mod harvest_tests { ); } - /// Every spelling `vendored_purl_keys` promises: the entry's map key + /// Every spelling `vendored_purl_keys` covers: the entry's map key /// (possibly qualified), its resolved base purl, and the - /// qualifier-stripped key. + /// qualifier-stripped key — one `PurlKey`. #[tokio::test] async fn vendored_purl_keys_lists_all_addressable_spellings() { let tmp = tempfile::tempdir().unwrap(); @@ -1835,12 +1835,15 @@ mod harvest_tests { write_ledger_entries(tmp.path(), &[(qualified, base, UUID, &rel)]); let keys = vendored_purl_keys(tmp.path()).await; - assert!(keys.contains(qualified), "map key spelling: {keys:?}"); assert!( - keys.contains(base), + purl_keys_cover(&keys, qualified), + "map key spelling: {keys:?}" + ); + assert!( + purl_keys_cover(&keys, base), "base purl / stripped spelling: {keys:?}" ); - assert_eq!(keys.len(), 2, "base and stripped coincide here: {keys:?}"); + assert_eq!(keys.len(), 1, "every spelling shares one key: {keys:?}"); } /// The documented fail-open degrade: no ledger yields the empty set, and diff --git a/crates/socket-patch-core/src/vendor/state.rs b/crates/socket-patch-core/src/vendor/state.rs index 39e43d955..4919f33cf 100644 --- a/crates/socket-patch-core/src/vendor/state.rs +++ b/crates/socket-patch-core/src/vendor/state.rs @@ -31,9 +31,9 @@ use serde::{Deserialize, Serialize}; use crate::constants::SOCKET_DIR; use crate::manifest::schema::PatchRecord; -use crate::utils::composer_version::{composer_purl_identity, composer_purls_equivalent}; use crate::utils::fs::{atomic_write_artifact, read_regular_to_bytes}; -use crate::utils::purl::{patch_matches, strip_purl_qualifiers}; +use crate::utils::purl::patch_matches; +use crate::utils::purl_key::PurlKey; use crate::utils::serde::serialize_sorted; use crate::utils::socket_dir::{prune_empty_dirs, remove_file_and_prune, write_json_ledger}; @@ -312,15 +312,13 @@ impl VendorEntry { } /// Does this entry, stored under ledger `key`, own the manifest purl - /// `purl`? The ledger-key / qualifier-stripped-key / base-purl triple, - /// plus composer release identity (`@3.0.2` owns `@3.0.2.0`) — the - /// per-entry form of the set [`VendorState::purl_keys`] flattens. + /// `purl`? By [`PurlKey`] of its ledger key or its base purl (any + /// qualifier variant, encoding, PyPI/NuGet name spelling or composer + /// release spelling) — the per-entry form of the set + /// [`VendorState::purl_keys`] flattens. pub fn covers_purl(&self, key: &str, purl: &str) -> bool { - key == purl - || strip_purl_qualifiers(key) == strip_purl_qualifiers(purl) - || self.base_purl == strip_purl_qualifiers(purl) - || composer_purls_equivalent(key, purl) - || composer_purls_equivalent(&self.base_purl, purl) + let purl = PurlKey::new(purl); + PurlKey::new(key) == purl || PurlKey::new(&self.base_purl) == purl } } @@ -340,27 +338,16 @@ impl VendorState { } } - /// Every purl spelling under which this ledger's entries are - /// addressable: each entry's map key (the manifest purl, possibly - /// qualified), its resolved base purl, the qualifier-stripped key, and - /// for composer the release identity of both - /// ([`composer_purl_identity`]). The one derivation behind every - /// whole-set vendor-ownership match (apply / rollback / remove / scan - /// prune); match against it with [`purl_keys_cover`]. + /// The [`PurlKey`]s under which this ledger's entries are addressable: + /// each entry's map key (the manifest purl, possibly qualified) and its + /// resolved base purl. The one derivation behind every whole-set + /// vendor-ownership match (apply / rollback / remove / scan prune); + /// match against it with [`purl_keys_cover`]. /// [`super::vendored_purl_keys`] is its load-then-derive convenience. - pub fn purl_keys(&self) -> HashSet { + pub fn purl_keys(&self) -> HashSet { self.entries .iter() - .flat_map(|(key, entry)| { - [ - Some(key.clone()), - Some(entry.base_purl.clone()), - Some(strip_purl_qualifiers(key).to_string()), - composer_purl_identity(key), - composer_purl_identity(&entry.base_purl), - ] - }) - .flatten() + .flat_map(|(key, entry)| [PurlKey::new(key), PurlKey::new(&entry.base_purl)]) .collect() } @@ -430,13 +417,11 @@ impl VendorState { } /// Whether `purl` is vendor-owned according to `keys`, a -/// [`VendorState::purl_keys`] set: by its own spelling, its -/// qualifier-stripped base, or (composer) its release identity, so a scan -/// that sees `@3.0.2` still finds the entry vendored as `@3.0.2.0`. -pub fn purl_keys_cover(keys: &HashSet, purl: &str) -> bool { - keys.contains(purl) - || keys.contains(strip_purl_qualifiers(purl)) - || composer_purl_identity(purl).is_some_and(|identity| keys.contains(&identity)) +/// [`VendorState::purl_keys`] set: by its [`PurlKey`], so a scan that sees +/// composer `@3.0.2` still finds the entry vendored as `@3.0.2.0`, and a +/// lowercase NuGet crawl the entry the API spelled `Newtonsoft.Json`. +pub fn purl_keys_cover(keys: &HashSet, purl: &str) -> bool { + keys.contains(&PurlKey::new(purl)) } impl Default for VendorState { @@ -604,10 +589,12 @@ fn binary_snapshot_identity_matches(a: &serde_json::Value, b: &serde_json::Value } /// The ledger entry addressable as `purl`: the exact map key first, then -/// any entry whose resolved `base_purl` equals it (a qualified manifest -/// key resolves to the entry recorded under the base PURL), then, for a -/// composer purl, the entry of the same release in another version -/// spelling (the smallest such key, so the pick is deterministic). +/// the entry (the smallest such key, so the pick is deterministic) whose +/// key is `purl` in another spelling ([`PurlKey::qualified`]: encoding, +/// PyPI/NuGet name spelling, composer release spelling — a qualified purl +/// still names only its own variant), or whose resolved `base_purl` is the +/// unqualified `purl` (a qualified manifest key resolves to the entry +/// recorded under the base PURL). pub fn lookup_entry<'a>( entries: &'a HashMap, purl: &str, @@ -620,18 +607,15 @@ pub fn lookup_entry_kv<'a>( entries: &'a HashMap, purl: &str, ) -> Option<(&'a String, &'a VendorEntry)> { - entries - .get_key_value(purl) - .or_else(|| entries.iter().find(|(_, e)| e.base_purl == purl)) - .or_else(|| { - entries - .iter() - .filter(|(key, e)| { - composer_purls_equivalent(key, purl) - || composer_purls_equivalent(&e.base_purl, purl) - }) - .min_by(|(a, _), (b, _)| a.cmp(b)) - }) + entries.get_key_value(purl).or_else(|| { + let want = PurlKey::qualified(purl); + entries + .iter() + .filter(|(key, e)| { + PurlKey::qualified(key) == want || PurlKey::qualified(&e.base_purl) == want + }) + .min_by(|(a, _), (b, _)| a.cmp(b)) + }) } fn state_path(project_root: &Path) -> PathBuf { @@ -1066,9 +1050,10 @@ mod tests { assert_eq!(npm.artifact.file_inventory, None, "cargo only"); } - /// Every spelling `purl_keys` promises: the (possibly qualified, + /// Every spelling `purl_keys` covers: the (possibly qualified, /// percent-encoded) map key, the entry's base purl and the - /// qualifier-stripped key; an empty ledger yields the empty set. + /// qualifier-stripped key all share one `PurlKey`; an empty ledger + /// yields the empty set. #[test] fn purl_keys_carry_every_spelling() { let mut state = VendorState::new(); @@ -1083,9 +1068,12 @@ mod tests { "pkg:npm/%40scope/pkg@1.0.0", "pkg:npm/@scope/pkg@1.0.0", ] { - assert!(keys.contains(spelling), "missing {spelling}: {keys:?}"); + assert!( + purl_keys_cover(&keys, spelling), + "missing {spelling}: {keys:?}" + ); } - assert_eq!(keys.len(), 3); + assert_eq!(keys.len(), 1); assert!(VendorState::new().purl_keys().is_empty()); } @@ -1132,6 +1120,27 @@ mod tests { assert!(!entry.covers_purl(key, "pkg:npm/other@1.0.0")); } + /// #553 / B20: a NuGet ledger entry recorded under the lockfile's + /// lowercase purl is the same package as the API's mixed-case purl, for + /// every ownership check (per-entry, whole-set, lookup). + #[test] + fn nuget_case_spellings_are_one_vendored_package() { + let mut entry = sample_entry(); + entry.base_purl = "pkg:nuget/newtonsoft.json@13.0.3".into(); + let key = "pkg:nuget/newtonsoft.json@13.0.3"; + let api = "pkg:nuget/Newtonsoft.Json@13.0.3"; + assert!(entry.covers_purl(key, api)); + let mut state = VendorState::new(); + state.entries.insert(key.to_string(), entry); + assert!(purl_keys_cover(&state.purl_keys(), api)); + assert!(!purl_keys_cover( + &state.purl_keys(), + "pkg:nuget/Newtonsoft.Json@13.0.4" + )); + let (found, _) = lookup_entry_kv(&state.entries, api).expect("case variant found"); + assert_eq!(found, key); + } + /// A maven-shaped entry: the wiring record holds the whole pom before /// and after the vendored `` was added. fn whole_file_entry(purl: &str, uuid: &str, before: &str, after: &str) -> VendorEntry { diff --git a/crates/socket-patch-core/src/vex/discover/composer.rs b/crates/socket-patch-core/src/vex/discover/composer.rs index dc62b5d4e..55e16c9cc 100644 --- a/crates/socket-patch-core/src/vex/discover/composer.rs +++ b/crates/socket-patch-core/src/vex/discover/composer.rs @@ -12,7 +12,7 @@ //! composer's leading-`v` normalization //! ([`crate::crawlers::composer_crawler::normalize_version`]: locks carry the //! pretty `v6.4.1`, purls the bare `6.4.1`). A vendored leaf matches it by -//! release identity ([`composer_purls_equivalent`]: a leaf keyed by the +//! release identity ([`PurlKey`]: a leaf keyed by the //! patch's padded `3.0.2.0` is the lock's `3.0.2`). The rewritten `dist` is //! what composer's default install consumes and the only block either //! backend rewrites, so it is what a ref is read from — unless composer @@ -72,12 +72,13 @@ use serde_json::Value; use super::{ - canonical_base_purl, composer_purl, names_vendor_dir, parse_json, vendored_leaf_purl, - DiscoverCtx, Discovery, LocateOpts, PatchedRef, DIAG_LOCKFILE_UNPARSEABLE, DIAG_REF_INVALID, + composer_purl, names_vendor_dir, parse_json, vendored_leaf_purl, DiscoverCtx, Discovery, + LocateOpts, PatchedRef, DIAG_LOCKFILE_UNPARSEABLE, DIAG_REF_INVALID, }; use crate::crawlers::composer_crawler::normalize_version; use crate::formats::composer::{ComposerLock, ComposerLockPackage}; -use crate::utils::composer_version::{composer_purls_equivalent, composer_version_normalize}; +use crate::utils::composer_version::composer_version_normalize; +use crate::utils::purl_key::PurlKey; /// The lock both backends rewrite (root-relative). const COMPOSER_LOCK: &str = "composer.lock"; @@ -285,9 +286,8 @@ fn entry_ref( if let Some(vref) = vendored { // The leaf carries the patch purl's spelling (`@3.0.2.0`), the lock // its own (`3.0.2`): the same release either way. - let leaf_matches = vendored_leaf_purl("composer", &vref.leaf).is_some_and(|leaf| { - leaf == canonical_base_purl(&purl) || composer_purls_equivalent(&leaf, &purl) - }); + let leaf_matches = vendored_leaf_purl("composer", &vref.leaf) + .is_some_and(|leaf| PurlKey::same(&leaf, &purl)); if vref.eco != "composer" || !leaf_matches { out.diag( DIAG_REF_INVALID, diff --git a/crates/socket-patch-core/src/vex/discover/mod.rs b/crates/socket-patch-core/src/vex/discover/mod.rs index 471e8344a..d0b9c76a5 100644 --- a/crates/socket-patch-core/src/vex/discover/mod.rs +++ b/crates/socket-patch-core/src/vex/discover/mod.rs @@ -183,10 +183,10 @@ use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use std::sync::Mutex; -use crate::crawlers::python_crawler::canonicalize_pypi_name; use crate::crawlers::Ecosystem; use crate::patch::path_safety::{is_canonical_uuid, is_safe_multi_segment}; -use crate::utils::purl::{normalize_purl, strip_purl_qualifiers}; +use crate::utils::purl_key::canonical_base_purl; +use crate::utils::purl_key::PurlKey; use crate::vendor::go_mod_edit::HOSTED_GO_MODULE_PREFIX; use crate::vendor::lock_inventory::{ inventory_project_every_lock, lookup, LockIntegrity, LockfileEntry, SourceKind, @@ -510,10 +510,10 @@ impl UnwiredCopy { return false; } match &self.target { - CopyTarget::Purl(purl) => *purl == r.purl, + CopyTarget::Purl(purl) => PurlKey::same(purl, &r.purl), CopyTarget::NpmName(name) => crate::utils::purl::purl_name_version(&r.purl) .and_then(|(_, version)| npm_purl(name, version)) - .is_some_and(|p| canonical_base_purl(&p) == r.purl), + .is_some_and(|p| PurlKey::same(&p, &r.purl)), CopyTarget::Any => true, } } @@ -641,7 +641,7 @@ impl Discovery { let key = canonical_base_purl(purl); self.refs .iter() - .any(|r| r.uuid == uuid && r.mode == mode && same_release(&r.purl, &key)) + .any(|r| r.uuid == uuid && r.mode == mode && PurlKey::same(&r.purl, &key)) } /// Whether some file discovery read mentions patch `uuid` as a `mode` @@ -848,7 +848,7 @@ impl Discovery { self.refs.iter().any(|r| { r.uuid == uuid && r.mode == WiringMode::Vendored - && same_release(&r.purl, &key) + && PurlKey::same(&r.purl, &key) && r.artifact_rel.as_deref() == Some(artifact) }) }) @@ -862,7 +862,7 @@ impl Discovery { pub fn vendored_contest(&self, purl: &str, uuid: &str) -> Option<&ContestedRef> { let key = canonical_base_purl(purl); self.contested.iter().find(|c| { - c.uuid == uuid && c.mode == WiringMode::Vendored && same_release(&c.purl, &key) + c.uuid == uuid && c.mode == WiringMode::Vendored && PurlKey::same(&c.purl, &key) }) } @@ -871,9 +871,9 @@ impl Discovery { /// ([`Discovery::resolved_elsewhere`]). pub fn resolves_package(&self, purl: &str) -> bool { let key = canonical_base_purl(purl); - self.refs.iter().any(|r| same_release(&r.purl, &key)) - || self.contested.iter().any(|c| same_release(&c.purl, &key)) - || self.elsewhere.iter().any(|e| same_release(&e.purl, &key)) + self.refs.iter().any(|r| PurlKey::same(&r.purl, &key)) + || self.contested.iter().any(|c| PurlKey::same(&c.purl, &key)) + || self.elsewhere.iter().any(|e| PurlKey::same(&e.purl, &key)) } fn recognize(&mut self, uuid: &str, mode: WiringMode, file: &str) { @@ -1725,43 +1725,6 @@ pub(crate) fn toml_or_diag( // ── purl helpers ───────────────────────────────────────────────────────── -/// The comparison key for "the same package" across purl spellings: -/// qualifiers and subpath stripped, components percent-decoded, the type -/// lowercased, and the name folded where the ecosystem's own resolution is -/// insensitive — pypi (PEP 503: case + `-`/`_`/`.` runs), composer and -/// nuget (case). Used to match discovered refs against manifest / ledger -/// keys and API purls; it is also the form [`PatchedRef::purl`] carries. -/// Never used to build filesystem paths. -pub fn canonical_base_purl(purl: &str) -> String { - let base = normalize_purl(strip_purl_qualifiers(purl.trim())).into_owned(); - let Some(rest) = base.strip_prefix("pkg:") else { - return base; - }; - let Some((ty, tail)) = rest.split_once('/') else { - return base; - }; - let ty = ty.to_ascii_lowercase(); - match ty.as_str() { - "pypi" => match tail.rsplit_once('@') { - Some((name, version)) => { - format!("pkg:pypi/{}@{version}", canonicalize_pypi_name(name)) - } - None => format!("pkg:pypi/{}", canonicalize_pypi_name(tail)), - }, - "composer" | "nuget" => format!("pkg:{ty}/{}", tail.to_lowercase()), - _ => format!("pkg:{ty}/{tail}"), - } -} - -/// Whether two [`canonical_base_purl`] spellings name the same package -/// release, whichever patch generation each side recorded: equal, or for -/// composer the same release in another version spelling (a ledger's -/// `@3.0.2.0` is the lock's `@3.0.2`). The one same-release predicate the -/// ledgers, `vex` and `remove` / `rollback` share. -pub fn same_release(a: &str, b: &str) -> bool { - a == b || crate::utils::composer_version::composer_purls_equivalent(a, b) -} - /// [`canonical_base_purl`] for a ref about to be pushed, plus shape checks: /// a known ecosystem type and a non-empty name and version (the version /// after the LAST `@`, containing no `/`). @@ -1792,7 +1755,7 @@ impl Discovery { let key = canonical_base_purl(purl); self.refs .iter() - .any(|r| r.mode == mode && same_release(&r.purl, &key)) + .any(|r| r.mode == mode && PurlKey::same(&r.purl, &key)) } /// Liveness of a VENDOR-ledger entry — the ONE rule every reader of the @@ -2641,35 +2604,6 @@ mod tests { assert_eq!(vendored_leaf_purl("npm", "not-a-tarball"), None); } - #[test] - fn canonical_base_purl_folds_only_insensitive_ecosystems() { - assert_eq!( - canonical_base_purl("pkg:pypi/Python_Dateutil@2.8.2?artifact_id=py3-none-any-whl"), - "pkg:pypi/python-dateutil@2.8.2" - ); - assert_eq!( - canonical_base_purl("pkg:npm/%40scope/Name@1.0.0"), - "pkg:npm/@scope/Name@1.0.0", - "npm is case-sensitive; only percent-decoding applies" - ); - assert_eq!( - canonical_base_purl("pkg:nuget/Newtonsoft.Json@13.0.1"), - "pkg:nuget/newtonsoft.json@13.0.1" - ); - assert_eq!( - canonical_base_purl("pkg:composer/Monolog/Monolog@2.0.0"), - "pkg:composer/monolog/monolog@2.0.0" - ); - assert_eq!( - canonical_base_purl("pkg:gem/nokogiri@1.16.5?platform=java"), - "pkg:gem/nokogiri@1.16.5" - ); - assert_eq!( - canonical_base_purl("pkg:golang/github.com/Foo/bar@v1.0.0#sub/dir"), - "pkg:golang/github.com/Foo/bar@v1.0.0" - ); - } - #[test] fn socket_patch_names_are_exact() { assert_eq!( diff --git a/crates/socket-patch-core/src/vex/mod.rs b/crates/socket-patch-core/src/vex/mod.rs index 0ce4b58ea..8db81df11 100644 --- a/crates/socket-patch-core/src/vex/mod.rs +++ b/crates/socket-patch-core/src/vex/mod.rs @@ -26,9 +26,9 @@ pub mod verify; pub use build::{build_document, BuildOptions}; pub use discover::{ - canonical_base_purl, discover_patched_refs, discover_patched_refs_in, - discover_patched_refs_with, Diag, DiscoverOptions, Discovery, PatchedRef, Recognized, - Unattested, UnattestedKind, UnlockedPin, WiringMode, + discover_patched_refs, discover_patched_refs_in, discover_patched_refs_with, Diag, + DiscoverOptions, Discovery, PatchedRef, Recognized, Unattested, UnattestedKind, UnlockedPin, + WiringMode, }; pub use product::{detect_product, DetectResult}; pub use schema::{