Fix vendored Pipenv re-vendor to a newer patch (#769)

crates/socket-patch-cli/src/commands/vendor.rs

@@ -34,9 +34,9 @@ use socket_patch_core::utils::group_commit::{CommittedFile, GroupCommit};
 use socket_patch_core::utils::purl::{canonical_purl, normalize_purl, strip_purl_qualifiers};
 use socket_patch_core::utils::socket_dir::remove_tree_and_prune;
 use socket_patch_core::vendor::{
-    self, ecosystem_dir_for_purl, load_state, lock_inventory, lookup_entry, save_state,
-    save_state_shared, PackageSource, RevertOpts, RevertOutcome, VendorEntry, VendorOutcome,
-    VendorServiceConfig, VendorState, VendorWarning,
+    self, ecosystem_dir_for_purl, load_state, lock_inventory, lookup_entry, lookup_entry_kv,
+    save_state, save_state_shared, PackageSource, RevertOpts, RevertOutcome, VendorEntry,
+    VendorOutcome, VendorServiceConfig, VendorState, VendorWarning,
 };
 use socket_patch_core::vex::time::now_rfc3339;
 use std::collections::{HashMap, HashSet};
@@ -2010,6 +2010,24 @@ impl StagedSource {
     }
 }
 
+/// Whether an installed copy that fails `candidate`'s variant probe is
+/// still `candidate` itself, superseded: the ledger vendored exactly this
+/// package at an OLDER patch uuid (#769), so the venv most likely holds
+/// that patch's bytes (`pipenv sync` from the vendored wheel), which are
+/// neither the pristine release nor this patch's output. The re-vendor
+/// then takes the pristine artifact from the lock / registry / service, as
+/// a lock-only checkout does, instead of reporting it not installed.
+fn superseded_install(
+    ledger: &VendorState,
+    candidate: &str,
+    record: &PatchRecord,
+    sole_candidate: bool,
+) -> bool {
+    lookup_entry_kv(&ledger.entries, candidate).is_some_and(|(key, entry)| {
+        entry.uuid != record.uuid && (sole_candidate || key == candidate)
+    })
+}
+
 #[allow(clippy::too_many_arguments)]
 async fn plan_service_downloads(
     cwd: &Path,
@@ -2026,6 +2044,8 @@ async fn plan_service_downloads(
         &vendor::pypi::InstalledSiteListings,
     ),
 ) -> Vec<socket_patch_core::api::client::PlannedDownload> {
+    // The loop's stand-in for a superseded install (see there).
+    let uninstalled = cwd.join(".socket/vendor/.uninstalled");
     // Each loop candidate that reaches its backend, in loop order.
     let mut reaching: Vec<(&str, &PatchRecord, &Path)> = Vec::new();
     let mut handled_bases: HashSet<String> = HashSet::new();
@@ -2057,14 +2077,19 @@ async fn plan_service_downloads(
                 && !matches!(Ecosystem::from_purl(candidate), Some(Ecosystem::Maven));
             let ledger_answers_probe =
                 lookup_entry(&ledger.entries, candidate).is_some_and(|e| e.uuid == record.uuid);
+            let mut source_path = source.path();
             if probe_applicable && !force && !ledger_answers_probe {
                 if matches!(staged, StagedSource::Installed(_)) {
                     if let Some((file, info)) = representative_file(&record.files) {
                         let dir = source.path();
                         if !variant_matches_installed(Some(
                             &verify_file_patch(dir, file, info).await.status,
                         )) {
-                            continue;
+                            if !superseded_install(ledger, candidate, record, candidates.len() == 1)
+                            {
+                                continue;
+                            }
+                            source_path = &uninstalled;
                         }
                     }
                 } else if candidates.len() > 1 && lookup_entry(&ledger.entries, candidate).is_none()
@@ -2102,7 +2127,7 @@ async fn plan_service_downloads(
             if lookup_entry(&ledger.entries, candidate).is_some_and(|e| e.uuid == record.uuid) {
                 continue;
             }
-            reaching.push((candidate.as_str(), record, source.path()));
+            reaching.push((candidate.as_str(), record, source_path));
         }
     }
 
@@ -2459,6 +2484,8 @@ pub(crate) async fn vendor_records_reusing(
         && !socket_patch_core::utils::failpoint::switched_off("group_commit"))
     .then(|| GroupCommit::begin(&common.cwd));
     let mut stale_artifacts: Vec<StaleArtifact> = Vec::new();
+    // The source of a superseded install (see [`superseded_install`]).
+    let uninstalled = common.cwd.join(".socket/vendor/.uninstalled");
     for (index, (purl, staged)) in all_packages.iter().enumerate() {
         let pkg_source = staged.as_source();
         let is_variant_eco =
@@ -2500,6 +2527,7 @@ pub(crate) async fn vendor_records_reusing(
             // without downloading the pristine tree just to read one file.
             let ledger_answers_probe =
                 lookup_entry(&state.entries, candidate).is_some_and(|e| e.uuid == record.uuid);
+            let mut candidate_source = pkg_source;
             if probe_applicable && !force && !ledger_answers_probe {
                 if matches!(staged, StagedSource::Installed(_)) {
                     if let Some((file, info)) = representative_file(&record.files) {
@@ -2508,7 +2536,11 @@ pub(crate) async fn vendor_records_reusing(
                                 .await
                                 .status,
                         )) {
-                            continue;
+                            if !superseded_install(&state, candidate, record, candidates.len() == 1)
+                            {
+                                continue;
+                            }
+                            candidate_source = PackageSource::Installed(&uninstalled);
                         }
                     }
                 } else if candidates.len() > 1 && lookup_entry(&state.entries, candidate).is_none()
@@ -2815,7 +2847,7 @@ pub(crate) async fn vendor_records_reusing(
             ));
             let outcome = dispatch_vendor_one(
                 candidate,
-                pkg_source,
+                candidate_source,
                 &common.cwd,
                 record,
                 sources,

crates/socket-patch-cli/tests/mode_migration_pypi.rs

@@ -456,6 +456,94 @@ async fn pipenv_vendored_to_hosted() {
     assert_vendored_to_hosted(&root, files).await;
 }
 
+/// A superseding patch for the same release (a fixed patch, or one
+/// covering more CVEs).
+const UUID_B: &str = "6d4f2b3c-8e5a-4f7b-9c9d-2e3f4a5b6c7d";
+const PATCHED_B: &[u8] = b"# six\nVERSION = '1.16.0'\nSOCKET_PATCHED = 2\n";
+
+/// A virtualenv whose six was installed from the vendored wheel of patch
+/// A (`pipenv sync` after the first vendor): its files are A's patched
+/// bytes, not the pristine release patch B is diffed against.
+fn venv_installed_from_patch_a(venv: &Path) {
+    let site = venv.join("lib/python3.11/site-packages");
+    let dist = site.join("six-1.16.0.dist-info");
+    std::fs::create_dir_all(&dist).unwrap();
+    std::fs::write(site.join("six.py"), PATCHED).unwrap();
+    std::fs::write(
+        dist.join("METADATA"),
+        "Metadata-Version: 2.1\nName: six\nVersion: 1.16.0\n\n",
+    )
+    .unwrap();
+    std::fs::write(
+        dist.join("WHEEL"),
+        "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py2-none-any\nTag: py3-none-any\n",
+    )
+    .unwrap();
+    std::fs::write(dist.join("INSTALLER"), "pip\n").unwrap();
+    std::fs::write(
+        dist.join("RECORD"),
+        "six.py,,\nsix-1.16.0.dist-info/METADATA,,\nsix-1.16.0.dist-info/WHEEL,,\nsix-1.16.0.dist-info/INSTALLER,,\nsix-1.16.0.dist-info/RECORD,,\n",
+    )
+    .unwrap();
+}
+
+/// #769: a Pipenv project vendored at patch A moves to the superseding
+/// patch B when the manifest offers it, as the `would_revendor` preview
+/// and the CLI contract promise, whether the checkout is lock-only or its
+/// venv was installed from A's vendored wheel. Pipfile.lock is rewired to
+/// B, A's artifact is swept, and reverting B restores the registry pin.
+#[tokio::test]
+async fn pipenv_revendors_to_a_superseding_patch() {
+    for venv_present in [false, true] {
+        let lane = if venv_present {
+            "venv from A"
+        } else {
+            "lock-only"
+        };
+        let (_tmp, root) = project();
+        stage_pipenv(&root);
+        let registry = std::fs::read_to_string(root.join("Pipfile.lock")).unwrap();
+        vendor_project(&root, &["Pipfile.lock"]);
+
+        let venv = root.join("../patched-venv");
+        let mut extra: Vec<(&str, &str)> = Vec::new();
+        if venv_present {
+            venv_installed_from_patch_a(&venv);
+            extra.push(("VIRTUAL_ENV", venv.to_str().unwrap()));
+        }
+        stage_manifest_with(&root, UUID_B, PATCHED_B);
+        let (code, env) = run_cli(&root, &["vendor"], &extra);
+        assert_eq!(code, 0, "{lane}: re-vendor to B: {env:#}");
+        assert!(
+            env.to_string().contains("vendor_stale_artifact_removed"),
+            "{lane}: A's artifact is swept: {env:#}"
+        );
+        let lock = std::fs::read_to_string(root.join("Pipfile.lock")).unwrap();
+        assert!(
+            lock.contains(&format!(".socket/vendor/pypi/{UUID_B}/")) && !lock.contains(UUID),
+            "{lane}: Pipfile.lock is rewired to B:\n{lock}"
+        );
+        assert!(!root.join(format!(".socket/vendor/pypi/{UUID}")).exists());
+        let wheels: Vec<_> = std::fs::read_dir(root.join(format!(".socket/vendor/pypi/{UUID_B}")))
+            .unwrap()
+            .flatten()
+            .filter(|e| e.file_name().to_string_lossy().ends_with(".whl"))
+            .collect();
+        assert_eq!(wheels.len(), 1, "{lane}: B's wheel is vendored");
+
+        let (code, env) = run_cli(&root, &["vendor", "--revert"], &extra);
+        assert_eq!(code, 0, "{lane}: revert B: {env:#}");
+        let reverted: Value =
+            serde_json::from_str(&std::fs::read_to_string(root.join("Pipfile.lock")).unwrap())
+                .unwrap();
+        let registry: Value = serde_json::from_str(&registry).unwrap();
+        assert_eq!(
+            reverted, registry,
+            "{lane}: revert restores the registry pin"
+        );
+    }
+}
+
 const UV_LOCK: &str = r#"version = 1
 revision = 2
 requires-python = ">=3.9"

crates/socket-patch-core/src/crawlers/gradle_cache.rs

@@ -70,8 +70,7 @@ pub fn hash_eq(dir_name: &str, sha1_hex: &str) -> bool {
 /// Whether `bytes` are the pristine download Gradle stored in the hash
 /// directory `dir_name` (their sha1 names it).
 pub fn pristine(dir_name: &str, bytes: &[u8]) -> bool {
-    use sha1::{Digest, Sha1};
-    hash_eq(dir_name, &hex::encode(Sha1::digest(bytes)))
+    hash_eq(dir_name, &crate::utils::digest::sha1_hex_of(bytes))
 }
 
 /// Whether `path` is a version directory of a `files-2.1` tree
@@ -432,8 +431,6 @@ impl DerivedIndex {
     /// The [`DerivedCopies`] of the jar `jar_leaf` whose pristine bytes
     /// hash to `pristine_sha1`.
     pub fn query(&self, jar_leaf: &str, pristine_sha1: &str) -> DerivedCopies {
-        use sha1::{Digest, Sha1};
-
         let instrumented = format!("instrumented-{jar_leaf}");
         let mut out = DerivedCopies {
             incomplete: self.incomplete,
@@ -460,7 +457,9 @@ impl DerivedIndex {
                 out.stale.push(path.clone());
             } else if name == jar_leaf || name == instrumented {
                 match crate::utils::fs::read_regular_to_bytes_sync(path) {
-                    Ok(bytes) if hash_eq(&hex::encode(Sha1::digest(&bytes)), pristine_sha1) => {
+                    Ok(bytes)
+                        if hash_eq(&crate::utils::digest::sha1_hex_of(&bytes), pristine_sha1) =>
+                    {
                         out.stale.push(path.clone())
                     }
                     Ok(_) => out.unknown.push(path.clone()),

crates/socket-patch-core/src/patch/jvm_jar.rs

@@ -25,8 +25,6 @@
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 
-use sha1::Digest as _;
-
 use crate::crawlers::gradle_cache;
 use crate::hash::git_sha256::compute_git_sha256_from_bytes;
 use crate::manifest::schema::PatchFileInfo;
@@ -353,12 +351,11 @@ fn unpatched_members(
 }
 
 fn sha256_hex(bytes: &[u8]) -> String {
-    use sha2::Digest as _;
-    hex::encode(sha2::Sha256::digest(bytes))
+    crate::utils::digest::sha256_hex_of(bytes)
 }
 
 fn sha1_hex(bytes: &[u8]) -> String {
-    hex::encode(sha1::Sha1::digest(bytes))
+    crate::utils::digest::sha1_hex_of(bytes)
 }
 
 /// `<socket_dir>/jvm-originals/<sha256>.jar`.

crates/socket-patch-core/src/patch/sidecars/maven.rs

@@ -17,8 +17,6 @@
 
 use std::path::{Path, PathBuf};
 
-use sha1::Digest as _;
-
 use super::{
     SidecarAdvisory, SidecarAdvisoryCode, SidecarError, SidecarFile, SidecarFileAction,
     SidecarPayload, SidecarSeverity,
@@ -44,7 +42,7 @@ impl Algo {
 
     fn digest(self, bytes: &[u8]) -> String {
         match self {
-            Algo::Sha1 => hex::encode(sha1::Sha1::digest(bytes)),
+            Algo::Sha1 => crate::utils::digest::sha1_hex_of(bytes),
             Algo::Md5 => hex::encode(md5(bytes)),
         }
     }

crates/socket-patch-core/src/vendor/mod.rs

@@ -120,8 +120,8 @@ pub use source::PackageSource;
 pub(crate) use npm_common::is_safe_npm_name;
 pub use pypi_requirements::requirements_include_names;
 pub use state::{
-    carry_forward_wiring, load_state, lookup_entry, purl_keys_cover, save_state, save_state_shared,
-    VendorEntry, VendorState, VENDOR_STATE_REL,
+    carry_forward_wiring, load_state, lookup_entry, lookup_entry_kv, purl_keys_cover, save_state,
+    save_state_shared, VendorEntry, VendorState, VENDOR_STATE_REL,
 };
 pub use verify::{
     artifact_is_file_shaped, check_vendored_artifact, compute_dir_inventory,