Fix Python crawler missing .egg-info installs (#447)

crates/socket-patch-cli/src/commands/scan/hosted/python.rs

@@ -213,4 +213,28 @@ mod tests {
         assert!(out.stale_purls.is_empty());
         assert!(out.warnings.is_empty());
     }
+
+    /// A legacy `.egg-info` install (pip < 23.1 building an sdist without
+    /// `wheel`) is a real copy pip keeps on `install -r`, so the hosted
+    /// stale-install guard must judge it like a `.dist-info` one (#447).
+    #[tokio::test]
+    async fn egg_info_install_gets_the_stale_install_warning() {
+        let tmp = tempfile::tempdir().unwrap();
+        let site = tmp.path().join("site-packages");
+        let egg = site.join("six-1.16.0-py3.11.egg-info");
+        std::fs::create_dir_all(&egg).unwrap();
+        std::fs::write(egg.join("PKG-INFO"), "Name: six\nVersion: 1.16.0\n").unwrap();
+        std::fs::write(site.join("six.py"), b"upstream").unwrap();
+        let common = crate::args::GlobalArgs {
+            cwd: tmp.path().to_path_buf(),
+            global_prefix: Some(site.clone()),
+            ..Default::default()
+        };
+        let purl = "pkg:pypi/six@1.16.0";
+        let confirmed = vec![(purl.to_string(), "six-uuid".to_string())];
+        let ledger = BTreeMap::from([("k".into(), record("six-uuid", "six.py", b"patched"))]);
+        let out = stale_install_warnings(&common, &confirmed, &BTreeSet::new(), &ledger).await;
+        assert_eq!(out.stale_purls, BTreeSet::from([purl.to_string()]));
+        assert_eq!(out.warnings[0]["code"], "redirect_pypi_stale_install");
+    }
 }

crates/socket-patch-cli/tests/e2e_vex_build/hatch.rs

@@ -414,15 +414,28 @@ fn flow(flavor: Flavor, mode: Mode) {
     );
 
     if mode == Mode::Hosted {
-        // Not installed as far as the crawler knows (no VIRTUAL_ENV, no
-        // in-project venv): the declaration's sha256 pin is the basis.
+        // Not installed as far as the crawler knows: the declaration's
+        // sha256 pin is the basis. The run points VIRTUAL_ENV at an EMPTY
+        // virtualenv. With no venv at all, a Python project falls back to
+        // the global interpreters, and on Ubuntu runners those carry apt's
+        // python3-six 1.16.0 (`six-1.16.0.egg-info` in
+        // /usr/lib/python3/dist-packages) — a real unpatched copy that vex
+        // rightly refuses to attest over.
+        let empty_venv = tmp.path().join("empty-venv");
+        std::fs::create_dir_all(empty_venv.join(if cfg!(windows) {
+            "Lib/site-packages"
+        } else {
+            "lib/python3.11/site-packages"
+        }))
+        .unwrap();
         let patch_api = vex_e2e_common::PatchApi::start(vec![(
             mode.uuid().to_string(),
             view(mode.uuid(), &pristine, &patched),
         )]);
         let run = vex_e2e_common::VexRun {
             patch_server_url: Some(api.uri()),
             product: Some(PRODUCT.into()),
+            envs: vec![("VIRTUAL_ENV".into(), empty_venv.into_os_string())],
             ..vex_e2e_common::VexRun::online(&patch_api)
         };
         let out = vex_e2e_common::run_vex(&vex_e2e_common::binary(), &fresh, &run);

crates/socket-patch-cli/tests/in_process_pypi_apply.rs

@@ -613,3 +613,90 @@ async fn pypi_crawler_finds_real_installed_six() {
         "batch request did not include the discovered six PURL {purl}; bodies: {batch_bodies:?}"
     );
 }
+
+// ---------------------------------------------------------------------------
+// Legacy `.egg-info` installs (#447)
+// ---------------------------------------------------------------------------
+
+/// pip < 23.1 installing an sdist without `wheel` (the default state of a
+/// fresh venv on CPython <= 3.11) records the install as
+/// `<name>-<version>-pyX.Y.egg-info` instead of `.dist-info`. Agent mode
+/// must find and patch that copy instead of skipping it as "not installed".
+/// The layouts are planted directly so the test needs no interpreter.
+#[tokio::test]
+#[serial]
+async fn pypi_scan_sync_patches_egg_info_install() {
+    // (a) the pip/setuptools directory form with `PKG-INFO`;
+    // (b) the bare distutils / apt `.egg-info` FILE form.
+    for bare_file in [false, true] {
+        let tmp = tempfile::tempdir().expect("tempdir");
+        let venv = tmp.path().join(".venv");
+        let site = if cfg!(windows) {
+            venv.join("Lib").join("site-packages")
+        } else {
+            venv.join("lib").join("python3.11").join("site-packages")
+        };
+        std::fs::create_dir_all(&site).unwrap();
+        let pkg_info =
+            format!("Metadata-Version: 1.2\nName: {PYPI_PACKAGE}\nVersion: {PYPI_VERSION}\n");
+        if bare_file {
+            std::fs::write(
+                site.join(format!("{PYPI_PACKAGE}-{PYPI_VERSION}.egg-info")),
+                &pkg_info,
+            )
+            .unwrap();
+        } else {
+            let egg = site.join(format!("{PYPI_PACKAGE}-{PYPI_VERSION}-py3.11.egg-info"));
+            std::fs::create_dir_all(&egg).unwrap();
+            std::fs::write(egg.join("PKG-INFO"), &pkg_info).unwrap();
+        }
+        let six_path = site.join("six.py");
+        let original = b"# six from an sdist\n".to_vec();
+        std::fs::write(&six_path, &original).unwrap();
+        let mut patched = original.clone();
+        patched.extend_from_slice(b"# SOCKET-PATCH-E2E-MARKER\n");
+
+        let server = MockServer::start().await;
+        setup_pypi_apply_mock(
+            &server,
+            &git_sha256(&original),
+            &git_sha256(&patched),
+            &patched,
+        )
+        .await;
+
+        std::env::remove_var("VIRTUAL_ENV");
+        let code = scan_run(ScanArgs {
+            socket_yml: Default::default(),
+            paths: Vec::new(),
+            packages: Vec::new(),
+            common: socket_patch_cli::args::GlobalArgs {
+                cwd: tmp.path().to_path_buf(),
+                org: Some(ORG.to_string()),
+                json: true,
+                yes: true,
+                api_url: Some(server.uri()),
+                api_token: Some("fake".to_string()),
+                ecosystems: Some(vec!["pypi".to_string()]),
+                download_mode: "diff".to_string(),
+                ..socket_patch_cli::args::GlobalArgs::default()
+            },
+            batch_size: Some(100),
+            apply: false,
+            prune: false,
+            sync: true,
+            vendor: false,
+            mode: None,
+            all_releases: false,
+            vex: Default::default(),
+            rollout: Default::default(),
+        })
+        .await;
+        assert_eq!(code, 0, "bare_file={bare_file}: scan --sync should succeed");
+        assert_eq!(
+            std::fs::read(&six_path).unwrap(),
+            patched,
+            "bare_file={bare_file}: the egg-info install must be patched"
+        );
+    }
+}

crates/socket-patch-cli/tests/in_process_python_envs.rs

@@ -456,38 +456,41 @@ async fn pypi_egg_info_layout_handled() {
     let tmp = tempfile::tempdir().unwrap();
     let site = venv_site_packages(&tmp.path().join(".venv"), "python3.11");
     std::fs::create_dir_all(&site).unwrap();
-    // egg-info — older format. The crawler only recognizes `.dist-info`
-    // dirs, so the egg-info package is NOT discovered. Pin that current
-    // contract: scan exits cleanly (like the empty-site-packages case) and
-    // ships no PURL for it. If egg-info support is added later this fails
-    // loudly and the assertion should be flipped to `assert_discovered`.
-    let egg = site.join("legacy_pkg-1.0.0.egg-info");
+    // egg-info — the legacy layout pip < 23.1 writes for an sdist built
+    // without `wheel` (and distutils / distro packages write as a bare
+    // FILE). It is a real, importable install, so the crawler must report
+    // it (#447). Three shapes: a `-pyX.Y`-suffixed directory with
+    // `PKG-INFO`, a bare `.egg-info` file, and a directory whose PKG-INFO
+    // is missing (the filename carries the identity).
+    let egg = site.join("legacy_pkg-1.0.0-py3.11.egg-info");
     std::fs::create_dir_all(&egg).unwrap();
     std::fs::write(
         egg.join("PKG-INFO"),
         "Metadata-Version: 1.0\nName: legacy_pkg\nVersion: 1.0.0\n",
     )
     .unwrap();
+    std::fs::write(
+        site.join("distro_pkg-2.1.egg-info"),
+        "Metadata-Version: 1.1\nName: distro-pkg\nVersion: 2.1\n",
+    )
+    .unwrap();
+    std::fs::create_dir_all(site.join("bare_dir_pkg-0.3-py3.11.egg-info")).unwrap();
 
-    // Positive control in the SAME site-packages: a real `.dist-info`
-    // package the crawler must discover. Without it, the negative
-    // assertions below are vacuous — they pass even if the crawler never
-    // walked this directory at all (e.g. a regression that stops probing
-    // `.venv`). The control proves the dir WAS walked, so a missing
-    // `legacy_pkg` means egg-info was specifically not recognized, not that
-    // scanning silently no-op'd.
+    // A `.dist-info` sibling in the SAME site-packages: both layouts are
+    // listed side by side.
     write_dist_info(&site, "modern_sibling", "2.0.0");
 
     let server = MockServer::start().await;
     mock_batch_empty(&server).await;
     let res = scan_scrubbed(default_args(tmp.path(), server.uri())).await;
-    assert_eq!(res, 0, "egg-info layout must scan cleanly without crashing");
+    assert_eq!(res, 0, "egg-info layout must scan cleanly");
     let bodies = batch_bodies(&server).await;
-    // Control: proves the crawler genuinely walked this site-packages dir.
     assert_discovered(&bodies, "pkg:pypi/modern-sibling@2.0.0");
-    // Not discovered today; neither the canonical nor raw name may appear.
-    assert_not_discovered(&bodies, "pkg:pypi/legacy-pkg@1.0.0");
-    assert_not_discovered(&bodies, "pkg:pypi/legacy_pkg@1.0.0");
+    assert_discovered(&bodies, "pkg:pypi/legacy-pkg@1.0.0");
+    assert_discovered(&bodies, "pkg:pypi/distro-pkg@2.1");
+    assert_discovered(&bodies, "pkg:pypi/bare-dir-pkg@0.3");
+    // The `-pyX.Y` suffix is not part of the version.
+    assert_not_discovered(&bodies, "py3.11");
 }
 
 // ---------------------------------------------------------------------------

crates/socket-patch-cli/tests/vendor_eject_fresh_checkout.rs

@@ -370,11 +370,25 @@ async fn pypi_eject_needs_no_virtualenv() {
     )
     .unwrap();
 
+    // Nothing installed for the project: VIRTUAL_ENV names an EMPTY
+    // virtualenv. With no venv at all, a Python project falls back to the
+    // global interpreters, and on Ubuntu those carry apt's python3-six
+    // 1.16.0 (`six-1.16.0.egg-info`), whose bytes are not this fixture's.
+    let empty_venv = tmp.path().join("empty-venv");
+    std::fs::create_dir_all(empty_venv.join(if cfg!(windows) {
+        "Lib/site-packages"
+    } else {
+        "lib/python3.11/site-packages"
+    }))
+    .unwrap();
     let (code, env) = run_json_with(
         &root,
         &server,
         &["vendor"],
-        &[("SOCKET_PYPI_JSON_API", format!("{}/pypi", server.uri()))],
+        &[
+            ("SOCKET_PYPI_JSON_API", format!("{}/pypi", server.uri())),
+            ("VIRTUAL_ENV", empty_venv.display().to_string()),
+        ],
     );
     assert_eq!(code, 0, "a fresh hosted pypi checkout ejects: {env:#}");
     assert!(applied(&env, PURL), "{env:#}");