Fix Python crawler missing .egg-info installs (#447) (#452)

* Start fix for #447

Assisted-by: Claude Code:claude-opus-5-5

* Find Python packages installed as .egg-info

pip before 23.1 installs an sdist without the wheel package via
setup.py install, which records the install as
<name>-<version>-pyX.Y.egg-info rather than .dist-info. That is the
default state of a fresh venv on CPython 3.11 and older, and distro
packages (Debian's python3-*) and distutils ship bare .egg-info
files. The crawler only listed .dist-info, so these real, importable
installs were reported "not installed" in agent mode, never got the
hosted stale-install warning, and were missing from scan -g.

The site-packages listing now also reads .egg-info directories
(PKG-INFO, falling back to the directory name) and bare .egg-info
files that carry metadata headers. Every consumer (scan, apply,
rollback, the stale-install guards, -g) shares that listing.

Fixes #447

Assisted-by: Claude Code:claude-opus-5-5

* Make the Hatch hosted pin-basis e2e hermetic

The step asserts vex attests from the lock pin when nothing is
installed, but it ran with no venv, so the Hatch project fell back to
the global interpreters. On Ubuntu runners those carry apt's
python3-six 1.16.0, the fixture's exact release, as an .egg-info
install. Now that the crawler sees .egg-info, vex correctly refused
to attest over that unpatched copy. Point VIRTUAL_ENV at an empty
venv so the step tests what it says: a package that is not installed.

Assisted-by: Claude Code:claude-opus-5-5

* Keep the pypi eject test off the system six

The test vendors a hosted six 1.16.0 from a fresh checkout with
nothing installed, but it ran with no venv, so the project fell back
to the global interpreters. On Ubuntu those carry apt's python3-six
1.16.0 as an .egg-info install. Now that the crawler sees it, the
variant probe compared it with the fixture's bytes and skipped it.
Point VIRTUAL_ENV at an empty venv so nothing is installed.

Assisted-by: Claude Code:claude-opus-5-5

* Use the Windows venv layout in the egg-info test

The new agent-mode egg-info test planted its site-packages at
lib/python3.11/site-packages, which is where a venv keeps them on
Unix. On Windows a venv uses Lib\site-packages, so the crawler never
found the package and the Windows test job failed.

Assisted-by: Claude Code:claude-opus-5-5

---------

Co-authored-by: Claude <noreply@anthropic.com>

Author: mikolalysenko

crates/socket-patch-cli/src/commands/scan/hosted/python.rs

@@ -213,4 +213,28 @@ mod tests {
         assert!(out.stale_purls.is_empty());
         assert!(out.warnings.is_empty());
     }
+
+    /// A legacy `.egg-info` install (pip < 23.1 building an sdist without
+    /// `wheel`) is a real copy pip keeps on `install -r`, so the hosted
+    /// stale-install guard must judge it like a `.dist-info` one (#447).
+    #[tokio::test]
+    async fn egg_info_install_gets_the_stale_install_warning() {
+        let tmp = tempfile::tempdir().unwrap();
+        let site = tmp.path().join("site-packages");
+        let egg = site.join("six-1.16.0-py3.11.egg-info");
+        std::fs::create_dir_all(&egg).unwrap();
+        std::fs::write(egg.join("PKG-INFO"), "Name: six\nVersion: 1.16.0\n").unwrap();
+        std::fs::write(site.join("six.py"), b"upstream").unwrap();
+        let common = crate::args::GlobalArgs {
+            cwd: tmp.path().to_path_buf(),
+            global_prefix: Some(site.clone()),
+            ..Default::default()
+        };
+        let purl = "pkg:pypi/six@1.16.0";
+        let confirmed = vec![(purl.to_string(), "six-uuid".to_string())];
+        let ledger = BTreeMap::from([("k".into(), record("six-uuid", "six.py", b"patched"))]);
+        let out = stale_install_warnings(&common, &confirmed, &BTreeSet::new(), &ledger).await;
+        assert_eq!(out.stale_purls, BTreeSet::from([purl.to_string()]));
+        assert_eq!(out.warnings[0]["code"], "redirect_pypi_stale_install");
+    }
 }

crates/socket-patch-cli/tests/e2e_vex_build/hatch.rs

@@ -414,15 +414,28 @@ fn flow(flavor: Flavor, mode: Mode) {
     );
 
     if mode == Mode::Hosted {
-        // Not installed as far as the crawler knows (no VIRTUAL_ENV, no
-        // in-project venv): the declaration's sha256 pin is the basis.
+        // Not installed as far as the crawler knows: the declaration's
+        // sha256 pin is the basis. The run points VIRTUAL_ENV at an EMPTY
+        // virtualenv. With no venv at all, a Python project falls back to
+        // the global interpreters, and on Ubuntu runners those carry apt's
+        // python3-six 1.16.0 (`six-1.16.0.egg-info` in
+        // /usr/lib/python3/dist-packages) — a real unpatched copy that vex
+        // rightly refuses to attest over.
+        let empty_venv = tmp.path().join("empty-venv");
+        std::fs::create_dir_all(empty_venv.join(if cfg!(windows) {
+            "Lib/site-packages"
+        } else {
+            "lib/python3.11/site-packages"
+        }))
+        .unwrap();
         let patch_api = vex_e2e_common::PatchApi::start(vec![(
             mode.uuid().to_string(),
             view(mode.uuid(), &pristine, &patched),
         )]);
         let run = vex_e2e_common::VexRun {
             patch_server_url: Some(api.uri()),
             product: Some(PRODUCT.into()),
+            envs: vec![("VIRTUAL_ENV".into(), empty_venv.into_os_string())],
             ..vex_e2e_common::VexRun::online(&patch_api)
         };
         let out = vex_e2e_common::run_vex(&vex_e2e_common::binary(), &fresh, &run);

crates/socket-patch-cli/tests/in_process_pypi_apply.rs

@@ -613,3 +613,90 @@ async fn pypi_crawler_finds_real_installed_six() {
         "batch request did not include the discovered six PURL {purl}; bodies: {batch_bodies:?}"
     );
 }
+
+// ---------------------------------------------------------------------------
+// Legacy `.egg-info` installs (#447)
+// ---------------------------------------------------------------------------
+
+/// pip < 23.1 installing an sdist without `wheel` (the default state of a
+/// fresh venv on CPython <= 3.11) records the install as
+/// `<name>-<version>-pyX.Y.egg-info` instead of `.dist-info`. Agent mode
+/// must find and patch that copy instead of skipping it as "not installed".
+/// The layouts are planted directly so the test needs no interpreter.
+#[tokio::test]
+#[serial]
+async fn pypi_scan_sync_patches_egg_info_install() {
+    // (a) the pip/setuptools directory form with `PKG-INFO`;
+    // (b) the bare distutils / apt `.egg-info` FILE form.
+    for bare_file in [false, true] {
+        let tmp = tempfile::tempdir().expect("tempdir");
+        let venv = tmp.path().join(".venv");
+        let site = if cfg!(windows) {
+            venv.join("Lib").join("site-packages")
+        } else {
+            venv.join("lib").join("python3.11").join("site-packages")
+        };
+        std::fs::create_dir_all(&site).unwrap();
+        let pkg_info =
+            format!("Metadata-Version: 1.2\nName: {PYPI_PACKAGE}\nVersion: {PYPI_VERSION}\n");
+        if bare_file {
+            std::fs::write(
+                site.join(format!("{PYPI_PACKAGE}-{PYPI_VERSION}.egg-info")),
+                &pkg_info,
+            )
+            .unwrap();
+        } else {
+            let egg = site.join(format!("{PYPI_PACKAGE}-{PYPI_VERSION}-py3.11.egg-info"));
+            std::fs::create_dir_all(&egg).unwrap();
+            std::fs::write(egg.join("PKG-INFO"), &pkg_info).unwrap();
+        }
+        let six_path = site.join("six.py");
+        let original = b"# six from an sdist\n".to_vec();
+        std::fs::write(&six_path, &original).unwrap();
+        let mut patched = original.clone();
+        patched.extend_from_slice(b"# SOCKET-PATCH-E2E-MARKER\n");
+
+        let server = MockServer::start().await;
+        setup_pypi_apply_mock(
+            &server,
+            &git_sha256(&original),
+            &git_sha256(&patched),
+            &patched,
+        )
+        .await;
+
+        std::env::remove_var("VIRTUAL_ENV");
+        let code = scan_run(ScanArgs {
+            socket_yml: Default::default(),
+            paths: Vec::new(),
+            packages: Vec::new(),
+            common: socket_patch_cli::args::GlobalArgs {
+                cwd: tmp.path().to_path_buf(),
+                org: Some(ORG.to_string()),
+                json: true,
+                yes: true,
+                api_url: Some(server.uri()),
+                api_token: Some("fake".to_string()),
+                ecosystems: Some(vec!["pypi".to_string()]),
+                download_mode: "diff".to_string(),
+                ..socket_patch_cli::args::GlobalArgs::default()
+            },
+            batch_size: Some(100),
+            apply: false,
+            prune: false,
+            sync: true,
+            vendor: false,
+            mode: None,
+            all_releases: false,
+            vex: Default::default(),
+            rollout: Default::default(),
+        })
+        .await;
+        assert_eq!(code, 0, "bare_file={bare_file}: scan --sync should succeed");
+        assert_eq!(
+            std::fs::read(&six_path).unwrap(),
+            patched,
+            "bare_file={bare_file}: the egg-info install must be patched"
+        );
+    }
+}

crates/socket-patch-cli/tests/in_process_python_envs.rs

@@ -456,38 +456,41 @@ async fn pypi_egg_info_layout_handled() {
     let tmp = tempfile::tempdir().unwrap();
     let site = venv_site_packages(&tmp.path().join(".venv"), "python3.11");
     std::fs::create_dir_all(&site).unwrap();
-    // egg-info — older format. The crawler only recognizes `.dist-info`
-    // dirs, so the egg-info package is NOT discovered. Pin that current
-    // contract: scan exits cleanly (like the empty-site-packages case) and
-    // ships no PURL for it. If egg-info support is added later this fails
-    // loudly and the assertion should be flipped to `assert_discovered`.
-    let egg = site.join("legacy_pkg-1.0.0.egg-info");
+    // egg-info — the legacy layout pip < 23.1 writes for an sdist built
+    // without `wheel` (and distutils / distro packages write as a bare
+    // FILE). It is a real, importable install, so the crawler must report
+    // it (#447). Three shapes: a `-pyX.Y`-suffixed directory with
+    // `PKG-INFO`, a bare `.egg-info` file, and a directory whose PKG-INFO
+    // is missing (the filename carries the identity).
+    let egg = site.join("legacy_pkg-1.0.0-py3.11.egg-info");
     std::fs::create_dir_all(&egg).unwrap();
     std::fs::write(
         egg.join("PKG-INFO"),
         "Metadata-Version: 1.0\nName: legacy_pkg\nVersion: 1.0.0\n",
     )
     .unwrap();
+    std::fs::write(
+        site.join("distro_pkg-2.1.egg-info"),
+        "Metadata-Version: 1.1\nName: distro-pkg\nVersion: 2.1\n",
+    )
+    .unwrap();
+    std::fs::create_dir_all(site.join("bare_dir_pkg-0.3-py3.11.egg-info")).unwrap();
 
-    // Positive control in the SAME site-packages: a real `.dist-info`
-    // package the crawler must discover. Without it, the negative
-    // assertions below are vacuous — they pass even if the crawler never
-    // walked this directory at all (e.g. a regression that stops probing
-    // `.venv`). The control proves the dir WAS walked, so a missing
-    // `legacy_pkg` means egg-info was specifically not recognized, not that
-    // scanning silently no-op'd.
+    // A `.dist-info` sibling in the SAME site-packages: both layouts are
+    // listed side by side.
     write_dist_info(&site, "modern_sibling", "2.0.0");
 
     let server = MockServer::start().await;
     mock_batch_empty(&server).await;
     let res = scan_scrubbed(default_args(tmp.path(), server.uri())).await;
-    assert_eq!(res, 0, "egg-info layout must scan cleanly without crashing");
+    assert_eq!(res, 0, "egg-info layout must scan cleanly");
     let bodies = batch_bodies(&server).await;
-    // Control: proves the crawler genuinely walked this site-packages dir.
     assert_discovered(&bodies, "pkg:pypi/modern-sibling@2.0.0");
-    // Not discovered today; neither the canonical nor raw name may appear.
-    assert_not_discovered(&bodies, "pkg:pypi/legacy-pkg@1.0.0");
-    assert_not_discovered(&bodies, "pkg:pypi/legacy_pkg@1.0.0");
+    assert_discovered(&bodies, "pkg:pypi/legacy-pkg@1.0.0");
+    assert_discovered(&bodies, "pkg:pypi/distro-pkg@2.1");
+    assert_discovered(&bodies, "pkg:pypi/bare-dir-pkg@0.3");
+    // The `-pyX.Y` suffix is not part of the version.
+    assert_not_discovered(&bodies, "py3.11");
 }
 
 // ---------------------------------------------------------------------------

crates/socket-patch-cli/tests/vendor_eject_fresh_checkout.rs

@@ -370,11 +370,25 @@ async fn pypi_eject_needs_no_virtualenv() {
     )
     .unwrap();
 
+    // Nothing installed for the project: VIRTUAL_ENV names an EMPTY
+    // virtualenv. With no venv at all, a Python project falls back to the
+    // global interpreters, and on Ubuntu those carry apt's python3-six
+    // 1.16.0 (`six-1.16.0.egg-info`), whose bytes are not this fixture's.
+    let empty_venv = tmp.path().join("empty-venv");
+    std::fs::create_dir_all(empty_venv.join(if cfg!(windows) {
+        "Lib/site-packages"
+    } else {
+        "lib/python3.11/site-packages"
+    }))
+    .unwrap();
     let (code, env) = run_json_with(
         &root,
         &server,
         &["vendor"],
-        &[("SOCKET_PYPI_JSON_API", format!("{}/pypi", server.uri()))],
+        &[
+            ("SOCKET_PYPI_JSON_API", format!("{}/pypi", server.uri())),
+            ("VIRTUAL_ENV", empty_venv.display().to_string()),
+        ],
     );
     assert_eq!(code, 0, "a fresh hosted pypi checkout ejects: {env:#}");
     assert!(applied(&env, PURL), "{env:#}");