diff --git a/crates/socket-patch-cli/src/commands/scan/hosted/python.rs b/crates/socket-patch-cli/src/commands/scan/hosted/python.rs index dd96fdce6..8a5445673 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted/python.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted/python.rs @@ -213,4 +213,28 @@ mod tests { assert!(out.stale_purls.is_empty()); assert!(out.warnings.is_empty()); } + + /// A legacy `.egg-info` install (pip < 23.1 building an sdist without + /// `wheel`) is a real copy pip keeps on `install -r`, so the hosted + /// stale-install guard must judge it like a `.dist-info` one (#447). + #[tokio::test] + async fn egg_info_install_gets_the_stale_install_warning() { + let tmp = tempfile::tempdir().unwrap(); + let site = tmp.path().join("site-packages"); + let egg = site.join("six-1.16.0-py3.11.egg-info"); + std::fs::create_dir_all(&egg).unwrap(); + std::fs::write(egg.join("PKG-INFO"), "Name: six\nVersion: 1.16.0\n").unwrap(); + std::fs::write(site.join("six.py"), b"upstream").unwrap(); + let common = crate::args::GlobalArgs { + cwd: tmp.path().to_path_buf(), + global_prefix: Some(site.clone()), + ..Default::default() + }; + let purl = "pkg:pypi/six@1.16.0"; + let confirmed = vec![(purl.to_string(), "six-uuid".to_string())]; + let ledger = BTreeMap::from([("k".into(), record("six-uuid", "six.py", b"patched"))]); + let out = stale_install_warnings(&common, &confirmed, &BTreeSet::new(), &ledger).await; + assert_eq!(out.stale_purls, BTreeSet::from([purl.to_string()])); + assert_eq!(out.warnings[0]["code"], "redirect_pypi_stale_install"); + } } diff --git a/crates/socket-patch-cli/tests/e2e_vex_build/hatch.rs b/crates/socket-patch-cli/tests/e2e_vex_build/hatch.rs index a7877c9ed..436ad1875 100644 --- a/crates/socket-patch-cli/tests/e2e_vex_build/hatch.rs +++ b/crates/socket-patch-cli/tests/e2e_vex_build/hatch.rs @@ -414,8 +414,20 @@ fn flow(flavor: Flavor, mode: Mode) { ); if mode == Mode::Hosted { - // Not installed as far as the crawler knows (no VIRTUAL_ENV, no - // in-project venv): the declaration's sha256 pin is the basis. + // Not installed as far as the crawler knows: the declaration's + // sha256 pin is the basis. The run points VIRTUAL_ENV at an EMPTY + // virtualenv. With no venv at all, a Python project falls back to + // the global interpreters, and on Ubuntu runners those carry apt's + // python3-six 1.16.0 (`six-1.16.0.egg-info` in + // /usr/lib/python3/dist-packages) — a real unpatched copy that vex + // rightly refuses to attest over. + let empty_venv = tmp.path().join("empty-venv"); + std::fs::create_dir_all(empty_venv.join(if cfg!(windows) { + "Lib/site-packages" + } else { + "lib/python3.11/site-packages" + })) + .unwrap(); let patch_api = vex_e2e_common::PatchApi::start(vec![( mode.uuid().to_string(), view(mode.uuid(), &pristine, &patched), @@ -423,6 +435,7 @@ fn flow(flavor: Flavor, mode: Mode) { let run = vex_e2e_common::VexRun { patch_server_url: Some(api.uri()), product: Some(PRODUCT.into()), + envs: vec![("VIRTUAL_ENV".into(), empty_venv.into_os_string())], ..vex_e2e_common::VexRun::online(&patch_api) }; let out = vex_e2e_common::run_vex(&vex_e2e_common::binary(), &fresh, &run); diff --git a/crates/socket-patch-cli/tests/in_process_pypi_apply.rs b/crates/socket-patch-cli/tests/in_process_pypi_apply.rs index 4cbac4574..1595f7596 100644 --- a/crates/socket-patch-cli/tests/in_process_pypi_apply.rs +++ b/crates/socket-patch-cli/tests/in_process_pypi_apply.rs @@ -613,3 +613,90 @@ async fn pypi_crawler_finds_real_installed_six() { "batch request did not include the discovered six PURL {purl}; bodies: {batch_bodies:?}" ); } + +// --------------------------------------------------------------------------- +// Legacy `.egg-info` installs (#447) +// --------------------------------------------------------------------------- + +/// pip < 23.1 installing an sdist without `wheel` (the default state of a +/// fresh venv on CPython <= 3.11) records the install as +/// `--pyX.Y.egg-info` instead of `.dist-info`. Agent mode +/// must find and patch that copy instead of skipping it as "not installed". +/// The layouts are planted directly so the test needs no interpreter. +#[tokio::test] +#[serial] +async fn pypi_scan_sync_patches_egg_info_install() { + // (a) the pip/setuptools directory form with `PKG-INFO`; + // (b) the bare distutils / apt `.egg-info` FILE form. + for bare_file in [false, true] { + let tmp = tempfile::tempdir().expect("tempdir"); + let venv = tmp.path().join(".venv"); + let site = if cfg!(windows) { + venv.join("Lib").join("site-packages") + } else { + venv.join("lib").join("python3.11").join("site-packages") + }; + std::fs::create_dir_all(&site).unwrap(); + let pkg_info = + format!("Metadata-Version: 1.2\nName: {PYPI_PACKAGE}\nVersion: {PYPI_VERSION}\n"); + if bare_file { + std::fs::write( + site.join(format!("{PYPI_PACKAGE}-{PYPI_VERSION}.egg-info")), + &pkg_info, + ) + .unwrap(); + } else { + let egg = site.join(format!("{PYPI_PACKAGE}-{PYPI_VERSION}-py3.11.egg-info")); + std::fs::create_dir_all(&egg).unwrap(); + std::fs::write(egg.join("PKG-INFO"), &pkg_info).unwrap(); + } + let six_path = site.join("six.py"); + let original = b"# six from an sdist\n".to_vec(); + std::fs::write(&six_path, &original).unwrap(); + let mut patched = original.clone(); + patched.extend_from_slice(b"# SOCKET-PATCH-E2E-MARKER\n"); + + let server = MockServer::start().await; + setup_pypi_apply_mock( + &server, + &git_sha256(&original), + &git_sha256(&patched), + &patched, + ) + .await; + + std::env::remove_var("VIRTUAL_ENV"); + let code = scan_run(ScanArgs { + socket_yml: Default::default(), + paths: Vec::new(), + packages: Vec::new(), + common: socket_patch_cli::args::GlobalArgs { + cwd: tmp.path().to_path_buf(), + org: Some(ORG.to_string()), + json: true, + yes: true, + api_url: Some(server.uri()), + api_token: Some("fake".to_string()), + ecosystems: Some(vec!["pypi".to_string()]), + download_mode: "diff".to_string(), + ..socket_patch_cli::args::GlobalArgs::default() + }, + batch_size: Some(100), + apply: false, + prune: false, + sync: true, + vendor: false, + mode: None, + all_releases: false, + vex: Default::default(), + rollout: Default::default(), + }) + .await; + assert_eq!(code, 0, "bare_file={bare_file}: scan --sync should succeed"); + assert_eq!( + std::fs::read(&six_path).unwrap(), + patched, + "bare_file={bare_file}: the egg-info install must be patched" + ); + } +} diff --git a/crates/socket-patch-cli/tests/in_process_python_envs.rs b/crates/socket-patch-cli/tests/in_process_python_envs.rs index 97a730bc3..22553fd8f 100644 --- a/crates/socket-patch-cli/tests/in_process_python_envs.rs +++ b/crates/socket-patch-cli/tests/in_process_python_envs.rs @@ -456,38 +456,41 @@ async fn pypi_egg_info_layout_handled() { let tmp = tempfile::tempdir().unwrap(); let site = venv_site_packages(&tmp.path().join(".venv"), "python3.11"); std::fs::create_dir_all(&site).unwrap(); - // egg-info — older format. The crawler only recognizes `.dist-info` - // dirs, so the egg-info package is NOT discovered. Pin that current - // contract: scan exits cleanly (like the empty-site-packages case) and - // ships no PURL for it. If egg-info support is added later this fails - // loudly and the assertion should be flipped to `assert_discovered`. - let egg = site.join("legacy_pkg-1.0.0.egg-info"); + // egg-info — the legacy layout pip < 23.1 writes for an sdist built + // without `wheel` (and distutils / distro packages write as a bare + // FILE). It is a real, importable install, so the crawler must report + // it (#447). Three shapes: a `-pyX.Y`-suffixed directory with + // `PKG-INFO`, a bare `.egg-info` file, and a directory whose PKG-INFO + // is missing (the filename carries the identity). + let egg = site.join("legacy_pkg-1.0.0-py3.11.egg-info"); std::fs::create_dir_all(&egg).unwrap(); std::fs::write( egg.join("PKG-INFO"), "Metadata-Version: 1.0\nName: legacy_pkg\nVersion: 1.0.0\n", ) .unwrap(); + std::fs::write( + site.join("distro_pkg-2.1.egg-info"), + "Metadata-Version: 1.1\nName: distro-pkg\nVersion: 2.1\n", + ) + .unwrap(); + std::fs::create_dir_all(site.join("bare_dir_pkg-0.3-py3.11.egg-info")).unwrap(); - // Positive control in the SAME site-packages: a real `.dist-info` - // package the crawler must discover. Without it, the negative - // assertions below are vacuous — they pass even if the crawler never - // walked this directory at all (e.g. a regression that stops probing - // `.venv`). The control proves the dir WAS walked, so a missing - // `legacy_pkg` means egg-info was specifically not recognized, not that - // scanning silently no-op'd. + // A `.dist-info` sibling in the SAME site-packages: both layouts are + // listed side by side. write_dist_info(&site, "modern_sibling", "2.0.0"); let server = MockServer::start().await; mock_batch_empty(&server).await; let res = scan_scrubbed(default_args(tmp.path(), server.uri())).await; - assert_eq!(res, 0, "egg-info layout must scan cleanly without crashing"); + assert_eq!(res, 0, "egg-info layout must scan cleanly"); let bodies = batch_bodies(&server).await; - // Control: proves the crawler genuinely walked this site-packages dir. assert_discovered(&bodies, "pkg:pypi/modern-sibling@2.0.0"); - // Not discovered today; neither the canonical nor raw name may appear. - assert_not_discovered(&bodies, "pkg:pypi/legacy-pkg@1.0.0"); - assert_not_discovered(&bodies, "pkg:pypi/legacy_pkg@1.0.0"); + assert_discovered(&bodies, "pkg:pypi/legacy-pkg@1.0.0"); + assert_discovered(&bodies, "pkg:pypi/distro-pkg@2.1"); + assert_discovered(&bodies, "pkg:pypi/bare-dir-pkg@0.3"); + // The `-pyX.Y` suffix is not part of the version. + assert_not_discovered(&bodies, "py3.11"); } // --------------------------------------------------------------------------- diff --git a/crates/socket-patch-cli/tests/vendor_eject_fresh_checkout.rs b/crates/socket-patch-cli/tests/vendor_eject_fresh_checkout.rs index 3f5d3ae49..2e029af81 100644 --- a/crates/socket-patch-cli/tests/vendor_eject_fresh_checkout.rs +++ b/crates/socket-patch-cli/tests/vendor_eject_fresh_checkout.rs @@ -370,11 +370,25 @@ async fn pypi_eject_needs_no_virtualenv() { ) .unwrap(); + // Nothing installed for the project: VIRTUAL_ENV names an EMPTY + // virtualenv. With no venv at all, a Python project falls back to the + // global interpreters, and on Ubuntu those carry apt's python3-six + // 1.16.0 (`six-1.16.0.egg-info`), whose bytes are not this fixture's. + let empty_venv = tmp.path().join("empty-venv"); + std::fs::create_dir_all(empty_venv.join(if cfg!(windows) { + "Lib/site-packages" + } else { + "lib/python3.11/site-packages" + })) + .unwrap(); let (code, env) = run_json_with( &root, &server, &["vendor"], - &[("SOCKET_PYPI_JSON_API", format!("{}/pypi", server.uri()))], + &[ + ("SOCKET_PYPI_JSON_API", format!("{}/pypi", server.uri())), + ("VIRTUAL_ENV", empty_venv.display().to_string()), + ], ); assert_eq!(code, 0, "a fresh hosted pypi checkout ejects: {env:#}"); assert!(applied(&env, PURL), "{env:#}"); diff --git a/crates/socket-patch-core/src/crawlers/python_crawler.rs b/crates/socket-patch-core/src/crawlers/python_crawler.rs index 03814ab23..2aabb1f61 100644 --- a/crates/socket-patch-core/src/crawlers/python_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/python_crawler.rs @@ -123,6 +123,59 @@ fn dist_info_dir_name_fallback(dist_info_path: &Path, is_dir: bool) -> Option<(S parse_dist_info_dir_name(&dir_name) } +/// Read `Name` and `Version` for a legacy `.egg-info` entry: the layout +/// pip < 23.1 writes when it builds an sdist without `wheel` +/// (`setup.py install`), and the one distutils and distro packages +/// (Debian's `python3-*`) ship. Two shapes: +/// +/// * a DIRECTORY holding `PKG-INFO` (the same `Name:`/`Version:` header +/// block as `METADATA`), falling back to the +/// `-[-pyX.Y].egg-info` directory name like the +/// `.dist-info` reader does; +/// * a bare FILE that IS the `PKG-INFO` (distutils). It has no directory +/// to vouch for it, so it counts only when its headers parse. +pub async fn read_egg_info_metadata(egg_info_path: &Path) -> Option<(String, String)> { + if is_dir(egg_info_path).await { + let content = read_regular_to_string(&egg_info_path.join("PKG-INFO")) + .await + .ok(); + return content + .and_then(|c| parse_metadata_text(&c)) + .or_else(|| parse_egg_info_dir_name(&egg_info_path.file_name()?.to_string_lossy())); + } + // FIFO-safe like the METADATA read: the regular-file reader rejects + // FIFOs, devices and directories. + let content = read_regular_to_string(egg_info_path).await.ok()?; + parse_metadata_text(&content) +} + +/// Blocking twin of [`read_egg_info_metadata`] for the walk-pool scan. +fn read_egg_info_metadata_sync(egg_info_path: &Path) -> Option<(String, String)> { + if is_dir_sync(egg_info_path) { + let content = read_regular_to_string_sync(&egg_info_path.join("PKG-INFO")).ok(); + return content + .and_then(|c| parse_metadata_text(&c)) + .or_else(|| parse_egg_info_dir_name(&egg_info_path.file_name()?.to_string_lossy())); + } + let content = read_regular_to_string_sync(egg_info_path).ok()?; + parse_metadata_text(&content) +} + +/// Derive `(name, version)` from a `-[-pyX.Y].egg-info` +/// name. setuptools and distutils escape `-` to `_` in both the name and +/// the version (`to_filename`), so the FIRST `-` ends the name and the +/// second one (if any) starts the `-pyX.Y` interpreter tag. +fn parse_egg_info_dir_name(dir_name: &str) -> Option<(String, String)> { + let base = dir_name.strip_suffix(".egg-info")?; + let mut parts = base.split('-'); + let name = parts.next()?; + let version = parts.next()?; + if name.is_empty() || version.is_empty() { + return None; + } + Some((name.to_string(), version.to_string())) +} + /// The `Name`/`Version` header parse of a METADATA body (see /// [`parse_metadata_headers`]). fn parse_metadata_text(content: &str) -> Option<(String, String)> { @@ -1565,10 +1618,13 @@ impl PythonCrawler { } } -/// Scan a `site-packages` directory for `.dist-info` entries, returning -/// `(canonicalized name, version)` for each package that yields metadata, -/// in listing order. One blocking-pool task for the listing and every -/// METADATA read, rather than a runtime hop per open, read and stat. +/// Scan a `site-packages` directory for installed distributions — the +/// `.dist-info` entries wheels install, and the legacy `.egg-info` +/// entries an sdist built without `wheel`, distutils or a distro package +/// leaves — returning `(canonicalized name, version)` for each one that +/// yields metadata, in listing order. One blocking-pool task for the +/// listing and every metadata read, rather than a runtime hop per open, +/// read and stat. pub(crate) async fn list_dist_info_packages(site_packages_path: &Path) -> Vec<(String, String)> { let site_packages_path = site_packages_path.to_path_buf(); run_blocking(move || list_dist_info_packages_sync(&site_packages_path)).await @@ -1576,21 +1632,20 @@ pub(crate) async fn list_dist_info_packages(site_packages_path: &Path) -> Vec<(S /// Blocking body of [`list_dist_info_packages`]. fn list_dist_info_packages_sync(site_packages_path: &Path) -> Vec<(String, String)> { - let dist_infos: Vec = list_dir_sync(site_packages_path) + list_dir_sync(site_packages_path) .into_iter() .filter_map(|entry| { let name_str = entry.name.to_string_lossy(); - name_str - .ends_with(".dist-info") - .then(|| site_packages_path.join(&*name_str)) - }) - .collect(); - dist_infos - .iter() - .filter_map(|dist_info_path| { - read_python_metadata_sync(dist_info_path) - .map(|(raw_name, version)| (canonicalize_pypi_name(&raw_name), version)) + let path = site_packages_path.join(&*name_str); + if name_str.ends_with(".dist-info") { + read_python_metadata_sync(&path) + } else if name_str.ends_with(".egg-info") { + read_egg_info_metadata_sync(&path) + } else { + None + } }) + .map(|(raw_name, version)| (canonicalize_pypi_name(&raw_name), version)) .collect() } @@ -2248,6 +2303,62 @@ mod tests { assert!(read_python_metadata(&dist_info).await.is_none()); } + #[test] + fn test_parse_egg_info_dir_name() { + assert_eq!( + parse_egg_info_dir_name("six-1.16.0-py3.11.egg-info"), + Some(("six".into(), "1.16.0".into())) + ); + assert_eq!( + parse_egg_info_dir_name("Flask_SQLAlchemy-3.0.5.egg-info"), + Some(("Flask_SQLAlchemy".into(), "3.0.5".into())) + ); + assert!(parse_egg_info_dir_name("noversion.egg-info").is_none()); + assert!(parse_egg_info_dir_name("-1.0.egg-info").is_none()); + assert!(parse_egg_info_dir_name("six-1.16.0.dist-info").is_none()); + } + + /// Legacy `.egg-info` installs (#447): a `PKG-INFO` directory, a + /// headerless directory (named fallback), and a bare distutils FILE are + /// installs; a bare file without headers is not. + #[tokio::test] + async fn egg_info_entries_are_listed() { + let dir = tempfile::tempdir().unwrap(); + let sp = dir.path(); + let egg = sp.join("six-1.16.0-py3.11.egg-info"); + tokio::fs::create_dir_all(&egg).await.unwrap(); + tokio::fs::write(egg.join("PKG-INFO"), "Name: six\nVersion: 1.16.0\n") + .await + .unwrap(); + tokio::fs::create_dir_all(sp.join("zope.interface-5.4.0-py3.11.egg-info")) + .await + .unwrap(); + tokio::fs::write( + sp.join("PyGObject-3.48.2.egg-info"), + "Metadata-Version: 1.1\nName: PyGObject\nVersion: 3.48.2\n", + ) + .await + .unwrap(); + tokio::fs::write(sp.join("ghost-1.0.egg-info"), "not metadata") + .await + .unwrap(); + let mut listed = list_dist_info_packages(sp).await; + listed.sort(); + assert_eq!( + listed, + vec![ + ("pygobject".to_string(), "3.48.2".to_string()), + ("six".to_string(), "1.16.0".to_string()), + ("zope-interface".to_string(), "5.4.0".to_string()), + ] + ); + let found = PythonCrawler::new() + .find_by_purls(sp, &["pkg:pypi/six@1.16.0".to_string()]) + .await + .unwrap(); + assert_eq!(found["pkg:pypi/six@1.16.0"].path, sp); + } + #[test] fn test_parse_dist_info_dir_name() { // Modern pip escapes `-` in the name to `_`. diff --git a/crates/socket-patch-core/src/crawlers/python_crawler/oracle.rs b/crates/socket-patch-core/src/crawlers/python_crawler/oracle.rs index 4aab29ea4..3bcdf4ac1 100644 --- a/crates/socket-patch-core/src/crawlers/python_crawler/oracle.rs +++ b/crates/socket-patch-core/src/crawlers/python_crawler/oracle.rs @@ -6,7 +6,7 @@ use std::collections::{HashMap, HashSet}; use std::path::Path; -use super::{canonicalize_pypi_name, read_python_metadata, PythonCrawler}; +use super::{canonicalize_pypi_name, read_egg_info_metadata, read_python_metadata, PythonCrawler}; use crate::crawlers::types::{CrawledPackage, CrawlerOptions}; pub(super) struct LegacyPythonCrawler; @@ -78,17 +78,22 @@ impl LegacyPythonCrawler { } } -/// The old per-entry async `list_dist_info_packages`. +/// The old per-entry async `list_dist_info_packages` (with the same +/// `.egg-info` support as the parallel scan). pub(super) async fn list_dist_info_packages(site_packages_path: &Path) -> Vec<(String, String)> { let mut out = Vec::new(); for entry in crate::utils::fs::list_dir_entries(site_packages_path).await { let name = entry.file_name(); let name_str = name.to_string_lossy(); - if !name_str.ends_with(".dist-info") { - continue; - } - let dist_info_path = site_packages_path.join(&*name_str); - if let Some((raw_name, version)) = read_python_metadata(&dist_info_path).await { + let entry_path = site_packages_path.join(&*name_str); + let found = if name_str.ends_with(".dist-info") { + read_python_metadata(&entry_path).await + } else if name_str.ends_with(".egg-info") { + read_egg_info_metadata(&entry_path).await + } else { + None + }; + if let Some((raw_name, version)) = found { out.push((canonicalize_pypi_name(&raw_name), version)); } }