diff --git a/crates/socket-patch-cli/CLI_CONTRACT.md b/crates/socket-patch-cli/CLI_CONTRACT.md index 40700537e..f12073687 100644 --- a/crates/socket-patch-cli/CLI_CONTRACT.md +++ b/crates/socket-patch-cli/CLI_CONTRACT.md @@ -477,7 +477,7 @@ the model is **not uniform** today: `workspaces`). One repo-root invocation discovers every member. A member that is itself a workspace root is recursed into (bounded depth). - **cwd-only (single project):** gem, pypi, composer. The crawler inspects only the project - rooted at `--cwd` (pypi looks at `$VIRTUAL_ENV`, `/.venv` / `venv`, then a Poetry project's out-of-tree virtualenv(s) under Poetry's `virtualenvs.path`; composer at the vendor tree); it does **not** + rooted at `--cwd` (pypi first takes the env the project's manager records: PDM's `.pdm-python` interpreter, meaning its venv or, for a base interpreter, PEP 582 `__pypackages__//lib`, and uv's `UV_PROJECT_ENVIRONMENT`. Otherwise it looks at `$VIRTUAL_ENV`, `/.venv` / `venv`, then a Poetry project's out-of-tree virtualenv(s) under Poetry's `virtualenvs.path`; composer at the vendor tree); it does **not** descend into sibling subprojects. A monorepo with several independent lockfiles in subdirectories (`backend/Gemfile.lock` + `frontend/Gemfile.lock`, multiple `.venv`, multiple `go.mod` / `composer.json`) is handled by invoking the tool **once per subproject** (`--cwd` each), as a diff --git a/crates/socket-patch-cli/tests/in_process_python_envs.rs b/crates/socket-patch-cli/tests/in_process_python_envs.rs index dc9624f52..f9deaa42d 100644 --- a/crates/socket-patch-cli/tests/in_process_python_envs.rs +++ b/crates/socket-patch-cli/tests/in_process_python_envs.rs @@ -109,6 +109,7 @@ fn assert_not_discovered(bodies: &[String], needle: &str) { /// `scan_run` directly. async fn scan_scrubbed(args: ScanArgs) -> i32 { std::env::remove_var("VIRTUAL_ENV"); + std::env::remove_var("UV_PROJECT_ENVIRONMENT"); scan_run(args).await } @@ -637,3 +638,115 @@ async fn pipenv_stray_venv_dirs_do_not_shadow_the_pipenv_venv() { assert_not_discovered(&bodies, "pkg:pypi/stray-decoy@6.6.6"); } } + +// --------------------------------------------------------------------------- +// Package-manager-recorded envs: PDM's saved interpreter / PEP 582, and uv's +// UV_PROJECT_ENVIRONMENT, ahead of a stray `./.venv` the manager never uses +// --------------------------------------------------------------------------- + +/// A project at `/app` with `pyproject` and an in-project `.venv` +/// holding `stray_decoy 6.6.6` that the project's manager does not use. +fn project_with_stray_venv(pyproject: &str) -> (tempfile::TempDir, std::path::PathBuf) { + let tmp = tempfile::tempdir().unwrap(); + let project = tmp.path().join("app"); + std::fs::create_dir_all(&project).unwrap(); + std::fs::write(project.join("pyproject.toml"), pyproject).unwrap(); + let stray = venv_site_packages(&project.join(".venv"), "python3.12"); + std::fs::create_dir_all(&stray).unwrap(); + write_dist_info(&stray, "stray_decoy", "6.6.6"); + (tmp, project) +} + +/// A venv at `root` holding `pkg 1.0.0`; returns its interpreter path. +fn venv_with(root: &Path, pkg: &str) -> std::path::PathBuf { + let site = venv_site_packages(root, "python3.12"); + std::fs::create_dir_all(&site).unwrap(); + write_dist_info(&site, pkg, "1.0.0"); + std::fs::write(root.join("pyvenv.cfg"), "home = /usr/bin\n").unwrap(); + if cfg!(windows) { + root.join("Scripts").join("python.exe") + } else { + root.join("bin").join("python") + } +} + +async fn assert_scan_finds(project: &Path, wanted: &str) { + let server = MockServer::start().await; + mock_batch_empty(&server).await; + assert_eq!(scan_scrubbed(default_args(project, server.uri())).await, 0); + let bodies = batch_bodies(&server).await; + assert_discovered(&bodies, wanted); + assert_not_discovered(&bodies, "pkg:pypi/stray-decoy@6.6.6"); +} + +/// #502: PDM's `.pdm-python` names an out-of-tree venv +/// (`venv.in_project = false`, or `pdm use `); that is the env scanned. +#[tokio::test] +#[serial] +async fn pdm_saved_interpreter_venv_is_scanned_not_a_stray_dot_venv() { + let (tmp, project) = + project_with_stray_venv("[project]\nname = \"app\"\n[tool.pdm]\ndistribution = false\n"); + std::fs::write(project.join("pdm.lock"), "[metadata]\n").unwrap(); + let python = venv_with( + &tmp.path().join("pdm").join("venvs").join("app-AbCd-3.12"), + "pdm_pkg", + ); + std::fs::write(project.join(".pdm-python"), python.display().to_string()).unwrap(); + assert_scan_finds(&project, "pkg:pypi/pdm-pkg@1.0.0").await; +} + +/// #528: a PEP 582 PDM project installs into `__pypackages__//lib`. +#[tokio::test] +#[serial] +async fn pdm_pep582_pypackages_is_scanned_not_a_stray_dot_venv() { + let (tmp, project) = project_with_stray_venv("[project]\nname = \"app\"\n"); + std::fs::write(project.join("pdm.lock"), "[metadata]\n").unwrap(); + let lib = project.join("__pypackages__").join("3.11").join("lib"); + std::fs::create_dir_all(&lib).unwrap(); + write_dist_info(&lib, "pep582_pkg", "1.0.0"); + // The saved interpreter is a base Python, not a venv, and the project + // turned PDM's venvs off (`pdm config -l python.use_venv false`). + let base = tmp.path().join("usr").join("bin").join("python3.11"); + std::fs::write(project.join(".pdm-python"), base.display().to_string()).unwrap(); + std::fs::write(project.join("pdm.toml"), "[python]\nuse_venv = false\n").unwrap(); + assert_scan_finds(&project, "pkg:pypi/pep582-pkg@1.0.0").await; +} + +/// #525: uv syncs into `UV_PROJECT_ENVIRONMENT`, absolute or relative to the +/// project, and ignores an activated `VIRTUAL_ENV` for project commands. +#[tokio::test] +#[serial] +async fn uv_project_environment_is_scanned_not_a_stray_dot_venv() { + let other = tempfile::tempdir().unwrap(); + let decoy = other.path().join("tool-venv"); + let decoy_site = venv_site_packages(&decoy, "python3.12"); + std::fs::create_dir_all(&decoy_site).unwrap(); + write_dist_info(&decoy_site, "activated_decoy", "6.6.6"); + + for relative in [false, true] { + let (tmp, project) = project_with_stray_venv("[project]\nname = \"app\"\n"); + std::fs::write(project.join("uv.lock"), "version = 1\n").unwrap(); + let env = if relative { + project.join(".venv-ci") + } else { + tmp.path().join("opt").join("venv") + }; + venv_with(&env, "uv_pkg"); + let server = MockServer::start().await; + mock_batch_empty(&server).await; + std::env::set_var("VIRTUAL_ENV", &decoy); + if relative { + std::env::set_var("UV_PROJECT_ENVIRONMENT", ".venv-ci"); + } else { + std::env::set_var("UV_PROJECT_ENVIRONMENT", &env); + } + let code = scan_run(default_args(&project, server.uri())).await; + std::env::remove_var("VIRTUAL_ENV"); + std::env::remove_var("UV_PROJECT_ENVIRONMENT"); + assert_eq!(code, 0); + let bodies = batch_bodies(&server).await; + assert_discovered(&bodies, "pkg:pypi/uv-pkg@1.0.0"); + assert_not_discovered(&bodies, "pkg:pypi/stray-decoy@6.6.6"); + assert_not_discovered(&bodies, "pkg:pypi/activated-decoy@6.6.6"); + } +} diff --git a/crates/socket-patch-cli/tests/in_process_redirect_pdm.rs b/crates/socket-patch-cli/tests/in_process_redirect_pdm.rs index 97f4a171d..c7adfe131 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect_pdm.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect_pdm.rs @@ -594,3 +594,114 @@ async fn assert_relock_roundtrip(lock: &str, relocked: &str) { "rollback restores the relocked lock byte for byte" ); } + +/// Spawn `scan --mode hosted --json` on `root` with a scrubbed environment +/// (no ambient `SOCKET_*`, `PDM_*` or `VIRTUAL_ENV`). +async fn scan_json( + root: &Path, + server: &MockServer, + extra: &[&str], +) -> (Option, serde_json::Value) { + let mut cmd = tokio::process::Command::new(binary()); + for (key, _) in std::env::vars_os() { + let name = key.to_string_lossy(); + if name.starts_with("SOCKET_") + || name.starts_with("PDM_") + || name == "VIRTUAL_ENV" + || name == "UV_PROJECT_ENVIRONMENT" + { + cmd.env_remove(key); + } + } + cmd.env("SOCKET_TELEMETRY_DISABLED", "1") + .args(["scan", "--mode", "hosted", "--yes", "--json", "--cwd"]) + .arg(root) + .args([ + "--api-url", + &server.uri(), + "--org", + ORG, + "--api-token", + "fake", + ]) + .args(extra); + let out = cmd.output().await.unwrap(); + let json = serde_json::from_slice(&out.stdout).unwrap_or_else(|error| { + panic!( + "{error}: stdout={} stderr={}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ) + }); + (out.status.code(), json) +} + +fn stale_warning(json: &serde_json::Value) -> bool { + json["redirect"]["warnings"] + .as_array() + .is_some_and(|warnings| { + warnings + .iter() + .any(|warning| warning["code"] == "redirect_pypi_stale_install") + }) +} + +/// Lay `urllib3 1.26.18` with `bytes` as its `response.py` into `site`. +fn install_urllib3(site: &Path, bytes: &[u8]) { + std::fs::create_dir_all(site.join("urllib3-1.26.18.dist-info")).unwrap(); + std::fs::create_dir_all(site.join("urllib3")).unwrap(); + std::fs::write(site.join("urllib3").join("response.py"), bytes).unwrap(); +} + +/// #502 / #528: the env PDM installs into is the one its `.pdm-python` +/// records: an out-of-tree venv, or `__pypackages__//lib` when the +/// interpreter is a base Python (PEP 582). A warm, still-upstream copy there +/// must raise the stale-install warning and block the VEX attestation, as an +/// in-project `.venv` does; the empty stray `.venv` from `write_project` +/// must not stand in for it. Once PDM's env holds the patched bytes, the +/// warning is gone and the redirect attests. +#[tokio::test] +async fn pdm_recorded_env_is_probed_for_stale_hosted_installs() { + for pep582 in [false, true] { + let server = MockServer::start().await; + mock_api(&server).await; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("app"); + std::fs::create_dir_all(&root).unwrap(); + write_project(&root, LOCK); + let site = if pep582 { + let base = tmp.path().join("usr").join("bin").join("python3.11"); + std::fs::write(root.join(".pdm-python"), base.display().to_string()).unwrap(); + std::fs::write(root.join("pdm.toml"), "[python]\nuse_venv = false\n").unwrap(); + root.join("__pypackages__").join("3.11").join("lib") + } else { + let venv = tmp.path().join("pdm-venvs").join("app-AbCd-3.12"); + std::fs::create_dir_all(&venv).unwrap(); + std::fs::write(venv.join("pyvenv.cfg"), "home = /usr/bin\n").unwrap(); + let python = if cfg!(windows) { + venv.join("Scripts").join("python.exe") + } else { + venv.join("bin").join("python") + }; + std::fs::write(root.join(".pdm-python"), python.display().to_string()).unwrap(); + if cfg!(windows) { + venv.join("Lib").join("site-packages") + } else { + venv.join("lib").join("python3.12").join("site-packages") + } + }; + install_urllib3(&site, UPSTREAM); + + let vex = tmp.path().join("out.vex.json"); + let (code, json) = scan_json(&root, &server, &["--vex", vex.to_str().unwrap()]).await; + assert_eq!(code, Some(1), "pep582={pep582}: {json}"); + assert!(stale_warning(&json), "pep582={pep582}: {json}"); + assert!(!vex.exists(), "stale bytes cannot produce a VEX file"); + + install_urllib3(&site, PATCHED); + let (code, json) = scan_json(&root, &server, &["--vex", vex.to_str().unwrap()]).await; + assert_eq!(code, Some(0), "pep582={pep582}: {json}"); + assert!(!stale_warning(&json), "pep582={pep582}: {json}"); + assert_eq!(json["vex"]["statements"], 1, "pep582={pep582}: {json}"); + } +} diff --git a/crates/socket-patch-core/src/crawlers/python_crawler.rs b/crates/socket-patch-core/src/crawlers/python_crawler.rs index 1dfcee0db..dfdee4f52 100644 --- a/crates/socket-patch-core/src/crawlers/python_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/python_crawler.rs @@ -318,9 +318,14 @@ async fn find_site_packages_under( /// Find local virtual environment `site-packages` directories. /// /// Checks (in order): -/// 1. `VIRTUAL_ENV` environment variable (for a Pipenv or Poetry project, -/// only when that tool itself would use it; Poetry also takes a conda -/// `CONDA_PREFIX`) +/// 0. The env the project's package manager records for it, which that +/// manager uses ahead of an activated venv or a stray `./.venv` (see +/// [`package_manager_recorded_site_packages`]): PDM's `.pdm-python` +/// interpreter (its venv, or `__pypackages__` for PEP 582) and uv's +/// `UV_PROJECT_ENVIRONMENT` +/// 1. `VIRTUAL_ENV` environment variable (for a Pipenv, Poetry or PDM +/// project, only when that tool itself would use it; Poetry also takes a +/// conda `CONDA_PREFIX`) /// 2. For a Pipenv project, the venv(s) Pipenv resolves for it (see /// [`pipenv_project_site_packages`]), and nothing else /// 3. Poetry's out-of-tree virtualenv(s), when Poetry itself would not use @@ -339,6 +344,13 @@ async fn find_local_venv_site_packages_with( var: &impl Fn(&str) -> Option, ) -> Vec { let mut results = Vec::new(); + + // 0. PDM and uv record where they install a project, and that record + // beats both an activated `VIRTUAL_ENV` and a `./.venv` they don't use. + if let Some(found) = package_manager_recorded_site_packages(cwd, var).await { + return found; + } + let pipenv = is_pipenv_project(cwd); let poetry = if pipenv { None @@ -351,9 +363,13 @@ async fn find_local_venv_site_packages_with( // `PIPENV_IGNORE_VIRTUALENVS`, so for a Pipenv project the activated venv // then belongs to something else and must not be patched. Poetry ignores // it once `poetry env use` recorded an env for the project (see - // [`poetry_active_prefix`]). + // [`poetry_active_prefix`]). PDM likewise skips an activated venv under + // `PDM_IGNORE_ACTIVE_VENV`. + let pdm_ignores_active = pdm_env_flag(var, "PDM_IGNORE_ACTIVE_VENV") + && pdm_drives_project(cwd).await; let active_prefix = match &poetry { Some(project) => poetry_active_prefix(project, var), + None if pdm_ignores_active => None, None if !pipenv || pipenv_uses_virtual_env(var) => var("VIRTUAL_ENV"), None => None, }; @@ -396,9 +412,216 @@ async fn find_local_venv_site_packages_with( results.extend(matches); } + // 5. A PDM project with no recorded interpreter and no venv for PDM to + // pick (an activated one, `./.venv`) is a PEP 582 project (PDM 1.x's + // default): its packages live in `__pypackages__//lib`. + if results.is_empty() && pdm_drives_project(cwd).await { + results = pdm_pep582_dirs(cwd).await; + } + results } +/// The `site-packages` of the env the project's package manager records for +/// `cwd`, when that env exists. `None` means the manager records nothing +/// (or nothing installed yet), and the generic probes decide. +/// +/// - **PDM** installs into the interpreter saved in `.pdm-python` (PDM +/// 2.x; `[python] path` in `.pdm.toml` before that), ahead of an +/// activated venv. That interpreter's venv is the env (an out-of-tree +/// `venv.in_project = false` venv, or one picked with `pdm use`). An +/// interpreter that is not a venv means PEP 582: PDM installs into +/// `__pypackages__//lib`, which PDM 1.x also uses with no saved +/// interpreter at all. +/// - **uv** syncs a project into `UV_PROJECT_ENVIRONMENT` (absolute, or +/// relative to the project) instead of `./.venv`, and ignores an +/// activated `VIRTUAL_ENV` for project commands. +async fn package_manager_recorded_site_packages( + cwd: &Path, + var: &impl Fn(&str) -> Option, +) -> Option> { + if let Some(found) = pdm_project_site_packages(cwd, var).await { + return Some(found); + } + uv_project_environment_site_packages(cwd, var).await +} + +/// The env of PDM's interpreter for `cwd` (see +/// [`package_manager_recorded_site_packages`]): `PDM_PYTHON`, else the saved +/// one unless `PDM_IGNORE_SAVED_PYTHON`. With neither, PDM picks an active +/// or project venv first, so the generic probes decide (PEP 582 is their +/// last resort, see [`find_local_venv_site_packages_with`]). +async fn pdm_project_site_packages( + cwd: &Path, + var: &impl Fn(&str) -> Option, +) -> Option> { + if !pdm_drives_project(cwd).await { + return None; + } + // `PDM_PYTHON` outranks the saved interpreter. The first of them that + // is an environment (a venv, or a conda env PDM reuses) is where PDM + // installs; a base interpreter (CI often points `PDM_PYTHON` at the + // system Python) only picks the Python a venv is made from. + let overridden = var("PDM_PYTHON") + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) + .map(|python| cwd.join(python)); + let saved = if pdm_env_flag(var, "PDM_IGNORE_SAVED_PYTHON") { + None + } else { + pdm_saved_interpreter(cwd).await + }; + let interpreters: Vec = overridden.into_iter().chain(saved).collect(); + if let Some(root) = interpreters.iter().find_map(|i| env_root_of_interpreter(i)) { + let found = find_site_packages_under(&root, "site-packages").await; + return (!found.is_empty()).then_some(found); + } + // A base interpreter means PEP 582 only with `python.use_venv` off; + // with it on (PDM 2.x's default) PDM uses a venv the generic probes + // find, with `__pypackages__` as their last resort. + if interpreters.is_empty() || pdm_uses_venv(cwd, var).await { + return None; + } + let found = pdm_pep582_dirs(cwd).await; + (!found.is_empty()).then_some(found) +} + +/// PDM's `python.use_venv` for `cwd`: `PDM_USE_VENV`, else `[python] +/// use_venv` in the project's `pdm.toml` (PDM 2.x) or legacy `.pdm.toml`. +/// Unset, it is on, except for a legacy PDM 1.x project (a `.pdm.toml` and +/// no `.pdm-python`), where it defaulted to off. +async fn pdm_uses_venv(cwd: &Path, var: &impl Fn(&str) -> Option) -> bool { + if var("PDM_USE_VENV").is_some() { + return pdm_env_flag(var, "PDM_USE_VENV"); + } + for config in ["pdm.toml", ".pdm.toml"] { + let Ok(text) = read_regular_to_string(&cwd.join(config)).await else { + continue; + }; + let setting = text + .parse::() + .ok() + .and_then(|doc| doc.get("python")?.get("use_venv")?.as_bool()); + if let Some(on) = setting { + return on; + } + } + !cwd.join(".pdm.toml").is_file() || cwd.join(".pdm-python").is_file() +} + +/// A boolean PDM environment setting, parsed like PDM's `ensure_boolean`: +/// set and non-empty, and not `false` / `no` / `0` (any case). +fn pdm_env_flag(var: &impl Fn(&str) -> Option, name: &str) -> bool { + var(name).is_some_and(|v| { + !v.is_empty() && !matches!(v.to_ascii_lowercase().as_str(), "false" | "no" | "0") + }) +} + +/// Whether PDM installs the project at `cwd`: a PDM project (see +/// [`is_pdm_project`], or a `.pdm-python`) with no `uv.lock` or +/// `poetry.lock`, which drive installs ahead of `pdm.lock` (the hosted +/// rewriters' precedence). +async fn pdm_drives_project(cwd: &Path) -> bool { + if cwd.join("uv.lock").is_file() || cwd.join("poetry.lock").is_file() { + return false; + } + cwd.join(".pdm-python").is_file() || is_pdm_project(cwd).await +} + +/// PEP 582 package dirs: `__pypackages__//lib`. +async fn pdm_pep582_dirs(cwd: &Path) -> Vec { + find_python_dirs(&cwd.join("__pypackages__"), &["*", "lib"]).await +} + +/// The interpreter PDM saved for `cwd`: `.pdm-python` (PDM 2.x), else +/// `[python] path` in the legacy `.pdm.toml`. A relative path is taken +/// against the project. +async fn pdm_saved_interpreter(cwd: &Path) -> Option { + let saved = match read_regular_to_string(&cwd.join(".pdm-python")).await { + Ok(text) => text.trim().to_string(), + Err(_) => { + let text = read_regular_to_string(&cwd.join(".pdm.toml")) + .await + .ok()?; + let doc = text.parse::().ok()?; + doc.get("python")?.get("path")?.as_str()?.trim().to_string() + } + }; + (!saved.is_empty()).then(|| cwd.join(saved)) +} + +/// Whether `cwd` is a PDM project: `pdm.lock`, `.pdm.toml`, or a +/// `[tool.pdm]` table in `pyproject.toml` with settings beyond `build` (a +/// `[tool.pdm.build]` table alone only configures the pdm-backend build +/// backend, which projects driven by other managers use too). +async fn is_pdm_project(cwd: &Path) -> bool { + if cwd.join("pdm.lock").is_file() || cwd.join(".pdm.toml").is_file() { + return true; + } + let Ok(text) = read_regular_to_string(&cwd.join("pyproject.toml")).await else { + return false; + }; + text.parse::() + .ok() + .and_then(|doc| { + let pdm = doc.get("tool")?.get("pdm")?.as_table_like()?; + pdm.iter().any(|(key, _)| key != "build").then_some(()) + }) + .is_some() +} + +/// The environment a Python interpreter path belongs to: a venv +/// (`/bin/python…` or `\Scripts\python.exe` with a +/// `/pyvenv.cfg`), or a conda env (`conda-meta/` at ``, whose +/// interpreter is `/bin/python…` or `\python.exe`). The path is +/// not resolved, since a venv's interpreter is a symlink to its base Python. +fn env_root_of_interpreter(python: &Path) -> Option { + let parent = python.parent()?; + let grandparent = parent.parent(); + if let Some(root) = grandparent.filter(|root| root.join("pyvenv.cfg").is_file()) { + return Some(root.to_path_buf()); + } + grandparent + .into_iter() + .chain(std::iter::once(parent)) + .find(|root| root.join("conda-meta").is_dir()) + .map(Path::to_path_buf) +} + +/// uv's `UV_PROJECT_ENVIRONMENT` for a uv project at `cwd` (see +/// [`package_manager_recorded_site_packages`]). Ambient in shells and +/// images, so it only counts for a project uv drives: one with `uv.lock`, +/// or a `pyproject.toml` that no other manager's lock or record claims. +async fn uv_project_environment_site_packages( + cwd: &Path, + var: &impl Fn(&str) -> Option, +) -> Option> { + let env = var("UV_PROJECT_ENVIRONMENT").filter(|v| !v.trim().is_empty())?; + let other_lock = [ + "poetry.lock", + "poetry.toml", + "pdm.lock", + ".pdm-python", + "Pipfile", + "Pipfile.lock", + ] + .iter() + .any(|marker| cwd.join(marker).exists()); + // A lockless Poetry (`[tool.poetry]`) or PDM project is still theirs. + let other_manager = other_lock + || is_pdm_project(cwd).await + || read_regular_to_string(&cwd.join("pyproject.toml")) + .await + .is_ok_and(|text| text.contains("[tool.poetry")); + let uv_project = + cwd.join("uv.lock").is_file() || (cwd.join("pyproject.toml").is_file() && !other_manager); + if !uv_project { + return None; + } + let found = find_site_packages_under(&cwd.join(env), "site-packages").await; + (!found.is_empty()).then_some(found) +} + /// Whether `cwd` is a Pipenv project: a `Pipfile` or a `Pipfile.lock`. fn is_pipenv_project(cwd: &Path) -> bool { cwd.join("Pipfile").is_file() || cwd.join("Pipfile.lock").is_file() @@ -2293,6 +2516,452 @@ mod tests { site } + /// A venv at `root` as the crawler sees it: `pyvenv.cfg`, an interpreter + /// path under `bin/` (`Scripts\\` on Windows), and its site-packages. + /// Returns `(interpreter, site_packages)`. + fn fake_venv_root(root: &Path) -> (PathBuf, PathBuf) { + let parent = root.parent().unwrap(); + let leaf = root.file_name().unwrap().to_str().unwrap(); + let site = fake_venv(parent, leaf); + std::fs::write(root.join("pyvenv.cfg"), "home = /usr/bin\n").unwrap(); + let python = if cfg!(windows) { + root.join("Scripts").join("python.exe") + } else { + root.join("bin").join("python") + }; + (python, site) + } + + fn env_of(pairs: &[(&str, String)]) -> impl Fn(&str) -> Option { + let pairs: Vec<(String, String)> = pairs + .iter() + .map(|(k, v)| (k.to_string(), v.clone())) + .collect(); + move |name: &str| { + pairs + .iter() + .find(|(k, _)| k == name) + .map(|(_, v)| v.clone()) + } + } + + /// #502: PDM installs into the interpreter saved in `.pdm-python` (an + /// out-of-tree `venv.in_project = false` venv, or one bound with + /// `pdm use`), ahead of a stray `./.venv` and an activated venv. + #[tokio::test] + async fn pdm_saved_interpreter_venv_is_the_project_env() { + let tmp = tempfile::tempdir().unwrap(); + let project = tmp.path().join("app"); + std::fs::create_dir_all(&project).unwrap(); + std::fs::write( + project.join("pyproject.toml"), + "[project]\nname = \"app\"\n[tool.pdm]\ndistribution = false\n", + ) + .unwrap(); + std::fs::write(project.join("pdm.lock"), "[metadata]\n").unwrap(); + let (python, pdm_site) = + fake_venv_root(&tmp.path().join("pdm-venvs").join("app-AbCd-3.12")); + std::fs::write( + project.join(".pdm-python"), + format!("{}\n", python.display()), + ) + .unwrap(); + let no_env = env_of(&[]); + + // Out-of-tree venv, nothing else around: found (was: skipped). + assert_eq!( + find_local_venv_site_packages_with(&project, &no_env).await, + vec![pdm_site.clone()] + ); + + // A stray `./.venv` PDM does not use is not patched. + let stray = fake_venv(&project, ".venv"); + assert_eq!( + find_local_venv_site_packages_with(&project, &no_env).await, + vec![pdm_site.clone()] + ); + + // PDM prefers its saved interpreter over an activated venv. + let other = tempfile::tempdir().unwrap(); + fake_venv(other.path(), "tool-venv"); + let activated = env_of(&[( + "VIRTUAL_ENV", + other + .path() + .join("tool-venv") + .to_string_lossy() + .into_owned(), + )]); + assert_eq!( + find_local_venv_site_packages_with(&project, &activated).await, + vec![pdm_site.clone()] + ); + + // `PDM_IGNORE_SAVED_PYTHON` makes PDM disregard `.pdm-python`. + let ignored = env_of(&[("PDM_IGNORE_SAVED_PYTHON", "1".to_string())]); + assert_eq!( + find_local_venv_site_packages_with(&project, &ignored).await, + vec![stray.clone()] + ); + // ...a boolean PDM parses: false values keep `.pdm-python`. + for falsy in ["0", "false", "NO"] { + let kept = env_of(&[("PDM_IGNORE_SAVED_PYTHON", falsy.to_string())]); + assert_eq!( + find_local_venv_site_packages_with(&project, &kept).await, + vec![pdm_site.clone()], + "PDM_IGNORE_SAVED_PYTHON={falsy:?}" + ); + } + + // A base `PDM_PYTHON` (CI's system Python) does not displace the + // saved venv PDM installs into. + let system = tmp.path().join("usr").join("bin").join("python3"); + let base_override = env_of(&[("PDM_PYTHON", system.to_string_lossy().into_owned())]); + assert_eq!( + find_local_venv_site_packages_with(&project, &base_override).await, + vec![pdm_site.clone()] + ); + + // A conda env PDM reuses (`conda-meta/`, no pyvenv.cfg) is an env. + let conda = tmp.path().join("conda").join("envs").join("app"); + let conda_site = fake_venv(conda.parent().unwrap(), "app"); + std::fs::create_dir_all(conda.join("conda-meta")).unwrap(); + let conda_python = if cfg!(windows) { + conda.join("python.exe") + } else { + conda.join("bin").join("python") + }; + let conda_env = env_of(&[("PDM_PYTHON", conda_python.to_string_lossy().into_owned())]); + assert_eq!( + find_local_venv_site_packages_with(&project, &conda_env).await, + vec![conda_site] + ); + + // `PDM_PYTHON` outranks `.pdm-python`. + let (ci_python, ci_site) = fake_venv_root(&tmp.path().join("ci-venv")); + let pinned = env_of(&[("PDM_PYTHON", ci_python.to_string_lossy().into_owned())]); + assert_eq!( + find_local_venv_site_packages_with(&project, &pinned).await, + vec![ci_site] + ); + + // Next to `uv.lock` or `poetry.lock` (which drive installs ahead of + // `pdm.lock`) a leftover PDM record is not the project's env. + for lock in ["uv.lock", "poetry.lock"] { + std::fs::write(project.join(lock), "").unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &no_env).await, + vec![stray.clone()], + "{lock}" + ); + std::fs::remove_file(project.join(lock)).unwrap(); + } + + // Legacy PDM (`.pdm.toml` `[python] path`) records the same thing. + std::fs::remove_file(project.join(".pdm-python")).unwrap(); + let mut doc = toml_edit::DocumentMut::new(); + doc["python"]["path"] = toml_edit::value(python.to_string_lossy().into_owned()); + std::fs::write(project.join(".pdm.toml"), doc.to_string()).unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &no_env).await, + vec![pdm_site.clone()] + ); + std::fs::remove_file(project.join(".pdm.toml")).unwrap(); + + // Saved interpreter whose venv is gone: the generic probes decide. + std::fs::write( + project.join(".pdm-python"), + tmp.path() + .join("gone") + .join("bin") + .join("python") + .display() + .to_string(), + ) + .unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &no_env).await, + vec![stray] + ); + } + + /// #528: a PDM interpreter that is not a venv means PEP 582, and PDM + /// installs into `__pypackages__//lib` (PDM 1.x does so with no + /// saved interpreter at all). The PATH Python is never the env. + #[tokio::test] + async fn pdm_pep582_pypackages_is_the_project_env() { + let tmp = tempfile::tempdir().unwrap(); + let project = tmp.path().join("demo"); + let lib = project.join("__pypackages__").join("3.11").join("lib"); + std::fs::create_dir_all(&lib).unwrap(); + std::fs::write( + project.join("pyproject.toml"), + "[project]\nname = \"demo\"\n", + ) + .unwrap(); + std::fs::write(project.join("pdm.lock"), "[metadata]\n").unwrap(); + // A base interpreter (no pyvenv.cfg next to it) with + // `pdm config -l python.use_venv false`, as in #528. + let base = tmp.path().join("usr").join("bin").join("python3.11"); + std::fs::create_dir_all(base.parent().unwrap()).unwrap(); + std::fs::write(project.join(".pdm-python"), base.display().to_string()).unwrap(); + std::fs::write(project.join("pdm.toml"), "[python]\nuse_venv = false\n").unwrap(); + let no_env = env_of(&[]); + assert_eq!( + find_local_venv_site_packages_with(&project, &no_env).await, + vec![lib.clone()] + ); + // ...even beside a `./.venv` PDM does not use. + let unused = fake_venv(&project, ".venv"); + assert_eq!( + find_local_venv_site_packages_with(&project, &no_env).await, + vec![lib.clone()] + ); + // With `use_venv` on (PDM 2.x's default, or PDM_USE_VENV), a base + // interpreter only seeds the venv: `./.venv` is the env. + std::fs::remove_file(project.join("pdm.toml")).unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &no_env).await, + vec![unused.clone()] + ); + let env_off = env_of(&[("PDM_USE_VENV", "0".to_string())]); + assert_eq!( + find_local_venv_site_packages_with(&project, &env_off).await, + vec![lib.clone()] + ); + std::fs::write(project.join("pdm.toml"), "[python]\nuse_venv = false\n").unwrap(); + let env_on = env_of(&[("PDM_USE_VENV", "1".to_string())]); + assert_eq!( + find_local_venv_site_packages_with(&project, &env_on).await, + vec![unused] + ); + std::fs::remove_dir_all(project.join(".venv")).unwrap(); + + // PDM 1.x: no saved interpreter, still a PDM project. + std::fs::remove_file(project.join(".pdm-python")).unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &no_env).await, + vec![lib.clone()] + ); + + // With no saved interpreter (or one PDM ignores), PDM picks an + // activated venv or `./.venv` before PEP 582. + let other = tempfile::tempdir().unwrap(); + let active_site = fake_venv(other.path(), "active"); + let active = env_of(&[( + "VIRTUAL_ENV", + other.path().join("active").to_string_lossy().into_owned(), + )]); + assert_eq!( + find_local_venv_site_packages_with(&project, &active).await, + vec![active_site.clone()] + ); + // ...unless PDM_IGNORE_ACTIVE_VENV tells PDM to skip it. + let opted_out = env_of(&[ + ( + "VIRTUAL_ENV", + other.path().join("active").to_string_lossy().into_owned(), + ), + ("PDM_IGNORE_ACTIVE_VENV", "1".to_string()), + ]); + assert_eq!( + find_local_venv_site_packages_with(&project, &opted_out).await, + vec![lib.clone()] + ); + // PDM parses the flag as a boolean: false values keep the venv. + for falsy in ["0", "false", "No", ""] { + let kept = env_of(&[ + ( + "VIRTUAL_ENV", + other.path().join("active").to_string_lossy().into_owned(), + ), + ("PDM_IGNORE_ACTIVE_VENV", falsy.to_string()), + ]); + assert_eq!( + find_local_venv_site_packages_with(&project, &kept).await, + vec![active_site.clone()], + "PDM_IGNORE_ACTIVE_VENV={falsy:?}" + ); + } + let dot_venv = fake_venv(&project, ".venv"); + assert_eq!( + find_local_venv_site_packages_with(&project, &no_env).await, + vec![dot_venv] + ); + std::fs::write(project.join(".pdm-python"), base.display().to_string()).unwrap(); + let ignored = env_of(&[("PDM_IGNORE_SAVED_PYTHON", "1".to_string())]); + assert_eq!( + find_local_venv_site_packages_with(&project, &ignored).await, + vec![fake_venv(&project, ".venv")] + ); + // ...while a saved base interpreter still means PEP 582. + assert_eq!( + find_local_venv_site_packages_with(&project, &no_env).await, + vec![lib.clone()] + ); + std::fs::remove_file(project.join(".pdm-python")).unwrap(); + std::fs::remove_dir_all(project.join(".venv")).unwrap(); + + // A lockless PDM project's `__pypackages__` is not handed to an + // ambient UV_PROJECT_ENVIRONMENT. + std::fs::remove_file(project.join("pdm.lock")).unwrap(); + std::fs::write( + project.join("pyproject.toml"), + "[project]\nname = \"demo\"\n[tool.pdm]\ndistribution = false\n", + ) + .unwrap(); + fake_venv(&tmp.path().join("uv-env"), "venv"); + let uv_env = env_of(&[( + "UV_PROJECT_ENVIRONMENT", + tmp.path().join("uv-env").join("venv").to_string_lossy().into_owned(), + )]); + assert_eq!( + find_local_venv_site_packages_with(&project, &uv_env).await, + vec![lib.clone()] + ); + std::fs::write( + project.join("pyproject.toml"), + "[project]\nname = \"demo\"\n", + ) + .unwrap(); + std::fs::write(project.join("pdm.lock"), "[metadata]\n").unwrap(); + + // A uv project with a leftover PDM lock is uv's, not PEP 582. + std::fs::write(project.join("uv.lock"), "version = 1\n").unwrap(); + assert!(find_local_venv_site_packages_with(&project, &no_env) + .await + .is_empty()); + std::fs::remove_file(project.join("uv.lock")).unwrap(); + + // `__pypackages__` outside a PDM project is not PDM's. + std::fs::remove_file(project.join("pdm.lock")).unwrap(); + assert!(find_local_venv_site_packages_with(&project, &no_env) + .await + .is_empty()); + } + + /// #525: uv syncs a project into `UV_PROJECT_ENVIRONMENT` (absolute, or + /// relative to the project) instead of `./.venv`, ignoring an + /// activated venv; the variable means nothing outside a uv project. + #[tokio::test] + async fn uv_project_environment_is_the_project_env() { + let tmp = tempfile::tempdir().unwrap(); + let project = tmp.path().join("app"); + std::fs::create_dir_all(&project).unwrap(); + std::fs::write( + project.join("pyproject.toml"), + "[project]\nname = \"app\"\n", + ) + .unwrap(); + std::fs::write(project.join("uv.lock"), "version = 1\n").unwrap(); + let abs_site = fake_venv(&tmp.path().join("opt"), "venv"); + let abs = tmp.path().join("opt").join("venv"); + let rel_site = fake_venv(&project, ".venv-ci"); + let stray = fake_venv(&project, ".venv"); + let other = tempfile::tempdir().unwrap(); + fake_venv(other.path(), "tool-venv"); + let activated = other + .path() + .join("tool-venv") + .to_string_lossy() + .into_owned(); + + let abs_env = env_of(&[("UV_PROJECT_ENVIRONMENT", abs.to_string_lossy().into_owned())]); + assert_eq!( + find_local_venv_site_packages_with(&project, &abs_env).await, + vec![abs_site.clone()] + ); + let rel_env = env_of(&[ + ("UV_PROJECT_ENVIRONMENT", ".venv-ci".to_string()), + ("VIRTUAL_ENV", activated.clone()), + ]); + assert_eq!( + find_local_venv_site_packages_with(&project, &rel_env).await, + vec![rel_site.clone()] + ); + + // Lock-less uv project (just pyproject.toml) counts too. + std::fs::remove_file(project.join("uv.lock")).unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &abs_env).await, + vec![abs_site.clone()] + ); + + // A lockless Poetry project (`[tool.poetry]`, or `poetry.toml`) is + // Poetry's: an ambient UV_PROJECT_ENVIRONMENT does not take it over. + std::fs::write( + project.join("pyproject.toml"), + "[tool.poetry]\nname = \"app\"\n", + ) + .unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &abs_env).await, + vec![stray.clone()] + ); + std::fs::write( + project.join("pyproject.toml"), + "[project]\nname = \"app\"\n", + ) + .unwrap(); + std::fs::write(project.join("poetry.toml"), "").unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &abs_env).await, + vec![stray.clone()] + ); + std::fs::remove_file(project.join("poetry.toml")).unwrap(); + + // A lockless PDM project is PDM's, while a `[tool.pdm.build]` table + // alone (the pdm-backend build backend) does not make one. + std::fs::write( + project.join("pyproject.toml"), + "[project]\nname = \"app\"\n[tool.pdm]\ndistribution = false\n", + ) + .unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &abs_env).await, + vec![stray.clone()] + ); + std::fs::write( + project.join("pyproject.toml"), + "[project]\nname = \"app\"\n[tool.pdm.build]\nincludes = [\"app\"]\n", + ) + .unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &abs_env).await, + vec![abs_site.clone()] + ); + std::fs::write( + project.join("pyproject.toml"), + "[project]\nname = \"app\"\n", + ) + .unwrap(); + + // ...but not a project another manager drives, nor a non-project. + std::fs::write(project.join("poetry.lock"), "").unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &abs_env).await, + vec![stray.clone()] + ); + std::fs::remove_file(project.join("poetry.lock")).unwrap(); + std::fs::remove_file(project.join("pyproject.toml")).unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &abs_env).await, + vec![stray.clone()] + ); + + // An env that does not exist yet leaves the generic probes in charge. + std::fs::write( + project.join("pyproject.toml"), + "[project]\nname = \"app\"\n", + ) + .unwrap(); + let missing = env_of(&[("UV_PROJECT_ENVIRONMENT", "not-synced".to_string())]); + assert_eq!( + find_local_venv_site_packages_with(&project, &missing).await, + vec![stray] + ); + } + /// The end-to-end shape: a Pipenv project with NO in-project venv and /// Pipenv's default out-of-tree placement under WORKON_HOME is found by /// name+hash (with and without the `-` suffix), while a diff --git a/docs/testing/pdm-compatibility.md b/docs/testing/pdm-compatibility.md index 4541634b3..99f57e19c 100644 --- a/docs/testing/pdm-compatibility.md +++ b/docs/testing/pdm-compatibility.md @@ -69,11 +69,19 @@ Measured details: - **Hosted mode verifies the lock's file hash** on install for every supported release (tamper the hash and `pdm sync` fails closed). Vendored mode's protection is the committed wheel bytes, verified by the same hash. -- **`__pypackages__` (PEP 582) projects are not covered.** PDM 0.x/1.x default - to `__pypackages__`, and 2.x does so under `python.use_venv = false`; the - installed-set crawler probes virtualenvs (`VIRTUAL_ENV`, `./.venv`), so agent - and vendored mode need a virtualenv install. Run PDM with `python.use_venv` - on, or use hosted mode. +- **The installed env is the one PDM records.** The crawler follows the + interpreter in `PDM_PYTHON`, else `.pdm-python` (`[python] path` in + `.pdm.toml` on older PDM), ahead of an activated venv or a stray `./.venv`. + The first of them that is an environment wins: a venv (an out-of-tree + `venv.in_project = false` venv, or one bound with `pdm use `) or a + conda env. A project with `uv.lock` or `poetry.lock` is not treated as + PDM's. A base interpreter means PEP 582 (`__pypackages__//lib`) only + when `python.use_venv` is off (`PDM_USE_VENV`, or `[python] use_venv` in + `pdm.toml` / `.pdm.toml`; off by default only for PDM 1.x). Otherwise the + activated or in-project venv is used, with `__pypackages__` as the last + resort. `PDM_IGNORE_SAVED_PYTHON` and `PDM_IGNORE_ACTIVE_VENV` are + honored, parsed as PDM parses booleans. Agent mode patches the env found, + and the hosted stale-install warning and `vex` check it. - **A non-default lock filename (`pdm lock -L custom.lock`) is invisible** to the scan, which only reads `pdm.lock`. A package locked at two versions (a marker fork) is refused (`pypi_pdm_lock_forked_package` / a version-mismatch refusal), diff --git a/docs/testing/uv-compatibility.md b/docs/testing/uv-compatibility.md index 16f1a67ba..42d248742 100644 --- a/docs/testing/uv-compatibility.md +++ b/docs/testing/uv-compatibility.md @@ -32,6 +32,13 @@ frozen, locked, and ordinary installation outcomes separately where supported. ## Limits +- The installed env is the one uv syncs into. With `UV_PROJECT_ENVIRONMENT` set + (absolute, or relative to the project), the crawler uses that env in place of + `./.venv` or an activated `VIRTUAL_ENV`, as `uv sync` / `uv run` do. This + holds for a project with `uv.lock`, or a `pyproject.toml` that no other + manager's lock claims. Agent mode patches that env, and the hosted + stale-install warning and `vex` check it. Run the scan with the same + `UV_PROJECT_ENVIRONMENT` that `uv sync` used. - uv 0.0 has no native `uv.lock`; its compatibility lane is compiled requirements. `uv pip sync` rejects the bare local wheel paths emitted by vendored requirements through uv 0.1.23 (`Unexpected '.', expected '-c', '-e', '-r'