From b7e12f1a4aa80213eff696853544d8015c0f386b Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 1 Aug 2026 20:53:45 +0200 Subject: [PATCH 01/12] terminalwriter: add OSC 8 hyperlink support Adds `TerminalWriter.hyperlink()` plus `write_link()`/`line_link()`, so paths pytest prints can be made clickable. The escapes are applied inside the shared `_write()` helper, after the `_current_line` bookkeeping and after `markup()`, so they stay invisible to `width_of_current_line`. `write()`/`line()` keep their exact signatures, since a keyword-only `link` parameter would collide with every existing `**markup` splat. `sep()` deliberately gets no hyperlink variant, as it computes fill counts from `len(title)`. Detection is allow-by-default with a deny-list: terminals without OSC 8 support almost universally ignore unknown OSC sequences rather than printing them, and an allow-list of known-good terminals would be permanently out of date. `should_do_markup` already excludes files, pipes and NO_COLOR. There are no callers yet. Co-Authored-By: Claude Opus 5 (1M context) --- changelog/1089.improvement.rst | 5 ++ src/_pytest/_io/terminalwriter.py | 74 ++++++++++++++++++++ testing/io/test_terminalwriter.py | 111 ++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+) create mode 100644 changelog/1089.improvement.rst diff --git a/changelog/1089.improvement.rst b/changelog/1089.improvement.rst new file mode 100644 index 00000000000..d8db4c32c90 --- /dev/null +++ b/changelog/1089.improvement.rst @@ -0,0 +1,5 @@ +:class:`~_pytest.config.Config`'s terminal writer can now emit OSC 8 terminal hyperlinks, so that +paths printed by pytest are clickable in terminals which support them. + +Hyperlinks are emitted only when colored output is enabled and the terminal is not known to mishandle +them. They can be forced on or off with the ``PYTEST_HYPERLINKS`` environment variable (``1`` or ``0``). diff --git a/src/_pytest/_io/terminalwriter.py b/src/_pytest/_io/terminalwriter.py index 9191b4edace..8047c559b64 100644 --- a/src/_pytest/_io/terminalwriter.py +++ b/src/_pytest/_io/terminalwriter.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Mapping from collections.abc import Sequence import os import shutil @@ -47,6 +48,26 @@ def should_do_markup(file: TextIO) -> bool: ) +def should_do_hyperlinks(file: TextIO) -> bool: + """Whether OSC 8 hyperlinks should be emitted to ``file``. + + There is no way to ask a terminal whether it understands OSC 8, so this is + allow-by-default with a deny-list: terminals which do not implement it + almost universally *ignore* unknown OSC sequences rather than printing + them, and an allow-list of known-good terminals would be permanently out of + date. ``should_do_markup`` already excludes files, pipes and ``NO_COLOR``. + """ + if os.environ.get("PYTEST_HYPERLINKS") == "1": + return True + if os.environ.get("PYTEST_HYPERLINKS") == "0": + return False + if not should_do_markup(file): + return False + term = os.environ.get("TERM", "") + # The Linux virtual console and GNU screen do not pass OSC 8 through. + return term not in ("dumb", "linux") and not term.startswith(("screen", "eterm")) + + @final class TerminalWriter: _esctable = dict( @@ -85,6 +106,7 @@ def __init__(self, file: TextIO | None = None) -> None: assert file is not None self._file = file self.hasmarkup = should_do_markup(file) + self.haslinks = should_do_hyperlinks(file) self._current_line = "" self._terminal_width: int | None = None self.code_highlight = True @@ -114,6 +136,28 @@ def markup(self, text: str, **markup: bool) -> str: text = "".join(f"\x1b[{cod}m" for cod in esc) + text + "\x1b[0m" return text + def hyperlink(self, text: str, url: str) -> str: + """Return ``text`` as an OSC 8 terminal hyperlink to ``url``. + + Returns ``text`` unchanged when hyperlinks are disabled, so callers may + use this unconditionally. + + The result contains zero-width escape sequences, so it must not be used + for width calculations - prefer ``write(..., link=...)``, which applies + the escapes after the line-width bookkeeping. + """ + if not self.haslinks or not text: + return text + # A ";" would terminate the parameter field, and ESC/BEL the sequence + # itself; rather than mangle the terminal, degrade to plain text. + if any(c in url for c in "\x1b\x07;"): + return text + # ST ("\x1b\\") rather than BEL as the terminator, as the spec uses it + # and it survives tmux/screen passthrough. Any SGR markup is nested + # inside the link, as some terminals drop the link when the SGR reset + # comes after the closing sequence. + return f"\x1b]8;;{url}\x1b\\{text}\x1b]8;;\x1b\\" + def sep( self, sepchar: str, @@ -152,6 +196,27 @@ def sep( self.line(line, **markup) def write(self, msg: str, *, flush: bool = False, **markup: bool) -> None: + self._write(msg, flush=flush, link=None, markup=markup) + + def write_link( + self, text: str, url: str, *, flush: bool = False, **markup: bool + ) -> None: + """Write ``text`` as an OSC 8 terminal hyperlink to ``url``. + + Prefer this over wrapping the text with ``hyperlink()`` yourself: the + escapes are applied after the line-width bookkeeping, so they stay + invisible to ``width_of_current_line``. + """ + self._write(text, flush=flush, link=url, markup=markup) + + def _write( + self, + msg: str, + *, + flush: bool, + link: str | None, + markup: Mapping[str, bool], + ) -> None: if msg: current_line = msg.rsplit("\n", 1)[-1] if "\n" in msg: @@ -159,7 +224,11 @@ def write(self, msg: str, *, flush: bool = False, **markup: bool) -> None: else: self._current_line += current_line + # The bookkeeping above deliberately uses the raw message, so that + # the escapes added below stay invisible to the width accounting. msg = self.markup(msg, **markup) + if link is not None: + msg = self.hyperlink(msg, link) self.write_raw(msg, flush=flush) @@ -183,6 +252,11 @@ def line(self, s: str = "", **markup: bool) -> None: self.write(s, **markup) self.write("\n") + def line_link(self, s: str, url: str, **markup: bool) -> None: + """Like ``line()``, but rendering ``s`` as a hyperlink to ``url``.""" + self.write_link(s, url, **markup) + self.write("\n") + def flush(self) -> None: self._file.flush() diff --git a/testing/io/test_terminalwriter.py b/testing/io/test_terminalwriter.py index 9aa89da0e41..a9c38036ce8 100644 --- a/testing/io/test_terminalwriter.py +++ b/testing/io/test_terminalwriter.py @@ -231,6 +231,117 @@ def test_empty_NO_COLOR_and_FORCE_COLOR_ignored(monkeypatch: MonkeyPatch) -> Non assert_color(False, False) +class TestHyperlinks: + @pytest.fixture + def file(self) -> StringIO: + return StringIO() + + @pytest.fixture + def tw(self, file: StringIO) -> terminalwriter.TerminalWriter: + tw = terminalwriter.TerminalWriter(file) + tw.haslinks = True + return tw + + def test_hyperlink(self, tw: terminalwriter.TerminalWriter) -> None: + assert ( + tw.hyperlink("pytest", "file:///tmp/x") + == "\x1b]8;;file:///tmp/x\x1b\\pytest\x1b]8;;\x1b\\" + ) + + def test_hyperlink_disabled(self, tw: terminalwriter.TerminalWriter) -> None: + tw.haslinks = False + assert tw.hyperlink("pytest", "file:///tmp/x") == "pytest" + + def test_hyperlink_empty_text(self, tw: terminalwriter.TerminalWriter) -> None: + assert tw.hyperlink("", "file:///tmp/x") == "" + + @pytest.mark.parametrize("url", ["file:///a;b", "file:///a\x1bb", "file:///a\x07b"]) + def test_hyperlink_rejects_unsafe_url( + self, tw: terminalwriter.TerminalWriter, url: str + ) -> None: + assert tw.hyperlink("pytest", url) == "pytest" + + def test_write_link( + self, tw: terminalwriter.TerminalWriter, file: StringIO + ) -> None: + tw.write_link("pytest", "file:///tmp/x") + assert file.getvalue() == "\x1b]8;;file:///tmp/x\x1b\\pytest\x1b]8;;\x1b\\" + + def test_line_link(self, tw: terminalwriter.TerminalWriter, file: StringIO) -> None: + tw.line_link("pytest", "file:///tmp/x") + assert file.getvalue() == "\x1b]8;;file:///tmp/x\x1b\\pytest\x1b]8;;\x1b\\\n" + + def test_write_link_disabled( + self, tw: terminalwriter.TerminalWriter, file: StringIO + ) -> None: + tw.haslinks = False + tw.write_link("pytest", "file:///tmp/x") + assert file.getvalue() == "pytest" + + def test_write_link_with_markup_nests_sgr_inside( + self, tw: terminalwriter.TerminalWriter, file: StringIO + ) -> None: + tw.hasmarkup = True + tw.write_link("pytest", "file:///tmp/x", bold=True) + assert ( + file.getvalue() + == "\x1b]8;;file:///tmp/x\x1b\\\x1b[1mpytest\x1b[0m\x1b]8;;\x1b\\" + ) + + def test_write_link_keeps_width_accounting( + self, tw: terminalwriter.TerminalWriter + ) -> None: + tw.write_link("abc", "file:///tmp/x") + assert tw.width_of_current_line == 3 + + +def assert_hyperlinks(expected: bool) -> None: + file = io.StringIO() + file.isatty = lambda: True # type: ignore + assert terminalwriter.should_do_hyperlinks(file) is expected + + +def test_should_do_hyperlinks_PYTEST_HYPERLINKS_eq_1(monkeypatch: MonkeyPatch) -> None: + # Forced on even though markup is off. + monkeypatch.setitem(os.environ, "NO_COLOR", "1") + monkeypatch.setitem(os.environ, "PYTEST_HYPERLINKS", "1") + assert_hyperlinks(True) + + +def test_should_not_do_hyperlinks_PYTEST_HYPERLINKS_eq_0( + monkeypatch: MonkeyPatch, +) -> None: + monkeypatch.setitem(os.environ, "PY_COLORS", "1") + monkeypatch.setitem(os.environ, "PYTEST_HYPERLINKS", "0") + assert_hyperlinks(False) + + +def test_should_not_do_hyperlinks_without_markup(monkeypatch: MonkeyPatch) -> None: + monkeypatch.delenv("PYTEST_HYPERLINKS", raising=False) + monkeypatch.setitem(os.environ, "NO_COLOR", "1") + assert_hyperlinks(False) + + +@pytest.mark.parametrize( + ["term", "expected"], + [ + ("xterm-256color", True), + ("foot", True), + ("linux", False), + ("screen-256color", False), + ("eterm-color", False), + ], +) +def test_should_do_hyperlinks_for_term( + monkeypatch: MonkeyPatch, term: str, expected: bool +) -> None: + monkeypatch.delenv("PYTEST_HYPERLINKS", raising=False) + monkeypatch.delenv("NO_COLOR", raising=False) + monkeypatch.setitem(os.environ, "PY_COLORS", "1") + monkeypatch.setenv("TERM", term) + assert_hyperlinks(expected) + + class TestTerminalWriterLineWidth: def test_init(self) -> None: tw = terminalwriter.TerminalWriter() From 621f85d585fa3b3e33ffc3dbd01b9f34f9bdf04e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 1 Aug 2026 23:21:57 +0200 Subject: [PATCH 02/12] pytester: isolate the user-level cache directory Pytester redirected HOME/USERPROFILE but nothing else, which is not enough to keep inner runs away from the developer's real user cache: - XDG_CACHE_HOME takes precedence over HOME on Linux, so it leaks whenever it is set in the outer environment; - LOCALAPPDATA is not derived from USERPROFILE, so on Windows it always leaks. XDG_CACHE_HOME is unset rather than redirected, so inner runs exercise the same platform-native path real users get. Prerequisite for the cache_policy work, which makes pytest actually read these variables. Co-Authored-By: Claude Opus 5 (1M context) --- changelog/1089.improvement.1.rst | 6 ++++++ src/_pytest/pytester.py | 8 ++++++++ testing/test_pytester.py | 23 +++++++++++++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 changelog/1089.improvement.1.rst diff --git a/changelog/1089.improvement.1.rst b/changelog/1089.improvement.1.rst new file mode 100644 index 00000000000..4a38db8005f --- /dev/null +++ b/changelog/1089.improvement.1.rst @@ -0,0 +1,6 @@ +The :fixture:`pytester` fixture now also isolates the user-level cache directory of inner test runs, +by unsetting ``XDG_CACHE_HOME`` and ``PYTEST_CACHE_HOME`` and pointing ``LOCALAPPDATA`` inside the +temporary home directory. + +Previously only ``HOME``/``USERPROFILE`` were redirected, which is not enough: ``XDG_CACHE_HOME`` takes +precedence over ``HOME``, and on Windows ``LOCALAPPDATA`` is not derived from ``USERPROFILE`` at all. diff --git a/src/_pytest/pytester.py b/src/_pytest/pytester.py index b69b58732ef..e8198e79fc4 100644 --- a/src/_pytest/pytester.py +++ b/src/_pytest/pytester.py @@ -709,6 +709,14 @@ def __init__( tmphome = str(self.path) mp.setenv("HOME", tmphome) mp.setenv("USERPROFILE", tmphome) + # Ensure the user-level cache directory is isolated as well. HOME alone + # is not enough: XDG_CACHE_HOME takes precedence over it, and on Windows + # LOCALAPPDATA is not derived from USERPROFILE at all. XDG_CACHE_HOME is + # unset rather than pointed at tmphome, so that inner runs exercise the + # same platform-native path that real users get. + mp.delenv("XDG_CACHE_HOME", raising=False) + mp.delenv("PYTEST_CACHE_HOME", raising=False) + mp.setenv("LOCALAPPDATA", os.path.join(tmphome, "AppData", "Local")) # Do not use colors for inner runs by default. mp.setenv("PY_COLORS", "0") diff --git a/testing/test_pytester.py b/testing/test_pytester.py index f641e9ee8bb..c2f1b2cfc3f 100644 --- a/testing/test_pytester.py +++ b/testing/test_pytester.py @@ -720,6 +720,29 @@ def test(): assert child.wait() == 0, out.decode("utf8") +def test_pytester_isolates_user_cache_env(pytester: Pytester) -> None: + tmphome = str(pytester.path) + assert "XDG_CACHE_HOME" not in os.environ + assert "PYTEST_CACHE_HOME" not in os.environ + assert os.environ["LOCALAPPDATA"] == os.path.join(tmphome, "AppData", "Local") + + # The isolation must survive into subprocess runs, which copy os.environ. + p1 = pytester.makepyfile( + f""" + import os + + def test(): + assert "XDG_CACHE_HOME" not in os.environ + assert "PYTEST_CACHE_HOME" not in os.environ + assert os.environ["LOCALAPPDATA"] == os.path.join( + {tmphome!r}, "AppData", "Local" + ) + """ + ) + result = pytester.runpytest_subprocess(str(p1)) + result.assert_outcomes(passed=1) + + def test_run_result_repr() -> None: outlines = ["some", "normal", "output"] errlines = ["some", "nasty", "errors", "happened"] From adec6cadfa6acc57bb499bad2ab7dfd71ed09287 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 2 Aug 2026 07:14:56 +0200 Subject: [PATCH 03/12] pathlib: resolve the user-level cache root behind an xdg extra Adds `pytest_user_cache_dir()` and `check_user_cache_root()`. Nothing calls them yet; `cache_policy = user` will. platformdirs is an optional dependency (`pytest[xdg]`) rather than a hard one, since the feature it serves is opt-in, and it is imported lazily so a plain install never touches it. PYTEST_CACHE_HOME is checked first, so the escape hatch - and most of the test suite - works without the extra. The per-platform conventions are delegated verbatim: taking the dependency is precisely so we stop having opinions about them. The hardening is deliberately weaker than TempPathFactory.getbasetemp's. That guards a directory in world-writable, shared /tmp where name-squatting is a real attack; the user cache home is neither shared nor world-writable, and two of those checks would actively cause harm here - rejecting a symlinked root would break pointing ~/.cache at another volume, and forcing 0o700 would fight GH-12308. Only ownership and a non-sticky world-writable root are rejected. Co-Authored-By: Claude Opus 5 (1M context) --- changelog/1089.packaging.rst | 8 ++++ pyproject.toml | 8 +++- src/_pytest/pathlib.py | 79 +++++++++++++++++++++++++++++++ testing/test_pathlib.py | 91 ++++++++++++++++++++++++++++++++++++ uv.lock | 19 ++++++-- 5 files changed, 201 insertions(+), 4 deletions(-) create mode 100644 changelog/1089.packaging.rst diff --git a/changelog/1089.packaging.rst b/changelog/1089.packaging.rst new file mode 100644 index 00000000000..5734337c88a --- /dev/null +++ b/changelog/1089.packaging.rst @@ -0,0 +1,8 @@ +A new ``xdg`` extra is available: ``pip install pytest[xdg]`` pulls in platformdirs_, which pytest uses +to locate the platform's user cache directory. + +The extra is only needed to store the cache outside the project directory; a plain ``pip install pytest`` +is unaffected, and platformdirs is imported lazily so it is never needed otherwise. Setting +``PYTEST_CACHE_HOME`` to an explicit directory also works without the extra. + +.. _platformdirs: https://pypi.org/project/platformdirs/ diff --git a/pyproject.toml b/pyproject.toml index b467ed0fba0..4f05e61bea2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,10 @@ optional-dependencies.dev = [ "setuptools", "xmlschema", ] +# Needed to locate the platform's user cache directory, for `cache_policy = user`. +optional-dependencies.xdg = [ + "platformdirs>=4", +] urls.Changelog = "https://docs.pytest.org/en/stable/changelog.html" urls.Contact = "https://docs.pytest.org/en/stable/contact.html" urls.Funding = "https://docs.pytest.org/en/stable/sponsor.html" @@ -87,8 +91,10 @@ scripts.pytest = "_pytest.config:_console_main" [dependency-groups] # Preferred entry point for developing pytest; pulls from optional-dependencies.dev. +# `xdg` is included so the user-level cache is covered locally and in CI; the +# tests which need it skip when it is absent, so a minimal install stays green. dev = [ - "pytest[dev]", + "pytest[dev,xdg]", ] [tool.setuptools.package-data] diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py index 10326e1c9a6..ab9ced5e81c 100644 --- a/src/_pytest/pathlib.py +++ b/src/_pytest/pathlib.py @@ -3,6 +3,7 @@ from collections.abc import Callable from collections.abc import Iterable from collections.abc import Iterator +from collections.abc import Mapping import contextlib from enum import Enum from errno import EBADF @@ -34,6 +35,7 @@ import warnings from _pytest.compat import assert_never +from _pytest.compat import get_user_id from _pytest.outcomes import skip from _pytest.warning_types import PytestWarning @@ -473,6 +475,83 @@ def resolve_from_str(input: str, rootpath: Path) -> Path: return rootpath.joinpath(input) +def pytest_user_cache_dir(*, environ: Mapping[str, str] | None = None) -> Path: + """Return the root directory for pytest's user-level caches. + + Creates nothing. + + ``PYTEST_CACHE_HOME`` overrides the location entirely; it is what pytest's + own test suite uses, and what CI setups wanting an explicit shared location + should set. Otherwise the platform convention is delegated to + ``platformdirs``, which is an optional dependency (``pytest[xdg]``). + + :raises UsageError: + If ``platformdirs`` is not installed and ``PYTEST_CACHE_HOME`` is unset. + """ + if environ is None: + environ = os.environ + override = environ.get("PYTEST_CACHE_HOME") + if override: + return Path(expanduser(expandvars(override))) + try: + import platformdirs + except ImportError: + # Imported lazily and only on this path, so that installations which + # never opt into the user-level cache do not need the dependency. + from _pytest.config.exceptions import UsageError + + raise UsageError( + "the user-level pytest cache requires the 'platformdirs' package; " + "install it with `pip install pytest[xdg]`, or set PYTEST_CACHE_HOME " + "to an explicit directory." + ) from None + # platformdirs is an optional dependency and so is untyped as far as our + # checker is concerned; re-wrap to keep the declared return annotation. + return Path(platformdirs.user_cache_path("pytest", appauthor=False)) + + +def check_user_cache_root(root: Path) -> None: + """Reject a user cache root we clearly should not be writing into. + + Deliberately weaker than :meth:`TempPathFactory.getbasetemp`'s equivalent + checks. That guards a directory in the world-writable, shared ``/tmp``, + where name-squatting is a real attack; the user cache home is neither + shared nor world-writable. Two of those checks would actively cause harm + here: + + * rejecting a symlinked root would break pointing ``~/.cache`` at another + volume, which is legitimate and common; + * forcing ``0o700`` would fight #12308, which is exactly why + ``_make_cachedir`` re-applies the umask default instead. + + So only ownership is checked, plus a world-writable root without the sticky + bit - which catches the one genuinely dangerous configuration, + ``XDG_CACHE_HOME=/tmp``. + """ + uid = get_user_id() + if uid is None: # Windows, emscripten, ... + return + try: + # Follow symlinks: a symlinked root is fine, what matters is the target. + st = root.stat() + except OSError: + # Does not exist yet, or is unreadable; creating it will report that. + return + + from _pytest.config.exceptions import UsageError + + if st.st_uid != uid: + raise UsageError( + f"The pytest user cache directory {root} is not owned by the " + f"current user. Fix this, or set PYTEST_CACHE_HOME." + ) + if st.st_mode & stat.S_IWOTH and not st.st_mode & stat.S_ISVTX: + raise UsageError( + f"The pytest user cache directory {root} is world-writable without " + f"the sticky bit set. Refusing to use it." + ) + + def fnmatch_ex(pattern: str, path: str | os.PathLike[str]) -> bool: """A port of FNMatcher from py.path.common which works with PurePath() instances. diff --git a/testing/test_pathlib.py b/testing/test_pathlib.py index bd85b7e8fb4..80886d40f0c 100644 --- a/testing/test_pathlib.py +++ b/testing/test_pathlib.py @@ -17,10 +17,14 @@ from typing import Any import unittest.mock +from _pytest.compat import get_user_id from _pytest.config import ExitCode +from _pytest.config import UsageError from _pytest.monkeypatch import MonkeyPatch +import _pytest.pathlib as pathlib_module from _pytest.pathlib import _import_module_using_spec from _pytest.pathlib import bestrelpath +from _pytest.pathlib import check_user_cache_root from _pytest.pathlib import commonpath from _pytest.pathlib import compute_module_name from _pytest.pathlib import CouldNotResolvePathError @@ -35,6 +39,7 @@ from _pytest.pathlib import is_importable from _pytest.pathlib import maybe_delete_a_numbered_dir from _pytest.pathlib import module_name_from_path +from _pytest.pathlib import pytest_user_cache_dir from _pytest.pathlib import resolve_package_path from _pytest.pathlib import resolve_pkg_root_and_module_name from _pytest.pathlib import safe_exists @@ -529,6 +534,92 @@ def test_bestrelpath() -> None: assert bestrelpath(curdir, Path("hello")) == "hello" +class TestUserCacheDir: + def test_cache_home_override(self, tmp_path: Path) -> None: + environ = {"PYTEST_CACHE_HOME": str(tmp_path / "explicit")} + assert pytest_user_cache_dir(environ=environ) == tmp_path / "explicit" + + def test_cache_home_override_is_expanded(self, monkeypatch: MonkeyPatch) -> None: + monkeypatch.setenv("SOMEWHERE", "elsewhere") + environ = {"PYTEST_CACHE_HOME": os.path.join("~", "$SOMEWHERE")} + assert pytest_user_cache_dir(environ=environ) == Path( + os.path.expanduser("~") + ).joinpath("elsewhere") + + def test_cache_home_override_wins_over_xdg(self, tmp_path: Path) -> None: + environ = { + "PYTEST_CACHE_HOME": str(tmp_path / "explicit"), + "XDG_CACHE_HOME": str(tmp_path / "xdg"), + } + assert pytest_user_cache_dir(environ=environ) == tmp_path / "explicit" + + def test_cache_home_override_needs_no_platformdirs( + self, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: + # The escape hatch has to work on installs without the `xdg` extra. + monkeypatch.setitem(sys.modules, "platformdirs", None) + environ = {"PYTEST_CACHE_HOME": str(tmp_path / "explicit")} + assert pytest_user_cache_dir(environ=environ) == tmp_path / "explicit" + + def test_delegates_to_platformdirs(self, monkeypatch: MonkeyPatch) -> None: + # The per-platform conventions are platformdirs' business; all we test + # is that we ask it the right question. + platformdirs = pytest.importorskip("platformdirs") + calls = [] + + def user_cache_path(*args: object, **kwargs: object) -> Path: + calls.append((args, kwargs)) + return Path("/somewhere/pytest") + + monkeypatch.setattr(platformdirs, "user_cache_path", user_cache_path) + assert pytest_user_cache_dir(environ={}) == Path("/somewhere/pytest") + assert calls == [(("pytest",), {"appauthor": False})] + + def test_without_platformdirs(self, monkeypatch: MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "platformdirs", None) + with pytest.raises(UsageError, match=r"pip install pytest\[xdg\]"): + pytest_user_cache_dir(environ={}) + + +class TestCheckUserCacheRoot: + def test_missing_root_is_accepted(self, tmp_path: Path) -> None: + check_user_cache_root(tmp_path / "does-not-exist") + + def test_plain_root_is_accepted(self, tmp_path: Path) -> None: + check_user_cache_root(tmp_path) + + @pytest.mark.skipif(sys.platform == "win32", reason="no symlink ownership on win32") + def test_symlinked_root_is_accepted(self, tmp_path: Path) -> None: + # Pointing ~/.cache at another volume is legitimate; unlike the tmpdir + # equivalent we must not reject it. + target = tmp_path / "target" + target.mkdir() + symlink_or_skip(target, tmp_path / "link") + check_user_cache_root(tmp_path / "link") + + def test_foreign_owner_is_rejected( + self, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: + uid = get_user_id() + if uid is None: + pytest.skip("no user id on this platform") + monkeypatch.setattr(pathlib_module, "get_user_id", lambda: uid + 1) + with pytest.raises(UsageError, match="not owned by the current user"): + check_user_cache_root(tmp_path) + + @pytest.mark.skipif(sys.platform == "win32", reason="no mode bits on win32") + def test_world_writable_root_is_rejected(self, tmp_path: Path) -> None: + tmp_path.chmod(0o777) + with pytest.raises(UsageError, match="world-writable"): + check_user_cache_root(tmp_path) + + @pytest.mark.skipif(sys.platform == "win32", reason="no mode bits on win32") + def test_world_writable_sticky_root_is_accepted(self, tmp_path: Path) -> None: + # /tmp itself is world-writable but sticky, which is safe enough. + tmp_path.chmod(0o1777) + check_user_cache_root(tmp_path) + + def test_commonpath() -> None: path = Path("/foo/bar/baz/path") subpath = path / "sampledir" diff --git a/uv.lock b/uv.lock index 065a1fdcde9..98e8d031c9c 100644 --- a/uv.lock +++ b/uv.lock @@ -477,6 +477,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, ] +[[package]] +name = "platformdirs" +version = "4.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/78/9b/560e4be8e26f6fd133a03630a8df0c663b9e8d61b4ade152b72005aec83b/platformdirs-4.11.0.tar.gz", hash = "sha256:0555d18370482847566ffabcaa53ad7c6c1c29f195989ae1ed634a05f76ea1e0", size = 31953, upload-time = "2026-07-21T13:09:36.565Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/68/d8d58938dfb1370b266a1a729e6d77a985be23689a0496498ee17b2cbf90/platformdirs-4.11.0-py3-none-any.whl", hash = "sha256:360ccded2b7fce0af0ff80cc8f5942a1c5d99b0e856033acb030bfc634709e74", size = 23247, upload-time = "2026-07-21T13:09:35.422Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -534,10 +543,13 @@ dev = [ { name = "setuptools" }, { name = "xmlschema" }, ] +xdg = [ + { name = "platformdirs" }, +] [package.dev-dependencies] dev = [ - { name = "pytest", extra = ["dev"] }, + { name = "pytest", extra = ["dev", "xdg"] }, ] [package.metadata] @@ -554,6 +566,7 @@ requires-dist = [ { name = "numpy", marker = "extra == 'dev'", specifier = ">=1.26" }, { name = "packaging", specifier = ">=24" }, { name = "pexpect", marker = "extra == 'dev'", specifier = ">=4.9" }, + { name = "platformdirs", marker = "extra == 'xdg'", specifier = ">=4" }, { name = "pluggy", specifier = ">=1.5,<2" }, { name = "pygments", specifier = ">=2.15" }, { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.5" }, @@ -563,10 +576,10 @@ requires-dist = [ { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2" }, { name = "xmlschema", marker = "extra == 'dev'" }, ] -provides-extras = ["dev"] +provides-extras = ["dev", "xdg"] [package.metadata.requires-dev] -dev = [{ name = "pytest", extras = ["dev"] }] +dev = [{ name = "pytest", extras = ["dev", "xdg"] }] [[package]] name = "pytest-xdist" From ff1a17aab15ee0a00b8e46f3e07046c56d3137f6 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 2 Aug 2026 08:06:06 +0200 Subject: [PATCH 04/12] cacheprovider: funnel cache directory resolution through one function Pure refactor, no behaviour change. `Cache.for_config` resolved the directory via `Cache.cache_dir_from_config`, which is also public API third-party plugins call; introducing `_resolve_cache_dir` gives both a single implementation to share, so the two cannot drift once the location grows more than one possible answer. Co-Authored-By: Claude Opus 5 (1M context) --- src/_pytest/cacheprovider.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index 6bcac1ad97a..4641877ea3c 100644 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -86,6 +86,14 @@ def _make_cachedir(target: Path) -> None: shutil.rmtree(path, ignore_errors=True) +def _resolve_cache_dir(config: Config) -> Path: + """Determine the cache directory for a Config. + + The single place where the cache directory location is decided. + """ + return resolve_from_str(config.getini("cache_dir"), config.rootpath) + + @final @dataclasses.dataclass class Cache: @@ -114,7 +122,7 @@ def for_config(cls, config: Config, *, _ispytest: bool = False) -> Cache: :meta private: """ check_ispytest(_ispytest) - cachedir = cls.cache_dir_from_config(config, _ispytest=True) + cachedir = _resolve_cache_dir(config) if config.getoption("cacheclear") and cachedir.is_dir(): cls.clear_cache(cachedir, _ispytest=True) return cls(cachedir, config, _ispytest=True) @@ -138,7 +146,7 @@ def cache_dir_from_config(config: Config, *, _ispytest: bool = False) -> Path: :meta private: """ check_ispytest(_ispytest) - return resolve_from_str(config.getini("cache_dir"), config.rootpath) + return _resolve_cache_dir(config) def warn(self, fmt: str, *, _ispytest: bool = False, **args: object) -> None: """Issue a cache warning. From 1ff5417c59e8be0077ba2e109d1e74aa7885ce83 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 2 Aug 2026 10:24:00 +0200 Subject: [PATCH 05/12] cacheprovider: introduce cache scopes Cached values are not all equally portable: `cache/lastfailed` and `cache/nodeids` depend on what the interpreter in use actually collects, while most plugin data does not care. Today the only way to express that is to move the whole cache directory, which is what the TOX_ENV_DIR special case does - multiplying entire cache trees, keyed off an environment variable belonging to one specific tool. CacheScope names the property directly, so interpreter matching happens inside the one cache directory belonging to the project rather than by having many directories. Scoped data lives under `s//{v,d}/`; the shared scope keeps the existing flat `v/` and `d/` layout, so existing caches and third-party plugins need no migration. `scope` is keyword-only and defaults to SHARED, so this commit changes no behaviour - nothing passes a non-default scope yet. PYTHON deliberately tracks major.minor only, so an in-place patch upgrade does not invalidate the cache, and ENV uses sys.prefix rather than sys.base_prefix because a venv is the unit users think in. Co-Authored-By: Claude Opus 5 (1M context) --- changelog/1089.feature.rst | 11 ++++ src/_pytest/cacheprovider.py | 118 +++++++++++++++++++++++++++++++--- src/pytest/__init__.py | 2 + testing/test_cacheprovider.py | 106 ++++++++++++++++++++++++++++++ 4 files changed, 228 insertions(+), 9 deletions(-) create mode 100644 changelog/1089.feature.rst diff --git a/changelog/1089.feature.rst b/changelog/1089.feature.rst new file mode 100644 index 00000000000..55689503280 --- /dev/null +++ b/changelog/1089.feature.rst @@ -0,0 +1,11 @@ +New :class:`pytest.CacheScope`, and a ``scope`` argument on :meth:`Cache.get `, +:meth:`Cache.set ` and :meth:`Cache.mkdir `. + +Cached data is not always valid everywhere a project is: last-failed test ids, for example, depend on +what the interpreter actually collects. ``scope`` lets a value be pinned to the running Python version +(:attr:`CacheScope.PYTHON `) or environment +(:attr:`CacheScope.ENV `) inside the project's single cache directory, instead of +needing a separate cache directory per environment. + +The default is :attr:`CacheScope.SHARED `, which behaves exactly as before, so +existing cache directories and plugins are unaffected. diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index 4641877ea3c..dd5177bc755 100644 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -8,11 +8,15 @@ from collections.abc import Generator from collections.abc import Iterable import dataclasses +import enum import errno +import hashlib import json import os from pathlib import Path +import re import shutil +import sys import tempfile from typing import final @@ -21,6 +25,7 @@ from .reports import CollectReport from _pytest import nodes from _pytest._io import TerminalWriter +from _pytest.compat import assert_never from _pytest.config import Config from _pytest.config import ExitCode from _pytest.config import hookimpl @@ -94,6 +99,68 @@ def _resolve_cache_dir(config: Config) -> Path: return resolve_from_str(config.getini("cache_dir"), config.rootpath) +class CacheScope(enum.Enum): + """How far a cached value travels. + + Cached data is not always valid everywhere the project is: last-failed test + ids, for instance, depend on what the interpreter in use actually collects. + Rather than giving each environment a whole cache directory of its own, + scoped values live in separate sub-directories of the one cache directory + belonging to the project. + + .. versionadded:: 9.0 + """ + + #: Valid for the project regardless of interpreter or environment. + SHARED = "shared" + #: Valid only for the running Python implementation and ``major.minor`` + #: version. The patch version is deliberately not part of this, so that an + #: in-place upgrade does not invalidate the cache. + PYTHON = "python" + #: Valid only for the running environment, i.e. :data:`sys.prefix`. + ENV = "env" + + +def _realpath_or_self(path: str) -> str: + try: + return os.path.realpath(path) + except OSError: + # E.g. a dead NFS mount. A stable-but-unresolved key beats crashing. + return path + + +_LABEL_UNSAFE = re.compile(r"[^A-Za-z0-9._-]+") + + +def _label(name: str, *, maxlen: int = 32) -> str: + """Make ``name`` safe and readable as a single path component. + + A whitelist rather than a blacklist, so spaces, separators, colons and + non-ASCII all collapse to ``-``. Leading dots are stripped so the result is + not hidden. Callers always combine this with a digest, so losing + information here is fine - and it also means Windows reserved device names + (``CON``, ``NUL``, ...) are harmless, since the label is never the whole + basename. + """ + return _LABEL_UNSAFE.sub("-", name)[:maxlen].strip("-.") or "root" + + +def _scope_id(scope: CacheScope) -> str | None: + """Return the sub-directory name for ``scope``, or None if unscoped.""" + if scope is CacheScope.SHARED: + return None + if scope is CacheScope.PYTHON: + major, minor = sys.version_info[:2] + return f"py-{_label(sys.implementation.name)}-{major}.{minor}" + if scope is CacheScope.ENV: + # normcase because on Windows `C:\Foo` and `c:\foo` are one directory + # and the casing varies with how the shell was launched. + prefix = os.path.normcase(_realpath_or_self(sys.prefix)) + digest = hashlib.sha256(prefix.encode("utf-8", "surrogatepass")).hexdigest() + return f"env-{_label(os.path.basename(prefix), maxlen=16)}-{digest[:8]}" + assert_never(scope) + + @final @dataclasses.dataclass class Cache: @@ -108,6 +175,10 @@ class Cache: # Sub-directory under cache-dir for values created by `set()`. _CACHE_PREFIX_VALUES = "v" + # Sub-directory under cache-dir holding one directory per non-shared + # CacheScope, each with its own `d` and `v` sub-directories. + _CACHE_PREFIX_SCOPES = "s" + def __init__( self, cachedir: Path, config: Config, *, _ispytest: bool = False ) -> None: @@ -134,7 +205,11 @@ def clear_cache(cls, cachedir: Path, _ispytest: bool = False) -> None: :meta private: """ check_ispytest(_ispytest) - for prefix in (cls._CACHE_PREFIX_DIRS, cls._CACHE_PREFIX_VALUES): + for prefix in ( + cls._CACHE_PREFIX_DIRS, + cls._CACHE_PREFIX_VALUES, + cls._CACHE_PREFIX_SCOPES, + ): d = cachedir / prefix if d.is_dir(): rm_rf(d) @@ -168,7 +243,13 @@ def _mkdir(self, path: Path) -> None: self._ensure_cache_dir_and_supporting_files() path.mkdir(exist_ok=True, parents=True) - def mkdir(self, name: str) -> Path: + def _scope_root(self, scope: CacheScope) -> Path: + scope_id = _scope_id(scope) + if scope_id is None: + return self._cachedir + return self._cachedir.joinpath(self._CACHE_PREFIX_SCOPES, scope_id) + + def mkdir(self, name: str, *, scope: CacheScope = CacheScope.SHARED) -> Path: """Return a directory path object with the given name. If the directory does not yet exist, it will be created. You can use @@ -181,18 +262,23 @@ def mkdir(self, name: str) -> Path: Must be a string not containing a ``/`` separator. Make sure the name contains your plugin or application identifiers to prevent clashes with other cache users. + :param scope: + How far the directory's contents travel; see :class:`CacheScope`. + Defaults to :attr:`CacheScope.SHARED`. + + .. versionadded:: 9.0 """ path = Path(name) if len(path.parts) > 1: raise ValueError("name is not allowed to contain path separators") - res = self._cachedir.joinpath(self._CACHE_PREFIX_DIRS, path) + res = self._scope_root(scope).joinpath(self._CACHE_PREFIX_DIRS, path) self._mkdir(res) return res - def _getvaluepath(self, key: str) -> Path: - return self._cachedir.joinpath(self._CACHE_PREFIX_VALUES, Path(key)) + def _getvaluepath(self, key: str, scope: CacheScope = CacheScope.SHARED) -> Path: + return self._scope_root(scope).joinpath(self._CACHE_PREFIX_VALUES, Path(key)) - def get(self, key: str, default): + def get(self, key: str, default, *, scope: CacheScope = CacheScope.SHARED): """Return the cached value for the given key. If no value was yet cached or the value cannot be read, the specified @@ -203,15 +289,23 @@ def get(self, key: str, default): name is the name of your plugin or your application. :param default: The value to return in case of a cache-miss or invalid cache value. + :param scope: + Which scope to read the value from; see :class:`CacheScope`. Must + match the scope it was written with. Defaults to + :attr:`CacheScope.SHARED`. + + .. versionadded:: 9.0 """ - path = self._getvaluepath(key) + path = self._getvaluepath(key, scope) try: with path.open("r", encoding="UTF-8") as f: return json.load(f) except (ValueError, OSError): return default - def set(self, key: str, value: object) -> None: + def set( + self, key: str, value: object, *, scope: CacheScope = CacheScope.SHARED + ) -> None: """Save value for the given key. :param key: @@ -220,8 +314,14 @@ def set(self, key: str, value: object) -> None: :param value: Must be of any combination of basic python types, including nested types like lists of dictionaries. + :param scope: + How far the value travels; see :class:`CacheScope`. Pin values + which are not valid across interpreters or environments, such as + collected test ids. Defaults to :attr:`CacheScope.SHARED`. + + .. versionadded:: 9.0 """ - path = self._getvaluepath(key) + path = self._getvaluepath(key, scope) try: self._mkdir(path.parent) except OSError as exc: diff --git a/src/pytest/__init__.py b/src/pytest/__init__.py index 6ff39f05a45..d5ba47ed2ac 100644 --- a/src/pytest/__init__.py +++ b/src/pytest/__init__.py @@ -10,6 +10,7 @@ from _pytest.approx import approx from _pytest.assertion import register_assert_rewrite from _pytest.cacheprovider import Cache +from _pytest.cacheprovider import CacheScope from _pytest.capture import CaptureFixture from _pytest.config import cmdline from _pytest.config import Config @@ -101,6 +102,7 @@ "HIDDEN_PARAM", "Approx", "Cache", + "CacheScope", "CallInfo", "CaptureFixture", "Class", diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py index 7ac3f38ab64..3b19384354b 100644 --- a/testing/test_cacheprovider.py +++ b/testing/test_cacheprovider.py @@ -7,8 +7,13 @@ import os from pathlib import Path import shutil +import sys from typing import Any +from _pytest.cacheprovider import _label +from _pytest.cacheprovider import _scope_id +from _pytest.cacheprovider import Cache +from _pytest.cacheprovider import CacheScope from _pytest.compat import assert_never from _pytest.config import ExitCode from _pytest.monkeypatch import MonkeyPatch @@ -1359,6 +1364,107 @@ def test_cachedir_tag(pytester: Pytester) -> None: assert cachedir_tag_path.read_bytes() == CACHEDIR_FILES["CACHEDIR.TAG"] +class TestCacheScopes: + @pytest.fixture + def cache(self, pytester: Pytester) -> Cache: + return Cache.for_config(pytester.parseconfig(), _ispytest=True) + + def test_shared_scope_keeps_the_flat_layout(self, cache: Cache) -> None: + # The shared scope must stay where it has always been, so that existing + # caches and third-party plugins need no migration. + cache.set("foo/bar", 1) + assert (cache._cachedir / "v" / "foo" / "bar").is_file() + assert not (cache._cachedir / "s").exists() + + @pytest.mark.parametrize("scope", [CacheScope.PYTHON, CacheScope.ENV]) + def test_scoped_values_round_trip(self, cache: Cache, scope: CacheScope) -> None: + cache.set("foo/bar", 1, scope=scope) + assert cache.get("foo/bar", None, scope=scope) == 1 + + scope_id = _scope_id(scope) + assert scope_id is not None + assert (cache._cachedir / "s" / scope_id / "v" / "foo" / "bar").is_file() + + def test_scopes_do_not_see_each_other(self, cache: Cache) -> None: + for scope in CacheScope: + cache.set("foo", scope.value, scope=scope) + for scope in CacheScope: + assert cache.get("foo", None, scope=scope) == scope.value + + def test_mkdir_is_scoped(self, cache: Cache) -> None: + shared = cache.mkdir("name") + scoped = cache.mkdir("name", scope=CacheScope.ENV) + assert shared.is_dir() and scoped.is_dir() + assert shared != scoped + + def test_mkdir_rejects_separators_in_any_scope(self, cache: Cache) -> None: + with pytest.raises(ValueError): + cache.mkdir("key/name", scope=CacheScope.ENV) + + def test_clear_cache_removes_scopes(self, cache: Cache) -> None: + cache.set("foo", 1) + cache.set("foo", 1, scope=CacheScope.ENV) + Cache.clear_cache(cache._cachedir, _ispytest=True) + assert not (cache._cachedir / "s").exists() + assert not (cache._cachedir / "v").exists() + # ... but the supporting files survive, as for `d` and `v` (#6290). + assert (cache._cachedir / "CACHEDIR.TAG").is_file() + + def test_scope_ids_are_stable(self) -> None: + assert _scope_id(CacheScope.SHARED) is None + for scope in (CacheScope.PYTHON, CacheScope.ENV): + assert _scope_id(scope) == _scope_id(scope) + + def test_python_scope_id_tracks_minor_version_only( + self, monkeypatch: MonkeyPatch + ) -> None: + before = _scope_id(CacheScope.PYTHON) + + # A patch release upgrade must not invalidate the cache. + major, minor, micro = sys.version_info[:3] + monkeypatch.setattr(sys, "version_info", (major, minor, micro + 1)) + assert _scope_id(CacheScope.PYTHON) == before + + monkeypatch.setattr(sys, "version_info", (major, minor + 1, 0)) + assert _scope_id(CacheScope.PYTHON) != before + + def test_env_scope_id_tracks_sys_prefix(self, monkeypatch: MonkeyPatch) -> None: + monkeypatch.setattr(sys, "prefix", "/somewhere/one/.venv") + one = _scope_id(CacheScope.ENV) + monkeypatch.setattr(sys, "prefix", "/somewhere/two/.venv") + two = _scope_id(CacheScope.ENV) + + assert one != two + # Both stay readable, and the leading dot is stripped so they are not + # hidden directories. + assert one is not None and two is not None + assert one.startswith("env-venv-") and two.startswith("env-venv-") + + def test_env_scope_id_ignores_python_version( + self, monkeypatch: MonkeyPatch + ) -> None: + before = _scope_id(CacheScope.ENV) + monkeypatch.setattr(sys, "version_info", (99, 9, 9)) + assert _scope_id(CacheScope.ENV) == before + + @pytest.mark.parametrize( + ("name", "expected"), + [ + ("myproj", "myproj"), + ("my project (v2)", "my-project-v2"), + (".hidden", "hidden"), + ("with/sep", "with-sep"), + ("ünïcode", "n-code"), + ("", "root"), + ("/", "root"), + ("!!!", "root"), + ("x" * 60, "x" * 32), + ], + ) + def test_label_sanitisation(self, name: str, expected: str) -> None: + assert _label(name) == expected + + def test_clioption_with_cacheshow_and_help(pytester: Pytester) -> None: result = pytester.runpytest("--cache-show", "--help") assert result.ret == 0 From 6e62687829cd1ac9f43fb18b6a1b3e91437bb237 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 2 Aug 2026 18:46:13 +0200 Subject: [PATCH 06/12] cacheprovider: pin lastfailed, nodeids and stepwise to the env scope Which tests an interpreter collects is not portable, so `--lf`, `--nf` and `--sw` state must not be shared between environments. Now that CacheScope exists, say so directly instead of relying on the cache directory having been moved out from under us. Running one project under several environments no longer has each run overwrite the previous one's last-failed set. That is what the TOX_ENV_DIR special case was for, and it now happens for every environment rather than only for tox. `--cache-show` learns to walk the scope directories too, reading values through the path rather than Cache.get so that scopes belonging to other environments are listed as well, tagged with the scope they came from. Existing entries are not migrated; the first run after upgrading behaves as if the cache were empty. test_stepwise.py's `cache_dir = .cache` workaround, added because tox's cache directory made the module flaky, is no longer needed. Co-Authored-By: Claude Opus 5 (1M context) --- changelog/1089.improvement.2.rst | 9 +++ src/_pytest/cacheprovider.py | 81 +++++++++++++++++++-------- src/_pytest/stepwise.py | 11 +++- testing/test_cacheprovider.py | 94 ++++++++++++++++++++++++++++---- testing/test_stepwise.py | 47 +++++++++------- 5 files changed, 185 insertions(+), 57 deletions(-) create mode 100644 changelog/1089.improvement.2.rst diff --git a/changelog/1089.improvement.2.rst b/changelog/1089.improvement.2.rst new file mode 100644 index 00000000000..211679896c7 --- /dev/null +++ b/changelog/1089.improvement.2.rst @@ -0,0 +1,9 @@ +The cache entries backing ``--lf``/``--ff``, ``--nf`` and ``--sw`` are now pinned to the environment they +were recorded in, using the new :attr:`CacheScope.ENV ` scope. + +Running the same project under several environments - a tox or nox matrix, or simply two virtualenvs - no +longer has each run overwrite the previous one's last-failed set, without needing a separate cache +directory per environment. + +Existing ``lastfailed``, ``nodeids`` and ``stepwise`` cache entries are not migrated, so the first run +after upgrading behaves as if the cache were empty. diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index dd5177bc755..b0ff921a048 100644 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -426,7 +426,9 @@ def __init__(self, config: Config) -> None: active_keys = "lf", "failedfirst" self.active = any(config.getoption(key) for key in active_keys) assert config.cache - self.lastfailed: dict[str, bool] = config.cache.get("cache/lastfailed", {}) + self.lastfailed: dict[str, bool] = config.cache.get( + "cache/lastfailed", {}, scope=CacheScope.ENV + ) self._previously_failed_count: int | None = None self._report_status: str | None = None self._skipped_files = 0 # count skipped files during collection due to --lf @@ -526,9 +528,11 @@ def pytest_sessionfinish(self, session: Session) -> None: return assert config.cache is not None - saved_lastfailed = config.cache.get("cache/lastfailed", {}) + saved_lastfailed = config.cache.get( + "cache/lastfailed", {}, scope=CacheScope.ENV + ) if saved_lastfailed != self.lastfailed: - config.cache.set("cache/lastfailed", self.lastfailed) + config.cache.set("cache/lastfailed", self.lastfailed, scope=CacheScope.ENV) class NFPlugin: @@ -538,7 +542,9 @@ def __init__(self, config: Config) -> None: self.config = config self.active = config.option.newfirst assert config.cache is not None - self.cached_nodeids = set(config.cache.get("cache/nodeids", [])) + self.cached_nodeids = set( + config.cache.get("cache/nodeids", [], scope=CacheScope.ENV) + ) @hookimpl(wrapper=True, tryfirst=True) def pytest_collection_modifyitems(self, items: list[nodes.Item]) -> Generator[None]: @@ -574,7 +580,9 @@ def pytest_sessionfinish(self) -> None: return assert config.cache is not None - config.cache.set("cache/nodeids", sorted(self.cached_nodeids)) + config.cache.set( + "cache/nodeids", sorted(self.cached_nodeids), scope=CacheScope.ENV + ) def pytest_addoption(parser: Parser) -> None: @@ -696,6 +704,20 @@ def pytest_report_header(config: Config) -> str | None: return None +def _cache_roots(basedir: Path) -> list[tuple[str | None, Path]]: + """Yield ``(scope_id, root)`` for the shared scope and every scope present. + + ``scope_id`` is None for the shared scope. Scopes are read off the + filesystem rather than from :class:`CacheScope`, so that scopes belonging + to other environments are included. + """ + roots: list[tuple[str | None, Path]] = [(None, basedir)] + scopesdir = basedir / Cache._CACHE_PREFIX_SCOPES + if scopesdir.is_dir(): + roots.extend((p.name, p) for p in sorted(scopesdir.iterdir()) if p.is_dir()) + return roots + + def cacheshow(config: Config, session: Session) -> int: """Display cache contents when --cache-show is used. @@ -723,26 +745,37 @@ def cacheshow(config: Config, session: Session) -> int: dummy = object() basedir = config.cache._cachedir - vdir = basedir / Cache._CACHE_PREFIX_VALUES tw.sep("-", f"cache values for {glob!r}") - for valpath in sorted(x for x in vdir.rglob(glob) if x.is_file()): - key = str(valpath.relative_to(vdir)) - val = config.cache.get(key, dummy) - if val is dummy: - tw.line(f"{key} contains unreadable content, will be ignored") - else: - tw.line(f"{key} contains:") - for line in pformat(val).splitlines(): - tw.line(" " + line) + for scope_id, root in _cache_roots(basedir): + vdir = root / Cache._CACHE_PREFIX_VALUES + if not vdir.is_dir(): + continue + for valpath in sorted(x for x in vdir.rglob(glob) if x.is_file()): + key = str(valpath.relative_to(vdir)) + if scope_id is not None: + key = f"{key} ({scope_id})" + # Read through the path rather than Cache.get, so that scopes + # belonging to other environments are shown too. + try: + with valpath.open("r", encoding="UTF-8") as f: + val = json.load(f) + except (ValueError, OSError): + val = dummy + if val is dummy: + tw.line(f"{key} contains unreadable content, will be ignored") + else: + tw.line(f"{key} contains:") + for line in pformat(val).splitlines(): + tw.line(" " + line) - ddir = basedir / Cache._CACHE_PREFIX_DIRS - if ddir.is_dir(): - contents = sorted(ddir.rglob(glob)) + ddirs = [root / Cache._CACHE_PREFIX_DIRS for _, root in _cache_roots(basedir)] + if any(ddir.is_dir() for ddir in ddirs): tw.sep("-", f"cache directories for {glob!r}") - for p in contents: - # if p.is_dir(): - # print("%s/" % p.relative_to(basedir)) - if p.is_file(): - key = str(p.relative_to(basedir)) - tw.line(f"{key} is a file of length {p.stat().st_size}") + for ddir in ddirs: + if not ddir.is_dir(): + continue + for p in sorted(ddir.rglob(glob)): + if p.is_file(): + key = str(p.relative_to(basedir)) + tw.line(f"{key} is a file of length {p.stat().st_size}") return 0 diff --git a/src/_pytest/stepwise.py b/src/_pytest/stepwise.py index 8901540eb59..5afea883386 100644 --- a/src/_pytest/stepwise.py +++ b/src/_pytest/stepwise.py @@ -8,6 +8,7 @@ from _pytest import nodes from _pytest.cacheprovider import Cache +from _pytest.cacheprovider import CacheScope from _pytest.config import Config from _pytest.config.argparsing import Parser from _pytest.main import Session @@ -108,7 +109,9 @@ def __init__(self, config: Config) -> None: self.cached_info = self._load_cached_info() def _load_cached_info(self) -> StepwiseCacheInfo: - cached_dict: dict[str, Any] | None = self.cache.get(STEPWISE_CACHE_DIR, None) + cached_dict: dict[str, Any] | None = self.cache.get( + STEPWISE_CACHE_DIR, None, scope=CacheScope.ENV + ) if cached_dict: try: return StepwiseCacheInfo( @@ -206,4 +209,8 @@ def pytest_sessionfinish(self) -> None: # race conditions (#10641). return self.cached_info.update_date_to_now() - self.cache.set(STEPWISE_CACHE_DIR, dataclasses.asdict(self.cached_info)) + self.cache.set( + STEPWISE_CACHE_DIR, + dataclasses.asdict(self.cached_info), + scope=CacheScope.ENV, + ) diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py index 3b19384354b..fa3193587a6 100644 --- a/testing/test_cacheprovider.py +++ b/testing/test_cacheprovider.py @@ -25,6 +25,13 @@ pytest_plugins = ("pytester",) +def env_scope_values(cachedir: str | Path = ".pytest_cache") -> Path: + """Path of the env-scoped value directory inside ``cachedir``.""" + scope_id = _scope_id(CacheScope.ENV) + assert scope_id is not None + return Path(cachedir, "s", scope_id, "v") + + class TestNewAPI: def test_config_cache_mkdir(self, pytester: Pytester) -> None: pytester.makeini("[pytest]") @@ -117,8 +124,8 @@ def test_cache_failure_warns( "*= warnings summary =*", "*/cacheprovider.py:*", " */cacheprovider.py:*: PytestCacheWarning: could not create cache path " - f"{unwritable_cache_dir}/v/cache/nodeids: *", - ' config.cache.set("cache/nodeids", sorted(self.cached_nodeids))', + f"{env_scope_values(unwritable_cache_dir)}/cache/nodeids: *", + " config.cache.set(", "*1 failed, 2 warnings in*", ] ) @@ -257,11 +264,13 @@ def pytest_configure(config): [ "*cachedir:*", "*- cache values for '[*]' -*", - "cache/nodeids contains:", "my/name contains:", " [1, 2, 3]", "other/some contains:", " {*'1': 2}", + # Env-scoped values are listed after the shared ones, tagged with + # the scope they belong to. + "cache/nodeids (env-*) contains:", "*- cache directories for '[*]' -*", "*mydb/hello*length 0*", "*mydb/world*length 0*", @@ -424,7 +433,7 @@ def test_hello(): ) config = pytester.parseconfigure() assert config.cache is not None - lastfailed = config.cache.get("cache/lastfailed", -1) + lastfailed = config.cache.get("cache/lastfailed", -1, scope=CacheScope.ENV) assert lastfailed == -1 def test_non_serializable_parametrize(self, pytester: Pytester) -> None: @@ -543,7 +552,7 @@ def rlf(fail_import: int, fail_run: int) -> Any: pytester.runpytest("-q") config = pytester.parseconfigure() assert config.cache is not None - lastfailed = config.cache.get("cache/lastfailed", -1) + lastfailed = config.cache.get("cache/lastfailed", -1, scope=CacheScope.ENV) return lastfailed lastfailed = rlf(fail_import=0, fail_run=0) @@ -593,7 +602,7 @@ def rlf( result = pytester.runpytest("-q", "--lf", *args) config = pytester.parseconfigure() assert config.cache is not None - lastfailed = config.cache.get("cache/lastfailed", -1) + lastfailed = config.cache.get("cache/lastfailed", -1, scope=CacheScope.ENV) return result, lastfailed result, lastfailed = rlf(fail_import=0, fail_run=0) @@ -615,17 +624,19 @@ def rlf( def test_lastfailed_creates_cache_when_needed(self, pytester: Pytester) -> None: # Issue #1342 + lastfailed = env_scope_values() / "cache" / "lastfailed" + pytester.makepyfile(test_empty="") pytester.runpytest("-q", "--lf") - assert not os.path.exists(".pytest_cache/v/cache/lastfailed") + assert not lastfailed.exists() pytester.makepyfile(test_successful="def test_success():\n assert True") pytester.runpytest("-q", "--lf") - assert not os.path.exists(".pytest_cache/v/cache/lastfailed") + assert not lastfailed.exists() pytester.makepyfile(test_errored="def test_error():\n assert False") pytester.runpytest("-q", "--lf") - assert os.path.exists(".pytest_cache/v/cache/lastfailed") + assert lastfailed.exists() def test_xfail_not_considered_failure(self, pytester: Pytester) -> None: pytester.makepyfile( @@ -703,7 +714,7 @@ def test_lf_and_ff_prints_no_needless_message( def get_cached_last_failed(self, pytester: Pytester) -> list[str]: config = pytester.parseconfigure() assert config.cache is not None - return sorted(config.cache.get("cache/lastfailed", {})) + return sorted(config.cache.get("cache/lastfailed", {}, scope=CacheScope.ENV)) def test_cache_cumulative(self, pytester: Pytester) -> None: """Test workflow where user fixes errors gradually file by file using --lf.""" @@ -1364,6 +1375,69 @@ def test_cachedir_tag(pytester: Pytester) -> None: assert cachedir_tag_path.read_bytes() == CACHEDIR_FILES["CACHEDIR.TAG"] +class TestEnvScopedBuiltins: + """The built-in cache keys are pinned to the environment. + + Which environment collected which tests is not portable, so `--lf`, `--nf` + and `--sw` state must not be shared between them. Previously the only way + to get that was to move the whole cache directory, which is what the + TOX_ENV_DIR special case does. + """ + + def test_lastfailed_lives_in_the_env_scope(self, pytester: Pytester) -> None: + pytester.makepyfile(test_a="def test_error(): assert False") + pytester.runpytest("-q") + + assert (env_scope_values() / "cache" / "lastfailed").is_file() + assert not (Path(".pytest_cache") / "v" / "cache" / "lastfailed").exists() + + def test_nodeids_lives_in_the_env_scope(self, pytester: Pytester) -> None: + pytester.makepyfile(test_a="def test_ok(): pass") + pytester.runpytest("-q") + + assert (env_scope_values() / "cache" / "nodeids").is_file() + assert not (Path(".pytest_cache") / "v" / "cache" / "nodeids").exists() + + def test_environments_do_not_clobber_each_other( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + pytester.makepyfile( + test_a=""" + import os + def test_one(): assert not os.environ.get("FAIL_ONE") + def test_two(): assert not os.environ.get("FAIL_TWO") + """ + ) + # Two runs which fail different tests, as two different environments. + # Patched at conftest import time, i.e. before cacheprovider's + # tryfirst pytest_configure builds the Cache. + pytester.makeconftest( + """ + import os, sys + sys.prefix = os.environ["FAKE_PREFIX"] + """ + ) + + monkeypatch.setenv("FAKE_PREFIX", str(pytester.path / "venv-one")) + monkeypatch.setenv("FAIL_ONE", "1") + pytester.runpytest_subprocess("-q").assert_outcomes(passed=1, failed=1) + + monkeypatch.delenv("FAIL_ONE") + monkeypatch.setenv("FAKE_PREFIX", str(pytester.path / "venv-two")) + monkeypatch.setenv("FAIL_TWO", "1") + pytester.runpytest_subprocess("-q").assert_outcomes(passed=1, failed=1) + + # Each environment still remembers its own failure, rather than the + # second run having overwritten the first. + monkeypatch.delenv("FAIL_TWO") + monkeypatch.setenv("FAKE_PREFIX", str(pytester.path / "venv-one")) + result = pytester.runpytest_subprocess("--lf", "-v") + result.stdout.fnmatch_lines( + ["*rerun previous 1 failure*", "*test_a.py::test_one*PASSED*"] + ) + result.assert_outcomes(passed=1) + + class TestCacheScopes: @pytest.fixture def cache(self, pytester: Pytester) -> Cache: diff --git a/testing/test_stepwise.py b/testing/test_stepwise.py index d2ad3bae500..13893f3f2a9 100644 --- a/testing/test_stepwise.py +++ b/testing/test_stepwise.py @@ -5,13 +5,32 @@ import json from pathlib import Path +from _pytest.cacheprovider import _scope_id from _pytest.cacheprovider import Cache +from _pytest.cacheprovider import CacheScope from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import Pytester from _pytest.stepwise import STEPWISE_CACHE_DIR import pytest +def stepwise_cache_file(cachedir: Path) -> Path: + """Path of the stepwise cache value inside ``cachedir``. + + Stepwise state is pinned to the environment, so it lives in a scope + sub-directory rather than directly under ``v/``. + """ + scope_id = _scope_id(CacheScope.ENV) + assert scope_id is not None + return ( + cachedir + / Cache._CACHE_PREFIX_SCOPES + / scope_id + / Cache._CACHE_PREFIX_VALUES + / STEPWISE_CACHE_DIR + ) + + @pytest.fixture def stepwise_pytester(pytester: Pytester) -> Pytester: # Rather than having to modify our testfile between tests, we introduce @@ -52,14 +71,6 @@ def test_success(): """ ) - # customize cache directory so we don't use the tox's cache directory, which makes tests in this module flaky - pytester.makeini( - """ - [pytest] - cache_dir = .cache - """ - ) - return pytester @@ -316,10 +327,7 @@ def test_one(): result = pytester.runpytest("--stepwise") assert result.ret == pytest.ExitCode.INTERRUPTED - stepwise_cache_file = ( - pytester.path / Cache._CACHE_PREFIX_VALUES / STEPWISE_CACHE_DIR - ) - assert not Path(stepwise_cache_file).exists() + assert not stepwise_cache_file(pytester.path).exists() def test_disabled_stepwise_xdist_dont_clear_cache(pytester: Pytester) -> None: @@ -328,13 +336,10 @@ def test_disabled_stepwise_xdist_dont_clear_cache(pytester: Pytester) -> None: pytest=f"[pytest]\ncache_dir = {pytester.path}\n", ) - stepwise_cache_file = ( - pytester.path / Cache._CACHE_PREFIX_VALUES / STEPWISE_CACHE_DIR - ) - stepwise_cache_dir = stepwise_cache_file.parent - stepwise_cache_dir.mkdir(exist_ok=True, parents=True) + cache_file = stepwise_cache_file(pytester.path) + cache_file.parent.mkdir(exist_ok=True, parents=True) - stepwise_cache_file_relative = f"{Cache._CACHE_PREFIX_VALUES}/{STEPWISE_CACHE_DIR}" + stepwise_cache_file_relative = cache_file.relative_to(pytester.path).as_posix() expected_value = '"test_one.py::test_one"' content = {f"{stepwise_cache_file_relative}": expected_value} @@ -359,8 +364,8 @@ def test_one(): result = pytester.runpytest() assert result.ret == 0 - assert Path(stepwise_cache_file).exists() - with stepwise_cache_file.open(encoding="utf-8") as file_handle: + assert cache_file.exists() + with cache_file.open(encoding="utf-8") as file_handle: observed_value = file_handle.readlines() assert [expected_value] == observed_value @@ -529,7 +534,7 @@ def test_1(): pass ) # Corrupt the cache. - cache_file = pytester.path / f".pytest_cache/v/{STEPWISE_CACHE_DIR}" + cache_file = stepwise_cache_file(pytester.path / ".pytest_cache") assert cache_file.is_file() cache_file.write_text(json.dumps({"invalid": True}), encoding="UTF-8") From 6f88b98808765954f071fc55b652a48fbbbe59e0 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Mon, 3 Aug 2026 10:39:05 +0200 Subject: [PATCH 07/12] cacheprovider: record cache metadata Writes `cache-info.json` at the top level of every cache directory: where the directory came from, when it was created and last used, and which scopes live in it. This is the backlink that makes a cache directory self-describing. Once a cache can live outside the project - which is the point of the next commit - its lifetime is no longer tied to the worktree's, so listing and pruning need something on disk that says what a directory belongs to. Recording scopes as well means a stale environment can be collected without touching the rest of the project's cache. Deliberate choices: - Top level, so `--cache-show`'s globbing never sees it and `--cache-clear` preserves it, exactly as for README.md and CACHEDIR.TAG (GH-6290). Clearing a cache must not make it anonymous. - Not dot-prefixed: someone browsing a cache directory far from the project it belongs to needs to see what it is for. - `origin` records paths unresolved, i.e. as the user sees them, since that is what gets displayed. - Epoch floats rather than ISO-8601, so a hand-edited timestamp cannot raise mid-listing. Rendering them for humans is the listing's job. - Unknown keys are preserved on rewrite, so a newer pytest's fields survive an older pytest touching the same directory. - Write failures are silent, unlike `set()`: the metadata only feeds listing and pruning, and whatever made it fail will have made the value writes warn already. A second warning for one cause is noise. Creation goes through _make_cachedir's existing temp-and-rename, so the file appears atomically with the rest; refreshes use their own same-directory temp-and-replace. Laziness is preserved - a run which never touches the cache still creates nothing. Co-Authored-By: Claude Opus 5 (1M context) --- src/_pytest/cacheprovider.py | 123 +++++++++++++++++++++++++-- testing/test_cacheprovider.py | 152 +++++++++++++++++++++++++++++++--- 2 files changed, 257 insertions(+), 18 deletions(-) diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index b0ff921a048..3f2983b8ee4 100644 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -7,6 +7,7 @@ from collections.abc import Generator from collections.abc import Iterable +from collections.abc import Mapping import dataclasses import enum import errno @@ -18,11 +19,14 @@ import shutil import sys import tempfile +import time +from typing import Any from typing import final from .pathlib import resolve_from_str from .pathlib import rm_rf from .reports import CollectReport +from _pytest import __version__ from _pytest import nodes from _pytest._io import TerminalWriter from _pytest.compat import assert_never @@ -60,12 +64,15 @@ } -def _make_cachedir(target: Path) -> None: +def _make_cachedir( + target: Path, extra_files: Mapping[str, bytes] | None = None +) -> None: """Create the pytest cache directory atomically with supporting files. Creates a temporary directory with README.md, .gitignore, and CACHEDIR.TAG, - then atomically renames it to the target location. If another process wins - the race, the temporary directory is cleaned up. + plus any ``extra_files``, then atomically renames it to the target + location. If another process wins the race, the temporary directory is + cleaned up. """ target.parent.mkdir(parents=True, exist_ok=True) path = Path(tempfile.mkdtemp(prefix="pytest-cache-files-", dir=target.parent)) @@ -76,7 +83,7 @@ def _make_cachedir(target: Path) -> None: os.umask(umask) path.chmod(0o777 - umask) - for name, content in CACHEDIR_FILES.items(): + for name, content in {**CACHEDIR_FILES, **(extra_files or {})}.items(): path.joinpath(name).write_bytes(content) path.rename(target) @@ -161,6 +168,62 @@ def _scope_id(scope: CacheScope) -> str | None: assert_never(scope) +def _scope_info(scope: CacheScope) -> dict[str, str]: + """Describe ``scope`` for the cache metadata.""" + major, minor = sys.version_info[:2] + info = { + "scope": scope.value, + "python": f"{sys.implementation.name}-{major}.{minor}", + } + if scope is CacheScope.ENV: + # Recorded unresolved, i.e. as the user sees it, since this is what + # gets shown when listing caches. + info["prefix"] = sys.prefix + return info + + +#: Name of the metadata file written at the top level of a cache directory. +#: Not dot-prefixed: someone browsing a cache directory far from the project it +#: belongs to needs to be able to see what it is for. +CACHE_INFO_NAME = "cache-info.json" + +#: Version of the `cache-info.json` format. Readers must tolerate a missing, +#: unparsable or newer file, and still offer to remove the directory. +CACHE_INFO_SCHEMA = 1 + + +def _now() -> float: + """Indirection so that tests can freeze time.""" + return time.time() + + +def _write_json_atomic(path: Path, data: object) -> None: + payload = json.dumps(data, ensure_ascii=False, indent=2) + # A temp file in the same directory guarantees os.replace is atomic, as it + # is then guaranteed to be on the same filesystem. + fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f"{path.name}-", suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="UTF-8") as f: + f.write(payload) + os.replace(tmp, path) + except BaseException: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +def read_cache_info(cachedir: Path) -> dict[str, Any] | None: + """Read a cache directory's metadata, or None if it has none readable.""" + try: + with (cachedir / CACHE_INFO_NAME).open("r", encoding="UTF-8") as f: + data = json.load(f) + except (ValueError, OSError): + return None + return data if isinstance(data, dict) else None + + @final @dataclasses.dataclass class Cache: @@ -185,6 +248,10 @@ def __init__( check_ispytest(_ispytest) self._cachedir = cachedir self._config = config + # Scopes touched this session, and the set already recorded in the + # metadata file, so a scope used later still gets recorded. + self._used_scopes: dict[str, dict[str, str]] = {} + self._recorded_scopes: frozenset[str] | None = None @classmethod def for_config(cls, config: Config, *, _ispytest: bool = False) -> Cache: @@ -247,6 +314,7 @@ def _scope_root(self, scope: CacheScope) -> Path: scope_id = _scope_id(scope) if scope_id is None: return self._cachedir + self._used_scopes[scope_id] = _scope_info(scope) return self._cachedir.joinpath(self._CACHE_PREFIX_SCOPES, scope_id) def mkdir(self, name: str, *, scope: CacheScope = CacheScope.SHARED) -> Path: @@ -342,10 +410,53 @@ def set( with f: f.write(data) + def _cache_info(self, previous: dict[str, Any] | None) -> dict[str, Any]: + """Build the metadata to record, merged over ``previous`` if any. + + Unknown keys in ``previous`` are preserved, so that a newer pytest's + fields survive an older pytest touching the same directory. + """ + now = _now() + info: dict[str, Any] = dict(previous) if previous else {} + info["schema"] = CACHE_INFO_SCHEMA + info["origin"] = { + "rootdir": str(self._config.rootpath), + "inipath": str(self._config.inipath) if self._config.inipath else None, + } + info["pytest_version"] = __version__ + info.setdefault("created_at", now) + info["last_used_at"] = now + + recorded = info.get("scopes") + scopes: dict[str, Any] = dict(recorded) if isinstance(recorded, dict) else {} + for scope_id, scope_info in self._used_scopes.items(): + scopes[scope_id] = {**scope_info, "last_used_at": now} + info["scopes"] = scopes + return info + def _ensure_cache_dir_and_supporting_files(self) -> None: - """Create the cache dir and its supporting files.""" + """Create the cache dir, its supporting files and its metadata.""" + used_scopes = frozenset(self._used_scopes) if not self._cachedir.is_dir(): - _make_cachedir(self._cachedir) + info = json.dumps(self._cache_info(None), ensure_ascii=False, indent=2) + _make_cachedir(self._cachedir, {CACHE_INFO_NAME: info.encode("UTF-8")}) + elif self._recorded_scopes != used_scopes: + # Either the metadata has not been refreshed this session yet, or a + # scope has been used since it last was. Note this also backfills + # the file into cache directories created by an older pytest. + try: + _write_json_atomic( + self._cachedir / CACHE_INFO_NAME, + self._cache_info(read_cache_info(self._cachedir)), + ) + except OSError: + # Deliberately silent, unlike `set()`. The metadata only feeds + # listing and pruning, so failing to write it costs the user + # nothing they asked for - and whatever made it fail will have + # made the actual cache writes warn already. Such a directory + # simply lists as having no metadata. + pass + self._recorded_scopes = used_scopes class LFPluginCollWrapper: diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py index fa3193587a6..462bb756596 100644 --- a/testing/test_cacheprovider.py +++ b/testing/test_cacheprovider.py @@ -4,16 +4,21 @@ from collections.abc import Sequence from enum import auto from enum import Enum +import json import os from pathlib import Path import shutil import sys from typing import Any +from _pytest import cacheprovider from _pytest.cacheprovider import _label from _pytest.cacheprovider import _scope_id from _pytest.cacheprovider import Cache +from _pytest.cacheprovider import CACHE_INFO_NAME +from _pytest.cacheprovider import CACHE_INFO_SCHEMA from _pytest.cacheprovider import CacheScope +from _pytest.cacheprovider import read_cache_info from _pytest.compat import assert_never from _pytest.config import ExitCode from _pytest.monkeypatch import MonkeyPatch @@ -25,6 +30,19 @@ pytest_plugins = ("pytester",) +@pytest.fixture +def unwritable_cache_dir(pytester: Pytester) -> Generator[Path]: + cache_dir = pytester.path.joinpath(".pytest_cache") + cache_dir.mkdir() + mode = cache_dir.stat().st_mode + cache_dir.chmod(0) + if os.access(cache_dir, os.W_OK): + pytest.skip("Failed to make cache dir unwritable") + + yield cache_dir + cache_dir.chmod(mode) + + def env_scope_values(cachedir: str | Path = ".pytest_cache") -> Path: """Path of the env-scoped value directory inside ``cachedir``.""" scope_id = _scope_id(CacheScope.ENV) @@ -81,18 +99,6 @@ def test_cache_writefail_cachefile_silent(self, pytester: Pytester) -> None: assert cache is not None cache.set("test/broken", []) - @pytest.fixture - def unwritable_cache_dir(self, pytester: Pytester) -> Generator[Path]: - cache_dir = pytester.path.joinpath(".pytest_cache") - cache_dir.mkdir() - mode = cache_dir.stat().st_mode - cache_dir.chmod(0) - if os.access(cache_dir, os.W_OK): - pytest.skip("Failed to make cache dir unwritable") - - yield cache_dir - cache_dir.chmod(mode) - @pytest.mark.filterwarnings( "ignore:could not create cache path:pytest.PytestWarning" ) @@ -1375,6 +1381,128 @@ def test_cachedir_tag(pytester: Pytester) -> None: assert cachedir_tag_path.read_bytes() == CACHEDIR_FILES["CACHEDIR.TAG"] +class TestCacheInfo: + @pytest.fixture + def cache(self, pytester: Pytester) -> Cache: + return Cache.for_config(pytester.parseconfig(), _ispytest=True) + + def info(self, cache: Cache) -> dict[str, Any]: + info = read_cache_info(cache._cachedir) + assert info is not None + return info + + def test_written_on_creation(self, pytester: Pytester) -> None: + pytester.makeini("[pytest]") + cache = Cache.for_config(pytester.parseconfig(), _ispytest=True) + cache.set("foo", 1) + info = self.info(cache) + + assert info["schema"] == CACHE_INFO_SCHEMA + assert info["origin"] == { + "rootdir": str(pytester.path), + "inipath": str(pytester.path / "tox.ini"), + } + assert info["pytest_version"] == pytest.__version__ + assert info["created_at"] == info["last_used_at"] + assert info["scopes"] == {} + + def test_origin_inipath_is_null_without_a_config_file(self, cache: Cache) -> None: + cache.set("foo", 1) + assert self.info(cache)["origin"]["inipath"] is None + + def test_not_written_when_cache_is_unused(self, cache: Cache) -> None: + # A run which never writes to the cache must still not create it. + assert not cache._cachedir.exists() + + def test_records_scopes_as_they_are_used(self, cache: Cache) -> None: + cache.set("foo", 1) + assert self.info(cache)["scopes"] == {} + + cache.set("foo", 1, scope=CacheScope.ENV) + scopes = self.info(cache)["scopes"] + scope_id = _scope_id(CacheScope.ENV) + assert set(scopes) == {scope_id} + assert scopes[scope_id]["scope"] == "env" + assert scopes[scope_id]["prefix"] == sys.prefix + + cache.set("foo", 1, scope=CacheScope.PYTHON) + assert set(self.info(cache)["scopes"]) == { + scope_id, + _scope_id(CacheScope.PYTHON), + } + + def test_last_used_at_refreshed_but_created_at_kept( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + monkeypatch.setattr(cacheprovider, "_now", lambda: 1000.0) + first = Cache.for_config(pytester.parseconfig(), _ispytest=True) + first.set("foo", 1) + + monkeypatch.setattr(cacheprovider, "_now", lambda: 2000.0) + second = Cache.for_config(pytester.parseconfig(), _ispytest=True) + second.set("foo", 2) + + info = self.info(second) + assert info["created_at"] == 1000.0 + assert info["last_used_at"] == 2000.0 + + def test_preserves_unknown_keys(self, cache: Cache) -> None: + # A newer pytest's fields must survive an older pytest touching the + # same directory. + cache.set("foo", 1) + path = cache._cachedir / CACHE_INFO_NAME + info = json.loads(path.read_text(encoding="UTF-8")) + info["from_the_future"] = {"hello": "world"} + path.write_text(json.dumps(info), encoding="UTF-8") + + cache.set("foo", 1, scope=CacheScope.ENV) + assert self.info(cache)["from_the_future"] == {"hello": "world"} + + def test_backfilled_into_a_preexisting_dir(self, cache: Cache) -> None: + cache.set("foo", 1) + (cache._cachedir / CACHE_INFO_NAME).unlink() + + later = Cache.for_config(cache._config, _ispytest=True) + later.set("foo", 2) + assert self.info(later)["schema"] == CACHE_INFO_SCHEMA + + def test_survives_cache_clear(self, pytester: Pytester) -> None: + # Like README.md and CACHEDIR.TAG, the metadata is a supporting file: + # clearing a cache must not make it anonymous (#6290). + pytester.makepyfile(test_a="def test_error(): assert False") + pytester.runpytest("-q") + cachedir = pytester.path / ".pytest_cache" + before = read_cache_info(cachedir) + assert before is not None + + pytester.runpytest("-q", "--cache-clear") + + after = read_cache_info(cachedir) + assert after is not None + assert after["created_at"] == before["created_at"] + + def test_unreadable_metadata_is_tolerated(self, cache: Cache) -> None: + cache.set("foo", 1) + (cache._cachedir / CACHE_INFO_NAME).write_text("{not json", encoding="UTF-8") + assert read_cache_info(cache._cachedir) is None + + # ... and gets rewritten rather than making the run fail. + later = Cache.for_config(cache._config, _ispytest=True) + later.set("foo", 2) + assert self.info(later)["schema"] == CACHE_INFO_SCHEMA + + @pytest.mark.filterwarnings("default") + def test_write_failure_is_silent( + self, pytester: Pytester, unwritable_cache_dir: Path + ) -> None: + # The value writes warn about the same cause already; a second warning + # for the metadata would be noise. + pytester.makepyfile(test_a="def test_ok(): pass") + result = pytester.runpytest() + assert result.ret == 0 + result.stdout.no_fnmatch_line("*cache metadata*") + + class TestEnvScopedBuiltins: """The built-in cache keys are pinned to the environment. From fe9960581c731558f54531207967fc1042e0e51e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 5 Aug 2026 09:43:39 +0200 Subject: [PATCH 08/12] cacheprovider: add the cache_policy option Names where the cache directory lives, instead of making everyone spell out a path: `local` (the default, unchanged) and `user` (the platform user cache directory, keyed by project). This is what GH-1089 has been asking for since 2015. What blocked it was that a cache outside the worktree no longer shares the worktree's lifetime; the metadata from the previous commit is what makes such a directory collectable, and the listing and pruning commands follow. `cache_dir` keeps working and always wins - it is an explicit path, the policy only decides the location when it is unset, and it stays the way to reach anywhere the policies do not name. To make "unset" detectable its default becomes empty rather than ".pytest_cache". `PYTEST_CACHE_POLICY` feeds the *default* of `cache_policy`, exactly as TOX_ENV_DIR feeds `cache_dir`'s, so a machine-wide opt-in still loses to an explicit setting. The project key is the rootdir alone - not the interpreter, which now lives in scopes - so one project gets one directory however many environments run it, and ephemeral environments no longer mint a cache tree per invocation. It is symlink-resolved so that reaching a project through two paths does not give it two caches; both the digest and the readable label come from the same normalised path. A directory name collision is detected via the full digest in the metadata and falls back to a longer name. pytest_report_header now compares the resolved path against the default rather than the configured string against a hardcoded ".pytest_cache", which also fixes setting cache_dir to the default explicitly forcing the line to show. The path is hyperlinked. Co-Authored-By: Claude Opus 5 (1M context) --- changelog/1089.feature.1.rst | 24 +++++ src/_pytest/cacheprovider.py | 125 +++++++++++++++++++++--- src/_pytest/pathlib.py | 2 +- testing/test_cacheprovider.py | 176 ++++++++++++++++++++++++++++++++++ testing/test_pathlib.py | 14 +-- 5 files changed, 318 insertions(+), 23 deletions(-) create mode 100644 changelog/1089.feature.1.rst diff --git a/changelog/1089.feature.1.rst b/changelog/1089.feature.1.rst new file mode 100644 index 00000000000..49597d7d123 --- /dev/null +++ b/changelog/1089.feature.1.rst @@ -0,0 +1,24 @@ +New :confval:`cache_policy` config option, choosing *where* the cache directory lives by name rather than +by spelling out a path: + +.. code-block:: ini + + [pytest] + cache_policy = user + +``local`` (the default) keeps the current ``/.pytest_cache``. ``user`` puts the cache in the +platform's user cache directory - ``$XDG_CACHE_HOME/pytest`` on Linux - under a sub-directory keyed by the +project, so nothing is written into the project at all. + +The ``user`` policy requires the ``xdg`` extra: ``pip install pytest[xdg]``. + +It can also be set for a whole machine with the ``PYTEST_CACHE_POLICY`` environment variable, so that +individual projects need no configuration. :confval:`cache_dir` remains an explicit path override and +always wins over the policy. + +Because the interpreter in use is now distinguished by :class:`pytest.CacheScope` within a single cache +directory, one project gets one cache directory whatever runs it. + +The ``cachedir:`` line in the report header now compares the resolved path against the default, so setting +:confval:`cache_dir` to ``.pytest_cache`` explicitly no longer forces the line to be shown. Where it is +shown, it is a clickable link in terminals which support hyperlinks. diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index 3f2983b8ee4..f721dc2d146 100644 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -22,9 +22,12 @@ import time from typing import Any from typing import final +from typing import Literal +from .pathlib import check_user_cache_root from .pathlib import resolve_from_str from .pathlib import rm_rf +from .pathlib import user_cache_root from .reports import CollectReport from _pytest import __version__ from _pytest import nodes @@ -98,12 +101,71 @@ def _make_cachedir( shutil.rmtree(path, ignore_errors=True) +def _project_key_path(rootpath: Path) -> str: + """Normalised rootdir, as used to identify a project. + + Symlinks are resolved even though ``Config.rootpath`` itself is not: + reaching one project through two paths must not give it two caches, which + is the duplication the ``user`` policy exists to avoid. Cache *content* is + unaffected either way, since node ids are stored relative to the rootdir. + + normcase matters on Windows, where two spellings differing only in case + are one directory, and the casing varies with how the shell was launched. + """ + return os.path.normcase(_realpath_or_self(str(rootpath))) + + +def _project_digest(rootpath: Path) -> str: + """Digest identifying a project for the ``user`` cache policy.""" + key = _project_key_path(rootpath) + return hashlib.sha256(key.encode("utf-8", "surrogatepass")).hexdigest() + + +def _user_cache_dir(config: Config) -> Path: + root = user_cache_root() + check_user_cache_root(root) + + digest = _project_digest(config.rootpath) + # From the same normalised path as the digest, so that reaching a project + # through a symlink gives the same directory name and not just the same + # digest. + label = _label(os.path.basename(_project_key_path(config.rootpath))) + candidate = root / f"{label}-{digest[:16]}" + + existing = read_cache_info(candidate) + if existing is None or existing.get("digest") in (None, digest): + return candidate + # A genuine 64-bit collision, or a hand-made directory. Deterministic and + # stateless: no counters, no lock files. + return root / f"{label}-{digest[:32]}" + + +def _cache_policy(config: Config) -> str: + """The effective cache policy, or ``explicit`` if ``cache_dir`` is set.""" + if config.getini("cache_dir"): + return "explicit" + policy: str = config.getini("cache_policy") + return policy + + def _resolve_cache_dir(config: Config) -> Path: """Determine the cache directory for a Config. The single place where the cache directory location is decided. + + ``cache_dir`` is an explicit path override and always wins; ``cache_policy`` + only decides the location when ``cache_dir`` is unset. """ - return resolve_from_str(config.getini("cache_dir"), config.rootpath) + cache_dir = config.getini("cache_dir") + if cache_dir: + # resolve_from_str applies expanduser/expandvars. + return resolve_from_str(cache_dir, config.rootpath) + + policy: str = config.getini("cache_policy") + if policy == "local": + return config.rootpath / ".pytest_cache" + assert policy == "user", policy + return _user_cache_dir(config) class CacheScope(enum.Enum): @@ -419,6 +481,12 @@ def _cache_info(self, previous: dict[str, Any] | None) -> dict[str, Any]: now = _now() info: dict[str, Any] = dict(previous) if previous else {} info["schema"] = CACHE_INFO_SCHEMA + policy = _cache_policy(self._config) + info["policy"] = policy + if policy == "user": + # Recorded in full, so that a directory name collision can be + # detected rather than silently sharing a cache. + info["digest"] = _project_digest(self._config.rootpath) info["origin"] = { "rootdir": str(self._config.rootpath), "inipath": str(self._config.inipath) if self._config.inipath else None, @@ -742,10 +810,26 @@ def pytest_addoption(parser: Parser) -> None: dest="cacheclear", help="Remove all cache contents at start of test run", ) - cache_dir_default = ".pytest_cache" + # Empty by default so that "not configured" is detectable, in which case + # cache_policy decides the location. + cache_dir_default = "" if "TOX_ENV_DIR" in os.environ: - cache_dir_default = os.path.join(os.environ["TOX_ENV_DIR"], cache_dir_default) - parser.addini("cache_dir", default=cache_dir_default, help="Cache directory path") + cache_dir_default = os.path.join(os.environ["TOX_ENV_DIR"], ".pytest_cache") + parser.addini( + "cache_dir", + default=cache_dir_default, + help="Cache directory path; overrides cache_policy", + ) + parser.addini( + "cache_policy", + type=Literal["local", "user"], + default=os.environ.get("PYTEST_CACHE_POLICY") or "local", + help=( + "Where the cache directory lives: 'local' (rootdir/.pytest_cache) " + "or 'user' (the platform's user cache directory, keyed by " + "project). Ignored if cache_dir is set." + ), + ) group.addoption( "--lfnf", "--last-failed-no-failures", @@ -801,18 +885,29 @@ def cache(request: FixtureRequest) -> Cache: def pytest_report_header(config: Config) -> str | None: """Display cachedir with --cache-show and if non-default.""" - if config.option.verbose > 0 or config.getini("cache_dir") != ".pytest_cache": - assert config.cache is not None - cachedir = config.cache._cachedir - # TODO: evaluate generating upward relative paths - # starting with .., ../.. if sensible + assert config.cache is not None + cachedir = config.cache._cachedir + # Compare the resolved path rather than the configured value, so that + # setting cache_dir to the default explicitly is treated as the default. + if config.option.verbose <= 0 and cachedir == config.rootpath / ".pytest_cache": + return None - try: - displaypath = cachedir.relative_to(config.rootpath) - except ValueError: - displaypath = cachedir - return f"cachedir: {displaypath}" - return None + # TODO: evaluate generating upward relative paths + # starting with .., ../.. if sensible + try: + displaypath: Path | str = cachedir.relative_to(config.rootpath) + except ValueError: + # Not below the rootdir (#3745); show the absolute path. + displaypath = cachedir + + # A plain string is returned to the hook caller, so the escapes have to be + # applied here rather than via write_link(). The line is flushed with a + # newline immediately, which resets the writer's width bookkeeping, so the + # transient over-count does not affect anything downstream. + if config.pluginmanager.get_plugin("terminalreporter") is not None: + tw = config.get_terminal_writer() + displaypath = tw.hyperlink(str(displaypath), cachedir.as_uri()) + return f"cachedir: {displaypath}" def _cache_roots(basedir: Path) -> list[tuple[str | None, Path]]: diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py index ab9ced5e81c..2c539daab03 100644 --- a/src/_pytest/pathlib.py +++ b/src/_pytest/pathlib.py @@ -475,7 +475,7 @@ def resolve_from_str(input: str, rootpath: Path) -> Path: return rootpath.joinpath(input) -def pytest_user_cache_dir(*, environ: Mapping[str, str] | None = None) -> Path: +def user_cache_root(*, environ: Mapping[str, str] | None = None) -> Path: """Return the root directory for pytest's user-level caches. Creates nothing. diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py index 462bb756596..b95e592d8bf 100644 --- a/testing/test_cacheprovider.py +++ b/testing/test_cacheprovider.py @@ -7,12 +7,14 @@ import json import os from pathlib import Path +import re import shutil import sys from typing import Any from _pytest import cacheprovider from _pytest.cacheprovider import _label +from _pytest.cacheprovider import _project_digest from _pytest.cacheprovider import _scope_id from _pytest.cacheprovider import Cache from _pytest.cacheprovider import CACHE_INFO_NAME @@ -22,6 +24,7 @@ from _pytest.compat import assert_never from _pytest.config import ExitCode from _pytest.monkeypatch import MonkeyPatch +from _pytest.pathlib import symlink_or_skip from _pytest.pytester import Pytester from _pytest.tmpdir import TempPathFactory import pytest @@ -229,6 +232,45 @@ def test_cache_reportheader( result.stdout.fnmatch_lines([f"cachedir: {expected}"]) +def test_cache_reportheader_hidden_by_default(pytester: Pytester) -> None: + pytester.makepyfile("""def test_foo(): pass""") + result = pytester.runpytest() + result.stdout.no_fnmatch_line("cachedir:*") + + +def test_cache_reportheader_hidden_for_explicit_default(pytester: Pytester) -> None: + """Setting cache_dir to the default explicitly is still the default. + + The check compares the resolved path, rather than the configured string + against a hardcoded ".pytest_cache". + """ + pytester.makepyfile("""def test_foo(): pass""") + pytester.makeini("[pytest]\ncache_dir = .pytest_cache\n") + result = pytester.runpytest() + result.stdout.no_fnmatch_line("cachedir:*") + + +def test_cache_reportheader_user_policy( + pytester: Pytester, monkeypatch: MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("PYTEST_CACHE_HOME", str(tmp_path / "user-cache")) + pytester.makepyfile("""def test_foo(): pass""") + pytester.makeini("[pytest]\ncache_policy = user\n") + result = pytester.runpytest("-v") + # Outside the rootdir, so shown as an absolute path (#3745). + result.stdout.fnmatch_lines([f"cachedir: {tmp_path / 'user-cache'}*"]) + + +def test_cache_reportheader_hyperlinked( + pytester: Pytester, monkeypatch: MonkeyPatch +) -> None: + monkeypatch.setenv("PY_COLORS", "1") + monkeypatch.setenv("PYTEST_HYPERLINKS", "1") + pytester.makepyfile("""def test_foo(): pass""") + result = pytester.runpytest_subprocess("-v") + result.stdout.fnmatch_lines(["*\x1b]8;;file://*.pytest_cache*"]) + + def test_cache_reportheader_external_abspath( pytester: Pytester, tmp_path_factory: TempPathFactory ) -> None: @@ -1381,6 +1423,140 @@ def test_cachedir_tag(pytester: Pytester) -> None: assert cachedir_tag_path.read_bytes() == CACHEDIR_FILES["CACHEDIR.TAG"] +class TestCachePolicy: + @pytest.fixture + def user_cache(self, monkeypatch: MonkeyPatch, tmp_path: Path) -> Path: + root = tmp_path / "user-cache" + monkeypatch.setenv("PYTEST_CACHE_HOME", str(root)) + return root + + def resolve(self, pytester: Pytester, **ini: str) -> Path: + body = "".join(f"{k} = {v}\n" for k, v in ini.items()) + pytester.makeini(f"[pytest]\n{body}") + return Cache.for_config(pytester.parseconfig(), _ispytest=True)._cachedir + + def test_local_is_the_default(self, pytester: Pytester) -> None: + assert self.resolve(pytester) == pytester.path / ".pytest_cache" + + def test_user_policy(self, pytester: Pytester, user_cache: Path) -> None: + cachedir = self.resolve(pytester, cache_policy="user") + assert cachedir.parent == user_cache + assert cachedir.name.startswith(f"{pytester.path.name}-") + + def test_cache_dir_wins_over_policy( + self, pytester: Pytester, user_cache: Path + ) -> None: + cachedir = self.resolve(pytester, cache_policy="user", cache_dir="explicit") + assert cachedir == pytester.path / "explicit" + + def test_unknown_policy_is_a_usage_error(self, pytester: Pytester) -> None: + pytester.makeini("[pytest]\ncache_policy = bogus\n") + pytester.makepyfile(test_a="def test_ok(): pass") + result = pytester.runpytest() + assert result.ret == ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines(["*cache_policy*expects one of*got 'bogus'*"]) + + def test_env_var_sets_the_default( + self, pytester: Pytester, user_cache: Path, monkeypatch: MonkeyPatch + ) -> None: + monkeypatch.setenv("PYTEST_CACHE_POLICY", "user") + assert self.resolve(pytester).parent == user_cache + + def test_explicit_setting_beats_the_env_var( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + monkeypatch.setenv("PYTEST_CACHE_POLICY", "user") + assert self.resolve(pytester, cache_policy="local") == ( + pytester.path / ".pytest_cache" + ) + + def test_project_key_is_stable(self, pytester: Pytester, user_cache: Path) -> None: + first = self.resolve(pytester, cache_policy="user") + second = self.resolve(pytester, cache_policy="user") + assert first == second + assert re.fullmatch(r"[A-Za-z0-9._-]{1,32}-[0-9a-f]{16}", first.name) + + def test_project_key_ignores_the_environment( + self, pytester: Pytester, user_cache: Path, monkeypatch: MonkeyPatch + ) -> None: + # The whole point of scopes: one project gets one directory, however + # many interpreters run it. + before = self.resolve(pytester, cache_policy="user") + monkeypatch.setattr(sys, "prefix", "/somewhere/else") + monkeypatch.setattr(sys, "version_info", (9, 9, 9)) + assert self.resolve(pytester, cache_policy="user") == before + + def test_project_key_resolves_symlinks( + self, pytester: Pytester, user_cache: Path, tmp_path: Path + ) -> None: + real = self.resolve(pytester, cache_policy="user") + + link = tmp_path / "link" + symlink_or_skip(pytester.path, link) + pytester.makeini("[pytest]\ncache_policy = user\n") + config = pytester.parseconfig(f"--rootdir={link}", str(link)) + assert Cache.for_config(config, _ispytest=True)._cachedir == real + + def test_project_key_collision_extends_the_name( + self, pytester: Pytester, user_cache: Path + ) -> None: + cachedir = self.resolve(pytester, cache_policy="user") + # Squat the short name with a directory belonging to something else. + cachedir.mkdir(parents=True) + (cachedir / CACHE_INFO_NAME).write_text( + json.dumps({"schema": 1, "digest": "f" * 64}), encoding="UTF-8" + ) + + extended = self.resolve(pytester, cache_policy="user") + assert extended != cachedir + assert extended.name.endswith(_project_digest(pytester.path)[:32]) + + def test_user_policy_records_the_digest( + self, pytester: Pytester, user_cache: Path + ) -> None: + cache = Cache.for_config( + pytester.parseconfig("-o", "cache_policy=user"), _ispytest=True + ) + cache.set("foo", 1) + info = read_cache_info(cache._cachedir) + assert info is not None + assert info["policy"] == "user" + assert info["digest"] == _project_digest(pytester.path) + + def test_user_policy_creates_nothing_when_unused( + self, pytester: Pytester, user_cache: Path + ) -> None: + pytester.makeini("[pytest]\ncache_policy = user\n") + pytester.makepyfile(test_a="def test_ok(): pass") + pytester.runpytest("--collect-only") + assert not user_cache.exists() + + def test_lastfailed_round_trips(self, pytester: Pytester, user_cache: Path) -> None: + pytester.makeini("[pytest]\ncache_policy = user\n") + pytester.makepyfile( + test_a="def test_ok(): pass\ndef test_bad(): assert False\n" + ) + pytester.runpytest("-q").assert_outcomes(passed=1, failed=1) + assert not (pytester.path / ".pytest_cache").exists() + + result = pytester.runpytest("-q", "--lf") + result.assert_outcomes(failed=1) + + def test_user_policy_without_platformdirs( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + monkeypatch.delenv("PYTEST_CACHE_HOME", raising=False) + pytester.makeini("[pytest]\ncache_policy = user\n") + pytester.makepyfile(test_a="def test_ok(): pass") + pytester.syspathinsert() + # Hide platformdirs from the inner run. + pytester.makepyfile(platformdirs="raise ImportError('hidden')") + + result = pytester.runpytest_subprocess() + assert result.ret == ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines(["*pip install pytest?xdg?*"]) + + class TestCacheInfo: @pytest.fixture def cache(self, pytester: Pytester) -> Cache: diff --git a/testing/test_pathlib.py b/testing/test_pathlib.py index 80886d40f0c..60a2e9b2d5a 100644 --- a/testing/test_pathlib.py +++ b/testing/test_pathlib.py @@ -39,13 +39,13 @@ from _pytest.pathlib import is_importable from _pytest.pathlib import maybe_delete_a_numbered_dir from _pytest.pathlib import module_name_from_path -from _pytest.pathlib import pytest_user_cache_dir from _pytest.pathlib import resolve_package_path from _pytest.pathlib import resolve_pkg_root_and_module_name from _pytest.pathlib import safe_exists from _pytest.pathlib import scandir from _pytest.pathlib import spec_matches_module_path from _pytest.pathlib import symlink_or_skip +from _pytest.pathlib import user_cache_root from _pytest.pathlib import visit from _pytest.pytester import Pytester from _pytest.pytester import RunResult @@ -537,12 +537,12 @@ def test_bestrelpath() -> None: class TestUserCacheDir: def test_cache_home_override(self, tmp_path: Path) -> None: environ = {"PYTEST_CACHE_HOME": str(tmp_path / "explicit")} - assert pytest_user_cache_dir(environ=environ) == tmp_path / "explicit" + assert user_cache_root(environ=environ) == tmp_path / "explicit" def test_cache_home_override_is_expanded(self, monkeypatch: MonkeyPatch) -> None: monkeypatch.setenv("SOMEWHERE", "elsewhere") environ = {"PYTEST_CACHE_HOME": os.path.join("~", "$SOMEWHERE")} - assert pytest_user_cache_dir(environ=environ) == Path( + assert user_cache_root(environ=environ) == Path( os.path.expanduser("~") ).joinpath("elsewhere") @@ -551,7 +551,7 @@ def test_cache_home_override_wins_over_xdg(self, tmp_path: Path) -> None: "PYTEST_CACHE_HOME": str(tmp_path / "explicit"), "XDG_CACHE_HOME": str(tmp_path / "xdg"), } - assert pytest_user_cache_dir(environ=environ) == tmp_path / "explicit" + assert user_cache_root(environ=environ) == tmp_path / "explicit" def test_cache_home_override_needs_no_platformdirs( self, tmp_path: Path, monkeypatch: MonkeyPatch @@ -559,7 +559,7 @@ def test_cache_home_override_needs_no_platformdirs( # The escape hatch has to work on installs without the `xdg` extra. monkeypatch.setitem(sys.modules, "platformdirs", None) environ = {"PYTEST_CACHE_HOME": str(tmp_path / "explicit")} - assert pytest_user_cache_dir(environ=environ) == tmp_path / "explicit" + assert user_cache_root(environ=environ) == tmp_path / "explicit" def test_delegates_to_platformdirs(self, monkeypatch: MonkeyPatch) -> None: # The per-platform conventions are platformdirs' business; all we test @@ -572,13 +572,13 @@ def user_cache_path(*args: object, **kwargs: object) -> Path: return Path("/somewhere/pytest") monkeypatch.setattr(platformdirs, "user_cache_path", user_cache_path) - assert pytest_user_cache_dir(environ={}) == Path("/somewhere/pytest") + assert user_cache_root(environ={}) == Path("/somewhere/pytest") assert calls == [(("pytest",), {"appauthor": False})] def test_without_platformdirs(self, monkeypatch: MonkeyPatch) -> None: monkeypatch.setitem(sys.modules, "platformdirs", None) with pytest.raises(UsageError, match=r"pip install pytest\[xdg\]"): - pytest_user_cache_dir(environ={}) + user_cache_root(environ={}) class TestCheckUserCacheRoot: From b5ce3b6b6719c04bfdc7f86c8c8d35f3d32c887a Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 5 Aug 2026 11:07:51 +0200 Subject: [PATCH 09/12] cacheprovider: deprecate the TOX_ENV_DIR cache_dir default Now redundant: scope=ENV keeps --lf/--nf/--sw state apart between environments without moving the cache directory, and it does so for every tool rather than only for tox. Behaviour is unchanged this cycle, with a PytestRemovedIn10Warning. Nothing needs to replace it. Anyone who wants the location anyway can spell out `cache_dir = $TOX_ENV_DIR/.pytest_cache`, which is the same expression the default was built from - cache_dir has expanded environment variables for years - so it resolves to the same directory by construction, whatever TOX_ENV_DIR points at. Two other things this shook out: The legacy path is now consulted by the `local` policy rather than being the default of `cache_dir`. As a cache_dir default it silently beat every policy, since cache_dir takes precedence - so `cache_policy = user` would have done nothing at all under tox, which is exactly the trap this series is removing. The warning goes through Config.issue_config_time_warning rather than warnings.warn, because warnings raised during pytest_configure escape the reporter and would never have been seen. `cache_policy = local` written out explicitly still gets the legacy path, since `local` is the default value and there is nothing to tell the two apart. The warning names the way out; there is a test documenting this. Co-Authored-By: Claude Opus 5 (1M context) --- changelog/1089.deprecation.rst | 15 ++++++ doc/en/deprecations.rst | 31 ++++++++++++ src/_pytest/cacheprovider.py | 40 +++++++++++++-- src/_pytest/deprecated.py | 10 ++++ testing/test_cacheprovider.py | 89 +++++++++++++++++++++++++++++++++- 5 files changed, 180 insertions(+), 5 deletions(-) create mode 100644 changelog/1089.deprecation.rst diff --git a/changelog/1089.deprecation.rst b/changelog/1089.deprecation.rst new file mode 100644 index 00000000000..cf4c8289cc3 --- /dev/null +++ b/changelog/1089.deprecation.rst @@ -0,0 +1,15 @@ +Defaulting :confval:`cache_dir` to ``$TOX_ENV_DIR/.pytest_cache`` when ``TOX_ENV_DIR`` is set in the +environment is deprecated, and will be removed in pytest 10. + +It existed because ``--lf``, ``--nf`` and ``--sw`` state is not valid across interpreters, so a tox matrix +would otherwise have each environment overwrite the previous one's. pytest now keeps that state apart on +its own using :attr:`CacheScope.ENV `, for every tool rather than only for tox. + +Nothing needs to replace it. To keep the cache in the tox environment anyway, spell out the location that +was previously implied - :confval:`cache_dir` expands environment variables, so this is the same +expression the old default was built from:: + + [pytest] + cache_dir = $TOX_ENV_DIR/.pytest_cache + +See :ref:`tox-env-dir-cache-dir`. diff --git a/doc/en/deprecations.rst b/doc/en/deprecations.rst index feaeaa09736..5075fa37801 100644 --- a/doc/en/deprecations.rst +++ b/doc/en/deprecations.rst @@ -15,6 +15,37 @@ Below is a complete list of all pytest features which are considered deprecated. :class:`~pytest.PytestWarning` or subclasses, which can be filtered using :ref:`standard warning filters `. +.. _tox-env-dir-cache-dir: + +Defaulting ``cache_dir`` to ``$TOX_ENV_DIR/.pytest_cache`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 9.2 + +When ``TOX_ENV_DIR`` is set in the environment, :confval:`cache_dir` has defaulted to +``$TOX_ENV_DIR/.pytest_cache`` rather than to a directory inside the project. This is deprecated and +will be removed in pytest 10. + +It existed because ``--lf``, ``--nf`` and ``--sw`` state is not valid across interpreters, so a tox +matrix would otherwise have each environment overwrite the previous one's. pytest now keeps that state +apart on its own, using :attr:`CacheScope.ENV ` within a single cache directory, +so the special case is no longer needed - and it only ever helped tox, not nox or a plain second +virtualenv. + +**Nothing needs to replace it.** Once the warning is gone, the cache lands in the project like any +other, and per-environment state stays separate as it does everywhere else. + +To keep the cache in the tox environment anyway, spell out the location that was previously implied: + +.. code-block:: ini + + [pytest] + cache_dir = $TOX_ENV_DIR/.pytest_cache + +:confval:`cache_dir` expands environment variables, so this is the same expression the old default was +built from and resolves to exactly the same directory. + + .. _callspec2-renamed: ``_pytest.python.CallSpec2`` renamed to ``CallSpec`` diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index f721dc2d146..1fb70dd8402 100644 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -38,6 +38,7 @@ from _pytest.config import hookimpl from _pytest.config.argparsing import Parser from _pytest.deprecated import check_ispytest +from _pytest.deprecated import TOX_ENV_DIR_CACHE_DIR from _pytest.fixtures import fixture from _pytest.fixtures import FixtureRequest from _pytest.main import Session @@ -140,6 +141,28 @@ def _user_cache_dir(config: Config) -> Path: return root / f"{label}-{digest[:32]}" +def _tox_legacy_cache_dir(config: Config) -> Path | None: + """The legacy ``$TOX_ENV_DIR/.pytest_cache`` location, if it applies. + + Only consulted by the ``local`` policy, so that setting ``cache_policy`` + explicitly is not silently overridden when running under tox. + + .. deprecated:: 9.2 + Superseded by :attr:`CacheScope.ENV`, which keeps ``--lf``/``--nf``/ + ``--sw`` state apart between environments without moving the cache + directory - and does so for every tool rather than only for tox. + Anyone who wants this exact location can spell it out as + ``cache_dir = $TOX_ENV_DIR/.pytest_cache``. + """ + tox_env_dir = os.environ.get("TOX_ENV_DIR") + if not tox_env_dir: + return None + # Warnings raised during pytest_configure escape the reporter, so this has + # to go through the config rather than warnings.warn. + config.issue_config_time_warning(TOX_ENV_DIR_CACHE_DIR, stacklevel=3) + return resolve_from_str(os.path.join(tox_env_dir, ".pytest_cache"), config.rootpath) + + def _cache_policy(config: Config) -> str: """The effective cache policy, or ``explicit`` if ``cache_dir`` is set.""" if config.getini("cache_dir"): @@ -163,6 +186,9 @@ def _resolve_cache_dir(config: Config) -> Path: policy: str = config.getini("cache_policy") if policy == "local": + legacy = _tox_legacy_cache_dir(config) + if legacy is not None: + return legacy return config.rootpath / ".pytest_cache" assert policy == "user", policy return _user_cache_dir(config) @@ -810,14 +836,20 @@ def pytest_addoption(parser: Parser) -> None: dest="cacheclear", help="Remove all cache contents at start of test run", ) + group.addoption( + "--cache-list", + action="store_true", + dest="cachelist", + help=( + "List the cache directories under the user-level cache root, " + "don't perform collection or tests." + ), + ) # Empty by default so that "not configured" is detectable, in which case # cache_policy decides the location. - cache_dir_default = "" - if "TOX_ENV_DIR" in os.environ: - cache_dir_default = os.path.join(os.environ["TOX_ENV_DIR"], ".pytest_cache") parser.addini( "cache_dir", - default=cache_dir_default, + default="", help="Cache directory path; overrides cache_policy", ) parser.addini( diff --git a/src/_pytest/deprecated.py b/src/_pytest/deprecated.py index f46b036affe..9f6371b0847 100644 --- a/src/_pytest/deprecated.py +++ b/src/_pytest/deprecated.py @@ -134,6 +134,16 @@ "Use parsefactories(holder=obj, node=node) instead." ) +TOX_ENV_DIR_CACHE_DIR = PytestRemovedIn10Warning( + "Defaulting the cache directory to $TOX_ENV_DIR/.pytest_cache is deprecated and " + "will be removed in pytest 10.\n" + "It existed to keep --lf/--nf/--sw state apart between tox environments, which " + "pytest now does on its own within a single cache directory.\n" + "Nothing needs to replace it. To keep the current location anyway, set\n" + " cache_dir = $TOX_ENV_DIR/.pytest_cache\n" + "See https://docs.pytest.org/en/stable/deprecations.html#tox-env-dir-cache-dir" +) + CALLSPEC2_RENAMED = PytestRemovedIn10Warning( "_pytest.python.CallSpec2 has been renamed to CallSpec.\n" "The CallSpec2 alias will be removed in pytest 10.\n" diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py index b95e592d8bf..06608b18b2d 100644 --- a/testing/test_cacheprovider.py +++ b/testing/test_cacheprovider.py @@ -217,6 +217,7 @@ def test_custom_cache_dir_with_env_var( assert pytester.path.joinpath("custom_cache_dir").is_dir() +@pytest.mark.filterwarnings("ignore::pytest.PytestRemovedIn10Warning") @pytest.mark.parametrize("env", ((), ("TOX_ENV_DIR", "mydir/tox-env"))) def test_cache_reportheader( env: Sequence[str], pytester: Pytester, monkeypatch: MonkeyPatch @@ -228,10 +229,96 @@ def test_cache_reportheader( else: monkeypatch.delenv("TOX_ENV_DIR", raising=False) expected = ".pytest_cache" - result = pytester.runpytest("-v") + result = pytester.runpytest("-v", "-W", "ignore::pytest.PytestRemovedIn10Warning") result.stdout.fnmatch_lines([f"cachedir: {expected}"]) +class TestToxEnvDirDeprecation: + def test_warns_when_the_legacy_default_is_used( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + monkeypatch.setenv("TOX_ENV_DIR", str(pytester.path / "tox-env")) + pytester.makepyfile(test_a="def test_ok(): pass") + result = pytester.runpytest("-W", "default") + result.stdout.fnmatch_lines( + [ + "*PytestRemovedIn10Warning: Defaulting the cache directory to " + "$TOX_ENV_DIR/.pytest_cache is deprecated*", + "*cache_dir = $TOX_ENV_DIR/.pytest_cache*", + ] + ) + + def test_the_advice_reproduces_the_legacy_location( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + """The deprecation advice has to actually be true. + + It names the very expression the legacy default was built from, so it + lands in the same place by construction - whatever TOX_ENV_DIR happens + to point at. + """ + monkeypatch.setenv("TOX_ENV_DIR", str(pytester.path / "tox-env")) + pytester.makepyfile(test_a="def test_bad(): assert False") + + legacy = Cache.for_config(pytester.parseconfig(), _ispytest=True)._cachedir + pytester.makeini("[pytest]\ncache_dir = $TOX_ENV_DIR/.pytest_cache\n") + migrated = Cache.for_config(pytester.parseconfig(), _ispytest=True)._cachedir + assert legacy == migrated == pytester.path / "tox-env" / ".pytest_cache" + + def test_silent_without_tox_env_dir( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + monkeypatch.delenv("TOX_ENV_DIR", raising=False) + pytester.makepyfile(test_a="def test_ok(): pass") + result = pytester.runpytest("-W", "default") + result.stdout.no_fnmatch_line("*TOX_ENV_DIR*") + + @pytest.mark.parametrize("ini", ["cache_policy = user", "cache_dir = elsewhere"]) + def test_silent_when_configured_explicitly( + self, pytester: Pytester, monkeypatch: MonkeyPatch, tmp_path: Path, ini: str + ) -> None: + monkeypatch.setenv("PYTEST_CACHE_HOME", str(tmp_path / "user-cache")) + monkeypatch.setenv("TOX_ENV_DIR", str(pytester.path / "tox-env")) + pytester.makeini(f"[pytest]\n{ini}\n") + pytester.makepyfile(test_a="def test_ok(): pass") + result = pytester.runpytest("-W", "default") + result.stdout.no_fnmatch_line("*TOX_ENV_DIR*") + + def test_configured_policy_is_not_overridden_by_tox( + self, pytester: Pytester, monkeypatch: MonkeyPatch, tmp_path: Path + ) -> None: + """A configured cache_policy must win over the legacy tox location. + + The legacy path is only consulted by the `local` policy for exactly + this reason - as a `cache_dir` default it would have silently beaten + every policy, since cache_dir takes precedence. + """ + user_cache = tmp_path / "user-cache" + monkeypatch.setenv("PYTEST_CACHE_HOME", str(user_cache)) + monkeypatch.setenv("TOX_ENV_DIR", str(pytester.path / "tox-env")) + pytester.makeini("[pytest]\ncache_policy = user\n") + pytester.makepyfile(test_a="def test_bad(): assert False") + pytester.runpytest("-q") + + assert list(user_cache.iterdir()) + assert not (pytester.path / "tox-env").exists() + + def test_local_policy_is_indistinguishable_from_the_default( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + """Writing `cache_policy = local` under tox still gets the legacy path. + + `local` is the default value, so there is nothing to tell an explicit + setting apart from an absent one. The warning points at `cache_dir` for + anyone who wants out. + """ + monkeypatch.setenv("TOX_ENV_DIR", str(pytester.path / "tox-env")) + pytester.makeini("[pytest]\ncache_policy = local\n") + pytester.makepyfile(test_a="def test_bad(): assert False") + pytester.runpytest("-q", "-W", "ignore::pytest.PytestRemovedIn10Warning") + assert (pytester.path / "tox-env" / ".pytest_cache").is_dir() + + def test_cache_reportheader_hidden_by_default(pytester: Pytester) -> None: pytester.makepyfile("""def test_foo(): pass""") result = pytester.runpytest() From d3d118b1a55d53f8cd0cfe95a9abdb361d6a1e33 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Mon, 3 Aug 2026 15:16:36 +0200 Subject: [PATCH 10/12] cacheprovider: add --cache-list Shows what has accumulated under the user-level cache root: size, age, origin project, and the scopes each directory holds, with `orphaned` for a directory whose project is gone and `stale` for a scope whose environment is gone. That is the CI-DOS scenario from GH-1089 made visible - and reduced from orphaned trees to orphaned scopes, since one project now keeps one directory. Uses the session-less short-circuit from helpconfig rather than wrap_session. Without a Session, pytest_sessionfinish never fires, so LFPlugin/NFPlugin cannot rewrite the very state being listed; that is structural rather than a guard which would have to grow with every new flag. It also means listing works without a collectable project, which matters for a command about the machine rather than about this project. --help is hoisted to the top of pytest_cmdline_main, which keeps the existing `--cache-show --help` contract and extends it to the new flag for free. A directory whose metadata is missing, unparsable or newer than we understand is still listed, as `broken` - otherwise it would be invisible but undeletable. Column widths are computed from the plain text before any link escapes are applied, and ORIGIN is last so it can overflow rather than be truncated. Co-Authored-By: Claude Opus 5 (1M context) --- changelog/1089.feature.2.rst | 20 +++ src/_pytest/cacheprovider.py | 270 +++++++++++++++++++++++++++++++++- testing/test_cacheprovider.py | 153 +++++++++++++++++++ 3 files changed, 440 insertions(+), 3 deletions(-) create mode 100644 changelog/1089.feature.2.rst diff --git a/changelog/1089.feature.2.rst b/changelog/1089.feature.2.rst new file mode 100644 index 00000000000..b8aed13dc31 --- /dev/null +++ b/changelog/1089.feature.2.rst @@ -0,0 +1,20 @@ +New ``--cache-list`` option, listing the cache directories under the user-level cache root together with +their size, age, origin project and the environments they hold state for: + +.. code-block:: text + + user cache directory: /home/ronny/.cache/pytest + + DIRECTORY SIZE LAST USED STATUS ORIGIN + pytest-1a2b3c4d5e6f7a 12.4 MiB 2 days ok /home/ronny/Projects/pytest + env-venv-9f8e7d6c 1.1 MiB 2 days ok /home/ronny/Projects/pytest/.venv + env-py312-0a1b2c3d 840.1 KiB 94 days stale /home/ronny/Projects/pytest/.tox/py312 + myproj-9f8e7d6c5b4a3 840 KiB 31 days orphaned /home/ronny/src/myproj + + 2 directories, 3 scopes, 13.2 MiB total + +``orphaned`` means the project it belongs to is gone; ``stale`` means the environment a scope holds state +for is gone. Directory and origin paths are clickable in terminals which support hyperlinks. + +It lists the user-level cache root regardless of the current project's :confval:`cache_policy`, so that +caches can still be found after switching back to ``local``. diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index 1fb70dd8402..b564807a1d4 100644 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -729,7 +729,7 @@ def pytest_collection_modifyitems( def pytest_sessionfinish(self, session: Session) -> None: config = self.config - if config.getoption("cacheshow") or hasattr(config, "workerinput"): + if _cache_command_active(config) or hasattr(config, "workerinput"): return assert config.cache is not None @@ -778,7 +778,7 @@ def _get_increasing_order(self, items: Iterable[nodes.Item]) -> list[nodes.Item] def pytest_sessionfinish(self) -> None: config = self.config - if config.getoption("cacheshow") or hasattr(config, "workerinput"): + if _cache_command_active(config) or hasattr(config, "workerinput"): return if config.getoption("collectonly"): @@ -877,11 +877,34 @@ def pytest_addoption(parser: Parser) -> None: ) +def _cache_command_active(config: Config) -> bool: + """Whether a short-circuiting cache command is running. + + Such a run must not write to the cache: merely inspecting it should not + rewrite the very state being inspected. + """ + return bool(config.getoption("cacheshow") or config.getoption("cachelist")) + + def pytest_cmdline_main(config: Config) -> int | ExitCode | None: - if config.option.cacheshow and not config.option.help: + if config.option.help: + # Let helpconfig's implementation handle it. The hook is firstresult, + # so returning None here is what lets --help win over these. + return None + if config.option.cacheshow: from _pytest.main import wrap_session return wrap_session(config, cacheshow) + if config.option.cachelist: + # The session-less pattern from helpconfig: no Session means + # pytest_sessionfinish never fires, so LFPlugin/NFPlugin cannot clobber + # the cache as a side effect of listing it. There is also nothing to + # collect - the command is about the machine, not this project. + config._do_configure() + try: + return cache_list(config) + finally: + config._ensure_unconfigure() return None @@ -1017,3 +1040,244 @@ def cacheshow(config: Config, session: Session) -> int: key = str(p.relative_to(basedir)) tw.line(f"{key} is a file of length {p.stat().st_size}") return 0 + + +@dataclasses.dataclass(frozen=True) +class _CacheDirEntry: + """A cache directory found under the user-level cache root.""" + + path: Path + info: dict[str, Any] | None + size: int + + @property + def origin(self) -> str | None: + if self.info is None: + return None + origin = self.info.get("origin") + if not isinstance(origin, dict): + return None + rootdir = origin.get("rootdir") + return rootdir if isinstance(rootdir, str) else None + + @property + def last_used_at(self) -> float | None: + return _as_timestamp( + None if self.info is None else self.info.get("last_used_at") + ) + + @property + def status(self) -> str: + if self.info is None or self.info.get("schema") != CACHE_INFO_SCHEMA: + # Unreadable, absent, or written by a pytest which knows more than + # we do. Still listed, so that it can still be removed. + return "broken" + origin = self.origin + if origin is None or not os.path.exists(origin): + return "orphaned" + return "ok" + + def scopes(self) -> list[_CacheScopeEntry]: + recorded = {} if self.info is None else self.info.get("scopes") + if not isinstance(recorded, dict): + recorded = {} + scopedir = self.path / Cache._CACHE_PREFIX_SCOPES + found = sorted(p.name for p in scopedir.iterdir()) if scopedir.is_dir() else [] + return [ + _CacheScopeEntry( + name=name, + info=recorded.get(name) + if isinstance(recorded.get(name), dict) + else None, + size=_dir_size(scopedir / name), + ) + for name in found + ] + + +@dataclasses.dataclass(frozen=True) +class _CacheScopeEntry: + """One scope directory inside a cache directory.""" + + name: str + info: dict[str, Any] | None + size: int + + @property + def prefix(self) -> str | None: + if self.info is None: + return None + prefix = self.info.get("prefix") + return prefix if isinstance(prefix, str) else None + + @property + def last_used_at(self) -> float | None: + return _as_timestamp( + None if self.info is None else self.info.get("last_used_at") + ) + + @property + def status(self) -> str: + if self.info is None: + return "broken" + prefix = self.prefix + # Only env scopes name an environment which can go away; a python + # scope stays meaningful as long as that interpreter is around. + if prefix is not None and not os.path.exists(prefix): + return "stale" + return "ok" + + +def _as_timestamp(value: object) -> float | None: + """Coerce a recorded timestamp, tolerating anything hand-edited into it.""" + return ( + value + if isinstance(value, (int, float)) and not isinstance(value, bool) + else None + ) + + +def _dir_size(path: Path) -> int: + """Total size of ``path``, ignoring anything unreadable.""" + total = 0 + stack = [path] + while stack: + try: + entries = list(os.scandir(stack.pop())) + except OSError: + continue + for entry in entries: + try: + if entry.is_dir(follow_symlinks=False): + stack.append(Path(entry.path)) + else: + total += entry.stat(follow_symlinks=False).st_size + except OSError: + continue + return total + + +def _format_size(size: int) -> str: + value = float(size) + for unit in ("B", "KiB", "MiB", "GiB"): + if value < 1024 or unit == "GiB": + return f"{value:.0f} {unit}" if unit == "B" else f"{value:.1f} {unit}" + value /= 1024 + raise AssertionError("unreachable") + + +def _format_age(last_used_at: float | None, now: float) -> str: + if last_used_at is None: + return "-" + seconds = max(now - last_used_at, 0) + for amount, unit in ((86400, "day"), (3600, "hour"), (60, "minute")): + if seconds >= amount: + count = int(seconds // amount) + return f"{count} {unit}{'s' if count != 1 else ''}" + return "just now" + + +def collect_cache_dirs(root: Path) -> list[_CacheDirEntry]: + """Every cache directory under ``root``, newest first.""" + try: + candidates = sorted(p for p in root.iterdir() if p.is_dir()) + except OSError: + return [] + entries = [ + _CacheDirEntry(path=p, info=read_cache_info(p), size=_dir_size(p)) + for p in candidates + ] + # Oldest last, so the ones worth pruning sit next to the totals. + return sorted(entries, key=lambda e: e.last_used_at or 0.0, reverse=True) + + +@dataclasses.dataclass(frozen=True) +class _ListRow: + """One rendered row of ``--cache-list``.""" + + name: str + size: str + age: str + status: str + origin: str | None + #: Set for a cache directory, None for a scope row nested under one. + link: Path | None + + @property + def cells(self) -> tuple[str, str, str, str]: + """The width-relevant cells, as plain text.""" + return (self.name, self.size, self.age, self.status) + + +def cache_list(config: Config) -> int: + """Display the cache directories under the user-level cache root.""" + tw = config.get_terminal_writer() + root = user_cache_root() + tw.line(f"user cache directory: {root}") + + entries = collect_cache_dirs(root) + if not entries: + tw.line("no managed cache directories found") + return 0 + + now = _now() + rows: list[_ListRow] = [] + for entry in entries: + rows.append( + _ListRow( + name=entry.path.name, + size=_format_size(entry.size), + age=_format_age(entry.last_used_at, now), + status=entry.status, + origin=entry.origin, + link=entry.path, + ) + ) + rows.extend( + _ListRow( + name=f" {scope.name}", + size=_format_size(scope.size), + age=_format_age(scope.last_used_at, now), + status=scope.status, + origin=scope.prefix, + link=None, + ) + for scope in entry.scopes() + ) + + header = ("DIRECTORY", "SIZE", "LAST USED", "STATUS") + # Widths come from the plain text, before any link escapes are applied. + # ORIGIN is last and so may overflow rather than being truncated: a + # truncated path is useless. + columns = zip(*(row.cells for row in rows), strict=True) + widths = [ + max(len(head), *(len(cell) for cell in column)) + for head, column in zip(header, columns, strict=True) + ] + + tw.line("") + heading = " ".join(h.ljust(w) for h, w in zip(header, widths, strict=True)) + tw.line(f" {heading} ORIGIN") + for row in rows: + tw.write(" ") + if row.link is not None: + tw.write_link(row.name.ljust(widths[0]), row.link.as_uri()) + else: + tw.write(row.name.ljust(widths[0])) + tw.write(f" {row.size.rjust(widths[1])}") + tw.write(f" {row.age.ljust(widths[2])}") + tw.write(f" {row.status.ljust(widths[3])}") + tw.write(" ") + # Only link an origin which is still there; a dangling link is worse + # than no link at all. + if row.origin is not None and os.path.isdir(row.origin): + tw.write_link(row.origin, Path(row.origin).as_uri()) + else: + tw.write(row.origin or "-") + tw.line("") + + scopes = sum(1 for row in rows if row.link is None) + total = sum(entry.size for entry in entries) + tw.line("") + tw.line(f"{len(entries)} directories, {scopes} scopes, {_format_size(total)} total") + return 0 diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py index 06608b18b2d..e5bcd943d9e 100644 --- a/testing/test_cacheprovider.py +++ b/testing/test_cacheprovider.py @@ -1644,6 +1644,159 @@ def test_user_policy_without_platformdirs( result.stderr.fnmatch_lines(["*pip install pytest?xdg?*"]) +class TestCacheList: + @pytest.fixture + def user_cache(self, monkeypatch: MonkeyPatch, tmp_path: Path) -> Path: + root = tmp_path / "user-cache" + monkeypatch.setenv("PYTEST_CACHE_HOME", str(root)) + return root + + def populate(self, pytester: Pytester, name: str) -> Path: + """Run a failing test in a fresh project, returning its rootdir.""" + project = pytester.path / name + project.mkdir() + project.joinpath("test_x.py").write_text( + "def test_bad(): assert False\n", encoding="UTF-8" + ) + project.joinpath("tox.ini").write_text( + "[pytest]\ncache_policy = user\n", encoding="UTF-8" + ) + pytester.runpytest_subprocess(str(project), "--rootdir", str(project)) + return project + + def test_empty(self, pytester: Pytester, user_cache: Path) -> None: + result = pytester.runpytest("--cache-list") + assert result.ret == 0 + result.stdout.fnmatch_lines( + [ + f"user cache directory: {user_cache}", + "no managed cache directories found", + ] + ) + + def test_lists_entries_with_scopes( + self, pytester: Pytester, user_cache: Path + ) -> None: + project = self.populate(pytester, "alpha") + + result = pytester.runpytest("--cache-list") + assert result.ret == 0 + result.stdout.fnmatch_lines( + [ + "*DIRECTORY*SIZE*LAST USED*STATUS*ORIGIN*", + f" alpha-* * ok *{project}", + " env-* * ok *", + "1 directories, 1 scopes, * total", + ] + ) + + def test_marks_orphaned_when_the_origin_is_gone( + self, pytester: Pytester, user_cache: Path + ) -> None: + project = self.populate(pytester, "alpha") + shutil.rmtree(project) + + result = pytester.runpytest("--cache-list") + result.stdout.fnmatch_lines([f" alpha-*orphaned*{project}"]) + + def test_marks_scope_stale_when_the_env_is_gone( + self, pytester: Pytester, user_cache: Path + ) -> None: + self.populate(pytester, "alpha") + cachedir = next(user_cache.iterdir()) + info = json.loads((cachedir / CACHE_INFO_NAME).read_text(encoding="UTF-8")) + for scope in info["scopes"].values(): + scope["prefix"] = str(pytester.path / "deleted-venv") + (cachedir / CACHE_INFO_NAME).write_text(json.dumps(info), encoding="UTF-8") + + result = pytester.runpytest("--cache-list") + # The directory itself stays fine; only the scope is collectable. + result.stdout.fnmatch_lines([" alpha-* ok *", " env-* stale *"]) + + @pytest.mark.parametrize("content", ["", "{not json", '{"schema": 99}']) + def test_tolerates_unusable_metadata( + self, pytester: Pytester, user_cache: Path, content: str + ) -> None: + # A directory we cannot understand must still be listed, or it becomes + # invisible but undeletable. + cachedir = user_cache / "mystery-0123456789abcdef" + cachedir.mkdir(parents=True) + if content: + (cachedir / CACHE_INFO_NAME).write_text(content, encoding="UTF-8") + + result = pytester.runpytest("--cache-list") + assert result.ret == 0 + result.stdout.fnmatch_lines([" mystery-*broken*"]) + + def test_works_regardless_of_this_project_policy( + self, pytester: Pytester, user_cache: Path + ) -> None: + # Listing must work after switching back to the local policy, or you + # could never clean up what you left behind. + self.populate(pytester, "alpha") + pytester.makeini("[pytest]\ncache_policy = local\n") + + result = pytester.runpytest("--cache-list") + result.stdout.fnmatch_lines([" alpha-*"]) + + def test_creates_nothing(self, pytester: Pytester, user_cache: Path) -> None: + pytester.makeini("[pytest]\ncache_policy = user\n") + pytester.makepyfile(test_a="def test_ok(): pass") + pytester.runpytest("--cache-list") + assert not user_cache.exists() + + def test_does_not_collect_or_run_tests( + self, pytester: Pytester, user_cache: Path + ) -> None: + pytester.makepyfile(test_a="raise RuntimeError('should not be collected')") + result = pytester.runpytest("--cache-list") + assert result.ret == 0 + result.stdout.no_fnmatch_line("*RuntimeError*") + + def test_does_not_clobber_the_cache( + self, pytester: Pytester, user_cache: Path + ) -> None: + """Listing must not rewrite the state being listed. + + LFPlugin/NFPlugin write on pytest_sessionfinish, which is why this + command deliberately runs without a Session. + """ + self.populate(pytester, "alpha") + cachedir = next(user_cache.iterdir()) + lastfailed = next(cachedir.glob("s/*/v/cache/lastfailed")) + before = lastfailed.read_bytes() + + pytester.runpytest("--cache-list") + assert lastfailed.read_bytes() == before + + def test_with_help(self, pytester: Pytester, user_cache: Path) -> None: + result = pytester.runpytest("--cache-list", "--help") + assert result.ret == 0 + result.stdout.fnmatch_lines(["*--cache-list*"]) + + def test_hyperlinks( + self, pytester: Pytester, user_cache: Path, monkeypatch: MonkeyPatch + ) -> None: + project = self.populate(pytester, "alpha") + # Pytester forces PY_COLORS=0 for inner runs, so ask explicitly. + monkeypatch.setenv("PY_COLORS", "1") + monkeypatch.setenv("PYTEST_HYPERLINKS", "1") + + result = pytester.runpytest_subprocess("--cache-list") + assert result.ret == 0 + cachedir = next(user_cache.iterdir()) + stdout = result.stdout.str() + assert f"\x1b]8;;{cachedir.as_uri()}\x1b\\" in stdout + assert f"\x1b]8;;{project.as_uri()}\x1b\\" in stdout + + def test_no_hyperlinks_when_piped( + self, pytester: Pytester, user_cache: Path + ) -> None: + self.populate(pytester, "alpha") + result = pytester.runpytest_subprocess("--cache-list") + assert "\x1b]8;;" not in result.stdout.str() + + class TestCacheInfo: @pytest.fixture def cache(self, pytester: Pytester) -> Cache: From e76ab4691d0ba151bce4d8c89e398ea7182facae Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Mon, 3 Aug 2026 22:20:19 +0200 Subject: [PATCH 11/12] cacheprovider: add --cache-prune Completes the lifetime-management story GH-1089 was blocked on since 2015: a cache outside the worktree no longer shares the worktree's lifetime, so there has to be a way to collect what accumulates. Selectors are `all`, `orphaned` (project gone, or metadata unreadable), `stale` and a glob over the directory name and origin path, repeatable. `stale` works at *scope* level rather than directory level - it drops the state belonging to a deleted virtualenv while leaving the project's cache alone, which is the common maintenance case and the practical payoff of having scopes at all. A selector is required; there is no default, so no bare invocation can delete anything, and --cache-list is the preview. Removal is not interactive, matching --cache-clear; failures are reported per entry and the command exits non-zero rather than stopping. `all` never removes the directory the invoking project would itself use. That self-exclusion deliberately guards whole-directory removal only: a stale scope can never be the one in use, since the running environment exists by definition, and clearing out a deleted virtualenv of the project you are standing in is the most likely reason to run this at all. Nothing locks the cache, so pruning concurrently with a run using it can race. rm_rf failures are reported rather than fatal. Co-Authored-By: Claude Opus 5 (1M context) --- changelog/1089.feature.3.rst | 15 +++ src/_pytest/cacheprovider.py | 95 ++++++++++++++++- testing/test_cacheprovider.py | 191 ++++++++++++++++++++++++++++++++++ 3 files changed, 297 insertions(+), 4 deletions(-) create mode 100644 changelog/1089.feature.3.rst diff --git a/changelog/1089.feature.3.rst b/changelog/1089.feature.3.rst new file mode 100644 index 00000000000..6eed82bc85f --- /dev/null +++ b/changelog/1089.feature.3.rst @@ -0,0 +1,15 @@ +New ``--cache-prune=SELECTOR`` option, removing cache directories or scopes under the user-level cache +root. ``SELECTOR`` is one of: + +* ``all`` - every cache directory except the one this invocation would itself use; +* ``orphaned`` - directories whose project no longer exists, and directories with unreadable metadata; +* ``stale`` - *scopes* whose environment no longer exists, leaving the project's cache directory itself + in place; +* anything else - a glob, matched against the directory name and against the origin project path. + +The option may be given more than once, and requires a selector: there is no default, so no bare +invocation can delete anything. Use ``--cache-list`` to preview. + +Removal is not interactive, matching ``--cache-clear``. Note that nothing locks the cache, so pruning +while another pytest run is using the same cache can race; failures are reported per directory and the +command exits non-zero. diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index b564807a1d4..d82338d77d3 100644 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -8,9 +8,11 @@ from collections.abc import Generator from collections.abc import Iterable from collections.abc import Mapping +from collections.abc import Sequence import dataclasses import enum import errno +import fnmatch import hashlib import json import os @@ -845,6 +847,18 @@ def pytest_addoption(parser: Parser) -> None: "don't perform collection or tests." ), ) + group.addoption( + "--cache-prune", + action="append", + dest="cacheprune", + metavar="SELECTOR", + help=( + "Remove cache directories or scopes under the user-level cache root, " + "don't perform collection or tests. SELECTOR is 'all', 'orphaned', " + "'stale', or a glob matched against the directory name and the origin " + "path. May be given more than once. See --cache-list first." + ), + ) # Empty by default so that "not configured" is detectable, in which case # cache_policy decides the location. parser.addini( @@ -883,7 +897,11 @@ def _cache_command_active(config: Config) -> bool: Such a run must not write to the cache: merely inspecting it should not rewrite the very state being inspected. """ - return bool(config.getoption("cacheshow") or config.getoption("cachelist")) + return bool( + config.getoption("cacheshow") + or config.getoption("cachelist") + or config.getoption("cacheprune") + ) def pytest_cmdline_main(config: Config) -> int | ExitCode | None: @@ -895,13 +913,15 @@ def pytest_cmdline_main(config: Config) -> int | ExitCode | None: from _pytest.main import wrap_session return wrap_session(config, cacheshow) - if config.option.cachelist: + if config.option.cachelist or config.option.cacheprune: # The session-less pattern from helpconfig: no Session means # pytest_sessionfinish never fires, so LFPlugin/NFPlugin cannot clobber - # the cache as a side effect of listing it. There is also nothing to - # collect - the command is about the machine, not this project. + # the cache as a side effect of inspecting it. There is also nothing to + # collect - these commands are about the machine, not this project. config._do_configure() try: + if config.option.cacheprune: + return cache_prune(config, config.option.cacheprune) return cache_list(config) finally: config._ensure_unconfigure() @@ -1281,3 +1301,70 @@ def cache_list(config: Config) -> int: tw.line("") tw.line(f"{len(entries)} directories, {scopes} scopes, {_format_size(total)} total") return 0 + + +def _matches_selector(entry: _CacheDirEntry, selector: str) -> bool: + if selector == "all": + return True + if selector == "orphaned": + return entry.status in ("orphaned", "broken") + if selector == "stale": + # Handled per scope rather than per directory. + return False + return fnmatch.fnmatch(entry.path.name, selector) or ( + entry.origin is not None and fnmatch.fnmatch(entry.origin, selector) + ) + + +def cache_prune(config: Config, selectors: Sequence[str]) -> int: + """Remove cache directories and scopes matching ``selectors``.""" + tw = config.get_terminal_writer() + root = user_cache_root() + tw.line(f"user cache directory: {root}") + + # Never remove the directory this very invocation would use. + assert config.cache is not None + current = config.cache._cachedir + + removed = 0 + failed = False + prune_stale = "stale" in selectors + for entry in collect_cache_dirs(root): + # Never remove the directory this run is itself using. Note this + # guards whole-directory removal only: a *stale* scope can never be + # the one in use, since the running environment exists by definition, + # and pruning the current project's dead environments is the most + # likely thing anyone wants. + if entry.path != current and any( + _matches_selector(entry, selector) for selector in selectors + ): + tw.line(f"removing {entry.path.name} ({_format_size(entry.size)})") + try: + rm_rf(entry.path) + except OSError as exc: + tw.line(f" failed: {exc}") + failed = True + else: + removed += entry.size + continue + + if not prune_stale: + continue + for scope in entry.scopes(): + if scope.status != "stale": + continue + path = entry.path / Cache._CACHE_PREFIX_SCOPES / scope.name + tw.line( + f"removing {entry.path.name}/{scope.name} ({_format_size(scope.size)})" + ) + try: + rm_rf(path) + except OSError as exc: + tw.line(f" failed: {exc}") + failed = True + else: + removed += scope.size + + if removed or not failed: + tw.line(f"reclaimed {_format_size(removed)}") + return ExitCode.USAGE_ERROR if failed else ExitCode.OK diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py index e5bcd943d9e..6f4193daae4 100644 --- a/testing/test_cacheprovider.py +++ b/testing/test_cacheprovider.py @@ -1797,6 +1797,197 @@ def test_no_hyperlinks_when_piped( assert "\x1b]8;;" not in result.stdout.str() +class TestCachePrune: + @pytest.fixture + def user_cache(self, monkeypatch: MonkeyPatch, tmp_path: Path) -> Path: + root = tmp_path / "user-cache" + monkeypatch.setenv("PYTEST_CACHE_HOME", str(root)) + return root + + def populate(self, pytester: Pytester, name: str) -> Path: + project = pytester.path / name + project.mkdir() + project.joinpath("test_x.py").write_text( + "def test_bad(): assert False\n", encoding="UTF-8" + ) + project.joinpath("tox.ini").write_text( + "[pytest]\ncache_policy = user\n", encoding="UTF-8" + ) + pytester.runpytest_subprocess(str(project), "--rootdir", str(project)) + return project + + def names(self, user_cache: Path) -> set[str]: + return {p.name for p in user_cache.iterdir()} + + def test_requires_a_selector(self, pytester: Pytester, user_cache: Path) -> None: + # No default, so a bare invocation can never be destructive. + result = pytester.runpytest("--cache-prune") + assert result.ret == ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines(["*--cache-prune: expected one argument*"]) + + def test_all(self, pytester: Pytester, user_cache: Path) -> None: + self.populate(pytester, "alpha") + self.populate(pytester, "beta") + + result = pytester.runpytest("--cache-prune=all") + assert result.ret == ExitCode.OK + result.stdout.fnmatch_lines(["removing alpha-*", "reclaimed *"]) + assert self.names(user_cache) == set() + + def test_all_skips_the_current_directory( + self, pytester: Pytester, user_cache: Path + ) -> None: + self.populate(pytester, "alpha") + pytester.makeini("[pytest]\ncache_policy = user\n") + # Give this project a cache directory of its own to protect. + pytester.makepyfile(test_a="def test_bad(): assert False") + pytester.runpytest("-q") + mine = _label(pytester.path.name) + + pytester.runpytest("--cache-prune=all") + remaining = self.names(user_cache) + assert not any(n.startswith("alpha-") for n in remaining) + assert any(n.startswith(f"{mine}-") for n in remaining) + + def test_orphaned(self, pytester: Pytester, user_cache: Path) -> None: + alpha = self.populate(pytester, "alpha") + self.populate(pytester, "beta") + shutil.rmtree(alpha) + + pytester.runpytest("--cache-prune=orphaned") + assert not any(n.startswith("alpha-") for n in self.names(user_cache)) + assert any(n.startswith("beta-") for n in self.names(user_cache)) + + def test_orphaned_removes_broken_directories( + self, pytester: Pytester, user_cache: Path + ) -> None: + user_cache.mkdir(parents=True, exist_ok=True) + (user_cache / "mystery-0123456789abcdef").mkdir() + + pytester.runpytest("--cache-prune=orphaned") + assert self.names(user_cache) == set() + + def test_stale_removes_only_the_scope( + self, pytester: Pytester, user_cache: Path + ) -> None: + self.populate(pytester, "alpha") + cachedir = next(user_cache.iterdir()) + info = json.loads((cachedir / CACHE_INFO_NAME).read_text(encoding="UTF-8")) + for scope in info["scopes"].values(): + scope["prefix"] = str(pytester.path / "deleted-venv") + (cachedir / CACHE_INFO_NAME).write_text(json.dumps(info), encoding="UTF-8") + + result = pytester.runpytest("--cache-prune=stale") + assert result.ret == ExitCode.OK + result.stdout.fnmatch_lines(["removing alpha-*/env-*"]) + # The project's own cache directory survives; only the dead + # environment's state goes. + assert cachedir.is_dir() + assert not list((cachedir / "s").iterdir()) + + def test_stale_applies_to_the_current_project_too( + self, pytester: Pytester, user_cache: Path + ) -> None: + """Self-exclusion must not block pruning your own stale scopes. + + It guards whole-directory removal; a stale scope can never be the one + in use, since the running environment exists by definition - and + clearing out a deleted virtualenv of the project you are standing in + is the most likely reason to run this at all. + """ + pytester.makeini("[pytest]\ncache_policy = user\n") + pytester.makepyfile(test_a="def test_bad(): assert False") + pytester.runpytest("-q") + + cachedir = next(user_cache.iterdir()) + info = json.loads((cachedir / CACHE_INFO_NAME).read_text(encoding="UTF-8")) + for scope in info["scopes"].values(): + scope["prefix"] = str(pytester.path / "deleted-venv") + (cachedir / CACHE_INFO_NAME).write_text(json.dumps(info), encoding="UTF-8") + + result = pytester.runpytest("--cache-prune=stale") + result.stdout.fnmatch_lines(["removing *env-*"]) + assert cachedir.is_dir() + assert not list((cachedir / "s").iterdir()) + + def test_glob_matches_name_or_origin( + self, pytester: Pytester, user_cache: Path + ) -> None: + self.populate(pytester, "alpha") + self.populate(pytester, "beta") + + pytester.runpytest("--cache-prune=alpha-*") + assert not any(n.startswith("alpha-") for n in self.names(user_cache)) + assert any(n.startswith("beta-") for n in self.names(user_cache)) + + pytester.runpytest(f"--cache-prune={pytester.path}/beta") + assert self.names(user_cache) == set() + + def test_selectors_are_repeatable( + self, pytester: Pytester, user_cache: Path + ) -> None: + self.populate(pytester, "alpha") + self.populate(pytester, "beta") + self.populate(pytester, "gamma") + + pytester.runpytest("--cache-prune=alpha-*", "--cache-prune=beta-*") + assert {n.split("-")[0] for n in self.names(user_cache)} == {"gamma"} + + def test_no_match_removes_nothing( + self, pytester: Pytester, user_cache: Path + ) -> None: + self.populate(pytester, "alpha") + result = pytester.runpytest("--cache-prune=nothing-matches-this") + assert result.ret == ExitCode.OK + assert any(n.startswith("alpha-") for n in self.names(user_cache)) + + def test_does_not_clobber_the_cache( + self, pytester: Pytester, user_cache: Path + ) -> None: + """Pruning must not rewrite the caches it leaves alone. + + The direct regression test for the LFPlugin/NFPlugin sessionfinish + hazard: a no-match prune has to be a complete no-op. + """ + project = self.populate(pytester, "alpha") + cachedir = next(user_cache.iterdir()) + values = { + p: p.read_bytes() for p in cachedir.glob("s/*/v/cache/*") if p.is_file() + } + assert values + + pytester.runpytest("--cache-prune=nothing-matches-this") + assert {p: p.read_bytes() for p in values} == values + + # ... and --lf still works afterwards. + result = pytester.runpytest_subprocess( + str(project), "--rootdir", str(project), "--lf", "-v" + ) + result.stdout.fnmatch_lines(["*rerun previous 1 failure*"]) + + def test_with_help(self, pytester: Pytester, user_cache: Path) -> None: + result = pytester.runpytest("--cache-prune=all", "--help") + assert result.ret == 0 + result.stdout.fnmatch_lines(["*--cache-prune*"]) + + @pytest.mark.skipif(sys.platform == "win32", reason="no chmod on win32") + def test_reports_failures_and_continues( + self, pytester: Pytester, user_cache: Path + ) -> None: + self.populate(pytester, "alpha") + self.populate(pytester, "beta") + alpha = next(p for p in user_cache.iterdir() if p.name.startswith("alpha-")) + user_cache.chmod(0o500) + try: + result = pytester.runpytest("--cache-prune=all") + finally: + user_cache.chmod(0o700) + + assert result.ret == ExitCode.USAGE_ERROR + result.stdout.fnmatch_lines(["*failed: *"]) + assert alpha.exists() + + class TestCacheInfo: @pytest.fixture def cache(self, pytester: Pytester) -> Cache: From 2ed96075f8cf0c6ae734570e18265a761d17a741 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 5 Aug 2026 11:11:46 +0200 Subject: [PATCH 12/12] docs: document cache policy, scopes and pruning Three new sections in the cache how-to - scopes, where the cache is stored, and listing/pruning - plus the cache_policy confval, the PYTEST_CACHE_POLICY, PYTEST_CACHE_HOME and PYTEST_HYPERLINKS environment variables, the two new options, and CacheScope in the API reference. The sample --cache-list output is a literal code block rather than regendoc output, since it is full of machine-specific paths. The regendoc-captured --help and ini listings are updated; nothing else moved, because the default resolved location is unchanged. One caveat is called out because it is not obvious and does bite: a project on an unmounted volume looks `orphaned`, which is the main reason pruning is never automatic. Co-Authored-By: Claude Opus 5 (1M context) --- doc/en/how-to/cache.rst | 152 +++++++++++++++++++++++++++++++++ doc/en/reference/reference.rst | 86 ++++++++++++++++++- 2 files changed, 235 insertions(+), 3 deletions(-) diff --git a/doc/en/how-to/cache.rst b/doc/en/how-to/cache.rst index ef0919ef1c8..4c40632a43a 100644 --- a/doc/en/how-to/cache.rst +++ b/doc/en/how-to/cache.rst @@ -332,6 +332,158 @@ servers where isolation and correctness is more important than speed. +.. _cache_scopes: + +Cache scopes +------------ + +.. versionadded:: 9.2 + +Not everything in the cache is equally portable. Which tests exist, and which are skipped, depends on the +interpreter running them, so the state behind :option:`--lf`, :option:`--nf` and :option:`--sw <--sw>` is +only meaningful for the environment that recorded it. + +pytest keeps such values apart *within* a single cache directory, rather than needing one cache directory +per environment. Every cached value has a :class:`~pytest.CacheScope`: + +.. list-table:: + :header-rows: 1 + + * - Scope + - Valid for + * - :attr:`CacheScope.SHARED ` + - the project, whatever runs it. The default. + * - :attr:`CacheScope.PYTHON ` + - one Python implementation and ``major.minor`` version + * - :attr:`CacheScope.ENV ` + - one environment, i.e. one :data:`sys.prefix` + +``--lf``, ``--nf`` and ``--sw`` use ``ENV``. Running a project under a tox or nox matrix, or simply under +two virtualenvs, therefore no longer has each run overwrite the previous one's last-failed set. + +Plugins can do the same: + +.. code-block:: python + + def pytest_configure(config): + # Valid anywhere the project is. + config.cache.set("myplugin/schema-version", 3) + + # Only valid for the environment that collected it. + config.cache.set("myplugin/collected", ids, scope=pytest.CacheScope.ENV) + +Reads must use the same scope they were written with. ``SHARED`` is the default, so existing plugins keep +working and keep their existing on-disk location. + + +.. _cache_location: + +Where the cache is stored +------------------------- + +.. versionadded:: 9.2 + +By default the cache lives in ``.pytest_cache`` inside the :ref:`rootdir `. The +:confval:`cache_policy` option chooses somewhere else by name: + +.. code-block:: ini + + [pytest] + cache_policy = user + +``local`` + ``/.pytest_cache``. The default. + +``user`` + A directory keyed by project inside the platform's user cache directory - ``$XDG_CACHE_HOME/pytest`` + or ``~/.cache/pytest`` on Linux, ``~/Library/Caches/pytest`` on macOS, ``%LOCALAPPDATA%\pytest\Cache`` + on Windows. Nothing at all is written into the project. + + This requires the ``xdg`` extra:: + + pip install pytest[xdg] + +:confval:`cache_dir` remains available and always wins: it is an explicit path, while ``cache_policy`` +only chooses a location when ``cache_dir`` is unset. Use it for anywhere the two policies do not name - +it expands environment variables, so a cache inside the current virtualenv is: + +.. code-block:: ini + + [pytest] + cache_dir = $VIRTUAL_ENV/.pytest_cache + +To opt in for a whole machine without editing every project, set the environment variable instead: + +.. code-block:: bash + + export PYTEST_CACHE_POLICY=user + +An explicit ``cache_policy`` in a config file still wins over it. + +Under the ``user`` policy the directory is named after the project and a digest of its path, so it stays +recognisable:: + + ~/.cache/pytest/myproject-1a2b3c4d5e6f7a8b/ + +The key is the rootdir alone - not the interpreter, which is handled by :ref:`cache scopes ` +instead. One project therefore gets one cache directory however many environments run it. A *new* +directory only appears if the project itself moves. + + +.. _cache_pruning: + +Listing and pruning caches +-------------------------- + +.. versionadded:: 9.2 + +A cache stored inside the project is deleted along with the project. One stored under the ``user`` policy +is not, so pytest can tell you what has accumulated: + +.. code-block:: bash + + pytest --cache-list + +.. code-block:: text + + user cache directory: /home/ronny/.cache/pytest + + DIRECTORY SIZE LAST USED STATUS ORIGIN + myproject-1a2b3c4d5e6f 12.4 MiB 2 days ok /home/ronny/src/myproject + env-venv-9f8e7d6c 1.1 MiB 2 days ok /home/ronny/src/myproject/.venv + env-py312-0a1b2c3d 840.1 KiB 94 days stale /home/ronny/src/myproject/.tox/py312 + oldthing-9f8e7d6c5b4a 840.0 KiB 31 days orphaned /home/ronny/src/oldthing + + 2 directories, 3 scopes, 13.2 MiB total + +``orphaned`` means the origin project no longer exists, and ``stale`` means the environment a scope holds +state for no longer exists. In terminals which support it, the directory and origin columns are clickable +links. + +Nothing is ever removed automatically. To remove things, say which: + +.. code-block:: bash + + pytest --cache-prune=stale # state for environments that are gone + pytest --cache-prune=orphaned # caches for projects that are gone + pytest --cache-prune='oldthing-*' # by name, or by origin path + pytest --cache-prune=all + +``--cache-prune`` requires a selector, so a bare invocation cannot delete anything, and it does not ask +for confirmation once given one - use ``--cache-list`` as the preview. ``all`` never removes the cache +directory of the project you run it from. + +.. note:: + + A project on an unmounted network or removable volume looks ``orphaned``, and a virtualenv on one + looks ``stale``. This is the main reason pruning is never automatic. + +.. note:: + + ``--cache-list`` and ``--cache-prune`` only see the user-level cache root. Caches placed somewhere + else with :confval:`cache_dir` are not tracked, since pytest has no way to know where they all are. + + .. _cache stepwise: Stepwise diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index 2627824624e..895f53bb46f 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -457,6 +457,11 @@ Under the hood, the cache plugin uses the simple .. autoclass:: pytest.Cache() :members: +How far a cached value travels is given by its :class:`pytest.CacheScope`; see :ref:`cache_scopes`. + +.. autoclass:: pytest.CacheScope() + :members: + .. fixture:: doctest_namespace @@ -1215,6 +1220,20 @@ Environment variables that can be used to change pytest's behavior. This is not meant to be set by users, but is set by pytest internally with the name of the current test so other processes can inspect it, see :ref:`pytest current test env` for more information. +.. envvar:: PYTEST_CACHE_HOME + + Overrides the user-level cache root used by ``cache_policy = user``, instead of asking the platform + for it. Useful for a shared, explicit location in CI. See :ref:`cache_location`. + + .. versionadded:: 9.2 + +.. envvar:: PYTEST_CACHE_POLICY + + Sets the default for :confval:`cache_policy`, so that the cache can be relocated for a whole machine + without editing every project. An explicit :confval:`cache_policy` still wins over it. + + .. versionadded:: 9.2 + .. envvar:: PYTEST_DEBUG When set, pytest will print tracing and debug information. @@ -1246,6 +1265,14 @@ Environment variables that can be used to change pytest's behavior. Entry point names of installed plugins are now also accepted, in addition to importable module names. +.. envvar:: PYTEST_HYPERLINKS + + When set to ``1``, pytest emits OSC 8 terminal hyperlinks, so that paths it prints are clickable. + When set to ``0``, it never does. Otherwise they are emitted whenever terminal color is in use and + the terminal is not one known to mishandle them. + + .. versionadded:: 9.2 + .. envvar:: PYTEST_THEME Sets a `pygment style `_ to use for the code output. @@ -1370,14 +1397,38 @@ passed multiple times. The expected format is ``name=value``. For example:: .. confval:: cache_dir :type: ``str`` - :default: ``".pytest_cache"`` + :default: ``""`` - Sets the directory where the cache plugin's content is stored. + Sets the directory where the cache plugin's content is stored, overriding :confval:`cache_policy`. Directory may be relative or absolute path. If setting relative path, then directory is created relative to :ref:`rootdir `. Additionally, a path may contain environment variables, that will be expanded. For more information about cache plugin please refer to :ref:`cache_provider`. + When unset, the location is decided by :confval:`cache_policy` instead. + + .. versionchanged:: 9.2 + + The default is now empty rather than ``".pytest_cache"``, so that leaving it unset can be told + apart from setting it. The resulting location is unchanged. + +.. confval:: cache_policy + :type: ``Literal["local", "user"]`` + :default: ``"local"`` + + .. versionadded:: 9.2 + + Chooses where the cache directory lives, without spelling out a path: + + * ``local`` - ``/.pytest_cache``. + * ``user`` - a per-project directory inside the platform's user cache directory. Requires the + ``xdg`` extra (``pip install pytest[xdg]``). + + Ignored when :confval:`cache_dir` is set. The default can be set for a whole machine with the + :envvar:`PYTEST_CACHE_POLICY` environment variable. + + See :ref:`cache_location` and :ref:`cache_pruning`. + .. confval:: collect_imported_tests :type: ``bool`` :default: ``true`` @@ -3237,6 +3288,21 @@ Cache Remove all cache contents at start of test run. See :ref:`cache`. +.. option:: --cache-list + + List the cache directories under the user-level cache root, don't perform collection or tests. + See :ref:`cache_pruning`. + + .. versionadded:: 9.2 + +.. option:: --cache-prune=SELECTOR + + Remove cache directories or scopes under the user-level cache root, don't perform collection or + tests. ``SELECTOR`` is ``all``, ``orphaned``, ``stale``, or a glob matched against the directory + name and the origin path. May be given more than once. See :ref:`cache_pruning`. + + .. versionadded:: 9.2 + Warnings ~~~~~~~~ @@ -3482,6 +3548,15 @@ All the command-line flags can also be obtained by running ``pytest --help``:: Show cache contents, don't perform collection or tests. Optional argument: glob (default: '*'). --cache-clear Remove all cache contents at start of test run + --cache-list List the cache directories under the user-level + cache root, don't perform collection or tests. + --cache-prune=SELECTOR + Remove cache directories or scopes under the user- + level cache root, don't perform collection or tests. + SELECTOR is 'all', 'orphaned', 'stale', or a glob + matched against the directory name and the origin + path. May be given more than once. See --cache-list + first. --lfnf, --last-failed-no-failures={all,none} With ``--lf``, determines whether to execute tests when there are no previously (known) failures or @@ -3747,7 +3822,12 @@ All the command-line flags can also be obtained by running ``pytest --help``:: Option flags for doctests doctest_encoding (string): Encoding used for doctest files - cache_dir (string): Cache directory path + cache_dir (string): Cache directory path; overrides cache_policy + cache_policy ('local' | 'user'): + Where the cache directory lives: 'local' + (rootdir/.pytest_cache) or 'user' (the platform's + user cache directory, keyed by project). Ignored if + cache_dir is set. log_level (string): Default value for --log-level log_format (string): Default value for --log-format log_date_format (string):