From bcb44254e6223313e1f34aeb70b7e6d7ff6ad1d0 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:50:36 +0300 Subject: [PATCH 1/7] Recreate venv for each build --- build_docs.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/build_docs.py b/build_docs.py index f76cf1a..358ed2d 100755 --- a/build_docs.py +++ b/build_docs.py @@ -804,24 +804,30 @@ def build(self) -> None: def build_venv(self) -> None: """Build a venv for the specific Python version. - So we can reuse them from builds to builds, while they contain - different Sphinx versions. + The venv is recreated from scratch for every build: pip considers + a requirement satisfied when the installed version number matches, + even if the requirement is a direct URL now pointing at different + code, so a reused venv can silently keep outdated packages + (see python/cpython#153227). """ requirements = list(self.build_meta.dependencies) if self.includes_html: # opengraph previews requirements.append("matplotlib>=3") - venv_path = self.build_root / self.build_meta.venv_name - venv.create(venv_path, symlinks=os.name != "nt", with_pip=True) + venv_name = self.build_meta.venv_name + if self.select_output is not None: + # Never share a venv with a concurrent differently-selected + # build, which may recreate it mid-build. + venv_name += f"-{self.select_output}" + venv_path = self.build_root / venv_name + venv.create(venv_path, symlinks=os.name != "nt", with_pip=True, clear=True) run( ( venv_path / "bin" / "python", "-m", "pip", "install", - "--upgrade", - "--upgrade-strategy=eager", self.theme, *requirements, ), From 8e93c0ecb4fe8bb4c013f91cc9b2fdf98a9460f4 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:59:10 +0300 Subject: [PATCH 2/7] Install from pylock.toml when available --- build_docs.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/build_docs.py b/build_docs.py index 358ed2d..0fd4a8e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -822,18 +822,20 @@ def build_venv(self) -> None: venv_name += f"-{self.select_output}" venv_path = self.build_root / venv_name venv.create(venv_path, symlinks=os.name != "nt", with_pip=True, clear=True) + python = venv_path / "bin" / "python" + run((python, "-m", "pip", "install", "--upgrade", "pip")) + + if (self.checkout / "Doc" / "pylock.toml").exists(): + requirements.remove("-rrequirements.txt") + run( + (python, "-m", "pip", "install", "-rpylock.toml"), + cwd=self.checkout / "Doc", + ) run( - ( - venv_path / "bin" / "python", - "-m", - "pip", - "install", - self.theme, - *requirements, - ), + (python, "-m", "pip", "install", self.theme, *requirements), cwd=self.checkout / "Doc", ) - run((venv_path / "bin" / "python", "-m", "pip", "freeze", "--all")) + run((python, "-m", "pip", "freeze", "--all")) self.venv = venv_path def setup_indexsidebar(self) -> None: From da1589e21c5f4a37b3fab07526e121cf37990597 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:12:20 +0300 Subject: [PATCH 3/7] Upgrade pip via upgrade_deps=True --- build_docs.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 0fd4a8e..251a1ec 100755 --- a/build_docs.py +++ b/build_docs.py @@ -821,9 +821,14 @@ def build_venv(self) -> None: # build, which may recreate it mid-build. venv_name += f"-{self.select_output}" venv_path = self.build_root / venv_name - venv.create(venv_path, symlinks=os.name != "nt", with_pip=True, clear=True) + venv.create( + venv_path, + symlinks=os.name != "nt", + with_pip=True, + clear=True, + upgrade_deps=True, + ) python = venv_path / "bin" / "python" - run((python, "-m", "pip", "install", "--upgrade", "pip")) if (self.checkout / "Doc" / "pylock.toml").exists(): requirements.remove("-rrequirements.txt") From 4d072db7e0dbfa63a0f40b696de5a8025742269a Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 30 Jul 2026 16:40:00 +0300 Subject: [PATCH 4/7] Remove reference to issue Co-authored-by: Stan Ulbrych --- build_docs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 251a1ec..48b4100 100755 --- a/build_docs.py +++ b/build_docs.py @@ -807,8 +807,7 @@ def build_venv(self) -> None: The venv is recreated from scratch for every build: pip considers a requirement satisfied when the installed version number matches, even if the requirement is a direct URL now pointing at different - code, so a reused venv can silently keep outdated packages - (see python/cpython#153227). + code, so a reused venv can silently keep outdated packages. """ requirements = list(self.build_meta.dependencies) if self.includes_html: From 3a8b60e747b8c83eb5bac6d129bb36dc456ea679 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:01:27 +0300 Subject: [PATCH 5/7] Create each venv at most once per run --- build_docs.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/build_docs.py b/build_docs.py index 48b4100..7cb2eb0 100755 --- a/build_docs.py +++ b/build_docs.py @@ -641,6 +641,7 @@ class DocBuilder: cpython_repo: Repository docs_by_version_content: bytes switchers_content: bytes + built_venvs: set[Path] build_root: Path www_root: Path select_output: Literal["no-html", "only-html", "only-html-en"] | None @@ -804,22 +805,28 @@ def build(self) -> None: def build_venv(self) -> None: """Build a venv for the specific Python version. - The venv is recreated from scratch for every build: pip considers - a requirement satisfied when the installed version number matches, - even if the requirement is a direct URL now pointing at different - code, so a reused venv can silently keep outdated packages. + The venv is created at most once per run and reused by later + builds of the same version: reusing a venv across runs can + silently keep outdated packages, because pip considers a + requirement satisfied when the installed version number matches, + even if the requirement is a direct URL now pointing at + different code. """ - requirements = list(self.build_meta.dependencies) - if self.includes_html: - # opengraph previews - requirements.append("matplotlib>=3") - venv_name = self.build_meta.venv_name if self.select_output is not None: # Never share a venv with a concurrent differently-selected # build, which may recreate it mid-build. venv_name += f"-{self.select_output}" venv_path = self.build_root / venv_name + if venv_path in self.built_venvs: + self.venv = venv_path + return + + requirements = list(self.build_meta.dependencies) + if self.includes_html: + # opengraph previews + requirements.append("matplotlib>=3") + venv.create( venv_path, symlinks=os.name != "nt", @@ -840,6 +847,7 @@ def build_venv(self) -> None: cwd=self.checkout / "Doc", ) run((python, "-m", "pip", "freeze", "--all")) + self.built_venvs.add(venv_path) self.venv = venv_path def setup_indexsidebar(self) -> None: @@ -1275,6 +1283,7 @@ def build_docs(args: argparse.Namespace) -> int: "https://github.com/python/cpython.git", args.build_root / _checkout_name(args.select_output), ) + built_venvs: set[Path] = set() while todo: build_props = todo.pop() logging.root.handlers[0].setFormatter( @@ -1292,6 +1301,7 @@ def build_docs(args: argparse.Namespace) -> int: cpython_repo, docs_by_version_content, switchers_content, + built_venvs, **vars(args), ) built_successfully = builder.run(http, force_build=force_build) From c236c899348cfaac65461631d2e232f4e9aa4e2c Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:03:03 +0300 Subject: [PATCH 6/7] Delete venvs at the end of the run --- build_docs.py | 64 +++++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/build_docs.py b/build_docs.py index 7cb2eb0..0319e62 100755 --- a/build_docs.py +++ b/build_docs.py @@ -805,12 +805,12 @@ def build(self) -> None: def build_venv(self) -> None: """Build a venv for the specific Python version. - The venv is created at most once per run and reused by later - builds of the same version: reusing a venv across runs can - silently keep outdated packages, because pip considers a - requirement satisfied when the installed version number matches, - even if the requirement is a direct URL now pointing at - different code. + The venv is created at most once per run, reused by later builds + of the same version, and removed at the end of the run: reusing + a venv across runs can silently keep outdated packages, because + pip considers a requirement satisfied when the installed version + number matches, even if the requirement is a direct URL now + pointing at different code. """ venv_name = self.build_meta.venv_name if self.select_output is not None: @@ -1284,31 +1284,35 @@ def build_docs(args: argparse.Namespace) -> int: args.build_root / _checkout_name(args.select_output), ) built_venvs: set[Path] = set() - while todo: - build_props = todo.pop() - logging.root.handlers[0].setFormatter( - logging.Formatter( - f"%(asctime)s %(levelname)s {build_props.slug}: %(message)s" + try: + while todo: + build_props = todo.pop() + logging.root.handlers[0].setFormatter( + logging.Formatter( + f"%(asctime)s %(levelname)s {build_props.slug}: %(message)s" + ) ) - ) - if sentry_sdk: - scope = sentry_sdk.get_isolation_scope() - scope.set_tag("version", build_props.version) - scope.set_tag("language", build_props.language) - cpython_repo.update() - builder = DocBuilder( - build_props, - cpython_repo, - docs_by_version_content, - switchers_content, - built_venvs, - **vars(args), - ) - built_successfully = builder.run(http, force_build=force_build) - if built_successfully: - build_succeeded.add(build_props.slug) - elif built_successfully is not None: - any_build_failed = True + if sentry_sdk: + scope = sentry_sdk.get_isolation_scope() + scope.set_tag("version", build_props.version) + scope.set_tag("language", build_props.language) + cpython_repo.update() + builder = DocBuilder( + build_props, + cpython_repo, + docs_by_version_content, + switchers_content, + built_venvs, + **vars(args), + ) + built_successfully = builder.run(http, force_build=force_build) + if built_successfully: + build_succeeded.add(build_props.slug) + elif built_successfully is not None: + any_build_failed = True + finally: + for venv_path in built_venvs: + shutil.rmtree(venv_path, ignore_errors=True) logging.root.handlers[0].setFormatter( logging.Formatter("%(asctime)s %(levelname)s: %(message)s") From a4cfdda8faff2fccb8eefeedad77e9d0c498c45d Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 30 Jul 2026 23:19:30 +0300 Subject: [PATCH 7/7] Check pylock.toml is_file instead of exists Co-authored-by: Stan Ulbrych --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 0319e62..eb304b6 100755 --- a/build_docs.py +++ b/build_docs.py @@ -836,7 +836,7 @@ def build_venv(self) -> None: ) python = venv_path / "bin" / "python" - if (self.checkout / "Doc" / "pylock.toml").exists(): + if (self.checkout / "Doc" / "pylock.toml").is_file(): requirements.remove("-rrequirements.txt") run( (python, "-m", "pip", "install", "-rpylock.toml"),