From 19cf03b6198878047533b2c6c054951b29c468ef Mon Sep 17 00:00:00 2001 From: "Jonathan J. Helmus" Date: Fri, 17 Jul 2026 14:09:01 -0500 Subject: [PATCH 01/12] require zlib on macOS --- cpython-unix/targets.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpython-unix/targets.yml b/cpython-unix/targets.yml index c39f20baf..911ca72f5 100644 --- a/cpython-unix/targets.yml +++ b/cpython-unix/targets.yml @@ -113,6 +113,7 @@ aarch64-apple-darwin: - tk - uuid - xz + - zlib - zstd openssl_target: darwin64-arm64-cc @@ -664,6 +665,7 @@ x86_64-apple-darwin: - tk - uuid - xz + - zlib - zstd openssl_target: darwin64-x86_64-cc From 5e98fadf38c2bddefbe5eccdd773a09a3fa84b13 Mon Sep 17 00:00:00 2001 From: "Jonathan J. Helmus" Date: Fri, 17 Jul 2026 13:51:36 -0500 Subject: [PATCH 02/12] add pkgconf and require it on macOS for CPython --- cpython-unix/Makefile | 4 ++++ cpython-unix/build-pkgconf.sh | 29 +++++++++++++++++++++++++++++ cpython-unix/build.py | 3 ++- cpython-unix/targets.yml | 2 ++ pythonbuild/downloads.py | 6 ++++++ scripts/update_downloads.py | 5 +++++ 6 files changed, 48 insertions(+), 1 deletion(-) create mode 100755 cpython-unix/build-pkgconf.sh diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile index 25a6b2aa2..adaf953fc 100644 --- a/cpython-unix/Makefile +++ b/cpython-unix/Makefile @@ -186,6 +186,9 @@ $(OUTDIR)/libedit-$(LIBEDIT_VERSION)-$(PACKAGE_SUFFIX).tar: $(LIBEDIT_DEPENDS) $(OUTDIR)/patchelf-$(PATCHELF_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-patchelf.sh $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) patchelf +$(OUTDIR)/pkgconf-$(PKGCONF_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-pkgconf.sh + $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) pkgconf + $(OUTDIR)/sqlite-$(SQLITE_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-sqlite.sh $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) sqlite @@ -265,6 +268,7 @@ PYTHON_DEPENDS_$(1) := \ $$(if $$(NEED_OPENSSL_1_1),$$(OUTDIR)/openssl-1.1-$$(OPENSSL_1.1_VERSION)-$$(PACKAGE_SUFFIX).tar) \ $$(if $$(NEED_OPENSSL_3_5),$$(OUTDIR)/openssl-3.5-$$(OPENSSL_3.5_VERSION)-$$(PACKAGE_SUFFIX).tar) \ $$(if $$(NEED_PATCHELF),$$(OUTDIR)/patchelf-$$(PATCHELF_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_PKGCONF),$$(OUTDIR)/pkgconf-$$(PKGCONF_VERSION)-$$(PACKAGE_SUFFIX).tar) \ $$(if $$(NEED_SQLITE),$$(OUTDIR)/sqlite-$$(SQLITE_VERSION)-$$(PACKAGE_SUFFIX).tar) \ $$(if $$(NEED_TCL),$$(OUTDIR)/tcl-$$(TCL_VERSION)-$$(PACKAGE_SUFFIX).tar) \ $$(if $$(NEED_TK),$$(OUTDIR)/tk-$$(TK_VERSION)-$$(PACKAGE_SUFFIX).tar) \ diff --git a/cpython-unix/build-pkgconf.sh b/cpython-unix/build-pkgconf.sh new file mode 100755 index 000000000..c15ae88c7 --- /dev/null +++ b/cpython-unix/build-pkgconf.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +set -ex + +ROOT=$(pwd) + +export PATH=${TOOLS_PATH}/${TOOLCHAIN}/bin:${TOOLS_PATH}/host/bin:$PATH + +tar -xf "pkgconf-${PKGCONF_VERSION}.tar.xz" + +pushd "pkgconf-${PKGCONF_VERSION}" + +CC="${HOST_CC}" CXX="${HOST_CXX}" CFLAGS="${EXTRA_HOST_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_HOST_CFLAGS} -fPIC" LDFLAGS="${EXTRA_HOST_LDFLAGS}" ./configure \ + --build="${BUILD_TRIPLE}" \ + --prefix=/tools/host \ + --disable-shared \ + --disable-dependency-tracking \ + --with-system-libdir=/usr/lib \ + --with-system-includedir=/usr/include + +make -j "${NUM_CPUS}" +make -j "${NUM_CPUS}" install DESTDIR="${ROOT}/out" + +ln -s pkgconf "${ROOT}/out/tools/host/bin/pkg-config" + +"${ROOT}/out/tools/host/bin/pkg-config" --version diff --git a/cpython-unix/build.py b/cpython-unix/build.py index c3bc0bbfa..50be46024 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -1154,6 +1154,7 @@ def main(): "ncurses", "openssl-3.5", "patchelf", + "pkgconf", "sqlite", "tcl", "uuid", @@ -1164,7 +1165,7 @@ def main(): "zlib", "zstd", ): - tools_path = "host" if action in ("m4", "patchelf") else "deps" + tools_path = "host" if action in ("m4", "patchelf", "pkgconf") else "deps" extra_archives = { "tcl": {"zlib"}, }.get(action) diff --git a/cpython-unix/targets.yml b/cpython-unix/targets.yml index 911ca72f5..b0f6c5196 100644 --- a/cpython-unix/targets.yml +++ b/cpython-unix/targets.yml @@ -108,6 +108,7 @@ aarch64-apple-darwin: - m4 - mpdecimal - openssl-3.5 + - pkgconf - sqlite - tcl - tk @@ -660,6 +661,7 @@ x86_64-apple-darwin: - m4 - mpdecimal - openssl-3.5 + - pkgconf - sqlite - tcl - tk diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py index b07138d4d..ddd6e3a18 100644 --- a/pythonbuild/downloads.py +++ b/pythonbuild/downloads.py @@ -286,6 +286,12 @@ "sha256": "39e8aeccd7495d54df094d2b4a7c08010ff7777036faaf24f28e07777d1598e2", "version": "0.13.1", }, + "pkgconf": { + "url": "https://github.com/pkgconf/pkgconf/releases/download/pkgconf-3.0.3/pkgconf-3.0.3.tar.xz", + "size": 415512, + "sha256": "aa033abb2b777ba4e66635495a931e53c49d86e4e4e38af68c0f76d666cbd8cf", + "version": "3.0.3", + }, "pip": { "url": "https://files.pythonhosted.org/packages/5d/95/6b5cb3461ea5673ba0995989746db58eb18b91b54dbf331e72f569540946/pip-26.1.2-py3-none-any.whl", "size": 1813144, diff --git a/scripts/update_downloads.py b/scripts/update_downloads.py index 9015e0fab..aea18d120 100644 --- a/scripts/update_downloads.py +++ b/scripts/update_downloads.py @@ -351,6 +351,11 @@ def xorg_policy(category: str, name: str, extension: str = "tar.gz") -> Policy: r"(?P[0-9.]+)", artifact_name="patchelf-{version}.tar.bz2", ), + "pkgconf": github_policy( + "pkgconf/pkgconf", + r"pkgconf-(?P[0-9.]+)", + artifact_name="pkgconf-{version}.tar.xz", + ), "pip": Policy(PyPIDiscovery("pip", r"pip-[0-9A-Za-z.+-]+-py3-none-any\.whl")), "setuptools": Policy( PyPIDiscovery("setuptools", r"setuptools-[0-9A-Za-z.+-]+-py3-none-any\.whl") From 04430c6beceb890ada2e35029c7a780b5aba6e33 Mon Sep 17 00:00:00 2001 From: "Jonathan J. Helmus" Date: Fri, 17 Jul 2026 14:10:41 -0500 Subject: [PATCH 03/12] configure pkgconf to report temp dir Configure pkgconf to report the temporary directories of the prefix that is used on macOS during CPython builds. --- cpython-unix/build-cpython.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index 18e765840..02e5679ed 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -15,6 +15,13 @@ export PKG_CONFIG_PATH=${TOOLS_PATH}/deps/share/pkgconfig:${TOOLS_PATH}/deps/lib # Ensure that `pkg-config` invocations include the static libraries export PKG_CONFIG="pkg-config --static" +# Dependency pkg-config files use /tools/deps as their prefix. macOS builds +# extract dependencies into a temporary directory, so have pkgconf relocate +# their prefix based on the location of each .pc file. +if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then + export PKG_CONFIG="${PKG_CONFIG} --define-prefix" +fi + # configure somehow has problems locating llvm-profdata even though it is in # PATH. The macro it is using allows us to specify its path via an # environment variable. From 607aa62d7cf674474912bced1c431db87c6be2e4 Mon Sep 17 00:00:00 2001 From: "Jonathan J. Helmus" Date: Fri, 17 Jul 2026 15:27:45 -0500 Subject: [PATCH 04/12] build _testclinic, _testclinic_limited and xxlimited_3_13 modules --- cpython-unix/extension-modules.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cpython-unix/extension-modules.yml b/cpython-unix/extension-modules.yml index 05165eb0a..3e8471ae3 100644 --- a/cpython-unix/extension-modules.yml +++ b/cpython-unix/extension-modules.yml @@ -725,6 +725,16 @@ _testcapi: - source: _testcapi/weakref.c minimum-python-version: "3.13" +_testclinic: + minimum-python-version: '3.12' + sources: + - _testclinic.c + +_testclinic_limited: + minimum-python-version: '3.13' + sources: + - _testclinic_limited.c + _testexternalinspection: minimum-python-version: '3.13' maximum-python-version: '3.13' @@ -1102,6 +1112,12 @@ xxlimited_35: disabled-targets: - .* +xxlimited_3_13: + minimum-python-version: '3.15' + build-mode: shared-or-disabled + sources: + - xxlimited_3_13.c + xxsubtype: setup-enabled-conditional: - enabled: true From 178ca66454d1d191ca51e35c074fca1c45dce4f7 Mon Sep 17 00:00:00 2001 From: "Jonathan J. Helmus" Date: Fri, 17 Jul 2026 15:32:15 -0500 Subject: [PATCH 05/12] Use Setup.stdlib for CPython 3.12+ extensions Build standard-library extensions through CPython's configure and Setup.stdlib mechanism for Python 3.12 and newer. Default to statically linking these modules into libpython/python with a few overrides speficied in a Setup.local file created from extension-modules.yaml. This allows CPython's extension checks to run. Python 3.11 and 3.10 contrinue to use the existing process to configure and build extension modules. --- cpython-unix/build-cpython.sh | 56 ++++++------------ .../patch-checksharedmods-disable-3.15.patch | 13 ----- .../patch-checksharedmods-disable.patch | 13 ----- ...ch-configure-disable-stdlib-mod-3.12.patch | 26 --------- pythonbuild/cpython.py | 57 ++++++++++++++++--- 5 files changed, 66 insertions(+), 99 deletions(-) delete mode 100644 cpython-unix/patch-checksharedmods-disable-3.15.patch delete mode 100644 cpython-unix/patch-checksharedmods-disable.patch delete mode 100644 cpython-unix/patch-configure-disable-stdlib-mod-3.12.patch diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index 02e5679ed..f808c2a18 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -210,23 +210,12 @@ if [[ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_10}" ]]; then patch -p1 -i "${ROOT}/patch-posixmodule-remove-system.patch" fi -# Python 3.11 has configure support for configuring extension modules. We really, -# really, really want to use this feature because it looks promising. But at the -# time we added this code the functionality didn't support all extension modules -# nor did it easily support static linking, including static linking of extra -# libraries (which appears to be a limitation of `makesetup`). So for now we -# disable the functionality and require our auto-generated Setup.local to provide -# everything. -if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_11}" ]; then - if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then - # This sets MODULE__STATE=disabled in the Makefile for all extension - # modules that are not unavailable (n/a) based on the platform. - # Valid STATE variables are needed to create the _missing_stdlib_info.py - # file during the build in Python 3.15+ - patch -p1 -i "${ROOT}/patch-configure-disable-stdlib-mod-3.12.patch" - else - patch -p1 -i "${ROOT}/patch-configure-disable-stdlib-mod.patch" - fi +# CPython 3.11 does not configure every standard-library extension, so disable +# its incomplete generated rules and supply all rules through Setup.local. +# CPython 3.12+ uses Setup.stdlib with a small Setup.local for per-module +# link-type overrides. +if [ "${PYTHON_MAJMIN_VERSION}" = "3.11" ]; then + patch -p1 -i "${ROOT}/patch-configure-disable-stdlib-mod.patch" # This hack also prevents the conditional definition of the pwd module in # Setup.bootstrap.in from working. So we remove that conditional. @@ -254,17 +243,6 @@ else patch -p1 -i "${ROOT}/patch-pgo-file-pool-3.11.patch" fi -# There's a post-build Python script that verifies modules were -# built correctly. Ideally we'd invoke this. But our nerfing of -# the configure-based module building and replacing it with our -# own Setup-derived version completely breaks assumptions in this -# script. So leave it off for now... at our own peril. -if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_15}" ]; then - patch -p1 -i "${ROOT}/patch-checksharedmods-disable-3.15.patch" -elif [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then - patch -p1 -i "${ROOT}/patch-checksharedmods-disable.patch" -fi - # CPython < 3.11 always linked against libcrypt. We backport part of # upstream commit be21706f3760bec8bd11f85ce02ed6792b07f51f to avoid this # behavior. @@ -371,13 +349,6 @@ if [[ "${CC}" = "clang" && -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]]; then EXTRA_CONFIGURE_FLAGS="${EXTRA_CONFIGURE_FLAGS} --with-tail-call-interp" fi -# On Python 3.12+ we need to link the special hacl library provided some SHA-256 -# implementations. Since we hack up the regular extension building mechanism, we -# need to reinvent this wheel. -if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then - LDFLAGS="${LDFLAGS} -LModules/_hacl" -fi - # On PPC we need to prevent the glibc 2.22 __tls_get_addr_opt symbol # from being introduced to preserve runtime compatibility with older # glibc. @@ -407,6 +378,12 @@ CONFIGURE_FLAGS=" --without-ensurepip ${EXTRA_CONFIGURE_FLAGS}" +if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then + # Build standard-library extensions as built-ins by default. Setup.local + # overrides only the extensions that must remain shared or disabled. + CONFIGURE_FLAGS="${CONFIGURE_FLAGS} MODULE_BUILDTYPE=static --enable-loadable-sqlite-extensions" +fi + # Build a libpython3.x.so, but statically link the interpreter against # libpython. @@ -788,6 +765,12 @@ BOLT_COMMON_FLAGS="${BOLT_COMMON_FLAGS:-}" BOLT_APPLY_FLAGS="${BOLT_APPLY_FLAGS: # Supplement produced Makefile with our modifications. cat ../Makefile.extra >> Makefile +if [[ "${PYBUILD_PLATFORM}" = macos* && -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]]; then + # configure-derived MODLIBS uses plain -l flags. Hide symbols from bundled + # static dependency libraries in both libpython and the interpreter. + printf '\nMODLIBS := $(subst -l,-Xlinker -hidden-l,$(MODLIBS))\n' >> Makefile +fi + # Debian's PPC64LE GCC 6 defaults to PIE but selects the non-PIE startup object unless -pie is # passed explicitly. Add it only to LINKFORSHARED, which CPython uses when linking executables. # See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81170. @@ -1123,9 +1106,6 @@ if linux_uapi_include_arch: "", ) replace_in_all("-isystem %s/deps/linux-uapi/usr/include" % tools_path, "") -# See https://github.com/python/cpython/issues/145810#issuecomment-4068139183 -replace_in_all("-LModules/_hacl", "") - EOF ${BUILD_PYTHON} "${ROOT}/hack_sysconfig.py" "${ROOT}/out/python" diff --git a/cpython-unix/patch-checksharedmods-disable-3.15.patch b/cpython-unix/patch-checksharedmods-disable-3.15.patch deleted file mode 100644 index 18952a030..000000000 --- a/cpython-unix/patch-checksharedmods-disable-3.15.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/Makefile.pre.in b/Makefile.pre.in -index a6beb96d12a..bb4e1ecd49b 100644 ---- a/Makefile.pre.in -+++ b/Makefile.pre.in -@@ -1610,7 +1610,7 @@ checksharedmods: sharedmods $(PYTHON_FOR_BUILD_DEPS) $(BUILDPYTHON) - else \ - $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py --generate-missing-stdlib-info; \ - fi -- @$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py -+ @echo "module checking disabled" - - .PHONY: rundsymutil - rundsymutil: sharedmods $(PYTHON_FOR_BUILD_DEPS) $(BUILDPYTHON) diff --git a/cpython-unix/patch-checksharedmods-disable.patch b/cpython-unix/patch-checksharedmods-disable.patch deleted file mode 100644 index 52b52a2e8..000000000 --- a/cpython-unix/patch-checksharedmods-disable.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/Makefile.pre.in b/Makefile.pre.in -index 09ceccda1d..6a3a84976e 100644 ---- a/Makefile.pre.in -+++ b/Makefile.pre.in -@@ -990,7 +990,7 @@ sharedmods: $(SHAREDMODS) pybuilddir.txt - # dependency on BUILDPYTHON ensures that the target is run last - .PHONY: checksharedmods - checksharedmods: sharedmods $(PYTHON_FOR_BUILD_DEPS) $(BUILDPYTHON) -- @$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py -+ @echo "module checking disabled" - - .PHONY: rundsymutil - rundsymutil: sharedmods $(PYTHON_FOR_BUILD_DEPS) $(BUILDPYTHON) diff --git a/cpython-unix/patch-configure-disable-stdlib-mod-3.12.patch b/cpython-unix/patch-configure-disable-stdlib-mod-3.12.patch deleted file mode 100644 index 675c31071..000000000 --- a/cpython-unix/patch-configure-disable-stdlib-mod-3.12.patch +++ /dev/null @@ -1,26 +0,0 @@ -diff --git a/configure.ac b/configure.ac -index a284a118f02..7e536c41fda 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -7859,11 +7859,7 @@ AC_DEFUN([PY_STDLIB_MOD], [ - m4_pushdef([modcond], [MODULE_]m4_toupper([$1]))dnl - m4_pushdef([modstate], [py_cv_module_$1])dnl - dnl Check if module has been disabled by PY_STDLIB_MOD_SET_NA() -- AS_IF([test "$modstate" != "n/a"], [ -- AS_IF([m4_ifblank([$2], [true], [$2])], -- [AS_IF([m4_ifblank([$3], [true], [$3])], [modstate=yes], [modstate=missing])], -- [modstate=disabled]) -- ]) -+ AS_IF([test "$modstate" != "n/a"], [modstate=disabled]) - _MODULE_BLOCK_ADD(modcond[_STATE], [$modstate]) - AS_VAR_IF([modstate], [yes], [ - m4_ifblank([$4], [], [_MODULE_BLOCK_ADD([MODULE_]m4_toupper([$1])[_CFLAGS], [$4])]) -@@ -7883,7 +7879,7 @@ AC_DEFUN([PY_STDLIB_MOD_SIMPLE], [ - m4_pushdef([modcond], [MODULE_]m4_toupper([$1]))dnl - m4_pushdef([modstate], [py_cv_module_$1])dnl - dnl Check if module has been disabled by PY_STDLIB_MOD_SET_NA() -- AS_IF([test "$modstate" != "n/a"], [modstate=yes]) -+ AS_IF([test "$modstate" != "n/a"], [modstate=disabled]) - AM_CONDITIONAL(modcond, [test "$modstate" = yes]) - _MODULE_BLOCK_ADD(modcond[_STATE], [$modstate]) - AS_VAR_IF([modstate], [yes], [ diff --git a/pythonbuild/cpython.py b/pythonbuild/cpython.py index 3221760ab..b830f512c 100644 --- a/pythonbuild/cpython.py +++ b/pythonbuild/cpython.py @@ -255,6 +255,8 @@ def derive_setup_local( ): """Derive the content of the Modules/Setup.local file.""" + use_setup_stdlib = meets_python_minimum_version(python_version, "3.12") + # The first part of this function validates that our extension modules YAML # based metadata is in sync with the various files declaring extension # modules in the Python distribution. @@ -343,6 +345,12 @@ def derive_setup_local( except KeyError: setup_bootstrap_in = [] + if use_setup_stdlib: + ifh = tf.extractfile(f"Python-{python_version}/Modules/Setup.stdlib.in") + setup_stdlib_in = ifh.readlines() + else: + setup_stdlib_in = [] + ifh = tf.extractfile("Python-%s/Modules/config.c.in" % python_version) config_c_in = ifh.read() @@ -353,6 +361,18 @@ def derive_setup_local( RE_VARIABLE = re.compile(rb"^[a-zA-Z_]+\s*=") RE_EXTENSION_MODULE = re.compile(rb"^([a-z_]+)\s.*[a-zA-Z/_-]+\.c\b") + RE_STDLIB_EXTENSION_MODULE = re.compile( + rb"^@MODULE_[A-Z0-9_]+_TRUE@([a-z0-9_]+\s+.*)$" + ) + + setup_stdlib_lines = {} + + for line in setup_stdlib_in: + if m := RE_STDLIB_EXTENSION_MODULE.match(line.rstrip()): + line = m.group(1) + name = line.split()[0].decode("ascii") + dist_modules.add(name) + setup_stdlib_lines[name] = line # Setup.bootstrap.in has a simple format. for line in setup_bootstrap_in: @@ -460,10 +480,11 @@ def derive_setup_local( % ", ".join(sorted(extra)) ) - # And with verification out of way, now we generate a Setup.local file - # from our metadata. The verification above ensured that our metadata - # agrees fully with the distribution's knowledge of extensions. So we can - # treat our metadata as canonical. + # And with verification out of way, now we generate a Setup.local file. + # Python 3.12+ builds extensions from Setup.stdlib using configure-derived + # compiler and linker flags. In that case Setup.local only contains the + # shared and disabled overrides; older versions still use the YAML-derived + # compilation rules for every extension. RE_DEFINE = re.compile(rb"-D[^=]+=[^\s]+") @@ -540,7 +561,10 @@ def derive_setup_local( enabled_extensions[name]["setup_line"] = setup_enabled_lines[name] continue - log(f"extension {name} being configured via YAML metadata") + if use_setup_stdlib and name in setup_stdlib_lines: + log(f"extension {name} being configured via Modules/Setup.stdlib") + else: + log(f"extension {name} being configured via YAML metadata") line = name @@ -667,9 +691,10 @@ def derive_setup_local( # around this by detecting the syntax we'd like to support and move the # variable defines to a Makefile supplement that overrides variables for # specific targets. - for m in RE_DEFINE.finditer(parsed["line"]): - for obj_path in sorted(parsed["posix_obj_paths"]): - extra_cflags.setdefault(bytes(obj_path), []).append(m.group(0)) + if not use_setup_stdlib: + for m in RE_DEFINE.finditer(parsed["line"]): + for obj_path in sorted(parsed["posix_obj_paths"]): + extra_cflags.setdefault(bytes(obj_path), []).append(m.group(0)) line = RE_DEFINE.sub(b"", line) @@ -679,9 +704,23 @@ def derive_setup_local( "makesetup: %s" % line.decode("utf-8") ) - section_lines[section].append(line) + # The YAML-derived line is still used to describe packaged object files + # and dependency libraries in PYTHON.json, even when compilation is + # driven by Setup.stdlib. enabled_extensions[name]["setup_line"] = line + if not use_setup_stdlib: + section_lines[section].append(line) + elif section == "shared": + if name not in setup_stdlib_lines: + raise Exception( + f"shared extension {name} has no Modules/Setup.stdlib.in entry" + ) + + # A rule without explicit compiler/linker flags inherits the + # MODULE__CFLAGS/LDFLAGS values produced by configure. + section_lines[section].append(setup_stdlib_lines[name]) + dest_lines = [] for section, lines in sorted(section_lines.items()): From 694718597f5502a640aa1e9470321552e95e21fc Mon Sep 17 00:00:00 2001 From: "Jonathan J. Helmus" Date: Fri, 17 Jul 2026 15:59:13 -0500 Subject: [PATCH 06/12] remove hacl patch Configuration correctly detect the SIMD support in the toolchain and applies the appropiate flags when building libhacl. --- cpython-unix/build-cpython.sh | 7 ------ .../patch-python-configure-hacl-no-simd.patch | 24 ------------------- 2 files changed, 31 deletions(-) delete mode 100644 cpython-unix/patch-python-configure-hacl-no-simd.patch diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index f808c2a18..acd1987eb 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -621,13 +621,6 @@ if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" && "${TARGET_TRIPLE}" == x86_64* PROFILE_TASK="${PROFILE_TASK} --ignore test.test_bytes.BytesTest.test_from_format" fi -# ./configure tries to auto-detect whether it can build 128-bit and 256-bit SIMD helpers for HACL, -# but on x86-64 that requires v2 and v3 respectively, and on arm64 the performance is bad as noted -# in the comments, so just don't even try. (We should check if we can make this conditional) -if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]]; then - patch -p1 -i "${ROOT}/patch-python-configure-hacl-no-simd.patch" -fi - # We use ndbm on macOS and BerkeleyDB elsewhere. if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --with-dbmliborder=ndbm" diff --git a/cpython-unix/patch-python-configure-hacl-no-simd.patch b/cpython-unix/patch-python-configure-hacl-no-simd.patch deleted file mode 100644 index 125aea33f..000000000 --- a/cpython-unix/patch-python-configure-hacl-no-simd.patch +++ /dev/null @@ -1,24 +0,0 @@ -diff --git a/configure.ac b/configure.ac -index a7b2f62579b..06c0c0c0da0 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -7897,8 +7897,7 @@ AC_SUBST([LIBHACL_LDFLAGS]) - # The SIMD files use aligned_alloc, which is not available on older versions of - # Android. - # The *mmintrin.h headers are x86-family-specific, so can't be used on WASI. --if test "$ac_sys_system" != "Linux-android" -a "$ac_sys_system" != "WASI" || \ -- { test -n "$ANDROID_API_LEVEL" && test "$ANDROID_API_LEVEL" -ge 28; } -+if false - then - dnl This can be extended here to detect e.g. Power8, which HACL* should also support. - AX_CHECK_COMPILE_FLAG([-msse -msse2 -msse3 -msse4.1 -msse4.2],[ -@@ -7930,8 +7929,7 @@ AC_SUBST([LIBHACL_BLAKE2_SIMD128_OBJS]) - # Although AVX support is not guaranteed on Android - # (https://developer.android.com/ndk/guides/abis#86-64), this is safe because we do a - # runtime CPUID check. --if test "$ac_sys_system" != "Linux-android" -a "$ac_sys_system" != "WASI" || \ -- { test -n "$ANDROID_API_LEVEL" && test "$ANDROID_API_LEVEL" -ge 28; } -+if false - then - AX_CHECK_COMPILE_FLAG([-mavx2],[ - [LIBHACL_SIMD256_FLAGS="-mavx2"] From b461f2152853056006407a725e1297a21f562fd9 Mon Sep 17 00:00:00 2001 From: "Jonathan J. Helmus" Date: Fri, 17 Jul 2026 16:45:47 -0500 Subject: [PATCH 07/12] do not require zlib on macOS for CPython, use the system --- cpython-unix/targets.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpython-unix/targets.yml b/cpython-unix/targets.yml index b0f6c5196..dca2efa3c 100644 --- a/cpython-unix/targets.yml +++ b/cpython-unix/targets.yml @@ -114,7 +114,6 @@ aarch64-apple-darwin: - tk - uuid - xz - - zlib - zstd openssl_target: darwin64-arm64-cc @@ -667,7 +666,6 @@ x86_64-apple-darwin: - tk - uuid - xz - - zlib - zstd openssl_target: darwin64-x86_64-cc From eae10cdd17b91e0c95d472566c57db72f32a2762 Mon Sep 17 00:00:00 2001 From: "Jonathan J. Helmus" Date: Fri, 17 Jul 2026 16:47:25 -0500 Subject: [PATCH 08/12] do not require a pkg-config compatible zlib on macOS for tcl --- cpython-unix/build-tcl.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpython-unix/build-tcl.sh b/cpython-unix/build-tcl.sh index 140aaeb76..54cd9e14c 100755 --- a/cpython-unix/build-tcl.sh +++ b/cpython-unix/build-tcl.sh @@ -80,6 +80,12 @@ fi export CFLAGS LDFLAGS export CPPFLAGS="${CFLAGS}" +# CPython uses the macOS SDK's zlib without a zlib.pc file. Keep the -lz in +# Libs.private, but do not make pkg-config require the missing metadata. +if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then + sed -i '' -e 's/ zlib >= 1.2.3//' tcl.pc.in +fi + ./configure \ --build="${BUILD_TRIPLE}" \ --host="${TARGET_TRIPLE}" \ From 19914133c5d29b432d1f42cdf97134395a5ade37 Mon Sep 17 00:00:00 2001 From: "Jonathan J. Helmus" Date: Fri, 17 Jul 2026 16:47:53 -0500 Subject: [PATCH 09/12] require x11 for tk when static linking --- cpython-unix/build-tk.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpython-unix/build-tk.sh b/cpython-unix/build-tk.sh index d90f47ce3..3c7fedcd6 100755 --- a/cpython-unix/build-tk.sh +++ b/cpython-unix/build-tk.sh @@ -31,6 +31,10 @@ if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then else LDFLAGS="${LDFLAGS} -Wl,--exclude-libs,ALL" EXTRA_CONFIGURE_FLAGS="--x-includes=${TOOLS_PATH}/deps/include --x-libraries=${TOOLS_PATH}/deps/lib" + + # Make pkg-config --static resolve libX11's private xcb and Xau dependencies. + sed -i '/^Requires: /a\ +Requires.private: x11' tk.pc.in fi CFLAGS="${CFLAGS}" CPPFLAGS="${CFLAGS}" LDFLAGS="${LDFLAGS}" ./configure \ From 059aed07b08fac53b8dfddc6bcc9cda2d3a3b767 Mon Sep 17 00:00:00 2001 From: "Jonathan J. Helmus" Date: Fri, 17 Jul 2026 16:51:17 -0500 Subject: [PATCH 10/12] add _testclinic, _testclinic_limited, and xxlimited_3_13 to validation --- src/validation.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/validation.rs b/src/validation.rs index d130b43e2..84d0e2135 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -1826,12 +1826,24 @@ fn validate_extension_modules( "_xxtestfuzz", ]); + if matches!(python_major_minor, "3.12" | "3.13" | "3.14" | "3.15") { + wanted.insert("_testclinic"); + } + + if matches!(python_major_minor, "3.13" | "3.14" | "3.15") { + wanted.insert("_testclinic_limited"); + } + if !static_crt { wanted.insert("_testcapi"); if !matches!(python_major_minor, "3.10" | "3.11" | "3.12") { wanted.insert("_testlimitedcapi"); } + + if python_major_minor == "3.15" { + wanted.insert("xxlimited_3_13"); + } } } From 06367f8b4eab8c1dc9ccd2b181bbaaaff7290c09 Mon Sep 17 00:00:00 2001 From: "Jonathan J. Helmus" Date: Tue, 21 Jul 2026 13:58:26 -0500 Subject: [PATCH 11/12] disable _testclinic[_limited] and xxlimited_3_13 extensions These extensions were not enabled when using a custom build configuration + build. Maintain this policy. --- cpython-unix/extension-modules.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cpython-unix/extension-modules.yml b/cpython-unix/extension-modules.yml index 3e8471ae3..f34ab2254 100644 --- a/cpython-unix/extension-modules.yml +++ b/cpython-unix/extension-modules.yml @@ -727,11 +727,15 @@ _testcapi: _testclinic: minimum-python-version: '3.12' + disabled-targets: + - .* sources: - _testclinic.c _testclinic_limited: minimum-python-version: '3.13' + disabled-targets: + - .* sources: - _testclinic_limited.c @@ -1114,7 +1118,8 @@ xxlimited_35: xxlimited_3_13: minimum-python-version: '3.15' - build-mode: shared-or-disabled + disabled-targets: + - .* sources: - xxlimited_3_13.c From 6402dfeb79b8dbbaa03eaa8050fab1788ebd4d10 Mon Sep 17 00:00:00 2001 From: "Jonathan J. Helmus" Date: Tue, 21 Jul 2026 13:59:56 -0500 Subject: [PATCH 12/12] remove _testclinic, _testclinic_limited, and xxlimited_3_13 to validation Reverts commit 059aed07b08fac53b8dfddc6bcc9cda2d3a3b767 --- src/validation.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/validation.rs b/src/validation.rs index 84d0e2135..d130b43e2 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -1826,24 +1826,12 @@ fn validate_extension_modules( "_xxtestfuzz", ]); - if matches!(python_major_minor, "3.12" | "3.13" | "3.14" | "3.15") { - wanted.insert("_testclinic"); - } - - if matches!(python_major_minor, "3.13" | "3.14" | "3.15") { - wanted.insert("_testclinic_limited"); - } - if !static_crt { wanted.insert("_testcapi"); if !matches!(python_major_minor, "3.10" | "3.11" | "3.12") { wanted.insert("_testlimitedcapi"); } - - if python_major_minor == "3.15" { - wanted.insert("xxlimited_3_13"); - } } }