From 51652826d7e7a6811297d05f6dec4d94c4397b9a Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 6 Aug 2026 23:20:07 -0400 Subject: [PATCH] feat: expose the precompiled-mode sources to non-CMake builds Adds pybind11.get_source_dir() / python -m pybind11 --srcdir, a srcdir variable in pybind11.pc, and Pybind11Extension(precompile=True) which compiles src/pybind11_combined.cpp into the extension and defines PYBIND11_PRECOMPILED (hard error if the sources are missing). Assisted-by: ClaudeCode:claude-fable-5 --- CMakeLists.txt | 9 +- pybind11/__init__.py | 3 +- pybind11/__main__.py | 9 ++ pybind11/commands.py | 18 ++++ pybind11/setup_helpers.py | 64 +++++++++++++ tests/extra_python_package/test_files.py | 1 + tests/extra_setuptools/test_setuphelper.py | 103 +++++++++++++++++++++ tools/pybind11.pc.in | 1 + 8 files changed, 203 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ea1b3a157b..f53a646b67 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -343,8 +343,8 @@ if(PYBIND11_INSTALL) install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION "${SKBUILD_HEADERS_DIR}") endif() install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) - install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/ - DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src") + set(pybind11_install_srcdir "${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src") + install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/ DESTINATION "${pybind11_install_srcdir}") set(PYBIND11_CMAKECONFIG_INSTALL_DIR "${CMAKE_INSTALL_DATAROOTDIR}/cmake/${PROJECT_NAME}" CACHE STRING "install path for pybind11Config.cmake") @@ -355,9 +355,9 @@ if(PYBIND11_INSTALL) set(pybind11_INCLUDEDIR "\$\{PACKAGE_PREFIX_DIR\}/${CMAKE_INSTALL_INCLUDEDIR}") endif() if(IS_ABSOLUTE "${CMAKE_INSTALL_DATAROOTDIR}") - set(pybind11_SRCDIR "${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src") + set(pybind11_SRCDIR "${pybind11_install_srcdir}") else() - set(pybind11_SRCDIR "\$\{PACKAGE_PREFIX_DIR\}/${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src") + set(pybind11_SRCDIR "\$\{PACKAGE_PREFIX_DIR\}/${pybind11_install_srcdir}") endif() configure_package_config_file( @@ -410,6 +410,7 @@ if(PYBIND11_INSTALL) endif() endif() join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}") + join_paths(srcdir_for_pc_file "\${prefix}" "${pybind11_install_srcdir}") configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" @ONLY) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" diff --git a/pybind11/__init__.py b/pybind11/__init__.py index 3882b2b17b..1d66e119d6 100644 --- a/pybind11/__init__.py +++ b/pybind11/__init__.py @@ -8,12 +8,13 @@ from ._version import __version__, version_info -from .commands import get_cmake_dir, get_include, get_pkgconfig_dir +from .commands import get_cmake_dir, get_include, get_pkgconfig_dir, get_source_dir __all__ = ( "__version__", "get_cmake_dir", "get_include", "get_pkgconfig_dir", + "get_source_dir", "version_info", ) diff --git a/pybind11/__main__.py b/pybind11/__main__.py index ce597c781a..98e9184807 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -17,6 +17,7 @@ get_include_dirs, get_ldflags, get_pkgconfig_dir, + get_source_dir, ) @@ -50,6 +51,12 @@ def main() -> None: action="store_true", help="Print the pkgconfig directory, ideal for setting $PKG_CONFIG_PATH.", ) + parser.add_argument( + "--srcdir", + action="store_true", + help="Print the directory containing the library sources for the optional" + " precompiled mode.", + ) parser.add_argument( "--extension-suffix", action="store_true", @@ -101,6 +108,8 @@ def main() -> None: print(quote(get_cmake_dir())) if args.pkgconfigdir: print(quote(get_pkgconfig_dir())) + if args.srcdir: + print(quote(get_source_dir())) if args.extension_suffix: print(ext_suffix) diff --git a/pybind11/commands.py b/pybind11/commands.py index 8bd0a9bf13..573b7cf4ac 100644 --- a/pybind11/commands.py +++ b/pybind11/commands.py @@ -52,6 +52,24 @@ def get_include(user: bool = False) -> str: # noqa: ARG001 return installed_path if os.path.exists(installed_path) else source_path +def get_source_dir() -> str: + """ + Return the path to the pybind11 library sources, for the optional + precompiled mode. Compile ``pybind11_combined.cpp`` (or the individual + ``.cpp`` files) with ``PYBIND11_PRECOMPILED`` defined, and define that + macro for every translation unit that includes pybind11. + """ + installed_path = os.path.join(DIR, "share", "pybind11", "src") + source_path = os.path.join(os.path.dirname(DIR), "src") + if os.path.exists(installed_path): + return installed_path + if os.path.exists(source_path): + return source_path + + msg = "pybind11 library sources not found (pybind11 not installed?)" + raise ImportError(msg) + + def get_cmake_dir() -> str: """ Return the path to the pybind11 CMake module directory. diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 66a3eec0d4..90c51f8f46 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -108,6 +108,14 @@ class Pybind11Extension(_Extension): If you want to add pybind11 headers manually, for example for an exact git checkout, then set ``include_pybind11=False``. + + Set ``precompile=True`` to compile the pybind11 library sources into the + extension (one extra translation unit) instead of instantiating everything + inline in every file; this usually builds faster. Requires an installed + pybind11 package that ships the library sources. Use the ``build_ext`` + from this module when you build more than one precompiled extension in + one ``setup()``; it gives each extension its own copy of the library + translation unit, so each gets its own object file. """ # flags are prepended, so that they can be further overridden, e.g. by @@ -127,6 +135,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: kwargs["language"] = "c++" include_pybind11 = kwargs.pop("include_pybind11", True) + precompile = kwargs.pop("precompile", False) super().__init__(*args, **kwargs) @@ -143,6 +152,40 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: except ModuleNotFoundError: pass + self._precompile_source: str | None = None + if precompile: + if not include_pybind11: + # The shipped sources must match the shipped headers; mixing + # them with a different checkout gives confusing errors. + msg = ( + "precompile=True compiles the sources of the installed " + "pybind11 package, so it cannot be combined with " + "include_pybind11=False. Instead, add " + "src/pybind11_combined.cpp from your pybind11 checkout " + "to sources and define PYBIND11_PRECOMPILED." + ) + raise ValueError(msg) + # No silent fallback: failing to precompile would quietly rebuild + # everything inline, so a missing source tree is an error. + try: + import pybind11 + + combined = os.path.join( + pybind11.get_source_dir(), "pybind11_combined.cpp" + ) + except (ImportError, AttributeError) as err: + msg = ( + "precompile=True requires an installed pybind11 package " + "that provides the library sources" + ) + raise ValueError(msg) from err + if not os.path.exists(combined): + msg = f"pybind11 library sources not found: {combined}" + raise ValueError(msg) + self._precompile_source = combined + self.sources.append(combined) + self.define_macros.append(("PYBIND11_PRECOMPILED", None)) + self.cxx_std = cxx_std cflags = [] @@ -278,9 +321,30 @@ def build_extensions(self) -> None: for ext in self.extensions: if hasattr(ext, "_cxx_level") and ext._cxx_level == 0: ext.cxx_std = auto_cpp_level(self.compiler) + self._isolate_precompile_source(ext) super().build_extensions() + def _isolate_precompile_source(self, ext: _Extension) -> None: + # Each precompiled extension needs its own combined source file: + # setuptools maps a shared absolute source to one shared object file, + # which races in parallel builds and can silently reuse an object + # compiled with another extension's macros. + src = getattr(ext, "_precompile_source", None) + if src is None: + return + dest_dir = Path(self.build_temp) / "pybind11_precompile" + dest_dir.mkdir(parents=True, exist_ok=True) + dest = dest_dir / (ext.name.replace(".", "_") + "_combined.cpp") + # A shim #include keeps the original's relative sibling includes valid + contents = f'#include "{Path(src).resolve().as_posix()}"\n' + if not dest.exists() or dest.read_text(encoding="utf-8") != contents: + dest.write_text(contents, encoding="utf-8") + # Give the shim the original's mtime, so mtime-based recompile checks + # follow the real source. + shutil.copystat(src, dest) + ext.sources[ext.sources.index(src)] = str(dest) + def intree_extensions( paths: Iterable[str], package_dir: dict[str, str] | None = None diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 58dbca4bc1..a14f057a68 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -33,6 +33,7 @@ PKGCONFIG = """\ prefix=${{pcfiledir}}/../../ includedir=${{prefix}}/include +srcdir=${{prefix}}/share/pybind11/src Name: pybind11 Description: Seamless operability between C++11 and Python diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py index 22c0c76f5d..4cd002d4f1 100644 --- a/tests/extra_setuptools/test_setuphelper.py +++ b/tests/extra_setuptools/test_setuphelper.py @@ -110,6 +110,109 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std): ) +def test_precompile_setup_py(monkeypatch, tmpdir): + # Two precompiled extensions with different configuration macros: each + # must compile its own copy of the combined translation unit, or the + # mtime-based recompile check reuses the first extension's object and the + # link-time config guard fails. + monkeypatch.chdir(tmpdir) + monkeypatch.syspath_prepend(MAIN_DIR) + + (tmpdir / "setup.py").write_text( + dedent( + f"""\ + import sys + sys.path.append({MAIN_DIR!r}) + + from setuptools import setup + from pybind11.setup_helpers import ( + ParallelCompile, + Pybind11Extension, + build_ext, + naive_recompile, + ) + + ParallelCompile(needs_recompile=naive_recompile).install() + + ext_modules = [ + Pybind11Extension( + "precompile_a", + ["a.cpp"], + cxx_std=17, + precompile=True, + ), + Pybind11Extension( + "precompile_b", + ["b.cpp"], + cxx_std=17, + precompile=True, + define_macros=[("PYBIND11_DETAILED_ERROR_MESSAGES", None)], + ), + ] + + setup( + name="precompile_setup_package", + cmdclass={{"build_ext": build_ext}}, + ext_modules=ext_modules, + ) + """ + ), + encoding="ascii", + ) + + for name, mult in [("a", 3), ("b", 5)]: + (tmpdir / f"{name}.cpp").write_text( + dedent( + f"""\ + #include + + #ifndef PYBIND11_PRECOMPILED + # error "expected PYBIND11_PRECOMPILED to be defined" + #endif + + int f(int x) {{ + return x * {mult}; + }} + PYBIND11_MODULE(precompile_{name}, m, pybind11::mod_gil_used()) {{ + m.def("f", &f); + }} + """ + ), + encoding="ascii", + ) + + subprocess.check_call( + [sys.executable, "setup.py", "build_ext", "--inplace"], + stdout=sys.stdout, + stderr=sys.stderr, + ) + + (tmpdir / "test.py").write_text( + dedent( + """\ + import precompile_a + import precompile_b + assert precompile_a.f(3) == 9 + assert precompile_b.f(3) == 15 + """ + ), + encoding="ascii", + ) + + subprocess.check_call( + [sys.executable, "test.py"], stdout=sys.stdout, stderr=sys.stderr + ) + + +def test_precompile_include_pybind11_false(monkeypatch): + monkeypatch.syspath_prepend(MAIN_DIR) + + from pybind11.setup_helpers import Pybind11Extension + + with pytest.raises(ValueError, match="include_pybind11=False"): + Pybind11Extension("bad", ["bad.cpp"], precompile=True, include_pybind11=False) + + def test_intree_extensions(monkeypatch, tmpdir): monkeypatch.syspath_prepend(MAIN_DIR) diff --git a/tools/pybind11.pc.in b/tools/pybind11.pc.in index 402f0b357d..8b5af0a991 100644 --- a/tools/pybind11.pc.in +++ b/tools/pybind11.pc.in @@ -1,5 +1,6 @@ prefix=@prefix_for_pc_file@ includedir=@includedir_for_pc_file@ +srcdir=@srcdir_for_pc_file@ Name: @PROJECT_NAME@ Description: Seamless operability between C++11 and Python