[feat/precompile-pybind11-h] feat: expose the precompiled-mode sources to non-CMake builds

CMakeLists.txt

@@ -343,8 +343,8 @@ if(PYBIND11_INSTALL)
     install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION "${SKBUILD_HEADERS_DIR}")
   endif()
   install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-  install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/
-          DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src")
+  set(pybind11_install_srcdir "${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src")
+  install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/ DESTINATION "${pybind11_install_srcdir}")
   set(PYBIND11_CMAKECONFIG_INSTALL_DIR
       "${CMAKE_INSTALL_DATAROOTDIR}/cmake/${PROJECT_NAME}"
       CACHE STRING "install path for pybind11Config.cmake")
@@ -355,9 +355,9 @@ if(PYBIND11_INSTALL)
     set(pybind11_INCLUDEDIR "\$\{PACKAGE_PREFIX_DIR\}/${CMAKE_INSTALL_INCLUDEDIR}")
   endif()
   if(IS_ABSOLUTE "${CMAKE_INSTALL_DATAROOTDIR}")
-    set(pybind11_SRCDIR "${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src")
+    set(pybind11_SRCDIR "${pybind11_install_srcdir}")
   else()
-    set(pybind11_SRCDIR "\$\{PACKAGE_PREFIX_DIR\}/${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src")
+    set(pybind11_SRCDIR "\$\{PACKAGE_PREFIX_DIR\}/${pybind11_install_srcdir}")
   endif()
 
   configure_package_config_file(
@@ -410,6 +410,7 @@ if(PYBIND11_INSTALL)
     endif()
   endif()
   join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}")
+  join_paths(srcdir_for_pc_file "\${prefix}" "${pybind11_install_srcdir}")
   configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11.pc.in"
                  "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" @ONLY)
   install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc"

pybind11/__init__.py

@@ -8,12 +8,13 @@
 
 
 from ._version import __version__, version_info
-from .commands import get_cmake_dir, get_include, get_pkgconfig_dir
+from .commands import get_cmake_dir, get_include, get_pkgconfig_dir, get_source_dir
 
 __all__ = (
     "__version__",
     "get_cmake_dir",
     "get_include",
     "get_pkgconfig_dir",
+    "get_source_dir",
     "version_info",
 )

pybind11/__main__.py

@@ -17,6 +17,7 @@
     get_include_dirs,
     get_ldflags,
     get_pkgconfig_dir,
+    get_source_dir,
 )
 
 
@@ -50,6 +51,12 @@ def main() -> None:
         action="store_true",
         help="Print the pkgconfig directory, ideal for setting $PKG_CONFIG_PATH.",
     )
+    parser.add_argument(
+        "--srcdir",
+        action="store_true",
+        help="Print the directory containing the library sources for the optional"
+        " precompiled mode.",
+    )
     parser.add_argument(
         "--extension-suffix",
         action="store_true",
@@ -101,6 +108,8 @@ def main() -> None:
         print(quote(get_cmake_dir()))
     if args.pkgconfigdir:
         print(quote(get_pkgconfig_dir()))
+    if args.srcdir:
+        print(quote(get_source_dir()))
     if args.extension_suffix:
         print(ext_suffix)
 

pybind11/commands.py

@@ -52,6 +52,24 @@ def get_include(user: bool = False) -> str:  # noqa: ARG001
     return installed_path if os.path.exists(installed_path) else source_path
 
 
+def get_source_dir() -> str:
+    """
+    Return the path to the pybind11 library sources, for the optional
+    precompiled mode. Compile ``pybind11_combined.cpp`` (or the individual
+    ``.cpp`` files) with ``PYBIND11_PRECOMPILED`` defined, and define that
+    macro for every translation unit that includes pybind11.
+    """
+    installed_path = os.path.join(DIR, "share", "pybind11", "src")
+    source_path = os.path.join(os.path.dirname(DIR), "src")
+    if os.path.exists(installed_path):
+        return installed_path
+    if os.path.exists(source_path):
+        return source_path
+
+    msg = "pybind11 library sources not found (pybind11 not installed?)"
+    raise ImportError(msg)
+
+
 def get_cmake_dir() -> str:
     """
     Return the path to the pybind11 CMake module directory.

pybind11/setup_helpers.py

@@ -108,6 +108,14 @@ class Pybind11Extension(_Extension):
 
     If you want to add pybind11 headers manually, for example for an exact
     git checkout, then set ``include_pybind11=False``.
+
+    Set ``precompile=True`` to compile the pybind11 library sources into the
+    extension (one extra translation unit) instead of instantiating everything
+    inline in every file; this usually builds faster. Requires an installed
+    pybind11 package that ships the library sources. Use the ``build_ext``
+    from this module when you build more than one precompiled extension in
+    one ``setup()``; it gives each extension its own copy of the library
+    translation unit, so each gets its own object file.
     """
 
     # flags are prepended, so that they can be further overridden, e.g. by
@@ -127,6 +135,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
             kwargs["language"] = "c++"
 
         include_pybind11 = kwargs.pop("include_pybind11", True)
+        precompile = kwargs.pop("precompile", False)
 
         super().__init__(*args, **kwargs)
 
@@ -143,6 +152,40 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
             except ModuleNotFoundError:
                 pass
 
+        self._precompile_source: str | None = None
+        if precompile:
+            if not include_pybind11:
+                # The shipped sources must match the shipped headers; mixing
+                # them with a different checkout gives confusing errors.
+                msg = (
+                    "precompile=True compiles the sources of the installed "
+                    "pybind11 package, so it cannot be combined with "
+                    "include_pybind11=False. Instead, add "
+                    "src/pybind11_combined.cpp from your pybind11 checkout "
+                    "to sources and define PYBIND11_PRECOMPILED."
+                )
+                raise ValueError(msg)
+            # No silent fallback: failing to precompile would quietly rebuild
+            # everything inline, so a missing source tree is an error.
+            try:
+                import pybind11
+
+                combined = os.path.join(
+                    pybind11.get_source_dir(), "pybind11_combined.cpp"
+                )
+            except (ImportError, AttributeError) as err:
+                msg = (
+                    "precompile=True requires an installed pybind11 package "
+                    "that provides the library sources"
+                )
+                raise ValueError(msg) from err
+            if not os.path.exists(combined):
+                msg = f"pybind11 library sources not found: {combined}"
+                raise ValueError(msg)
+            self._precompile_source = combined
+            self.sources.append(combined)
+            self.define_macros.append(("PYBIND11_PRECOMPILED", None))
+
         self.cxx_std = cxx_std
 
         cflags = []
@@ -278,9 +321,30 @@ def build_extensions(self) -> None:
         for ext in self.extensions:
             if hasattr(ext, "_cxx_level") and ext._cxx_level == 0:
                 ext.cxx_std = auto_cpp_level(self.compiler)
+            self._isolate_precompile_source(ext)
 
         super().build_extensions()
 
+    def _isolate_precompile_source(self, ext: _Extension) -> None:
+        # Each precompiled extension needs its own combined source file:
+        # setuptools maps a shared absolute source to one shared object file,
+        # which races in parallel builds and can silently reuse an object
+        # compiled with another extension's macros.
+        src = getattr(ext, "_precompile_source", None)
+        if src is None:
+            return
+        dest_dir = Path(self.build_temp) / "pybind11_precompile"
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        dest = dest_dir / (ext.name.replace(".", "_") + "_combined.cpp")
+        # A shim #include keeps the original's relative sibling includes valid
+        contents = f'#include "{Path(src).resolve().as_posix()}"\n'
+        if not dest.exists() or dest.read_text(encoding="utf-8") != contents:
+            dest.write_text(contents, encoding="utf-8")
+        # Give the shim the original's mtime, so mtime-based recompile checks
+        # follow the real source.
+        shutil.copystat(src, dest)
+        ext.sources[ext.sources.index(src)] = str(dest)
+
 
 def intree_extensions(
     paths: Iterable[str], package_dir: dict[str, str] | None = None

tests/extra_python_package/test_files.py

@@ -33,6 +33,7 @@
 PKGCONFIG = """\
 prefix=${{pcfiledir}}/../../
 includedir=${{prefix}}/include
+srcdir=${{prefix}}/share/pybind11/src
 
 Name: pybind11
 Description: Seamless operability between C++11 and Python

tests/extra_setuptools/test_setuphelper.py

@@ -110,6 +110,109 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std):
     )
 
 
+def test_precompile_setup_py(monkeypatch, tmpdir):
+    # Two precompiled extensions with different configuration macros: each
+    # must compile its own copy of the combined translation unit, or the
+    # mtime-based recompile check reuses the first extension's object and the
+    # link-time config guard fails.
+    monkeypatch.chdir(tmpdir)
+    monkeypatch.syspath_prepend(MAIN_DIR)
+
+    (tmpdir / "setup.py").write_text(
+        dedent(
+            f"""\
+            import sys
+            sys.path.append({MAIN_DIR!r})
+
+            from setuptools import setup
+            from pybind11.setup_helpers import (
+                ParallelCompile,
+                Pybind11Extension,
+                build_ext,
+                naive_recompile,
+            )
+
+            ParallelCompile(needs_recompile=naive_recompile).install()
+
+            ext_modules = [
+                Pybind11Extension(
+                    "precompile_a",
+                    ["a.cpp"],
+                    cxx_std=17,
+                    precompile=True,
+                ),
+                Pybind11Extension(
+                    "precompile_b",
+                    ["b.cpp"],
+                    cxx_std=17,
+                    precompile=True,
+                    define_macros=[("PYBIND11_DETAILED_ERROR_MESSAGES", None)],
+                ),
+            ]
+
+            setup(
+                name="precompile_setup_package",
+                cmdclass={{"build_ext": build_ext}},
+                ext_modules=ext_modules,
+            )
+            """
+        ),
+        encoding="ascii",
+    )
+
+    for name, mult in [("a", 3), ("b", 5)]:
+        (tmpdir / f"{name}.cpp").write_text(
+            dedent(
+                f"""\
+                #include <pybind11/pybind11.h>
+
+                #ifndef PYBIND11_PRECOMPILED
+                #    error "expected PYBIND11_PRECOMPILED to be defined"
+                #endif
+
+                int f(int x) {{
+                    return x * {mult};
+                }}
+                PYBIND11_MODULE(precompile_{name}, m, pybind11::mod_gil_used()) {{
+                    m.def("f", &f);
+                }}
+                """
+            ),
+            encoding="ascii",
+        )
+
+    subprocess.check_call(
+        [sys.executable, "setup.py", "build_ext", "--inplace"],
+        stdout=sys.stdout,
+        stderr=sys.stderr,
+    )
+
+    (tmpdir / "test.py").write_text(
+        dedent(
+            """\
+            import precompile_a
+            import precompile_b
+            assert precompile_a.f(3) == 9
+            assert precompile_b.f(3) == 15
+            """
+        ),
+        encoding="ascii",
+    )
+
+    subprocess.check_call(
+        [sys.executable, "test.py"], stdout=sys.stdout, stderr=sys.stderr
+    )
+
+
+def test_precompile_include_pybind11_false(monkeypatch):
+    monkeypatch.syspath_prepend(MAIN_DIR)
+
+    from pybind11.setup_helpers import Pybind11Extension
+
+    with pytest.raises(ValueError, match="include_pybind11=False"):
+        Pybind11Extension("bad", ["bad.cpp"], precompile=True, include_pybind11=False)
+
+
 def test_intree_extensions(monkeypatch, tmpdir):
     monkeypatch.syspath_prepend(MAIN_DIR)
 

tools/pybind11.pc.in

@@ -1,5 +1,6 @@
 prefix=@prefix_for_pc_file@
 includedir=@includedir_for_pc_file@
+srcdir=@srcdir_for_pc_file@
 
 Name: @PROJECT_NAME@
 Description: Seamless operability between C++11 and Python