refactor: use ThreadPoolExecutor in extract_all

.github/workflows/ci.yml

@@ -38,7 +38,7 @@ jobs:
     - uses: astral-sh/setup-uv@v9.0.0
 
     - name: Test package
-      run: uv run --with "clang<19" --group test pytest --forked
+      run: uv run --with "clang<19" --group test pytest
 
 # Commented for now -- msys2 Clang (v15) and the clang Python package (v14) are incompatible
 #

pybind11_mkdoc/mkdoc_lib.py

@@ -15,9 +15,9 @@
 import sys
 import textwrap
 from collections import OrderedDict
+from concurrent.futures import ThreadPoolExecutor
 from glob import glob
-from multiprocessing import cpu_count
-from threading import Semaphore, Thread
+from itertools import repeat
 
 from clang import cindex
 from clang.cindex import CursorKind
@@ -94,9 +94,6 @@
 
 CPP_OPERATORS = OrderedDict(sorted(CPP_OPERATORS.items(), key=lambda t: -len(t[0])))
 
-job_count = cpu_count()
-job_semaphore = Semaphore(job_count)
-errors_detected = False
 docstring_width = 70
 
 
@@ -555,25 +552,12 @@ def extract(filename, node, prefix, output, file_cache):
     return None
 
 
-class ExtractionThread(Thread):
-    def __init__(self, filename, parameters, output):
-        Thread.__init__(self)
-        self.filename = filename
-        self.parameters = parameters
-        self.output = output
-        job_semaphore.acquire()
-
-    def run(self):
-        global errors_detected
-        try:
-            index = cindex.Index(cindex.conf.lib.clang_createIndex(False, True))
-            tu = index.parse(self.filename, self.parameters)
-            extract(self.filename, tu.cursor, "", self.output, {})
-        except BaseException:
-            errors_detected = True
-            raise
-        finally:
-            job_semaphore.release()
+def _extract_file(filename, parameters):
+    index = cindex.Index(cindex.conf.lib.clang_createIndex(False, True))
+    tu = index.parse(filename, parameters)
+    output = []
+    extract(filename, tu.cursor, "", output, {})
+    return output
 
 
 def read_args(args):
@@ -591,7 +575,8 @@ def read_args(args):
         sdk_dir = dev_path + "Platforms/MacOSX.platform/Developer/SDKs"
         libclang = lib_dir + "libclang.dylib"
 
-        if os.path.exists(libclang):
+        # cindex forbids (re)configuring the library once it has been loaded
+        if os.path.exists(libclang) and not cindex.Config.loaded:
             cindex.Config.set_library_path(os.path.dirname(libclang))
 
         if os.path.exists(sdk_dir):
@@ -602,13 +587,14 @@ def read_args(args):
         if "LIBCLANG_PATH" in os.environ:
             library_file = os.environ["LIBCLANG_PATH"]
             if os.path.isfile(library_file):
-                cindex.Config.set_library_file(library_file)
+                if not cindex.Config.loaded:
+                    cindex.Config.set_library_file(library_file)
             else:
                 msg = "Failed to find libclang.dll! Set the LIBCLANG_PATH environment variable to provide a path to it."
                 raise FileNotFoundError(msg)
         else:
             library_file = ctypes.util.find_library("libclang.dll")
-            if library_file is not None:
+            if library_file is not None and not cindex.Config.loaded:
                 cindex.Config.set_library_file(library_file)
     elif platform.system() == "Linux":
         # LLVM switched to a monolithical setup that includes everything under
@@ -650,7 +636,8 @@ def folder_version(d):
         else:
             libclang_dir = os.path.join(llvm_dir, "lib", "libclang.so.1")
 
-        cindex.Config.set_library_file(libclang_dir)
+        if not cindex.Config.loaded:
+            cindex.Config.set_library_file(libclang_dir)
         cpp_dirs = []
 
         if "-stdlib=libc++" not in args:
@@ -696,15 +683,9 @@ def folder_version(d):
 
 def extract_all(args):
     parameters, filenames = read_args(args)
-    output = []
-    for filename in filenames:
-        thr = ExtractionThread(filename, parameters, output)
-        thr.start()
-
-    for _i in range(job_count):
-        job_semaphore.acquire()
-
-    return output
+    with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor:
+        results = executor.map(_extract_file, filenames, repeat(parameters))
+        return [comment for output in results for comment in output]
 
 
 def write_header(comments, out_file=sys.stdout):
@@ -765,8 +746,6 @@ def mkdoc(args, width, output=None):
         global docstring_width
         docstring_width = int(width)
     comments = extract_all(args)
-    if errors_detected:
-        return
 
     if output:
         try:

pyproject.toml

@@ -31,7 +31,6 @@ dynamic = ["version"]
 test = [
     "pytest",
     "pybind11",
-    "pytest-forked",
 ]
 dev = [{ include-group = "test" }]
 

tests/cli_test.py

@@ -25,3 +25,15 @@ def test_simple_header_cli(tmp_path: Path, name: str) -> None:
     res = tf.read_text(encoding="utf-8")
 
     assert res == expected
+
+
+def test_parse_failure_sets_exit_code(tmp_path: Path) -> None:
+    tf = tmp_path / "tmp.h"
+    result = subprocess.run(
+        [sys.executable, "-m", "pybind11_mkdoc", "-o", tf, tmp_path / "does_not_exist.h"],
+        check=False,
+        capture_output=True,
+    )
+
+    assert result.returncode != 0
+    assert not tf.exists()

tests/sample_header_test.py

@@ -23,6 +23,13 @@ def test_generate_headers(capsys, tmp_path):
     assert output.read_text() == expected
 
 
+def test_extract_all_twice_in_one_process():
+    header = str(DIR / "sample_header_docs" / "sample_header.h")
+    first = pybind11_mkdoc.mkdoc_lib.extract_all([header])
+    second = pybind11_mkdoc.mkdoc_lib.extract_all([header])
+    assert first == second
+
+
 def test_generate_headers_2(capsys, tmp_path):
     with open(DIR / "sample_header_docs" / "sample_header_2_truth.h") as f:
         expected = f.read()