Skip to content

refactor: simplify and modernize mkdoc_lib #63

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
henryiii merged 1 commit into from
Aug 7, 2026
Merged

refactor: simplify and modernize mkdoc_lib #63

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 37 additions & 36 deletions pybind11_mkdoc/mkdoc_lib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import re
import sys
import textwrap
from collections import OrderedDict
from concurrent.futures import ThreadPoolExecutor
from glob import glob
from itertools import repeat
Expand Down Expand Up @@ -92,7 +91,8 @@
"()": "call",
}

CPP_OPERATORS = OrderedDict(sorted(CPP_OPERATORS.items(), key=lambda t: -len(t[0])))
# Longest operators first, so that e.g. "<<=" is matched before "<<".
CPP_OPERATORS = dict(sorted(CPP_OPERATORS.items(), key=lambda t: -len(t[0])))

docstring_width = 70

Expand All @@ -101,25 +101,20 @@ class NoFilenamesError(ValueError):
pass


def d(s):
return s if isinstance(s, str) else s.decode("utf8")


def sanitize_name(name):
name = re.sub(r"type-parameter-0-([0-9]+)", r"T\1", name)
for k, v in CPP_OPERATORS.items():
name = name.replace(f"operator{k}", f"operator_{v}")
name = re.sub("<.*>", "", name)
name = "".join([ch if ch.isalnum() else "_" for ch in name])
name = re.sub("_$", "", re.sub("_+", "_", name))
name = re.sub("_+", "_", name).removesuffix("_")
return "mkd_doc_" + name


section_command_re = re.compile(r"\s*[\\@](\w+)(?:\[([^\]]+)\])?(?:\s+(.*))?$")
code_segment_re = re.compile(r"(```)")
prefix_re = re.compile(r"(\s*)((?:[*\-鈥\s)|(?:\(?\d+[\.)]\s)|(?:[\w:]+(?:\s+\[[^\]]+\])?:(?:\s+|$)))")
param_arg_re = re.compile(r"([\w:]+)\s*(.*)")
raises_arg_re = re.compile(r"([\w:]+)\s*(.*)")
named_arg_re = re.compile(r"([\w:]+)\s*(.*)")

IGNORED_DOXYGEN_COMMANDS = {
"addtogroup",
Expand Down Expand Up @@ -184,6 +179,23 @@ def sanitize_name(name):
(re.compile(r"[\\@]ref\s+([^\s]+)"), r"\1"),
]

# Applied before the Doxygen sections are consumed.
BLOCK_DOXYGEN_REPLACEMENTS = [
(re.compile(r"[\\@]code\s?(.*?)\s?[\\@]endcode", flags=re.DOTALL), "```\n\\1\n```\n"),
(re.compile(r"[\\@]verbatim\s?(.*?)\s?[\\@]endverbatim", flags=re.DOTALL), "```\n\\1\n```\n"),
]

MARKUP_REPLACEMENTS = [
(re.compile(r"<tt>(.*?)</tt>", flags=re.DOTALL), r"``\1``"),
(re.compile(r"<pre>(.*?)</pre>", flags=re.DOTALL), "```\n\\1\n```\n"),
(re.compile(r"<em>(.*?)</em>", flags=re.DOTALL), r"*\1*"),
(re.compile(r"<b>(.*?)</b>", flags=re.DOTALL), r"**\1**"),
(re.compile(r"[\\@]f\$(.*?)[\\@]f\$", flags=re.DOTALL), r":math:`\1`"),
(re.compile(r"<li>"), "\n\n* "),
(re.compile(r"</?ul>"), ""),
(re.compile(r"</li>"), "\n\n"),
]


def _format_named_entries(heading, entries):
if not entries:
Expand Down Expand Up @@ -292,7 +304,7 @@ def _consume_doxygen_sections(s):
continue

if command in {"param", "arg"}:
arg = param_arg_re.match(rest)
arg = named_arg_re.match(rest)
if arg:
name, text = arg.groups()
if option:
Expand All @@ -304,7 +316,7 @@ def _consume_doxygen_sections(s):
continue

if command in {"tparam", "typeparam"}:
arg = param_arg_re.match(rest)
arg = named_arg_re.match(rest)
if arg:
name, text = arg.groups()
t_params.append((name, text.strip()))
Expand All @@ -322,7 +334,7 @@ def _consume_doxygen_sections(s):
continue

if command == "retval":
arg = param_arg_re.match(rest)
arg = named_arg_re.match(rest)
if arg:
name, text = arg.groups()
returns.append(f"{name}: {text.strip()}" if text else name)
Expand All @@ -332,7 +344,7 @@ def _consume_doxygen_sections(s):
continue

if command in {"exception", "throw", "throws"}:
arg = raises_arg_re.match(rest)
arg = named_arg_re.match(rest)
if arg:
name, text = arg.groups()
raises.append((name, text.strip()))
Expand Down Expand Up @@ -404,19 +416,14 @@ def process_comment(comment):
for pattern, replacement in INLINE_DOXYGEN_REPLACEMENTS:
s = pattern.sub(replacement, s)

s = re.sub(r"[\\@]code\s?(.*?)\s?[\\@]endcode", r"```\n\1\n```\n", s, flags=re.DOTALL)
s = re.sub(r"[\\@]verbatim\s?(.*?)\s?[\\@]endverbatim", r"```\n\1\n```\n", s, flags=re.DOTALL)
for pattern, replacement in BLOCK_DOXYGEN_REPLACEMENTS:
s = pattern.sub(replacement, s)

s = _consume_doxygen_sections(s)

# HTML/TeX tags
s = re.sub(r"<tt>(.*?)</tt>", r"``\1``", s, flags=re.DOTALL)
s = re.sub(r"<pre>(.*?)</pre>", r"```\n\1\n```\n", s, flags=re.DOTALL)
s = re.sub(r"<em>(.*?)</em>", r"*\1*", s, flags=re.DOTALL)
s = re.sub(r"<b>(.*?)</b>", r"**\1**", s, flags=re.DOTALL)
s = re.sub(r"[\\@]f\$(.*?)[\\@]f\$", r":math:`\1`", s, flags=re.DOTALL)
s = re.sub(r"<li>", r"\n\n* ", s)
s = re.sub(r"</?ul>", r"", s)
s = re.sub(r"</li>", r"\n\n", s)
for pattern, replacement in MARKUP_REPLACEMENTS:
s = pattern.sub(replacement, s)

s = s.replace("``true``", "``True``")
s = s.replace("``false``", "``False``")
Expand Down Expand Up @@ -518,7 +525,7 @@ def _is_cursor_from_file(node, filename, file_cache):
if node.location.file is None:
return True

node_filename = d(node.location.file.name)
node_filename = node.location.file.name
# libclang often reports many cursors from the same file; avoid repeated stat calls by caching.
if node_filename not in file_cache:
file_cache[node_filename] = os.path.samefile(node_filename, filename)
Expand All @@ -527,29 +534,25 @@ def _is_cursor_from_file(node, filename, file_cache):

def extract(filename, node, prefix, output, file_cache):
if not _is_cursor_from_file(node, filename, file_cache):
return 0
return
if node.kind in RECURSE_LIST:
sub_prefix = prefix
if node.kind not in PREFIX_BLACKLIST:
if len(sub_prefix) > 0:
sub_prefix += "_"
sub_prefix += d(node.spelling)
sub_prefix += node.spelling
for i in node.get_children():
extract(filename, i, sub_prefix, output, file_cache)
if node.kind in PRINT_LIST:
comment = d(node.raw_comment) if node.raw_comment is not None else ""
comment = process_comment(comment)
comment = process_comment(node.raw_comment or "")
if node.kind in FUNCTION_DOCSTRING_LIST:
comment = format_function_docstring(comment)
sub_prefix = prefix
if len(sub_prefix) > 0:
sub_prefix += "_"
if len(node.spelling) > 0:
name = sanitize_name(sub_prefix + d(node.spelling))
name = sanitize_name(sub_prefix + node.spelling)
output.append((name, filename, comment))
return None
return None
return None


def _extract_file(filename, parameters):
Expand Down Expand Up @@ -726,10 +729,8 @@ def write_header(comments, out_file=sys.stdout):
else:
name_prev = name
name_ctr = 1
print(
'\nstatic const char *{} ={}R"doc({})doc";'.format(name, "\n" if "\n" in comment else " ", comment),
file=out_file,
)
sep = "\n" if "\n" in comment else " "
print(f'\nstatic const char *{name} ={sep}R"doc({comment})doc";', file=out_file)

print(
"""
Expand Down