From 52495376e8195c0957578ae6a61abdfdefd8ffae Mon Sep 17 00:00:00 2001 From: Leonardo Salas Date: Sat, 15 Aug 2026 21:53:26 -0600 Subject: [PATCH 01/14] feat: render mermaid and svg code blocks as diagrams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A ```mermaid block is now rendered to SVG and embedded as a picture, and a ```svg block is passed through, since Typst renders SVG natively. Everything else keeps rendering as syntax-highlighted code. Mermaid goes through mermaidx, which runs Mermaid on an embedded JavaScript engine — no Node install and no headless browser, so the tool stays a plain pip install. Wheels cover Linux, macOS, and Windows on Python 3.10 through 3.14. Invalid diagram sources are reported with Mermaid's own parse message instead of aborting with a traceback. Conversion now carries generated content alongside copied assets, since a rendered diagram has no file of its own. Pictures also stop being forced to full width. A new fit-image helper keeps an image at its natural size and shrinks it only when it is too wide for the text block, which fixes small figures being blown up to fill the page. Closes #8. --- doc_engine/cli.py | 10 ++++++- doc_engine/compiler.py | 18 ++++++++++++ doc_engine/converter.py | 23 +++++++++++++-- doc_engine/diagrams.py | 62 +++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 1 + tests/test_converter.py | 2 +- tests/test_diagrams.py | 58 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 doc_engine/diagrams.py create mode 100644 tests/test_diagrams.py diff --git a/doc_engine/cli.py b/doc_engine/cli.py index 834c540..975214c 100644 --- a/doc_engine/cli.py +++ b/doc_engine/cli.py @@ -12,6 +12,7 @@ from doc_engine import __version__, frontmatter from doc_engine.compiler import DEFAULT_TEMPLATE, available_templates, compile_pdf from doc_engine.converter import convert_document, extract_title, strip_first_heading +from doc_engine.diagrams import DiagramError from doc_engine.linter import has_errors, lint if sys.platform == "win32": @@ -260,7 +261,13 @@ def run() -> bool: console.print() with console.status("[bold blue]Converting Markdown → Typst…[/bold blue]"): - conversion = convert_document(strip_first_heading(content), base_dir=input_path.parent) + try: + conversion = convert_document(strip_first_heading(content), base_dir=input_path.parent) + except DiagramError as exc: + console.print( + f"\n[bold red]Diagram failed:[/bold red] {exc.language} block — {exc.message}" + ) + return False with console.status("[bold blue]Compiling PDF…[/bold blue]"): try: @@ -277,6 +284,7 @@ def run() -> bool: branding=not no_branding, version=__version__, assets=conversion.assets, + generated=conversion.generated, ) except Exception as exc: message = getattr(exc, "message", None) or str(exc) diff --git a/doc_engine/compiler.py b/doc_engine/compiler.py index 3a6090b..aebe285 100644 --- a/doc_engine/compiler.py +++ b/doc_engine/compiler.py @@ -38,6 +38,7 @@ def compile_pdf( subtitle: str = "", date: str | None = None, assets: dict[str, str] | None = None, + generated: dict[str, str] | None = None, ) -> None: source = template_path(template) if not source.exists(): @@ -61,6 +62,11 @@ def compile_pdf( destination.parent.mkdir(parents=True, exist_ok=True) shutil.copy(origin, destination) + for name, content in (generated or {}).items(): + destination = tmp / name + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(content, encoding="utf-8") + accent_inject = f'rgb("{accent}")' if accent else "none" main_file = tmp / "main.typ" @@ -75,6 +81,17 @@ def compile_pdf( typst.compile(str(main_file), output=resolved_output) +# Pictures keep their natural size unless they are too wide for the text block, +# in which case they shrink to fit. Forcing every image to full width blows up +# small diagrams; leaving them unconstrained lets big ones run off the page. +_FIT_IMAGE = """#let fit-image(path) = context { + let img = image(path) + let natural = measure(img) + layout(area => if natural.width > area.width { image(path, width: 100%) } else { img }) +} +""" + + def _build_main( body: str, title: str, @@ -89,6 +106,7 @@ def _build_main( date_line = f' date: "{_escape(date)}",\n' if date else "" return ( '#import "template.typ": setup_doc\n\n' + f"{_FIT_IMAGE}\n" "#show: setup_doc.with(\n" f' title: "{_escape(title)}",\n' f' subtitle: "{_escape(subtitle)}",\n' diff --git a/doc_engine/converter.py b/doc_engine/converter.py index c71c613..bdaf4cb 100644 --- a/doc_engine/converter.py +++ b/doc_engine/converter.py @@ -17,6 +17,8 @@ import mistune +from doc_engine import diagrams + _TYPST_ESCAPES = { "\\": "\\\\", "#": "\\#", @@ -62,8 +64,16 @@ def _render_children(renderer: mistune.BaseRenderer, token: dict, state: Any) -> @dataclass class Conversion: + """Typst markup plus everything the compiler must place beside it. + + `assets` maps a sandbox-relative name to a file on disk to copy. `generated` + maps a name to content produced during conversion, such as a rendered + diagram, which has no file of its own. + """ + body: str assets: dict[str, str] = field(default_factory=dict) + generated: dict[str, str] = field(default_factory=dict) class TypstRenderer(mistune.BaseRenderer): @@ -76,6 +86,7 @@ def __init__(self, base_dir: Path | None = None) -> None: self._footnotes: dict[str, str] = {} self._asset_names: dict[str, str] = {} self.assets: dict[str, str] = {} + self.generated: dict[str, str] = {} def text(self, token: dict, state: Any) -> str: return _escape(token["raw"]) @@ -101,7 +112,7 @@ def image(self, token: dict, state: Any) -> str: asset = self._register_image(url) if asset is None: return f"[{alt}]" if alt else "" - return f'#image("{asset}")' + return f'#fit-image("{asset}")' def linebreak(self, token: dict, state: Any) -> str: return "\\\n" @@ -142,8 +153,16 @@ def block_code(self, token: dict, state: Any) -> str: info = token.get("attrs", {}).get("info", "") or "" lang = info.split()[0] if info else "" code = token["raw"] + if lang and diagrams.is_diagram(lang): + return self._render_diagram(lang, code) return f"\n```{lang}\n{code}```\n\n" + def _render_diagram(self, language: str, source: str) -> str: + svg = diagrams.render(language, source) + name = f"assets/diagram_{len(self.generated)}.svg" + self.generated[name] = svg + return f'\n#align(center)[#fit-image("{name}")]\n\n' + def block_quote(self, token: dict, state: Any) -> str: content = _render_children(self, token, state).strip() return ( @@ -273,7 +292,7 @@ def convert_document(markdown: str, base_dir: Path | None = None) -> Conversion: renderer.load_footnotes(tokens, state) body = renderer.render_tokens(tokens, state) body = _CITATION.sub(r"@\1", body) - return Conversion(body=body, assets=renderer.assets) + return Conversion(body=body, assets=renderer.assets, generated=renderer.generated) def convert(markdown: str, base_dir: Path | None = None) -> str: diff --git a/doc_engine/diagrams.py b/doc_engine/diagrams.py new file mode 100644 index 0000000..cbc8e72 --- /dev/null +++ b/doc_engine/diagrams.py @@ -0,0 +1,62 @@ +"""Fenced blocks that become pictures instead of code. + +A ```mermaid block is rendered to SVG through mermaidx, which runs Mermaid on an +embedded JavaScript engine — no Node install, no headless browser. A ```svg block +is passed straight through, since Typst renders SVG natively. +""" + +from __future__ import annotations + +MERMAID_LANGUAGES = ("mermaid", "mmd") +SVG_LANGUAGES = ("svg",) + + +class DiagramError(Exception): + """A diagram source that could not be rendered.""" + + def __init__(self, language: str, message: str) -> None: + super().__init__(message) + self.language = language + self.message = message + + +def is_diagram(language: str) -> bool: + lang = language.strip().lower() + return lang in MERMAID_LANGUAGES or lang in SVG_LANGUAGES + + +def render(language: str, source: str) -> str: + """Return the SVG for a diagram block, raising DiagramError if it is invalid.""" + lang = language.strip().lower() + if lang in SVG_LANGUAGES: + return source + if lang in MERMAID_LANGUAGES: + return _render_mermaid(source) + raise DiagramError(language, f"unsupported diagram language: {language}") + + +def _render_mermaid(source: str) -> str: + try: + import mermaidx + except ImportError as exc: + raise DiagramError( + "mermaid", + "mermaid rendering needs the 'mermaidx' package, which ships with " + "doc-engine-cli — reinstall with 'pip install --upgrade doc-engine-cli'", + ) from exc + + try: + return mermaidx.render(source).svg() + except DiagramError: + raise + except Exception as exc: + raise DiagramError("mermaid", _clean(str(exc))) from exc + + +def _clean(message: str) -> str: + """Trim mermaidx's wrapper so the user sees Mermaid's own complaint.""" + text = message.strip() + for prefix in ("Mermaid rendering failed:", "Error:"): + if text.startswith(prefix): + text = text[len(prefix) :].strip() + return " ".join(text.split()) diff --git a/pyproject.toml b/pyproject.toml index 052a789..a9c8250 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dependencies = [ "rich>=13.0", "mistune>=3.0", "typst>=0.11", + "mermaidx>=0.9", ] [project.optional-dependencies] diff --git a/tests/test_converter.py b/tests/test_converter.py index 91cf2b7..eb6877c 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -141,7 +141,7 @@ def test_local_image_is_embedded(self, tmp_path) -> None: (tmp_path / "logo.png").write_bytes(b"\x89PNG\r\n") conversion = convert_document("![logo](logo.png)", base_dir=tmp_path) name = next(iter(conversion.assets)) - assert f'#image("{name}")' in conversion.body + assert f'#fit-image("{name}")' in conversion.body assert conversion.assets[name].endswith("logo.png") def test_repeated_image_is_registered_once(self, tmp_path) -> None: diff --git a/tests/test_diagrams.py b/tests/test_diagrams.py new file mode 100644 index 0000000..424a99c --- /dev/null +++ b/tests/test_diagrams.py @@ -0,0 +1,58 @@ +import pytest + +from doc_engine import diagrams +from doc_engine.converter import convert_document + + +class TestDetection: + def test_diagram_languages_are_recognized(self) -> None: + for language in ("mermaid", "mmd", "svg", "SVG", "Mermaid"): + assert diagrams.is_diagram(language) + + def test_other_languages_are_not(self) -> None: + for language in ("python", "bash", "", "json"): + assert not diagrams.is_diagram(language) + + +class TestSvg: + def test_svg_passes_through(self) -> None: + source = '' + assert diagrams.render("svg", source) == source + + +class TestMermaid: + def test_renders_to_svg(self) -> None: + svg = diagrams.render("mermaid", "flowchart TD\n A-->B\n") + assert svg.lstrip().startswith(" None: + with pytest.raises(diagrams.DiagramError) as caught: + diagrams.render("mermaid", "flowchart TD\n A --> ((( broken") + assert caught.value.language == "mermaid" + assert "Mermaid rendering failed" not in caught.value.message + assert caught.value.message + + +class TestConversion: + def test_mermaid_block_becomes_a_picture(self) -> None: + conversion = convert_document("```mermaid\nflowchart TD\n A-->B\n```\n") + name = next(iter(conversion.generated)) + assert f'#fit-image("{name}")' in conversion.body + assert " None: + source = '' + conversion = convert_document(f"```svg\n{source}\n```\n") + name = next(iter(conversion.generated)) + assert conversion.generated[name].strip() == source + + def test_ordinary_code_is_left_alone(self) -> None: + conversion = convert_document("```python\nprint('hi')\n```\n") + assert conversion.generated == {} + assert "```python" in conversion.body + + def test_several_diagrams_get_distinct_names(self) -> None: + conversion = convert_document( + "```mermaid\nflowchart TD\n A-->B\n```\n\n```mermaid\nflowchart TD\n C-->D\n```\n" + ) + assert len(conversion.generated) == 2 From 633a48851771f12aae9ff4b6a7b96fe4ccd9f69b Mon Sep 17 00:00:00 2001 From: Leonardo Salas Date: Wed, 19 Aug 2026 13:37:24 -0600 Subject: [PATCH 02/14] feat!: add page size support and default to A4 Every template hardcoded us-letter, which is the standard in the US and Canada and nowhere else. Page size is now a real option: --paper on the command line, or a paper key in front matter, covering A3 through A6, both ISO B5 and JIS B5, and the US letter, legal, and tabloid sizes. BREAKING CHANGE: the default page size is now a4 rather than us-letter. Pass --paper us-letter, or set paper: us-letter in front matter, to keep the previous output. Templates also gain an ..options sink, so adding an option in a later release no longer breaks a user-supplied .typ that has not been updated. --- doc_engine/cli.py | 35 ++++++++++++++++++++++++++++-- doc_engine/compiler.py | 18 ++++++++++++++- doc_engine/templates/academic.typ | 4 +++- doc_engine/templates/book.typ | 4 +++- doc_engine/templates/minimal.typ | 4 +++- doc_engine/templates/modern.typ | 4 +++- doc_engine/templates/technical.typ | 4 +++- tests/test_cli.py | 28 ++++++++++++++++++++++-- 8 files changed, 91 insertions(+), 10 deletions(-) diff --git a/doc_engine/cli.py b/doc_engine/cli.py index 975214c..230cef6 100644 --- a/doc_engine/cli.py +++ b/doc_engine/cli.py @@ -10,7 +10,13 @@ from rich.panel import Panel from doc_engine import __version__, frontmatter -from doc_engine.compiler import DEFAULT_TEMPLATE, available_templates, compile_pdf +from doc_engine.compiler import ( + DEFAULT_PAPER, + DEFAULT_TEMPLATE, + PAPER_SIZES, + available_templates, + compile_pdf, +) from doc_engine.converter import convert_document, extract_title, strip_first_heading from doc_engine.diagrams import DiagramError from doc_engine.linter import has_errors, lint @@ -114,6 +120,16 @@ def _template_callback(ctx: click.Context, param: click.Parameter, value: str | return resolved +def _resolve_paper(flag: str | None, from_meta: str | None) -> str | None: + """Return the page size to use, or None when the front matter names an unknown one.""" + if flag: + return flag.lower() + if from_meta: + key = from_meta.strip().lower() + return key if key in PAPER_SIZES else None + return DEFAULT_PAPER + + def _template_label(template: str) -> str: return Path(template).stem if template.endswith(".typ") else template @@ -162,6 +178,12 @@ def cli(ctx: click.Context) -> None: callback=_normalize_accent, help="Accent color as a hex value (#2563eb) or a name (blue, teal, rose...).", ) +@click.option( + "--paper", + default=None, + type=click.Choice(PAPER_SIZES, case_sensitive=False), + help=f"Page size (default: {DEFAULT_PAPER}).", +) @click.option("--bib", default=None, help="Path to a custom .bib file.") @click.option("--no-branding", "no_branding", is_flag=True, help="Hide the doc-engine attribution in the PDF.") @click.option("--dry-run", "dry_run", is_flag=True, help="Check the Markdown for errors without producing a PDF.") @@ -177,6 +199,7 @@ def build( date: str | None, template: str | None, accent: str | None, + paper: str | None, bib: str | None, no_branding: bool, dry_run: bool, @@ -253,10 +276,17 @@ def run() -> bool: resolved_author = author or meta.get("author") or _detect_git_user() resolved_subtitle = subtitle or meta.get("subtitle") or "" resolved_date = date or meta.get("date") + resolved_paper = _resolve_paper(paper, meta.get("paper")) + if resolved_paper is None: + console.print(f"[bold red]Error:[/bold red] Unknown paper size — {meta.get('paper')}") + return False console.print(f" [dim]Title:[/dim] [white]{resolved_title}[/white]") console.print(f" [dim]Author:[/dim] [white]{resolved_author}[/white]") - console.print(f" [dim]Template:[/dim] [white]{_template_label(resolved_template)}[/white]") + console.print( + f" [dim]Template:[/dim] [white]{_template_label(resolved_template)}[/white]" + f" [dim]on[/dim] [white]{resolved_paper}[/white]" + ) console.print(f" [dim]Output:[/dim] [cyan]{output_path}[/cyan]") console.print() @@ -285,6 +315,7 @@ def run() -> bool: version=__version__, assets=conversion.assets, generated=conversion.generated, + paper=resolved_paper, ) except Exception as exc: message = getattr(exc, "message", None) or str(exc) diff --git a/doc_engine/compiler.py b/doc_engine/compiler.py index aebe285..02a1100 100644 --- a/doc_engine/compiler.py +++ b/doc_engine/compiler.py @@ -6,6 +6,19 @@ _TEMPLATES_DIR = Path(__file__).parent / "templates" DEFAULT_TEMPLATE = "academic" +DEFAULT_PAPER = "a4" + +PAPER_SIZES = ( + "a3", + "a4", + "a5", + "a6", + "iso-b5", + "jis-b5", + "us-legal", + "us-letter", + "us-tabloid", +) def available_templates() -> list[str]: @@ -39,6 +52,7 @@ def compile_pdf( date: str | None = None, assets: dict[str, str] | None = None, generated: dict[str, str] | None = None, + paper: str = DEFAULT_PAPER, ) -> None: source = template_path(template) if not source.exists(): @@ -73,7 +87,7 @@ def compile_pdf( main_file.write_text( _build_main( typst_body, title, author, bib_inject, accent_inject, - branding, version, subtitle, date, + branding, version, subtitle, date, paper, ), encoding="utf-8", ) @@ -102,6 +116,7 @@ def _build_main( version: str, subtitle: str = "", date: str | None = None, + paper: str = DEFAULT_PAPER, ) -> str: date_line = f' date: "{_escape(date)}",\n' if date else "" return ( @@ -116,6 +131,7 @@ def _build_main( f" accent: {accent_inject},\n" f" branding: {'true' if branding else 'false'},\n" f' version: "{version}",\n' + f' paper: "{paper}",\n' ")\n\n" f"{body}" ) diff --git a/doc_engine/templates/academic.typ b/doc_engine/templates/academic.typ index f4d9140..1f790d1 100644 --- a/doc_engine/templates/academic.typ +++ b/doc_engine/templates/academic.typ @@ -7,6 +7,8 @@ accent: none, branding: true, version: "", + paper: "a4", + ..options, body, ) = { let accent-color = if accent == none { rgb("#2563eb") } else { accent } @@ -18,7 +20,7 @@ set document(author: author, title: title) set page( - paper: "us-letter", + paper: paper, margin: (top: 1.2in, bottom: 1.2in, left: 1.1in, right: 1.1in), header: context { if counter(page).get().first() > 1 [ diff --git a/doc_engine/templates/book.typ b/doc_engine/templates/book.typ index 67dab72..5695ea8 100644 --- a/doc_engine/templates/book.typ +++ b/doc_engine/templates/book.typ @@ -7,6 +7,8 @@ accent: none, branding: true, version: "", + paper: "a4", + ..options, body, ) = { let accent-color = if accent == none { rgb("#7a1f2b") } else { accent } @@ -17,7 +19,7 @@ set document(author: author, title: title) set page( - paper: "us-letter", + paper: paper, margin: (top: 1.3in, bottom: 1.3in, left: 1.4in, right: 1.4in), header: context { if counter(page).get().first() > 1 [ diff --git a/doc_engine/templates/minimal.typ b/doc_engine/templates/minimal.typ index d382c1d..abbbfd1 100644 --- a/doc_engine/templates/minimal.typ +++ b/doc_engine/templates/minimal.typ @@ -7,6 +7,8 @@ accent: none, branding: true, version: "", + paper: "a4", + ..options, body, ) = { let accent-color = if accent == none { rgb("#111827") } else { accent } @@ -17,7 +19,7 @@ set document(author: author, title: title) set page( - paper: "us-letter", + paper: paper, margin: (top: 1in, bottom: 1in, left: 1in, right: 1in), footer: context { set text(font: "Inter", size: 8pt, fill: muted) diff --git a/doc_engine/templates/modern.typ b/doc_engine/templates/modern.typ index 87c6077..26a8661 100644 --- a/doc_engine/templates/modern.typ +++ b/doc_engine/templates/modern.typ @@ -7,6 +7,8 @@ accent: none, branding: true, version: "", + paper: "a4", + ..options, body, ) = { let accent-color = if accent == none { rgb("#0ea5e9") } else { accent } @@ -17,7 +19,7 @@ set document(author: author, title: title) set page( - paper: "us-letter", + paper: paper, margin: (top: 1in, bottom: 1in, left: 1.3in, right: 1.3in), footer: context { set text(font: "Inter", size: 8pt, fill: muted) diff --git a/doc_engine/templates/technical.typ b/doc_engine/templates/technical.typ index 35d3e29..b59d609 100644 --- a/doc_engine/templates/technical.typ +++ b/doc_engine/templates/technical.typ @@ -7,6 +7,8 @@ accent: none, branding: true, version: "", + paper: "a4", + ..options, body, ) = { let accent-color = if accent == none { rgb("#7c3aed") } else { accent } @@ -18,7 +20,7 @@ set document(author: author, title: title) set page( - paper: "us-letter", + paper: paper, margin: (top: 1.1in, bottom: 1.1in, left: 1.1in, right: 1.1in), header: context { if counter(page).get().first() > 1 [ diff --git a/tests/test_cli.py b/tests/test_cli.py index d1481af..950b96a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,8 +1,8 @@ from click.testing import CliRunner from doc_engine import __version__ -from doc_engine.cli import _unique_path, cli -from doc_engine.compiler import _build_main, available_templates +from doc_engine.cli import _resolve_paper, _unique_path, cli +from doc_engine.compiler import DEFAULT_PAPER, _build_main, available_templates class TestTemplates: @@ -32,6 +32,30 @@ def test_custom_template_path_is_rejected_when_missing(self, tmp_path) -> None: assert result.exit_code == 2 +class TestPaper: + def test_defaults_to_a4(self) -> None: + assert _resolve_paper(None, None) == DEFAULT_PAPER == "a4" + + def test_flag_wins_over_front_matter(self) -> None: + assert _resolve_paper("us-letter", "a5") == "us-letter" + + def test_front_matter_is_used_when_no_flag(self) -> None: + assert _resolve_paper(None, "A5") == "a5" + + def test_unknown_front_matter_size_is_rejected(self) -> None: + assert _resolve_paper(None, "tabloid-xl") is None + + def test_build_main_injects_paper(self) -> None: + main = _build_main("body", "T", "Me", "none", "none", True, "2.0.0", "", None, "a5") + assert 'paper: "a5"' in main + + def test_invalid_paper_flag_is_rejected(self, tmp_path) -> None: + doc = tmp_path / "doc.md" + doc.write_text("# T\n\nText.\n") + result = CliRunner().invoke(cli, ["build", str(doc), "--paper", "banana", "--dry-run"]) + assert result.exit_code == 2 + + class TestUniquePath: def test_returns_same_when_absent(self, tmp_path) -> None: target = tmp_path / "out.pdf" From 97400b0cad2a9d6e1d4ca330ca76ad7591631938 Mon Sep 17 00:00:00 2001 From: Leonardo Salas Date: Wed, 19 Aug 2026 13:52:03 -0600 Subject: [PATCH 03/14] feat: translate LaTeX math into Typst math MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Markdown written for other toolchains carries LaTeX math, and Typst does not speak it: a backslash escapes the next character, so $\frac{a}{b}$ reaches the compiler as an unknown variable named rac. A new latex module rewrites it — Greek letters, relations and operators, fractions and roots, scripts, font commands, matrices, and cases. Unrecognized commands drop their backslash, which lands on the right Typst symbol most of the time. Two differences between the languages needed real handling rather than a lookup table. Typst reads a run of letters as one identifier and rejects the ones it does not know, while LaTeX reads xy as x times y, so runs are split into single letters unless they name an operator such as sin or lim. And matrix columns are separated by & in LaTeX but by a comma in Typst, with rows moving from \\ to a semicolon, which made environments context-sensitive. Inline math also gets a stricter rule than the one mistune ships, whose closing guard is a lookahead where it needs a lookbehind. Without that fix "it costs $10 and $20" parsed the middle as math and mangled it — the kind of line that appears in nearly every README this tool is pointed at. Every symbol in the table was checked against the compiler, and the tests compile their translations rather than only comparing strings. --- doc_engine/converter.py | 33 +++- doc_engine/latex.py | 329 ++++++++++++++++++++++++++++++++++++++++ tests/test_latex.py | 152 +++++++++++++++++++ 3 files changed, 512 insertions(+), 2 deletions(-) create mode 100644 doc_engine/latex.py create mode 100644 tests/test_latex.py diff --git a/doc_engine/converter.py b/doc_engine/converter.py index bdaf4cb..b3b1034 100644 --- a/doc_engine/converter.py +++ b/doc_engine/converter.py @@ -17,7 +17,7 @@ import mistune -from doc_engine import diagrams +from doc_engine import diagrams, latex _TYPST_ESCAPES = { "\\": "\\\\", @@ -35,6 +35,14 @@ } _PLUGINS = ["table", "strikethrough", "task_lists", "footnotes"] + +_BLOCK_MATH = r"^ {0,3}\$\$[ \t]*\n(?P[\s\S]+?)\n\$\$[ \t]*$" + +# mistune's own inline rule writes the closing guard as a lookahead where it +# needs a lookbehind, so "it costs $10 and $20" parses the middle as math and +# mangles it. Requiring no space beside either delimiter, and no digit after the +# closing one, keeps prices and shell variables out of math mode. +_INLINE_MATH = r"\$(?!\s)(?P(?:[^$\\\n]|\\.)+?)(? str: return "".join(_TYPST_ESCAPES.get(ch, ch) for ch in text) +def _math_plugin(md: mistune.Markdown) -> None: + """Register math parsing with a stricter inline rule than mistune ships.""" + + def parse_block(block: Any, match: Any, state: Any) -> int: + state.append_token({"type": "block_math", "raw": match.group("math_text")}) + return match.end() + 1 + + def parse_inline(inline: Any, match: Any, state: Any) -> int: + state.append_token({"type": "inline_math", "raw": match.group("math_text")}) + return match.end() + + md.block.register("block_math", _BLOCK_MATH, parse_block, before="list") + md.inline.register("inline_math", _INLINE_MATH, parse_inline, before="link") + + def _render_children(renderer: mistune.BaseRenderer, token: dict, state: Any) -> str: children = token.get("children") if not children: @@ -129,6 +152,12 @@ def inline_html(self, token: dict, state: Any) -> str: return "\\\n" return "" + def inline_math(self, token: dict, state: Any) -> str: + return f"${latex.to_typst(token['raw'])}$" + + def block_math(self, token: dict, state: Any) -> str: + return f"\n$ {latex.to_typst(token['raw'])} $\n\n" + def footnote_ref(self, token: dict, state: Any) -> str: key = token["raw"] body = self._footnotes.get(key, "") @@ -287,7 +316,7 @@ def finalize(self, data: str, state: Any) -> str: def convert_document(markdown: str, base_dir: Path | None = None) -> Conversion: renderer = TypstRenderer(base_dir=base_dir) - md = mistune.create_markdown(renderer=None, plugins=_PLUGINS) + md = mistune.create_markdown(renderer=None, plugins=[*_PLUGINS, _math_plugin]) tokens, state = md.parse(markdown) renderer.load_footnotes(tokens, state) body = renderer.render_tokens(tokens, state) diff --git a/doc_engine/latex.py b/doc_engine/latex.py new file mode 100644 index 0000000..6274f2a --- /dev/null +++ b/doc_engine/latex.py @@ -0,0 +1,329 @@ +"""LaTeX math translated into Typst math. + +Typst has its own math language rather than a LaTeX dialect, so `$\\frac{a}{b}$` +means nothing to it: a backslash escapes the next character, and the compiler +ends up complaining about an unknown variable named `rac`. This module rewrites +the LaTeX people already have in their Markdown into the equivalent Typst. + +Coverage is the common ground of real documents — Greek letters, operators and +relations, fractions and roots, sub- and superscripts, matrices, cases, and the +usual font commands. Anything unrecognized is passed through with its backslash +removed, which is usually right, since most LaTeX command names match a Typst +symbol of the same name. +""" + +from __future__ import annotations + +import re + +# Every name here was checked against the Typst compiler; see tests/test_latex.py. +_SYMBOLS = { + "varepsilon": "epsilon.alt", + "vartheta": "theta.alt", + "varphi": "phi.alt", + "varrho": "rho.alt", + "varsigma": "sigma.alt", + "varpi": "pi.alt", + "infty": "infinity", + "cdot": "dot.op", + "cdots": "dots.c", + "ldots": "dots.h", + "dots": "dots.h", + "vdots": "dots.v", + "ddots": "dots.down", + "pm": "plus.minus", + "mp": "minus.plus", + "le": "<=", + "leq": "<=", + "ge": ">=", + "geq": ">=", + "ne": "!=", + "neq": "!=", + "ll": "<<", + "gg": ">>", + "sim": "tilde.op", + "simeq": "tilde.eq", + "cong": "tilde.equiv", + "propto": "prop", + "notin": "in.not", + "ni": "in.rev", + "subseteq": "subset.eq", + "supseteq": "supset.eq", + "cup": "union", + "cap": "inter", + "setminus": "without", + "oplus": "xor", + "otimes": "times.o", + "circ": "compose", + "to": "arrow.r", + "rightarrow": "arrow.r", + "leftarrow": "arrow.l", + "leftrightarrow": "arrow.l.r", + "Rightarrow": "arrow.r.double", + "Leftarrow": "arrow.l.double", + "Leftrightarrow": "arrow.l.r.double", + "implies": "arrow.r.double", + "iff": "arrow.l.r.double", + "mapsto": "arrow.r.bar", + "langle": "chevron.l", + "rangle": "chevron.r", + "mid": "|", + "vert": "|", + "Vert": "||", + "lvert": "|", + "rvert": "|", + "lVert": "||", + "rVert": "||", + "emptyset": "emptyset", + "varnothing": "emptyset", + "prod": "product", + "bigcup": "union.big", + "bigcap": "inter.big", + "int": "integral", + "iint": "integral.double", + "oint": "integral.cont", + "nabla": "nabla", + "partial": "partial", + "ast": "ast", + "star": "star", + "bullet": "circle.filled.small", + "top": "top", + "bot": "bot", + "perp": "perp", + "angle": "angle", + "triangle": "triangle", + "aleph": "aleph", + "hbar": "planck.reduce", + "ell": "ell", + "Re": "Re", + "Im": "Im", + "quad": "quad", + "qquad": "wide", +} + +# Commands whose LaTeX braces become Typst call arguments. +_ONE_ARG = { + "sqrt": "sqrt", + "text": None, + "textrm": None, + "textbf": "bold", + "textit": "italic", + "mathbb": "bb", + "mathcal": "cal", + "mathfrak": "frak", + "mathbf": "bold", + "mathit": "italic", + "mathrm": "upright", + "mathsf": "sans", + "mathtt": "mono", + "operatorname": None, + "hat": "hat", + "widehat": "hat", + "bar": "macron", + "overline": "overline", + "underline": "underline", + "vec": "arrow", + "tilde": "tilde", + "widetilde": "tilde", + "dot": "dot", + "ddot": "dot.double", + "boldsymbol": "bold", +} + +_TWO_ARG = { + "frac": "frac", + "dfrac": "frac", + "tfrac": "frac", + "binom": "binom", +} + +# LaTeX environments and the Typst constructor that replaces them. +_MATRIX_ENVIRONMENTS = { + "matrix": ("mat", None), + "pmatrix": ("mat", "#none"), + "bmatrix": ("mat", '"["'), + "Bmatrix": ("mat", '"{"'), + "vmatrix": ("mat", '"|"'), + "Vmatrix": ("mat", '"||"'), +} + +# How `\\` and `&` translate, per context: LaTeX separates matrix columns with +# `&` and rows with `\\`, while Typst uses `,` and `;`. In cases, every row is a +# single Typst argument, so `&` is only alignment and collapses to a space. +_ROW_SEPARATORS = { + "normal": ("\\", "&"), + "matrix": (";", ","), + "cases": (",", " "), +} + +_SPACING = {",": "thin", ";": "med", ":": "med", "!": "", " ": " "} +_COMMAND = re.compile(r"[A-Za-z]+") +_LETTERS = re.compile(r"[A-Za-z]+") + +# Typst reads a run of letters as one identifier and fails when it does not know +# it, while LaTeX reads `xy` as x times y. Runs are therefore split into single +# letters unless they name one of these operators, which Typst does define. +_OPERATORS = frozenset( + """ + arccos arcsin arctan arg ceil cos cosh cot coth csc deg det dif dim exp floor + gcd hom id im inf ker lcm lim liminf limsup ln log max min mod Pr round sec + sin sinh sup tan tanh tr upright + """.split() +) + + +def to_typst(latex: str) -> str: + """Translate a LaTeX math expression into Typst math markup.""" + return _convert(latex.strip()).strip() + + +def _convert(source: str, mode: str = "normal") -> str: + out: list[str] = [] + index = 0 + while index < len(source): + char = source[index] + if char == "\\": + text, index = _command(source, index, mode) + _append(out, text) + elif char == "{": + group, index = _group(source, index) + _append(out, f"({_convert(group, mode)})") + elif char == "&": + _append(out, _ROW_SEPARATORS[mode][1]) + index += 1 + elif char.isalpha(): + match = _LETTERS.match(source, index) + run = match.group() + index = match.end() + _append(out, run if run in _OPERATORS else " ".join(run)) + else: + _append(out, char) + index += 1 + return "".join(out) + + +def _append(out: list[str], text: str) -> None: + """Add *text*, keeping a space between neighbours that would otherwise merge. + + Typst would read the result of `i` followed by `\\pi` as one identifier + called `ipi`, so adjacent names have to stay separated. + """ + if text and out: + previous = out[-1] + if previous and (previous[-1].isalnum() or previous[-1] == ".") and text[0].isalnum(): + out.append(" ") + out.append(text) + + +def _command(source: str, index: int, mode: str) -> tuple[str, int]: + """Translate the command starting at the backslash in *index*.""" + index += 1 + if index >= len(source): + return "", index + + char = source[index] + if char == "\\": + return _ROW_SEPARATORS[mode][0], index + 1 + if char in _SPACING: + return _SPACING[char], index + 1 + if char in "{}|": + return {"{": "{", "}": "}", "|": "||"}[char], index + 1 + + match = _COMMAND.match(source, index) + if not match: + return char, index + 1 + + name = match.group() + index = match.end() + + if name == "begin": + return _environment(source, index) + if name == "end": + _, index = _next_argument(source, index) + return "", index + if name in ("left", "right"): + return "", index + if name in _TWO_ARG: + first, index = _next_argument(source, index) + second, index = _next_argument(source, index) + return f"{_TWO_ARG[name]}({_convert(first, mode)}, {_convert(second, mode)})", index + if name in _ONE_ARG: + if name == "sqrt" and _peek(source, index) == "[": + degree, index = _bracket(source, index) + radicand, index = _next_argument(source, index) + return f"root({_convert(degree, mode)}, {_convert(radicand, mode)})", index + argument, index = _next_argument(source, index) + function = _ONE_ARG[name] + if function is None: + return f'"{argument.strip()}"', index + return f"{function}({_convert(argument, mode)})", index + + return _SYMBOLS.get(name, name), index + + +def _environment(source: str, index: int) -> tuple[str, int]: + """Translate a `\\begin{env}...\\end{env}` block into its Typst constructor.""" + name, index = _next_argument(source, index) + name = name.strip() + body, index = _environment_body(source, index, name) + + if name == "cases": + return f"cases({_convert(body, 'cases').strip()})", index + if name in _MATRIX_ENVIRONMENTS: + constructor, delimiter = _MATRIX_ENVIRONMENTS[name] + inner = _convert(body, "matrix").strip().strip(";").strip() + if delimiter is None: + return f"{constructor}(delim: #none, {inner})", index + if delimiter == "#none": + return f"{constructor}({inner})", index + return f"{constructor}(delim: {delimiter}, {inner})", index + return _convert(body, "normal"), index + + +def _environment_body(source: str, index: int, name: str) -> tuple[str, int]: + """Return everything up to the matching `\\end{name}`, and the index past it.""" + closing = f"\\end{{{name}}}" + end = source.find(closing, index) + if end == -1: + return source[index:], len(source) + return source[index:end], end + len(closing) + + +def _next_argument(source: str, index: int) -> tuple[str, int]: + """Read the next `{...}` group, or a single token when there are no braces.""" + while index < len(source) and source[index] == " ": + index += 1 + if index >= len(source): + return "", index + if source[index] == "{": + return _group(source, index) + if source[index] == "\\": + match = _COMMAND.match(source, index + 1) + end = match.end() if match else index + 2 + return source[index:end], end + return source[index], index + 1 + + +def _group(source: str, index: int) -> tuple[str, int]: + depth = 0 + start = index + 1 + while index < len(source): + if source[index] == "{": + depth += 1 + elif source[index] == "}": + depth -= 1 + if depth == 0: + return source[start:index], index + 1 + index += 1 + return source[start:], index + + +def _bracket(source: str, index: int) -> tuple[str, int]: + end = source.find("]", index) + if end == -1: + return "", index + return source[index + 1 : end], end + 1 + + +def _peek(source: str, index: int) -> str: + return source[index] if index < len(source) else "" diff --git a/tests/test_latex.py b/tests/test_latex.py new file mode 100644 index 0000000..6db8d70 --- /dev/null +++ b/tests/test_latex.py @@ -0,0 +1,152 @@ +import pytest + +from doc_engine.converter import convert +from doc_engine.latex import to_typst + +typst = pytest.importorskip("typst") + + +def compiles(expression: str, tmp_path) -> bool: + """Compile a Typst math expression, so the tests check real output.""" + source = tmp_path / "probe.typ" + source.write_text(f"$ {expression} $", encoding="utf-8") + try: + typst.compile(str(source), output=str(tmp_path / "probe.pdf")) + except Exception: + return False + return True + + +class TestStructure: + def test_fraction(self) -> None: + assert to_typst(r"\frac{a}{b}") == "frac(a, b)" + + def test_nested_fraction(self) -> None: + assert to_typst(r"\frac{x^2}{\sqrt{y}}") == "frac(x^2, sqrt(y))" + + def test_root_with_degree(self) -> None: + assert to_typst(r"\sqrt[3]{x}") == "root(3, x)" + + def test_binomial(self) -> None: + assert to_typst(r"\binom{n}{k}") == "binom(n, k)" + + def test_braces_become_grouping_parentheses(self) -> None: + assert to_typst(r"x^{2n}") == "x^(2 n)" + + def test_left_and_right_are_dropped(self) -> None: + assert "left" not in to_typst(r"\left( x \right)") + + +class TestSymbols: + def test_greek(self) -> None: + assert to_typst(r"\alpha \beta \gamma") == "alpha beta gamma" + + def test_variant_greek_maps_to_alternate(self) -> None: + assert to_typst(r"\varepsilon") == "epsilon.alt" + + def test_relations(self) -> None: + assert to_typst(r"a \leq b \neq c") == "a <= b != c" + + def test_set_operations(self) -> None: + assert to_typst(r"A \cup B \cap C") == "A union B inter C" + + def test_mid_becomes_a_bar(self) -> None: + assert to_typst(r"P(A \mid B)") == "P(A | B)" + + def test_unknown_command_keeps_its_name(self) -> None: + assert to_typst(r"\zeta") == "zeta" + + +class TestLetterRuns: + def test_runs_are_split_into_separate_variables(self) -> None: + assert to_typst("xy") == "x y" + + def test_operators_stay_whole(self) -> None: + assert to_typst(r"\sin x") == "sin x" + assert to_typst("lim") == "lim" + + def test_symbol_does_not_merge_with_neighbour(self) -> None: + assert to_typst(r"i\pi") == "i pi" + + +class TestEnvironments: + def test_matrix_columns_and_rows(self) -> None: + assert to_typst(r"\begin{pmatrix} a & b \\ c & d \end{pmatrix}") == "mat(a , b ; c , d)" + + def test_bracket_matrix_sets_delimiter(self) -> None: + assert 'delim: "["' in to_typst(r"\begin{bmatrix} a \\ b \end{bmatrix}") + + def test_cases_rows_are_arguments(self) -> None: + result = to_typst(r"\begin{cases} x & \text{if } y \\ z & \text{else} \end{cases}") + assert result.startswith("cases(") + assert '"if"' in result + + +class TestFonts: + def test_blackboard_bold(self) -> None: + assert to_typst(r"\mathbb{R}") == "bb(R)" + + def test_calligraphic(self) -> None: + assert to_typst(r"\mathcal{L}") == "cal(L)" + + def test_text_becomes_a_string(self) -> None: + assert to_typst(r"\text{if}") == '"if"' + + +class TestCompiles: + """Everything above is only useful if Typst actually accepts the output.""" + + @pytest.mark.parametrize( + "expression", + [ + r"\frac{-b \pm \sqrt{b^2-4ac}}{2a}", + r"e^{i\pi} + 1 = 0", + r"\int_{-\infty}^{\infty} e^{-x^2}\,dx = \sqrt{\pi}", + r"\sum_{i=1}^{n} i = \frac{n(n+1)}{2}", + r"\lim_{x \to 0} \frac{\sin x}{x} = 1", + r"P(A \mid B) = \frac{P(B \mid A)P(A)}{P(B)}", + r"\begin{pmatrix} a & b \\ c & d \end{pmatrix}", + r"\begin{cases} x & \text{if } x > 0 \\ -x & \text{otherwise} \end{cases}", + r"\forall \epsilon > 0, \exists \delta", + r"x \in \mathbb{Z} \setminus \{0\}", + r"\vec{v} \cdot \hat{n}", + r"\max_{i} x_i", + ], + ) + def test_translation_compiles(self, expression: str, tmp_path) -> None: + assert compiles(to_typst(expression), tmp_path) + + +class TestMarkdownIntegration: + def test_inline_math_is_translated(self) -> None: + assert "frac(a, b)" in convert(r"Value is $\frac{a}{b}$ here.") + + def test_block_math_is_translated(self) -> None: + result = convert("$$\n\\frac{a}{b}\n$$\n") + assert "$ frac(a, b) $" in result + + +class TestDollarsThatAreNotMath: + """Prices and shell variables must survive having math enabled.""" + + @pytest.mark.parametrize( + "text", + [ + "Price is $10", + "It costs $10 and $20 total", + "Between $5 and $500 per month", + "Use $HOME and $PATH variables", + "Run: export $FOO && echo $BAR", + "A $ B $ C", + ], + ) + def test_literal_dollars_are_escaped_not_parsed(self, text: str) -> None: + result = convert(text) + assert "\\$" in result + for word in text.replace("$", "").split(): + assert word in result + + def test_real_inline_math_still_works_alongside(self) -> None: + result = convert(r"Costs $10 but $x^2$ is math") + assert "\\$10" in result + assert "$x^2$" in result From b7fb5c1907e29400d2a2bdf7681c605eff118fa0 Mon Sep 17 00:00:00 2001 From: Leonardo Salas Date: Wed, 19 Aug 2026 14:34:37 -0600 Subject: [PATCH 04/14] feat: rewrite the help screens with Rich MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `doc-engine --help` listed two commands and two global flags, so finding the options that matter meant knowing to ask for `build --help` first. Since building is what nearly every run does, the top-level screen now shows those flags directly, grouped by what they affect — output, document details, appearance, and the flags used while writing — instead of one flat column. Both screens open with worked examples. Descriptions are read from the Click parameters, so the help text still has a single source of truth and cannot drift from the flags themselves. `info` moves to a grid so wrapped values line up under their label, and it now reports what the build can actually do: page sizes, accents, diagrams, math. --- doc_engine/cli.py | 45 ++++++---- doc_engine/help.py | 202 +++++++++++++++++++++++++++++++++++++++++++++ tests/test_cli.py | 34 ++++++++ 3 files changed, 264 insertions(+), 17 deletions(-) create mode 100644 doc_engine/help.py diff --git a/doc_engine/cli.py b/doc_engine/cli.py index 230cef6..beff317 100644 --- a/doc_engine/cli.py +++ b/doc_engine/cli.py @@ -6,8 +6,9 @@ from pathlib import Path import click -from rich.console import Console +from rich.console import Console, Group from rich.panel import Panel +from rich.table import Table from doc_engine import __version__, frontmatter from doc_engine.compiler import ( @@ -19,6 +20,7 @@ ) from doc_engine.converter import convert_document, extract_title, strip_first_heading from doc_engine.diagrams import DiagramError +from doc_engine.help import RichCommand, RichGroup from doc_engine.linter import has_errors, lint if sys.platform == "win32": @@ -151,15 +153,15 @@ def _print_issues(issues: list, filename: str) -> None: console.print(f" [{color}]{issue.format(filename)}[/{color}]") -@click.group(invoke_without_command=True) +@click.group(cls=RichGroup, invoke_without_command=True) @click.version_option(__version__, prog_name="doc-engine") @click.pass_context def cli(ctx: click.Context) -> None: if ctx.invoked_subcommand is None: - click.echo(ctx.get_help()) + ctx.get_help() -@cli.command() +@cli.command(cls=RichCommand) @click.argument("input_file", required=False, type=click.Path(exists=False)) @click.option("-o", "--output", default=None, help="Output PDF file path.") @click.option("-t", "--title", default=None, help="Document title override.") @@ -373,20 +375,29 @@ def _mtime(path: Path) -> float: return 0.0 -@cli.command() +@cli.command(cls=RichCommand) def info() -> None: - """Show version, repository, and available templates.""" - templates = ", ".join(available_templates()) - body = ( - f"[bold white]doc-engine-cli[/bold white] [dim]v{__version__}[/dim]\n" - "Zero-config Markdown → PDF documentation engine.\n\n" - f"[dim]Repository:[/dim] [cyan]{REPO_URL}[/cyan]\n" - f"[dim]Templates:[/dim] {templates}\n\n" - "[dim]Common usage[/dim]\n" - " doc-engine build\n" - " doc-engine build README.md --template modern --accent teal\n" - " doc-engine build --watch\n\n" - "[dim]Run[/dim] [cyan]doc-engine --help[/cyan] [dim]for every command and flag.[/dim]" + """Show version, repository, and what this build supports.""" + facts = Table.grid(padding=(0, 2)) + facts.add_column(style="dim", no_wrap=True, vertical="top") + facts.add_column() + facts.add_row("Repository", f"[cyan]{REPO_URL}[/cyan]") + facts.add_row("Templates", ", ".join(available_templates())) + facts.add_row("Page sizes", f"{', '.join(PAPER_SIZES)} [dim](default {DEFAULT_PAPER})[/dim]") + facts.add_row("Accents", f"{', '.join(sorted(_NAMED_ACCENTS))} [dim]or any hex[/dim]") + + body = Group( + f"[bold white]doc-engine-cli[/bold white] [dim]v{__version__}[/dim]", + "Turn Markdown into a polished PDF. No LaTeX, no config.", + "", + facts, + "", + "[dim]Markdown support[/dim]", + " tables · task lists · footnotes · local images · bibliography", + " mermaid and svg code blocks rendered as diagrams", + " LaTeX math, inline with $…$ and display with $$…$$", + "", + "[dim]Run[/dim] [cyan]doc-engine --help[/cyan] [dim]for every command and flag.[/dim]", ) console.print(Panel(body, border_style="blue", padding=(1, 2), title="info")) diff --git a/doc_engine/help.py b/doc_engine/help.py new file mode 100644 index 0000000..c54236e --- /dev/null +++ b/doc_engine/help.py @@ -0,0 +1,202 @@ +"""Help screens rendered with Rich. + +Click's default help splits the tool in two: `--help` lists the commands, and +the flags people actually want live behind `build --help`. Since building is +what nearly every run does, the top-level screen shows those flags directly, +grouped by what they affect rather than in one flat column. + +Help text is read from the Click parameters themselves, so there is one source +of truth for every description. +""" + +from __future__ import annotations + +import click +from rich.console import Console +from rich.panel import Panel +from rich.table import Table +from rich.text import Text + +# Which build options belong under which heading, in the order they are shown. +_GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("Input and output", ("output", "force", "open_pdf")), + ("Document details", ("title", "subtitle", "author", "date", "bib")), + ("Appearance", ("template", "accent", "paper", "no_branding")), + ("While you work", ("watch", "dry_run")), +) + +_EXAMPLES = ( + ("doc-engine build", "Build README.md from the current folder"), + ("doc-engine build notes.md", "Build a specific file"), + ("doc-engine build --watch", "Rebuild every time you save"), + ("doc-engine build --template modern --accent teal", "Pick a layout and color"), + ("doc-engine build --paper us-letter", "Use US Letter instead of A4"), + ("doc-engine build --dry-run", "Check for problems without writing a PDF"), +) + + +class RichGroup(click.Group): + """The top-level command, whose help doubles as the guide to `build`.""" + + def format_help(self, ctx: click.Context, formatter: click.HelpFormatter) -> None: + console = Console() + _banner(console, ctx) + _usage(console) + _examples(console) + + build = self.get_command(ctx, "build") + if build is not None: + _options(console, build) + + _commands(console, self, ctx) + _footer(console) + + +class RichCommand(click.Command): + """A subcommand whose options are grouped the same way.""" + + def format_help(self, ctx: click.Context, formatter: click.HelpFormatter) -> None: + console = Console() + console.print() + console.print( + Text.assemble( + ("Usage ", "bold"), + (f"doc-engine {self.name} ", "bold cyan"), + (_argument_hint(self), "cyan"), + ("[OPTIONS]", "dim"), + ) + ) + if self.help: + console.print(Text(f" {self.help.strip()}", style="dim")) + if self.name == "build": + _examples(console) + _options(console, self) + _footer(console) + + +def _banner(console: Console, ctx: click.Context) -> None: + from doc_engine import __version__ + + console.print() + console.print( + Panel( + Text.assemble( + ("doc-engine ", "bold white"), + (f"v{__version__}", "dim"), + ("\nTurn Markdown into a polished PDF. No LaTeX, no config.", "dim"), + ), + border_style="blue", + padding=(0, 2), + ) + ) + + +def _usage(console: Console) -> None: + console.print() + console.print( + Text.assemble( + ("Usage ", "bold"), + ("doc-engine build ", "bold cyan"), + ("[FILE] ", "cyan"), + ("[OPTIONS]", "dim"), + ) + ) + console.print( + Text.assemble( + (" ", ""), + ("FILE", "cyan"), + (" defaults to the README.md in the current folder.", "dim"), + ) + ) + + +def _examples(console: Console) -> None: + console.print() + console.print(Text("Examples", style="bold")) + table = Table.grid(padding=(0, 2)) + table.add_column(style="cyan", no_wrap=True) + table.add_column(style="dim") + for command, description in _EXAMPLES: + table.add_row(f" {command}", description) + console.print(table) + + +def _options(console: Console, command: click.Command) -> None: + by_name = {param.name: param for param in command.params} + shown: set[str] = set() + + for heading, names in _GROUPS: + rows = [(by_name[name], name) for name in names if name in by_name] + if not rows: + continue + console.print() + console.print(Text(heading, style="bold")) + table = Table.grid(padding=(0, 2)) + table.add_column(style="cyan", no_wrap=True) + table.add_column() + for param, name in rows: + shown.add(name) + table.add_row(f" {_flags(param)}", _describe(param)) + console.print(table) + + extra = [ + param + for param in command.params + if param.name not in shown + and isinstance(param, click.Option) + and param.name not in ("help",) + ] + if extra: + console.print() + console.print(Text("Other", style="bold")) + table = Table.grid(padding=(0, 2)) + table.add_column(style="cyan", no_wrap=True) + table.add_column() + for param in extra: + table.add_row(f" {_flags(param)}", _describe(param)) + console.print(table) + + +def _commands(console: Console, group: click.Group, ctx: click.Context) -> None: + console.print() + console.print(Text("Commands", style="bold")) + table = Table.grid(padding=(0, 2)) + table.add_column(style="cyan", no_wrap=True) + table.add_column(style="dim") + for name in sorted(group.list_commands(ctx)): + command = group.get_command(ctx, name) + if command is None: + continue + table.add_row(f" doc-engine {name}", (command.short_help or command.help or "").strip()) + table.add_row(" doc-engine --version", "Print the version and exit") + console.print(table) + + +def _footer(console: Console) -> None: + console.print() + console.print( + Text.assemble( + ("Docs ", "dim"), + ("https://github.com/leonardosalasd/doc-engine-cli", "cyan"), + ) + ) + console.print() + + +def _flags(param: click.Parameter) -> str: + if isinstance(param, click.Argument): + return param.name.upper() + return ", ".join(param.opts) + + +def _describe(param: click.Parameter) -> Text: + text = Text(getattr(param, "help", "") or "") + choices = param.type.choices if isinstance(param.type, click.Choice) else None + if choices: + text.append(f"\n{', '.join(choices)}", style="dim") + return text + + +def _argument_hint(command: click.Command) -> str: + names = [p.name.upper() for p in command.params if isinstance(p, click.Argument)] + return f"[{names[0]}] " if names else "" diff --git a/tests/test_cli.py b/tests/test_cli.py index 950b96a..ce787c8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -120,3 +120,37 @@ def test_invalid_template_is_rejected(self, tmp_path) -> None: good.write_text("# T\n\nText.\n") result = CliRunner().invoke(cli, ["build", str(good), "--template", "nope", "--dry-run"]) assert result.exit_code == 2 + + +class TestHelp: + def test_top_level_help_lists_build_options(self) -> None: + result = CliRunner().invoke(cli, ["--help"]) + assert result.exit_code == 0 + for expected in ("--template", "--accent", "--paper", "--watch"): + assert expected in result.output + + def test_top_level_help_groups_options(self) -> None: + result = CliRunner().invoke(cli, ["--help"]) + for heading in ("Input and output", "Document details", "Appearance"): + assert heading in result.output + + def test_top_level_help_shows_examples_and_commands(self) -> None: + result = CliRunner().invoke(cli, ["--help"]) + assert "Examples" in result.output + assert "doc-engine info" in result.output + + def test_build_help_still_works(self) -> None: + result = CliRunner().invoke(cli, ["build", "--help"]) + assert result.exit_code == 0 + assert "--output" in result.output + + def test_bare_invocation_shows_help(self) -> None: + result = CliRunner().invoke(cli, []) + assert result.exit_code == 0 + assert "Usage" in result.output + + def test_info_lists_capabilities(self) -> None: + result = CliRunner().invoke(cli, ["info"]) + flat = " ".join(result.output.split()) + assert "mermaid" in flat + assert "math" in flat.lower() From d5276ab4fe922f8a29a8a89bac98191c3d2e2654 Mon Sep 17 00:00:00 2001 From: Leonardo Salas Date: Wed, 19 Aug 2026 14:42:58 -0600 Subject: [PATCH 05/14] feat: build one PDF from many files with a manifest A project that outgrew a single file can now list its parts in doc-engine.md as ordinary Markdown links, so the manifest still reads as a table of contents on GitHub. Entries are handled by what they are: Markdown files become sections, .mmd sources are rendered where they appear, images are placed as captioned figures, and a .bib registers as the bibliography for the document. Front matter at the top carries the metadata for the whole thing. Each entry converts against its own folder, so a picture referenced from inside an included file resolves exactly as it does when that file is built alone. Asset names are namespaced per entry so two folders can both hold a logo.png without one overwriting the other. Watch mode follows every file the manifest names, and re-reads the list after each rebuild in case the manifest itself changed. Also fixes tables, which have been wrong since 1.0.0: the header row was read as a table_row when a table_head holds its cells directly, so the column count came from the first cell's text and every table collapsed into one column. --- doc_engine/cli.py | 76 +++++++++++++---- doc_engine/converter.py | 22 +++-- doc_engine/manifest.py | 179 ++++++++++++++++++++++++++++++++++++++++ tests/test_converter.py | 17 ++++ tests/test_manifest.py | 106 ++++++++++++++++++++++++ 5 files changed, 375 insertions(+), 25 deletions(-) create mode 100644 doc_engine/manifest.py create mode 100644 tests/test_manifest.py diff --git a/doc_engine/cli.py b/doc_engine/cli.py index beff317..5ef0a2c 100644 --- a/doc_engine/cli.py +++ b/doc_engine/cli.py @@ -10,7 +10,7 @@ from rich.panel import Panel from rich.table import Table -from doc_engine import __version__, frontmatter +from doc_engine import __version__, frontmatter, manifest from doc_engine.compiler import ( DEFAULT_PAPER, DEFAULT_TEMPLATE, @@ -223,11 +223,12 @@ def build( if input_file: input_path = Path(input_file) else: - input_path = _find_file(cwd, _README_CANDIDATES) + input_path = manifest.find(cwd) or _find_file(cwd, _README_CANDIDATES) if not input_path: console.print( - "[bold red]Error:[/bold red] No README.md found in current directory.\n" - "[dim]Provide an input file or run from a directory containing a README.md.[/dim]" + "[bold red]Error:[/bold red] Nothing to build in this directory.\n" + "[dim]Add a README.md, write a doc-engine.md manifest, " + "or name a file to convert.[/dim]" ) raise SystemExit(1) console.print(f" [dim]Auto-detected:[/dim] [cyan]{input_path.name}[/cyan]") @@ -251,16 +252,28 @@ def build( output_path = _unique_path(output_path) def run() -> bool: - raw = input_path.read_text(encoding="utf-8") - meta, content = frontmatter.parse(raw) - - issues = lint(raw) - if issues: - _print_issues(issues, str(input_path)) - console.print() - if has_errors(issues): - console.print("[bold red]Aborted:[/bold red] fix the errors above, or run [cyan]--dry-run[/cyan] to recheck.") + collected: manifest.Manifest | None = None + if manifest.is_manifest(input_path): + try: + collected = manifest.load(input_path) + except manifest.ManifestError as exc: + console.print(f"[bold red]Error:[/bold red] {exc}") return False + meta = collected.metadata + content = "" + to_lint = [entry.path for entry in collected.entries if entry.kind == "markdown"] + else: + meta, content = frontmatter.parse(input_path.read_text(encoding="utf-8")) + to_lint = [input_path] + + for source in to_lint: + issues = lint(source.read_text(encoding="utf-8")) + if issues: + _print_issues(issues, str(source)) + console.print() + if has_errors(issues): + console.print("[bold red]Aborted:[/bold red] fix the errors above, or run [cyan]--dry-run[/cyan] to recheck.") + return False resolved_template = template or _resolve_template(meta.get("template", DEFAULT_TEMPLATE)) if resolved_template is None: @@ -274,7 +287,9 @@ def run() -> bool: console.print(f"[bold yellow]Warning:[/bold yellow] Ignoring unknown accent — {meta['accent']}") resolved_bib = _resolve_bib(bib or meta.get("bib"), cwd) - resolved_title = title or meta.get("title") or extract_title(content) + if resolved_bib is None and collected is not None: + resolved_bib = collected.bibliography + resolved_title = title or meta.get("title") or extract_title(content) or input_path.stem resolved_author = author or meta.get("author") or _detect_git_user() resolved_subtitle = subtitle or meta.get("subtitle") or "" resolved_date = date or meta.get("date") @@ -294,7 +309,12 @@ def run() -> bool: with console.status("[bold blue]Converting Markdown → Typst…[/bold blue]"): try: - conversion = convert_document(strip_first_heading(content), base_dir=input_path.parent) + if collected is not None: + conversion = manifest.assemble(collected) + else: + conversion = convert_document( + strip_first_heading(content), base_dir=input_path.parent + ) except DiagramError as exc: console.print( f"\n[bold red]Diagram failed:[/bold red] {exc.language} block — {exc.message}" @@ -354,20 +374,40 @@ def _resolve_bib(bib: str | None, cwd: Path) -> Path | None: def _watch(input_path: Path, run) -> None: - console.print("\n[dim]Watching for changes — press Ctrl+C to stop.[/dim]") - last = _mtime(input_path) + watched = _watch_list(input_path) + count = len(watched) + noun = "file" if count == 1 else "files" + console.print(f"\n[dim]Watching {count} {noun} — press Ctrl+C to stop.[/dim]") + last = _fingerprint(watched) try: while True: time.sleep(0.5) - current = _mtime(input_path) + current = _fingerprint(watched) if current != last: last = current console.rule(f"[dim]{time.strftime('%H:%M:%S')} — rebuilding[/dim]") run() + # A rebuild can change which files the manifest pulls in. + watched = _watch_list(input_path) + last = _fingerprint(watched) except KeyboardInterrupt: console.print("\n[dim]Stopped.[/dim]") +def _watch_list(input_path: Path) -> list[Path]: + """The manifest and everything it pulls in, so editing any part rebuilds.""" + if not manifest.is_manifest(input_path): + return [input_path] + try: + return [input_path, *manifest.load(input_path).sources] + except (manifest.ManifestError, OSError): + return [input_path] + + +def _fingerprint(paths: list[Path]) -> tuple[float, ...]: + return tuple(_mtime(path) for path in paths) + + def _mtime(path: Path) -> float: try: return path.stat().st_mtime diff --git a/doc_engine/converter.py b/doc_engine/converter.py index b3b1034..a240f26 100644 --- a/doc_engine/converter.py +++ b/doc_engine/converter.py @@ -102,10 +102,11 @@ class Conversion: class TypstRenderer(mistune.BaseRenderer): NAME = "typst" - def __init__(self, base_dir: Path | None = None) -> None: + def __init__(self, base_dir: Path | None = None, namespace: str = "") -> None: super().__init__() self._ordered_stack: list[bool] = [] self._base_dir = base_dir + self._namespace = namespace self._footnotes: dict[str, str] = {} self._asset_names: dict[str, str] = {} self.assets: dict[str, str] = {} @@ -188,7 +189,7 @@ def block_code(self, token: dict, state: Any) -> str: def _render_diagram(self, language: str, source: str) -> str: svg = diagrams.render(language, source) - name = f"assets/diagram_{len(self.generated)}.svg" + name = f"assets/{self._namespace}diagram_{len(self.generated)}.svg" self.generated[name] = svg return f'\n#align(center)[#fit-image("{name}")]\n\n' @@ -251,8 +252,8 @@ def table(self, token: dict, state: Any) -> str: if not head: return "" - head_row = head.get("children", [{}])[0] - cells = head_row.get("children", []) + # A table_head holds its cells directly, without a table_row in between. + cells = head.get("children", []) n = len(cells) if n == 0: return "" @@ -305,7 +306,7 @@ def _register_image(self, url: str) -> str | None: resolved = str(source.resolve()) if resolved in self._asset_names: return self._asset_names[resolved] - name = f"assets/{len(self.assets)}_{_UNSAFE.sub('_', source.name)}" + name = f"assets/{self._namespace}{len(self.assets)}_{_UNSAFE.sub('_', source.name)}" self._asset_names[resolved] = name self.assets[name] = resolved return name @@ -314,8 +315,15 @@ def finalize(self, data: str, state: Any) -> str: return data -def convert_document(markdown: str, base_dir: Path | None = None) -> Conversion: - renderer = TypstRenderer(base_dir=base_dir) +def convert_document( + markdown: str, base_dir: Path | None = None, namespace: str = "" +) -> Conversion: + """Convert Markdown to Typst. + + *namespace* prefixes the names of assets and rendered diagrams, so several + documents assembled into one PDF cannot overwrite each other's files. + """ + renderer = TypstRenderer(base_dir=base_dir, namespace=namespace) md = mistune.create_markdown(renderer=None, plugins=[*_PLUGINS, _math_plugin]) tokens, state = md.parse(markdown) renderer.load_footnotes(tokens, state) diff --git a/doc_engine/manifest.py b/doc_engine/manifest.py new file mode 100644 index 0000000..f76fae7 --- /dev/null +++ b/doc_engine/manifest.py @@ -0,0 +1,179 @@ +"""A manifest that builds one PDF out of many files. + +A project that outgrew a single Markdown file lists its parts in a manifest — +`doc-engine.md` by default — as ordinary Markdown links, so the file still reads +and renders as a table of contents on GitHub: + + --- + title: Payments API + template: academic + --- + + - [Overview](doc/overview.md) + - [Architecture](diagrams/architecture.mmd) + - [Schema](img/schema.png) + - [References](bib/references.bib) + +Each entry is handled by what it is. Markdown files are appended as sections, +diagram sources are rendered where they appear, images are placed as figures, +and a bibliography is registered for the whole document. Paths resolve against +the manifest's own folder, and headings inside an included file keep their +levels, so a `##` stays a subsection of the entry above it. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path + +from doc_engine import diagrams, frontmatter +from doc_engine.converter import Conversion, convert_document + +MANIFEST_NAMES = ("doc-engine.md", "docengine.md", "SUMMARY.md") + +_LINK = re.compile(r"\[(?P