From f2ec3e98f2a188274cbb0b53ba078b2b0c3a11cb Mon Sep 17 00:00:00 2001 From: Tobias Kaestner Date: Sat, 26 Sep 2026 20:49:11 +0200 Subject: [PATCH 1/5] feat: registry: let a document opt out of cross-referencing Every document is a link peer of every other: intersphinx, doxylink, external needs and Doxygen TAGFILES are derived for all pairs. That is wrong for a large reference build that documents a superset of its peers' symbols, such as a project's full API beside a scoped subset. When two Doxygen projects that both document a symbol import each other's tag files, each defers the symbol to the other and neither generates its page, leaving dead links on both sides. Add a per-document `crossref: false`. Such a document is still built, deployed and listed in the navigation, but it is removed from the link graph in both directions: no peer gets an entry for it, and it gets none for its peers. A quoted "false" is rejected at configure time instead of being read as a truthy string. First unit tests for docrefs: 6 tests over load(), tagfiles() and validation. Against the unchanged code 5 fail; the sixth pins that the navigation entry is kept, which was already true. Suite 147 -> 153. Found on the Zephyr safety docset, where the full Zephyr API and the safety-scoped API both document k_queue: 23 dead cross-project links, all gone with the full API isolated. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tobias Kaestner --- doc/manual/reference/registry-schema.rst | 11 +++ scripts/docrefs.py | 28 +++++- .../_tests/test_docrefs_crossref.py | 91 +++++++++++++++++++ 3 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 sphinx/_extensions/_tests/test_docrefs_crossref.py diff --git a/doc/manual/reference/registry-schema.rst b/doc/manual/reference/registry-schema.rst index 5eb18f6..32647eb 100644 --- a/doc/manual/reference/registry-schema.rst +++ b/doc/manual/reference/registry-schema.rst @@ -182,6 +182,17 @@ Each entry in ``documents:``, keyed by its id: Downloaded at build time and stored locally as ``doxygen.tag``, whatever the remote file is actually called. +``crossref`` + Boolean, default ``true``. ``false`` takes the document out of the + cross-reference graph in both directions: no peer gets an intersphinx, + doxylink, external-needs or Doxygen ``TAGFILES`` entry for it, and it gets + none for its peers. It is still built, deployed and listed in the + navigation. Meant for a large reference build that documents a superset of + its peers' symbols — a project's full API beside a scoped subset. Doxygen + projects that import each other's tag files leave shared symbols to the + other project, so neither generates their pages. A quoted ``"false"`` is a + configure-time error rather than a truthy string. + ``needs`` Opt-in sub-block; presence is what makes this document importable as external needs by every peer. Two shapes: diff --git a/scripts/docrefs.py b/scripts/docrefs.py index 40c026a..195a182 100644 --- a/scripts/docrefs.py +++ b/scripts/docrefs.py @@ -62,6 +62,20 @@ DOXYGEN_TAGFILE = "doxygen.tag" +def _crossref(meta): + """Whether a document takes part in cross-document linking. + + ``crossref: false`` keeps a document's build, deploy tree and navigation + entry but removes it from the link graph in both directions: no peer gets + an intersphinx, doxylink, needs or ``TAGFILES`` entry for it, and it gets + none for its peers. For a large reference build that documents a superset + of its peers' symbols (e.g. a project's full API beside a scoped subset), + importing each other's tag files makes each Doxygen project defer the + shared symbols to the other, so neither generates their pages. + """ + return meta.get("crossref", True) + + def _registry(registry): """Load, parse and validate the registry YAML. @@ -164,6 +178,13 @@ def _validate(data): raise ValueError( f"docrefs: document '{doc_id}' has kind '{kind}', which is not one of {_KINDS}" ) + if not isinstance(meta.get("crossref", True), bool): + # A quoted "false" is a truthy string: accepting it would leave + # the document linked while the registry says it is not. + raise ValueError( + f"docrefs: document '{doc_id}' has crossref " + f"'{meta['crossref']}', which is not a boolean (true/false)" + ) if ( kind in ("external", "sphinx-external", "doxygen-external") and meta.get("remote-url") is None @@ -721,8 +742,9 @@ def _nav_href(doc_id, meta): needs_external_needs = [] doxylink = {} + this_crossref = _crossref(documents.get(this_doc, {})) for doc_id, meta in documents.items(): - if doc_id == this_doc: + if doc_id == this_doc or not (this_crossref and _crossref(meta)): continue kind = meta.get("kind", "sphinx") @@ -865,8 +887,10 @@ def tagfiles(this_doc, deploy_dir, registry=None): this_html = (deploy / this_path).as_posix() entries = [] + if not _crossref(documents.get(this_doc, {})): + return "" for doc_id, meta in documents.items(): - if doc_id == this_doc: + if doc_id == this_doc or not _crossref(meta): continue kind = meta.get("kind") if kind == "doxygen": diff --git a/sphinx/_extensions/_tests/test_docrefs_crossref.py b/sphinx/_extensions/_tests/test_docrefs_crossref.py new file mode 100644 index 0000000..7caa929 --- /dev/null +++ b/sphinx/_extensions/_tests/test_docrefs_crossref.py @@ -0,0 +1,91 @@ +# Copyright (c) 2026 inovex GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +"""``crossref: false`` removes a document from the link graph in both +directions, while keeping it in the navigation.""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "scripts")) + +import docrefs # noqa: E402 + +REGISTRY = """\ +base_url: "http://localhost:8000/" +groups: + - id: reference + title: Reference +documents: + guide: + kind: sphinx + group: reference + builders: [html] + needs: + source: json + api: + kind: doxygen + group: reference + design: + kind: doxygen + group: reference + full-api: + kind: doxygen + group: reference + crossref: false +""" + + +@pytest.fixture +def registry(tmp_path): + path = tmp_path / "documents.yaml" + path.write_text(REGISTRY) + return path + + +@pytest.fixture +def load(registry, tmp_path, monkeypatch): + def _load(doc_id): + # load() derives the build root from the per-target OUTPUT_DIR. + monkeypatch.setenv("OUTPUT_DIR", str(tmp_path / "deploy" / "html" / doc_id)) + return docrefs.load(doc_id, registry=registry) + + return _load + + +def test_tagfiles_skip_an_isolated_peer(registry, tmp_path): + entries = docrefs.tagfiles("api", tmp_path / "deploy", registry=registry) + assert "html/design/doxygen.tag" in entries + assert "full-api" not in entries + + +def test_isolated_document_imports_no_tagfiles(registry, tmp_path): + assert docrefs.tagfiles("full-api", tmp_path / "deploy", registry=registry) == "" + + +def test_load_skips_an_isolated_peer(load): + refs = load("guide") + assert set(refs.doxylink) == {"api", "design"} + + +def test_isolated_document_gets_no_link_maps(load): + refs = load("full-api") + assert refs.doxylink == {} + assert refs.intersphinx_mapping == {} + assert refs.needs_external_needs == [] + + +def test_isolated_document_stays_in_navigation(load): + refs = load("guide") + hrefs = [link["href"] for group in refs.reference_groups for link in group["links"]] + assert "../full-api/index.html" in hrefs + + +def test_crossref_must_be_a_boolean(tmp_path): + path = tmp_path / "documents.yaml" + path.write_text(REGISTRY.replace("crossref: false", 'crossref: "false"')) + with pytest.raises(ValueError, match="not a boolean"): + docrefs.tagfiles("api", tmp_path / "deploy", registry=path) From 5cf873a7fe631e7bc0ff1711e43f5c4109147bd2 Mon Sep 17 00:00:00 2001 From: Tobias Kaestner Date: Sat, 26 Sep 2026 22:00:41 +0200 Subject: [PATCH 2/5] fix: doxygen: silence warnings in stage 1 Stage 1 runs Doxygen over the same inputs as stage 2, only to produce the tag file and XML, so every warning was printed twice. Its copy is also partly false: TAGFILES is cleared in stage 1, so every reference into a peer document warns as unresolved. That makes it both noise and a hazard for any consumer who sets WARN_AS_ERROR, because the warnings that count are stage 2's. Turn warnings off in the stage-1 overlay. WARNINGS = NO alone is not enough: the WARN_IF_* switches warn independently of it and left 40 of 341 warnings on one document. With the full set, 0 remain and the tag file is unchanged (same compounds). WARN_AS_ERROR = NO keeps stage 1 from ever failing the build on a consumer's setting. On the Zephyr safety docset a full build goes from 3439 to 975 warning lines (together with the consumer fixing its own stale aliases); the test specification is unchanged. The acceptance suite (zdocs-tests) was not available in that workspace and has not been run. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tobias Kaestner --- cmake/doxygen.cmake | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cmake/doxygen.cmake b/cmake/doxygen.cmake index 5cfbe18..4c134b5 100644 --- a/cmake/doxygen.cmake +++ b/cmake/doxygen.cmake @@ -325,6 +325,18 @@ function(add_doxygen_target name) "GENERATE_MAN = NO\n" "GENERATE_RTF = NO\n" "GENERATE_DOCBOOK = NO\n" + # Stage 2 parses the same inputs and reports every warning again, so + # stage 1's copy is pure duplication. Worse, stage 1 warns falsely: with + # TAGFILES cleared, every reference into a peer is "unresolved". Its + # warnings must therefore never fail the build either. WARNINGS alone is + # not enough: the WARN_IF_* switches warn independently of it. + "WARNINGS = NO\n" + "WARN_IF_UNDOCUMENTED = NO\n" + "WARN_IF_DOC_ERROR = NO\n" + "WARN_IF_INCOMPLETE_DOC = NO\n" + "WARN_NO_PARAMDOC = NO\n" + "WARN_IF_UNDOC_ENUM_VAL = NO\n" + "WARN_AS_ERROR = NO\n" ) # Doxygen is invoked through run_doxygen.cmake (a `cmake -P` wrapper) so a From c7797b601b21be320ddc9c896159d4c6b12235b3 Mon Sep 17 00:00:00 2001 From: Tobias Kaestner Date: Sat, 26 Sep 2026 22:03:47 +0200 Subject: [PATCH 3/5] feat: doccheck: accept named findings from the registry doc-check fails on any finding, including defects the consumer cannot fix, such as a dead link inside a generated upstream API. The only way to a passing build was then to stop running the check, which hides every other finding along with it. Add an optional registry list, doc_check_accepted, of findings to let through, each with a required reason. Matching is on the exact printed finding, never a pattern, so an acceptance cannot swallow a new, different finding. Accepted findings no longer fail the check but are still printed under "accepted (N)" with their reason. An entry that matches nothing is reported as "accepted but no longer found" without failing, so fixing the defect never breaks the build and the list does not go stale unnoticed. A malformed entry exits 2. First tests for doccheck: 5, run as real invocations against a small deploy tree. Against the unchanged code 4 fail; the fifth pins that an unaccepted dead link still fails. Suite 153 -> 158. The acceptance suite (zdocs-tests) was not available and has not been run. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tobias Kaestner --- doc/manual/reference/registry-schema.rst | 18 +++++ scripts/doccheck.py | 56 ++++++++++++- .../_tests/test_doccheck_accepted.py | 79 +++++++++++++++++++ 3 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 sphinx/_extensions/_tests/test_doccheck_accepted.py diff --git a/doc/manual/reference/registry-schema.rst b/doc/manual/reference/registry-schema.rst index 32647eb..618f6ef 100644 --- a/doc/manual/reference/registry-schema.rst +++ b/doc/manual/reference/registry-schema.rst @@ -54,6 +54,24 @@ Top level or empty — see :doc:`cli`. Optional; without it, ``doc-check`` simply skips that one check. +``doc_check_accepted`` + Optional list of ``doccheck`` findings to let through, for defects the + consumer cannot fix (e.g. one inside a generated upstream API): + + .. code-block:: yaml + + doc_check_accepted: + - finding: "html/api/structfoo.html: dead link -> structfoo_1_1_0d13.html" + reason: "Doxygen does not generate nested anonymous struct pages" + + ``finding`` is the finding's exact printed text, never a pattern, so an + acceptance cannot swallow a new, different finding. ``reason`` is required. + Accepted findings do not fail the check, but they are still printed under + ``accepted (N)`` with their reason. An entry that no longer matches anything + is printed under ``accepted but no longer found`` without failing, so fixing + the defect never breaks the build and the list does not rot. A malformed + entry is a bad invocation (exit 2). + ``doxygen_xml`` Project-scoped boolean, default off. When true, **every** ``kind: doxygen`` document in the registry generates XML into diff --git a/scripts/doccheck.py b/scripts/doccheck.py index 5876652..cad4f87 100755 --- a/scripts/doccheck.py +++ b/scripts/doccheck.py @@ -200,6 +200,29 @@ def check_deploy_links(deploy: Path, base_url: str) -> list[str]: return findings +def _accepted_findings(entries) -> dict[str, str] | None: + """The registry's ``doc_check_accepted:`` list as ``{finding: reason}``. + + Each entry names one finding by its exact printed text, never a pattern, + so an acceptance cannot swallow a new, different finding. ``reason`` is + required: an exception nobody can explain is one nobody can retire. + Returns ``None`` (after printing why) for a malformed list. + """ + accepted: dict[str, str] = {} + for i, entry in enumerate(entries or []): + finding = entry.get("finding") if isinstance(entry, dict) else None + reason = entry.get("reason") if isinstance(entry, dict) else None + if not (isinstance(finding, str) and finding and isinstance(reason, str) and reason): + print( + f"doccheck: doc_check_accepted[{i}] needs a non-empty 'finding' " + "and 'reason'", + file=sys.stderr, + ) + return None + accepted[finding] = reason + return accepted + + def main() -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--registry", type=Path, required=True) @@ -223,6 +246,10 @@ def main() -> int: print(f"doccheck: no deploy tree at {args.deploy}", file=sys.stderr) return 2 + accepted = _accepted_findings(raw.get("doc_check_accepted")) + if accepted is None: + return 2 + groups: list[tuple[str, list[str]]] = [] if smoke_page: groups.append( @@ -231,13 +258,34 @@ def main() -> int: groups.append(("dead deploy links", check_deploy_links(args.deploy, base_url))) total = 0 + matched: set[str] = set() for title, findings in groups: - if findings: - total += len(findings) - print(f"\ndoccheck: {title} ({len(findings)}):", file=sys.stderr) - for f in findings: + failing = [f for f in findings if f not in accepted] + matched.update(f for f in findings if f in accepted) + if failing: + total += len(failing) + print(f"\ndoccheck: {title} ({len(failing)}):", file=sys.stderr) + for f in failing: print(f" {f}", file=sys.stderr) + # Accepted findings are printed, never hidden: the build passes, but the + # log still says what is broken and why it was let through. + if matched: + print(f"\ndoccheck: accepted ({len(matched)}):", file=sys.stderr) + for f in sorted(matched): + print(f" {f}\n reason: {accepted[f]}", file=sys.stderr) + stale = sorted(set(accepted) - matched) + if stale: + # Not a failure — fixing the underlying problem must not break the + # build — but said out loud, so the list does not rot. + print( + f"\ndoccheck: accepted but no longer found ({len(stale)}) — " + "remove from doc_check_accepted:", + file=sys.stderr, + ) + for f in stale: + print(f" {f}", file=sys.stderr) + if total: print(f"\ndoccheck: FAILED with {total} finding(s)", file=sys.stderr) return 1 diff --git a/sphinx/_extensions/_tests/test_doccheck_accepted.py b/sphinx/_extensions/_tests/test_doccheck_accepted.py new file mode 100644 index 0000000..2078771 --- /dev/null +++ b/sphinx/_extensions/_tests/test_doccheck_accepted.py @@ -0,0 +1,79 @@ +# Copyright (c) 2026 inovex GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +"""``doc_check_accepted:`` lets named findings through without hiding them.""" + +import subprocess +import sys +from pathlib import Path + +import pytest + +DOCCHECK = Path(__file__).resolve().parents[3] / "scripts" / "doccheck.py" + +DEAD = "html/api/index.html: dead link -> missing.html" + + +@pytest.fixture +def deploy(tmp_path): + page = tmp_path / "deploy" / "html" / "api" / "index.html" + page.parent.mkdir(parents=True) + page.write_text('x self') + return tmp_path / "deploy" + + +def run(tmp_path, deploy, accepted_yaml=""): + registry = tmp_path / "documents.yaml" + registry.write_text('base_url: "http://localhost:8000/"\n' + accepted_yaml) + return subprocess.run( + [sys.executable, DOCCHECK, "--registry", registry, "--deploy", deploy], + capture_output=True, + text=True, + ) + + +def test_dead_link_fails_without_acceptance(tmp_path, deploy): + result = run(tmp_path, deploy) + assert result.returncode == 1 + assert DEAD in result.stderr + + +def test_accepted_finding_passes_but_is_still_reported(tmp_path, deploy): + result = run( + tmp_path, + deploy, + f'doc_check_accepted:\n - finding: "{DEAD}"\n reason: "issue 042"\n', + ) + assert result.returncode == 0 + assert "accepted (1)" in result.stderr + assert DEAD in result.stderr + assert "issue 042" in result.stderr + + +def test_acceptance_is_exact_not_a_pattern(tmp_path, deploy): + result = run( + tmp_path, + deploy, + 'doc_check_accepted:\n - finding: "html/api/index.html: dead link"\n' + ' reason: "prefix only"\n', + ) + assert result.returncode == 1 + assert "no longer found (1)" in result.stderr + + +def test_stale_acceptance_is_reported_but_does_not_fail(tmp_path, deploy): + (deploy / "html" / "api" / "missing.html").write_text("now it exists") + result = run( + tmp_path, + deploy, + f'doc_check_accepted:\n - finding: "{DEAD}"\n reason: "issue 042"\n', + ) + assert result.returncode == 0 + assert "no longer found (1)" in result.stderr + + +def test_acceptance_without_reason_is_a_bad_invocation(tmp_path, deploy): + result = run(tmp_path, deploy, f'doc_check_accepted:\n - finding: "{DEAD}"\n') + assert result.returncode == 2 + assert "needs a non-empty 'finding' and 'reason'" in result.stderr From 896ee1e7fc1db554ab0321474600086e07564553 Mon Sep 17 00:00:00 2001 From: Tobias Kaestner Date: Sun, 27 Sep 2026 17:36:21 +0200 Subject: [PATCH 4/5] feat: registry: publish a sphinx-needs document's needs as a Doxygen tag file A `kind: sphinx` document with `needs: {source: json}` can now declare `doxygen_tag: true` or `doxygen_tag: {types: [...]}`. The new `-needstag` target (`docrefs.py needs-tag`) turns the document's stage-1 needs.json into deploy/html//needs.tag, one `` per need, and every doxygen peer lists it in TAGFILES. `\verifies` and `\satisfies` then resolve against requirements authored in reStructuredText and link to their Sphinx pages. An unknown UID still trips the stage-2 "Reference to unknown requirement" gate. The target joins doc-index but not doc-tags. Every Sphinx index waits on doc-tags, so a Doxygen input derived from a Sphinx index could not be built there without a cycle. Nothing needs the tag earlier, because stage-1 Doxygen blanks its TAGFILES. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tobias Kaestner --- cmake/registry.cmake | 28 ++- doc/manual/reference/registry-schema.rst | 30 +++ scripts/docrefs.py | 181 +++++++++++++++++ .../_tests/test_docrefs_needs_tag.py | 185 ++++++++++++++++++ 4 files changed, 423 insertions(+), 1 deletion(-) create mode 100644 sphinx/_extensions/_tests/test_docrefs_needs_tag.py diff --git a/cmake/registry.cmake b/cmake/registry.cmake index 7a4f41c..e912898 100644 --- a/cmake/registry.cmake +++ b/cmake/registry.cmake @@ -75,6 +75,9 @@ endmacro() # with zero output formats (fail loudly at configure # time, per this engine's own convention — see # add_sphinx_target's identical BUILDERS check). +# With `doxygen_tag:`, also -needstag: its needs +# as deploy/html//needs.tag, after -index, in +# doc-index but not doc-tags. # - depends on every qualifying document's own # - target. # --nodeps the same, but against each document's @@ -118,7 +121,9 @@ endmacro() - ``sphinx`` (or omitted) — :cmake:command:`add_sphinx_target` ``( BUILDERS REGISTRY [DOCDIR ...])``. A configure-time ``FATAL_ERROR`` naming the document if ``builders:`` is - empty or missing. + empty or missing. A document with ``doxygen_tag:`` also gets + ``-needstag``, which writes its needs as a Doxygen tag file after its + stage-1 index (in ``doc-index``, not ``doc-tags``). - ``doxygen`` — :cmake:command:`add_doxygen_target` ``( REGISTRY [DOCDIR ...])``. - ``external`` / ``sphinx-external`` — no CMake target of any kind. @@ -306,6 +311,27 @@ function(add_docs_from_registry) if(NOT _zdocs_testmodule_spec STREQUAL "") add_dependencies(${_zdocs_id}-index ${_zdocs_testmodule_spec}-index) endif() + + # doxygen_tag: this document's needs, as a Doxygen tag file every + # doxygen peer lists in TAGFILES (docrefs.py tagfiles), so `\verifies` + # and `\satisfies` resolve against requirements authored in rst. + # Generated from the needs.json that -index exports, so it runs + # after that. It joins doc-index but deliberately NOT doc-tags: every + # Sphinx index waits on doc-tags, so membership there would be a cycle + # (-needstag -> -index -> doc-tags -> -needstag). Nothing + # needs it earlier, because stage-1 doxygen blanks its TAGFILES. + string(JSON _zdocs_doxygen_tag GET "${_zdocs_entry}" "doxygen_tag") + if(_zdocs_doxygen_tag) + add_custom_target( + ${_zdocs_id}-needstag + COMMAND + ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../scripts/docrefs.py + needs-tag ${_zdocs_id} ${CMAKE_CURRENT_BINARY_DIR}/deploy --registry ${ARGS_REGISTRY} + COMMENT "Doxygen needs tag for ${_zdocs_id}..." + ) + add_dependencies(${_zdocs_id}-needstag ${_zdocs_id}-index) + add_dependencies(doc-index ${_zdocs_id}-needstag) + endif() endif() endforeach() diff --git a/doc/manual/reference/registry-schema.rst b/doc/manual/reference/registry-schema.rst index 618f6ef..fbf05a8 100644 --- a/doc/manual/reference/registry-schema.rst +++ b/doc/manual/reference/registry-schema.rst @@ -238,6 +238,36 @@ Each entry in ``documents:``, keyed by its id: rejected at configure time as a ``spec:`` target, because ``testreport`` correlates against ``needs.json``, and the synthesized stub is not that. +``doxygen_tag`` + Opt-in; publishes this document's needs as Doxygen requirements, so a + ``\verifies`` or ``\satisfies`` in any ``kind: doxygen`` peer resolves + against requirements authored in reStructuredText, and links to their + Sphinx pages. Two shapes: + + .. code-block:: yaml + + doxygen_tag: true # every need this document defines + + .. code-block:: yaml + + doxygen_tag: + types: [requirement] # only needs of these types + + Requires ``kind: sphinx`` and ``needs: {source: json}``, because the tag file + is generated from this document's own ``needs.json``. Anything else is a + configure-time error, as are an unknown key and an empty or non-list + ``types``. The engine adds a ``-needstag`` target that writes + ``deploy/html//needs.tag`` after the document's stage-1 index, and lists + that file in every Doxygen peer's ``TAGFILES``. Imported (external) needs are + left out, because their own document publishes them. + + Nothing but Sphinx parses the requirements, so no ``.dox`` is generated and + no Doxygen project is built for them. A ``\verifies`` naming an id the tag + does not declare warns ``Reference to unknown requirement``, which the + stage-2 warning gate (``ZDOCS_DOXYGEN_WARN_FAIL_PATTERNS``) fails on. + ``crossref: false`` on either side removes the entry, as for any tag file. + Needs Doxygen 1.16 or newer. + ``testmodule`` Opt-in sub-block (see :doc:`../explanation/testmodule-and-twister` and :doc:`directives-and-roles`); its presence is the sole trigger that loads diff --git a/scripts/docrefs.py b/scripts/docrefs.py index 195a182..0c74fa5 100644 --- a/scripts/docrefs.py +++ b/scripts/docrefs.py @@ -61,6 +61,13 @@ #: the engine renames it to this on download. DOXYGEN_TAGFILE = "doxygen.tag" +#: Filename of the Doxygen tag file a ``doxygen_tag:`` Sphinx document publishes +#: beside its ``needs.json`` (see :func:`needs_tag`). Named apart from +#: :data:`DOXYGEN_TAGFILE` because it is not written by Doxygen and describes +#: needs, not symbols. Shared with ``cmake/registry.cmake`` only through the +#: ``needs-tag`` CLI, which writes it. +NEEDS_TAGFILE = "needs.tag" + def _crossref(meta): """Whether a document takes part in cross-document linking. @@ -140,6 +147,50 @@ def _registry(registry): _TESTMODULE_KEYS = ("doxygen_source", "api_reference", "spec") +#: Allowed keys inside a document's ``doxygen_tag:`` block. +_DOXYGEN_TAG_KEYS = ("types",) + + +def _validate_doxygen_tag(doc_id, meta, kind, doxygen_tag): + """Raise ``ValueError`` unless ``doxygen_tag:`` is usable on ``doc_id``.""" + if kind != "sphinx": + raise ValueError( + f"docrefs: document '{doc_id}' has 'doxygen_tag:' but is kind " + f"'{kind}' — only a 'kind: sphinx' document publishes its needs " + f"as a Doxygen tag file" + ) + needs = meta.get("needs") + if needs is None or needs.get("source") != "json": + raise ValueError( + f"docrefs: document '{doc_id}' has 'doxygen_tag:' but no " + f"'needs: {{source: json}}' — the tag file is generated from this " + f"document's own needs.json export" + ) + if doxygen_tag is True: + return + if not isinstance(doxygen_tag, dict): + # `false` included: absence already means "off", and a second spelling + # of it only invites a quoted "false" that would read as true. + raise ValueError( + f"docrefs: document '{doc_id}' has doxygen_tag '{doxygen_tag}' — " + f"use 'true' or a mapping such as '{{types: [requirement]}}'" + ) + unknown = sorted(set(doxygen_tag) - set(_DOXYGEN_TAG_KEYS)) + if unknown: + raise ValueError( + f"docrefs: document '{doc_id}' has unknown key(s) {unknown} in its " + f"'doxygen_tag:' block — allowed keys are {_DOXYGEN_TAG_KEYS}" + ) + types = doxygen_tag.get("types") + if types is not None and ( + not isinstance(types, list) or not types or not all(isinstance(t, str) for t in types) + ): + raise ValueError( + f"docrefs: document '{doc_id}' has doxygen_tag.types '{types}' — " + f"it must be a non-empty list of need type names" + ) + + def _validate(data): """Raise ``ValueError`` on any registry validation problem.""" groups = data.get("groups") @@ -208,6 +259,16 @@ def _validate(data): f"docrefs: {kind} document '{doc_id}' is missing a 'remote-tagfile:' field" ) + # -- doxygen_tag: publish the needs as a Doxygen tag file --------- + # + # Only a Sphinx document that exports its own needs.json can publish + # one: the tag is generated FROM that export. Rejected here rather than + # at build time, where a missing needs.json would leave every + # `\verifies` in the peers "unknown" with no hint why. + doxygen_tag = meta.get("doxygen_tag") + if doxygen_tag is not None: + _validate_doxygen_tag(doc_id, meta, kind, doxygen_tag) + # -- testmodule: sub-block (step 27) ----------------------------- # # Rejected at CONFIGURE time (rule 6), naming the offending document, @@ -903,9 +964,102 @@ def tagfiles(this_doc, deploy_dir, registry=None): tag = (deploy / path / DOXYGEN_TAGFILE).as_posix() base_dir_url = _intersphinx_target_dir(meta["remote-url"]) entries.append(f"{tag}={base_dir_url}") + elif kind in (None, "sphinx") and meta.get("doxygen_tag") is not None: + # `kind` is read raw above; omitted means sphinx. + # A needs tag (see needs_tag()): its compounds' filenames are the + # Sphinx pages, relative to that document's html root. + path = meta.get("path", f"html/{doc_id}") + tag = (deploy / path / NEEDS_TAGFILE).as_posix() + location = posixpath.relpath((deploy / path).as_posix(), this_html) + entries.append(f"{tag}={location}") return " ".join(entries) +def _needs_tag_xml(needs, types): + """Doxygen tag-file XML declaring each need in ``needs`` as a requirement. + + Doxygen >= 1.16 resolves ``\\verifies`` / ``\\satisfies`` against a + ```` from any tag file in ``TAGFILES``, and + links to ``/#`` — it appends the anchor itself. So + ``filename`` is the need's page, and the anchor is the need id, which is + what sphinx-needs uses for the need's target. An id no compound declares + still warns "Reference to unknown requirement", which is what the stage-2 + warn-log gate keys on. + + ``title`` is the bare need title: Doxygen already parenthesises it where it + renders a reference (``SD-REQ-001 (Title)``). It is XML-escaped only, + which is what Doxygen writes into its own tag files. Doxygen then reads the + title as markup, and inconsistently: the requirements page interprets + commands (``@c``, ``\\b``) that the reference text shows verbatim, and a + backslash escape is consumed in one place but printed in the other. No + encoding is right in both, so a title with Doxygen markup characters + renders as Doxygen would render the same ``\\requirement`` title, and a + ```` in it warns "Unsupported xml/html tag". The warning is harmless + to the build, because it doesn't match the stage-2 gate's pattern. + """ + from xml.sax.saxutils import escape + + lines = [ + "", + "", + "", + ] + for need_id in sorted(needs): + need = needs[need_id] + if need.get("is_external"): + # Imported from a peer: that peer publishes it, if anyone does. + continue + if types is not None and need.get("type") not in types: + continue + lines += [ + ' ', + f" {escape(need_id)}", + f" {escape(need.get('title') or '')}", + f" {escape(need['docname'])}.html", + " ", + ] + lines.append("") + return "\n".join(lines) + "\n" + + +def needs_tag(doc_id, deploy_dir, registry=None): + """Write ``doc_id``'s needs as a Doxygen tag file; return its path. + + Reads the ``needs.json`` that the document's own stage-1 ``xref`` build + exported, and writes :data:`NEEDS_TAGFILE` beside it. It is the only link + from sphinx-needs sources to Doxygen's ``\\requirement`` vocabulary, so + the requirements stay authored in reStructuredText and are parsed by + Sphinx alone. There is no second parser and no generated ``.dox``. + + Nothing reads the tag before stage 2 (stage-1 Doxygen blanks ``TAGFILES``), + so ``cmake/registry.cmake`` builds it after ``-index`` and outside + ``doc-tags``. That ordering is what keeps this cycle-free. + + The file is rewritten only when its content changes, so an unchanged + requirement set does not look new to anything that tracks timestamps. + """ + documents = _registry(registry)["documents"] + meta = documents.get(doc_id) + if meta is None or meta.get("doxygen_tag") is None: + raise ValueError(f"docrefs: document '{doc_id}' does not declare 'doxygen_tag:'") + html_dir = Path(deploy_dir) / meta.get("path", f"html/{doc_id}") + needs_json = html_dir / "needs.json" + if not needs_json.is_file(): + raise ValueError( + f"docrefs: {needs_json} not found — '{doc_id}' must be built " + f"(its stage-1 index) before its needs tag" + ) + export = json.loads(needs_json.read_text()) + needs = export["versions"][export["current_version"]]["needs"] + types = meta["doxygen_tag"].get("types") if isinstance(meta["doxygen_tag"], dict) else None + + out = html_dir / NEEDS_TAGFILE + content = _needs_tag_xml(needs, types) + if not out.is_file() or out.read_text() != content: + out.write_text(content) + return out + + def navlinks(this_doc, registry=None): """Grouped cross-document navigation entries for ``this_doc``. @@ -1022,6 +1176,9 @@ def manifest(registry=None): never its own spec), so this cannot cycle; it is deliberately NOT generalised to every needs-importer/publisher pair, which CAN cycle and is its own, deferred step. + * ``doxygen_tag`` — ``True`` when the document declares + ``doxygen_tag:``. ``add_docs_from_registry`` then adds the + ``-needstag`` target (see :func:`needs_tag`). """ documents = _registry(registry)["documents"] entries = [] @@ -1038,6 +1195,7 @@ def manifest(registry=None): "remote_tagfile": meta.get("remote-tagfile"), "testmodule_doxygen_source": testmodule.get("doxygen_source"), "testmodule_spec": testmodule.get("spec"), + "doxygen_tag": meta.get("doxygen_tag") is not None, } ) return entries @@ -1115,6 +1273,24 @@ def _cli(argv=None): help="path to the document registry (documents.yaml)", ) + p_needs_tag = sub.add_parser( + "needs-tag", + help="write a document's needs as a Doxygen tag file", + description="Read a doxygen_tag: document's needs.json and write " + f"{NEEDS_TAGFILE} beside it, declaring each need as a Doxygen " + "requirement so \\verifies / \\satisfies resolve against it.", + ) + p_needs_tag.add_argument("doc_id", help="registry id of the Sphinx document") + p_needs_tag.add_argument( + "deploy_dir", help="path to the deploy/ directory holding the built docs" + ) + p_needs_tag.add_argument( + "--registry", + metavar="documents.yaml", + required=True, + help="path to the document registry (documents.yaml)", + ) + p_version = sub.add_parser( "version", help="print a document's git-derived version string", @@ -1152,6 +1328,11 @@ def _cli(argv=None): sys.stdout.write("TRUE" if xml_enabled(args.registry) else "FALSE") elif args.command == "manifest": sys.stdout.write(json.dumps(manifest(args.registry))) + elif args.command == "needs-tag": + try: + needs_tag(args.doc_id, args.deploy_dir, args.registry) + except ValueError as exc: + sys.exit(str(exc)) elif args.command == "version": meta = _registry(args.registry)["documents"].get(args.doc_id, {}) ver = resolve_version( diff --git a/sphinx/_extensions/_tests/test_docrefs_needs_tag.py b/sphinx/_extensions/_tests/test_docrefs_needs_tag.py new file mode 100644 index 0000000..3910f82 --- /dev/null +++ b/sphinx/_extensions/_tests/test_docrefs_needs_tag.py @@ -0,0 +1,185 @@ +# Copyright (c) 2026 inovex GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +"""``doxygen_tag:`` publishes a Sphinx document's needs as a Doxygen tag file, +so ``\\verifies`` / ``\\satisfies`` resolve against requirements authored in rst.""" + +import json +import os +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "scripts")) + +import docrefs # noqa: E402 + +REGISTRY = """\ +groups: + - id: reference + title: Reference +documents: + requirements: + kind: sphinx + group: reference + builders: [html] + needs: + source: json + doxygen_tag: + types: [requirement, top_requirement] + api: + kind: doxygen + group: reference + full-api: + kind: doxygen + group: reference + crossref: false +""" + +NEEDS = { + "SD-REQ-001": { + "id": "SD-REQ-001", + "title": "Sealing initialisation & more", + "type": "requirement", + "docname": "detailed", + "is_external": False, + }, + "SD-TOP-001": { + "id": "SD-TOP-001", + "title": "Integrity detection", + "type": "top_requirement", + "docname": "sub/top-level", + "is_external": False, + }, + "TC_ONE": { + "id": "TC_ONE", + "title": "A test case", + "type": "test_case", + "docname": "tests", + "is_external": False, + }, + "PEER-001": { + "id": "PEER-001", + "title": "Imported", + "type": "requirement", + "docname": None, + "is_external": True, + }, +} + + +def _write_registry(tmp_path, text=REGISTRY): + path = tmp_path / "documents.yaml" + path.write_text(text) + return path + + +@pytest.fixture +def registry(tmp_path): + return _write_registry(tmp_path) + + +@pytest.fixture +def deploy(tmp_path): + html = tmp_path / "deploy" / "html" / "requirements" + html.mkdir(parents=True) + export = {"current_version": "v1", "versions": {"v1": {"needs": NEEDS}}} + (html / "needs.json").write_text(json.dumps(export)) + return tmp_path / "deploy" + + +def _compounds(path): + root = ET.parse(path).getroot() + return { + c.findtext("id"): (c.get("kind"), c.findtext("title"), c.findtext("filename")) + for c in root.iter("compound") + } + + +def test_tag_declares_each_own_need_of_the_selected_types(registry, deploy): + out = docrefs.needs_tag("requirements", deploy, registry=registry) + assert out == deploy / "html" / "requirements" / docrefs.NEEDS_TAGFILE + assert _compounds(out) == { + "SD-REQ-001": ("requirement", "Sealing initialisation & more", "detailed.html"), + "SD-TOP-001": ("requirement", "Integrity detection", "sub/top-level.html"), + } + + +def test_true_selects_every_type_but_never_imported_needs(tmp_path, deploy): + registry = _write_registry( + tmp_path, + REGISTRY.replace( + "doxygen_tag:\n types: [requirement, top_requirement]", "doxygen_tag: true" + ), + ) + ids = set(_compounds(docrefs.needs_tag("requirements", deploy, registry=registry))) + assert ids == {"SD-REQ-001", "SD-TOP-001", "TC_ONE"} + + +def test_unchanged_content_is_not_rewritten(registry, deploy): + out = docrefs.needs_tag("requirements", deploy, registry=registry) + stamp = out.stat().st_mtime_ns + + os.utime(out, ns=(stamp - 10**9, stamp - 10**9)) + docrefs.needs_tag("requirements", deploy, registry=registry) + assert out.stat().st_mtime_ns == stamp - 10**9 + + +def test_missing_export_names_the_document(registry, tmp_path): + with pytest.raises(ValueError, match="stage-1 index"): + docrefs.needs_tag("requirements", tmp_path / "nowhere", registry=registry) + + +def test_doxygen_peers_list_the_tag_relative_to_their_html(registry, tmp_path): + entries = docrefs.tagfiles("api", tmp_path / "deploy", registry=registry) + assert f"html/requirements/{docrefs.NEEDS_TAGFILE}=../requirements" in entries + + +def test_omitted_kind_counts_as_sphinx(tmp_path): + registry = _write_registry(tmp_path, REGISTRY.replace(" kind: sphinx\n", "", 1)) + entries = docrefs.tagfiles("api", tmp_path / "deploy", registry=registry) + assert f"html/requirements/{docrefs.NEEDS_TAGFILE}" in entries + + +def test_isolated_peer_gets_no_needs_tag(registry, tmp_path): + assert docrefs.tagfiles("full-api", tmp_path / "deploy", registry=registry) == "" + + +def test_manifest_flags_the_document(registry): + flags = {e["id"]: e["doxygen_tag"] for e in docrefs.manifest(registry)} + assert flags == {"requirements": True, "api": False, "full-api": False} + + +@pytest.mark.parametrize( + ("before", "after", "message"), + [ + (" needs:\n source: json\n", "", "needs: {source: json}"), + ("types: [requirement, top_requirement]", "typez: [requirement]", "unknown key"), + ("types: [requirement, top_requirement]", "types: []", "non-empty list"), + ("types: [requirement, top_requirement]", "types: requirement", "non-empty list"), + ( + " doxygen_tag:\n types: [requirement, top_requirement]\n", + " doxygen_tag: false\n", + "use 'true' or a mapping", + ), + ], +) +def test_invalid_declarations_fail_validation(tmp_path, before, after, message): + assert before in REGISTRY + registry = _write_registry(tmp_path, REGISTRY.replace(before, after)) + with pytest.raises(ValueError, match=message): + docrefs.manifest(registry) + + +def test_only_a_sphinx_document_can_publish_one(tmp_path): + registry = _write_registry( + tmp_path, + REGISTRY.replace( + " api:\n kind: doxygen\n", " api:\n kind: doxygen\n doxygen_tag: true\n" + ), + ) + with pytest.raises(ValueError, match="only a 'kind: sphinx' document"): + docrefs.manifest(registry) From 3ded634076fa99e678fc06acfb8d7ebf0a21f64d Mon Sep 17 00:00:00 2001 From: Tobias Kaestner Date: Tue, 29 Sep 2026 10:35:54 +0200 Subject: [PATCH 5/5] fix: sphinx: pass the found ZEPHYR_BASE to every Sphinx run zdocs_conf loads Zephyr's doc extensions, including external_content, which stages the sources, only when ZEPHYR_BASE is in the environment. Its comment assumed find_package(Zephyr) exports it, but find_package sets only the CMake variable. So a build worked only when the user's shell exported ZEPHYR_BASE. In a fresh west workspace, a plain `cmake -S doc -B build` staged nothing, and Sphinx failed with "unable to load the master document" (found in the safety-toolbox standalone workspace). Add ZEPHYR_BASE=${ZEPHYR_BASE} to SPHINX_ENV. Being explicit also overrides a stale shell value that points at a different Zephyr. Acceptance suite without ZEPHYR_BASE in the shell (Zephyr found via CMAKE_PREFIX_PATH): 276/276. Before this change: test_05 errors. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tobias Kaestner --- cmake/sphinx.cmake | 10 ++++++++++ sphinx/zdocs_conf.py | 5 +++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cmake/sphinx.cmake b/cmake/sphinx.cmake index 059ab50..dd80002 100644 --- a/cmake/sphinx.cmake +++ b/cmake/sphinx.cmake @@ -225,6 +225,16 @@ function(add_sphinx_target doc_name) if(NOT ZDOCS_TWISTER_OUT STREQUAL "") list(APPEND SPHINX_ENV ZDOCS_TWISTER_OUT=${ZDOCS_TWISTER_OUT}) endif() + # The Zephyr that find_package(Zephyr) found. zdocs_conf loads Zephyr's doc + # extensions from it, including external_content, which stages the sources + # into ${DOCS_SRC_DIR}. find_package sets only the CMake variable, never the + # environment, so without this the build depended on the user's shell + # exporting ZEPHYR_BASE. A shell without it staged nothing, and Sphinx failed + # with "unable to load the master document". Set explicitly, it also + # overrides a stale shell value that points at a different Zephyr. + if(NOT ZEPHYR_BASE STREQUAL "") + list(APPEND SPHINX_ENV ZEPHYR_BASE=${ZEPHYR_BASE}) + endif() if(ARGS_REGISTRY) set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${ARGS_REGISTRY}) diff --git a/sphinx/zdocs_conf.py b/sphinx/zdocs_conf.py index bbe9e6d..90be3be 100644 --- a/sphinx/zdocs_conf.py +++ b/sphinx/zdocs_conf.py @@ -37,8 +37,9 @@ ZDOCS_BASE = ZDOCS_DOC_DIR.parent # zdocs' own extensions and scripts, plus Zephyr's doc extensions (external_content -# assembles the Sphinx source tree). ZEPHYR_BASE is exported by the Zephyr build -# that ran find_package(Zephyr). +# assembles the Sphinx source tree). cmake/sphinx.cmake puts ZEPHYR_BASE into +# the environment of every Sphinx run, from the Zephyr that find_package(Zephyr) +# found. find_package itself sets only the CMake variable. sys.path.insert(0, str(ZDOCS_DOC_DIR / "_extensions")) sys.path.insert(0, str(ZDOCS_BASE / "scripts")) _zephyr_base = os.environ.get("ZEPHYR_BASE")