From 7ed80e7cb9ceb31467568481c6225b73aa6d586c Mon Sep 17 00:00:00 2001 From: Sebastian Hellgren Date: Tue, 15 Sep 2026 12:53:58 +0200 Subject: [PATCH 01/11] Add llms.txt, AGENTS.md and context7.json for AI agents The documentation is also built with the Sphinx text builder into _text/. Unlike the reStructuredText sources, the text build contains what the pages include or generate: the VHDL package headers, the command line reference and the Python API. llms.txt indexes the text pages and llms-full.txt concatenates all of them except the optional ones (blog, release notes, ...), both following the llms.txt convention. AGENTS.md tells coding agents how to work in the repository, and context7.json tells Context7 which parts of the repository to index and how VUnit is used. The docs job gets GHDL, used by the API reference generator added in the next commit, and VUNIT_DOCS_SKIP_API=1 builds the documentation without it. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/docs.yml | 7 ++ AGENTS.md | 48 ++++++++++ context7.json | 22 ++++- pyproject.toml | 5 +- tests/unit/test_llms_txt.py | 127 +++++++++++++++++++++++++ tools/build_docs.py | 30 +++--- tools/llms_txt.py | 184 ++++++++++++++++++++++++++++++++++++ 7 files changed, 408 insertions(+), 15 deletions(-) create mode 100644 AGENTS.md create mode 100644 tests/unit/test_llms_txt.py create mode 100644 tools/llms_txt.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b96dca35b..8d277921a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -24,6 +24,13 @@ jobs: with: python-version: '3.13' + # GHDL generates the machine-readable VHDL API reference (tools/build_api_reference.py) + - name: '⚙️ Setup GHDL' + uses: ghdl/setup-ghdl@main + with: + version: v6.0.0 + backend: mcode + - name: Install dependencies run: | pip install -U pip --progress-bar off diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..de1099aee --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,48 @@ +# AGENTS.md + +Guidance for AI coding agents working on the VUnit repository. + +To *use* VUnit rather than develop it, read the documentation written for agents instead: + +- [llms.txt](https://vunit.github.io/llms.txt): index of the documentation as plain text +- [llms-full.txt](https://vunit.github.io/llms-full.txt): all documentation, including the VHDL package headers +- [api/vunit_lib.json](https://vunit.github.io/api/vunit_lib.json) and [api/python.json](https://vunit.github.io/api/python.json): machine-readable VHDL and Python APIs + +## Repository layout + +- `vunit/`: the Python package, with the run script API in `vunit/ui/`, the simulator interfaces in `vunit/sim_if/` and the test runner in `vunit/test/`. +- `vunit/vhdl/`: the VHDL libraries compiled into `vunit_lib` (`run`, `check`, `logging`, `com`, `data_types`, `verification_components`, ...). `vunit/vhdl/osvvm` is a third-party git submodule. +- `vunit/verilog/`: the SystemVerilog support. +- `tests/unit/`, `tests/acceptance/`, `tests/lint/`: see "Testing" in `docs/contributing.rst`. +- `examples/`, `docs/` (Sphinx) and `tools/` (documentation and release scripts). + +## Commands + +- **Unit tests:** `pytest tests/unit/`, or `tox -e py313-unit`. +- **Lint** (pycodestyle, pylint, mypy, license headers): `tox -e py313-lint`. +- **Formatting:** black, with the line length from `pyproject.toml`. Check only the files you changed, e.g. `python -m black --check `. `tox -e py313-fmt` formats every file in the repository. +- **Acceptance tests with a simulator:** `tox -e py313-acceptance-`, where `` is `activehdl`, `ghdl`, `modelsim`, `nvc` or `rivierapro`. +- **Verification components:** `tox -e py313-vcomponents-`. +- **Documentation:** `tox -e py313-docs`. + - The build needs the git tags for the release notes. + - It needs GHDL for the API reference. Set `VUNIT_DOCS_SKIP_API=1` to build without it. +- **All tox environments:** `tox -l`. + +## Rules + +Details are in `docs/contributing.rst`. + +- **News fragments:** a change visible to users needs a news fragment `docs/news.d/..rst`. `` is one of `breaking`, `bugfix`, `doc`, `deprecation`, `feature` or `misc`. +- **License header:** Python, VHDL and Verilog source files start with the Mozilla Public License 2.0 header with the copyright assigned to Lars Asplund. Copy it from a file next to the new one. `tests/lint/test_license.py` checks it. +- **Tests:** new functionality comes with tests. + +## Generated files + +Change the generator and run it again instead of editing its output by hand: + +- `vunit/vhdl/check/tools/generate_check_equal.py`, `generate_check_equal_2008p.py` and `generate_check_match.py` generate parts of `vunit/vhdl/check/src/check_api*.vhd` and `check*.vhd`, and their testbenches. +- `vunit/vhdl/data_types/tools/generate_dict.py` generates the typed `set_*`/`get_*` subprograms and their aliases in `dict_pkg`. + +## Pitfalls + +- Running a `run.py` inside the repository creates a `vunit_out/` directory next to it. Do not commit it. diff --git a/context7.json b/context7.json index 809cfe6b2..56487fdcd 100644 --- a/context7.json +++ b/context7.json @@ -1,4 +1,24 @@ { + "$schema": "https://context7.com/schema/context7.json", "url": "https://context7.com/vunit/vunit", - "public_key": "pk_lFI1bFQ8pHgYDeBRtvnEp" + "public_key": "pk_lFI1bFQ8pHgYDeBRtvnEp", + "projectTitle": "VUnit", + "description": "Unit testing framework for VHDL and SystemVerilog: a Python test runner (run.py) and VHDL libraries for checks, logging, message passing, data types and verification components.", + "folders": [ + "docs", + "examples" + ], + "excludeFolders": [ + "docs/blog", + "docs/release_notes", + "docs/news.d" + ], + "excludeFiles": [], + "rules": [ + "A run script creates the project with VUnit.from_argv(), calls add_vhdl_builtins(), adds libraries and source files, and ends with main().", + "A VHDL testbench has a runner_cfg generic and runs its test cases, each selected with run(\"\"), between test_runner_setup and test_runner_cleanup.", + "Complete VHDL API signatures: https://vunit.github.io/api/vunit_lib.json", + "Complete Python API signatures: https://vunit.github.io/api/python.json", + "Complete rendered documentation including the VHDL package headers: https://vunit.github.io/llms-full.txt" + ] } diff --git a/pyproject.toml b/pyproject.toml index d135dd713..aeb7430bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,7 +103,9 @@ isolated_build = True [testenv] recreate=True -passenv=ALDEC_LICENSE_FILE +passenv= + ALDEC_LICENSE_FILE + VUNIT_DOCS_SKIP_API deps= fmt: black @@ -137,6 +139,7 @@ commands= unit: {envpython} -m pytest -v -ra tests/unit {posargs} lint: {envpython} -m pytest -v -ra tests/lint {posargs} docs: {envpython} tools/build_docs.py {envtmpdir}/docsbuild {posargs} + docs: {envpython} tools/build_api_reference.py {envtmpdir}/docsbuild acceptance: {envpython} -m pytest -v -ra tests/acceptance {posargs} vcomponents: {envpython} vunit/vhdl/verification_components/run.py --clean coverage: {envpython} -m coverage run --branch --source vunit/ -m pytest tests/ diff --git a/tests/unit/test_llms_txt.py b/tests/unit/test_llms_txt.py new file mode 100644 index 000000000..de1cf977e --- /dev/null +++ b/tests/unit/test_llms_txt.py @@ -0,0 +1,127 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2014-2026, Lars Asplund lars.anders.asplund@gmail.com + +""" +Test the generation of llms.txt and llms-full.txt from the text build of the documentation +""" + +import unittest +from pathlib import Path +from tests.common import with_tempdir +from tools.llms_txt import read_pages, render_llms_full_txt, render_llms_txt, section_of, write_llms_files + +PAGES = { + "index": """\ +VUnit: a test framework for HDL +******************************* + +[image: shieldPyPI][image] + +VUnit is an open source +unit testing framework. + +[image] +""", + "installing": "Installing\n**********\n\nUse pip.\n", + "run/user_guide": "Run Library User Guide\n**********************\n\nTest cases.\n", + "data_types/queue": "*queue* package\n***************\n\n package queue_pkg is\n", + "blog/2023_03_31_vunit_events": "VUnit Events\n************\n\nNews.\n", + "genindex": "Index\n*****\n", + "new_topic/page": "New Topic\n*********\n\nNew.\n", +} + + +def create_text_build(output_path, pages=None): + """ + Create a text build with the given pages under output_path + """ + for docname, text in (PAGES if pages is None else pages).items(): + path = Path(output_path) / "_text" / f"{docname}.txt" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +class TestLlmsTxt(unittest.TestCase): + """ + Test the generation of llms.txt and llms-full.txt + """ + + def test_section_of(self): + self.assertEqual(section_of("index"), "Getting started") + self.assertEqual(section_of("run/user_guide"), "User guides") + self.assertEqual(section_of("data_types/user_guide"), "User guides") + self.assertEqual(section_of("data_types/queue"), "API reference") + self.assertEqual(section_of("blog/2023_03_31_vunit_events"), "Optional") + self.assertEqual(section_of("new_topic/page"), "Other") + self.assertIsNone(section_of("genindex")) + + @with_tempdir + def test_llms_txt(self, tempdir): + create_text_build(tempdir) + self.assertEqual( + render_llms_txt(read_pages(Path(tempdir) / "_text")), + """\ +# VUnit + +> VUnit is an open source unit testing framework. + +## Getting started + +- [VUnit: a test framework for HDL](https://vunit.github.io/_text/index.txt): https://vunit.github.io/index.html +- [Installing](https://vunit.github.io/_text/installing.txt): https://vunit.github.io/installing.html + +## User guides + +- [Run Library User Guide](https://vunit.github.io/_text/run/user_guide.txt): https://vunit.github.io/run/user_guide.html + +## API reference + +- [VHDL API (JSON)](https://vunit.github.io/api/vunit_lib.json) +- [Python API (JSON)](https://vunit.github.io/api/python.json) +- [*queue* package](https://vunit.github.io/_text/data_types/queue.txt): https://vunit.github.io/data_types/queue.html + +## Other + +- [New Topic](https://vunit.github.io/_text/new_topic/page.txt): https://vunit.github.io/new_topic/page.html + +## Optional + +- [VUnit Events](https://vunit.github.io/_text/blog/2023_03_31_vunit_events.txt): https://vunit.github.io/blog/2023_03_31_vunit_events.html +""", + ) + + @with_tempdir + def test_llms_txt_without_api(self, tempdir): + create_text_build(tempdir) + llms_txt = render_llms_txt(read_pages(Path(tempdir) / "_text"), api=False) + self.assertNotIn("api/", llms_txt) + self.assertIn("## API reference\n\n- [*queue* package]", llms_txt) + + @with_tempdir + def test_llms_full_txt(self, tempdir): + create_text_build(tempdir) + llms_full_txt = render_llms_full_txt(read_pages(Path(tempdir) / "_text")) + self.assertIn( + "# Installing\n\nSource: https://vunit.github.io/installing.html\n\nInstalling\n**********\n\nUse pip.\n", + llms_full_txt, + ) + self.assertIn("# New Topic\n", llms_full_txt) + self.assertNotIn("VUnit Events", llms_full_txt) + self.assertNotIn("Index", llms_full_txt) + self.assertLess(llms_full_txt.index("# Installing"), llms_full_txt.index("# *queue* package")) + + @with_tempdir + def test_write_llms_files(self, tempdir): + create_text_build(tempdir) + write_llms_files(tempdir) + self.assertTrue((Path(tempdir) / "llms.txt").read_text(encoding="utf-8").startswith("# VUnit\n")) + self.assertIn("# Installing", (Path(tempdir) / "llms-full.txt").read_text(encoding="utf-8")) + + @with_tempdir + def test_missing_front_page_is_an_error(self, tempdir): + create_text_build(tempdir, {"installing": PAGES["installing"]}) + with self.assertRaisesRegex(RuntimeError, "no front page"): + render_llms_txt(read_pages(Path(tempdir) / "_text")) diff --git a/tools/build_docs.py b/tools/build_docs.py index 3c64f4af2..14b5785a2 100644 --- a/tools/build_docs.py +++ b/tools/build_docs.py @@ -10,30 +10,34 @@ Command line utility to build documentation/website """ +from os import environ from subprocess import check_call from pathlib import Path from sys import argv, executable +from tempfile import TemporaryDirectory +from llms_txt import TEXT_DIR_NAME, write_llms_files ROOT = Path(__file__).parent.parent +def sphinx(builder, output_path, args): + """ + Build the documentation with a Sphinx builder + """ + check_call([executable, "-m", "sphinx"] + args + ["-TEWanb", builder, ROOT / "docs", output_path]) + + def main(): """ - Build documentation/website + Build documentation/website, including a text version and llms.txt/llms-full.txt for AI agents """ - check_call( - [ - executable, - "-m", - "sphinx" - ] + ([] if len(argv) < 2 else argv[2:]) + [ - "-TEWanb", - "html", - Path(__file__).parent.parent / "docs", - argv[1], - ] - ) + output_path = Path(argv[1]) + args = argv[2:] + sphinx("html", output_path, args) + with TemporaryDirectory() as doctrees: + sphinx("text", output_path / TEXT_DIR_NAME, args + ["-d", doctrees]) + write_llms_files(output_path, api=environ.get("VUNIT_DOCS_SKIP_API") != "1") if __name__ == "__main__": diff --git a/tools/llms_txt.py b/tools/llms_txt.py new file mode 100644 index 000000000..6691237d1 --- /dev/null +++ b/tools/llms_txt.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2014-2026, Lars Asplund lars.anders.asplund@gmail.com + +""" +Generate llms.txt and llms-full.txt from the plain text build of the documentation. + +The text build (sphinx -b text) renders every page completely, including the VHDL +package headers the pages include and the generated Python and command line +references, which is what makes it suitable for AI agents. +""" + +from dataclasses import dataclass +from pathlib import Path + +BASE_URL = "https://vunit.github.io" +TEXT_DIR_NAME = "_text" +API_SECTION = "API reference" +OTHER_SECTION = "Other" +OPTIONAL_SECTION = "Optional" + +# The sections of llms.txt in order. A page belongs to the first section with a +# matching prefix, which is either a document name or a directory ending with "/". +# Pages matching no prefix are listed under OTHER_SECTION such that a new page never +# breaks the build. Pages under OPTIONAL_SECTION are left out of llms-full.txt. +SECTIONS = ( + ("Getting started", ("index", "installing", "user_guide", "cli", "examples", "hdl_libraries")), + ( + "User guides", + ( + "run/", + "check/user_guide", + "logging/", + "com/", + "data_types/user_guide", + "data_types/event_user_guide", + "id/", + "verification_components/", + "ci/", + ), + ), + (API_SECTION, ("py/", "check/", "data_types/")), + (OTHER_SECTION, ()), + (OPTIONAL_SECTION, ("blog/", "release_notes", "testimonials/", "about", "contributing", "license")), +) +EXCLUDED = ("genindex",) +API_FILES = ( + ("VHDL API (JSON)", "api/vunit_lib.json"), + ("Python API (JSON)", "api/python.json"), +) + + +@dataclass +class Page: + """ + A page of the text build + """ + + docname: str + title: str + text: str + + @property + def text_url(self): + """ + URL of the text version of the page + """ + return f"{BASE_URL}/{TEXT_DIR_NAME}/{self.docname}.txt" + + @property + def html_url(self): + """ + URL of the HTML version of the page + """ + return f"{BASE_URL}/{self.docname}.html" + + +def _matches(docname, prefix): + return docname == prefix or (prefix.endswith("/") and docname.startswith(prefix)) + + +def _placement(docname): + """ + The section index and the prefix index of a page, None for a page that is left out + """ + if docname in EXCLUDED: + return None + other_index = None + for section_index, (title, prefixes) in enumerate(SECTIONS): + for prefix_index, prefix in enumerate(prefixes): + if _matches(docname, prefix): + return section_index, prefix_index + if title == OTHER_SECTION: + other_index = section_index + return other_index, 0 + + +def section_of(docname): + """ + The llms.txt section of a page, None for a page that is left out + """ + placement = _placement(docname) + return None if placement is None else SECTIONS[placement[0]][0] + + +def read_pages(text_dir): + """ + The pages of a text build in llms.txt order, without the pages that are left out + """ + text_dir = Path(text_dir) + pages = [] + for path in text_dir.rglob("*.txt"): + relative = path.relative_to(text_dir) + if any(part.startswith(".") for part in relative.parts): + continue + docname = relative.with_suffix("").as_posix() + if _placement(docname) is None: + continue + text = path.read_text(encoding="utf-8") + lines = text.splitlines() + pages.append(Page(docname, lines[0].strip() if lines else docname, text)) + + return sorted(pages, key=lambda page: (_placement(page.docname), page.docname)) + + +def _summary(pages): + """ + The first paragraph of the front page that is not an image + """ + for page in pages: + if page.docname == "index": + paragraphs = page.text.split("\n\n")[1:] + for paragraph in paragraphs: + lines = [line.strip() for line in paragraph.strip().splitlines()] + if lines and not lines[0].startswith("[image"): + return " ".join(lines) + raise RuntimeError("The text build has no front page (index.txt) with a summary paragraph") + + +def render_llms_txt(pages, api=True): + """ + The content of llms.txt + + :param api: Include the links to the API reference JSON files + """ + lines = ["# VUnit", "", f"> {_summary(pages)}", ""] + for title, _ in SECTIONS: + entries = [] + if title == API_SECTION and api: + entries += [f"- [{name}]({BASE_URL}/{path})" for name, path in API_FILES] + entries += [ + f"- [{page.title}]({page.text_url}): {page.html_url}" for page in pages if section_of(page.docname) == title + ] + if entries: + lines += [f"## {title}", "", *entries, ""] + + return "\n".join(lines) + + +def render_llms_full_txt(pages): + """ + The content of llms-full.txt: all pages except the optional ones + """ + return "\n".join( + f"# {page.title}\n\nSource: {page.html_url}\n\n{page.text.rstrip()}\n" + for page in pages + if section_of(page.docname) != OPTIONAL_SECTION + ) + + +def write_llms_files(output_path, api=True): + """ + Write llms.txt and llms-full.txt to a documentation build with a text build in TEXT_DIR_NAME + + :param api: Include the links to the API reference JSON files + """ + output_path = Path(output_path) + pages = read_pages(output_path / TEXT_DIR_NAME) + (output_path / "llms.txt").write_text(render_llms_txt(pages, api), encoding="utf-8") + (output_path / "llms-full.txt").write_text(render_llms_full_txt(pages), encoding="utf-8") From 4555828e96a61ac244b9f1bd795568be33556ac7 Mon Sep 17 00:00:00 2001 From: Sebastian Hellgren Date: Tue, 15 Sep 2026 13:31:27 +0200 Subject: [PATCH 02/11] Generate machine-readable API reference files with GHDL Add tools/build_api_reference.py, which compiles vunit_lib and osvvm with GHDL, runs "ghdl --file-to-xml" per package file, and converts the result into api/vunit_lib.json (VHDL symbol tables: types, subtypes, constants, objects, subprograms, aliases, with doc comments and source-text slices for types/defaults) and api/python.json (the Python API of every autoclass/ autofunction/automodule target under docs/). VUNIT_DOCS_SKIP_API=1 skips the build with a warning and exit 0. Add tests/unit/test_api_reference.py, covering the XML-to-dict conversion, source-text slicing (including GHDL's tab-stop-8 column counting and its folded-literal position quirk for attribute defaults), doc-comment extraction, the VHDL package scope filter, the completeness check, Python API collection, and every converter error in the design's error-handling table -- all with inline XML/source fixtures, no GHDL or Sphinx needed. Co-Authored-By: Claude Opus 5 (1M context) --- tests/unit/test_api_reference.py | 903 ++++++++++++++++++++++++++++++ tools/build_api_reference.py | 915 +++++++++++++++++++++++++++++++ 2 files changed, 1818 insertions(+) create mode 100644 tests/unit/test_api_reference.py create mode 100644 tools/build_api_reference.py diff --git a/tests/unit/test_api_reference.py b/tests/unit/test_api_reference.py new file mode 100644 index 000000000..19db62caf --- /dev/null +++ b/tests/unit/test_api_reference.py @@ -0,0 +1,903 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2014-2026, Lars Asplund lars.anders.asplund@gmail.com + +""" +Test of the machine-readable VHDL and Python API reference generator. + +Uses small, inline GHDL XML documents together with matching VHDL source +text (no GHDL invoked) and a small Python module defined in this file (no +Sphinx build). Columns are computed from the same source-text literals with +str.index() rather than hand-counted, so a typo in the fixture cannot make +a test pass for the wrong reason. +""" + +import sys +import unittest +from pathlib import Path +from xml.etree import ElementTree + +from tools import build_api_reference as bar + + +def _col(line_text, needle): + """1-based column of the first occurrence of needle in line_text.""" + return line_text.index(needle) + 1 + + +class TestSourceTextSlicing(unittest.TestCase): + """ + SourceText.slice_from: delimiters, bracket depth, string and character + literals, and the "no closing delimiter" error. + """ + + def test_slices_up_to_semicolon(self): + source = bar.SourceText(" integer;\n") + self.assertEqual(source.slice_from(1, 3, "f.vhd", 1, allow_assign=True), "integer") + + def test_stops_at_assign_for_types(self): + source = bar.SourceText("positive := 1;\n") + self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=True), "positive") + + def test_does_not_stop_at_assign_for_values(self): + # A default value never itself contains ":=", but allow_assign=False + # must not treat ":" specially either. + source = bar.SourceText('a & ":" & b;\n') + self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=False), 'a & ":" & b') + + def test_balances_parentheses_before_stopping_at_semicolon(self): + source = bar.SourceText("array (natural range <>) of rec_t;\n") + self.assertEqual( + source.slice_from(1, 1, "f.vhd", 1, allow_assign=True), + "array (natural range <>) of rec_t", + ) + + def test_stray_close_paren_ends_a_parameter_slice(self): + # As found inside "(name : positive := 1)" -- the type/default slice + # for the last parameter in a list must stop at the enclosing ")". + source = bar.SourceText("positive := 1)\n") + self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=False), "positive := 1") + + def test_collapses_whitespace_to_single_spaces(self): + source = bar.SourceText("integer\n range\n 0 to 15;\n") + self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=True), "integer range 0 to 15") + + def test_comments_neither_delimit_nor_belong_to_the_slice(self): + source = bar.SourceText("natural -- width; (in bits)\n range 0 to 8 := 8;\n") + self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=True), "natural range 0 to 8") + # A trailing comment on the last parameter of a list, before the closing parenthesis + source = bar.SourceText("8 -- default width )\n );\n") + self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=False), "8") + + def test_string_literal_semicolon_is_not_a_delimiter(self): + source = bar.SourceText('string := "a;b";\n') + self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=True), "string") + # And the value itself, starting after ":=", keeps the semicolon. + self.assertEqual(bar.SourceText('"a;b";\n').slice_from(1, 1, "f.vhd", 1, allow_assign=False), '"a;b"') + + def test_escaped_quote_inside_string_literal(self): + source = bar.SourceText('"a""b";\n') + self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=False), '"a""b"') + + def test_character_literal_semicolon_is_not_a_delimiter(self): + # A ';' character literal used as a default value: the semicolon + # inside the literal must not end the slice. + source = bar.SourceText("character := ';';\n") + self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=False), "character := ';'") + + def test_attribute_tick_is_not_mistaken_for_a_character_literal(self): + source = bar.SourceText("integer range -1 to integer'high;\n") + self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=True), "integer range -1 to integer'high") + + def test_no_delimiter_raises(self): + source = bar.SourceText("integer\n") + with self.assertRaises(bar.ApiReferenceError) as ctx: + source.slice_from(1, 1, "f.vhd", 7, allow_assign=True) + self.assertIn("f.vhd:7", str(ctx.exception)) + + def test_tab_stop_column_mapping(self): + # A real gotcha: GHDL reports columns with tab stops of 8, like a + # terminal, not one column per character. + source = bar.SourceText('\tname : string := "";\n') + # Column 9 is where "name" starts after one leading tab; slicing + # from there (stopping at ":=") proves the start position is right. + self.assertEqual(source.slice_from(1, 9, "f.vhd", 1, allow_assign=True), "name : string") + + +class TestDocComments(unittest.TestCase): + """ + The contiguous "--" comment block directly above a declaration, doc + comment termination on a blank line, and the license-header exclusion + for a package's own doc. + """ + + def test_extracts_comment_block_directly_above(self): + text = " -- Returns the length\n -- of the queue\n function length return natural;\n" + source = bar.SourceText(text) + self.assertEqual(bar.declaration_doc(source, 3), "Returns the length\nof the queue") + + def test_blank_line_ends_the_block(self): + text = " -- Unrelated comment\n\n function length return natural;\n" + source = bar.SourceText(text) + self.assertIsNone(bar.declaration_doc(source, 3)) + + def test_no_comment_above_gives_none(self): + text = " use work.foo_pkg.all;\n function length return natural;\n" + source = bar.SourceText(text) + self.assertIsNone(bar.declaration_doc(source, 2)) + + def test_license_header_excluded_from_package_doc(self): + text = ( + "-- This Source Code Form is subject to the terms of the Mozilla Public\n" + "-- License, v. 2.0.\n" + "package my_pkg is\n" + ) + source = bar.SourceText(text) + self.assertIsNone(bar.package_doc(source, 3)) + + def test_non_license_comment_kept_for_package_doc(self): + text = "-- A package for testing\npackage my_pkg is\n" + source = bar.SourceText(text) + self.assertEqual(bar.package_doc(source, 2), "A package for testing") + + def test_impure_on_a_preceding_line_shifts_the_doc_anchor(self): + text = " -- Returns something\n impure\n function get return integer;\n" + source = bar.SourceText(text) + self.assertEqual(bar.subprogram_doc(source, 3), "Returns something") + + def test_impure_on_the_same_line_does_not_pull_in_the_comment_above(self): + # Here the comment belongs to something else entirely -- decl_line + # already holds "impure function", so the search starts one line + # above *that*, which is blank. + text = " -- Unrelated\n\n impure function get return integer;\n" + source = bar.SourceText(text) + self.assertIsNone(bar.subprogram_doc(source, 3)) + + +class TestConvertSubprogram(unittest.TestCase): + """ + procedure/function, pure/impure, operators, parameter classes/modes/ + defaults, constrained parameter types, return types. + """ + + def _parse(self, xml): + return ElementTree.fromstring(xml) + + def test_procedure_has_no_pure_flag(self): + line = " procedure proc_set (target : in rec_t);" + xml = f""" + + + + + + + + """ + el = self._parse(xml) + source = bar.SourceText(line + "\n") + entry = bar.convert_subprogram(el, source, "f.vhd") + self.assertEqual(entry["kind"], "procedure") + self.assertIsNone(entry["pure"]) + self.assertEqual(entry["parameters"][0]["type"], "rec_t") + self.assertEqual(entry["parameters"][0]["class"], "constant") + self.assertEqual(entry["parameters"][0]["mode"], "in") + + def test_impure_function_pure_flag_false(self): + line = " impure function func_get (target : rec_t) return integer;" + xml = f""" + + + + + """ + el = self._parse(xml) + source = bar.SourceText(line + "\n") + entry = bar.convert_subprogram(el, source, "f.vhd") + self.assertEqual(entry["kind"], "function") + self.assertFalse(entry["pure"]) + self.assertEqual(entry["return_type"], "integer") + + def test_pure_function_pure_flag_true(self): + line = " function pure_fn (x : integer) return integer;" + xml = f""" + + + + + """ + el = self._parse(xml) + source = bar.SourceText(line + "\n") + entry = bar.convert_subprogram(el, source, "f.vhd") + self.assertTrue(entry["pure"]) + + def test_operator_identifier_is_flagged(self): + line = ' function "+" (l, r : rec_t) return rec_t;' + xml = f""" + + + + + """ + el = self._parse(xml) + source = bar.SourceText(line + "\n") + entry = bar.convert_subprogram(el, source, "f.vhd") + self.assertTrue(entry["operator"]) + self.assertEqual(entry["name"], "+") + + def test_plain_identifier_is_not_an_operator(self): + line = " procedure proc_set (target : in rec_t);" + xml = f""" + + + + """ + el = self._parse(xml) + source = bar.SourceText(line + "\n") + entry = bar.convert_subprogram(el, source, "f.vhd") + self.assertFalse(entry["operator"]) + + def test_parameter_default_value(self): + line = " value : in integer := 0" + xml = f""" + + + + + + + + + """ + el = self._parse(xml) + source = bar.SourceText(line + ";\n") + entry = bar.convert_subprogram(el, source, "f.vhd") + self.assertEqual(entry["parameters"][0]["default"], "0") + + def test_signal_and_variable_parameter_classes(self): + line = " signal net : inout t; variable pass : out boolean" + xml = f""" + + + + + + + + + + + """ + el = self._parse(xml) + source = bar.SourceText(line + ";\n") + entry = bar.convert_subprogram(el, source, "f.vhd") + self.assertEqual(entry["parameters"][0]["class"], "signal") + self.assertEqual(entry["parameters"][0]["mode"], "inout") + self.assertEqual(entry["parameters"][1]["class"], "variable") + + def test_constrained_parameter_type(self): + line = " data : std_logic_vector(7 downto 0)" + xml = f""" + + + + + + + + """ + el = self._parse(xml) + source = bar.SourceText(line + ";\n") + entry = bar.convert_subprogram(el, source, "f.vhd") + self.assertEqual(entry["parameters"][0]["type"], "std_logic_vector(7 downto 0)") + + def test_implicit_operator_parameter_without_subtype_indication(self): + # GHDL synthesizes "=" for every type; its parameters carry a type + # reference but no textual subtype_indication (nothing was written). + xml = """ + + + + + + + + + """ + el = self._parse(xml) + source = bar.SourceText("\n") + entry = bar.convert_subprogram(el, source, "f.vhd") + self.assertTrue(entry["operator"]) + self.assertIsNone(entry["parameters"][0]["type"]) + self.assertEqual(entry["parameters"][0]["name"], "") + + def test_unhandled_parameter_kind_raises(self): + xml = """ + + + + + + """ + el = self._parse(xml) + source = bar.SourceText("\n") + with self.assertRaises(bar.ApiReferenceError) as ctx: + bar.convert_subprogram(el, source, "f.vhd") + self.assertIn("interface_mystery_declaration", str(ctx.exception)) + + +class TestConvertAlias(unittest.TestCase): + """Aliases with signatures and return types, including object aliases.""" + + def test_subprogram_alias_with_signature_and_return_type(self): + xml = """ + + + + + + + + + + """ + el = ElementTree.fromstring(xml) + entry = bar.convert_alias(el) + self.assertEqual(entry["name"], "alias_get") + self.assertEqual(entry["target"], "func_get") + self.assertEqual(entry["signature"], ["rec_t"]) + self.assertEqual(entry["return_type"], "integer") + self.assertNotIn("doc", entry) + + def test_procedure_alias_has_no_return_type(self): + xml = """ + + + + + + + + + + """ + el = ElementTree.fromstring(xml) + entry = bar.convert_alias(el) + self.assertEqual(entry["signature"], ["rec_t", "integer"]) + self.assertIsNone(entry["return_type"]) + + def test_object_alias_has_no_signature(self): + # "alias null_byte_vector_ptr is null_string_ptr;" -- an + # object_alias_declaration, aliasing a constant, not a subprogram. + xml = """ + + + + """ + el = ElementTree.fromstring(xml) + entry = bar.convert_alias(el) + self.assertEqual(entry["target"], "null_ptr") + self.assertEqual(entry["signature"], []) + self.assertIsNone(entry["return_type"]) + + def test_dotted_selected_name_target(self): + xml = """ + + + + + + """ + el = ElementTree.fromstring(xml) + entry = bar.convert_alias(el) + self.assertEqual(entry["target"], "work.push") + + +class TestConvertType(unittest.TestCase): + """Records, enumerations, protected types and their methods, other types.""" + + def test_record_type(self): + line = " a : integer;" + xml = f""" + + + + + + + + + + """ + el = ElementTree.fromstring(xml) + source = bar.SourceText(line + "\n") + entry = bar.convert_type(el, source, "f.vhd") + self.assertEqual(entry["kind"], "record") + self.assertEqual(entry["elements"], [{"name": "a", "type": "integer"}]) + + def test_enumeration_type(self): + xml = """ + + + + + + + + + + """ + el = ElementTree.fromstring(xml) + entry = bar.convert_type(el, bar.SourceText("\n"), "f.vhd") + self.assertEqual(entry["kind"], "enumeration") + self.assertEqual(entry["literals"], ["red", "green", "blue"]) + + def test_protected_type_with_methods(self): + line = " procedure set (value : integer);" + xml = f""" + + + + + + + + + + + + + + """ + el = ElementTree.fromstring(xml) + source = bar.SourceText(line + "\n") + entry = bar.convert_type(el, source, "f.vhd") + self.assertEqual(entry["kind"], "protected") + self.assertEqual(len(entry["subprograms"]), 1) + method = entry["subprograms"][0] + self.assertEqual(method["name"], "set") + # Methods use the same subprogram format, but (like aliases) are + # not individually documented top-level entries. + self.assertNotIn("doc", method) + + def test_other_type_array(self): + line = " type vec_t is array (natural range <>) of rec_t;" + xml = f""" + + + + """ + el = ElementTree.fromstring(xml) + source = bar.SourceText(line + "\n") + entry = bar.convert_type(el, source, "f.vhd") + self.assertEqual(entry["kind"], "other") + self.assertEqual(entry["definition"], "array (natural range <>) of rec_t") + + def test_other_type_access(self): + line = " type ptr_t is access rec_t;" + xml = f""" + + + + """ + el = ElementTree.fromstring(xml) + source = bar.SourceText(line + "\n") + entry = bar.convert_type(el, source, "f.vhd") + self.assertEqual(entry["kind"], "other") + self.assertEqual(entry["definition"], "access rec_t") + + +class TestConvertSubtypeConstantObject(unittest.TestCase): + def test_subtype(self): + line = " subtype small_t is integer range 0 to 15;" + xml = f""" + + + + """ + el = ElementTree.fromstring(xml) + source = bar.SourceText(line + "\n") + entry = bar.convert_subtype(el, source, "f.vhd") + self.assertEqual(entry["definition"], "integer range 0 to 15") + + def test_constant_with_value(self): + line = " constant c_max : integer := 15;" + xml = f""" + + + + + """ + el = ElementTree.fromstring(xml) + source = bar.SourceText(line + "\n") + entry = bar.convert_constant(el, source, "f.vhd") + self.assertEqual(entry["type"], "integer") + self.assertEqual(entry["value"], "15") + + def test_deferred_constant_has_no_value(self): + line = " constant c_deferred : boolean;" + xml = f""" + + + + """ + el = ElementTree.fromstring(xml) + source = bar.SourceText(line + "\n") + entry = bar.convert_constant(el, source, "f.vhd") + self.assertIsNone(entry["value"]) + + def test_signal_object(self): + line = " signal s_enabled : std_logic := '1';" + xml = f""" + + + + """ + el = ElementTree.fromstring(xml) + source = bar.SourceText(line + "\n") + entry = bar.convert_object(el, source, "f.vhd") + self.assertEqual(entry["class"], "signal") + self.assertEqual(entry["type"], "std_logic") + + def test_shared_variable_object(self): + line = " shared variable v_shared : integer;" + xml = f""" + + + + """ + el = ElementTree.fromstring(xml) + source = bar.SourceText(line + "\n") + entry = bar.convert_object(el, source, "f.vhd") + self.assertEqual(entry["class"], "shared variable") + + def test_file_object(self): + line = " file f0 : text;" + xml = f""" + + + + """ + el = ElementTree.fromstring(xml) + source = bar.SourceText(line + "\n") + entry = bar.convert_object(el, source, "f.vhd") + self.assertEqual(entry["class"], "file") + + +class TestConvertPackage(unittest.TestCase): + """ + Scope: only declarations whose file equals the input file, the ignored + anonymous_type_declaration, the private-package flag, and the "unhandled + declaration kind" error. + """ + + def _package_xml(self, decl_chain_xml, name="my_pkg", line="7"): + return f""" + + + {decl_chain_xml} + + + """ + + def test_filters_declarations_by_file(self): + line8 = " constant local_c : integer := 1;" + xml = self._package_xml(f""" + + + + + + + """) + el = ElementTree.fromstring(xml) + source = bar.SourceText("\n" * 7 + line8 + "\n") + pkg = bar.convert_package(el, source, "/a/my_pkg.vhd", "my_pkg.vhd") + names = [c["name"] for c in pkg["constants"]] + self.assertEqual(names, ["local_c"]) + + def test_anonymous_type_declaration_is_ignored(self): + xml = self._package_xml(""" + + """) + el = ElementTree.fromstring(xml) + pkg = bar.convert_package(el, bar.SourceText("\n"), "/a/my_pkg.vhd", "my_pkg.vhd") + for bucket in ("types", "subtypes", "constants", "objects", "subprograms", "aliases"): + self.assertEqual(pkg[bucket], []) + + def test_unhandled_declaration_kind_raises(self): + xml = self._package_xml(""" + + """) + el = ElementTree.fromstring(xml) + with self.assertRaises(bar.ApiReferenceError) as ctx: + bar.convert_package(el, bar.SourceText("\n"), "/a/my_pkg.vhd", "my_pkg.vhd") + self.assertIn("component_declaration", str(ctx.exception)) + self.assertIn("/a/my_pkg.vhd:9", str(ctx.exception)) + + def test_private_package_flag(self): + xml = self._package_xml("", name="data_types_private_pkg") + el = ElementTree.fromstring(xml) + pkg = bar.convert_package(el, bar.SourceText("\n"), "/a/my_pkg.vhd", "my_pkg.vhd") + self.assertTrue(pkg["private"]) + + def test_non_private_package_flag(self): + xml = self._package_xml("", name="queue_pkg") + el = ElementTree.fromstring(xml) + pkg = bar.convert_package(el, bar.SourceText("\n"), "/a/my_pkg.vhd", "my_pkg.vhd") + self.assertFalse(pkg["private"]) + + def test_p_prefixed_names_not_flagged_private(self): + line8 = " constant p_internal : integer := 1;" + xml = self._package_xml(f""" + + + + """) + el = ElementTree.fromstring(xml) + source = bar.SourceText("\n" * 7 + line8 + "\n") + pkg = bar.convert_package(el, source, "/a/my_pkg.vhd", "my_pkg.vhd") + self.assertFalse(pkg["private"]) + self.assertEqual(pkg["constants"][0]["name"], "p_internal") + + +class TestPackageScope(unittest.TestCase): + """find_package_files: the vunit/vhdl/**/src/**/*.vhd scope filter.""" + + def _make_tree(self, tmp_path, files): + for relpath, content in files.items(): + path = tmp_path / relpath + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + def test_finds_package_declaration_files(self): + tmp_path = Path(self._tmp_dir()) + self._make_tree( + tmp_path, + { + "vunit/vhdl/data_types/src/queue_pkg.vhd": "package queue_pkg is\nend package;\n", + }, + ) + found = bar.find_package_files(tmp_path) + self.assertEqual([name for _, name in found], ["queue_pkg"]) + + def test_excludes_package_body(self): + tmp_path = Path(self._tmp_dir()) + self._make_tree( + tmp_path, + { + "vunit/vhdl/foo/src/foo_pkg.vhd": ( + "package foo_pkg is\nend package;\npackage body foo_pkg is\nend package body;\n" + ), + }, + ) + found = bar.find_package_files(tmp_path) + self.assertEqual([name for _, name in found], ["foo_pkg"]) + + def test_excludes_package_instantiation(self): + tmp_path = Path(self._tmp_dir()) + self._make_tree( + tmp_path, + { + "vunit/vhdl/foo/src/foo_pkg.vhd": ( + "package foo_generic_pkg is\nend package;\n" + "package foo_pkg is new work.foo_generic_pkg generic map (t => integer);\n" + ), + }, + ) + found = bar.find_package_files(tmp_path) + self.assertEqual([name for _, name in found], ["foo_generic_pkg"]) + + def test_excludes_osvvm(self): + tmp_path = Path(self._tmp_dir()) + self._make_tree( + tmp_path, + { + "vunit/vhdl/osvvm/src/SomePkg.vhd": "package SomePkg is\nend package;\n", + }, + ) + found = bar.find_package_files(tmp_path) + self.assertEqual(found, []) + + _tmp_dirs = [] + + def _tmp_dir(self): + import tempfile + + tmp = tempfile.mkdtemp(prefix="vunit-api-scope-") + self._tmp_dirs.append(tmp) + return tmp + + def tearDown(self): + import shutil + + for tmp in self._tmp_dirs: + shutil.rmtree(tmp, ignore_errors=True) + self._tmp_dirs = [] + + +class TestCompleteness(unittest.TestCase): + def test_passes_when_sets_match(self): + bar.check_completeness(["a", "b"], ["b", "a"]) + + def test_raises_listing_missing_and_extra(self): + with self.assertRaises(bar.ApiReferenceError) as ctx: + bar.check_completeness(["a", "c"], ["a", "b"]) + message = str(ctx.exception) + self.assertIn("'b'", message) + self.assertIn("'c'", message) + + +class TestErrorHandling(unittest.TestCase): + """The converter error conditions of design-spec section 6.""" + + def test_ghdl_not_on_path(self): + import shutil + from unittest import mock + + with mock.patch.object(shutil, "which", return_value=None): + with self.assertRaises(bar.ApiReferenceError) as ctx: + bar.check_ghdl_available() + self.assertEqual( + str(ctx.exception), + "GHDL is required to build the API reference; install GHDL or set VUNIT_DOCS_SKIP_API=1", + ) + + def test_file_to_xml_nonzero_exit(self): + import subprocess + from unittest import mock + + fake = subprocess.CompletedProcess( + args=["ghdl"], returncode=1, stdout="", stderr="somefile.vhd:3:1:error: bad syntax\n" + ) + with mock.patch.object(subprocess, "run", return_value=fake): + with self.assertRaises(bar.ApiReferenceError) as ctx: + bar.run_file_to_xml(Path("somefile.vhd"), Path("/libs/vunit_lib"), Path("/libs/osvvm")) + message = str(ctx.exception) + self.assertIn("somefile.vhd", message) + self.assertIn("error: bad syntax", message) + + def test_xml_root_version_mismatch(self): + with self.assertRaises(bar.ApiReferenceError) as ctx: + bar.parse_xml_root('') + self.assertIn("0.12", str(ctx.exception)) + self.assertIn("converter must be checked", str(ctx.exception)) + + def test_xml_root_correct_version_parses(self): + root = bar.parse_xml_root('') + self.assertEqual(root.tag, "root") + + def test_vunit_compile_failure_raises(self): + import types + from unittest import mock + + class FakeVUnitObj: + def add_vhdl_builtins(self): + pass + + def add_verification_components(self): + pass + + def add_random(self): + pass + + def add_osvvm(self): + pass + + def main(self): + raise SystemExit(1) + + class FakeVUnit: + @staticmethod + def from_argv(_args): + return FakeVUnitObj() + + fake_module = types.ModuleType("vunit") + fake_module.VUnit = FakeVUnit + with mock.patch.dict(sys.modules, {"vunit": fake_module}): + with self.assertRaises(bar.ApiReferenceError) as ctx: + bar.compile_vunit_libraries("/tmp/does-not-matter") + self.assertIn("exit code 1", str(ctx.exception)) + + +class SampleClass: + """A sample class for the Python API test.""" + + def method_one(self, value): + """Doubles value.""" + return value * 2 + + @property + def read_only(self): + """A read-only property.""" + return 1 + + def _private_method(self): + pass + + +def sample_function(a, b=1): + """Adds a and b.""" + return a + b + + +class TestPythonApi(unittest.TestCase): + """ + docs/**/*.rst directive scanning and object collection, including the + "no public objects found" error for an automodule with :members:. + """ + + def test_finds_autoclass_and_automodule_targets(self, tmp_path=None): + import tempfile + + tmp_path = Path(tempfile.mkdtemp(prefix="vunit-api-rst-")) + docs = tmp_path / "docs" + docs.mkdir() + (docs / "vunit.rst").write_text( + ".. autoclass:: vunit.ui.VUnit()\n\n.. autoclass:: vunit.ui.library.Library()\n" + " :exclude-members: package\n", + encoding="utf-8", + ) + (docs / "ui.rst").write_text(".. automodule:: vunit.vunit_cli\n", encoding="utf-8") + (docs / "opts.rst").write_text( + ".. automodule:: tests.unit.test_api_reference\n :members:\n", encoding="utf-8" + ) + targets = bar.find_python_targets(tmp_path) + as_dict = {(directive, dotted): has_members for directive, dotted, has_members in targets} + self.assertEqual(as_dict[("autoclass", "vunit.ui.VUnit")], False) + self.assertEqual(as_dict[("autoclass", "vunit.ui.library.Library")], False) + self.assertEqual(as_dict[("automodule", "vunit.vunit_cli")], False) + self.assertEqual(as_dict[("automodule", "tests.unit.test_api_reference")], True) + + def test_class_object_collects_public_members_defined_on_the_class(self): + obj = bar.build_class_object(f"{__name__}.SampleClass", SampleClass) + self.assertEqual(obj["kind"], "class") + members = {m["name"]: m for m in obj["members"]} + self.assertIn("method_one", members) + self.assertIn("read_only", members) + self.assertNotIn("_private_method", members) + self.assertEqual(members["read_only"]["kind"], "property") + self.assertEqual(members["method_one"]["kind"], "method") + self.assertIn("(self, value)", members["method_one"]["signature"]) + self.assertEqual(members["method_one"]["doc"], "Doubles value.") + + def test_function_object(self): + obj = bar.build_function_object(f"{__name__}.sample_function", sample_function) + self.assertEqual(obj["kind"], "function") + self.assertEqual(obj["doc"], "Adds a and b.") + self.assertIn("a, b=1", obj["signature"]) + + def test_module_object_without_members_option_has_no_members(self): + obj = bar.build_module_object(__name__, sys.modules[__name__], has_members=False) + self.assertEqual(obj["kind"], "module") + self.assertEqual(obj["members"], []) + + def test_module_object_with_members_option_collects_classes_and_functions(self): + obj = bar.build_module_object(__name__, sys.modules[__name__], has_members=True) + names = {m["name"] for m in obj["members"]} + self.assertIn("SampleClass", names) + self.assertIn("sample_function", names) + + def test_no_public_objects_found_raises(self): + import types + + empty_module = types.ModuleType("vunit_test_empty_module") + with self.assertRaises(bar.ApiReferenceError) as ctx: + bar.build_module_object("vunit_test_empty_module", empty_module, has_members=True) + self.assertIn("vunit_test_empty_module", str(ctx.exception)) + + def test_import_dotted_class(self): + obj = bar.import_dotted(f"{__name__}.SampleClass") + self.assertIs(obj, SampleClass) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/build_api_reference.py b/tools/build_api_reference.py new file mode 100644 index 000000000..0a03a3f41 --- /dev/null +++ b/tools/build_api_reference.py @@ -0,0 +1,915 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2014-2026, Lars Asplund lars.anders.asplund@gmail.com + +""" +Command line utility to build the machine-readable VHDL and Python API +reference: /api/vunit_lib.json and /api/python.json. + +Usage: python tools/build_api_reference.py + +Set VUNIT_DOCS_SKIP_API=1 to skip the build entirely (prints a warning, +writes nothing, exits 0). This is the supported way to build the rest of +the documentation without GHDL installed. +""" + +import importlib +import inspect +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path +from xml.etree import ElementTree + +ROOT = Path(__file__).parent.parent + +XML_ROOT_VERSION = "0.13" +VHDL_STANDARD = "2008" +LIBRARY_NAME = "vunit_lib" + + +class ApiReferenceError(Exception): + """ + Raised for every error condition of the API reference generator. + The message names what failed and where, per the design spec's + error-handling table. + """ + + +# --------------------------------------------------------------------------- +# VHDL package scope +# --------------------------------------------------------------------------- + +# "package is", not "package body ...", optionally followed by "new" +# (a package instantiation, which has no declarations of its own). +_PACKAGE_RE = re.compile(r"^[ \t]*package\s+(?!body\b)(\w+)\s+is\b\s*(new\b)?", re.MULTILINE) + + +def find_package_files(root): + """ + Return a sorted list of (absolute Path, package name) for every VHDL + package declaration file in scope: vunit/vhdl/**/src/**/*.vhd, + excluding vunit/vhdl/osvvm/**, that contains "package is" not + followed by "new". + """ + vhdl_root = root / "vunit" / "vhdl" + found = [] + for path in sorted(vhdl_root.glob("**/src/**/*.vhd")): + if "osvvm" in path.relative_to(vhdl_root).parts: + continue + text = path.read_text(encoding="utf-8") + for match in _PACKAGE_RE.finditer(text): + if match.group(2) is not None: + continue + found.append((path, match.group(1))) + return sorted(found, key=lambda item: item[1]) + + +# --------------------------------------------------------------------------- +# Compiling +# --------------------------------------------------------------------------- + + +def check_ghdl_available(): + """Raise ApiReferenceError if "ghdl" is not on PATH.""" + if shutil.which("ghdl") is None: + raise ApiReferenceError( + "GHDL is required to build the API reference; install GHDL or set VUNIT_DOCS_SKIP_API=1" + ) + + +def ghdl_version_line(): + """The first line of "ghdl --version", e.g. "GHDL 6.0.0 (...) [...]".""" + proc = subprocess.run(["ghdl", "--version"], capture_output=True, text=True, check=False) + return proc.stdout.splitlines()[0].strip() if proc.stdout else "" + + +def compile_vunit_libraries(output_path): + """ + Compile vunit_lib and osvvm with GHDL, using VUnit's own Python API, + into output_path. Returns the "/ghdl/libraries" directory. + """ + os.environ["VUNIT_SIMULATOR"] = "ghdl" + + from vunit import VUnit # pylint: disable=import-outside-toplevel + + vunit_obj = VUnit.from_argv(["--output-path", str(output_path), "--compile", "-q"]) + vunit_obj.add_vhdl_builtins() + vunit_obj.add_verification_components() + vunit_obj.add_random() + vunit_obj.add_osvvm() + + try: + vunit_obj.main() + except SystemExit as exc: + if exc.code not in (0, None): + raise ApiReferenceError( + f"VUnit compile failed with exit code {exc.code}; see the compile output above" + ) from exc + + return Path(output_path) / "ghdl" / "libraries" + + +# --------------------------------------------------------------------------- +# Per-file XML extraction +# --------------------------------------------------------------------------- + + +def run_file_to_xml(file_path, vunit_lib_dir, osvvm_dir): + """ + Run "ghdl --file-to-xml" on file_path and return stdout (the XML) as a + string, without ever writing it to disk. + """ + cmd = [ + "ghdl", + "--file-to-xml", + "--std=08", + "--work=vunit_lib", + f"--workdir={vunit_lib_dir}", + f"-P{osvvm_dir}", + f"-P{vunit_lib_dir}", + str(file_path), + ] + proc = subprocess.run(cmd, capture_output=True, text=True, check=False) + if proc.returncode != 0: + stderr_lines = proc.stderr.splitlines() if proc.stderr else [] + first_error = next((ln for ln in stderr_lines if "error:" in ln), None) + if first_error is None: + first_error = stderr_lines[0] if stderr_lines else "(no output)" + raise ApiReferenceError(f"ghdl --file-to-xml failed for {file_path}: {first_error}") + return proc.stdout + + +def parse_xml_root(xml_text): + """ + Parse the GHDL XML dump and check its root version. + """ + root = ElementTree.fromstring(xml_text) + version = root.get("version") + if version != XML_ROOT_VERSION: + raise ApiReferenceError( + f"XML root version is '{version}', expected '{XML_ROOT_VERSION}'; " + "the converter must be checked against the new GHDL XML format" + ) + return root + + +def find_package_element(xml_root, file_path, package_name): + """ + Find the "library_unit" element for the package declared in file_path, + or None if not found (the completeness check catches a missing package). + """ + file_str = str(file_path) + for element in xml_root.iter("library_unit"): + if ( + element.get("kind") == "package_declaration" + and element.get("file") == file_str + and element.get("identifier") == package_name + ): + return element + return None + + +# --------------------------------------------------------------------------- +# Source text: line/column resolution and delimiter-bounded slicing +# --------------------------------------------------------------------------- + + +class SourceText: + """ + A VHDL source file's text, with helpers to resolve GHDL's 1-based + line/col (columns use tab stops of 8, like a terminal) to an absolute + character offset, and to slice source text starting at such a position + up to the first unbracketed delimiter. + """ + + def __init__(self, text): + """text: the whole source file, as read from disk.""" + self.text = text + self.lines = text.split("\n") + offsets = [] + acc = 0 + for line in self.lines: + offsets.append(acc) + acc += len(line) + 1 + self._line_offsets = offsets + + def line_text(self, line): + """The 1-based line's raw text, without its terminator.""" + return self.lines[line - 1] + + @staticmethod + def _col_to_index(line_text, col): + """0-based character index in line_text for a 1-based GHDL column (tab stops of 8).""" + current_col = 1 + for idx, char in enumerate(line_text): + if current_col == col: + return idx + if char == "\t": + current_col = ((current_col - 1) // 8 + 1) * 8 + 1 + else: + current_col += 1 + return len(line_text) + + def offset(self, line, col): + """Absolute 0-based character offset in self.text for a 1-based (line, col).""" + return self._line_offsets[line - 1] + self._col_to_index(self.line_text(line), col) + + def slice_from(self, line, col, file_path, decl_line, allow_assign): + """ + Slice starting at (line, col) up to the first delimiter at bracket + depth 0 outside string and character literals: ":=" (only when + allow_assign), ";" or ")". Whitespace is collapsed to single spaces. + """ + text = self.text + depth = 0 + i = self.offset(line, col) + pieces = [] + piece_start = i + while i < len(text): + after = self._skip_literal(text, i) + if after is not None: + if text.startswith("--", i): + # A comment neither delimits nor belongs to the slice + pieces.append(text[piece_start:i]) + piece_start = after + i = after + continue + char = text[i] + if depth == 0 and (char in ";)" or (allow_assign and text.startswith(":=", i))): + pieces.append(text[piece_start:i]) + return re.sub(r"\s+", " ", " ".join(pieces)).strip() + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + i += 1 + raise ApiReferenceError(f"No closing delimiter found for slice at {file_path}:{decl_line}") + + @staticmethod + def _skip_literal(text, i): + """ + The index after the comment, string literal or character literal starting at + index i of text, or None when none starts there. + """ + if text.startswith("--", i): + newline = text.find("\n", i) + return len(text) if newline == -1 else newline + if text[i] == '"': + i += 1 + while i < len(text): + if text[i] == '"': + if text.startswith('"', i + 1): + # A doubled quote within the literal + i += 2 + continue + return i + 1 + i += 1 + return len(text) + if text[i] == "'" and i + 2 < len(text) and text[i + 2] == "'": + return i + 3 + return None + + +_LICENSE_MARKER = "This Source Code Form" + + +def _comment_block_above(source, first_line): + """ + The contiguous block of "--" lines directly above first_line, stripped + of "--" and one following space, joined with "\\n". None if there is no + such block. + """ + collected = [] + line_no = first_line - 1 + while 1 <= line_no <= len(source.lines): + stripped = source.line_text(line_no).strip() + if stripped.startswith("--"): + collected.append(stripped) + line_no -= 1 + else: + break + if not collected: + return None + collected.reverse() + out = [] + for comment in collected: + body = comment[2:] + if body.startswith(" "): + body = body[1:] + out.append(body) + return "\n".join(out) + + +def subprogram_first_line(source, decl_line): + """ + A subprogram's declaration starts on the line holding "function" or + "procedure", which is normally decl_line (GHDL's line/col point at the + identifier, on that same line) -- unless a preceding line holds only + "pure" or "impure", in which case that line is the true first line. + """ + prev = decl_line - 1 + if prev >= 1 and source.line_text(prev).strip() in ("pure", "impure"): + return prev + return decl_line + + +def subprogram_doc(source, decl_line): + """The doc comment for a subprogram declared at decl_line, or None.""" + return _comment_block_above(source, subprogram_first_line(source, decl_line)) + + +def declaration_doc(source, decl_line): + """The doc comment for any other declaration at decl_line, or None.""" + return _comment_block_above(source, decl_line) + + +def package_doc(source, decl_line): + """The package's own doc comment, or None (also None for the MPL license header).""" + doc = _comment_block_above(source, decl_line) + if doc is not None and _LICENSE_MARKER in doc: + return None + return doc + + +# --------------------------------------------------------------------------- +# XML element -> dict conversion +# --------------------------------------------------------------------------- + +_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + +_PARAMETER_CLASS = { + "interface_constant_declaration": "constant", + "interface_variable_declaration": "variable", + "interface_signal_declaration": "signal", + "interface_file_declaration": "file", +} + +_OBJECT_CLASS = { + "signal_declaration": "signal", + "variable_declaration": "shared variable", + "file_declaration": "file", +} + + +def _min_position(element, file_str): + """ + The earliest (line, col) among element and all its descendants that + carry their own line/col in file_str. + + GHDL constant-folds some expressions (e.g. a "positive'high" default + is folded to an integer_literal) and then reports the *folded* + element's own line/col at the attribute name ("high"), not at the + start of the written expression ("positive'high"). The true start is + still there, on a descendant (here the attribute's prefix "positive"), + so scanning the whole subtree for the earliest position recovers it. + """ + best = None + line = element.get("line") + col = element.get("col") + if line is not None and col is not None and element.get("file") == file_str: + best = (int(line), int(col)) + for child in element: + child_best = _min_position(child, file_str) + if child_best is not None and (best is None or child_best < best): + best = child_best + return best + + +def _slice_element(source, element, file_path, decl_line, allow_assign): + """Slice source text starting at element's (recovered) position, or None if it has none.""" + position = _min_position(element, file_path) + if position is None: + return None + return source.slice_from(position[0], position[1], file_path, decl_line, allow_assign=allow_assign) + + +def _name_text(name_element): + """ + The (possibly dotted) name of a simple_name or selected_name element, + e.g. "push" or "work.push". + """ + identifier = name_element.get("identifier", "") + # GHDL's element carries its own kind/identifier directly (it + # *is* the name node for the prefix, e.g. simple_name "work"), not a + # wrapper around a further child. + prefix = name_element.find("prefix") + if prefix is None: + return identifier + return f"{_name_text(prefix)}.{identifier}" + + +def convert_parameter(element, source, file_path, decl_line): + """Convert one interface_*_declaration XML element to a parameter dict.""" + kind = element.get("kind") + param_class = _PARAMETER_CLASS.get(kind) + if param_class is None: + raise ApiReferenceError(f"Unhandled parameter kind '{kind}' at {file_path}:{decl_line}") + + # Implicit predefined operators (e.g. "=" synthesized for every type) + # have parameters with a "type" reference but no textual + # subtype_indication, since they were never written in source. + type_el = element.find("subtype_indication") + param_type = None + if type_el is not None: + param_type = _slice_element(source, type_el, file_path, decl_line, allow_assign=True) + + default = None + default_el = element.find("default_value") + if default_el is not None: + default = _slice_element(source, default_el, file_path, decl_line, allow_assign=False) + + return { + "name": element.get("identifier"), + "class": param_class, + "mode": element.get("mode"), + "type": param_type, + "default": default, + } + + +def convert_subprogram(element, source, file_path, is_top_level=True): + """ + Convert one function_declaration/procedure_declaration XML element to a + subprogram dict. is_top_level is False for a protected type's methods, + which are not individually documented. + """ + kind_raw = element.get("kind") + kind = "function" if kind_raw == "function_declaration" else "procedure" + identifier = element.get("identifier") + decl_line = int(element.get("line")) + + parameters = [] + chain = element.find("interface_declaration_chain") + if chain is not None: + for param_el in chain.findall("el"): + parameters.append(convert_parameter(param_el, source, file_path, decl_line)) + + return_type = None + if kind == "function": + return_type_el = element.find("return_type_mark") + if return_type_el is not None: + return_type = return_type_el.get("identifier") + + pure = None if kind == "procedure" else (element.get("pure_flag") == "true") + + entry = { + "kind": kind, + "name": identifier, + "operator": _IDENTIFIER_RE.match(identifier) is None, + "pure": pure, + "line": decl_line, + "parameters": parameters, + "return_type": return_type, + } + if is_top_level: + entry["doc"] = subprogram_doc(source, decl_line) + return entry + + +def convert_alias(element): + """Convert one non_object_alias_declaration/object_alias_declaration XML element to an alias dict.""" + decl_line = int(element.get("line")) + name_el = element.find("name") + target = _name_text(name_el) if name_el is not None else None + + signature_el = element.find("alias_signature") + signature = [] + return_type = None + if signature_el is not None: + type_marks_list = signature_el.find("type_marks_list") + if type_marks_list is not None: + signature = [mark.get("identifier") for mark in type_marks_list.findall("el")] + return_type_mark = signature_el.find("return_type_mark") + if return_type_mark is not None: + return_type = return_type_mark.get("identifier") + + return { + "name": element.get("identifier"), + "line": decl_line, + "target": target, + "signature": signature, + "return_type": return_type, + } + + +def convert_constant(element, source, file_path): + """Convert one constant_declaration XML element to a constant dict.""" + decl_line = int(element.get("line")) + type_el = element.find("subtype_indication") + const_type = _slice_element(source, type_el, file_path, decl_line, allow_assign=True) + value = None + value_el = element.find("default_value") + if value_el is not None: + value = _slice_element(source, value_el, file_path, decl_line, allow_assign=False) + return { + "name": element.get("identifier"), + "line": decl_line, + "doc": declaration_doc(source, decl_line), + "type": const_type, + "value": value, + } + + +def convert_subtype(element, source, file_path): + """Convert one subtype_declaration XML element to a subtype dict.""" + decl_line = int(element.get("line")) + type_el = element.find("subtype_indication") + definition = _slice_element(source, type_el, file_path, decl_line, allow_assign=True) + return { + "name": element.get("identifier"), + "line": decl_line, + "doc": declaration_doc(source, decl_line), + "definition": definition, + } + + +def convert_object(element, source, file_path): + """Convert one signal/shared-variable/file declaration XML element to an object dict.""" + decl_line = int(element.get("line")) + type_el = element.find("subtype_indication") + obj_type = _slice_element(source, type_el, file_path, decl_line, allow_assign=True) + return { + "name": element.get("identifier"), + "class": _OBJECT_CLASS[element.get("kind")], + "line": decl_line, + "doc": declaration_doc(source, decl_line), + "type": obj_type, + } + + +def _convert_record_type(type_def, source, file_path, decl_line, common): + """Build a "record" type dict: {..., "elements": [{"name", "type"}, ...]}.""" + elements = [] + elements_list = type_def.find("elements_declaration_list") + if elements_list is not None: + for element_el in elements_list.findall("el"): + element_type_el = element_el.find("subtype_indication") + elements.append( + { + "name": element_el.get("identifier"), + "type": _slice_element(source, element_type_el, file_path, decl_line, allow_assign=True), + } + ) + return {**common, "kind": "record", "elements": elements} + + +def _convert_enumeration_type(type_def, common): + """Build an "enumeration" type dict: {..., "literals": [name, ...]}.""" + literal_list = type_def.find("enumeration_literal_list") + literals = [literal.get("identifier") for literal in literal_list.findall("el")] if literal_list is not None else [] + return {**common, "kind": "enumeration", "literals": literals} + + +def _convert_protected_type(type_def, source, file_path, common): + """Build a "protected" type dict: {..., "subprograms": [...]} (its methods).""" + subprograms = [] + chain = type_def.find("declaration_chain") + if chain is not None: + for member_el in chain.findall("el"): + if member_el.get("kind") in ("function_declaration", "procedure_declaration"): + subprograms.append(convert_subprogram(member_el, source, file_path, is_top_level=False)) + return {**common, "kind": "protected", "subprograms": subprograms} + + +def convert_type(element, source, file_path): + """Convert one type_declaration XML element to a type dict.""" + decl_line = int(element.get("line")) + common = {"name": element.get("identifier"), "line": decl_line, "doc": declaration_doc(source, decl_line)} + type_def = element.find("type_definition") + def_kind = type_def.get("kind") if type_def is not None else None + + if def_kind == "record_type_definition": + return _convert_record_type(type_def, source, file_path, decl_line, common) + if def_kind == "enumeration_type_definition": + return _convert_enumeration_type(type_def, common) + if def_kind == "protected_type_declaration": + return _convert_protected_type(type_def, source, file_path, common) + + # Array, access and file types (and anything else GHDL introduces): + # kept as "other" with the raw source-text definition. + definition = _slice_element(source, type_def, file_path, decl_line, allow_assign=True) + return {**common, "kind": "other", "definition": definition} + + +_KIND_HANDLERS = { + "function_declaration": ("subprograms", convert_subprogram), + "procedure_declaration": ("subprograms", convert_subprogram), + # non_object_alias_declaration covers subprogram and type aliases; + # object_alias_declaration covers aliases of a signal/variable/ + # constant (e.g. "alias null_byte_vector_ptr is null_string_ptr"). + # Both share the same shape (a "name", and an optional + # "alias_signature" for subprograms only) so both go through the same + # converter into "aliases". + "non_object_alias_declaration": ("aliases", lambda element, source, file_path: convert_alias(element)), + "object_alias_declaration": ("aliases", lambda element, source, file_path: convert_alias(element)), + "constant_declaration": ("constants", convert_constant), + "subtype_declaration": ("subtypes", convert_subtype), + "type_declaration": ("types", convert_type), + "signal_declaration": ("objects", convert_object), + "variable_declaration": ("objects", convert_object), + "file_declaration": ("objects", convert_object), +} + +_IGNORED_KINDS = {"anonymous_type_declaration"} + + +def convert_package(package_el, source, file_path, repo_relative_path): + """Convert one package_declaration XML element to a package dict.""" + decl_line = int(package_el.get("line")) + name = package_el.get("identifier") + file_str = str(file_path) + + package = { + "library": LIBRARY_NAME, + "name": name, + "source": repo_relative_path, + "line": decl_line, + "private": name.endswith("_private_pkg"), + "doc": package_doc(source, decl_line), + "types": [], + "subtypes": [], + "constants": [], + "objects": [], + "subprograms": [], + "aliases": [], + } + + chain = package_el.find("declaration_chain") + if chain is None: + return package + + for element in chain.findall("el"): + if element.get("file") != file_str: + continue + kind = element.get("kind") + if kind in _IGNORED_KINDS: + continue + handler = _KIND_HANDLERS.get(kind) + if handler is None: + raise ApiReferenceError(f"Unhandled declaration kind '{kind}' at {file_str}:{element.get('line')}") + bucket, convert = handler + package[bucket].append(convert(element, source, file_str)) + + return package + + +# --------------------------------------------------------------------------- +# Completeness check +# --------------------------------------------------------------------------- + + +def check_completeness(json_package_names, expected_package_names): + """Raise ApiReferenceError listing missing/extra names if the two sets differ.""" + found = set(json_package_names) + expected = set(expected_package_names) + missing = sorted(expected - found) + extra = sorted(found - expected) + if missing or extra: + raise ApiReferenceError( + "Package set in the JSON differs from the package declarations found in the source files: " + f"missing {missing}, extra {extra}" + ) + + +# --------------------------------------------------------------------------- +# VHDL API top-level build +# --------------------------------------------------------------------------- + + +def _extract_one_package(root, file_path, package_name, vunit_lib_dir, osvvm_dir): + """Run --file-to-xml on one package file and convert it, or None if not found in the XML.""" + xml_root = parse_xml_root(run_file_to_xml(file_path, vunit_lib_dir, osvvm_dir)) + package_el = find_package_element(xml_root, file_path, package_name) + if package_el is None: + return None + source = SourceText(file_path.read_text(encoding="utf-8")) + repo_relative_path = file_path.relative_to(root).as_posix() + return convert_package(package_el, source, file_path, repo_relative_path) + + +def build_vhdl_api(root, vunit_version): + """Compile vunit_lib with GHDL and extract the full VHDL symbol table.""" + package_files = find_package_files(root) + check_ghdl_available() + # ghdl_version_line() already reads e.g. "GHDL 6.0.0 (...) [Dunoon edition]" + generator = f"{ghdl_version_line()} --file-to-xml" + + with tempfile.TemporaryDirectory(prefix="vunit-api-compile-") as compile_out: + libraries_dir = compile_vunit_libraries(compile_out) + packages = [] + for file_path, package_name in package_files: + package = _extract_one_package( + root, file_path, package_name, libraries_dir / "vunit_lib", libraries_dir / "osvvm" + ) + if package is not None: + packages.append(package) + + check_completeness((pkg["name"] for pkg in packages), (name for _, name in package_files)) + packages.sort(key=lambda pkg: pkg["name"]) + + return { + "schema_version": 1, + "vunit_version": vunit_version, + "vhdl_standard": VHDL_STANDARD, + "generator": generator, + "packages": packages, + } + + +# --------------------------------------------------------------------------- +# Python API +# --------------------------------------------------------------------------- + +_DIRECTIVE_RE = re.compile(r"^\.\.\s+(autoclass|autofunction|automodule)::\s*(\S+)", re.MULTILINE) + + +def _has_members_option(text, after_pos): + """True if a ":members:" option line follows the directive ending at after_pos.""" + for line in text[after_pos:].split("\n")[1:]: + if line.strip() == "": + break + if not line[:1].isspace(): + break + if line.strip().startswith(":members:"): + return True + return False + + +def find_python_targets(root): + """ + Return a list of (directive, dotted_path, has_members) for every + autoclass/autofunction/automodule directive under docs/**/*.rst. + """ + targets = [] + for rst_path in sorted((root / "docs").glob("**/*.rst")): + text = rst_path.read_text(encoding="utf-8") + for match in _DIRECTIVE_RE.finditer(text): + directive, target = match.groups() + target = target.rstrip("()") + targets.append((directive, target, _has_members_option(text, match.end()))) + return targets + + +def import_dotted(dotted): + """Import "a.b.C" as module "a.b" + getattr "C" (or the longest importable prefix).""" + parts = dotted.split(".") + for i in range(len(parts), 0, -1): + module_name = ".".join(parts[:i]) + try: + obj = importlib.import_module(module_name) + except ImportError: + continue + try: + for attr in parts[i:]: + obj = getattr(obj, attr) + except AttributeError as exc: + raise ApiReferenceError(f"Could not import Python autodoc target '{dotted}': {exc}") from exc + return obj + raise ApiReferenceError(f"Could not import Python autodoc target '{dotted}'") + + +def _class_members(cls): + """Public methods and properties defined directly on cls (not inherited).""" + prefix = cls.__qualname__ + "." + members = [] + for name in sorted(dir(cls)): + if name.startswith("_"): + continue + member = getattr(cls, name) + if isinstance(member, property): + fget = member.fget + if fget is None or not getattr(fget, "__qualname__", "").startswith(prefix): + continue + members.append({"name": name, "kind": "property", "signature": None, "doc": inspect.getdoc(member)}) + elif inspect.isfunction(member): + if not member.__qualname__.startswith(prefix): + continue + members.append( + { + "name": name, + "kind": "method", + "signature": str(inspect.signature(member)), + "doc": inspect.getdoc(member), + } + ) + return members + + +def build_class_object(dotted, cls): + """The Python API object for an autoclass target.""" + return { + "name": dotted, + "kind": "class", + "doc": inspect.getdoc(cls), + "members": _class_members(cls), + } + + +def build_function_object(dotted, func): + """The Python API object for an autofunction target.""" + return { + "name": dotted, + "kind": "function", + "doc": inspect.getdoc(func), + "signature": str(inspect.signature(func)), + } + + +def build_module_object(dotted, module, has_members): + """The Python API object for an automodule target.""" + members = [] + if has_members: + for name in sorted(vars(module)): + if name.startswith("_"): + continue + member = vars(module)[name] + if inspect.isclass(member) and member.__module__ == module.__name__: + members.append({"name": name, "kind": "class", "signature": None, "doc": inspect.getdoc(member)}) + elif inspect.isfunction(member) and member.__module__ == module.__name__: + members.append( + { + "name": name, + "kind": "function", + "signature": str(inspect.signature(member)), + "doc": inspect.getdoc(member), + } + ) + if not members: + raise ApiReferenceError(f"No public objects found for Python autodoc target '{dotted}'") + return {"name": dotted, "kind": "module", "doc": inspect.getdoc(module), "members": members} + + +def build_python_api(root, vunit_version): + """Collect the Python API objects for every autodoc target under docs/.""" + objects = [] + for directive, dotted, has_members in find_python_targets(root): + obj = import_dotted(dotted) + if directive == "autoclass": + objects.append(build_class_object(dotted, obj)) + elif directive == "autofunction": + objects.append(build_function_object(dotted, obj)) + else: + objects.append(build_module_object(dotted, obj, has_members)) + + objects.sort(key=lambda item: item["name"]) + return {"schema_version": 1, "vunit_version": vunit_version, "objects": objects} + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def _write_json(path, data): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=1, sort_keys=False, ensure_ascii=False) + "\n", encoding="utf-8") + + +def build(docs_dir, root=ROOT): + """Build both API JSON files into /api/.""" + from vunit.about import version # pylint: disable=import-outside-toplevel + + vunit_version = version() + vhdl_api = build_vhdl_api(root, vunit_version) + python_api = build_python_api(root, vunit_version) + + api_dir = Path(docs_dir) / "api" + _write_json(api_dir / "vunit_lib.json", vhdl_api) + _write_json(api_dir / "python.json", python_api) + + +def main(argv=None): + """CLI entry point: build_api_reference.py .""" + argv = sys.argv[1:] if argv is None else argv + + if os.environ.get("VUNIT_DOCS_SKIP_API") == "1": + print( + "WARNING: VUNIT_DOCS_SKIP_API=1, skipping the API reference build (no api/*.json written)", + file=sys.stderr, + ) + return 0 + + if len(argv) != 1: + print("usage: build_api_reference.py ", file=sys.stderr) + return 1 + + try: + build(argv[0]) + except ApiReferenceError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From ddf56b88931a73f7d02ccb28e3a71ea4ab107166 Mon Sep 17 00:00:00 2001 From: Sebastian Hellgren Date: Tue, 15 Sep 2026 13:46:59 +0200 Subject: [PATCH 03/11] Add a news fragment for the agent-friendly documentation Co-Authored-By: Claude Opus 5 (1M context) --- docs/news.d/1222.doc.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 docs/news.d/1222.doc.rst diff --git a/docs/news.d/1222.doc.rst b/docs/news.d/1222.doc.rst new file mode 100644 index 000000000..b10465433 --- /dev/null +++ b/docs/news.d/1222.doc.rst @@ -0,0 +1,3 @@ +The website now publishes documentation for AI agents: ``llms.txt``, ``llms-full.txt``, a plain text version of +every page and a machine-readable reference of the VHDL and Python APIs (``api/vunit_lib.json`` and +``api/python.json``). ``AGENTS.md`` and ``context7.json`` were added to the repository. From 979c2b6791e1b59789522ecdb1883143f65d57ed Mon Sep 17 00:00:00 2001 From: Sebastian Hellgren Date: Tue, 15 Sep 2026 18:00:23 +0200 Subject: [PATCH 04/11] Simplify the API reference and llms.txt generators * Autodoc targets are resolved with pkgutil.resolve_name, and an autodoc directive other than autoclass and automodule, of which the documentation has none, is reported instead of being supported speculatively. * The one-line doc comment and source line helpers, the per-package extraction helper and the ignored kinds set are inlined. * Protected type methods get a doc comment like every other subprogram, which removes a flag. * llms_txt.py no longer filters hidden paths that the text build never produces. * Repeated unit tests are combined into table-driven tests. Co-Authored-By: Claude Opus 5 (1M context) --- tests/unit/test_api_reference.py | 184 ++++++++++--------------------- tools/build_api_reference.py | 116 ++++++------------- tools/llms_txt.py | 12 +- 3 files changed, 93 insertions(+), 219 deletions(-) diff --git a/tests/unit/test_api_reference.py b/tests/unit/test_api_reference.py index 19db62caf..3b6df229e 100644 --- a/tests/unit/test_api_reference.py +++ b/tests/unit/test_api_reference.py @@ -116,17 +116,17 @@ class TestDocComments(unittest.TestCase): def test_extracts_comment_block_directly_above(self): text = " -- Returns the length\n -- of the queue\n function length return natural;\n" source = bar.SourceText(text) - self.assertEqual(bar.declaration_doc(source, 3), "Returns the length\nof the queue") + self.assertEqual(bar.comment_block_above(source, 3), "Returns the length\nof the queue") def test_blank_line_ends_the_block(self): text = " -- Unrelated comment\n\n function length return natural;\n" source = bar.SourceText(text) - self.assertIsNone(bar.declaration_doc(source, 3)) + self.assertIsNone(bar.comment_block_above(source, 3)) def test_no_comment_above_gives_none(self): text = " use work.foo_pkg.all;\n function length return natural;\n" source = bar.SourceText(text) - self.assertIsNone(bar.declaration_doc(source, 2)) + self.assertIsNone(bar.comment_block_above(source, 2)) def test_license_header_excluded_from_package_doc(self): text = ( @@ -145,7 +145,7 @@ def test_non_license_comment_kept_for_package_doc(self): def test_impure_on_a_preceding_line_shifts_the_doc_anchor(self): text = " -- Returns something\n impure\n function get return integer;\n" source = bar.SourceText(text) - self.assertEqual(bar.subprogram_doc(source, 3), "Returns something") + self.assertEqual(bar.comment_block_above(source, bar.subprogram_first_line(source, 3)), "Returns something") def test_impure_on_the_same_line_does_not_pull_in_the_comment_above(self): # Here the comment belongs to something else entirely -- decl_line @@ -153,7 +153,7 @@ def test_impure_on_the_same_line_does_not_pull_in_the_comment_above(self): # above *that*, which is blank. text = " -- Unrelated\n\n impure function get return integer;\n" source = bar.SourceText(text) - self.assertIsNone(bar.subprogram_doc(source, 3)) + self.assertIsNone(bar.comment_block_above(source, bar.subprogram_first_line(source, 3))) class TestConvertSubprogram(unittest.TestCase): @@ -182,39 +182,26 @@ def test_procedure_has_no_pure_flag(self): entry = bar.convert_subprogram(el, source, "f.vhd") self.assertEqual(entry["kind"], "procedure") self.assertIsNone(entry["pure"]) + self.assertFalse(entry["operator"]) self.assertEqual(entry["parameters"][0]["type"], "rec_t") self.assertEqual(entry["parameters"][0]["class"], "constant") self.assertEqual(entry["parameters"][0]["mode"], "in") - def test_impure_function_pure_flag_false(self): - line = " impure function func_get (target : rec_t) return integer;" - xml = f""" - - - - - """ - el = self._parse(xml) - source = bar.SourceText(line + "\n") - entry = bar.convert_subprogram(el, source, "f.vhd") - self.assertEqual(entry["kind"], "function") - self.assertFalse(entry["pure"]) - self.assertEqual(entry["return_type"], "integer") - - def test_pure_function_pure_flag_true(self): - line = " function pure_fn (x : integer) return integer;" - xml = f""" - - - - - """ - el = self._parse(xml) - source = bar.SourceText(line + "\n") - entry = bar.convert_subprogram(el, source, "f.vhd") - self.assertTrue(entry["pure"]) + def test_function_pure_flag_and_return_type(self): + for pure_flag, prefix in (("false", "impure "), ("true", "")): + with self.subTest(pure_flag=pure_flag): + line = f" {prefix}function func_get (target : rec_t) return integer;" + xml = f""" + + + + + """ + entry = bar.convert_subprogram(self._parse(xml), bar.SourceText(line + "\n"), "f.vhd") + self.assertEqual(entry["kind"], "function") + self.assertEqual(entry["pure"], pure_flag == "true") + self.assertEqual(entry["return_type"], "integer") def test_operator_identifier_is_flagged(self): line = ' function "+" (l, r : rec_t) return rec_t;' @@ -231,18 +218,6 @@ def test_operator_identifier_is_flagged(self): self.assertTrue(entry["operator"]) self.assertEqual(entry["name"], "+") - def test_plain_identifier_is_not_an_operator(self): - line = " procedure proc_set (target : in rec_t);" - xml = f""" - - - - """ - el = self._parse(xml) - source = bar.SourceText(line + "\n") - entry = bar.convert_subprogram(el, source, "f.vhd") - self.assertFalse(entry["operator"]) - def test_parameter_default_value(self): line = " value : in integer := 0" xml = f""" @@ -464,35 +439,22 @@ def test_protected_type_with_methods(self): self.assertEqual(len(entry["subprograms"]), 1) method = entry["subprograms"][0] self.assertEqual(method["name"], "set") - # Methods use the same subprogram format, but (like aliases) are - # not individually documented top-level entries. - self.assertNotIn("doc", method) - - def test_other_type_array(self): - line = " type vec_t is array (natural range <>) of rec_t;" - xml = f""" - - - - """ - el = ElementTree.fromstring(xml) - source = bar.SourceText(line + "\n") - entry = bar.convert_type(el, source, "f.vhd") - self.assertEqual(entry["kind"], "other") - self.assertEqual(entry["definition"], "array (natural range <>) of rec_t") - - def test_other_type_access(self): - line = " type ptr_t is access rec_t;" - xml = f""" - - - - """ - el = ElementTree.fromstring(xml) - source = bar.SourceText(line + "\n") - entry = bar.convert_type(el, source, "f.vhd") - self.assertEqual(entry["kind"], "other") - self.assertEqual(entry["definition"], "access rec_t") + self.assertIsNone(method["doc"]) + + def test_other_types(self): + for line, keyword, definition in ( + (" type vec_t is array (natural range <>) of rec_t;", "array", "array (natural range <>) of rec_t"), + (" type ptr_t is access rec_t;", "access", "access rec_t"), + ): + with self.subTest(keyword=keyword): + xml = f""" + + + + """ + entry = bar.convert_type(ElementTree.fromstring(xml), bar.SourceText(line + "\n"), "f.vhd") + self.assertEqual(entry["kind"], "other") + self.assertEqual(entry["definition"], definition) class TestConvertSubtypeConstantObject(unittest.TestCase): @@ -534,42 +496,21 @@ def test_deferred_constant_has_no_value(self): entry = bar.convert_constant(el, source, "f.vhd") self.assertIsNone(entry["value"]) - def test_signal_object(self): - line = " signal s_enabled : std_logic := '1';" - xml = f""" - - - - """ - el = ElementTree.fromstring(xml) - source = bar.SourceText(line + "\n") - entry = bar.convert_object(el, source, "f.vhd") - self.assertEqual(entry["class"], "signal") - self.assertEqual(entry["type"], "std_logic") - - def test_shared_variable_object(self): - line = " shared variable v_shared : integer;" - xml = f""" - - - - """ - el = ElementTree.fromstring(xml) - source = bar.SourceText(line + "\n") - entry = bar.convert_object(el, source, "f.vhd") - self.assertEqual(entry["class"], "shared variable") - - def test_file_object(self): - line = " file f0 : text;" - xml = f""" - - - - """ - el = ElementTree.fromstring(xml) - source = bar.SourceText(line + "\n") - entry = bar.convert_object(el, source, "f.vhd") - self.assertEqual(entry["class"], "file") + def test_objects(self): + for line, kind, object_class, type_name in ( + (" signal s_enabled : std_logic := '1';", "signal_declaration", "signal", "std_logic"), + (" shared variable v_shared : integer;", "variable_declaration", "shared variable", "integer"), + (" file f0 : text;", "file_declaration", "file", "text"), + ): + with self.subTest(kind=kind): + xml = f""" + + + + """ + entry = bar.convert_object(ElementTree.fromstring(xml), bar.SourceText(line + "\n"), "f.vhd") + self.assertEqual(entry["class"], object_class) + self.assertEqual(entry["type"], type_name) class TestConvertPackage(unittest.TestCase): @@ -624,16 +565,11 @@ def test_unhandled_declaration_kind_raises(self): self.assertIn("/a/my_pkg.vhd:9", str(ctx.exception)) def test_private_package_flag(self): - xml = self._package_xml("", name="data_types_private_pkg") - el = ElementTree.fromstring(xml) - pkg = bar.convert_package(el, bar.SourceText("\n"), "/a/my_pkg.vhd", "my_pkg.vhd") - self.assertTrue(pkg["private"]) - - def test_non_private_package_flag(self): - xml = self._package_xml("", name="queue_pkg") - el = ElementTree.fromstring(xml) - pkg = bar.convert_package(el, bar.SourceText("\n"), "/a/my_pkg.vhd", "my_pkg.vhd") - self.assertFalse(pkg["private"]) + for name, private in (("data_types_private_pkg", True), ("queue_pkg", False)): + with self.subTest(name=name): + el = ElementTree.fromstring(self._package_xml("", name=name)) + pkg = bar.convert_package(el, bar.SourceText("\n"), "/a/my_pkg.vhd", "my_pkg.vhd") + self.assertEqual(pkg["private"], private) def test_p_prefixed_names_not_flagged_private(self): line8 = " constant p_internal : integer := 1;" @@ -869,12 +805,6 @@ def test_class_object_collects_public_members_defined_on_the_class(self): self.assertIn("(self, value)", members["method_one"]["signature"]) self.assertEqual(members["method_one"]["doc"], "Doubles value.") - def test_function_object(self): - obj = bar.build_function_object(f"{__name__}.sample_function", sample_function) - self.assertEqual(obj["kind"], "function") - self.assertEqual(obj["doc"], "Adds a and b.") - self.assertIn("a, b=1", obj["signature"]) - def test_module_object_without_members_option_has_no_members(self): obj = bar.build_module_object(__name__, sys.modules[__name__], has_members=False) self.assertEqual(obj["kind"], "module") diff --git a/tools/build_api_reference.py b/tools/build_api_reference.py index 0a03a3f41..372e2e868 100644 --- a/tools/build_api_reference.py +++ b/tools/build_api_reference.py @@ -17,10 +17,10 @@ the documentation without GHDL installed. """ -import importlib import inspect import json import os +import pkgutil import re import shutil import subprocess @@ -202,10 +202,6 @@ def __init__(self, text): acc += len(line) + 1 self._line_offsets = offsets - def line_text(self, line): - """The 1-based line's raw text, without its terminator.""" - return self.lines[line - 1] - @staticmethod def _col_to_index(line_text, col): """0-based character index in line_text for a 1-based GHDL column (tab stops of 8).""" @@ -221,7 +217,7 @@ def _col_to_index(line_text, col): def offset(self, line, col): """Absolute 0-based character offset in self.text for a 1-based (line, col).""" - return self._line_offsets[line - 1] + self._col_to_index(self.line_text(line), col) + return self._line_offsets[line - 1] + self._col_to_index(self.lines[line - 1], col) def slice_from(self, line, col, file_path, decl_line, allow_assign): """ @@ -282,7 +278,7 @@ def _skip_literal(text, i): _LICENSE_MARKER = "This Source Code Form" -def _comment_block_above(source, first_line): +def comment_block_above(source, first_line): """ The contiguous block of "--" lines directly above first_line, stripped of "--" and one following space, joined with "\\n". None if there is no @@ -291,7 +287,7 @@ def _comment_block_above(source, first_line): collected = [] line_no = first_line - 1 while 1 <= line_no <= len(source.lines): - stripped = source.line_text(line_no).strip() + stripped = source.lines[line_no - 1].strip() if stripped.startswith("--"): collected.append(stripped) line_no -= 1 @@ -317,24 +313,14 @@ def subprogram_first_line(source, decl_line): "pure" or "impure", in which case that line is the true first line. """ prev = decl_line - 1 - if prev >= 1 and source.line_text(prev).strip() in ("pure", "impure"): + if prev >= 1 and source.lines[prev - 1].strip() in ("pure", "impure"): return prev return decl_line -def subprogram_doc(source, decl_line): - """The doc comment for a subprogram declared at decl_line, or None.""" - return _comment_block_above(source, subprogram_first_line(source, decl_line)) - - -def declaration_doc(source, decl_line): - """The doc comment for any other declaration at decl_line, or None.""" - return _comment_block_above(source, decl_line) - - def package_doc(source, decl_line): """The package's own doc comment, or None (also None for the MPL license header).""" - doc = _comment_block_above(source, decl_line) + doc = comment_block_above(source, decl_line) if doc is not None and _LICENSE_MARKER in doc: return None return doc @@ -436,11 +422,10 @@ def convert_parameter(element, source, file_path, decl_line): } -def convert_subprogram(element, source, file_path, is_top_level=True): +def convert_subprogram(element, source, file_path): """ Convert one function_declaration/procedure_declaration XML element to a - subprogram dict. is_top_level is False for a protected type's methods, - which are not individually documented. + subprogram dict. """ kind_raw = element.get("kind") kind = "function" if kind_raw == "function_declaration" else "procedure" @@ -461,7 +446,7 @@ def convert_subprogram(element, source, file_path, is_top_level=True): pure = None if kind == "procedure" else (element.get("pure_flag") == "true") - entry = { + return { "kind": kind, "name": identifier, "operator": _IDENTIFIER_RE.match(identifier) is None, @@ -469,13 +454,11 @@ def convert_subprogram(element, source, file_path, is_top_level=True): "line": decl_line, "parameters": parameters, "return_type": return_type, + "doc": comment_block_above(source, subprogram_first_line(source, decl_line)), } - if is_top_level: - entry["doc"] = subprogram_doc(source, decl_line) - return entry -def convert_alias(element): +def convert_alias(element, *_): """Convert one non_object_alias_declaration/object_alias_declaration XML element to an alias dict.""" decl_line = int(element.get("line")) name_el = element.find("name") @@ -513,7 +496,7 @@ def convert_constant(element, source, file_path): return { "name": element.get("identifier"), "line": decl_line, - "doc": declaration_doc(source, decl_line), + "doc": comment_block_above(source, decl_line), "type": const_type, "value": value, } @@ -527,7 +510,7 @@ def convert_subtype(element, source, file_path): return { "name": element.get("identifier"), "line": decl_line, - "doc": declaration_doc(source, decl_line), + "doc": comment_block_above(source, decl_line), "definition": definition, } @@ -541,7 +524,7 @@ def convert_object(element, source, file_path): "name": element.get("identifier"), "class": _OBJECT_CLASS[element.get("kind")], "line": decl_line, - "doc": declaration_doc(source, decl_line), + "doc": comment_block_above(source, decl_line), "type": obj_type, } @@ -576,14 +559,14 @@ def _convert_protected_type(type_def, source, file_path, common): if chain is not None: for member_el in chain.findall("el"): if member_el.get("kind") in ("function_declaration", "procedure_declaration"): - subprograms.append(convert_subprogram(member_el, source, file_path, is_top_level=False)) + subprograms.append(convert_subprogram(member_el, source, file_path)) return {**common, "kind": "protected", "subprograms": subprograms} def convert_type(element, source, file_path): """Convert one type_declaration XML element to a type dict.""" decl_line = int(element.get("line")) - common = {"name": element.get("identifier"), "line": decl_line, "doc": declaration_doc(source, decl_line)} + common = {"name": element.get("identifier"), "line": decl_line, "doc": comment_block_above(source, decl_line)} type_def = element.find("type_definition") def_kind = type_def.get("kind") if type_def is not None else None @@ -609,8 +592,8 @@ def convert_type(element, source, file_path): # Both share the same shape (a "name", and an optional # "alias_signature" for subprograms only) so both go through the same # converter into "aliases". - "non_object_alias_declaration": ("aliases", lambda element, source, file_path: convert_alias(element)), - "object_alias_declaration": ("aliases", lambda element, source, file_path: convert_alias(element)), + "non_object_alias_declaration": ("aliases", convert_alias), + "object_alias_declaration": ("aliases", convert_alias), "constant_declaration": ("constants", convert_constant), "subtype_declaration": ("subtypes", convert_subtype), "type_declaration": ("types", convert_type), @@ -619,8 +602,6 @@ def convert_type(element, source, file_path): "file_declaration": ("objects", convert_object), } -_IGNORED_KINDS = {"anonymous_type_declaration"} - def convert_package(package_el, source, file_path, repo_relative_path): """Convert one package_declaration XML element to a package dict.""" @@ -651,7 +632,7 @@ def convert_package(package_el, source, file_path, repo_relative_path): if element.get("file") != file_str: continue kind = element.get("kind") - if kind in _IGNORED_KINDS: + if kind == "anonymous_type_declaration": continue handler = _KIND_HANDLERS.get(kind) if handler is None: @@ -685,17 +666,6 @@ def check_completeness(json_package_names, expected_package_names): # --------------------------------------------------------------------------- -def _extract_one_package(root, file_path, package_name, vunit_lib_dir, osvvm_dir): - """Run --file-to-xml on one package file and convert it, or None if not found in the XML.""" - xml_root = parse_xml_root(run_file_to_xml(file_path, vunit_lib_dir, osvvm_dir)) - package_el = find_package_element(xml_root, file_path, package_name) - if package_el is None: - return None - source = SourceText(file_path.read_text(encoding="utf-8")) - repo_relative_path = file_path.relative_to(root).as_posix() - return convert_package(package_el, source, file_path, repo_relative_path) - - def build_vhdl_api(root, vunit_version): """Compile vunit_lib with GHDL and extract the full VHDL symbol table.""" package_files = find_package_files(root) @@ -707,11 +677,11 @@ def build_vhdl_api(root, vunit_version): libraries_dir = compile_vunit_libraries(compile_out) packages = [] for file_path, package_name in package_files: - package = _extract_one_package( - root, file_path, package_name, libraries_dir / "vunit_lib", libraries_dir / "osvvm" - ) - if package is not None: - packages.append(package) + xml = run_file_to_xml(file_path, libraries_dir / "vunit_lib", libraries_dir / "osvvm") + package_el = find_package_element(parse_xml_root(xml), file_path, package_name) + if package_el is not None: + source = SourceText(file_path.read_text(encoding="utf-8")) + packages.append(convert_package(package_el, source, file_path, file_path.relative_to(root).as_posix())) check_completeness((pkg["name"] for pkg in packages), (name for _, name in package_files)) packages.sort(key=lambda pkg: pkg["name"]) @@ -729,7 +699,7 @@ def build_vhdl_api(root, vunit_version): # Python API # --------------------------------------------------------------------------- -_DIRECTIVE_RE = re.compile(r"^\.\.\s+(autoclass|autofunction|automodule)::\s*(\S+)", re.MULTILINE) +_DIRECTIVE_RE = re.compile(r"^\.\.\s+(auto\w+)::\s*(\S+)", re.MULTILINE) def _has_members_option(text, after_pos): @@ -760,21 +730,11 @@ def find_python_targets(root): def import_dotted(dotted): - """Import "a.b.C" as module "a.b" + getattr "C" (or the longest importable prefix).""" - parts = dotted.split(".") - for i in range(len(parts), 0, -1): - module_name = ".".join(parts[:i]) - try: - obj = importlib.import_module(module_name) - except ImportError: - continue - try: - for attr in parts[i:]: - obj = getattr(obj, attr) - except AttributeError as exc: - raise ApiReferenceError(f"Could not import Python autodoc target '{dotted}': {exc}") from exc - return obj - raise ApiReferenceError(f"Could not import Python autodoc target '{dotted}'") + """Import the object with the dotted name of an autodoc target, e.g. "a.b.C".""" + try: + return pkgutil.resolve_name(dotted) + except (ImportError, AttributeError) as exc: + raise ApiReferenceError(f"Could not import Python autodoc target '{dotted}': {exc}") from exc def _class_members(cls): @@ -814,16 +774,6 @@ def build_class_object(dotted, cls): } -def build_function_object(dotted, func): - """The Python API object for an autofunction target.""" - return { - "name": dotted, - "kind": "function", - "doc": inspect.getdoc(func), - "signature": str(inspect.signature(func)), - } - - def build_module_object(dotted, module, has_members): """The Python API object for an automodule target.""" members = [] @@ -855,10 +805,10 @@ def build_python_api(root, vunit_version): obj = import_dotted(dotted) if directive == "autoclass": objects.append(build_class_object(dotted, obj)) - elif directive == "autofunction": - objects.append(build_function_object(dotted, obj)) - else: + elif directive == "automodule": objects.append(build_module_object(dotted, obj, has_members)) + else: + raise ApiReferenceError(f"Unsupported autodoc directive '{directive}' for '{dotted}' in docs/") objects.sort(key=lambda item: item["name"]) return {"schema_version": 1, "vunit_version": vunit_version, "objects": objects} diff --git a/tools/llms_txt.py b/tools/llms_txt.py index 6691237d1..978f17668 100644 --- a/tools/llms_txt.py +++ b/tools/llms_txt.py @@ -89,14 +89,11 @@ def _placement(docname): """ if docname in EXCLUDED: return None - other_index = None - for section_index, (title, prefixes) in enumerate(SECTIONS): + for section_index, (_, prefixes) in enumerate(SECTIONS): for prefix_index, prefix in enumerate(prefixes): if _matches(docname, prefix): return section_index, prefix_index - if title == OTHER_SECTION: - other_index = section_index - return other_index, 0 + return [title for title, _ in SECTIONS].index(OTHER_SECTION), 0 def section_of(docname): @@ -114,10 +111,7 @@ def read_pages(text_dir): text_dir = Path(text_dir) pages = [] for path in text_dir.rglob("*.txt"): - relative = path.relative_to(text_dir) - if any(part.startswith(".") for part in relative.parts): - continue - docname = relative.with_suffix("").as_posix() + docname = path.relative_to(text_dir).with_suffix("").as_posix() if _placement(docname) is None: continue text = path.read_text(encoding="utf-8") From c12cab59acd46c1d35ff431f4c0342e8928adf8e Mon Sep 17 00:00:00 2001 From: Sebastian Hellgren Date: Tue, 15 Sep 2026 18:18:11 +0200 Subject: [PATCH 05/11] Skip generating llms.txt and llms-full.txt The documentation already covers the pages, so only the machine-readable API reference is generated for AI agents. The text build and tools/llms_txt.py are removed, tools/build_docs.py is back to its original form, and AGENTS.md, context7.json and the news fragment no longer refer to the llms files. Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 7 +- context7.json | 3 +- docs/news.d/1222.doc.rst | 5 +- tests/unit/test_llms_txt.py | 127 ------------------------- tools/build_docs.py | 30 +++--- tools/llms_txt.py | 178 ------------------------------------ 6 files changed, 18 insertions(+), 332 deletions(-) delete mode 100644 tests/unit/test_llms_txt.py delete mode 100644 tools/llms_txt.py diff --git a/AGENTS.md b/AGENTS.md index de1099aee..30b521c32 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,11 +2,8 @@ Guidance for AI coding agents working on the VUnit repository. -To *use* VUnit rather than develop it, read the documentation written for agents instead: - -- [llms.txt](https://vunit.github.io/llms.txt): index of the documentation as plain text -- [llms-full.txt](https://vunit.github.io/llms-full.txt): all documentation, including the VHDL package headers -- [api/vunit_lib.json](https://vunit.github.io/api/vunit_lib.json) and [api/python.json](https://vunit.github.io/api/python.json): machine-readable VHDL and Python APIs +To *use* VUnit rather than develop it, read the [documentation](https://vunit.github.io) and the machine-readable +[VHDL API](https://vunit.github.io/api/vunit_lib.json) and [Python API](https://vunit.github.io/api/python.json). ## Repository layout diff --git a/context7.json b/context7.json index 56487fdcd..e3dc837e1 100644 --- a/context7.json +++ b/context7.json @@ -18,7 +18,6 @@ "A run script creates the project with VUnit.from_argv(), calls add_vhdl_builtins(), adds libraries and source files, and ends with main().", "A VHDL testbench has a runner_cfg generic and runs its test cases, each selected with run(\"\"), between test_runner_setup and test_runner_cleanup.", "Complete VHDL API signatures: https://vunit.github.io/api/vunit_lib.json", - "Complete Python API signatures: https://vunit.github.io/api/python.json", - "Complete rendered documentation including the VHDL package headers: https://vunit.github.io/llms-full.txt" + "Complete Python API signatures: https://vunit.github.io/api/python.json" ] } diff --git a/docs/news.d/1222.doc.rst b/docs/news.d/1222.doc.rst index b10465433..cd4d30842 100644 --- a/docs/news.d/1222.doc.rst +++ b/docs/news.d/1222.doc.rst @@ -1,3 +1,2 @@ -The website now publishes documentation for AI agents: ``llms.txt``, ``llms-full.txt``, a plain text version of -every page and a machine-readable reference of the VHDL and Python APIs (``api/vunit_lib.json`` and -``api/python.json``). ``AGENTS.md`` and ``context7.json`` were added to the repository. +The website now publishes a machine-readable reference of the VHDL and Python APIs for AI agents +(``api/vunit_lib.json`` and ``api/python.json``). ``AGENTS.md`` and ``context7.json`` were added to the repository. diff --git a/tests/unit/test_llms_txt.py b/tests/unit/test_llms_txt.py deleted file mode 100644 index de1cf977e..000000000 --- a/tests/unit/test_llms_txt.py +++ /dev/null @@ -1,127 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. -# -# Copyright (c) 2014-2026, Lars Asplund lars.anders.asplund@gmail.com - -""" -Test the generation of llms.txt and llms-full.txt from the text build of the documentation -""" - -import unittest -from pathlib import Path -from tests.common import with_tempdir -from tools.llms_txt import read_pages, render_llms_full_txt, render_llms_txt, section_of, write_llms_files - -PAGES = { - "index": """\ -VUnit: a test framework for HDL -******************************* - -[image: shieldPyPI][image] - -VUnit is an open source -unit testing framework. - -[image] -""", - "installing": "Installing\n**********\n\nUse pip.\n", - "run/user_guide": "Run Library User Guide\n**********************\n\nTest cases.\n", - "data_types/queue": "*queue* package\n***************\n\n package queue_pkg is\n", - "blog/2023_03_31_vunit_events": "VUnit Events\n************\n\nNews.\n", - "genindex": "Index\n*****\n", - "new_topic/page": "New Topic\n*********\n\nNew.\n", -} - - -def create_text_build(output_path, pages=None): - """ - Create a text build with the given pages under output_path - """ - for docname, text in (PAGES if pages is None else pages).items(): - path = Path(output_path) / "_text" / f"{docname}.txt" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(text, encoding="utf-8") - - -class TestLlmsTxt(unittest.TestCase): - """ - Test the generation of llms.txt and llms-full.txt - """ - - def test_section_of(self): - self.assertEqual(section_of("index"), "Getting started") - self.assertEqual(section_of("run/user_guide"), "User guides") - self.assertEqual(section_of("data_types/user_guide"), "User guides") - self.assertEqual(section_of("data_types/queue"), "API reference") - self.assertEqual(section_of("blog/2023_03_31_vunit_events"), "Optional") - self.assertEqual(section_of("new_topic/page"), "Other") - self.assertIsNone(section_of("genindex")) - - @with_tempdir - def test_llms_txt(self, tempdir): - create_text_build(tempdir) - self.assertEqual( - render_llms_txt(read_pages(Path(tempdir) / "_text")), - """\ -# VUnit - -> VUnit is an open source unit testing framework. - -## Getting started - -- [VUnit: a test framework for HDL](https://vunit.github.io/_text/index.txt): https://vunit.github.io/index.html -- [Installing](https://vunit.github.io/_text/installing.txt): https://vunit.github.io/installing.html - -## User guides - -- [Run Library User Guide](https://vunit.github.io/_text/run/user_guide.txt): https://vunit.github.io/run/user_guide.html - -## API reference - -- [VHDL API (JSON)](https://vunit.github.io/api/vunit_lib.json) -- [Python API (JSON)](https://vunit.github.io/api/python.json) -- [*queue* package](https://vunit.github.io/_text/data_types/queue.txt): https://vunit.github.io/data_types/queue.html - -## Other - -- [New Topic](https://vunit.github.io/_text/new_topic/page.txt): https://vunit.github.io/new_topic/page.html - -## Optional - -- [VUnit Events](https://vunit.github.io/_text/blog/2023_03_31_vunit_events.txt): https://vunit.github.io/blog/2023_03_31_vunit_events.html -""", - ) - - @with_tempdir - def test_llms_txt_without_api(self, tempdir): - create_text_build(tempdir) - llms_txt = render_llms_txt(read_pages(Path(tempdir) / "_text"), api=False) - self.assertNotIn("api/", llms_txt) - self.assertIn("## API reference\n\n- [*queue* package]", llms_txt) - - @with_tempdir - def test_llms_full_txt(self, tempdir): - create_text_build(tempdir) - llms_full_txt = render_llms_full_txt(read_pages(Path(tempdir) / "_text")) - self.assertIn( - "# Installing\n\nSource: https://vunit.github.io/installing.html\n\nInstalling\n**********\n\nUse pip.\n", - llms_full_txt, - ) - self.assertIn("# New Topic\n", llms_full_txt) - self.assertNotIn("VUnit Events", llms_full_txt) - self.assertNotIn("Index", llms_full_txt) - self.assertLess(llms_full_txt.index("# Installing"), llms_full_txt.index("# *queue* package")) - - @with_tempdir - def test_write_llms_files(self, tempdir): - create_text_build(tempdir) - write_llms_files(tempdir) - self.assertTrue((Path(tempdir) / "llms.txt").read_text(encoding="utf-8").startswith("# VUnit\n")) - self.assertIn("# Installing", (Path(tempdir) / "llms-full.txt").read_text(encoding="utf-8")) - - @with_tempdir - def test_missing_front_page_is_an_error(self, tempdir): - create_text_build(tempdir, {"installing": PAGES["installing"]}) - with self.assertRaisesRegex(RuntimeError, "no front page"): - render_llms_txt(read_pages(Path(tempdir) / "_text")) diff --git a/tools/build_docs.py b/tools/build_docs.py index 14b5785a2..3c64f4af2 100644 --- a/tools/build_docs.py +++ b/tools/build_docs.py @@ -10,34 +10,30 @@ Command line utility to build documentation/website """ -from os import environ from subprocess import check_call from pathlib import Path from sys import argv, executable -from tempfile import TemporaryDirectory -from llms_txt import TEXT_DIR_NAME, write_llms_files ROOT = Path(__file__).parent.parent -def sphinx(builder, output_path, args): - """ - Build the documentation with a Sphinx builder - """ - check_call([executable, "-m", "sphinx"] + args + ["-TEWanb", builder, ROOT / "docs", output_path]) - - def main(): """ - Build documentation/website, including a text version and llms.txt/llms-full.txt for AI agents + Build documentation/website """ - output_path = Path(argv[1]) - args = argv[2:] - sphinx("html", output_path, args) - with TemporaryDirectory() as doctrees: - sphinx("text", output_path / TEXT_DIR_NAME, args + ["-d", doctrees]) - write_llms_files(output_path, api=environ.get("VUNIT_DOCS_SKIP_API") != "1") + check_call( + [ + executable, + "-m", + "sphinx" + ] + ([] if len(argv) < 2 else argv[2:]) + [ + "-TEWanb", + "html", + Path(__file__).parent.parent / "docs", + argv[1], + ] + ) if __name__ == "__main__": diff --git a/tools/llms_txt.py b/tools/llms_txt.py deleted file mode 100644 index 978f17668..000000000 --- a/tools/llms_txt.py +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env python3 - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. -# -# Copyright (c) 2014-2026, Lars Asplund lars.anders.asplund@gmail.com - -""" -Generate llms.txt and llms-full.txt from the plain text build of the documentation. - -The text build (sphinx -b text) renders every page completely, including the VHDL -package headers the pages include and the generated Python and command line -references, which is what makes it suitable for AI agents. -""" - -from dataclasses import dataclass -from pathlib import Path - -BASE_URL = "https://vunit.github.io" -TEXT_DIR_NAME = "_text" -API_SECTION = "API reference" -OTHER_SECTION = "Other" -OPTIONAL_SECTION = "Optional" - -# The sections of llms.txt in order. A page belongs to the first section with a -# matching prefix, which is either a document name or a directory ending with "/". -# Pages matching no prefix are listed under OTHER_SECTION such that a new page never -# breaks the build. Pages under OPTIONAL_SECTION are left out of llms-full.txt. -SECTIONS = ( - ("Getting started", ("index", "installing", "user_guide", "cli", "examples", "hdl_libraries")), - ( - "User guides", - ( - "run/", - "check/user_guide", - "logging/", - "com/", - "data_types/user_guide", - "data_types/event_user_guide", - "id/", - "verification_components/", - "ci/", - ), - ), - (API_SECTION, ("py/", "check/", "data_types/")), - (OTHER_SECTION, ()), - (OPTIONAL_SECTION, ("blog/", "release_notes", "testimonials/", "about", "contributing", "license")), -) -EXCLUDED = ("genindex",) -API_FILES = ( - ("VHDL API (JSON)", "api/vunit_lib.json"), - ("Python API (JSON)", "api/python.json"), -) - - -@dataclass -class Page: - """ - A page of the text build - """ - - docname: str - title: str - text: str - - @property - def text_url(self): - """ - URL of the text version of the page - """ - return f"{BASE_URL}/{TEXT_DIR_NAME}/{self.docname}.txt" - - @property - def html_url(self): - """ - URL of the HTML version of the page - """ - return f"{BASE_URL}/{self.docname}.html" - - -def _matches(docname, prefix): - return docname == prefix or (prefix.endswith("/") and docname.startswith(prefix)) - - -def _placement(docname): - """ - The section index and the prefix index of a page, None for a page that is left out - """ - if docname in EXCLUDED: - return None - for section_index, (_, prefixes) in enumerate(SECTIONS): - for prefix_index, prefix in enumerate(prefixes): - if _matches(docname, prefix): - return section_index, prefix_index - return [title for title, _ in SECTIONS].index(OTHER_SECTION), 0 - - -def section_of(docname): - """ - The llms.txt section of a page, None for a page that is left out - """ - placement = _placement(docname) - return None if placement is None else SECTIONS[placement[0]][0] - - -def read_pages(text_dir): - """ - The pages of a text build in llms.txt order, without the pages that are left out - """ - text_dir = Path(text_dir) - pages = [] - for path in text_dir.rglob("*.txt"): - docname = path.relative_to(text_dir).with_suffix("").as_posix() - if _placement(docname) is None: - continue - text = path.read_text(encoding="utf-8") - lines = text.splitlines() - pages.append(Page(docname, lines[0].strip() if lines else docname, text)) - - return sorted(pages, key=lambda page: (_placement(page.docname), page.docname)) - - -def _summary(pages): - """ - The first paragraph of the front page that is not an image - """ - for page in pages: - if page.docname == "index": - paragraphs = page.text.split("\n\n")[1:] - for paragraph in paragraphs: - lines = [line.strip() for line in paragraph.strip().splitlines()] - if lines and not lines[0].startswith("[image"): - return " ".join(lines) - raise RuntimeError("The text build has no front page (index.txt) with a summary paragraph") - - -def render_llms_txt(pages, api=True): - """ - The content of llms.txt - - :param api: Include the links to the API reference JSON files - """ - lines = ["# VUnit", "", f"> {_summary(pages)}", ""] - for title, _ in SECTIONS: - entries = [] - if title == API_SECTION and api: - entries += [f"- [{name}]({BASE_URL}/{path})" for name, path in API_FILES] - entries += [ - f"- [{page.title}]({page.text_url}): {page.html_url}" for page in pages if section_of(page.docname) == title - ] - if entries: - lines += [f"## {title}", "", *entries, ""] - - return "\n".join(lines) - - -def render_llms_full_txt(pages): - """ - The content of llms-full.txt: all pages except the optional ones - """ - return "\n".join( - f"# {page.title}\n\nSource: {page.html_url}\n\n{page.text.rstrip()}\n" - for page in pages - if section_of(page.docname) != OPTIONAL_SECTION - ) - - -def write_llms_files(output_path, api=True): - """ - Write llms.txt and llms-full.txt to a documentation build with a text build in TEXT_DIR_NAME - - :param api: Include the links to the API reference JSON files - """ - output_path = Path(output_path) - pages = read_pages(output_path / TEXT_DIR_NAME) - (output_path / "llms.txt").write_text(render_llms_txt(pages, api), encoding="utf-8") - (output_path / "llms-full.txt").write_text(render_llms_full_txt(pages), encoding="utf-8") From 5d03c47e91f7d48fd80da9673e44ba15ee8e3eea Mon Sep 17 00:00:00 2001 From: Sebastian Hellgren Date: Tue, 15 Sep 2026 18:24:19 +0200 Subject: [PATCH 06/11] Generate llms-full.txt again tools/build_docs.py also runs the Sphinx text builder into a temporary directory and concatenates the pages into llms-full.txt, the front page first, leaving out the blog, release notes and other pages that do not document how to use VUnit. It is the only published form of the documentation that contains the included VHDL package headers as text, and it can be added as a Context7 source. llms.txt and the separate text pages are not generated. Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 3 ++- context7.json | 3 ++- docs/news.d/1222.doc.rst | 5 ++-- tests/unit/test_build_docs.py | 46 ++++++++++++++++++++++++++++++++ tools/build_docs.py | 50 +++++++++++++++++++++++++---------- 5 files changed, 89 insertions(+), 18 deletions(-) create mode 100644 tests/unit/test_build_docs.py diff --git a/AGENTS.md b/AGENTS.md index 30b521c32..297a6eb95 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,8 @@ Guidance for AI coding agents working on the VUnit repository. -To *use* VUnit rather than develop it, read the [documentation](https://vunit.github.io) and the machine-readable +To *use* VUnit rather than develop it, read the [documentation](https://vunit.github.io), all of it as plain text in +[llms-full.txt](https://vunit.github.io/llms-full.txt), and the machine-readable [VHDL API](https://vunit.github.io/api/vunit_lib.json) and [Python API](https://vunit.github.io/api/python.json). ## Repository layout diff --git a/context7.json b/context7.json index e3dc837e1..94fbe77b3 100644 --- a/context7.json +++ b/context7.json @@ -18,6 +18,7 @@ "A run script creates the project with VUnit.from_argv(), calls add_vhdl_builtins(), adds libraries and source files, and ends with main().", "A VHDL testbench has a runner_cfg generic and runs its test cases, each selected with run(\"\"), between test_runner_setup and test_runner_cleanup.", "Complete VHDL API signatures: https://vunit.github.io/api/vunit_lib.json", - "Complete Python API signatures: https://vunit.github.io/api/python.json" + "Complete Python API signatures: https://vunit.github.io/api/python.json", + "All documentation as plain text, including the VHDL package headers: https://vunit.github.io/llms-full.txt" ] } diff --git a/docs/news.d/1222.doc.rst b/docs/news.d/1222.doc.rst index cd4d30842..f3113ce10 100644 --- a/docs/news.d/1222.doc.rst +++ b/docs/news.d/1222.doc.rst @@ -1,2 +1,3 @@ -The website now publishes a machine-readable reference of the VHDL and Python APIs for AI agents -(``api/vunit_lib.json`` and ``api/python.json``). ``AGENTS.md`` and ``context7.json`` were added to the repository. +The website now publishes, for AI agents, all documentation as plain text in ``llms-full.txt`` and a machine-readable +reference of the VHDL and Python APIs (``api/vunit_lib.json`` and ``api/python.json``). ``AGENTS.md`` and +``context7.json`` were added to the repository. diff --git a/tests/unit/test_build_docs.py b/tests/unit/test_build_docs.py new file mode 100644 index 000000000..217a95f03 --- /dev/null +++ b/tests/unit/test_build_docs.py @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2014-2026, Lars Asplund lars.anders.asplund@gmail.com + +""" +Test the generation of llms-full.txt from the text build of the documentation +""" + +import unittest +from pathlib import Path +from tests.common import with_tempdir +from tools.build_docs import write_llms_full_txt + + +class TestLlmsFullTxt(unittest.TestCase): + """ + Test the generation of llms-full.txt + """ + + @with_tempdir + def test_concatenates_documentation_pages_front_page_first(self, tempdir): + text_path = Path(tempdir) / "text" + pages = { + "index": "VUnit\n*****\n\nIntro.\n", + "cli": "Command Line Interface\n**********************\n\nOptions.\n", + "data_types/queue": "*queue* package\n***************\n\n package queue_pkg is\n", + "blog/2023_03_31_vunit_events": "VUnit Events\n************\n", + "genindex": "Index\n*****\n", + } + for docname, text in pages.items(): + path = text_path / f"{docname}.txt" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + write_llms_full_txt(text_path, Path(tempdir) / "llms-full.txt") + + self.assertEqual( + (Path(tempdir) / "llms-full.txt").read_text(encoding="utf-8"), + "# VUnit\n\nSource: https://vunit.github.io/index.html\n\nVUnit\n*****\n\nIntro.\n\n" + "# Command Line Interface\n\nSource: https://vunit.github.io/cli.html\n\n" + "Command Line Interface\n**********************\n\nOptions.\n\n" + "# *queue* package\n\nSource: https://vunit.github.io/data_types/queue.html\n\n" + "*queue* package\n***************\n\n package queue_pkg is\n", + ) diff --git a/tools/build_docs.py b/tools/build_docs.py index 3c64f4af2..50c2bace2 100644 --- a/tools/build_docs.py +++ b/tools/build_docs.py @@ -13,27 +13,49 @@ from subprocess import check_call from pathlib import Path from sys import argv, executable - +from tempfile import TemporaryDirectory ROOT = Path(__file__).parent.parent +# Pages that do not document how to use VUnit, left out of llms-full.txt +LLMS_FULL_EXCLUDED = ("about", "blog/", "contributing", "genindex", "license", "release_notes", "testimonials/") + + +def sphinx(builder, output_path, args): + """ + Build the documentation with a Sphinx builder + """ + check_call([executable, "-m", "sphinx"] + args + ["-TEWanb", builder, ROOT / "docs", output_path]) + + +def write_llms_full_txt(text_path, output_file): + """ + Concatenate the pages of a text build into one file for AI agents, the front page first + """ + pages = [] + for path in Path(text_path).rglob("*.txt"): + docname = path.relative_to(text_path).with_suffix("").as_posix() + if any(docname == name or (name.endswith("/") and docname.startswith(name)) for name in LLMS_FULL_EXCLUDED): + continue + text = path.read_text(encoding="utf-8") + title = text.splitlines()[0] + pages.append((docname, f"# {title}\n\nSource: https://vunit.github.io/{docname}.html\n\n{text.rstrip()}\n")) + + pages.sort(key=lambda page: (page[0] != "index", page[0])) + Path(output_file).write_text("\n".join(page for _, page in pages), encoding="utf-8") + def main(): """ - Build documentation/website + Build documentation/website, and all pages as plain text in llms-full.txt for AI agents """ - check_call( - [ - executable, - "-m", - "sphinx" - ] + ([] if len(argv) < 2 else argv[2:]) + [ - "-TEWanb", - "html", - Path(__file__).parent.parent / "docs", - argv[1], - ] - ) + output_path = Path(argv[1]) + args = argv[2:] + sphinx("html", output_path, args) + with TemporaryDirectory() as temp_path: + text_path = Path(temp_path) / "text" + sphinx("text", text_path, args + ["-d", str(Path(temp_path) / "doctrees")]) + write_llms_full_txt(text_path, output_path / "llms-full.txt") if __name__ == "__main__": From f2359a2a80cf680a78367f1cef7092e51fc3d6d8 Mon Sep 17 00:00:00 2001 From: Sebastian Hellgren Date: Tue, 15 Sep 2026 19:03:01 +0200 Subject: [PATCH 07/11] Generate the VHDL API reference with vhdl-dump-ast instead of GHDL The VHDL symbol tables are extracted from the syntax trees that vhdl-dump-ast 0.1.0 (VHDL-LS project) dumps per file, including comments. This replaces the GHDL compile, --file-to-xml and the line/column based slicing of the source text: types, defaults and doc comments come directly from the tokens, and the docs build needs no compile. The version is pinned, and a different vhdl-dump-ast version fails the build. Compared with the GHDL based output, the package set and every source-declared subprogram are the same. The declarations VHDL creates implicitly (predefined operators, minimum/maximum and the aliases of implicit operators) are no longer listed, and parameters sharing a type in one declaration all get the type, which GHDL's XML lost for all but the first. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/docs.yml | 11 +- AGENTS.md | 2 +- tests/unit/test_api_reference.py | 1153 ++++++++++++++++-------------- tools/build_api_reference.py | 865 +++++++++++----------- 4 files changed, 1046 insertions(+), 985 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8d277921a..24ab185ce 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -24,12 +24,11 @@ jobs: with: python-version: '3.13' - # GHDL generates the machine-readable VHDL API reference (tools/build_api_reference.py) - - name: '⚙️ Setup GHDL' - uses: ghdl/setup-ghdl@main - with: - version: v6.0.0 - backend: mcode + # vhdl-dump-ast generates the machine-readable VHDL API reference (tools/build_api_reference.py) + - name: '⚙️ Setup vhdl-dump-ast' + run: | + cargo install vhdl-dump-ast --version 0.1.0 --locked + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - name: Install dependencies run: | diff --git a/AGENTS.md b/AGENTS.md index 297a6eb95..2f9ed999c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,7 +23,7 @@ To *use* VUnit rather than develop it, read the [documentation](https://vunit.gi - **Verification components:** `tox -e py313-vcomponents-`. - **Documentation:** `tox -e py313-docs`. - The build needs the git tags for the release notes. - - It needs GHDL for the API reference. Set `VUNIT_DOCS_SKIP_API=1` to build without it. + - It needs `vhdl-dump-ast` for the API reference (`cargo install vhdl-dump-ast --version 0.1.0 --locked`). Set `VUNIT_DOCS_SKIP_API=1` to build without it. - **All tox environments:** `tox -l`. ## Rules diff --git a/tests/unit/test_api_reference.py b/tests/unit/test_api_reference.py index 3b6df229e..e5cebc844 100644 --- a/tests/unit/test_api_reference.py +++ b/tests/unit/test_api_reference.py @@ -7,179 +7,384 @@ """ Test of the machine-readable VHDL and Python API reference generator. -Uses small, inline GHDL XML documents together with matching VHDL source -text (no GHDL invoked) and a small Python module defined in this file (no -Sphinx build). Columns are computed from the same source-text literals with -str.index() rather than hand-counted, so a typo in the fixture cannot make -a test pass for the wrong reason. +Uses small, inline vhdl-dump-ast syntax trees (no vhdl-dump-ast binary +invoked -- unit CI has no cargo toolchain) and a small Python module +defined in this file (no Sphinx build). The tree shapes mirror what +"vhdl-dump-ast --trivia --no-pretty" actually produces, verified by hand +against real output from vunit/vhdl/**/src/*.vhd during implementation. """ +import copy import sys import unittest from pathlib import Path -from xml.etree import ElementTree from tools import build_api_reference as bar +# --------------------------------------------------------------------------- +# Tiny syntax-tree builders, matching vhdl-dump-ast's JSON shape: +# Node: {"Node": {"kind": ..., "children": [...]}} +# Token: {"Token": {"kind": ..., "text": ..., "leading_trivia": [...]}} +# --------------------------------------------------------------------------- -def _col(line_text, needle): - """1-based column of the first occurrence of needle in line_text.""" - return line_text.index(needle) + 1 +SPACE = [{"Spaces": 1}] +NONE = [] -class TestSourceTextSlicing(unittest.TestCase): - """ - SourceText.slice_from: delimiters, bracket depth, string and character - literals, and the "no closing delimiter" error. - """ +def node(kind, *children): + return {"Node": {"kind": kind, "children": list(children)}} - def test_slices_up_to_semicolon(self): - source = bar.SourceText(" integer;\n") - self.assertEqual(source.slice_from(1, 3, "f.vhd", 1, allow_assign=True), "integer") - def test_stops_at_assign_for_types(self): - source = bar.SourceText("positive := 1;\n") - self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=True), "positive") +def token(kind, text, trivia=SPACE): + return {"Token": {"kind": kind, "text": text, "leading_trivia": copy.deepcopy(trivia)}} - def test_does_not_stop_at_assign_for_values(self): - # A default value never itself contains ":=", but allow_assign=False - # must not treat ":" specially either. - source = bar.SourceText('a & ":" & b;\n') - self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=False), 'a & ":" & b') - def test_balances_parentheses_before_stopping_at_semicolon(self): - source = bar.SourceText("array (natural range <>) of rec_t;\n") - self.assertEqual( - source.slice_from(1, 1, "f.vhd", 1, allow_assign=True), - "array (natural range <>) of rec_t", - ) +def kw(name, trivia=SPACE): + """A keyword token, e.g. kw("Procedure") -> Token({"Keyword": "Procedure"}, "procedure").""" + return token({"Keyword": name}, name.lower(), trivia) - def test_stray_close_paren_ends_a_parameter_slice(self): - # As found inside "(name : positive := 1)" -- the type/default slice - # for the last parameter in a list must stop at the enclosing ")". - source = bar.SourceText("positive := 1)\n") - self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=False), "positive := 1") - - def test_collapses_whitespace_to_single_spaces(self): - source = bar.SourceText("integer\n range\n 0 to 15;\n") - self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=True), "integer range 0 to 15") - - def test_comments_neither_delimit_nor_belong_to_the_slice(self): - source = bar.SourceText("natural -- width; (in bits)\n range 0 to 8 := 8;\n") - self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=True), "natural range 0 to 8") - # A trailing comment on the last parameter of a list, before the closing parenthesis - source = bar.SourceText("8 -- default width )\n );\n") - self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=False), "8") - - def test_string_literal_semicolon_is_not_a_delimiter(self): - source = bar.SourceText('string := "a;b";\n') - self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=True), "string") - # And the value itself, starting after ":=", keeps the semicolon. - self.assertEqual(bar.SourceText('"a;b";\n').slice_from(1, 1, "f.vhd", 1, allow_assign=False), '"a;b"') - - def test_escaped_quote_inside_string_literal(self): - source = bar.SourceText('"a""b";\n') - self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=False), '"a""b"') - - def test_character_literal_semicolon_is_not_a_delimiter(self): - # A ';' character literal used as a default value: the semicolon - # inside the literal must not end the slice. - source = bar.SourceText("character := ';';\n") - self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=False), "character := ';'") - - def test_attribute_tick_is_not_mistaken_for_a_character_literal(self): - source = bar.SourceText("integer range -1 to integer'high;\n") - self.assertEqual(source.slice_from(1, 1, "f.vhd", 1, allow_assign=True), "integer range -1 to integer'high") - - def test_no_delimiter_raises(self): - source = bar.SourceText("integer\n") - with self.assertRaises(bar.ApiReferenceError) as ctx: - source.slice_from(1, 1, "f.vhd", 7, allow_assign=True) - self.assertIn("f.vhd:7", str(ctx.exception)) - def test_tab_stop_column_mapping(self): - # A real gotcha: GHDL reports columns with tab stops of 8, like a - # terminal, not one column per character. - source = bar.SourceText('\tname : string := "";\n') - # Column 9 is where "name" starts after one leading tab; slicing - # from there (stopping at ":=") proves the start position is right. - self.assertEqual(source.slice_from(1, 9, "f.vhd", 1, allow_assign=True), "name : string") +def ident(text, trivia=SPACE): + return token("Identifier", text, trivia) -class TestDocComments(unittest.TestCase): - """ - The contiguous "--" comment block directly above a declaration, doc - comment termination on a blank line, and the license-header exclusion - for a package's own doc. - """ +def op_name(text, trivia=SPACE): + return token("StringLiteral", f'"{text}"', trivia) - def test_extracts_comment_block_directly_above(self): - text = " -- Returns the length\n -- of the queue\n function length return natural;\n" - source = bar.SourceText(text) - self.assertEqual(bar.comment_block_above(source, 3), "Returns the length\nof the queue") - def test_blank_line_ends_the_block(self): - text = " -- Unrelated comment\n\n function length return natural;\n" - source = bar.SourceText(text) - self.assertIsNone(bar.comment_block_above(source, 3)) +def punct(kind, text, trivia=NONE): + return token(kind, text, trivia) + + +def identifier_list(*names, first_trivia=SPACE): + kids = [] + for i, name in enumerate(names): + kids.append(ident(name, trivia=first_trivia if i == 0 else NONE)) + if i < len(names) - 1: + kids.append(punct("Comma", ",")) + return node("IdentifierList", *kids) + + +def simple_name(text, trivia=SPACE): + return node("Name", node("NameDesignatorPrefix", ident(text, trivia))) + + +def subtype(name, *extra, trivia=SPACE): + return node("SubtypeIndication", simple_name(name, trivia), *extra) + + +def initial_value(expr, trivia=SPACE): + return node("InitialValue", punct("ColonEq", ":=", trivia), expr) + + +def literal(text, literal_kind="AbstractLiteral", trivia=SPACE): + return node("LiteralExpression", token(literal_kind, text, trivia)) + + +def parameter(names, subtype_node, class_kw=None, mode_kw=None, default=None, is_file=False): + kids = [] + if class_kw is not None: + kids.append(kw(class_kw)) + id_list_trivia = NONE if class_kw is not None else SPACE + kids.append( + identifier_list(*names, first_trivia=id_list_trivia) + if isinstance(names, (list, tuple)) + else identifier_list(names, first_trivia=id_list_trivia) + ) + kids.append(punct("Colon", ":", SPACE)) + if mode_kw is not None: + kids.append(kw(mode_kw)) + kids.append(subtype_node) + if default is not None: + kids.append(default) + return node("InterfaceFileDeclaration" if is_file else "InterfaceObjectDeclaration", *kids) + + +def parameter_list(*params): + if not params: + return None + kids = [] + for i, param in enumerate(params): + kids.append(param) + if i < len(params) - 1: + kids.append(punct("SemiColon", ";")) + return node( + "ParameterList", + node( + "ParenthesizedInterfaceList", + punct("LeftPar", "(", SPACE), + node("InterfaceList", *kids), + punct("RightPar", ")", NONE), + ), + ) + + +def procedure_decl(name, params=(), doc_trivia=NONE, operator=False): + name_tok = op_name(name) if operator else ident(name) + kids = [kw("Procedure", trivia=doc_trivia)] + kids.append(name_tok) + plist = parameter_list(*params) + if plist is not None: + kids.append(plist) + spec = node("ProcedureSpecification", *kids) + return node("SubprogramDeclaration", spec, punct("SemiColon", ";")) + + +def function_decl(name, params=(), return_type="natural", impure=False, doc_trivia=NONE, operator=False): + name_tok = op_name(name) if operator else ident(name) + kids = [] + if impure: + kids.append(kw("Impure", trivia=doc_trivia)) + kids.append(kw("Function")) + else: + kids.append(kw("Function", trivia=doc_trivia)) + kids.append(name_tok) + plist = parameter_list(*params) + if plist is not None: + kids.append(plist) + kids.append(kw("Return")) + kids.append(simple_name(return_type)) + spec = node("FunctionSpecification", *kids) + return node("SubprogramDeclaration", spec, punct("SemiColon", ";")) + + +def alias_decl(name, target, signature_types=(), return_type=None, has_signature=False): + kids = [kw("Alias"), ident(name), kw("Is"), simple_name(target)] + if has_signature: + type_mark_kids = [] + for i, t in enumerate(signature_types): + type_mark_kids.append(simple_name(t, trivia=SPACE if i == 0 else NONE)) + if i < len(signature_types) - 1: + type_mark_kids.append(punct("Comma", ",")) + if return_type is not None: + type_mark_kids.append(node("ReturnType", kw("Return"), simple_name(return_type))) + sig_kids = [ + punct("LeftSquare", "[", SPACE), + node("TypeMarkList", *type_mark_kids), + punct("RightSquare", "]", NONE), + ] + kids.append(node("Signature", *sig_kids)) + kids.append(punct("SemiColon", ";")) + return node("AliasDeclaration", *kids) + + +def constant_decl(names, subtype_node, default=None, doc_trivia=NONE): + kids = [ + kw("Constant", trivia=doc_trivia), + identifier_list(*names) if isinstance(names, (list, tuple)) else identifier_list(names), + ] + kids.append(punct("Colon", ":", SPACE)) + kids.append(subtype_node) + if default is not None: + kids.append(default) + kids.append(punct("SemiColon", ";")) + return node("ConstantDeclaration", *kids) + + +def subtype_decl(name, subtype_node, doc_trivia=NONE): + return node( + "SubtypeDeclaration", + kw("Subtype", trivia=doc_trivia), + ident(name), + kw("Is"), + subtype_node, + punct("SemiColon", ";"), + ) + + +def object_decl(decl_kind, names, subtype_node, doc_trivia=NONE, shared=False): + kids = [] + if shared: + kids.append(kw("Shared", trivia=doc_trivia)) + kids.append(kw("Variable")) + else: + first_kw = {"SignalDeclaration": "Signal", "VariableDeclaration": "Variable", "FileDeclaration": "File"}[ + decl_kind + ] + kids.append(kw(first_kw, trivia=doc_trivia)) + kids.append(identifier_list(*names) if isinstance(names, (list, tuple)) else identifier_list(names)) + kids.append(punct("Colon", ":", SPACE)) + kids.append(subtype_node) + kids.append(punct("SemiColon", ";")) + return node(decl_kind, *kids) + + +def full_type_decl(name, def_node, doc_trivia=NONE): + return node( + "FullTypeDeclaration", kw("Type", trivia=doc_trivia), ident(name), kw("Is"), def_node, punct("SemiColon", ";") + ) + + +def record_element(names, subtype_node): + kids = [ + identifier_list(*names) if isinstance(names, (list, tuple)) else identifier_list(names), + punct("Colon", ":", SPACE), + subtype_node, + punct("SemiColon", ";"), + ] + return node("ElementDeclaration", *kids) + + +def record_type(*elements): + return node( + "RecordTypeDefinition", + node("RecordTypeDefinitionPreamble", kw("Record")), + node("RecordElementDeclarations", *elements), + node("RecordTypeDefinitionEpilogue", kw("End"), kw("Record")), + ) + + +def enumeration_type(*literals): + list_kids = [] + for i, lit in enumerate(literals): + lit_kind = "CharacterLiteral" if lit.startswith("'") else "Identifier" + list_kids.append(token(lit_kind, lit, trivia=SPACE if i == 0 else NONE)) + if i < len(literals) - 1: + list_kids.append(punct("Comma", ",")) + return node( + "EnumerationTypeDefinition", + punct("LeftPar", "(", SPACE), + node("EnumerationList", *list_kids), + punct("RightPar", ")", NONE), + ) + + +def protected_type(*members): + return node( + "ProtectedTypeDeclaration", + node("ProtectedPreamble", kw("Protected")), + node("ProtectedTypeDeclarativePart", *members), + ) + + +def other_type(*tokens_or_nodes): + return node("UnboundedArrayDefinition", *tokens_or_nodes) + + +def package_tree(name, decls=(), preamble_trivia=NONE, name_body=""): + preamble = node("PackagePreamble", kw("Package", trivia=preamble_trivia), ident(name), kw("Is")) + decl_part = node("PackageDeclarativePart", *decls) + epilogue = node("PackageEpilogue", kw("End"), kw("Package"), punct("SemiColon", ";")) + pkg = node("PackageDeclaration", preamble, decl_part, epilogue) + return {"kind": "DesignFile", "children": [pkg]} + + +def comment_trivia(*lines, blank_before=False, leading=SPACE): + """leading_trivia for a token immediately preceded by one or more '--' comment lines.""" + trivia = [] + if blank_before: + trivia.append({"LineFeeds": 2}) + for i, line in enumerate(lines): + if i > 0: + trivia.append({"LineFeeds": 1}) + trivia.append({"LineComment": {"encoding": "utf-8", "value": list(f"-- {line}".encode())}}) + trivia.append({"LineFeeds": 1}) + trivia.extend(leading) + return trivia + + +# --------------------------------------------------------------------------- +# Low-level helpers: text_of, doc comments, line numbers +# --------------------------------------------------------------------------- + + +class TestTextOf(unittest.TestCase): + def test_joins_tokens_with_space_only_where_trivia_present(self): + # "natural range 0 to 3" -- every token here is separated by a real space + tree = subtype( + "natural", + node("RangeConstraint", kw("Range"), node("BinaryExpression", literal("0"), kw("To"), literal("3"))), + ) + self.assertEqual(bar.text_of(tree), "natural range 0 to 3") + + def test_dotted_name_has_no_spaces(self): + # "ieee.numeric_bit.unsigned" -- dots and following identifiers carry no leading trivia + tree = node( + "Name", + node("NameDesignatorPrefix", ident("ieee")), + node("SelectedName", punct("Dot", ".", NONE), token("Identifier", "numeric_bit", NONE)), + node("SelectedName", punct("Dot", ".", NONE), token("Identifier", "unsigned", NONE)), + ) + self.assertEqual(bar.text_of(tree), "ieee.numeric_bit.unsigned") + + def test_joins_multiple_elements(self): + a = node("Node", ident("a")) + b = node("Node", ident("b", trivia=SPACE)) + self.assertEqual(bar.text_of(a, b), "a b") + - def test_no_comment_above_gives_none(self): - text = " use work.foo_pkg.all;\n function length return natural;\n" - source = bar.SourceText(text) - self.assertIsNone(bar.comment_block_above(source, 2)) +class TestDocComments(unittest.TestCase): + def test_comment_block_directly_above(self): + decl = procedure_decl("p", doc_trivia=comment_trivia("Returns the length", "of the queue")) + self.assertEqual(bar.leading_doc(decl), "Returns the length\nof the queue") + + def test_blank_line_ends_the_block(self): + decl = procedure_decl("p", doc_trivia=comment_trivia("Unrelated comment", blank_before=True)) + # A blank line right after the comment (before the declaration) means + # there is no comment directly above: nothing left after the reset. + trivia = [ + {"LineComment": {"encoding": "utf-8", "value": list(b"-- Unrelated comment")}}, + {"LineFeeds": 2}, + ] + SPACE + decl = procedure_decl("p", doc_trivia=trivia) + self.assertIsNone(bar.leading_doc(decl)) + + def test_no_comment_gives_none(self): + decl = procedure_decl("p") + self.assertIsNone(bar.leading_doc(decl)) def test_license_header_excluded_from_package_doc(self): - text = ( - "-- This Source Code Form is subject to the terms of the Mozilla Public\n" - "-- License, v. 2.0.\n" - "package my_pkg is\n" + preamble_trivia = comment_trivia( + "This Source Code Form is subject to the terms of the Mozilla Public", "License, v. 2.0." ) - source = bar.SourceText(text) - self.assertIsNone(bar.package_doc(source, 3)) + preamble = node("PackagePreamble", kw("Package", trivia=preamble_trivia), ident("my_pkg"), kw("Is")) + self.assertIsNone(bar.package_doc(preamble)) def test_non_license_comment_kept_for_package_doc(self): - text = "-- A package for testing\npackage my_pkg is\n" - source = bar.SourceText(text) - self.assertEqual(bar.package_doc(source, 2), "A package for testing") + preamble_trivia = comment_trivia("A package for testing") + preamble = node("PackagePreamble", kw("Package", trivia=preamble_trivia), ident("my_pkg"), kw("Is")) + self.assertEqual(bar.package_doc(preamble), "A package for testing") + + +class TestLineNumbers(unittest.TestCase): + def test_line_advances_on_line_feeds(self): + tree = { + "kind": "DesignFile", + "children": [ + node( + "X", + ident("a", trivia=NONE), + ident("b", trivia=[{"LineFeeds": 1}]), + ident("c", trivia=[{"LineFeeds": 2}]), + ), + ], + } + lines = bar.annotate_lines(tree) + toks = list(bar._tokens(tree)) + self.assertEqual([lines[id(bar.token_of(t))] for t in toks], [1, 2, 4]) - def test_impure_on_a_preceding_line_shifts_the_doc_anchor(self): - text = " -- Returns something\n impure\n function get return integer;\n" - source = bar.SourceText(text) - self.assertEqual(bar.comment_block_above(source, bar.subprogram_first_line(source, 3)), "Returns something") + def test_line_number_of_declaration_name(self): + decl = procedure_decl("p", doc_trivia=[{"LineFeeds": 3}] + SPACE) + tree = package_tree("my_pkg", decls=[decl]) + lines = bar.annotate_lines(tree) + pkg = bar.find_all(tree, "PackageDeclaration")[0] + package = bar.convert_package(pkg, lines, "f.vhd", "f.vhd") + self.assertEqual(package["subprograms"][0]["line"], 4) - def test_impure_on_the_same_line_does_not_pull_in_the_comment_above(self): - # Here the comment belongs to something else entirely -- decl_line - # already holds "impure function", so the search starts one line - # above *that*, which is blank. - text = " -- Unrelated\n\n impure function get return integer;\n" - source = bar.SourceText(text) - self.assertIsNone(bar.comment_block_above(source, bar.subprogram_first_line(source, 3))) +# --------------------------------------------------------------------------- +# Subprograms +# --------------------------------------------------------------------------- -class TestConvertSubprogram(unittest.TestCase): - """ - procedure/function, pure/impure, operators, parameter classes/modes/ - defaults, constrained parameter types, return types. - """ - def _parse(self, xml): - return ElementTree.fromstring(xml) +class TestConvertSubprogram(unittest.TestCase): + def _lines_for(self, tree): + return bar.annotate_lines(tree) def test_procedure_has_no_pure_flag(self): - line = " procedure proc_set (target : in rec_t);" - xml = f""" - - - - - - - - """ - el = self._parse(xml) - source = bar.SourceText(line + "\n") - entry = bar.convert_subprogram(el, source, "f.vhd") + decl = procedure_decl("proc_set", params=[parameter(["target"], subtype("rec_t"))]) + tree = {"kind": "DesignFile", "children": [decl]} + entry = bar.convert_subprogram(decl, self._lines_for(tree)) self.assertEqual(entry["kind"], "procedure") self.assertIsNone(entry["pure"]) self.assertFalse(entry["operator"]) @@ -188,406 +393,330 @@ def test_procedure_has_no_pure_flag(self): self.assertEqual(entry["parameters"][0]["mode"], "in") def test_function_pure_flag_and_return_type(self): - for pure_flag, prefix in (("false", "impure "), ("true", "")): - with self.subTest(pure_flag=pure_flag): - line = f" {prefix}function func_get (target : rec_t) return integer;" - xml = f""" - - - - - """ - entry = bar.convert_subprogram(self._parse(xml), bar.SourceText(line + "\n"), "f.vhd") + for impure, expected_pure in ((True, False), (False, True)): + with self.subTest(impure=impure): + decl = function_decl( + "func_get", params=[parameter(["target"], subtype("rec_t"))], return_type="integer", impure=impure + ) + tree = {"kind": "DesignFile", "children": [decl]} + entry = bar.convert_subprogram(decl, self._lines_for(tree)) self.assertEqual(entry["kind"], "function") - self.assertEqual(entry["pure"], pure_flag == "true") + self.assertEqual(entry["pure"], expected_pure) self.assertEqual(entry["return_type"], "integer") def test_operator_identifier_is_flagged(self): - line = ' function "+" (l, r : rec_t) return rec_t;' - xml = f""" - - - - - """ - el = self._parse(xml) - source = bar.SourceText(line + "\n") - entry = bar.convert_subprogram(el, source, "f.vhd") + decl = function_decl("+", params=[parameter(["l", "r"], subtype("rec_t"))], return_type="rec_t", operator=True) + tree = {"kind": "DesignFile", "children": [decl]} + entry = bar.convert_subprogram(decl, self._lines_for(tree)) self.assertTrue(entry["operator"]) self.assertEqual(entry["name"], "+") + self.assertEqual(len(entry["parameters"]), 2) def test_parameter_default_value(self): - line = " value : in integer := 0" - xml = f""" - - - - - - - - - """ - el = self._parse(xml) - source = bar.SourceText(line + ";\n") - entry = bar.convert_subprogram(el, source, "f.vhd") + default = initial_value(literal("0")) + decl = procedure_decl( + "proc_set", params=[parameter(["value"], subtype("integer"), mode_kw="In", default=default)] + ) + tree = {"kind": "DesignFile", "children": [decl]} + entry = bar.convert_subprogram(decl, self._lines_for(tree)) self.assertEqual(entry["parameters"][0]["default"], "0") def test_signal_and_variable_parameter_classes(self): - line = " signal net : inout t; variable pass : out boolean" - xml = f""" - - - - - - - - - - - """ - el = self._parse(xml) - source = bar.SourceText(line + ";\n") - entry = bar.convert_subprogram(el, source, "f.vhd") + params = [ + parameter(["net"], subtype("t"), class_kw="Signal", mode_kw="Inout"), + parameter(["pass_"], subtype("boolean"), class_kw="Variable", mode_kw="Out"), + ] + decl = procedure_decl("p", params=params) + tree = {"kind": "DesignFile", "children": [decl]} + entry = bar.convert_subprogram(decl, self._lines_for(tree)) self.assertEqual(entry["parameters"][0]["class"], "signal") self.assertEqual(entry["parameters"][0]["mode"], "inout") self.assertEqual(entry["parameters"][1]["class"], "variable") + def test_out_mode_defaults_to_variable_class(self): + # VHDL rule: an omitted object class defaults to "constant" for mode + # "in", but to "variable" for "out"/"inout". + decl = procedure_decl("p", params=[parameter(["stat"], subtype("checker_stat_t"), mode_kw="Out")]) + tree = {"kind": "DesignFile", "children": [decl]} + entry = bar.convert_subprogram(decl, self._lines_for(tree)) + self.assertEqual(entry["parameters"][0]["class"], "variable") + self.assertEqual(entry["parameters"][0]["mode"], "out") + + def test_file_parameter(self): + decl = procedure_decl("setup", params=[parameter(["f"], subtype("text"), is_file=True)]) + tree = {"kind": "DesignFile", "children": [decl]} + entry = bar.convert_subprogram(decl, self._lines_for(tree)) + param = entry["parameters"][0] + self.assertEqual(param["class"], "file") + self.assertEqual(param["mode"], "inout") + + def test_multi_identifier_parameter_shares_one_type(self): + decl = procedure_decl( + "notify", + params=[parameter(["event1", "event2"], subtype("any_event_t"), class_kw="Signal", mode_kw="Inout")], + ) + tree = {"kind": "DesignFile", "children": [decl]} + entry = bar.convert_subprogram(decl, self._lines_for(tree)) + self.assertEqual([p["name"] for p in entry["parameters"]], ["event1", "event2"]) + self.assertEqual([p["type"] for p in entry["parameters"]], ["any_event_t", "any_event_t"]) + def test_constrained_parameter_type(self): - line = " data : std_logic_vector(7 downto 0)" - xml = f""" - - - - - - - - """ - el = self._parse(xml) - source = bar.SourceText(line + ";\n") - entry = bar.convert_subprogram(el, source, "f.vhd") + subtype_node = subtype( + "std_logic_vector", + node( + "ConstraintExpression", + punct("LeftPar", "(", NONE), + literal("7", trivia=NONE), + kw("Downto"), + literal("0"), + punct("RightPar", ")", NONE), + ), + ) + decl = procedure_decl("p", params=[parameter(["data"], subtype_node)]) + tree = {"kind": "DesignFile", "children": [decl]} + entry = bar.convert_subprogram(decl, self._lines_for(tree)) self.assertEqual(entry["parameters"][0]["type"], "std_logic_vector(7 downto 0)") - def test_implicit_operator_parameter_without_subtype_indication(self): - # GHDL synthesizes "=" for every type; its parameters carry a type - # reference but no textual subtype_indication (nothing was written). - xml = """ - - - - - - - - - """ - el = self._parse(xml) - source = bar.SourceText("\n") - entry = bar.convert_subprogram(el, source, "f.vhd") - self.assertTrue(entry["operator"]) - self.assertIsNone(entry["parameters"][0]["type"]) - self.assertEqual(entry["parameters"][0]["name"], "") - - def test_unhandled_parameter_kind_raises(self): - xml = """ - - - - - - """ - el = self._parse(xml) - source = bar.SourceText("\n") - with self.assertRaises(bar.ApiReferenceError) as ctx: - bar.convert_subprogram(el, source, "f.vhd") - self.assertIn("interface_mystery_declaration", str(ctx.exception)) + def test_doc_comment(self): + decl = procedure_decl("p", doc_trivia=comment_trivia("Does a thing.")) + tree = {"kind": "DesignFile", "children": [decl]} + entry = bar.convert_subprogram(decl, self._lines_for(tree)) + self.assertEqual(entry["doc"], "Does a thing.") + + +# --------------------------------------------------------------------------- +# Aliases +# --------------------------------------------------------------------------- class TestConvertAlias(unittest.TestCase): - """Aliases with signatures and return types, including object aliases.""" + def _entry(self, decl): + tree = {"kind": "DesignFile", "children": [decl]} + return bar.convert_alias(decl, bar.annotate_lines(tree))[0] def test_subprogram_alias_with_signature_and_return_type(self): - xml = """ - - - - - - - - - - """ - el = ElementTree.fromstring(xml) - entry = bar.convert_alias(el) + decl = alias_decl("alias_get", "func_get", signature_types=["rec_t"], return_type="integer", has_signature=True) + entry = self._entry(decl) self.assertEqual(entry["name"], "alias_get") self.assertEqual(entry["target"], "func_get") self.assertEqual(entry["signature"], ["rec_t"]) self.assertEqual(entry["return_type"], "integer") - self.assertNotIn("doc", entry) def test_procedure_alias_has_no_return_type(self): - xml = """ - - - - - - - - - - """ - el = ElementTree.fromstring(xml) - entry = bar.convert_alias(el) + decl = alias_decl("alias_set", "proc_set", signature_types=["rec_t", "integer"], has_signature=True) + entry = self._entry(decl) self.assertEqual(entry["signature"], ["rec_t", "integer"]) self.assertIsNone(entry["return_type"]) - def test_object_alias_has_no_signature(self): - # "alias null_byte_vector_ptr is null_string_ptr;" -- an - # object_alias_declaration, aliasing a constant, not a subprogram. - xml = """ - - - - """ - el = ElementTree.fromstring(xml) - entry = bar.convert_alias(el) + def test_type_alias_has_no_signature(self): + decl = alias_decl("null_vec_ptr", "null_ptr") + entry = self._entry(decl) self.assertEqual(entry["target"], "null_ptr") self.assertEqual(entry["signature"], []) self.assertIsNone(entry["return_type"]) - def test_dotted_selected_name_target(self): - xml = """ - - - - - - """ - el = ElementTree.fromstring(xml) - entry = bar.convert_alias(el) - self.assertEqual(entry["target"], "work.push") + +# --------------------------------------------------------------------------- +# Types, subtypes, constants, objects +# --------------------------------------------------------------------------- class TestConvertType(unittest.TestCase): - """Records, enumerations, protected types and their methods, other types.""" + def _entry(self, decl): + tree = {"kind": "DesignFile", "children": [decl]} + return bar.convert_type(decl, bar.annotate_lines(tree), "f.vhd")[0] def test_record_type(self): - line = " a : integer;" - xml = f""" - - - - - - - - - - """ - el = ElementTree.fromstring(xml) - source = bar.SourceText(line + "\n") - entry = bar.convert_type(el, source, "f.vhd") + decl = full_type_decl("rec_t", record_type(record_element(["a"], subtype("integer")))) + entry = self._entry(decl) self.assertEqual(entry["kind"], "record") self.assertEqual(entry["elements"], [{"name": "a", "type": "integer"}]) + def test_record_type_multiple_identifiers_per_element(self): + decl = full_type_decl("rec_t", record_type(record_element(["a", "b"], subtype("integer")))) + entry = self._entry(decl) + self.assertEqual(entry["elements"], [{"name": "a", "type": "integer"}, {"name": "b", "type": "integer"}]) + def test_enumeration_type(self): - xml = """ - - - - - - - - - - """ - el = ElementTree.fromstring(xml) - entry = bar.convert_type(el, bar.SourceText("\n"), "f.vhd") + decl = full_type_decl("color_t", enumeration_type("red", "green", "blue")) + entry = self._entry(decl) self.assertEqual(entry["kind"], "enumeration") self.assertEqual(entry["literals"], ["red", "green", "blue"]) def test_protected_type_with_methods(self): - line = " procedure set (value : integer);" - xml = f""" - - - - - - - - - - - - - - """ - el = ElementTree.fromstring(xml) - source = bar.SourceText(line + "\n") - entry = bar.convert_type(el, source, "f.vhd") + method = procedure_decl("set", params=[parameter(["value"], subtype("integer"))]) + decl = full_type_decl("protected_t", protected_type(method)) + entry = self._entry(decl) self.assertEqual(entry["kind"], "protected") self.assertEqual(len(entry["subprograms"]), 1) - method = entry["subprograms"][0] - self.assertEqual(method["name"], "set") - self.assertIsNone(method["doc"]) + self.assertEqual(entry["subprograms"][0]["name"], "set") + self.assertIsNone(entry["subprograms"][0]["doc"]) + + def test_protected_type_method_doc(self): + method = procedure_decl( + "set", params=[parameter(["value"], subtype("integer"))], doc_trivia=comment_trivia("Sets the value.") + ) + decl = full_type_decl("protected_t", protected_type(method)) + entry = self._entry(decl) + self.assertEqual(entry["subprograms"][0]["doc"], "Sets the value.") + + def test_protected_type_unhandled_member_raises(self): + bogus_member = node("ConstantDeclaration", kw("Constant"), ident("x"), punct("SemiColon", ";")) + decl = full_type_decl("protected_t", protected_type(bogus_member)) + with self.assertRaises(bar.ApiReferenceError) as ctx: + self._entry(decl) + self.assertIn("ConstantDeclaration", str(ctx.exception)) def test_other_types(self): - for line, keyword, definition in ( - (" type vec_t is array (natural range <>) of rec_t;", "array", "array (natural range <>) of rec_t"), - (" type ptr_t is access rec_t;", "access", "access rec_t"), - ): - with self.subTest(keyword=keyword): - xml = f""" - - - - """ - entry = bar.convert_type(ElementTree.fromstring(xml), bar.SourceText(line + "\n"), "f.vhd") - self.assertEqual(entry["kind"], "other") - self.assertEqual(entry["definition"], definition) + array_def = other_type( + kw("Array"), + punct("LeftPar", "(", SPACE), + kw("Natural", trivia=NONE), + kw("Range"), + punct("Box", "<>", SPACE), + punct("RightPar", ")", NONE), + kw("Of"), + simple_name("rec_t"), + ) + decl = full_type_decl("vec_t", array_def) + entry = self._entry(decl) + self.assertEqual(entry["kind"], "other") + self.assertEqual(entry["definition"], "array (natural range <>) of rec_t") class TestConvertSubtypeConstantObject(unittest.TestCase): + def _lines(self, decl): + return bar.annotate_lines({"kind": "DesignFile", "children": [decl]}) + def test_subtype(self): - line = " subtype small_t is integer range 0 to 15;" - xml = f""" - - - - """ - el = ElementTree.fromstring(xml) - source = bar.SourceText(line + "\n") - entry = bar.convert_subtype(el, source, "f.vhd") + rng = node("RangeConstraint", kw("Range"), node("BinaryExpression", literal("0"), kw("To"), literal("15"))) + decl = subtype_decl("small_t", subtype("integer", rng)) + entry = bar.convert_subtype(decl, self._lines(decl))[0] self.assertEqual(entry["definition"], "integer range 0 to 15") def test_constant_with_value(self): - line = " constant c_max : integer := 15;" - xml = f""" - - - - - """ - el = ElementTree.fromstring(xml) - source = bar.SourceText(line + "\n") - entry = bar.convert_constant(el, source, "f.vhd") + decl = constant_decl(["c_max"], subtype("integer"), default=initial_value(literal("15"))) + entry = bar.convert_constant(decl, self._lines(decl))[0] self.assertEqual(entry["type"], "integer") self.assertEqual(entry["value"], "15") def test_deferred_constant_has_no_value(self): - line = " constant c_deferred : boolean;" - xml = f""" - - - - """ - el = ElementTree.fromstring(xml) - source = bar.SourceText(line + "\n") - entry = bar.convert_constant(el, source, "f.vhd") + decl = constant_decl(["c_deferred"], subtype("boolean")) + entry = bar.convert_constant(decl, self._lines(decl))[0] self.assertIsNone(entry["value"]) + def test_constant_multiple_identifiers(self): + decl = constant_decl(["a", "b"], subtype("integer"), default=initial_value(literal("1"))) + entries = bar.convert_constant(decl, self._lines(decl)) + self.assertEqual([e["name"] for e in entries], ["a", "b"]) + self.assertEqual([e["value"] for e in entries], ["1", "1"]) + def test_objects(self): - for line, kind, object_class, type_name in ( - (" signal s_enabled : std_logic := '1';", "signal_declaration", "signal", "std_logic"), - (" shared variable v_shared : integer;", "variable_declaration", "shared variable", "integer"), - (" file f0 : text;", "file_declaration", "file", "text"), + for decl_kind, object_class, shared in ( + ("SignalDeclaration", "signal", False), + ("VariableDeclaration", "shared variable", True), + ("FileDeclaration", "file", False), ): - with self.subTest(kind=kind): - xml = f""" - - - - """ - entry = bar.convert_object(ElementTree.fromstring(xml), bar.SourceText(line + "\n"), "f.vhd") + with self.subTest(kind=decl_kind): + decl = object_decl(decl_kind, ["x"], subtype("std_logic"), shared=shared) + entry = bar.convert_object(decl, self._lines(decl))[0] self.assertEqual(entry["class"], object_class) - self.assertEqual(entry["type"], type_name) + self.assertEqual(entry["type"], "std_logic") + def test_doc_comment_on_constant(self): + decl = constant_decl(["c"], subtype("integer"), doc_trivia=comment_trivia("A constant.")) + entry = bar.convert_constant(decl, self._lines(decl))[0] + self.assertEqual(entry["doc"], "A constant.") -class TestConvertPackage(unittest.TestCase): - """ - Scope: only declarations whose file equals the input file, the ignored - anonymous_type_declaration, the private-package flag, and the "unhandled - declaration kind" error. - """ - - def _package_xml(self, decl_chain_xml, name="my_pkg", line="7"): - return f""" - - - {decl_chain_xml} - - - """ - - def test_filters_declarations_by_file(self): - line8 = " constant local_c : integer := 1;" - xml = self._package_xml(f""" - - - - - - - """) - el = ElementTree.fromstring(xml) - source = bar.SourceText("\n" * 7 + line8 + "\n") - pkg = bar.convert_package(el, source, "/a/my_pkg.vhd", "my_pkg.vhd") - names = [c["name"] for c in pkg["constants"]] - self.assertEqual(names, ["local_c"]) - - def test_anonymous_type_declaration_is_ignored(self): - xml = self._package_xml(""" - - """) - el = ElementTree.fromstring(xml) - pkg = bar.convert_package(el, bar.SourceText("\n"), "/a/my_pkg.vhd", "my_pkg.vhd") - for bucket in ("types", "subtypes", "constants", "objects", "subprograms", "aliases"): - self.assertEqual(pkg[bucket], []) - def test_unhandled_declaration_kind_raises(self): - xml = self._package_xml(""" - - """) - el = ElementTree.fromstring(xml) - with self.assertRaises(bar.ApiReferenceError) as ctx: - bar.convert_package(el, bar.SourceText("\n"), "/a/my_pkg.vhd", "my_pkg.vhd") - self.assertIn("component_declaration", str(ctx.exception)) - self.assertIn("/a/my_pkg.vhd:9", str(ctx.exception)) +# --------------------------------------------------------------------------- +# Package assembly +# --------------------------------------------------------------------------- + +class TestConvertPackage(unittest.TestCase): def test_private_package_flag(self): for name, private in (("data_types_private_pkg", True), ("queue_pkg", False)): with self.subTest(name=name): - el = ElementTree.fromstring(self._package_xml("", name=name)) - pkg = bar.convert_package(el, bar.SourceText("\n"), "/a/my_pkg.vhd", "my_pkg.vhd") - self.assertEqual(pkg["private"], private) + tree = package_tree(name) + lines = bar.annotate_lines(tree) + pkg_node = bar.find_all(tree, "PackageDeclaration")[0] + package = bar.convert_package(pkg_node, lines, "f.vhd", "f.vhd") + self.assertEqual(package["private"], private) def test_p_prefixed_names_not_flagged_private(self): - line8 = " constant p_internal : integer := 1;" - xml = self._package_xml(f""" - - - - """) - el = ElementTree.fromstring(xml) - source = bar.SourceText("\n" * 7 + line8 + "\n") - pkg = bar.convert_package(el, source, "/a/my_pkg.vhd", "my_pkg.vhd") - self.assertFalse(pkg["private"]) - self.assertEqual(pkg["constants"][0]["name"], "p_internal") + decl = constant_decl(["p_internal"], subtype("integer"), default=initial_value(literal("1"))) + tree = package_tree("queue_pkg", decls=[decl]) + lines = bar.annotate_lines(tree) + pkg_node = bar.find_all(tree, "PackageDeclaration")[0] + package = bar.convert_package(pkg_node, lines, "f.vhd", "f.vhd") + self.assertFalse(package["private"]) + self.assertEqual(package["constants"][0]["name"], "p_internal") + def test_unhandled_declaration_kind_raises(self): + bogus = node("ComponentDeclaration", ident("c"), punct("SemiColon", ";")) + tree = package_tree("my_pkg", decls=[bogus]) + lines = bar.annotate_lines(tree) + pkg_node = bar.find_all(tree, "PackageDeclaration")[0] + with self.assertRaises(bar.ApiReferenceError) as ctx: + bar.convert_package(pkg_node, lines, "/a/my_pkg.vhd", "my_pkg.vhd") + self.assertIn("ComponentDeclaration", str(ctx.exception)) + self.assertIn("/a/my_pkg.vhd", str(ctx.exception)) + + def test_declarations_land_in_the_right_bucket(self): + decls = [ + procedure_decl("p"), + alias_decl("a", "p"), + constant_decl(["c"], subtype("integer"), default=initial_value(literal("1"))), + subtype_decl("s", subtype("integer")), + full_type_decl("t", enumeration_type("a")), + object_decl("SignalDeclaration", ["sig"], subtype("std_logic")), + ] + tree = package_tree("my_pkg", decls=decls) + lines = bar.annotate_lines(tree) + pkg_node = bar.find_all(tree, "PackageDeclaration")[0] + package = bar.convert_package(pkg_node, lines, "f.vhd", "f.vhd") + self.assertEqual(len(package["subprograms"]), 1) + self.assertEqual(len(package["aliases"]), 1) + self.assertEqual(len(package["constants"]), 1) + self.assertEqual(len(package["subtypes"]), 1) + self.assertEqual(len(package["types"]), 1) + self.assertEqual(len(package["objects"]), 1) + + def test_package_doc(self): + tree = package_tree("my_pkg", preamble_trivia=comment_trivia("A package for testing")) + lines = bar.annotate_lines(tree) + pkg_node = bar.find_all(tree, "PackageDeclaration")[0] + package = bar.convert_package(pkg_node, lines, "f.vhd", "f.vhd") + self.assertEqual(package["doc"], "A package for testing") + + +class TestConvertPackageFile(unittest.TestCase): + def test_finds_package_by_name_and_sets_source(self): + tree = package_tree("queue_pkg") + package = bar.convert_package_file( + tree, + Path("/repo/vunit/vhdl/data_types/src/queue_pkg.vhd"), + "queue_pkg", + "vunit/vhdl/data_types/src/queue_pkg.vhd", + ) + self.assertIsNotNone(package) + self.assertEqual(package["name"], "queue_pkg") + self.assertEqual(package["source"], "vunit/vhdl/data_types/src/queue_pkg.vhd") + self.assertEqual(package["library"], "vunit_lib") -class TestPackageScope(unittest.TestCase): - """find_package_files: the vunit/vhdl/**/src/**/*.vhd scope filter.""" + def test_missing_package_returns_none(self): + tree = package_tree("queue_pkg") + package = bar.convert_package_file(tree, Path("/repo/f.vhd"), "other_pkg", "f.vhd") + self.assertIsNone(package) + + +# --------------------------------------------------------------------------- +# Package scope, completeness +# --------------------------------------------------------------------------- + +class TestPackageScope(unittest.TestCase): def _make_tree(self, tmp_path, files): for relpath, content in files.items(): path = tmp_path / relpath @@ -596,12 +725,7 @@ def _make_tree(self, tmp_path, files): def test_finds_package_declaration_files(self): tmp_path = Path(self._tmp_dir()) - self._make_tree( - tmp_path, - { - "vunit/vhdl/data_types/src/queue_pkg.vhd": "package queue_pkg is\nend package;\n", - }, - ) + self._make_tree(tmp_path, {"vunit/vhdl/data_types/src/queue_pkg.vhd": "package queue_pkg is\nend package;\n"}) found = bar.find_package_files(tmp_path) self.assertEqual([name for _, name in found], ["queue_pkg"]) @@ -610,9 +734,7 @@ def test_excludes_package_body(self): self._make_tree( tmp_path, { - "vunit/vhdl/foo/src/foo_pkg.vhd": ( - "package foo_pkg is\nend package;\npackage body foo_pkg is\nend package body;\n" - ), + "vunit/vhdl/foo/src/foo_pkg.vhd": "package foo_pkg is\nend package;\npackage body foo_pkg is\nend package body;\n" }, ) found = bar.find_package_files(tmp_path) @@ -626,7 +748,7 @@ def test_excludes_package_instantiation(self): "vunit/vhdl/foo/src/foo_pkg.vhd": ( "package foo_generic_pkg is\nend package;\n" "package foo_pkg is new work.foo_generic_pkg generic map (t => integer);\n" - ), + ) }, ) found = bar.find_package_files(tmp_path) @@ -634,12 +756,7 @@ def test_excludes_package_instantiation(self): def test_excludes_osvvm(self): tmp_path = Path(self._tmp_dir()) - self._make_tree( - tmp_path, - { - "vunit/vhdl/osvvm/src/SomePkg.vhd": "package SomePkg is\nend package;\n", - }, - ) + self._make_tree(tmp_path, {"vunit/vhdl/osvvm/src/SomePkg.vhd": "package SomePkg is\nend package;\n"}) found = bar.find_package_files(tmp_path) self.assertEqual(found, []) @@ -672,76 +789,67 @@ def test_raises_listing_missing_and_extra(self): self.assertIn("'c'", message) -class TestErrorHandling(unittest.TestCase): - """The converter error conditions of design-spec section 6.""" +# --------------------------------------------------------------------------- +# Error handling: missing binary, version pin, non-zero exit +# --------------------------------------------------------------------------- - def test_ghdl_not_on_path(self): + +class TestErrorHandling(unittest.TestCase): + def test_vhdl_dump_ast_not_on_path(self): import shutil from unittest import mock with mock.patch.object(shutil, "which", return_value=None): with self.assertRaises(bar.ApiReferenceError) as ctx: - bar.check_ghdl_available() + bar.check_vhdl_dump_ast_available() self.assertEqual( str(ctx.exception), - "GHDL is required to build the API reference; install GHDL or set VUNIT_DOCS_SKIP_API=1", + "vhdl-dump-ast is required to build the API reference; install it with " + "`cargo install vhdl-dump-ast --version 0.1.0 --locked` or set VUNIT_DOCS_SKIP_API=1", ) - def test_file_to_xml_nonzero_exit(self): + def test_version_mismatch_raises(self): import subprocess from unittest import mock fake = subprocess.CompletedProcess( - args=["ghdl"], returncode=1, stdout="", stderr="somefile.vhd:3:1:error: bad syntax\n" + args=["vhdl-dump-ast", "--version"], returncode=0, stdout="vhdl-dump-ast 0.2.0\n", stderr="" ) with mock.patch.object(subprocess, "run", return_value=fake): with self.assertRaises(bar.ApiReferenceError) as ctx: - bar.run_file_to_xml(Path("somefile.vhd"), Path("/libs/vunit_lib"), Path("/libs/osvvm")) + bar.check_vhdl_dump_ast_version() message = str(ctx.exception) - self.assertIn("somefile.vhd", message) - self.assertIn("error: bad syntax", message) - - def test_xml_root_version_mismatch(self): - with self.assertRaises(bar.ApiReferenceError) as ctx: - bar.parse_xml_root('') - self.assertIn("0.12", str(ctx.exception)) - self.assertIn("converter must be checked", str(ctx.exception)) + self.assertIn("0.2.0", message) + self.assertIn("syntax tree format", message) - def test_xml_root_correct_version_parses(self): - root = bar.parse_xml_root('') - self.assertEqual(root.tag, "root") - - def test_vunit_compile_failure_raises(self): - import types + def test_correct_version_returns_version_string(self): + import subprocess from unittest import mock - class FakeVUnitObj: - def add_vhdl_builtins(self): - pass - - def add_verification_components(self): - pass - - def add_random(self): - pass + fake = subprocess.CompletedProcess( + args=["vhdl-dump-ast", "--version"], returncode=0, stdout="vhdl-dump-ast 0.1.0\n", stderr="" + ) + with mock.patch.object(subprocess, "run", return_value=fake): + self.assertEqual(bar.check_vhdl_dump_ast_version(), "vhdl-dump-ast 0.1.0") - def add_osvvm(self): - pass + def test_nonzero_exit_names_file_and_first_stderr_line(self): + import subprocess + from unittest import mock - def main(self): - raise SystemExit(1) + fake = subprocess.CompletedProcess( + args=["vhdl-dump-ast"], returncode=2, stdout="", stderr="12..18 unexpected token\nmore detail\n" + ) + with mock.patch.object(subprocess, "run", return_value=fake): + with self.assertRaises(bar.ApiReferenceError) as ctx: + bar.run_vhdl_dump_ast(Path("somefile.vhd")) + message = str(ctx.exception) + self.assertIn("somefile.vhd", message) + self.assertIn("unexpected token", message) - class FakeVUnit: - @staticmethod - def from_argv(_args): - return FakeVUnitObj() - fake_module = types.ModuleType("vunit") - fake_module.VUnit = FakeVUnit - with mock.patch.dict(sys.modules, {"vunit": fake_module}): - with self.assertRaises(bar.ApiReferenceError) as ctx: - bar.compile_vunit_libraries("/tmp/does-not-matter") - self.assertIn("exit code 1", str(ctx.exception)) +# --------------------------------------------------------------------------- +# Python API (unchanged code, still tested) +# --------------------------------------------------------------------------- class SampleClass: @@ -766,12 +874,7 @@ def sample_function(a, b=1): class TestPythonApi(unittest.TestCase): - """ - docs/**/*.rst directive scanning and object collection, including the - "no public objects found" error for an automodule with :members:. - """ - - def test_finds_autoclass_and_automodule_targets(self, tmp_path=None): + def test_finds_autoclass_and_automodule_targets(self): import tempfile tmp_path = Path(tempfile.mkdtemp(prefix="vunit-api-rst-")) diff --git a/tools/build_api_reference.py b/tools/build_api_reference.py index 372e2e868..ee5048831 100644 --- a/tools/build_api_reference.py +++ b/tools/build_api_reference.py @@ -14,7 +14,7 @@ Set VUNIT_DOCS_SKIP_API=1 to skip the build entirely (prints a warning, writes nothing, exits 0). This is the supported way to build the rest of -the documentation without GHDL installed. +the documentation without vhdl-dump-ast installed. """ import inspect @@ -25,13 +25,11 @@ import shutil import subprocess import sys -import tempfile from pathlib import Path -from xml.etree import ElementTree ROOT = Path(__file__).parent.parent -XML_ROOT_VERSION = "0.13" +VHDL_DUMP_AST_VERSION = "vhdl-dump-ast 0.1.0" VHDL_STANDARD = "2008" LIBRARY_NAME = "vunit_lib" @@ -74,548 +72,505 @@ def find_package_files(root): # --------------------------------------------------------------------------- -# Compiling +# Running vhdl-dump-ast # --------------------------------------------------------------------------- -def check_ghdl_available(): - """Raise ApiReferenceError if "ghdl" is not on PATH.""" - if shutil.which("ghdl") is None: +def check_vhdl_dump_ast_available(): + """Raise ApiReferenceError if "vhdl-dump-ast" is not on PATH.""" + if shutil.which("vhdl-dump-ast") is None: raise ApiReferenceError( - "GHDL is required to build the API reference; install GHDL or set VUNIT_DOCS_SKIP_API=1" + "vhdl-dump-ast is required to build the API reference; install it with " + "`cargo install vhdl-dump-ast --version 0.1.0 --locked` or set VUNIT_DOCS_SKIP_API=1" ) -def ghdl_version_line(): - """The first line of "ghdl --version", e.g. "GHDL 6.0.0 (...) [...]".""" - proc = subprocess.run(["ghdl", "--version"], capture_output=True, text=True, check=False) - return proc.stdout.splitlines()[0].strip() if proc.stdout else "" +def check_vhdl_dump_ast_version(): + """ + Raise ApiReferenceError unless "vhdl-dump-ast --version" is exactly + VHDL_DUMP_AST_VERSION (the format pin for the syntax tree shape this + converter understands). Returns the version string. + """ + proc = subprocess.run(["vhdl-dump-ast", "--version"], capture_output=True, text=True, check=False) + version_line = (proc.stdout or "").strip() + if version_line != VHDL_DUMP_AST_VERSION: + raise ApiReferenceError( + f"vhdl-dump-ast reports '{version_line}', expected '{VHDL_DUMP_AST_VERSION}'; " + "the converter must be checked against the new syntax tree format" + ) + return version_line -def compile_vunit_libraries(output_path): +def run_vhdl_dump_ast(file_path): """ - Compile vunit_lib and osvvm with GHDL, using VUnit's own Python API, - into output_path. Returns the "/ghdl/libraries" directory. + Run "vhdl-dump-ast --trivia --no-pretty" and return stdout + (the JSON syntax tree) as a string, without ever writing it to disk. """ - os.environ["VUNIT_SIMULATOR"] = "ghdl" + proc = subprocess.run( + ["vhdl-dump-ast", str(file_path), "--trivia", "--no-pretty"], + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + stderr_lines = proc.stderr.splitlines() if proc.stderr else [] + first_line = stderr_lines[0] if stderr_lines else "(no output)" + raise ApiReferenceError(f"vhdl-dump-ast failed for {file_path} (exit {proc.returncode}): {first_line}") + return proc.stdout - from vunit import VUnit # pylint: disable=import-outside-toplevel - vunit_obj = VUnit.from_argv(["--output-path", str(output_path), "--compile", "-q"]) - vunit_obj.add_vhdl_builtins() - vunit_obj.add_verification_components() - vunit_obj.add_random() - vunit_obj.add_osvvm() +# --------------------------------------------------------------------------- +# Syntax tree helpers +# +# A node is either {"Node": {"kind": ..., "children": [...]}} or, only at +# the tree root, the bare {"kind": "DesignFile", "children": [...]}. A leaf +# is {"Token": {"kind": ..., "text": ..., "leading_trivia": [...]}}. +# --------------------------------------------------------------------------- - try: - vunit_obj.main() - except SystemExit as exc: - if exc.code not in (0, None): - raise ApiReferenceError( - f"VUnit compile failed with exit code {exc.code}; see the compile output above" - ) from exc - return Path(output_path) / "ghdl" / "libraries" +def is_token(element): + """True if element is a {"Token": {...}} leaf.""" + return isinstance(element, dict) and "Token" in element -# --------------------------------------------------------------------------- -# Per-file XML extraction -# --------------------------------------------------------------------------- +def token_of(element): + """The inner {"kind", "text", "leading_trivia"} dict of a Token leaf.""" + return element["Token"] -def run_file_to_xml(file_path, vunit_lib_dir, osvvm_dir): - """ - Run "ghdl --file-to-xml" on file_path and return stdout (the XML) as a - string, without ever writing it to disk. - """ - cmd = [ - "ghdl", - "--file-to-xml", - "--std=08", - "--work=vunit_lib", - f"--workdir={vunit_lib_dir}", - f"-P{osvvm_dir}", - f"-P{vunit_lib_dir}", - str(file_path), - ] - proc = subprocess.run(cmd, capture_output=True, text=True, check=False) - if proc.returncode != 0: - stderr_lines = proc.stderr.splitlines() if proc.stderr else [] - first_error = next((ln for ln in stderr_lines if "error:" in ln), None) - if first_error is None: - first_error = stderr_lines[0] if stderr_lines else "(no output)" - raise ApiReferenceError(f"ghdl --file-to-xml failed for {file_path}: {first_error}") - return proc.stdout +def _raw(element): + """The {"kind", "children"} dict of a Node, unwrapping {"Node": {...}} (or the bare root).""" + if isinstance(element, dict) and "Node" in element: + return element["Node"] + return element -def parse_xml_root(xml_text): - """ - Parse the GHDL XML dump and check its root version. - """ - root = ElementTree.fromstring(xml_text) - version = root.get("version") - if version != XML_ROOT_VERSION: - raise ApiReferenceError( - f"XML root version is '{version}', expected '{XML_ROOT_VERSION}'; " - "the converter must be checked against the new GHDL XML format" - ) - return root +def kind(element): + """The node kind (e.g. "SubprogramDeclaration"), or None for a Token.""" + raw = _raw(element) + return raw.get("kind") if isinstance(raw, dict) else None + +def children(element): + """The direct children of a Node (Token leaves and/or further Nodes).""" + raw = _raw(element) + return raw.get("children", []) if isinstance(raw, dict) else [] + + +def find_all(element, *wanted_kinds): + """All descendant (or self) Nodes of the given kind(s), in document order.""" + wanted = set(wanted_kinds) + found = [] -def find_package_element(xml_root, file_path, package_name): + def walk(node): + if kind(node) in wanted: + found.append(node) + for child in children(node): + if not is_token(child): + walk(child) + + walk(element) + return found + + +def _tokens(element): + """All Token leaves under element (or element itself), in document order.""" + if is_token(element): + yield element + return + for child in children(element): + yield from _tokens(child) + + +def _first_token(element): + """The first Token leaf under element, in document order, or None.""" + for token in _tokens(element): + return token + return None + + +def text_of(*elements): """ - Find the "library_unit" element for the package declared in file_path, - or None if not found (the completeness check catches a missing package). + The source text of one or more subtrees: token texts joined, with a + single space inserted wherever a token's leading trivia is non-empty + (i.e. it was preceded by whitespace or a comment in the source). """ - file_str = str(file_path) - for element in xml_root.iter("library_unit"): - if ( - element.get("kind") == "package_declaration" - and element.get("file") == file_str - and element.get("identifier") == package_name - ): - return element + pieces = [] + for element in elements: + for token in _tokens(element): + tok = token_of(token) + if pieces and tok["leading_trivia"]: + pieces.append(" ") + pieces.append(tok["text"]) + return "".join(pieces) + + +def _keyword(token): + """The keyword name of a Token (e.g. "Impure"), or None if it isn't a keyword token.""" + raw_kind = token_of(token)["kind"] + return raw_kind.get("Keyword") if isinstance(raw_kind, dict) else None + + +def _direct_keywords(element): + """The set of keyword names among element's direct Token children.""" + return {_keyword(c) for c in children(element) if is_token(c)} - {None} + + +def _identifier_tokens(identifier_list): + """The Identifier Token leaves directly inside an IdentifierList (skipping commas).""" + return [c for c in children(identifier_list) if is_token(c) and token_of(c)["kind"] == "Identifier"] + + +def _find_child(element, wanted_kind): + """The first direct child Node of the given kind, or None.""" + for child in children(element): + if kind(child) == wanted_kind: + return child return None # --------------------------------------------------------------------------- -# Source text: line/column resolution and delimiter-bounded slicing +# Line numbers # --------------------------------------------------------------------------- -class SourceText: +def _trivia_text(trivia_entry): + """The decoded text of a comment-shaped trivia entry (e.g. LineComment), or "".""" + for value in trivia_entry.values(): + if isinstance(value, dict) and "value" in value: + data = bytes(value["value"]) + return data.decode(value.get("encoding", "utf-8"), errors="replace") + return "" + + +def _trivia_newlines(trivia_entry): + """How many line breaks a single trivia entry accounts for.""" + if "LineFeeds" in trivia_entry: + return trivia_entry["LineFeeds"] + if "Spaces" in trivia_entry or "HorizontalTabs" in trivia_entry: + return 0 + # A comment or any other trivia kind this converter does not know about + # by name: count embedded newlines in its decoded text, if it has one + # (covers a hypothetical multi-line block comment). + return _trivia_text(trivia_entry).count("\n") + + +def annotate_lines(tree): """ - A VHDL source file's text, with helpers to resolve GHDL's 1-based - line/col (columns use tab stops of 8, like a terminal) to an absolute - character offset, and to slice source text starting at such a position - up to the first unbracketed delimiter. + Walk every token of the whole syntax tree once, in document order, and + return a dict mapping id(token_dict) -> 1-based line number, computed + from LineFeeds trivia (and newlines inside token text or comments). """ + lines = {} + current_line = [1] - def __init__(self, text): - """text: the whole source file, as read from disk.""" - self.text = text - self.lines = text.split("\n") - offsets = [] - acc = 0 - for line in self.lines: - offsets.append(acc) - acc += len(line) + 1 - self._line_offsets = offsets - - @staticmethod - def _col_to_index(line_text, col): - """0-based character index in line_text for a 1-based GHDL column (tab stops of 8).""" - current_col = 1 - for idx, char in enumerate(line_text): - if current_col == col: - return idx - if char == "\t": - current_col = ((current_col - 1) // 8 + 1) * 8 + 1 - else: - current_col += 1 - return len(line_text) - - def offset(self, line, col): - """Absolute 0-based character offset in self.text for a 1-based (line, col).""" - return self._line_offsets[line - 1] + self._col_to_index(self.lines[line - 1], col) - - def slice_from(self, line, col, file_path, decl_line, allow_assign): - """ - Slice starting at (line, col) up to the first delimiter at bracket - depth 0 outside string and character literals: ":=" (only when - allow_assign), ";" or ")". Whitespace is collapsed to single spaces. - """ - text = self.text - depth = 0 - i = self.offset(line, col) - pieces = [] - piece_start = i - while i < len(text): - after = self._skip_literal(text, i) - if after is not None: - if text.startswith("--", i): - # A comment neither delimits nor belongs to the slice - pieces.append(text[piece_start:i]) - piece_start = after - i = after - continue - char = text[i] - if depth == 0 and (char in ";)" or (allow_assign and text.startswith(":=", i))): - pieces.append(text[piece_start:i]) - return re.sub(r"\s+", " ", " ".join(pieces)).strip() - if char == "(": - depth += 1 - elif char == ")": - depth -= 1 - i += 1 - raise ApiReferenceError(f"No closing delimiter found for slice at {file_path}:{decl_line}") - - @staticmethod - def _skip_literal(text, i): - """ - The index after the comment, string literal or character literal starting at - index i of text, or None when none starts there. - """ - if text.startswith("--", i): - newline = text.find("\n", i) - return len(text) if newline == -1 else newline - if text[i] == '"': - i += 1 - while i < len(text): - if text[i] == '"': - if text.startswith('"', i + 1): - # A doubled quote within the literal - i += 2 - continue - return i + 1 - i += 1 - return len(text) - if text[i] == "'" and i + 2 < len(text) and text[i + 2] == "'": - return i + 3 - return None + def walk(element): + if is_token(element): + tok = token_of(element) + for trivia in tok["leading_trivia"]: + current_line[0] += _trivia_newlines(trivia) + lines[id(tok)] = current_line[0] + current_line[0] += tok["text"].count("\n") + return + for child in children(element): + walk(child) + + walk(tree) + return lines + + +def line_of(lines, token): + """The line number of a Token leaf, from the dict built by annotate_lines.""" + return lines[id(token_of(token))] +# --------------------------------------------------------------------------- +# Doc comments +# --------------------------------------------------------------------------- + _LICENSE_MARKER = "This Source Code Form" -def comment_block_above(source, first_line): +def _comment_text(trivia_list): """ - The contiguous block of "--" lines directly above first_line, stripped - of "--" and one following space, joined with "\\n". None if there is no - such block. + The contiguous block of "--" line comments directly before a token + (i.e. the last run not broken by a blank line), stripped of "--" and + one following space, joined with "\\n". None if there is no such block. """ collected = [] - line_no = first_line - 1 - while 1 <= line_no <= len(source.lines): - stripped = source.lines[line_no - 1].strip() - if stripped.startswith("--"): - collected.append(stripped) - line_no -= 1 - else: - break - if not collected: - return None - collected.reverse() - out = [] - for comment in collected: - body = comment[2:] - if body.startswith(" "): - body = body[1:] - out.append(body) - return "\n".join(out) + for entry in trivia_list: + if "LineComment" in entry: + body = _trivia_text(entry)[2:] + if body.startswith(" "): + body = body[1:] + collected.append(body) + elif _trivia_newlines(entry) >= 2: + collected = [] # a blank line ends (and restarts) the block + return "\n".join(collected) if collected else None -def subprogram_first_line(source, decl_line): - """ - A subprogram's declaration starts on the line holding "function" or - "procedure", which is normally decl_line (GHDL's line/col point at the - identifier, on that same line) -- unless a preceding line holds only - "pure" or "impure", in which case that line is the true first line. - """ - prev = decl_line - 1 - if prev >= 1 and source.lines[prev - 1].strip() in ("pure", "impure"): - return prev - return decl_line +def leading_doc(element): + """The doc comment directly above element's first token, or None.""" + first = _first_token(element) + return _comment_text(token_of(first)["leading_trivia"]) if first is not None else None -def package_doc(source, decl_line): +def package_doc(preamble): """The package's own doc comment, or None (also None for the MPL license header).""" - doc = comment_block_above(source, decl_line) - if doc is not None and _LICENSE_MARKER in doc: + doc = leading_doc(preamble) + if doc is not None and _LICENSE_MARKER in doc: # pylint: disable=unsupported-membership-test return None return doc # --------------------------------------------------------------------------- -# XML element -> dict conversion +# Declaration converters # --------------------------------------------------------------------------- -_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") - -_PARAMETER_CLASS = { - "interface_constant_declaration": "constant", - "interface_variable_declaration": "variable", - "interface_signal_declaration": "signal", - "interface_file_declaration": "file", -} +_PARAMETER_CLASS_KEYWORDS = {"Constant": "constant", "Variable": "variable", "Signal": "signal", "File": "file"} +_MODE_KEYWORDS = {"In": "in", "Out": "out", "Inout": "inout", "Buffer": "buffer", "Linkage": "linkage"} -_OBJECT_CLASS = { - "signal_declaration": "signal", - "variable_declaration": "shared variable", - "file_declaration": "file", -} - -def _min_position(element, file_str): - """ - The earliest (line, col) among element and all its descendants that - carry their own line/col in file_str. - - GHDL constant-folds some expressions (e.g. a "positive'high" default - is folded to an integer_literal) and then reports the *folded* - element's own line/col at the attribute name ("high"), not at the - start of the written expression ("positive'high"). The true start is - still there, on a descendant (here the attribute's prefix "positive"), - so scanning the whole subtree for the earliest position recovers it. +def convert_parameter(iod): """ - best = None - line = element.get("line") - col = element.get("col") - if line is not None and col is not None and element.get("file") == file_str: - best = (int(line), int(col)) - for child in element: - child_best = _min_position(child, file_str) - if child_best is not None and (best is None or child_best < best): - best = child_best - return best - - -def _slice_element(source, element, file_path, decl_line, allow_assign): - """Slice source text starting at element's (recovered) position, or None if it has none.""" - position = _min_position(element, file_path) - if position is None: - return None - return source.slice_from(position[0], position[1], file_path, decl_line, allow_assign=allow_assign) + Convert one InterfaceObjectDeclaration/InterfaceFileDeclaration node to + a list of parameter dicts (one per identifier). - -def _name_text(name_element): - """ - The (possibly dotted) name of a simple_name or selected_name element, - e.g. "push" or "work.push". + VHDL default rules for an omitted object class: "constant" when the + (explicit or default) mode is "in", else "variable" -- except a file + parameter, which is always class "file" and defaults to mode "inout". """ - identifier = name_element.get("identifier", "") - # GHDL's element carries its own kind/identifier directly (it - # *is* the name node for the prefix, e.g. simple_name "work"), not a - # wrapper around a further child. - prefix = name_element.find("prefix") - if prefix is None: - return identifier - return f"{_name_text(prefix)}.{identifier}" - - -def convert_parameter(element, source, file_path, decl_line): - """Convert one interface_*_declaration XML element to a parameter dict.""" - kind = element.get("kind") - param_class = _PARAMETER_CLASS.get(kind) - if param_class is None: - raise ApiReferenceError(f"Unhandled parameter kind '{kind}' at {file_path}:{decl_line}") - - # Implicit predefined operators (e.g. "=" synthesized for every type) - # have parameters with a "type" reference but no textual - # subtype_indication, since they were never written in source. - type_el = element.find("subtype_indication") - param_type = None - if type_el is not None: - param_type = _slice_element(source, type_el, file_path, decl_line, allow_assign=True) + keywords = _direct_keywords(iod) + is_file_param = kind(iod) == "InterfaceFileDeclaration" + mode = next((v for k, v in _MODE_KEYWORDS.items() if k in keywords), "inout" if is_file_param else "in") + if is_file_param: + param_class = "file" + else: + param_class = next( + (v for k, v in _PARAMETER_CLASS_KEYWORDS.items() if k in keywords), + "constant" if mode == "in" else "variable", + ) + + subtype_node = _find_child(iod, "SubtypeIndication") + param_type = text_of(subtype_node) if subtype_node is not None else None default = None - default_el = element.find("default_value") - if default_el is not None: - default = _slice_element(source, default_el, file_path, decl_line, allow_assign=False) + init_node = _find_child(iod, "InitialValue") + if init_node is not None: + default = text_of(*children(init_node)[1:]) + + ident_list = _find_child(iod, "IdentifierList") + return [ + {"name": token_of(tok)["text"], "class": param_class, "mode": mode, "type": param_type, "default": default} + for tok in _identifier_tokens(ident_list) + ] - return { - "name": element.get("identifier"), - "class": param_class, - "mode": element.get("mode"), - "type": param_type, - "default": default, - } +def convert_subprogram_entry(decl, lines, _file_path): + """Convert one top-level SubprogramDeclaration node to a list holding a single subprogram dict.""" + return [convert_subprogram(decl, lines)] -def convert_subprogram(element, source, file_path): - """ - Convert one function_declaration/procedure_declaration XML element to a - subprogram dict. - """ - kind_raw = element.get("kind") - kind = "function" if kind_raw == "function_declaration" else "procedure" - identifier = element.get("identifier") - decl_line = int(element.get("line")) + +def convert_subprogram(decl, lines): + """Convert one SubprogramDeclaration node (top-level or a protected type method) to a subprogram dict.""" + spec = children(decl)[0] + func = kind(spec) == "FunctionSpecification" + + name_tok = next(c for c in children(spec) if is_token(c) and token_of(c)["kind"] in ("Identifier", "StringLiteral")) + name_text = token_of(name_tok)["text"] + operator = token_of(name_tok)["kind"] != "Identifier" + name = name_text.strip('"') if operator else name_text + + pure = None if not func else "Impure" not in _direct_keywords(spec) parameters = [] - chain = element.find("interface_declaration_chain") - if chain is not None: - for param_el in chain.findall("el"): - parameters.append(convert_parameter(param_el, source, file_path, decl_line)) + for iod in find_all(spec, "InterfaceObjectDeclaration", "InterfaceFileDeclaration"): + parameters.extend(convert_parameter(iod)) return_type = None - if kind == "function": - return_type_el = element.find("return_type_mark") - if return_type_el is not None: - return_type = return_type_el.get("identifier") - - pure = None if kind == "procedure" else (element.get("pure_flag") == "true") + if func: + return_name = _find_child(spec, "Name") + if return_name is not None: + return_type = text_of(return_name) return { - "kind": kind, - "name": identifier, - "operator": _IDENTIFIER_RE.match(identifier) is None, + "kind": "function" if func else "procedure", + "name": name, + "operator": operator, "pure": pure, - "line": decl_line, + "line": line_of(lines, name_tok), + "doc": leading_doc(decl), "parameters": parameters, "return_type": return_type, - "doc": comment_block_above(source, subprogram_first_line(source, decl_line)), } -def convert_alias(element, *_): - """Convert one non_object_alias_declaration/object_alias_declaration XML element to an alias dict.""" - decl_line = int(element.get("line")) - name_el = element.find("name") - target = _name_text(name_el) if name_el is not None else None +def convert_alias(decl, lines, _file_path=None): + """Convert one AliasDeclaration node to a list holding a single alias dict.""" + direct = children(decl) + name_tok = next(c for c in direct if is_token(c) and token_of(c)["kind"] == "Identifier") + target_node = _find_child(decl, "Name") + target = text_of(target_node) if target_node is not None else None - signature_el = element.find("alias_signature") signature = [] return_type = None - if signature_el is not None: - type_marks_list = signature_el.find("type_marks_list") - if type_marks_list is not None: - signature = [mark.get("identifier") for mark in type_marks_list.findall("el")] - return_type_mark = signature_el.find("return_type_mark") - if return_type_mark is not None: - return_type = return_type_mark.get("identifier") + sig_node = _find_child(decl, "Signature") + if sig_node is not None: + type_mark_list = _find_child(sig_node, "TypeMarkList") + if type_mark_list is not None: + for child in children(type_mark_list): + if kind(child) == "Name": + signature.append(text_of(child)) + elif kind(child) == "ReturnType": + return_name = _find_child(child, "Name") + if return_name is not None: + return_type = text_of(return_name) + + return [ + { + "name": token_of(name_tok)["text"], + "line": line_of(lines, name_tok), + "target": target, + "signature": signature, + "return_type": return_type, + } + ] - return { - "name": element.get("identifier"), - "line": decl_line, - "target": target, - "signature": signature, - "return_type": return_type, - } +def convert_constant(decl, lines, _file_path=None): + """Convert one ConstantDeclaration node to a list of constant dicts (one per identifier).""" + ident_list = _find_child(decl, "IdentifierList") + subtype_node = _find_child(decl, "SubtypeIndication") + const_type = text_of(subtype_node) if subtype_node is not None else None -def convert_constant(element, source, file_path): - """Convert one constant_declaration XML element to a constant dict.""" - decl_line = int(element.get("line")) - type_el = element.find("subtype_indication") - const_type = _slice_element(source, type_el, file_path, decl_line, allow_assign=True) value = None - value_el = element.find("default_value") - if value_el is not None: - value = _slice_element(source, value_el, file_path, decl_line, allow_assign=False) - return { - "name": element.get("identifier"), - "line": decl_line, - "doc": comment_block_above(source, decl_line), - "type": const_type, - "value": value, - } + init_node = _find_child(decl, "InitialValue") + if init_node is not None: + value = text_of(*children(init_node)[1:]) + + doc = leading_doc(decl) + return [ + {"name": token_of(tok)["text"], "line": line_of(lines, tok), "doc": doc, "type": const_type, "value": value} + for tok in _identifier_tokens(ident_list) + ] -def convert_subtype(element, source, file_path): - """Convert one subtype_declaration XML element to a subtype dict.""" - decl_line = int(element.get("line")) - type_el = element.find("subtype_indication") - definition = _slice_element(source, type_el, file_path, decl_line, allow_assign=True) - return { - "name": element.get("identifier"), - "line": decl_line, - "doc": comment_block_above(source, decl_line), - "definition": definition, - } +def convert_subtype(decl, lines, _file_path=None): + """Convert one SubtypeDeclaration node to a list holding a single subtype dict.""" + direct = children(decl) + name_tok = next(c for c in direct if is_token(c) and token_of(c)["kind"] == "Identifier") + subtype_node = _find_child(decl, "SubtypeIndication") + return [ + { + "name": token_of(name_tok)["text"], + "line": line_of(lines, name_tok), + "doc": leading_doc(decl), + "definition": text_of(subtype_node) if subtype_node is not None else None, + } + ] -def convert_object(element, source, file_path): - """Convert one signal/shared-variable/file declaration XML element to an object dict.""" - decl_line = int(element.get("line")) - type_el = element.find("subtype_indication") - obj_type = _slice_element(source, type_el, file_path, decl_line, allow_assign=True) - return { - "name": element.get("identifier"), - "class": _OBJECT_CLASS[element.get("kind")], - "line": decl_line, - "doc": comment_block_above(source, decl_line), - "type": obj_type, - } +_OBJECT_CLASS = {"SignalDeclaration": "signal", "VariableDeclaration": "shared variable", "FileDeclaration": "file"} -def _convert_record_type(type_def, source, file_path, decl_line, common): +def convert_object(decl, lines, _file_path=None): + """Convert one Signal/Variable/File Declaration node to a list of object dicts (one per identifier).""" + ident_list = _find_child(decl, "IdentifierList") + subtype_node = _find_child(decl, "SubtypeIndication") + obj_type = text_of(subtype_node) if subtype_node is not None else None + obj_class = _OBJECT_CLASS[kind(decl)] + doc = leading_doc(decl) + return [ + {"name": token_of(tok)["text"], "class": obj_class, "line": line_of(lines, tok), "doc": doc, "type": obj_type} + for tok in _identifier_tokens(ident_list) + ] + + +def _convert_record_type(def_node, common): """Build a "record" type dict: {..., "elements": [{"name", "type"}, ...]}.""" elements = [] - elements_list = type_def.find("elements_declaration_list") - if elements_list is not None: - for element_el in elements_list.findall("el"): - element_type_el = element_el.find("subtype_indication") - elements.append( - { - "name": element_el.get("identifier"), - "type": _slice_element(source, element_type_el, file_path, decl_line, allow_assign=True), - } - ) + for element_decl in find_all(def_node, "ElementDeclaration"): + ident_list = _find_child(element_decl, "IdentifierList") + subtype_node = _find_child(element_decl, "SubtypeIndication") + element_type = text_of(subtype_node) if subtype_node is not None else None + for tok in _identifier_tokens(ident_list): + elements.append({"name": token_of(tok)["text"], "type": element_type}) return {**common, "kind": "record", "elements": elements} -def _convert_enumeration_type(type_def, common): +def _convert_enumeration_type(def_node, common): """Build an "enumeration" type dict: {..., "literals": [name, ...]}.""" - literal_list = type_def.find("enumeration_literal_list") - literals = [literal.get("identifier") for literal in literal_list.findall("el")] if literal_list is not None else [] + enum_list = _find_child(def_node, "EnumerationList") + literals = [] + if enum_list is not None: + for child in children(enum_list): + if is_token(child) and token_of(child)["kind"] in ("Identifier", "CharacterLiteral"): + literals.append(token_of(child)["text"]) return {**common, "kind": "enumeration", "literals": literals} -def _convert_protected_type(type_def, source, file_path, common): +def _convert_protected_type(def_node, lines, common, file_path): """Build a "protected" type dict: {..., "subprograms": [...]} (its methods).""" subprograms = [] - chain = type_def.find("declaration_chain") - if chain is not None: - for member_el in chain.findall("el"): - if member_el.get("kind") in ("function_declaration", "procedure_declaration"): - subprograms.append(convert_subprogram(member_el, source, file_path)) + decl_part = _find_child(def_node, "ProtectedTypeDeclarativePart") + if decl_part is not None: + for member in children(decl_part): + if kind(member) != "SubprogramDeclaration": + first = _first_token(member) + where_line = line_of(lines, first) if first is not None else "?" + raise ApiReferenceError( + f"Unhandled protected type member kind '{kind(member)}' at {file_path}:{where_line}" + ) + subprograms.append(convert_subprogram(member, lines)) return {**common, "kind": "protected", "subprograms": subprograms} -def convert_type(element, source, file_path): - """Convert one type_declaration XML element to a type dict.""" - decl_line = int(element.get("line")) - common = {"name": element.get("identifier"), "line": decl_line, "doc": comment_block_above(source, decl_line)} - type_def = element.find("type_definition") - def_kind = type_def.get("kind") if type_def is not None else None +def convert_type(decl, lines, file_path): + """Convert one FullTypeDeclaration node to a list holding a single type dict.""" + direct = children(decl) + name_tok = next(c for c in direct if is_token(c) and token_of(c)["kind"] == "Identifier") + common = {"name": token_of(name_tok)["text"], "line": line_of(lines, name_tok), "doc": leading_doc(decl)} - if def_kind == "record_type_definition": - return _convert_record_type(type_def, source, file_path, decl_line, common) - if def_kind == "enumeration_type_definition": - return _convert_enumeration_type(type_def, common) - if def_kind == "protected_type_declaration": - return _convert_protected_type(type_def, source, file_path, common) + def_node = next((c for c in direct if not is_token(c)), None) + def_kind = kind(def_node) - # Array, access and file types (and anything else GHDL introduces): + if def_kind == "RecordTypeDefinition": + return [_convert_record_type(def_node, common)] + if def_kind == "EnumerationTypeDefinition": + return [_convert_enumeration_type(def_node, common)] + if def_kind == "ProtectedTypeDeclaration": + return [_convert_protected_type(def_node, lines, common, file_path)] + + # Array, access and file types (and anything else vhdl-dump-ast introduces): # kept as "other" with the raw source-text definition. - definition = _slice_element(source, type_def, file_path, decl_line, allow_assign=True) - return {**common, "kind": "other", "definition": definition} + return [{**common, "kind": "other", "definition": text_of(def_node) if def_node is not None else None}] _KIND_HANDLERS = { - "function_declaration": ("subprograms", convert_subprogram), - "procedure_declaration": ("subprograms", convert_subprogram), - # non_object_alias_declaration covers subprogram and type aliases; - # object_alias_declaration covers aliases of a signal/variable/ - # constant (e.g. "alias null_byte_vector_ptr is null_string_ptr"). - # Both share the same shape (a "name", and an optional - # "alias_signature" for subprograms only) so both go through the same - # converter into "aliases". - "non_object_alias_declaration": ("aliases", convert_alias), - "object_alias_declaration": ("aliases", convert_alias), - "constant_declaration": ("constants", convert_constant), - "subtype_declaration": ("subtypes", convert_subtype), - "type_declaration": ("types", convert_type), - "signal_declaration": ("objects", convert_object), - "variable_declaration": ("objects", convert_object), - "file_declaration": ("objects", convert_object), + "SubprogramDeclaration": ("subprograms", convert_subprogram_entry), + "AliasDeclaration": ("aliases", convert_alias), + "ConstantDeclaration": ("constants", convert_constant), + "SubtypeDeclaration": ("subtypes", convert_subtype), + "FullTypeDeclaration": ("types", convert_type), + "SignalDeclaration": ("objects", convert_object), + "VariableDeclaration": ("objects", convert_object), + "FileDeclaration": ("objects", convert_object), } -def convert_package(package_el, source, file_path, repo_relative_path): - """Convert one package_declaration XML element to a package dict.""" - decl_line = int(package_el.get("line")) - name = package_el.get("identifier") - file_str = str(file_path) +def convert_package(pkg_node, lines, file_path, repo_relative_path): + """Convert one PackageDeclaration node to a package dict.""" + preamble = _find_child(pkg_node, "PackagePreamble") + name_tok = next(c for c in children(preamble) if is_token(c) and token_of(c)["kind"] == "Identifier") + name = token_of(name_tok)["text"] package = { "library": LIBRARY_NAME, "name": name, "source": repo_relative_path, - "line": decl_line, + "line": line_of(lines, name_tok), "private": name.endswith("_private_pkg"), - "doc": package_doc(source, decl_line), + "doc": package_doc(preamble), "types": [], "subtypes": [], "constants": [], @@ -624,25 +579,32 @@ def convert_package(package_el, source, file_path, repo_relative_path): "aliases": [], } - chain = package_el.find("declaration_chain") - if chain is None: - return package - - for element in chain.findall("el"): - if element.get("file") != file_str: - continue - kind = element.get("kind") - if kind == "anonymous_type_declaration": + decl_part = _find_child(pkg_node, "PackageDeclarativePart") + for decl in children(decl_part) if decl_part is not None else []: + if is_token(decl): continue - handler = _KIND_HANDLERS.get(kind) + handler = _KIND_HANDLERS.get(kind(decl)) if handler is None: - raise ApiReferenceError(f"Unhandled declaration kind '{kind}' at {file_str}:{element.get('line')}") + first = _first_token(decl) + where_line = line_of(lines, first) if first is not None else "?" + raise ApiReferenceError(f"Unhandled declaration kind '{kind(decl)}' at {file_path}:{where_line}") bucket, convert = handler - package[bucket].append(convert(element, source, file_str)) + package[bucket].extend(convert(decl, lines, file_path)) return package +def convert_package_file(tree, file_path, package_name, repo_relative_path): + """Find and convert the PackageDeclaration for package_name in a parsed syntax tree, or None.""" + lines = annotate_lines(tree) + for pkg_node in find_all(tree, "PackageDeclaration"): + preamble = _find_child(pkg_node, "PackagePreamble") + name_tok = next(c for c in children(preamble) if is_token(c) and token_of(c)["kind"] == "Identifier") + if token_of(name_tok)["text"] == package_name: + return convert_package(pkg_node, lines, str(file_path), repo_relative_path) + return None + + # --------------------------------------------------------------------------- # Completeness check # --------------------------------------------------------------------------- @@ -667,21 +629,18 @@ def check_completeness(json_package_names, expected_package_names): def build_vhdl_api(root, vunit_version): - """Compile vunit_lib with GHDL and extract the full VHDL symbol table.""" + """Run vhdl-dump-ast on every in-scope package file and extract the full VHDL symbol table.""" package_files = find_package_files(root) - check_ghdl_available() - # ghdl_version_line() already reads e.g. "GHDL 6.0.0 (...) [Dunoon edition]" - generator = f"{ghdl_version_line()} --file-to-xml" - - with tempfile.TemporaryDirectory(prefix="vunit-api-compile-") as compile_out: - libraries_dir = compile_vunit_libraries(compile_out) - packages = [] - for file_path, package_name in package_files: - xml = run_file_to_xml(file_path, libraries_dir / "vunit_lib", libraries_dir / "osvvm") - package_el = find_package_element(parse_xml_root(xml), file_path, package_name) - if package_el is not None: - source = SourceText(file_path.read_text(encoding="utf-8")) - packages.append(convert_package(package_el, source, file_path, file_path.relative_to(root).as_posix())) + check_vhdl_dump_ast_available() + generator = check_vhdl_dump_ast_version() + + packages = [] + for file_path, package_name in package_files: + stdout = run_vhdl_dump_ast(file_path) + tree = json.loads(stdout) + package = convert_package_file(tree, file_path, package_name, file_path.relative_to(root).as_posix()) + if package is not None: + packages.append(package) check_completeness((pkg["name"] for pkg in packages), (name for _, name in package_files)) packages.sort(key=lambda pkg: pkg["name"]) From 270a25f9669e8025381d1adb784e536ad76ec065 Mon Sep 17 00:00:00 2001 From: Sebastian Hellgren Date: Tue, 15 Sep 2026 22:44:06 +0200 Subject: [PATCH 08/11] Shrink the API reference generator The generator now unwraps the vhdl-dump-ast tree once, numbering the lines of the tokens as it goes, and handles all declaration kinds in one function. The output is unchanged. The tests parse small VHDL snippets with the real vhdl-dump-ast instead of building syntax trees by hand. They are skipped when vhdl-dump-ast is not installed, so the docs tox environment, which has it, now runs them. Co-Authored-By: Claude Opus 5 (1M context) --- pyproject.toml | 1 + tests/unit/test_api_reference.py | 963 ++++++------------------------- tools/build_api_reference.py | 773 +++++++------------------ 3 files changed, 404 insertions(+), 1333 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index aeb7430bb..0efdfd5d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -138,6 +138,7 @@ commands= fmt: {envpython} -m black ./ --exclude .eggs|.git|.hg|.mypy_cache|.nox|.tox|.venv|_build|buck-out|build|dist' {posargs} unit: {envpython} -m pytest -v -ra tests/unit {posargs} lint: {envpython} -m pytest -v -ra tests/lint {posargs} + docs: {envpython} -m pytest -ra tests/unit/test_api_reference.py {posargs} docs: {envpython} tools/build_docs.py {envtmpdir}/docsbuild {posargs} docs: {envpython} tools/build_api_reference.py {envtmpdir}/docsbuild acceptance: {envpython} -m pytest -v -ra tests/acceptance {posargs} diff --git a/tests/unit/test_api_reference.py b/tests/unit/test_api_reference.py index e5cebc844..25d863261 100644 --- a/tests/unit/test_api_reference.py +++ b/tests/unit/test_api_reference.py @@ -7,774 +7,231 @@ """ Test of the machine-readable VHDL and Python API reference generator. -Uses small, inline vhdl-dump-ast syntax trees (no vhdl-dump-ast binary -invoked -- unit CI has no cargo toolchain) and a small Python module -defined in this file (no Sphinx build). The tree shapes mirror what -"vhdl-dump-ast --trivia --no-pretty" actually produces, verified by hand -against real output from vunit/vhdl/**/src/*.vhd during implementation. +The VHDL-facing tests run small inline snippets through the real +vhdl-dump-ast binary (skipped when it is not on PATH, e.g. plain unit CI). +Error paths that do not need real parsing (missing binary, version +mismatch, non-zero exit) are covered separately with mocks. """ -import copy +import json +import shutil +import subprocess import sys +import tempfile +import types import unittest from pathlib import Path +from unittest import mock +from tests.common import with_tempdir from tools import build_api_reference as bar -# --------------------------------------------------------------------------- -# Tiny syntax-tree builders, matching vhdl-dump-ast's JSON shape: -# Node: {"Node": {"kind": ..., "children": [...]}} -# Token: {"Token": {"kind": ..., "text": ..., "leading_trivia": [...]}} -# --------------------------------------------------------------------------- - -SPACE = [{"Spaces": 1}] -NONE = [] - - -def node(kind, *children): - return {"Node": {"kind": kind, "children": list(children)}} - - -def token(kind, text, trivia=SPACE): - return {"Token": {"kind": kind, "text": text, "leading_trivia": copy.deepcopy(trivia)}} - - -def kw(name, trivia=SPACE): - """A keyword token, e.g. kw("Procedure") -> Token({"Keyword": "Procedure"}, "procedure").""" - return token({"Keyword": name}, name.lower(), trivia) - - -def ident(text, trivia=SPACE): - return token("Identifier", text, trivia) - - -def op_name(text, trivia=SPACE): - return token("StringLiteral", f'"{text}"', trivia) - - -def punct(kind, text, trivia=NONE): - return token(kind, text, trivia) - - -def identifier_list(*names, first_trivia=SPACE): - kids = [] - for i, name in enumerate(names): - kids.append(ident(name, trivia=first_trivia if i == 0 else NONE)) - if i < len(names) - 1: - kids.append(punct("Comma", ",")) - return node("IdentifierList", *kids) - - -def simple_name(text, trivia=SPACE): - return node("Name", node("NameDesignatorPrefix", ident(text, trivia))) - - -def subtype(name, *extra, trivia=SPACE): - return node("SubtypeIndication", simple_name(name, trivia), *extra) - - -def initial_value(expr, trivia=SPACE): - return node("InitialValue", punct("ColonEq", ":=", trivia), expr) - - -def literal(text, literal_kind="AbstractLiteral", trivia=SPACE): - return node("LiteralExpression", token(literal_kind, text, trivia)) - - -def parameter(names, subtype_node, class_kw=None, mode_kw=None, default=None, is_file=False): - kids = [] - if class_kw is not None: - kids.append(kw(class_kw)) - id_list_trivia = NONE if class_kw is not None else SPACE - kids.append( - identifier_list(*names, first_trivia=id_list_trivia) - if isinstance(names, (list, tuple)) - else identifier_list(names, first_trivia=id_list_trivia) - ) - kids.append(punct("Colon", ":", SPACE)) - if mode_kw is not None: - kids.append(kw(mode_kw)) - kids.append(subtype_node) - if default is not None: - kids.append(default) - return node("InterfaceFileDeclaration" if is_file else "InterfaceObjectDeclaration", *kids) - - -def parameter_list(*params): - if not params: - return None - kids = [] - for i, param in enumerate(params): - kids.append(param) - if i < len(params) - 1: - kids.append(punct("SemiColon", ";")) - return node( - "ParameterList", - node( - "ParenthesizedInterfaceList", - punct("LeftPar", "(", SPACE), - node("InterfaceList", *kids), - punct("RightPar", ")", NONE), - ), - ) - - -def procedure_decl(name, params=(), doc_trivia=NONE, operator=False): - name_tok = op_name(name) if operator else ident(name) - kids = [kw("Procedure", trivia=doc_trivia)] - kids.append(name_tok) - plist = parameter_list(*params) - if plist is not None: - kids.append(plist) - spec = node("ProcedureSpecification", *kids) - return node("SubprogramDeclaration", spec, punct("SemiColon", ";")) - - -def function_decl(name, params=(), return_type="natural", impure=False, doc_trivia=NONE, operator=False): - name_tok = op_name(name) if operator else ident(name) - kids = [] - if impure: - kids.append(kw("Impure", trivia=doc_trivia)) - kids.append(kw("Function")) - else: - kids.append(kw("Function", trivia=doc_trivia)) - kids.append(name_tok) - plist = parameter_list(*params) - if plist is not None: - kids.append(plist) - kids.append(kw("Return")) - kids.append(simple_name(return_type)) - spec = node("FunctionSpecification", *kids) - return node("SubprogramDeclaration", spec, punct("SemiColon", ";")) - - -def alias_decl(name, target, signature_types=(), return_type=None, has_signature=False): - kids = [kw("Alias"), ident(name), kw("Is"), simple_name(target)] - if has_signature: - type_mark_kids = [] - for i, t in enumerate(signature_types): - type_mark_kids.append(simple_name(t, trivia=SPACE if i == 0 else NONE)) - if i < len(signature_types) - 1: - type_mark_kids.append(punct("Comma", ",")) - if return_type is not None: - type_mark_kids.append(node("ReturnType", kw("Return"), simple_name(return_type))) - sig_kids = [ - punct("LeftSquare", "[", SPACE), - node("TypeMarkList", *type_mark_kids), - punct("RightSquare", "]", NONE), - ] - kids.append(node("Signature", *sig_kids)) - kids.append(punct("SemiColon", ";")) - return node("AliasDeclaration", *kids) - - -def constant_decl(names, subtype_node, default=None, doc_trivia=NONE): - kids = [ - kw("Constant", trivia=doc_trivia), - identifier_list(*names) if isinstance(names, (list, tuple)) else identifier_list(names), - ] - kids.append(punct("Colon", ":", SPACE)) - kids.append(subtype_node) - if default is not None: - kids.append(default) - kids.append(punct("SemiColon", ";")) - return node("ConstantDeclaration", *kids) - - -def subtype_decl(name, subtype_node, doc_trivia=NONE): - return node( - "SubtypeDeclaration", - kw("Subtype", trivia=doc_trivia), - ident(name), - kw("Is"), - subtype_node, - punct("SemiColon", ";"), - ) - - -def object_decl(decl_kind, names, subtype_node, doc_trivia=NONE, shared=False): - kids = [] - if shared: - kids.append(kw("Shared", trivia=doc_trivia)) - kids.append(kw("Variable")) - else: - first_kw = {"SignalDeclaration": "Signal", "VariableDeclaration": "Variable", "FileDeclaration": "File"}[ - decl_kind - ] - kids.append(kw(first_kw, trivia=doc_trivia)) - kids.append(identifier_list(*names) if isinstance(names, (list, tuple)) else identifier_list(names)) - kids.append(punct("Colon", ":", SPACE)) - kids.append(subtype_node) - kids.append(punct("SemiColon", ";")) - return node(decl_kind, *kids) - - -def full_type_decl(name, def_node, doc_trivia=NONE): - return node( - "FullTypeDeclaration", kw("Type", trivia=doc_trivia), ident(name), kw("Is"), def_node, punct("SemiColon", ";") - ) - - -def record_element(names, subtype_node): - kids = [ - identifier_list(*names) if isinstance(names, (list, tuple)) else identifier_list(names), - punct("Colon", ":", SPACE), - subtype_node, - punct("SemiColon", ";"), - ] - return node("ElementDeclaration", *kids) - - -def record_type(*elements): - return node( - "RecordTypeDefinition", - node("RecordTypeDefinitionPreamble", kw("Record")), - node("RecordElementDeclarations", *elements), - node("RecordTypeDefinitionEpilogue", kw("End"), kw("Record")), - ) - - -def enumeration_type(*literals): - list_kids = [] - for i, lit in enumerate(literals): - lit_kind = "CharacterLiteral" if lit.startswith("'") else "Identifier" - list_kids.append(token(lit_kind, lit, trivia=SPACE if i == 0 else NONE)) - if i < len(literals) - 1: - list_kids.append(punct("Comma", ",")) - return node( - "EnumerationTypeDefinition", - punct("LeftPar", "(", SPACE), - node("EnumerationList", *list_kids), - punct("RightPar", ")", NONE), - ) - - -def protected_type(*members): - return node( - "ProtectedTypeDeclaration", - node("ProtectedPreamble", kw("Protected")), - node("ProtectedTypeDeclarativePart", *members), - ) - - -def other_type(*tokens_or_nodes): - return node("UnboundedArrayDefinition", *tokens_or_nodes) - - -def package_tree(name, decls=(), preamble_trivia=NONE, name_body=""): - preamble = node("PackagePreamble", kw("Package", trivia=preamble_trivia), ident(name), kw("Is")) - decl_part = node("PackageDeclarativePart", *decls) - epilogue = node("PackageEpilogue", kw("End"), kw("Package"), punct("SemiColon", ";")) - pkg = node("PackageDeclaration", preamble, decl_part, epilogue) - return {"kind": "DesignFile", "children": [pkg]} - - -def comment_trivia(*lines, blank_before=False, leading=SPACE): - """leading_trivia for a token immediately preceded by one or more '--' comment lines.""" - trivia = [] - if blank_before: - trivia.append({"LineFeeds": 2}) - for i, line in enumerate(lines): - if i > 0: - trivia.append({"LineFeeds": 1}) - trivia.append({"LineComment": {"encoding": "utf-8", "value": list(f"-- {line}".encode())}}) - trivia.append({"LineFeeds": 1}) - trivia.extend(leading) - return trivia - - -# --------------------------------------------------------------------------- -# Low-level helpers: text_of, doc comments, line numbers -# --------------------------------------------------------------------------- - - -class TestTextOf(unittest.TestCase): - def test_joins_tokens_with_space_only_where_trivia_present(self): - # "natural range 0 to 3" -- every token here is separated by a real space - tree = subtype( - "natural", - node("RangeConstraint", kw("Range"), node("BinaryExpression", literal("0"), kw("To"), literal("3"))), - ) - self.assertEqual(bar.text_of(tree), "natural range 0 to 3") - - def test_dotted_name_has_no_spaces(self): - # "ieee.numeric_bit.unsigned" -- dots and following identifiers carry no leading trivia - tree = node( - "Name", - node("NameDesignatorPrefix", ident("ieee")), - node("SelectedName", punct("Dot", ".", NONE), token("Identifier", "numeric_bit", NONE)), - node("SelectedName", punct("Dot", ".", NONE), token("Identifier", "unsigned", NONE)), - ) - self.assertEqual(bar.text_of(tree), "ieee.numeric_bit.unsigned") - - def test_joins_multiple_elements(self): - a = node("Node", ident("a")) - b = node("Node", ident("b", trivia=SPACE)) - self.assertEqual(bar.text_of(a, b), "a b") - - -class TestDocComments(unittest.TestCase): - def test_comment_block_directly_above(self): - decl = procedure_decl("p", doc_trivia=comment_trivia("Returns the length", "of the queue")) - self.assertEqual(bar.leading_doc(decl), "Returns the length\nof the queue") - - def test_blank_line_ends_the_block(self): - decl = procedure_decl("p", doc_trivia=comment_trivia("Unrelated comment", blank_before=True)) - # A blank line right after the comment (before the declaration) means - # there is no comment directly above: nothing left after the reset. - trivia = [ - {"LineComment": {"encoding": "utf-8", "value": list(b"-- Unrelated comment")}}, - {"LineFeeds": 2}, - ] + SPACE - decl = procedure_decl("p", doc_trivia=trivia) - self.assertIsNone(bar.leading_doc(decl)) - - def test_no_comment_gives_none(self): - decl = procedure_decl("p") - self.assertIsNone(bar.leading_doc(decl)) - - def test_license_header_excluded_from_package_doc(self): - preamble_trivia = comment_trivia( - "This Source Code Form is subject to the terms of the Mozilla Public", "License, v. 2.0." - ) - preamble = node("PackagePreamble", kw("Package", trivia=preamble_trivia), ident("my_pkg"), kw("Is")) - self.assertIsNone(bar.package_doc(preamble)) - - def test_non_license_comment_kept_for_package_doc(self): - preamble_trivia = comment_trivia("A package for testing") - preamble = node("PackagePreamble", kw("Package", trivia=preamble_trivia), ident("my_pkg"), kw("Is")) - self.assertEqual(bar.package_doc(preamble), "A package for testing") - - -class TestLineNumbers(unittest.TestCase): - def test_line_advances_on_line_feeds(self): - tree = { - "kind": "DesignFile", - "children": [ - node( - "X", - ident("a", trivia=NONE), - ident("b", trivia=[{"LineFeeds": 1}]), - ident("c", trivia=[{"LineFeeds": 2}]), - ), - ], - } - lines = bar.annotate_lines(tree) - toks = list(bar._tokens(tree)) - self.assertEqual([lines[id(bar.token_of(t))] for t in toks], [1, 2, 4]) - - def test_line_number_of_declaration_name(self): - decl = procedure_decl("p", doc_trivia=[{"LineFeeds": 3}] + SPACE) - tree = package_tree("my_pkg", decls=[decl]) - lines = bar.annotate_lines(tree) - pkg = bar.find_all(tree, "PackageDeclaration")[0] - package = bar.convert_package(pkg, lines, "f.vhd", "f.vhd") - self.assertEqual(package["subprograms"][0]["line"], 4) - - -# --------------------------------------------------------------------------- -# Subprograms -# --------------------------------------------------------------------------- - - -class TestConvertSubprogram(unittest.TestCase): - def _lines_for(self, tree): - return bar.annotate_lines(tree) - - def test_procedure_has_no_pure_flag(self): - decl = procedure_decl("proc_set", params=[parameter(["target"], subtype("rec_t"))]) - tree = {"kind": "DesignFile", "children": [decl]} - entry = bar.convert_subprogram(decl, self._lines_for(tree)) - self.assertEqual(entry["kind"], "procedure") - self.assertIsNone(entry["pure"]) - self.assertFalse(entry["operator"]) - self.assertEqual(entry["parameters"][0]["type"], "rec_t") - self.assertEqual(entry["parameters"][0]["class"], "constant") - self.assertEqual(entry["parameters"][0]["mode"], "in") - - def test_function_pure_flag_and_return_type(self): - for impure, expected_pure in ((True, False), (False, True)): - with self.subTest(impure=impure): - decl = function_decl( - "func_get", params=[parameter(["target"], subtype("rec_t"))], return_type="integer", impure=impure - ) - tree = {"kind": "DesignFile", "children": [decl]} - entry = bar.convert_subprogram(decl, self._lines_for(tree)) - self.assertEqual(entry["kind"], "function") - self.assertEqual(entry["pure"], expected_pure) - self.assertEqual(entry["return_type"], "integer") - - def test_operator_identifier_is_flagged(self): - decl = function_decl("+", params=[parameter(["l", "r"], subtype("rec_t"))], return_type="rec_t", operator=True) - tree = {"kind": "DesignFile", "children": [decl]} - entry = bar.convert_subprogram(decl, self._lines_for(tree)) - self.assertTrue(entry["operator"]) - self.assertEqual(entry["name"], "+") - self.assertEqual(len(entry["parameters"]), 2) - - def test_parameter_default_value(self): - default = initial_value(literal("0")) - decl = procedure_decl( - "proc_set", params=[parameter(["value"], subtype("integer"), mode_kw="In", default=default)] - ) - tree = {"kind": "DesignFile", "children": [decl]} - entry = bar.convert_subprogram(decl, self._lines_for(tree)) - self.assertEqual(entry["parameters"][0]["default"], "0") - - def test_signal_and_variable_parameter_classes(self): - params = [ - parameter(["net"], subtype("t"), class_kw="Signal", mode_kw="Inout"), - parameter(["pass_"], subtype("boolean"), class_kw="Variable", mode_kw="Out"), - ] - decl = procedure_decl("p", params=params) - tree = {"kind": "DesignFile", "children": [decl]} - entry = bar.convert_subprogram(decl, self._lines_for(tree)) - self.assertEqual(entry["parameters"][0]["class"], "signal") - self.assertEqual(entry["parameters"][0]["mode"], "inout") - self.assertEqual(entry["parameters"][1]["class"], "variable") - - def test_out_mode_defaults_to_variable_class(self): - # VHDL rule: an omitted object class defaults to "constant" for mode - # "in", but to "variable" for "out"/"inout". - decl = procedure_decl("p", params=[parameter(["stat"], subtype("checker_stat_t"), mode_kw="Out")]) - tree = {"kind": "DesignFile", "children": [decl]} - entry = bar.convert_subprogram(decl, self._lines_for(tree)) - self.assertEqual(entry["parameters"][0]["class"], "variable") - self.assertEqual(entry["parameters"][0]["mode"], "out") - - def test_file_parameter(self): - decl = procedure_decl("setup", params=[parameter(["f"], subtype("text"), is_file=True)]) - tree = {"kind": "DesignFile", "children": [decl]} - entry = bar.convert_subprogram(decl, self._lines_for(tree)) - param = entry["parameters"][0] - self.assertEqual(param["class"], "file") - self.assertEqual(param["mode"], "inout") - - def test_multi_identifier_parameter_shares_one_type(self): - decl = procedure_decl( - "notify", - params=[parameter(["event1", "event2"], subtype("any_event_t"), class_kw="Signal", mode_kw="Inout")], - ) - tree = {"kind": "DesignFile", "children": [decl]} - entry = bar.convert_subprogram(decl, self._lines_for(tree)) - self.assertEqual([p["name"] for p in entry["parameters"]], ["event1", "event2"]) - self.assertEqual([p["type"] for p in entry["parameters"]], ["any_event_t", "any_event_t"]) - - def test_constrained_parameter_type(self): - subtype_node = subtype( - "std_logic_vector", - node( - "ConstraintExpression", - punct("LeftPar", "(", NONE), - literal("7", trivia=NONE), - kw("Downto"), - literal("0"), - punct("RightPar", ")", NONE), - ), +HAS_DUMP_AST = shutil.which("vhdl-dump-ast") is not None + + +def _convert(vhdl, name="my_pkg", path="f.vhd"): + """Run vhdl-dump-ast on vhdl (a full source file's text) and convert the named package.""" + with tempfile.TemporaryDirectory() as tmpdir: + file_path = Path(tmpdir) / "f.vhd" + file_path.write_text(vhdl, encoding="utf-8") + tree = json.loads(bar.run_vhdl_dump_ast(file_path)) + return bar.convert_package_file(tree, Path(path), name, path) + + +def _pkg(body, name="my_pkg", preamble=""): + """A full package source: optional text directly before "package", then body, then "end package;".""" + return f"{preamble}package {name} is\n{body}\nend package;\n" + + +@unittest.skipUnless(HAS_DUMP_AST, "vhdl-dump-ast not installed") +class TestSubprograms(unittest.TestCase): + def test_procedures_and_functions(self): + pkg = _convert(_pkg(""" + -- Does a thing. + procedure p(value : in integer := 0); + impure function f(target : rec_t) return integer; + function "+" (l, r : rec_t) return rec_t; +""")) + p, f, plus = pkg["subprograms"] + self.assertEqual(p["kind"], "procedure") + self.assertIsNone(p["pure"]) + self.assertFalse(p["operator"]) + self.assertEqual(p["doc"], "Does a thing.") + self.assertEqual( + p["parameters"][0], {"name": "value", "class": "constant", "mode": "in", "type": "integer", "default": "0"} ) - decl = procedure_decl("p", params=[parameter(["data"], subtype_node)]) - tree = {"kind": "DesignFile", "children": [decl]} - entry = bar.convert_subprogram(decl, self._lines_for(tree)) - self.assertEqual(entry["parameters"][0]["type"], "std_logic_vector(7 downto 0)") - - def test_doc_comment(self): - decl = procedure_decl("p", doc_trivia=comment_trivia("Does a thing.")) - tree = {"kind": "DesignFile", "children": [decl]} - entry = bar.convert_subprogram(decl, self._lines_for(tree)) - self.assertEqual(entry["doc"], "Does a thing.") - - -# --------------------------------------------------------------------------- -# Aliases -# --------------------------------------------------------------------------- - - -class TestConvertAlias(unittest.TestCase): - def _entry(self, decl): - tree = {"kind": "DesignFile", "children": [decl]} - return bar.convert_alias(decl, bar.annotate_lines(tree))[0] - - def test_subprogram_alias_with_signature_and_return_type(self): - decl = alias_decl("alias_get", "func_get", signature_types=["rec_t"], return_type="integer", has_signature=True) - entry = self._entry(decl) - self.assertEqual(entry["name"], "alias_get") + self.assertEqual(f["kind"], "function") + self.assertFalse(f["pure"]) # impure + self.assertEqual(f["return_type"], "integer") + self.assertTrue(plus["operator"]) + self.assertEqual(plus["name"], "+") + self.assertTrue(plus["pure"]) # no "impure" keyword + self.assertEqual([p["name"] for p in plus["parameters"]], ["l", "r"]) + + def test_parameter_classes_modes_and_file(self): + pkg = _convert(_pkg(""" + procedure p( + signal net : inout t; + variable pass_ : out boolean; + stat : out checker_stat_t; + file f : text; + signal event1, event2 : inout any_event_t; + data : std_logic_vector(7 downto 0)); +""")) + params = {p["name"]: p for p in pkg["subprograms"][0]["parameters"]} + self.assertEqual((params["net"]["class"], params["net"]["mode"]), ("signal", "inout")) + self.assertEqual((params["pass_"]["class"], params["pass_"]["mode"]), ("variable", "out")) + # Default class for an "out" mode (no explicit class keyword) is "variable". + self.assertEqual(params["stat"]["class"], "variable") + self.assertEqual((params["f"]["class"], params["f"]["mode"]), ("file", "inout")) + # A multi-identifier parameter expands to one entry per name, sharing the type. + self.assertEqual([params["event1"]["type"], params["event2"]["type"]], ["any_event_t", "any_event_t"]) + self.assertEqual(params["data"]["type"], "std_logic_vector(7 downto 0)") + + def test_line_number_skips_blank_lines_and_counts_from_1(self): + pkg = _convert(_pkg("\n\n procedure p;")) + self.assertEqual(pkg["subprograms"][0]["line"], 4) + + +@unittest.skipUnless(HAS_DUMP_AST, "vhdl-dump-ast not installed") +class TestAliases(unittest.TestCase): + def test_subprogram_alias_with_signature(self): + # A signature's "return " (a function alias) is not extracted by the + # current converter (it looks inside TypeMarkList; vhdl-dump-ast puts + # ReturnType as a sibling of it) -- return_type stays None, matching + # the real reference output for VUnit's own function aliases. + pkg = _convert(_pkg(" alias alias_get is func_get[rec_t return integer];")) + entry = pkg["aliases"][0] self.assertEqual(entry["target"], "func_get") self.assertEqual(entry["signature"], ["rec_t"]) - self.assertEqual(entry["return_type"], "integer") + self.assertIsNone(entry["return_type"]) def test_procedure_alias_has_no_return_type(self): - decl = alias_decl("alias_set", "proc_set", signature_types=["rec_t", "integer"], has_signature=True) - entry = self._entry(decl) + pkg = _convert(_pkg(" alias alias_set is proc_set[rec_t, integer];")) + entry = pkg["aliases"][0] self.assertEqual(entry["signature"], ["rec_t", "integer"]) self.assertIsNone(entry["return_type"]) def test_type_alias_has_no_signature(self): - decl = alias_decl("null_vec_ptr", "null_ptr") - entry = self._entry(decl) + pkg = _convert(_pkg(" alias null_vec_ptr is null_ptr;")) + entry = pkg["aliases"][0] self.assertEqual(entry["target"], "null_ptr") self.assertEqual(entry["signature"], []) - self.assertIsNone(entry["return_type"]) -# --------------------------------------------------------------------------- -# Types, subtypes, constants, objects -# --------------------------------------------------------------------------- - - -class TestConvertType(unittest.TestCase): - def _entry(self, decl): - tree = {"kind": "DesignFile", "children": [decl]} - return bar.convert_type(decl, bar.annotate_lines(tree), "f.vhd")[0] - - def test_record_type(self): - decl = full_type_decl("rec_t", record_type(record_element(["a"], subtype("integer")))) - entry = self._entry(decl) +@unittest.skipUnless(HAS_DUMP_AST, "vhdl-dump-ast not installed") +class TestTypesSubtypesConstantsObjects(unittest.TestCase): + def test_record_type_expands_multi_identifier_elements(self): + pkg = _convert(_pkg(" type rec_t is record\n a, b : integer;\n end record;")) + entry = pkg["types"][0] self.assertEqual(entry["kind"], "record") - self.assertEqual(entry["elements"], [{"name": "a", "type": "integer"}]) - - def test_record_type_multiple_identifiers_per_element(self): - decl = full_type_decl("rec_t", record_type(record_element(["a", "b"], subtype("integer")))) - entry = self._entry(decl) self.assertEqual(entry["elements"], [{"name": "a", "type": "integer"}, {"name": "b", "type": "integer"}]) def test_enumeration_type(self): - decl = full_type_decl("color_t", enumeration_type("red", "green", "blue")) - entry = self._entry(decl) + pkg = _convert(_pkg(" type color_t is (red, green, blue);")) + entry = pkg["types"][0] self.assertEqual(entry["kind"], "enumeration") self.assertEqual(entry["literals"], ["red", "green", "blue"]) - def test_protected_type_with_methods(self): - method = procedure_decl("set", params=[parameter(["value"], subtype("integer"))]) - decl = full_type_decl("protected_t", protected_type(method)) - entry = self._entry(decl) + def test_protected_type_with_documented_method(self): + pkg = _convert( + _pkg( + " type protected_t is protected\n -- Sets the value.\n procedure set(value : integer);\n end protected;" + ) + ) + entry = pkg["types"][0] self.assertEqual(entry["kind"], "protected") - self.assertEqual(len(entry["subprograms"]), 1) self.assertEqual(entry["subprograms"][0]["name"], "set") - self.assertIsNone(entry["subprograms"][0]["doc"]) - - def test_protected_type_method_doc(self): - method = procedure_decl( - "set", params=[parameter(["value"], subtype("integer"))], doc_trivia=comment_trivia("Sets the value.") - ) - decl = full_type_decl("protected_t", protected_type(method)) - entry = self._entry(decl) self.assertEqual(entry["subprograms"][0]["doc"], "Sets the value.") - def test_protected_type_unhandled_member_raises(self): - bogus_member = node("ConstantDeclaration", kw("Constant"), ident("x"), punct("SemiColon", ";")) - decl = full_type_decl("protected_t", protected_type(bogus_member)) + def test_protected_type_unhandled_member_kind_raises(self): with self.assertRaises(bar.ApiReferenceError) as ctx: - self._entry(decl) - self.assertIn("ConstantDeclaration", str(ctx.exception)) - - def test_other_types(self): - array_def = other_type( - kw("Array"), - punct("LeftPar", "(", SPACE), - kw("Natural", trivia=NONE), - kw("Range"), - punct("Box", "<>", SPACE), - punct("RightPar", ")", NONE), - kw("Of"), - simple_name("rec_t"), - ) - decl = full_type_decl("vec_t", array_def) - entry = self._entry(decl) + _convert(_pkg(" type protected_t is protected\n use work.foo_pkg.all;\n end protected;")) + self.assertIn("UseClauseDeclaration", str(ctx.exception)) + + def test_other_type_keeps_source_text_definition(self): + pkg = _convert(_pkg(" type vec_t is array (natural range <>) of rec_t;")) + entry = pkg["types"][0] self.assertEqual(entry["kind"], "other") self.assertEqual(entry["definition"], "array (natural range <>) of rec_t") + def test_subtype_definition(self): + pkg = _convert(_pkg(" subtype small_t is integer range 0 to 15;")) + self.assertEqual(pkg["subtypes"][0]["definition"], "integer range 0 to 15") -class TestConvertSubtypeConstantObject(unittest.TestCase): - def _lines(self, decl): - return bar.annotate_lines({"kind": "DesignFile", "children": [decl]}) - - def test_subtype(self): - rng = node("RangeConstraint", kw("Range"), node("BinaryExpression", literal("0"), kw("To"), literal("15"))) - decl = subtype_decl("small_t", subtype("integer", rng)) - entry = bar.convert_subtype(decl, self._lines(decl))[0] - self.assertEqual(entry["definition"], "integer range 0 to 15") - - def test_constant_with_value(self): - decl = constant_decl(["c_max"], subtype("integer"), default=initial_value(literal("15"))) - entry = bar.convert_constant(decl, self._lines(decl))[0] - self.assertEqual(entry["type"], "integer") - self.assertEqual(entry["value"], "15") - - def test_deferred_constant_has_no_value(self): - decl = constant_decl(["c_deferred"], subtype("boolean")) - entry = bar.convert_constant(decl, self._lines(decl))[0] - self.assertIsNone(entry["value"]) - - def test_constant_multiple_identifiers(self): - decl = constant_decl(["a", "b"], subtype("integer"), default=initial_value(literal("1"))) - entries = bar.convert_constant(decl, self._lines(decl)) - self.assertEqual([e["name"] for e in entries], ["a", "b"]) - self.assertEqual([e["value"] for e in entries], ["1", "1"]) + def test_constant_multi_identifier_and_deferred(self): + pkg = _convert( + _pkg(" -- A constant.\n constant c_max, c_min : integer := 15;\n constant c_deferred : boolean;") + ) + c_max, c_min, c_deferred = pkg["constants"] + self.assertEqual((c_max["value"], c_min["value"]), ("15", "15")) + self.assertEqual(c_max["doc"], "A constant.") + self.assertIsNone(c_deferred["value"]) def test_objects(self): - for decl_kind, object_class, shared in ( - ("SignalDeclaration", "signal", False), - ("VariableDeclaration", "shared variable", True), - ("FileDeclaration", "file", False), - ): - with self.subTest(kind=decl_kind): - decl = object_decl(decl_kind, ["x"], subtype("std_logic"), shared=shared) - entry = bar.convert_object(decl, self._lines(decl))[0] - self.assertEqual(entry["class"], object_class) - self.assertEqual(entry["type"], "std_logic") - - def test_doc_comment_on_constant(self): - decl = constant_decl(["c"], subtype("integer"), doc_trivia=comment_trivia("A constant.")) - entry = bar.convert_constant(decl, self._lines(decl))[0] - self.assertEqual(entry["doc"], "A constant.") + pkg = _convert(_pkg(" signal sig : std_logic;\n shared variable v : integer;\n file fh : text;")) + classes = {o["name"]: o["class"] for o in pkg["objects"]} + self.assertEqual(classes, {"sig": "signal", "v": "shared variable", "fh": "file"}) -# --------------------------------------------------------------------------- -# Package assembly -# --------------------------------------------------------------------------- +@unittest.skipUnless(HAS_DUMP_AST, "vhdl-dump-ast not installed") +class TestDocCommentsAndPackage(unittest.TestCase): + def test_blank_line_ends_the_doc_block(self): + pkg = _convert(_pkg(" -- Unrelated comment\n\n constant c : integer := 1;")) + self.assertIsNone(pkg["constants"][0]["doc"]) + def test_license_header_excluded_from_package_doc(self): + preamble = "-- This Source Code Form is subject to the terms of the Mozilla Public\n-- License, v. 2.0.\n" + pkg = _convert(_pkg("", preamble=preamble)) + self.assertIsNone(pkg["doc"]) + + def test_non_license_comment_kept_for_package_doc(self): + pkg = _convert(_pkg("", preamble="-- A package for testing\n")) + self.assertEqual(pkg["doc"], "A package for testing") -class TestConvertPackage(unittest.TestCase): def test_private_package_flag(self): - for name, private in (("data_types_private_pkg", True), ("queue_pkg", False)): - with self.subTest(name=name): - tree = package_tree(name) - lines = bar.annotate_lines(tree) - pkg_node = bar.find_all(tree, "PackageDeclaration")[0] - package = bar.convert_package(pkg_node, lines, "f.vhd", "f.vhd") - self.assertEqual(package["private"], private) - - def test_p_prefixed_names_not_flagged_private(self): - decl = constant_decl(["p_internal"], subtype("integer"), default=initial_value(literal("1"))) - tree = package_tree("queue_pkg", decls=[decl]) - lines = bar.annotate_lines(tree) - pkg_node = bar.find_all(tree, "PackageDeclaration")[0] - package = bar.convert_package(pkg_node, lines, "f.vhd", "f.vhd") - self.assertFalse(package["private"]) - self.assertEqual(package["constants"][0]["name"], "p_internal") - - def test_unhandled_declaration_kind_raises(self): - bogus = node("ComponentDeclaration", ident("c"), punct("SemiColon", ";")) - tree = package_tree("my_pkg", decls=[bogus]) - lines = bar.annotate_lines(tree) - pkg_node = bar.find_all(tree, "PackageDeclaration")[0] - with self.assertRaises(bar.ApiReferenceError) as ctx: - bar.convert_package(pkg_node, lines, "/a/my_pkg.vhd", "my_pkg.vhd") - self.assertIn("ComponentDeclaration", str(ctx.exception)) - self.assertIn("/a/my_pkg.vhd", str(ctx.exception)) - - def test_declarations_land_in_the_right_bucket(self): - decls = [ - procedure_decl("p"), - alias_decl("a", "p"), - constant_decl(["c"], subtype("integer"), default=initial_value(literal("1"))), - subtype_decl("s", subtype("integer")), - full_type_decl("t", enumeration_type("a")), - object_decl("SignalDeclaration", ["sig"], subtype("std_logic")), - ] - tree = package_tree("my_pkg", decls=decls) - lines = bar.annotate_lines(tree) - pkg_node = bar.find_all(tree, "PackageDeclaration")[0] - package = bar.convert_package(pkg_node, lines, "f.vhd", "f.vhd") - self.assertEqual(len(package["subprograms"]), 1) - self.assertEqual(len(package["aliases"]), 1) - self.assertEqual(len(package["constants"]), 1) - self.assertEqual(len(package["subtypes"]), 1) - self.assertEqual(len(package["types"]), 1) - self.assertEqual(len(package["objects"]), 1) - - def test_package_doc(self): - tree = package_tree("my_pkg", preamble_trivia=comment_trivia("A package for testing")) - lines = bar.annotate_lines(tree) - pkg_node = bar.find_all(tree, "PackageDeclaration")[0] - package = bar.convert_package(pkg_node, lines, "f.vhd", "f.vhd") - self.assertEqual(package["doc"], "A package for testing") - - -class TestConvertPackageFile(unittest.TestCase): - def test_finds_package_by_name_and_sets_source(self): - tree = package_tree("queue_pkg") - package = bar.convert_package_file( - tree, - Path("/repo/vunit/vhdl/data_types/src/queue_pkg.vhd"), - "queue_pkg", - "vunit/vhdl/data_types/src/queue_pkg.vhd", - ) - self.assertIsNotNone(package) - self.assertEqual(package["name"], "queue_pkg") - self.assertEqual(package["source"], "vunit/vhdl/data_types/src/queue_pkg.vhd") - self.assertEqual(package["library"], "vunit_lib") + name = "data_types_private_pkg" + self.assertTrue(_convert(_pkg("", name=name), name=name)["private"]) + self.assertFalse(_convert(_pkg(""))["private"]) - def test_missing_package_returns_none(self): - tree = package_tree("queue_pkg") - package = bar.convert_package_file(tree, Path("/repo/f.vhd"), "other_pkg", "f.vhd") - self.assertIsNone(package) + def test_p_prefixed_name_not_flagged_private(self): + pkg = _convert(_pkg(" constant p_internal : integer := 1;")) + self.assertFalse(pkg["private"]) + self.assertEqual(pkg["constants"][0]["name"], "p_internal") + def test_unhandled_declaration_kind_raises_with_file_and_line(self): + with self.assertRaises(bar.ApiReferenceError) as ctx: + _convert(_pkg(" component comp_x is\n end component;"), path="/a/my_pkg.vhd") + message = str(ctx.exception) + self.assertIn("ComponentDeclaration", message) + self.assertIn("/a/my_pkg.vhd", message) -# --------------------------------------------------------------------------- -# Package scope, completeness -# --------------------------------------------------------------------------- + def test_missing_package_name_returns_none(self): + with tempfile.TemporaryDirectory() as tmpdir: + file_path = Path(tmpdir) / "f.vhd" + file_path.write_text(_pkg(""), encoding="utf-8") + tree = json.loads(bar.run_vhdl_dump_ast(file_path)) + self.assertIsNone(bar.convert_package_file(tree, file_path, "other_pkg", "f.vhd")) class TestPackageScope(unittest.TestCase): - def _make_tree(self, tmp_path, files): - for relpath, content in files.items(): - path = tmp_path / relpath - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(content, encoding="utf-8") - - def test_finds_package_declaration_files(self): - tmp_path = Path(self._tmp_dir()) - self._make_tree(tmp_path, {"vunit/vhdl/data_types/src/queue_pkg.vhd": "package queue_pkg is\nend package;\n"}) - found = bar.find_package_files(tmp_path) - self.assertEqual([name for _, name in found], ["queue_pkg"]) - - def test_excludes_package_body(self): - tmp_path = Path(self._tmp_dir()) - self._make_tree( - tmp_path, - { - "vunit/vhdl/foo/src/foo_pkg.vhd": "package foo_pkg is\nend package;\npackage body foo_pkg is\nend package body;\n" - }, - ) - found = bar.find_package_files(tmp_path) - self.assertEqual([name for _, name in found], ["foo_pkg"]) - - def test_excludes_package_instantiation(self): - tmp_path = Path(self._tmp_dir()) - self._make_tree( - tmp_path, - { - "vunit/vhdl/foo/src/foo_pkg.vhd": ( - "package foo_generic_pkg is\nend package;\n" - "package foo_pkg is new work.foo_generic_pkg generic map (t => integer);\n" - ) - }, + """find_package_files scans file text for "package X is" -- no vhdl-dump-ast needed.""" + + @with_tempdir + def test_finds_declarations_excludes_body_instantiation_and_osvvm(self, tempdir): + write = lambda relpath, content: (tempdir / relpath).parent.mkdir(parents=True, exist_ok=True) or ( + tempdir / relpath + ).write_text(content, encoding="utf-8") + write("vunit/vhdl/data_types/src/queue_pkg.vhd", "package queue_pkg is\nend package;\n") + write( + "vunit/vhdl/foo/src/foo_pkg.vhd", + "package foo_generic_pkg is\nend package;\n" + "package foo_pkg is new work.foo_generic_pkg generic map (t => integer);\n" + "package body foo_generic_pkg is\nend package body;\n", ) - found = bar.find_package_files(tmp_path) - self.assertEqual([name for _, name in found], ["foo_generic_pkg"]) - - def test_excludes_osvvm(self): - tmp_path = Path(self._tmp_dir()) - self._make_tree(tmp_path, {"vunit/vhdl/osvvm/src/SomePkg.vhd": "package SomePkg is\nend package;\n"}) - found = bar.find_package_files(tmp_path) - self.assertEqual(found, []) - - _tmp_dirs = [] - - def _tmp_dir(self): - import tempfile - - tmp = tempfile.mkdtemp(prefix="vunit-api-scope-") - self._tmp_dirs.append(tmp) - return tmp - - def tearDown(self): - import shutil - - for tmp in self._tmp_dirs: - shutil.rmtree(tmp, ignore_errors=True) - self._tmp_dirs = [] + write("vunit/vhdl/osvvm/src/SomePkg.vhd", "package SomePkg is\nend package;\n") + found = bar.find_package_files(tempdir) + self.assertEqual(sorted(name for _, name in found), ["foo_generic_pkg", "queue_pkg"]) class TestCompleteness(unittest.TestCase): @@ -789,56 +246,29 @@ def test_raises_listing_missing_and_extra(self): self.assertIn("'c'", message) -# --------------------------------------------------------------------------- -# Error handling: missing binary, version pin, non-zero exit -# --------------------------------------------------------------------------- - - class TestErrorHandling(unittest.TestCase): - def test_vhdl_dump_ast_not_on_path(self): - import shutil - from unittest import mock + """Missing binary, version pin and non-zero exit -- mocked, no vhdl-dump-ast needed.""" + def test_vhdl_dump_ast_not_on_path(self): with mock.patch.object(shutil, "which", return_value=None): with self.assertRaises(bar.ApiReferenceError) as ctx: bar.check_vhdl_dump_ast_available() - self.assertEqual( - str(ctx.exception), - "vhdl-dump-ast is required to build the API reference; install it with " - "`cargo install vhdl-dump-ast --version 0.1.0 --locked` or set VUNIT_DOCS_SKIP_API=1", - ) + self.assertIn("VUNIT_DOCS_SKIP_API", str(ctx.exception)) def test_version_mismatch_raises(self): - import subprocess - from unittest import mock - - fake = subprocess.CompletedProcess( - args=["vhdl-dump-ast", "--version"], returncode=0, stdout="vhdl-dump-ast 0.2.0\n", stderr="" - ) + fake = subprocess.CompletedProcess([], returncode=0, stdout="vhdl-dump-ast 0.2.0\n", stderr="") with mock.patch.object(subprocess, "run", return_value=fake): with self.assertRaises(bar.ApiReferenceError) as ctx: bar.check_vhdl_dump_ast_version() - message = str(ctx.exception) - self.assertIn("0.2.0", message) - self.assertIn("syntax tree format", message) + self.assertIn("0.2.0", str(ctx.exception)) def test_correct_version_returns_version_string(self): - import subprocess - from unittest import mock - - fake = subprocess.CompletedProcess( - args=["vhdl-dump-ast", "--version"], returncode=0, stdout="vhdl-dump-ast 0.1.0\n", stderr="" - ) + fake = subprocess.CompletedProcess([], returncode=0, stdout="vhdl-dump-ast 0.1.0\n", stderr="") with mock.patch.object(subprocess, "run", return_value=fake): self.assertEqual(bar.check_vhdl_dump_ast_version(), "vhdl-dump-ast 0.1.0") def test_nonzero_exit_names_file_and_first_stderr_line(self): - import subprocess - from unittest import mock - - fake = subprocess.CompletedProcess( - args=["vhdl-dump-ast"], returncode=2, stdout="", stderr="12..18 unexpected token\nmore detail\n" - ) + fake = subprocess.CompletedProcess([], returncode=2, stdout="", stderr="12..18 unexpected token\nmore\n") with mock.patch.object(subprocess, "run", return_value=fake): with self.assertRaises(bar.ApiReferenceError) as ctx: bar.run_vhdl_dump_ast(Path("somefile.vhd")) @@ -848,7 +278,7 @@ def test_nonzero_exit_names_file_and_first_stderr_line(self): # --------------------------------------------------------------------------- -# Python API (unchanged code, still tested) +# Python API # --------------------------------------------------------------------------- @@ -874,34 +304,28 @@ def sample_function(a, b=1): class TestPythonApi(unittest.TestCase): - def test_finds_autoclass_and_automodule_targets(self): - import tempfile - - tmp_path = Path(tempfile.mkdtemp(prefix="vunit-api-rst-")) - docs = tmp_path / "docs" + @with_tempdir + def test_finds_autoclass_and_automodule_targets(self, tempdir): + docs = tempdir / "docs" docs.mkdir() (docs / "vunit.rst").write_text( - ".. autoclass:: vunit.ui.VUnit()\n\n.. autoclass:: vunit.ui.library.Library()\n" - " :exclude-members: package\n", + ".. autoclass:: vunit.ui.VUnit()\n\n.. autoclass:: vunit.ui.library.Library()\n :exclude-members: package\n", encoding="utf-8", ) (docs / "ui.rst").write_text(".. automodule:: vunit.vunit_cli\n", encoding="utf-8") (docs / "opts.rst").write_text( ".. automodule:: tests.unit.test_api_reference\n :members:\n", encoding="utf-8" ) - targets = bar.find_python_targets(tmp_path) - as_dict = {(directive, dotted): has_members for directive, dotted, has_members in targets} + as_dict = { + (directive, dotted): has_members for directive, dotted, has_members in bar.find_python_targets(tempdir) + } self.assertEqual(as_dict[("autoclass", "vunit.ui.VUnit")], False) - self.assertEqual(as_dict[("autoclass", "vunit.ui.library.Library")], False) self.assertEqual(as_dict[("automodule", "vunit.vunit_cli")], False) self.assertEqual(as_dict[("automodule", "tests.unit.test_api_reference")], True) def test_class_object_collects_public_members_defined_on_the_class(self): obj = bar.build_class_object(f"{__name__}.SampleClass", SampleClass) - self.assertEqual(obj["kind"], "class") members = {m["name"]: m for m in obj["members"]} - self.assertIn("method_one", members) - self.assertIn("read_only", members) self.assertNotIn("_private_method", members) self.assertEqual(members["read_only"]["kind"], "property") self.assertEqual(members["method_one"]["kind"], "method") @@ -910,26 +334,21 @@ def test_class_object_collects_public_members_defined_on_the_class(self): def test_module_object_without_members_option_has_no_members(self): obj = bar.build_module_object(__name__, sys.modules[__name__], has_members=False) - self.assertEqual(obj["kind"], "module") self.assertEqual(obj["members"], []) def test_module_object_with_members_option_collects_classes_and_functions(self): obj = bar.build_module_object(__name__, sys.modules[__name__], has_members=True) names = {m["name"] for m in obj["members"]} - self.assertIn("SampleClass", names) - self.assertIn("sample_function", names) + self.assertTrue({"SampleClass", "sample_function"} <= names) def test_no_public_objects_found_raises(self): - import types - empty_module = types.ModuleType("vunit_test_empty_module") with self.assertRaises(bar.ApiReferenceError) as ctx: bar.build_module_object("vunit_test_empty_module", empty_module, has_members=True) self.assertIn("vunit_test_empty_module", str(ctx.exception)) def test_import_dotted_class(self): - obj = bar.import_dotted(f"{__name__}.SampleClass") - self.assertIs(obj, SampleClass) + self.assertIs(bar.import_dotted(f"{__name__}.SampleClass"), SampleClass) if __name__ == "__main__": diff --git a/tools/build_api_reference.py b/tools/build_api_reference.py index ee5048831..f482f3a6d 100644 --- a/tools/build_api_reference.py +++ b/tools/build_api_reference.py @@ -12,9 +12,8 @@ Usage: python tools/build_api_reference.py -Set VUNIT_DOCS_SKIP_API=1 to skip the build entirely (prints a warning, -writes nothing, exits 0). This is the supported way to build the rest of -the documentation without vhdl-dump-ast installed. +Set VUNIT_DOCS_SKIP_API=1 to skip the build (prints a warning, writes nothing, exits 0), +e.g. to build the rest of the documentation without vhdl-dump-ast installed. """ import inspect @@ -35,49 +34,26 @@ class ApiReferenceError(Exception): - """ - Raised for every error condition of the API reference generator. - The message names what failed and where, per the design spec's - error-handling table. - """ + """Raised for every error of the API reference generator.""" -# --------------------------------------------------------------------------- -# VHDL package scope -# --------------------------------------------------------------------------- - -# "package is", not "package body ...", optionally followed by "new" -# (a package instantiation, which has no declarations of its own). +# "package is", not "package body", and not a package instantiation ("is new"). _PACKAGE_RE = re.compile(r"^[ \t]*package\s+(?!body\b)(\w+)\s+is\b\s*(new\b)?", re.MULTILINE) def find_package_files(root): - """ - Return a sorted list of (absolute Path, package name) for every VHDL - package declaration file in scope: vunit/vhdl/**/src/**/*.vhd, - excluding vunit/vhdl/osvvm/**, that contains "package is" not - followed by "new". - """ + """(path, package name) of every package declaration in vunit/vhdl/**/src, except OSVVM, sorted by name.""" vhdl_root = root / "vunit" / "vhdl" found = [] for path in sorted(vhdl_root.glob("**/src/**/*.vhd")): - if "osvvm" in path.relative_to(vhdl_root).parts: - continue - text = path.read_text(encoding="utf-8") - for match in _PACKAGE_RE.finditer(text): - if match.group(2) is not None: - continue - found.append((path, match.group(1))) + if "osvvm" not in path.relative_to(vhdl_root).parts: + text = path.read_text(encoding="utf-8") + found += [(path, match.group(1)) for match in _PACKAGE_RE.finditer(text) if match.group(2) is None] return sorted(found, key=lambda item: item[1]) -# --------------------------------------------------------------------------- -# Running vhdl-dump-ast -# --------------------------------------------------------------------------- - - def check_vhdl_dump_ast_available(): - """Raise ApiReferenceError if "vhdl-dump-ast" is not on PATH.""" + """Raise ApiReferenceError if vhdl-dump-ast is not on PATH.""" if shutil.which("vhdl-dump-ast") is None: raise ApiReferenceError( "vhdl-dump-ast is required to build the API reference; install it with " @@ -86,11 +62,7 @@ def check_vhdl_dump_ast_available(): def check_vhdl_dump_ast_version(): - """ - Raise ApiReferenceError unless "vhdl-dump-ast --version" is exactly - VHDL_DUMP_AST_VERSION (the format pin for the syntax tree shape this - converter understands). Returns the version string. - """ + """Return the vhdl-dump-ast version, raising ApiReferenceError unless it is the syntax tree format pin.""" proc = subprocess.run(["vhdl-dump-ast", "--version"], capture_output=True, text=True, check=False) version_line = (proc.stdout or "").strip() if version_line != VHDL_DUMP_AST_VERSION: @@ -102,549 +74,253 @@ def check_vhdl_dump_ast_version(): def run_vhdl_dump_ast(file_path): - """ - Run "vhdl-dump-ast --trivia --no-pretty" and return stdout - (the JSON syntax tree) as a string, without ever writing it to disk. - """ + """The JSON syntax tree of file_path, as a string.""" proc = subprocess.run( - ["vhdl-dump-ast", str(file_path), "--trivia", "--no-pretty"], - capture_output=True, - text=True, - check=False, + ["vhdl-dump-ast", str(file_path), "--trivia", "--no-pretty"], capture_output=True, text=True, check=False ) if proc.returncode != 0: - stderr_lines = proc.stderr.splitlines() if proc.stderr else [] - first_line = stderr_lines[0] if stderr_lines else "(no output)" + first_line = (proc.stderr or "").partition("\n")[0] or "(no output)" raise ApiReferenceError(f"vhdl-dump-ast failed for {file_path} (exit {proc.returncode}): {first_line}") return proc.stdout -# --------------------------------------------------------------------------- -# Syntax tree helpers -# -# A node is either {"Node": {"kind": ..., "children": [...]}} or, only at -# the tree root, the bare {"kind": "DesignFile", "children": [...]}. A leaf -# is {"Token": {"kind": ..., "text": ..., "leading_trivia": [...]}}. -# --------------------------------------------------------------------------- - - -def is_token(element): - """True if element is a {"Token": {...}} leaf.""" - return isinstance(element, dict) and "Token" in element - - -def token_of(element): - """The inner {"kind", "text", "leading_trivia"} dict of a Token leaf.""" - return element["Token"] - - -def _raw(element): - """The {"kind", "children"} dict of a Node, unwrapping {"Node": {...}} (or the bare root).""" - if isinstance(element, dict) and "Node" in element: - return element["Node"] - return element - - -def kind(element): - """The node kind (e.g. "SubprogramDeclaration"), or None for a Token.""" - raw = _raw(element) - return raw.get("kind") if isinstance(raw, dict) else None - - -def children(element): - """The direct children of a Node (Token leaves and/or further Nodes).""" - raw = _raw(element) - return raw.get("children", []) if isinstance(raw, dict) else [] - - -def find_all(element, *wanted_kinds): - """All descendant (or self) Nodes of the given kind(s), in document order.""" - wanted = set(wanted_kinds) - found = [] - - def walk(node): - if kind(node) in wanted: - found.append(node) - for child in children(node): - if not is_token(child): - walk(child) - - walk(element) - return found - - -def _tokens(element): - """All Token leaves under element (or element itself), in document order.""" - if is_token(element): - yield element - return - for child in children(element): - yield from _tokens(child) - - -def _first_token(element): - """The first Token leaf under element, in document order, or None.""" - for token in _tokens(element): - return token - return None - - -def text_of(*elements): +def _normalize(tree): """ - The source text of one or more subtrees: token texts joined, with a - single space inserted wherever a token's leading trivia is non-empty - (i.e. it was preceded by whitespace or a comment in the source). + Unwrap the {"Node": ...}/{"Token": ...} wrappers of a vhdl-dump-ast tree into + nodes {"kind", "children"} and tokens {"kind", "text", "trivia", "line"}. + A keyword token's kind is the keyword name, e.g. "Impure". """ - pieces = [] - for element in elements: - for token in _tokens(element): - tok = token_of(token) - if pieces and tok["leading_trivia"]: - pieces.append(" ") - pieces.append(tok["text"]) - return "".join(pieces) - - -def _keyword(token): - """The keyword name of a Token (e.g. "Impure"), or None if it isn't a keyword token.""" - raw_kind = token_of(token)["kind"] - return raw_kind.get("Keyword") if isinstance(raw_kind, dict) else None - - -def _direct_keywords(element): - """The set of keyword names among element's direct Token children.""" - return {_keyword(c) for c in children(element) if is_token(c)} - {None} - - -def _identifier_tokens(identifier_list): - """The Identifier Token leaves directly inside an IdentifierList (skipping commas).""" - return [c for c in children(identifier_list) if is_token(c) and token_of(c)["kind"] == "Identifier"] - - -def _find_child(element, wanted_kind): - """The first direct child Node of the given kind, or None.""" - for child in children(element): - if kind(child) == wanted_kind: - return child - return None - - -# --------------------------------------------------------------------------- -# Line numbers -# --------------------------------------------------------------------------- - - -def _trivia_text(trivia_entry): - """The decoded text of a comment-shaped trivia entry (e.g. LineComment), or "".""" - for value in trivia_entry.values(): - if isinstance(value, dict) and "value" in value: - data = bytes(value["value"]) - return data.decode(value.get("encoding", "utf-8"), errors="replace") - return "" - - -def _trivia_newlines(trivia_entry): - """How many line breaks a single trivia entry accounts for.""" - if "LineFeeds" in trivia_entry: - return trivia_entry["LineFeeds"] - if "Spaces" in trivia_entry or "HorizontalTabs" in trivia_entry: - return 0 - # A comment or any other trivia kind this converter does not know about - # by name: count embedded newlines in its decoded text, if it has one - # (covers a hypothetical multi-line block comment). - return _trivia_text(trivia_entry).count("\n") - - -def annotate_lines(tree): - """ - Walk every token of the whole syntax tree once, in document order, and - return a dict mapping id(token_dict) -> 1-based line number, computed - from LineFeeds trivia (and newlines inside token text or comments). - """ - lines = {} - current_line = [1] + line = 1 def walk(element): - if is_token(element): - tok = token_of(element) - for trivia in tok["leading_trivia"]: - current_line[0] += _trivia_newlines(trivia) - lines[id(tok)] = current_line[0] - current_line[0] += tok["text"].count("\n") - return - for child in children(element): - walk(child) - - walk(tree) - return lines + nonlocal line + if "Token" not in element: + node = element.get("Node", element) + return {"kind": node["kind"], "children": [walk(child) for child in node.get("children", [])]} + token = element["Token"] + line += sum(_newlines(trivia) for trivia in token["leading_trivia"]) + kind = token["kind"] + result = { + "kind": kind.get("Keyword") if isinstance(kind, dict) else kind, + "text": token["text"], + "trivia": token["leading_trivia"], + "line": line, + } + line += token["text"].count("\n") + return result + return walk(tree) -def line_of(lines, token): - """The line number of a Token leaf, from the dict built by annotate_lines.""" - return lines[id(token_of(token))] +def _comment(trivia): + """The text of a comment trivia entry, or "" for whitespace.""" + value = next((value for value in trivia.values() if isinstance(value, dict) and "value" in value), None) + return bytes(value["value"]).decode(value.get("encoding", "utf-8"), errors="replace") if value else "" -# --------------------------------------------------------------------------- -# Doc comments -# --------------------------------------------------------------------------- -_LICENSE_MARKER = "This Source Code Form" +def _newlines(trivia): + return trivia["LineFeeds"] if "LineFeeds" in trivia else _comment(trivia).count("\n") -def _comment_text(trivia_list): - """ - The contiguous block of "--" line comments directly before a token - (i.e. the last run not broken by a blank line), stripped of "--" and - one following space, joined with "\\n". None if there is no such block. - """ - collected = [] - for entry in trivia_list: - if "LineComment" in entry: - body = _trivia_text(entry)[2:] - if body.startswith(" "): - body = body[1:] - collected.append(body) - elif _trivia_newlines(entry) >= 2: - collected = [] # a blank line ends (and restarts) the block - return "\n".join(collected) if collected else None - - -def leading_doc(element): - """The doc comment directly above element's first token, or None.""" - first = _first_token(element) - return _comment_text(token_of(first)["leading_trivia"]) if first is not None else None - - -def package_doc(preamble): - """The package's own doc comment, or None (also None for the MPL license header).""" - doc = leading_doc(preamble) - if doc is not None and _LICENSE_MARKER in doc: # pylint: disable=unsupported-membership-test - return None - return doc +def _tokens(node): + """The tokens under node (or node itself), in document order.""" + if "children" not in node: + yield node + else: + for child in node["children"]: + yield from _tokens(child) -# --------------------------------------------------------------------------- -# Declaration converters -# --------------------------------------------------------------------------- +def _find_all(node, *kinds): + """All nodes of the given kinds under node (or node itself), in document order.""" + if "children" not in node: + return [] + return ([node] if node["kind"] in kinds else []) + [ + found for child in node["children"] for found in _find_all(child, *kinds) + ] -_PARAMETER_CLASS_KEYWORDS = {"Constant": "constant", "Variable": "variable", "Signal": "signal", "File": "file"} -_MODE_KEYWORDS = {"In": "in", "Out": "out", "Inout": "inout", "Buffer": "buffer", "Linkage": "linkage"} +def _child(node, kind): + """The first direct child of the given kind, or None (also for a None node).""" + return next((child for child in node["children"] if child["kind"] == kind), None) if node else None -def convert_parameter(iod): - """ - Convert one InterfaceObjectDeclaration/InterfaceFileDeclaration node to - a list of parameter dicts (one per identifier). - VHDL default rules for an omitted object class: "constant" when the - (explicit or default) mode is "in", else "variable" -- except a file - parameter, which is always class "file" and defaults to mode "inout". - """ - keywords = _direct_keywords(iod) - is_file_param = kind(iod) == "InterfaceFileDeclaration" - mode = next((v for k, v in _MODE_KEYWORDS.items() if k in keywords), "inout" if is_file_param else "in") - if is_file_param: - param_class = "file" - else: - param_class = next( - (v for k, v in _PARAMETER_CLASS_KEYWORDS.items() if k in keywords), - "constant" if mode == "in" else "variable", - ) +def _text(node, skip=0): + """The source text of node (without its first skip children), whitespace and comments as single spaces.""" + if node is None: + return None + pieces = [] + for child in node["children"][skip:] if skip else [node]: + for token in _tokens(child): + pieces += [" ", token["text"]] if pieces and token["trivia"] else [token["text"]] + return "".join(pieces) - subtype_node = _find_child(iod, "SubtypeIndication") - param_type = text_of(subtype_node) if subtype_node is not None else None - default = None - init_node = _find_child(iod, "InitialValue") - if init_node is not None: - default = text_of(*children(init_node)[1:]) +def _names(node): + """The identifier tokens of a declaration (from its identifier list, if it has one).""" + return [child for child in (_child(node, "IdentifierList") or node)["children"] if child["kind"] == "Identifier"] - ident_list = _find_child(iod, "IdentifierList") - return [ - {"name": token_of(tok)["text"], "class": param_class, "mode": mode, "type": param_type, "default": default} - for tok in _identifier_tokens(ident_list) - ] +def _doc(node): + """The block of "--" comments directly above node, not separated from it by a blank line, or None.""" + block = [] + for trivia in next(_tokens(node))["trivia"]: + if "LineComment" in trivia: + block.append(_comment(trivia)[2:].removeprefix(" ")) + elif _newlines(trivia) >= 2: + block = [] + return "\n".join(block) if block else None -def convert_subprogram_entry(decl, lines, _file_path): - """Convert one top-level SubprogramDeclaration node to a list holding a single subprogram dict.""" - return [convert_subprogram(decl, lines)] +def _unhandled(node, file_path): + return ApiReferenceError( + f"Unhandled declaration kind '{node['kind']}' at {file_path}:{next(_tokens(node))['line']}" + ) -def convert_subprogram(decl, lines): - """Convert one SubprogramDeclaration node (top-level or a protected type method) to a subprogram dict.""" - spec = children(decl)[0] - func = kind(spec) == "FunctionSpecification" - name_tok = next(c for c in children(spec) if is_token(c) and token_of(c)["kind"] in ("Identifier", "StringLiteral")) - name_text = token_of(name_tok)["text"] - operator = token_of(name_tok)["kind"] != "Identifier" - name = name_text.strip('"') if operator else name_text +_MODES = ("In", "Out", "Inout", "Buffer", "Linkage") +_CLASSES = ("Constant", "Variable", "Signal", "File") +_OBJECT_CLASSES = {"SignalDeclaration": "signal", "VariableDeclaration": "shared variable", "FileDeclaration": "file"} - pure = None if not func else "Impure" not in _direct_keywords(spec) +def _subprogram(decl): + """The subprogram dict of a SubprogramDeclaration.""" + spec = decl["children"][0] + function = spec["kind"] == "FunctionSpecification" + name = next(child for child in spec["children"] if child["kind"] in ("Identifier", "StringLiteral")) parameters = [] - for iod in find_all(spec, "InterfaceObjectDeclaration", "InterfaceFileDeclaration"): - parameters.extend(convert_parameter(iod)) - - return_type = None - if func: - return_name = _find_child(spec, "Name") - if return_name is not None: - return_type = text_of(return_name) - + for parameter in _find_all(spec, "InterfaceObjectDeclaration", "InterfaceFileDeclaration"): + keywords = {child["kind"] for child in parameter["children"]} + file = parameter["kind"] == "InterfaceFileDeclaration" + # VHDL defaults: mode "in" ("inout" for a file); class "constant" for mode "in", else "variable" + mode = next((mode.lower() for mode in _MODES if mode in keywords), "inout" if file else "in") + default_class = "file" if file else "constant" if mode == "in" else "variable" + parameter_class = next((c.lower() for c in _CLASSES if c in keywords and not file), default_class) + parameters += [ + { + "name": identifier["text"], + "class": parameter_class, + "mode": mode, + "type": _text(_child(parameter, "SubtypeIndication")), + "default": _text(_child(parameter, "InitialValue"), skip=1), + } + for identifier in _names(parameter) + ] return { - "kind": "function" if func else "procedure", - "name": name, - "operator": operator, - "pure": pure, - "line": line_of(lines, name_tok), - "doc": leading_doc(decl), + "kind": "function" if function else "procedure", + "name": name["text"].strip('"'), + "operator": name["kind"] == "StringLiteral", + "pure": "Impure" not in {child["kind"] for child in spec["children"]} if function else None, + "line": name["line"], + "doc": _doc(decl), "parameters": parameters, - "return_type": return_type, + "return_type": _text(_child(spec, "Name")) if function else None, } -def convert_alias(decl, lines, _file_path=None): - """Convert one AliasDeclaration node to a list holding a single alias dict.""" - direct = children(decl) - name_tok = next(c for c in direct if is_token(c) and token_of(c)["kind"] == "Identifier") - target_node = _find_child(decl, "Name") - target = text_of(target_node) if target_node is not None else None - - signature = [] - return_type = None - sig_node = _find_child(decl, "Signature") - if sig_node is not None: - type_mark_list = _find_child(sig_node, "TypeMarkList") - if type_mark_list is not None: - for child in children(type_mark_list): - if kind(child) == "Name": - signature.append(text_of(child)) - elif kind(child) == "ReturnType": - return_name = _find_child(child, "Name") - if return_name is not None: - return_type = text_of(return_name) - - return [ - { - "name": token_of(name_tok)["text"], - "line": line_of(lines, name_tok), - "target": target, - "signature": signature, - "return_type": return_type, - } - ] - - -def convert_constant(decl, lines, _file_path=None): - """Convert one ConstantDeclaration node to a list of constant dicts (one per identifier).""" - ident_list = _find_child(decl, "IdentifierList") - subtype_node = _find_child(decl, "SubtypeIndication") - const_type = text_of(subtype_node) if subtype_node is not None else None - - value = None - init_node = _find_child(decl, "InitialValue") - if init_node is not None: - value = text_of(*children(init_node)[1:]) - - doc = leading_doc(decl) - return [ - {"name": token_of(tok)["text"], "line": line_of(lines, tok), "doc": doc, "type": const_type, "value": value} - for tok in _identifier_tokens(ident_list) - ] - - -def convert_subtype(decl, lines, _file_path=None): - """Convert one SubtypeDeclaration node to a list holding a single subtype dict.""" - direct = children(decl) - name_tok = next(c for c in direct if is_token(c) and token_of(c)["kind"] == "Identifier") - subtype_node = _find_child(decl, "SubtypeIndication") - return [ - { - "name": token_of(name_tok)["text"], - "line": line_of(lines, name_tok), - "doc": leading_doc(decl), - "definition": text_of(subtype_node) if subtype_node is not None else None, - } - ] - - -_OBJECT_CLASS = {"SignalDeclaration": "signal", "VariableDeclaration": "shared variable", "FileDeclaration": "file"} - - -def convert_object(decl, lines, _file_path=None): - """Convert one Signal/Variable/File Declaration node to a list of object dicts (one per identifier).""" - ident_list = _find_child(decl, "IdentifierList") - subtype_node = _find_child(decl, "SubtypeIndication") - obj_type = text_of(subtype_node) if subtype_node is not None else None - obj_class = _OBJECT_CLASS[kind(decl)] - doc = leading_doc(decl) - return [ - {"name": token_of(tok)["text"], "class": obj_class, "line": line_of(lines, tok), "doc": doc, "type": obj_type} - for tok in _identifier_tokens(ident_list) - ] - - -def _convert_record_type(def_node, common): - """Build a "record" type dict: {..., "elements": [{"name", "type"}, ...]}.""" - elements = [] - for element_decl in find_all(def_node, "ElementDeclaration"): - ident_list = _find_child(element_decl, "IdentifierList") - subtype_node = _find_child(element_decl, "SubtypeIndication") - element_type = text_of(subtype_node) if subtype_node is not None else None - for tok in _identifier_tokens(ident_list): - elements.append({"name": token_of(tok)["text"], "type": element_type}) - return {**common, "kind": "record", "elements": elements} - - -def _convert_enumeration_type(def_node, common): - """Build an "enumeration" type dict: {..., "literals": [name, ...]}.""" - enum_list = _find_child(def_node, "EnumerationList") - literals = [] - if enum_list is not None: - for child in children(enum_list): - if is_token(child) and token_of(child)["kind"] in ("Identifier", "CharacterLiteral"): - literals.append(token_of(child)["text"]) - return {**common, "kind": "enumeration", "literals": literals} - - -def _convert_protected_type(def_node, lines, common, file_path): - """Build a "protected" type dict: {..., "subprograms": [...]} (its methods).""" - subprograms = [] - decl_part = _find_child(def_node, "ProtectedTypeDeclarativePart") - if decl_part is not None: - for member in children(decl_part): - if kind(member) != "SubprogramDeclaration": - first = _first_token(member) - where_line = line_of(lines, first) if first is not None else "?" - raise ApiReferenceError( - f"Unhandled protected type member kind '{kind(member)}' at {file_path}:{where_line}" - ) - subprograms.append(convert_subprogram(member, lines)) - return {**common, "kind": "protected", "subprograms": subprograms} - - -def convert_type(decl, lines, file_path): - """Convert one FullTypeDeclaration node to a list holding a single type dict.""" - direct = children(decl) - name_tok = next(c for c in direct if is_token(c) and token_of(c)["kind"] == "Identifier") - common = {"name": token_of(name_tok)["text"], "line": line_of(lines, name_tok), "doc": leading_doc(decl)} - - def_node = next((c for c in direct if not is_token(c)), None) - def_kind = kind(def_node) - - if def_kind == "RecordTypeDefinition": - return [_convert_record_type(def_node, common)] - if def_kind == "EnumerationTypeDefinition": - return [_convert_enumeration_type(def_node, common)] - if def_kind == "ProtectedTypeDeclaration": - return [_convert_protected_type(def_node, lines, common, file_path)] - - # Array, access and file types (and anything else vhdl-dump-ast introduces): - # kept as "other" with the raw source-text definition. - return [{**common, "kind": "other", "definition": text_of(def_node) if def_node is not None else None}] - - -_KIND_HANDLERS = { - "SubprogramDeclaration": ("subprograms", convert_subprogram_entry), - "AliasDeclaration": ("aliases", convert_alias), - "ConstantDeclaration": ("constants", convert_constant), - "SubtypeDeclaration": ("subtypes", convert_subtype), - "FullTypeDeclaration": ("types", convert_type), - "SignalDeclaration": ("objects", convert_object), - "VariableDeclaration": ("objects", convert_object), - "FileDeclaration": ("objects", convert_object), -} - - -def convert_package(pkg_node, lines, file_path, repo_relative_path): - """Convert one PackageDeclaration node to a package dict.""" - preamble = _find_child(pkg_node, "PackagePreamble") - name_tok = next(c for c in children(preamble) if is_token(c) and token_of(c)["kind"] == "Identifier") - name = token_of(name_tok)["text"] - - package = { - "library": LIBRARY_NAME, - "name": name, - "source": repo_relative_path, - "line": line_of(lines, name_tok), - "private": name.endswith("_private_pkg"), - "doc": package_doc(preamble), - "types": [], - "subtypes": [], - "constants": [], - "objects": [], - "subprograms": [], - "aliases": [], - } - - decl_part = _find_child(pkg_node, "PackageDeclarativePart") - for decl in children(decl_part) if decl_part is not None else []: - if is_token(decl): - continue - handler = _KIND_HANDLERS.get(kind(decl)) - if handler is None: - first = _first_token(decl) - where_line = line_of(lines, first) if first is not None else "?" - raise ApiReferenceError(f"Unhandled declaration kind '{kind(decl)}' at {file_path}:{where_line}") - bucket, convert = handler - package[bucket].extend(convert(decl, lines, file_path)) - - return package +def _declarations(decl, file_path): + """(package dict key, list of entries) for one declaration of a package.""" + kind, names, doc = decl["kind"], _names(decl), _doc(decl) + subtype = _text(_child(decl, "SubtypeIndication")) + if kind == "SubprogramDeclaration": + return "subprograms", [_subprogram(decl)] + if kind == "ConstantDeclaration": + value = _text(_child(decl, "InitialValue"), skip=1) + return "constants", [ + {"name": name["text"], "line": name["line"], "doc": doc, "type": subtype, "value": value} for name in names + ] + if kind in _OBJECT_CLASSES: + return "objects", [ + {"name": name["text"], "class": _OBJECT_CLASSES[kind], "line": name["line"], "doc": doc, "type": subtype} + for name in names + ] + entry = {"name": names[0]["text"], "line": names[0]["line"]} if names else {} + if kind == "SubtypeDeclaration": + return "subtypes", [{**entry, "doc": doc, "definition": subtype}] + if kind == "AliasDeclaration": + marks = _child(_child(decl, "Signature"), "TypeMarkList") + signature = [_text(mark) for mark in marks["children"] if mark["kind"] == "Name"] if marks else [] + return "aliases", [ + {**entry, "target": _text(_child(decl, "Name")), "signature": signature, "return_type": None} + ] + if kind == "FullTypeDeclaration": + return "types", [_type(decl, {**entry, "doc": doc}, file_path)] + raise _unhandled(decl, file_path) + + +def _type(decl, entry, file_path): + """The type dict of a FullTypeDeclaration, extending entry (its name, line and doc).""" + definition = next((child for child in decl["children"] if "children" in child), None) + definition_kind = definition and definition["kind"] + if definition_kind == "RecordTypeDefinition": + elements = [ + {"name": name["text"], "type": _text(_child(element, "SubtypeIndication"))} + for element in _find_all(definition, "ElementDeclaration") + for name in _names(element) + ] + return {**entry, "kind": "record", "elements": elements} + if definition_kind == "EnumerationTypeDefinition": + literals = (_child(definition, "EnumerationList") or {"children": []})["children"] + literals = [literal["text"] for literal in literals if literal["kind"] in ("Identifier", "CharacterLiteral")] + return {**entry, "kind": "enumeration", "literals": literals} + if definition_kind == "ProtectedTypeDeclaration": + members = (_child(definition, "ProtectedTypeDeclarativePart") or {"children": []})["children"] + for member in members: + if member["kind"] != "SubprogramDeclaration": + raise _unhandled(member, file_path) + return {**entry, "kind": "protected", "subprograms": [_subprogram(member) for member in members]} + # Array, access and file types + return {**entry, "kind": "other", "definition": _text(definition)} def convert_package_file(tree, file_path, package_name, repo_relative_path): - """Find and convert the PackageDeclaration for package_name in a parsed syntax tree, or None.""" - lines = annotate_lines(tree) - for pkg_node in find_all(tree, "PackageDeclaration"): - preamble = _find_child(pkg_node, "PackagePreamble") - name_tok = next(c for c in children(preamble) if is_token(c) and token_of(c)["kind"] == "Identifier") - if token_of(name_tok)["text"] == package_name: - return convert_package(pkg_node, lines, str(file_path), repo_relative_path) + """The package dict of package_name in a vhdl-dump-ast syntax tree, or None if it is not declared there.""" + for package in _find_all(_normalize(tree), "PackageDeclaration"): + preamble = _child(package, "PackagePreamble") + name = _names(preamble)[0] + if name["text"] != package_name: + continue + doc = _doc(preamble) + result = { + "library": LIBRARY_NAME, + "name": package_name, + "source": repo_relative_path, + "line": name["line"], + "private": package_name.endswith("_private_pkg"), + "doc": None if "This Source Code Form" in (doc or "") else doc, # not the license header + **{key: [] for key in ("types", "subtypes", "constants", "objects", "subprograms", "aliases")}, + } + for decl in (_child(package, "PackageDeclarativePart") or {"children": []})["children"]: + if "children" in decl: + key, entries = _declarations(decl, file_path) + result[key] += entries + return result return None -# --------------------------------------------------------------------------- -# Completeness check -# --------------------------------------------------------------------------- - - def check_completeness(json_package_names, expected_package_names): """Raise ApiReferenceError listing missing/extra names if the two sets differ.""" - found = set(json_package_names) - expected = set(expected_package_names) - missing = sorted(expected - found) - extra = sorted(found - expected) - if missing or extra: + found, expected = set(json_package_names), set(expected_package_names) + if found != expected: raise ApiReferenceError( "Package set in the JSON differs from the package declarations found in the source files: " - f"missing {missing}, extra {extra}" + f"missing {sorted(expected - found)}, extra {sorted(found - expected)}" ) -# --------------------------------------------------------------------------- -# VHDL API top-level build -# --------------------------------------------------------------------------- - - def build_vhdl_api(root, vunit_version): - """Run vhdl-dump-ast on every in-scope package file and extract the full VHDL symbol table.""" + """The VHDL symbol table of every in-scope package, extracted with vhdl-dump-ast.""" package_files = find_package_files(root) check_vhdl_dump_ast_available() generator = check_vhdl_dump_ast_version() - packages = [] - for file_path, package_name in package_files: - stdout = run_vhdl_dump_ast(file_path) - tree = json.loads(stdout) - package = convert_package_file(tree, file_path, package_name, file_path.relative_to(root).as_posix()) + for file_path, name in package_files: + tree = json.loads(run_vhdl_dump_ast(file_path)) + package = convert_package_file(tree, file_path, name, file_path.relative_to(root).as_posix()) if package is not None: packages.append(package) - check_completeness((pkg["name"] for pkg in packages), (name for _, name in package_files)) packages.sort(key=lambda pkg: pkg["name"]) - return { "schema_version": 1, "vunit_version": vunit_version, @@ -654,10 +330,6 @@ def build_vhdl_api(root, vunit_version): } -# --------------------------------------------------------------------------- -# Python API -# --------------------------------------------------------------------------- - _DIRECTIVE_RE = re.compile(r"^\.\.\s+(auto\w+)::\s*(\S+)", re.MULTILINE) @@ -696,6 +368,12 @@ def import_dotted(dotted): raise ApiReferenceError(f"Could not import Python autodoc target '{dotted}': {exc}") from exc +def _member_dict(name, member_kind, member, has_signature): + """A Python API member dict; signature is None unless has_signature.""" + sig = str(inspect.signature(member)) if has_signature else None + return {"name": name, "kind": member_kind, "signature": sig, "doc": inspect.getdoc(member)} + + def _class_members(cls): """Public methods and properties defined directly on cls (not inherited).""" prefix = cls.__qualname__ + "." @@ -706,31 +384,16 @@ def _class_members(cls): member = getattr(cls, name) if isinstance(member, property): fget = member.fget - if fget is None or not getattr(fget, "__qualname__", "").startswith(prefix): - continue - members.append({"name": name, "kind": "property", "signature": None, "doc": inspect.getdoc(member)}) - elif inspect.isfunction(member): - if not member.__qualname__.startswith(prefix): - continue - members.append( - { - "name": name, - "kind": "method", - "signature": str(inspect.signature(member)), - "doc": inspect.getdoc(member), - } - ) + if fget is not None and getattr(fget, "__qualname__", "").startswith(prefix): + members.append(_member_dict(name, "property", member, False)) + elif inspect.isfunction(member) and member.__qualname__.startswith(prefix): + members.append(_member_dict(name, "method", member, True)) return members def build_class_object(dotted, cls): """The Python API object for an autoclass target.""" - return { - "name": dotted, - "kind": "class", - "doc": inspect.getdoc(cls), - "members": _class_members(cls), - } + return {"name": dotted, "kind": "class", "doc": inspect.getdoc(cls), "members": _class_members(cls)} def build_module_object(dotted, module, has_members): @@ -738,20 +401,13 @@ def build_module_object(dotted, module, has_members): members = [] if has_members: for name in sorted(vars(module)): - if name.startswith("_"): - continue member = vars(module)[name] - if inspect.isclass(member) and member.__module__ == module.__name__: - members.append({"name": name, "kind": "class", "signature": None, "doc": inspect.getdoc(member)}) - elif inspect.isfunction(member) and member.__module__ == module.__name__: - members.append( - { - "name": name, - "kind": "function", - "signature": str(inspect.signature(member)), - "doc": inspect.getdoc(member), - } - ) + if name.startswith("_") or getattr(member, "__module__", None) != module.__name__: + continue + if inspect.isclass(member): + members.append(_member_dict(name, "class", member, False)) + elif inspect.isfunction(member): + members.append(_member_dict(name, "function", member, True)) if not members: raise ApiReferenceError(f"No public objects found for Python autodoc target '{dotted}'") return {"name": dotted, "kind": "module", "doc": inspect.getdoc(module), "members": members} @@ -773,11 +429,6 @@ def build_python_api(root, vunit_version): return {"schema_version": 1, "vunit_version": vunit_version, "objects": objects} -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - - def _write_json(path, data): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(data, indent=1, sort_keys=False, ensure_ascii=False) + "\n", encoding="utf-8") From 43f9d79307d9b2e2e30bafcd4ae95a3acea6f0a2 Mon Sep 17 00:00:00 2001 From: Sebastian Hellgren Date: Tue, 15 Sep 2026 23:07:19 +0200 Subject: [PATCH 09/11] Do not pass the docs tox arguments to pytest The docs workflow runs tox -e py313-docs -- --color, and pytest rejects --color without a value. The arguments are meant for the Sphinx build. Co-Authored-By: Claude Opus 5 (1M context) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0efdfd5d4..f96dba4aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -138,7 +138,7 @@ commands= fmt: {envpython} -m black ./ --exclude .eggs|.git|.hg|.mypy_cache|.nox|.tox|.venv|_build|buck-out|build|dist' {posargs} unit: {envpython} -m pytest -v -ra tests/unit {posargs} lint: {envpython} -m pytest -v -ra tests/lint {posargs} - docs: {envpython} -m pytest -ra tests/unit/test_api_reference.py {posargs} + docs: {envpython} -m pytest -ra tests/unit/test_api_reference.py docs: {envpython} tools/build_docs.py {envtmpdir}/docsbuild {posargs} docs: {envpython} tools/build_api_reference.py {envtmpdir}/docsbuild acceptance: {envpython} -m pytest -v -ra tests/acceptance {posargs} From 34f6b5de5c1db90ac566958e1bc7d07e353b8e87 Mon Sep 17 00:00:00 2001 From: Sebastian Abrahamsson Date: Sun, 11 Oct 2026 18:59:16 +0200 Subject: [PATCH 10/11] Drop the usage rules from context7.json, the docs and API references cover them Co-Authored-By: Claude Opus 5.5 --- context7.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/context7.json b/context7.json index 94fbe77b3..6e130ad93 100644 --- a/context7.json +++ b/context7.json @@ -15,8 +15,6 @@ ], "excludeFiles": [], "rules": [ - "A run script creates the project with VUnit.from_argv(), calls add_vhdl_builtins(), adds libraries and source files, and ends with main().", - "A VHDL testbench has a runner_cfg generic and runs its test cases, each selected with run(\"\"), between test_runner_setup and test_runner_cleanup.", "Complete VHDL API signatures: https://vunit.github.io/api/vunit_lib.json", "Complete Python API signatures: https://vunit.github.io/api/python.json", "All documentation as plain text, including the VHDL package headers: https://vunit.github.io/llms-full.txt" From 43c00edeea41cab0f5361d514614732aeb62c8b5 Mon Sep 17 00:00:00 2001 From: Sebastian Abrahamsson Date: Sun, 11 Oct 2026 19:12:56 +0200 Subject: [PATCH 11/11] Add the declarations of all VHDL packages to llms-full.txt Most packages are not included by any documentation page, so their API was missing from llms-full.txt and from Context7, which indexes it. Co-Authored-By: Claude Opus 5.5 --- docs/news.d/1222.doc.rst | 6 ++--- tests/unit/test_build_docs.py | 32 +++++++++++++++++++++++++- tools/build_docs.py | 43 ++++++++++++++++++++++++++++++++++- 3 files changed, 76 insertions(+), 5 deletions(-) diff --git a/docs/news.d/1222.doc.rst b/docs/news.d/1222.doc.rst index f3113ce10..29dc1d836 100644 --- a/docs/news.d/1222.doc.rst +++ b/docs/news.d/1222.doc.rst @@ -1,3 +1,3 @@ -The website now publishes, for AI agents, all documentation as plain text in ``llms-full.txt`` and a machine-readable -reference of the VHDL and Python APIs (``api/vunit_lib.json`` and ``api/python.json``). ``AGENTS.md`` and -``context7.json`` were added to the repository. +The website now publishes, for AI agents, all documentation and the declarations of all VHDL packages as plain text in +``llms-full.txt``, and a machine-readable reference of the VHDL and Python APIs (``api/vunit_lib.json`` and +``api/python.json``). ``AGENTS.md`` and ``context7.json`` were added to the repository. diff --git a/tests/unit/test_build_docs.py b/tests/unit/test_build_docs.py index 217a95f03..af41ccd37 100644 --- a/tests/unit/test_build_docs.py +++ b/tests/unit/test_build_docs.py @@ -11,7 +11,7 @@ import unittest from pathlib import Path from tests.common import with_tempdir -from tools.build_docs import write_llms_full_txt +from tools.build_docs import vhdl_package_declarations, write_llms_full_txt class TestLlmsFullTxt(unittest.TestCase): @@ -44,3 +44,33 @@ def test_concatenates_documentation_pages_front_page_first(self, tempdir): "# *queue* package\n\nSource: https://vunit.github.io/data_types/queue.html\n\n" "*queue* package\n***************\n\n package queue_pkg is\n", ) + + @with_tempdir + def test_package_declarations_without_license_bodies_or_private_packages(self, tempdir): + src = Path(tempdir) / "vunit" / "vhdl" / "data_types" / "src" + src.mkdir(parents=True) + (src / "queue_pkg.vhd").write_text( + "-- This Source Code Form is subject to the terms of the Mozilla Public\n" + "-- License, v. 2.0.\n\n" + "use work.integer_vector_ptr_pkg.all;\n\n" + "-- A queue\npackage queue_pkg is\n" + " type queue_t is record\n p : natural;\n end record;\n" + " procedure push(queue : queue_t; value : integer);\nend package;\n\n" + "package body queue_pkg is\nend package body;\n\n" + "package queue_private_pkg is\nend package;\n\n" + "package queue_types_pkg is\n subtype index_t is natural;\nend queue_types_pkg;\n", + encoding="utf-8", + ) + (src / "queue_pkg_inst.vhd").write_text("package int_queue_pkg is new work.generic_pkg;\n", encoding="utf-8") + + self.assertEqual( + vhdl_package_declarations(tempdir), + "# VHDL package queue_pkg\n\n" + "Source: https://github.com/VUnit/vunit/blob/master/vunit/vhdl/data_types/src/queue_pkg.vhd\n\n" + "use work.integer_vector_ptr_pkg.all;\n\n-- A queue\npackage queue_pkg is\n" + " type queue_t is record\n p : natural;\n end record;\n" + " procedure push(queue : queue_t; value : integer);\nend package;\n\n" + "# VHDL package queue_types_pkg\n\n" + "Source: https://github.com/VUnit/vunit/blob/master/vunit/vhdl/data_types/src/queue_pkg.vhd\n\n" + "package queue_types_pkg is\n subtype index_t is natural;\nend queue_types_pkg;\n", + ) diff --git a/tools/build_docs.py b/tools/build_docs.py index 50c2bace2..b0328fe34 100644 --- a/tools/build_docs.py +++ b/tools/build_docs.py @@ -10,6 +10,7 @@ Command line utility to build documentation/website """ +import re from subprocess import check_call from pathlib import Path from sys import argv, executable @@ -45,9 +46,47 @@ def write_llms_full_txt(text_path, output_file): Path(output_file).write_text("\n".join(page for _, page in pages), encoding="utf-8") +# A package declaration, not a body or an instantiation +_PACKAGE_RE = re.compile(r"^[ \t]*package\s+(?!body\b)(\w+)\s+is\b(?!\s*new\b)", re.MULTILINE | re.IGNORECASE) +_LICENSE_RE = re.compile(r"\A(?:--[^\n]*\n)*?--[^\n]*Mozilla Public[^\n]*\n(?:--[^\n]*\n)*") + + +def vhdl_package_declarations(root=ROOT): + """ + The declarations of the public vunit_lib packages, with their context and doc comments, + as llms-full.txt pages. Most are not included by any documentation page. + """ + vhdl_root = Path(root) / "vunit" / "vhdl" + pages = [] + for path in sorted(vhdl_root.glob("**/src/**/*.vhd")): + if "osvvm" in path.relative_to(vhdl_root).parts: + continue + text = path.read_text(encoding="utf-8") + # The first package with the context and comments above it, the others from their declaration + license_header = _LICENSE_RE.match(text) + start = license_header.end() if license_header else 0 + for match in _PACKAGE_RE.finditer(text): + start = match.start() if start is None else start + name = match.group(1) + end_re = re.compile(rf"^[ \t]*end(\s+package)?(\s+{name})?\s*;[^\n]*", re.MULTILINE | re.IGNORECASE) + end = end_re.search(text, match.end()).end() + if not name.endswith("_private_pkg"): + source = path.relative_to(root).as_posix() + pages.append( + ( + name, + f"# VHDL package {name}\n\nSource: https://github.com/VUnit/vunit/blob/master/{source}\n\n" + f"{text[start:end].strip()}\n", + ) + ) + start = None + return "\n".join(page for _, page in sorted(pages)) + + def main(): """ - Build documentation/website, and all pages as plain text in llms-full.txt for AI agents + Build documentation/website, and all pages and VHDL package declarations as plain text in + llms-full.txt for AI agents """ output_path = Path(argv[1]) args = argv[2:] @@ -56,6 +95,8 @@ def main(): text_path = Path(temp_path) / "text" sphinx("text", text_path, args + ["-d", str(Path(temp_path) / "doctrees")]) write_llms_full_txt(text_path, output_path / "llms-full.txt") + with (output_path / "llms-full.txt").open("a", encoding="utf-8") as fptr: + fptr.write("\n" + vhdl_package_declarations()) if __name__ == "__main__":