Add sphinx-gp-llms: LLM-friendly documentation outputs

docs/_ext/package_reference.py

@@ -204,6 +204,7 @@ class PackageDocsRecord:
     "sphinx-vite-builder": "build-seo",
     "sphinx-gp-opengraph": "build-seo",
     "sphinx-gp-sitemap": "build-seo",
+    "sphinx-gp-llms": "build-seo",
 }
 
 

docs/packages/sphinx-gp-llms/index.md

@@ -0,0 +1,6 @@
+(sphinx-gp-llms)=
+
+# sphinx-gp-llms
+
+```{package-landing} sphinx-gp-llms
+```

docs/redirects.txt

@@ -1,5 +1,6 @@
 extensions/sphinx-gp-opengraph packages/sphinx-gp-opengraph
 extensions/sphinx-gp-sitemap packages/sphinx-gp-sitemap
+extensions/sphinx-gp-llms packages/sphinx-gp-llms
 extensions/gp-sphinx packages/gp-sphinx
 extensions/index packages/index
 extensions/sphinx-autodoc-argparse packages/sphinx-autodoc-argparse

packages/gp-furo-theme/src/gp_furo_theme/theme/gp-furo/page.html

@@ -184,9 +184,32 @@
             {%- endif -%}
             <div class="page-source">
               Source: <code>{{ _docpath }}</code>
-              {%- if theme_source_branch %}
+              {%- if theme_source_branch or llms_md_url is defined %}
               &#183;
-              Machine-readable: <a class="muted-link" href="{{ _repo }}/raw/{{ theme_source_branch }}/{{ _docpath }}">Raw source</a>
+              Machine-readable:
+              {%- if llms_md_url is defined %}
+              <a class="muted-link" href="{{ pathto(llms_md_url, 1) }}">Markdown</a>,
+              {%- endif %}
+              {%- if theme_source_branch %}
+              <a class="muted-link" href="{{ _repo }}/raw/{{ theme_source_branch }}/{{ _docpath }}">raw source</a>{%- if llms_json_url is defined %},{%- endif %}
+              {%- endif %}
+              {%- if llms_json_url is defined %}
+              <a class="muted-link" href="{{ pathto(llms_json_url, 1) }}">docs.json</a>,
+              <a class="muted-link" href="{{ pathto(llms_txt_url, 1) }}">llms.txt</a>,
+              <a class="muted-link" href="{{ pathto(llms_full_url, 1) }}">llms-full.txt</a>
+              {%- endif %}
+              {%- endif %}
+            </div>
+            {%- elif llms_md_url is defined or llms_json_url is defined -%}
+            <div class="page-source">
+              Machine-readable:
+              {%- if llms_md_url is defined %}
+              <a class="muted-link" href="{{ pathto(llms_md_url, 1) }}">Markdown</a>{%- if llms_json_url is defined %},{%- endif %}
+              {%- endif %}
+              {%- if llms_json_url is defined %}
+              <a class="muted-link" href="{{ pathto(llms_json_url, 1) }}">docs.json</a>,
+              <a class="muted-link" href="{{ pathto(llms_txt_url, 1) }}">llms.txt</a>,
+              <a class="muted-link" href="{{ pathto(llms_full_url, 1) }}">llms-full.txt</a>
               {%- endif %}
             </div>
             {%- endif %}

packages/gp-sphinx/src/gp_sphinx/defaults.py

@@ -86,6 +86,7 @@ class FontConfig(_FontConfigRequired, total=False):
     "sphinx_copybutton",
     "sphinx_gp_opengraph",
     "sphinx_gp_sitemap",
+    "sphinx_gp_llms",
     "sphinxext.rediraffe",
     "sphinx_design",
     "myst_parser",
@@ -96,7 +97,7 @@ class FontConfig(_FontConfigRequired, total=False):
 Examples
 --------
 >>> len(DEFAULT_EXTENSIONS)
-13
+14
 
 >>> DEFAULT_EXTENSIONS[0]
 'sphinx.ext.autodoc'

packages/sphinx-gp-llms/README.md

@@ -0,0 +1,6 @@
+# sphinx-gp-llms
+
+LLM-friendly documentation outputs for Sphinx.
+
+Generates `llms.txt`, `llms-full.txt`, `docs.json`, and per-page `.md`
+twin files during the standard HTML build.

packages/sphinx-gp-llms/pyproject.toml

@@ -0,0 +1,40 @@
+[project]
+name = "sphinx-gp-llms"
+version = "0.0.1a23"
+description = "LLM-friendly documentation outputs for Sphinx — llms.txt, llms-full.txt, docs.json, per-page Markdown"
+requires-python = ">=3.10,<4.0"
+authors = [
+  {name = "Tony Narlock", email = "tony@git-pull.com"}
+]
+license = { text = "MIT" }
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "License :: OSI Approved :: MIT License",
+  "Framework :: Sphinx",
+  "Framework :: Sphinx :: Extension",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
+  "Topic :: Documentation",
+  "Topic :: Documentation :: Sphinx",
+  "Typing :: Typed",
+]
+readme = "README.md"
+keywords = ["sphinx", "llm", "documentation", "ai", "llms-txt"]
+dependencies = [
+  "sphinx>=8.1",
+]
+
+[project.urls]
+Repository = "https://github.com/git-pull/gp-sphinx"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/sphinx_gp_llms"]

packages/sphinx-gp-llms/src/sphinx_gp_llms/__init__.py

@@ -0,0 +1,218 @@
+"""LLM-friendly documentation outputs for Sphinx.
+
+Generates ``llms.txt``, ``llms-full.txt``, ``docs.json``, and per-page
+``.md`` twin files during the standard HTML build, following conventions
+established by llmstxt.org (Jeremy Howard / Answer.AI), Cloudflare
+("Markdown for Agents"), Mintlify, and Lakebed (Ping).
+
+The extension hooks into ``build-finished`` to write output files and
+``html-page-context`` to inject footer link variables into the template
+context.
+
+Examples
+--------
+>>> from sphinx_gp_llms import setup
+>>> callable(setup)
+True
+"""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import typing as t
+
+from sphinx.errors import ExtensionError
+from sphinx.util.logging import getLogger
+
+if t.TYPE_CHECKING:
+    from docutils import nodes
+    from sphinx.application import Sphinx
+    from sphinx.util.typing import ExtensionMetadata
+
+_EXTENSION_VERSION = "0.0.1a23"
+
+logger = getLogger(__name__)
+logging.getLogger(__name__).addHandler(logging.NullHandler())
+
+__all__ = ["setup"]
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+    """Register config values and connect build hooks.
+
+    Parameters
+    ----------
+    app : Sphinx
+        Sphinx application instance.
+
+    Returns
+    -------
+    ExtensionMetadata
+        Extension metadata with version and parallel-build flags.
+
+    Examples
+    --------
+    >>> from sphinx_gp_llms import setup
+    >>> callable(setup)
+    True
+    """
+    app.add_config_value(
+        "llms_generate_txt",
+        default=True,
+        rebuild="",
+        types=frozenset({bool}),
+        description="Enable llms.txt generation.",
+    )
+    app.add_config_value(
+        "llms_generate_full",
+        default=True,
+        rebuild="",
+        types=frozenset({bool}),
+        description="Enable llms-full.txt generation.",
+    )
+    app.add_config_value(
+        "llms_generate_json",
+        default=True,
+        rebuild="",
+        types=frozenset({bool}),
+        description="Enable docs.json agent manifest generation.",
+    )
+    app.add_config_value(
+        "llms_generate_md_twins",
+        default=True,
+        rebuild="",
+        types=frozenset({bool}),
+        description="Enable per-page .md twin file generation.",
+    )
+    app.add_config_value(
+        "llms_txt_filename",
+        default="llms.txt",
+        rebuild="",
+        types=frozenset({str}),
+        description="Output filename for the llms.txt index.",
+    )
+    app.add_config_value(
+        "llms_full_filename",
+        default="llms-full.txt",
+        rebuild="",
+        types=frozenset({str}),
+        description="Output filename for the concatenated full-content file.",
+    )
+    app.add_config_value(
+        "llms_json_filename",
+        default="docs.json",
+        rebuild="",
+        types=frozenset({str}),
+        description="Output filename for the docs.json agent manifest.",
+    )
+    app.add_config_value(
+        "llms_excludes",
+        default=[],
+        rebuild="",
+        types=frozenset({list}),
+        description=(
+            "fnmatch patterns matched against each page's relative URL. "
+            "Matched pages are excluded from all LLM outputs."
+        ),
+    )
+    app.add_config_value(
+        "llms_description_length",
+        default=200,
+        rebuild="",
+        types=frozenset({int}),
+        description="Maximum character length for page descriptions.",
+    )
+
+    with contextlib.suppress(ExtensionError):
+        app.add_config_value(
+            "site_url",
+            default=None,
+            rebuild="",
+            types=frozenset({str, type(None)}),
+            description=(
+                "Site base URL — registered defensively; "
+                "sphinx-gp-sitemap usually registers this first."
+            ),
+        )
+
+    app.connect("build-finished", _write_llm_outputs)
+    app.connect("html-page-context", _inject_llms_context)
+
+    return {
+        "version": _EXTENSION_VERSION,
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }
+
+
+def _resolve_site_url(app: Sphinx) -> str | None:
+    """Resolve site URL from config, normalizing trailing slash."""
+    url: str | None = getattr(app.config, "site_url", None) or getattr(
+        app.config, "html_baseurl", None
+    )
+    if not url:
+        return None
+    return url if url.endswith("/") else url + "/"
+
+
+def _write_llm_outputs(app: Sphinx, exception: BaseException | None) -> None:
+    """Generate all enabled LLM output files at build-finished."""
+    if exception is not None:
+        return
+
+    if not hasattr(app.builder, "get_target_uri"):
+        return
+
+    site_url = _resolve_site_url(app)
+    if not site_url:
+        logger.info(
+            "sphinx-gp-llms: skipped — site_url and html_baseurl both unset",
+            type="llms",
+            subtype="configuration",
+        )
+        return
+
+    if app.config.llms_generate_txt:
+        from sphinx_gp_llms._llms_txt import write_llms_txt
+
+        write_llms_txt(app, site_url)
+
+    if app.config.llms_generate_full:
+        from sphinx_gp_llms._llms_full_txt import write_llms_full_txt
+
+        write_llms_full_txt(app, site_url)
+
+    if app.config.llms_generate_json:
+        from sphinx_gp_llms._docs_json import write_docs_json
+
+        write_docs_json(app, site_url)
+
+    if app.config.llms_generate_md_twins:
+        from sphinx_gp_llms._md_twins import write_md_twins
+
+        write_md_twins(app)
+
+
+def _inject_llms_context(
+    app: Sphinx,
+    pagename: str,
+    templatename: str,
+    context: dict[str, t.Any],
+    doctree: nodes.document | None,
+) -> None:
+    """Add LLM output link variables to the Jinja2 template context."""
+    del templatename, doctree
+
+    site_url = _resolve_site_url(app)
+    if not site_url:
+        return
+
+    if app.config.llms_generate_md_twins:
+        context["llms_md_url"] = pagename + ".md"
+    if app.config.llms_generate_txt:
+        context["llms_txt_url"] = app.config.llms_txt_filename
+    if app.config.llms_generate_full:
+        context["llms_full_url"] = app.config.llms_full_filename
+    if app.config.llms_generate_json:
+        context["llms_json_url"] = app.config.llms_json_filename