Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions generated/badges.readme.md
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
<!-- generated from tools/docs_audit/render_badges.py (readme) — edit the list, not this -->
<p align="center">
<a href="https://github.com/CTRLRun/ctrlrun/blob/badges/clones-history.json"><img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/CTRLRun/ctrlrun/badges/clones-badge.json" alt="Clones"></a>
<a href="https://pypi.org/project/ctrlrun/"><img src="https://img.shields.io/pypi/v/ctrlrun?color=B8730A&label=pypi" alt="PyPI"></a>
<a href="https://pypi.org/project/ctrlrun/"><img src="https://img.shields.io/pypi/pyversions/ctrlrun?color=B8730A" alt="Python versions"></a>
<a href="https://pypistats.org/packages/ctrlrun"><img src="https://img.shields.io/pypi/dm/ctrlrun?color=B8730A&label=downloads" alt="Downloads"></a>
<a href="https://ctrlrun.dev"><img src="https://img.shields.io/badge/docs-ctrlrun.dev-B8730A" alt="Docs"></a>
<a href="https://github.com/CTRLRun/ctrlrun/actions/workflows/ci.yml"><img src="https://github.com/CTRLRun/ctrlrun/actions/workflows/ci.yml/badge.svg?branch=main" alt="CI"></a>
<a href="https://github.com/CTRLRun/ctrlrun/actions/workflows/codeql.yml"><img src="https://github.com/CTRLRun/ctrlrun/actions/workflows/codeql.yml/badge.svg?branch=main" alt="CodeQL"></a>
<a href="https://github.com/CTRLRun/ctrlrun/actions/workflows/fuzz.yml"><img src="https://github.com/CTRLRun/ctrlrun/actions/workflows/fuzz.yml/badge.svg?branch=main" alt="Fuzz"></a>
<a href="https://ctrlrun.dev/docs/how-this-is-built"><img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/CTRLRun/ctrlrun/badges/tests-badge.json" alt="Tests"></a>
<a href="https://ctrlrun.dev/docs/security/verify-guarantees"><img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/CTRLRun/ctrlrun/badges/verify-badge.json" alt="CTRLRun verified"></a>
<a href="https://scorecard.dev/viewer/?uri=github.com/CTRLRun/ctrlrun"><img src="https://api.scorecard.dev/projects/github.com/CTRLRun/ctrlrun/badge" alt="OpenSSF Scorecard"></a>
<a href="https://github.com/astral-sh/ruff"><img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json" alt="Ruff"></a>
<a href="https://github.com/CTRLRun/ctrlrun/blob/main/scripts/check.sh"><img src="https://img.shields.io/badge/mypy-strict-B8730A" alt="Checked with mypy --strict"></a>
<a href="https://github.com/CTRLRun/ctrlrun/blob/main/LICENSE"><img src="https://img.shields.io/pypi/l/ctrlrun?color=B8730A" alt="License"></a>
</p>
<!-- end generated -->
23 changes: 23 additions & 0 deletions tests/test_docs_audit.py
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,29 @@ def test_a_github_blob_url_into_either_repository_is_an_internal_link(tmp_path):
]


def test_a_link_into_the_badges_branch_is_external(tmp_path):
"""The `badges` branch is an orphan CI writes and no checkout holds, so a link into it is
somebody else's to keep however internal the URL looks. `clones-history.json` is the
receipt the README's clones badge links to: resolved against a worktree that holds `main`
it is a working link reported as broken, and the rest of the branch must stay skipped with
it. A path missing on a ref this checkout *does* hold is still broken, which is the half
that fails if the skip is widened to every absolute github.com URL.
"""
page = tmp_path / "a.md"
page.write_text(
"[clones](https://github.com/CTRLRun/ctrlrun/blob/badges/clones-history.json)\n"
"[tests](https://github.com/CTRLRun/ctrlrun/blob/badges/tests-badge.json)\n"
"[bad](https://github.com/CTRLRun/ctrlrun/blob/main/clones-history.json)\n",
encoding="utf-8",
)

broken = links.check_text(page.read_text(), page)

assert [b.target for b in broken] == [
"https://github.com/CTRLRun/ctrlrun/blob/main/clones-history.json"
]


def test_a_root_relative_docs_path_resolves_under_docs(tmp_path, monkeypatch):
monkeypatch.setattr(links, "REPO_ROOT", tmp_path)
(tmp_path / "docs" / "concepts").mkdir(parents=True)
Expand Down
12 changes: 10 additions & 2 deletions tools/docs_audit/links.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
- `[text](/path)` and `href="/path"` — the docs site's own root-relative form, resolved against
`docs/` with `.mdx` then `.md` appended, and against the repository root as a fallback;
- `https://github.com/CTRLRun/ctrlrun/blob/<ref>/<path>` and `/tree/<ref>/<path>`, which are
internal links wearing an absolute URL, resolved against the checkout.
internal links wearing an absolute URL, resolved against the checkout. A `<ref>` no checkout
holds is skipped instead: the `badges` branch is an orphan CI writes and nobody clones.

Every other absolute URL is skipped: an external link is somebody else's to keep, and a check
that opened sockets would be a check that failed in CI for reasons nobody here can fix.
Expand Down Expand Up @@ -49,8 +50,13 @@ def documents_to_check() -> list[Path]:
#: repository, so a page can cite `LICENSE` in the library and `docs.json` here and
#: both are checked. Named groups rather than two regexes: one place decides.
_GITHUB = re.compile(
r"^https://github\.com/CTRLRun/(?P<repo>ctrlrun|ctrlrun-docs)/(?:blob|tree)/[^/]+/(?P<path>.*)$"
r"^https://github\.com/CTRLRun/(?P<repo>ctrlrun|ctrlrun-docs)/(?:blob|tree)/(?P<ref>[^/]+)/(?P<path>.*)$"
)
#: Refs no checkout contains, so a link into one is external however much it looks internal.
#: `badges` is the orphan branch CI publishes the endpoint documents to: `clones-history.json`
#: is real and is the receipt the clones badge links to, and resolving it against a worktree
#: that holds `main` would report a working link as broken.
_DETACHED_REFS: frozenset[str] = frozenset({"badges"})
_HEADING = re.compile(r"^\s{0,3}(#{1,6})\s+(.*?)\s*#*\s*$")
_EXPLICIT_ID = re.compile(r"\{#([A-Za-z0-9_-]+)\}\s*$")
_ID_ATTRIBUTE = re.compile(r"""\bid=["']([A-Za-z0-9_-]+)["']""")
Expand Down Expand Up @@ -133,6 +139,8 @@ def _resolve(target: str, source: Path) -> tuple[Path | None, str | None] | None
matched = _GITHUB.match(path_part)
if matched is None:
return None
if matched.group("ref") in _DETACHED_REFS:
return None
root = REPO_ROOT if matched.group("repo") == "ctrlrun-docs" else CORE_ROOT
return root / unquote(matched.group("path")), anchor
if not path_part:
Expand Down
65 changes: 42 additions & 23 deletions tools/docs_audit/render_badges.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,30 @@
- `tests-badge.json`, written by `--write-count` **after** `scripts/check.sh` has passed, so no
number is published for a run whose suite was red.

Two are static, and a static badge is a self-assertion unless something enforces it. These two
are enforced: `scripts/check.sh` runs `ruff format --check`, `ruff check` and `mypy --strict
src`, CI calls that file rather than naming the tools itself, and `test_ci_runs_the_check_script`
fails if it stops. The `docs` badge is a link and claims nothing.

Downloads and stars are deliberately absent. `STYLE.md` forbids social proof that does not
exist, and a count published four days after the first release measures mirrors.
One badge is static: `docs`, which is a link and claims nothing. The row carried `ruff` and
`mypy --strict` as well until 2026-09-11. Both were enforced rather than asserted, and both are
still enforced now that the badges are gone: `scripts/check.sh` runs `ruff format --check`,
`ruff check` and `mypy --strict src`, CI calls that file rather than naming the tools itself,
and `test_ci_runs_the_check_script` fails if it stops. They were removed because how a library
is written is not what a stranger decides in the first five seconds, and because a row that
wraps to three lines on a phone is a row nobody reads to the end. `pypi/pyversions` went with
them: it was metadata rather than a claim, and the PyPI page the badge beside it links to
carries it anyway.

Stars are deliberately absent: `STYLE.md` forbids social proof that does not exist, and a star
count is a popularity number with no reading behind it. The two adoption counts that are here
each name what they measure and link to the data rather than to a page that repeats them:

- `downloads` is PyPI's own monthly figure via shields.io, which reads pypistats. It counts
installs by mirrors and by CI as well as by people, so it is an upper bound on adoption and
is labelled `downloads/month` rather than users.
- `clones` is this repository's own, published to the `badges` branch by `.github/workflows/
traffic.yml`. GitHub's traffic API keeps fourteen days and needs push access, so the workflow
reads it daily with a token, merges each day into `clones-history.json` on that branch, and
the badge links to that file: the number is a sum of days anybody can re-add. It counts every
`git clone`, and `actions/checkout` is one, so this repository's own CI is in the figure
alongside everybody else's; a reader who wants people rather than clones has the per-day
file and the workflow run history to subtract with.

`--write-count` is what CI calls, and what it counts is what `pytest` **collects**. That is not
the same as what passed: the suite skips a handful of tests on a machine without a framework
Expand Down Expand Up @@ -61,19 +78,26 @@ class Badge:
href: str


#: The row, in reading order: what it is, where it is documented, that it builds and is
#: analysed, that its own suite is this big, that its guarantees were checked, how its supply
#: chain scores, how it is written, and the licence.
#: The row, in reading order: how often it is cloned, what it is, how often it is installed,
#: where it is documented, that it builds, is analysed and is fuzzed, that its own suite is this
#: big, that its guarantees were checked, how its supply chain scores, and the licence. Every entry is a
#: claim a reader can follow to the thing that measured it; an entry that is not stops being a
#: badge and becomes decoration, which is the test the three removed ones failed.
BADGES: tuple[Badge, ...] = (
Badge(
"Clones",
f"https://img.shields.io/endpoint?url={BADGES_BRANCH}/clones-badge.json",
"https://github.com/CTRLRun/ctrlrun/blob/badges/clones-history.json",
),
Badge(
"PyPI",
"https://img.shields.io/pypi/v/ctrlrun?color=B8730A&label=pypi",
"https://pypi.org/project/ctrlrun/",
),
Badge(
"Python versions",
"https://img.shields.io/pypi/pyversions/ctrlrun?color=B8730A",
"https://pypi.org/project/ctrlrun/",
"Downloads",
"https://img.shields.io/pypi/dm/ctrlrun?color=B8730A&label=downloads",
"https://pypistats.org/packages/ctrlrun",
),
Badge(
"Docs",
Expand All @@ -90,6 +114,11 @@ class Badge:
"https://github.com/CTRLRun/ctrlrun/actions/workflows/codeql.yml/badge.svg?branch=main",
"https://github.com/CTRLRun/ctrlrun/actions/workflows/codeql.yml",
),
Badge(
"Fuzz",
"https://github.com/CTRLRun/ctrlrun/actions/workflows/fuzz.yml/badge.svg?branch=main",
"https://github.com/CTRLRun/ctrlrun/actions/workflows/fuzz.yml",
),
Badge(
"Tests",
f"https://img.shields.io/endpoint?url={BADGES_BRANCH}/tests-badge.json",
Expand All @@ -105,16 +134,6 @@ class Badge:
"https://api.scorecard.dev/projects/github.com/CTRLRun/ctrlrun/badge",
"https://scorecard.dev/viewer/?uri=github.com/CTRLRun/ctrlrun",
),
Badge(
"Ruff",
"https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json",
"https://github.com/astral-sh/ruff",
),
Badge(
"Checked with mypy --strict",
"https://img.shields.io/badge/mypy-strict-B8730A",
"https://github.com/CTRLRun/ctrlrun/blob/main/scripts/check.sh",
),
Badge(
"License",
"https://img.shields.io/pypi/l/ctrlrun?color=B8730A",
Expand Down
Loading