From b7e1f96968bfab2a0dcca5ac22674f1ee4e1bed7 Mon Sep 17 00:00:00 2001 From: mrbobbytables Date: Thu, 29 Oct 2026 16:01:58 -0500 Subject: [PATCH 1/3] docs: record focused validation for content-only PRs Document the proposed CI split for authored-document-only changes, retaining relevant content checks and the production build while reserving broad suites for code, generated data, mixed, and uncertain changes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: mrbobbytables --- ...nt-focused-validation-for-documentation.md | 120 ++++++++++++++++++ adr/README.md | 1 + 2 files changed, 121 insertions(+) create mode 100644 adr/0011-content-focused-validation-for-documentation.md diff --git a/adr/0011-content-focused-validation-for-documentation.md b/adr/0011-content-focused-validation-for-documentation.md new file mode 100644 index 00000000..38ee18ee --- /dev/null +++ b/adr/0011-content-focused-validation-for-documentation.md @@ -0,0 +1,120 @@ +# ADR 0011: Content-only pull requests use focused validation + +- **Status**: Accepted +- **Date**: 2026-10-29 +- **Related**: + [ADR 0008](./0008-scope-agent-findings-to-user-observable-change.md), + [ADR 0009](./0009-refresh-generated-data-only-on-content-change.md), + [ADR 0010](./0010-architecture-pages-are-the-metadata-source.md) + +## Context + +Every pull request currently runs the repository-wide validation and lint +workflows, the full end-to-end suite, and end-to-end coverage. That includes +unit coverage thresholds, data validators, and code checks even when a +contributor changes only authored documentation. The production build is useful +for documentation changes because it catches broken internal links and invalid +content, but the code and browser suites provide little additional value for a +prose-only change. + +Contributors updating end-user documentation should not need to understand +unrelated data pipelines or wait for broad test suites. At the same time, +generated content and mixed changes must keep the checks that validate their +sources and outputs. This decision covers human-maintained documents, not +generated data or pages such as imported architecture pages and generated +catalog content. Updates to generated case-study or radar-report data and other +generated snapshots remain on their existing validation path. + +## Decision drivers + +- Keep changes to end-user documentation easy to contribute and review. +- Catch formatting, spelling, Markdown, document-contract, and internal-link + regressions before publication. +- Preserve full validation for code, data, generated content, and mixed changes. +- Preserve the required check names and the merge-queue contract. +- Make uncertain change classification take the safe, full-validation path. +- Do not lower or change any existing coverage or quality thresholds. + +## Options considered + +1. **Run the existing full workflow for every pull request.** This is simple and + safe, but requires documentation-only changes to pass unrelated unit, data, + lint, and browser suites. Rejected because it imposes avoidable contributor + friction. +2. **Use focused checks for authored-document-only changes and retain full + checks otherwise.** This keeps the production build and relevant content + checks while skipping suites that do not exercise prose. Chosen. +3. **Skip CI or the production build for documentation-only changes.** This is + the fastest option, but can publish broken internal links or invalid + documentation. Rejected. + +## Decision + +The CI workflow will classify the changed paths and use two validation paths. + +### Authored-document-only changes + +A pull request qualifies only when every changed path is a human-maintained +Markdown or MDX document. This includes authored documentation, blog posts, +ADRs, and repository guidance. Generated pages and snapshots are not eligible. +Any code, test, workflow, configuration, data, generated file, non-document +asset, or other unrecognized path makes the change ineligible; a mixed change +uses full validation. + +For eligible changes: + +- Run formatting, spelling, and Markdown checks on the changed documents. +- Run the applicable document-contract checks, including checks for Markdown + references and documentation structure. +- Build the production site to validate the rendered content and internal links. +- Skip full unit coverage, data validators unrelated to changed documents, broad + code lint/duplication/unused checks, and browser tests and coverage. + +The exact generated paths and document-contract checks will be maintained +explicitly in the classifier and workflow. Content authored from generated data +remains subject to its existing data validation lane; changes to generated data +or generated outputs always use full validation. + +### All other changes + +Code, test, workflow, configuration, data, generated-content, mixed, and +unrecognized changes continue to run the existing full validation, lint, +end-to-end, and end-to-end coverage checks. Manual workflow dispatch also keeps +the full path. + +The classifier must support both `pull_request` and `merge_group` events. It +must evaluate the complete applicable diff, including additions, deletions, and +renames. If it cannot determine the changed paths or whether an output is +generated, it must select full validation. + +The existing required check names remain stable. For document-only changes, the +required validation check still runs the focused content checks and production +build, and the lint check runs the changed-document checks. Browser jobs may be +skipped only when that required content-validation path succeeds. The merge +queue must apply the same classification and safety rules. + +## Implementation plan + +1. Add and test a change classifier for authored-document-only, generated, + mixed, code, unknown, and merge-queue diffs. Default classification failures + to full validation. +2. Adapt the existing CI jobs to use the classifier while retaining their + required check names. Keep the production build and content-specific checks + on the document-only path; skip only the unrelated broad suites. +3. Update contributor guidance to explain which checks run for document-only + changes and when the full suite applies. +4. Verify both paths, including merge-queue behavior, and ensure the + document-only path cannot pass when its build or focused checks fail. + +## Consequences + +- A prose-only contribution receives formatting, spelling, Markdown, + document-contract, and production-build validation without waiting on broad + code and browser suites. +- Generated-data changes and changes that mix content with code or data retain + the current full checks. +- Change classification becomes part of the CI trust boundary. Unknown paths, + missing diff information, or uncertain generated-file status must never + silently skip validation. +- This decision changes which checks run by change type; it does not change test + coverage requirements or any other frozen threshold. diff --git a/adr/README.md b/adr/README.md index e3ac563d..d6eb365e 100644 --- a/adr/README.md +++ b/adr/README.md @@ -21,6 +21,7 @@ still pending) so future contributors understand why, not just what. | [0008](./0008-scope-agent-findings-to-user-observable-change.md) | Scope agent findings to user-observable change | Accepted | | [0009](./0009-refresh-generated-data-only-on-content-change.md) | Refresh generated data only when its content changes | Accepted | | [0010](./0010-architecture-pages-are-the-metadata-source.md) | Architecture pages are the source of their catalog metadata | Accepted | +| [0011](./0011-content-focused-validation-for-documentation.md) | Content-only pull requests use focused validation | Accepted | Supporting material is not part of the numbered ADR sequence and is not assigned an ADR number, but is listed here for discoverability: From 7058eac378548fdcf066667d43e82dab3472f0c8 Mon Sep 17 00:00:00 2001 From: mrbobbytables Date: Thu, 29 Oct 2026 16:49:32 -0500 Subject: [PATCH 2/3] ci: focus validation for content-only pull requests Route authored-document-only changes through focused checks while preserving full validation for generated, mixed, and uncertain changes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: mrbobbytables --- .github/workflows/ci.yml | 135 +++-- CONTRIBUTING.md | 51 +- scripts/lib/content-only-ci.mjs | 245 +++++++++ tests/content-only-ci-workflow.test.mjs | 97 ++++ tests/content-only-ci.test.mjs | 472 ++++++++++++++++++ tests/contributing-just-build-parity.test.mjs | 4 +- 6 files changed, 942 insertions(+), 62 deletions(-) create mode 100644 scripts/lib/content-only-ci.mjs create mode 100644 tests/content-only-ci-workflow.test.mjs create mode 100644 tests/content-only-ci.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63670806..a3e5e2dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,8 +18,44 @@ concurrency: cancel-in-progress: true jobs: + classify: + name: Classify changed paths + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + document_only: ${{ steps.classify.outputs.document_only }} + documents_b64: ${{ steps.classify.outputs.documents_b64 }} + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: Classify changed documents + id: classify + run: + node --input-type=module -e "import { runClassifier } from + './scripts/lib/content-only-ci.mjs'; runClassifier()" + env: + GITHUB_EVENT_NAME: ${{ github.event_name }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + MERGE_BASE_SHA: ${{ github.event.merge_group.base_sha }} + validate: name: Validate repository + needs: classify + outputs: + document_only: + ${{ needs.classify.result == 'success' && + needs.classify.outputs.document_only == 'true' }} + if: ${{ !cancelled() }} + env: + FULL_VALIDATION: + ${{ needs.classify.result != 'success' || + needs.classify.outputs.document_only != 'true' }} + CONTENT_ONLY: + ${{ needs.classify.result == 'success' && + needs.classify.outputs.document_only == 'true' }} runs-on: ubuntu-24.04 timeout-minutes: 15 steps: @@ -30,8 +66,10 @@ jobs: - name: Set up Node.js and install dependencies uses: ./.github/actions/setup-node-deps - name: Audit dependencies against the advisory allowlist + if: ${{ !cancelled() && env.FULL_VALIDATION == 'true' }} run: npm run check:audit - name: Run unit tests with coverage + if: ${{ !cancelled() && env.FULL_VALIDATION == 'true' }} # A zone fourteen hours ahead of UTC, so a date formatted in the # host zone instead of UTC lands on a different calendar day for most # of the day; the suite used to pin TZ=UTC, which hid exactly that. @@ -40,49 +78,47 @@ jobs: run: npm run test:unit:coverage:check | tee coverage-summary.txt shell: bash -o pipefail {0} - name: Publish coverage summary - if: always() + if: ${{ !cancelled() && env.FULL_VALIDATION == 'true' }} run: | if [ -f coverage-summary.txt ]; then { echo '## Unit test coverage'; echo '```'; cat coverage-summary.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY" fi - name: Upload coverage report - if: always() + if: ${{ !cancelled() && env.FULL_VALIDATION == 'true' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: coverage-summary path: coverage-summary.txt retention-days: 90 - - name: Validate metrics data - run: npm run validate:metrics - - name: Validate architectures data - run: npm run validate:architectures - - name: Validate architecture assets - run: npm run validate:architecture-assets - - name: Validate awards data - run: npm run validate:awards - - name: Validate community people data - run: npm run validate:community-people - - name: Validate community group links - run: npm run validate:community-groups - - name: Validate case studies data - run: npm run validate:case-studies - - name: Validate radar reports data - run: npm run validate:radar-reports - - name: Validate projects-born data - run: npm run validate:projects-born - - name: Validate End User talks data - run: npm run validate:end-user-talks - - name: Validate button contrast - run: npm run validate:button-contrast - - name: Validate End User landscape data - run: npm run validate:enduser-members - - name: Validate members data - run: npm run validate:members + - name: Validate repository data + if: ${{ !cancelled() && env.FULL_VALIDATION == 'true' }} + run: | + npm run validate:metrics + npm run validate:architectures + npm run validate:architecture-assets + npm run validate:awards + npm run validate:community-people + npm run validate:community-groups + npm run validate:case-studies + npm run validate:radar-reports + npm run validate:projects-born + npm run validate:end-user-talks + npm run validate:button-contrast + npm run validate:enduser-members + npm run validate:members + - name: Check content contracts + if: ${{ !cancelled() && env.CONTENT_ONLY == 'true' }} + run: >- + node --test tests/docs-contract.test.mjs + tests/markdown-references.test.mjs + tests/community-group-coverage.test.mjs - name: Build production site run: npm run build:production lint: name: Lint repository + needs: classify + if: ${{ !cancelled() }} runs-on: ubuntu-24.04 timeout-minutes: 10 steps: @@ -92,21 +128,35 @@ jobs: persist-credentials: false - name: Set up Node.js and install dependencies uses: ./.github/actions/setup-node-deps - - name: Check formatting - run: npm run check:format - - name: Check spelling - run: npm run check:spelling - - name: Check markdown - run: npm run check:markdown - - name: Check duplication - run: npm run check:duplication - - name: Lint JavaScript - run: npm run check:lint - - name: Check unused code and dependencies - run: npm run check:unused + - name: Check changed-document formatting, spelling, and Markdown + if: + ${{ !cancelled() && needs.classify.result == 'success' && + needs.classify.outputs.document_only == 'true' }} + run: + node --input-type=module -e "import { runDocumentLint } from + './scripts/lib/content-only-ci.mjs'; + runDocumentLint(process.env.CHANGED_DOCUMENTS_B64)" + env: + CHANGED_DOCUMENTS_B64: ${{ needs.classify.outputs.documents_b64 }} + - name: Run full repository lint + if: + ${{ !cancelled() && (needs.classify.result != 'success' || + needs.classify.outputs.document_only != 'true') }} + run: | + npm run check:format + npm run check:spelling + npm run check:markdown + npm run check:duplication + npm run check:lint + npm run check:unused e2e: name: End-to-end tests + needs: [validate, lint] + if: >- + ${{ !cancelled() && (needs.validate.result != 'success' || + needs.lint.result != 'success' || needs.validate.outputs.document_only != + 'true') }} runs-on: ubuntu-24.04 timeout-minutes: 15 steps: @@ -146,6 +196,11 @@ jobs: e2e-coverage: name: End-to-end coverage + needs: [classify, validate, lint] + if: >- + ${{ !cancelled() && (needs.classify.result != 'success' || + needs.classify.outputs.document_only != 'true' || needs.validate.result != + 'success' || needs.lint.result != 'success') }} runs-on: ubuntu-24.04 timeout-minutes: 20 env: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 280e0577..c7b12953 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,17 +64,19 @@ These recipes are a superset of the validation steps the "Refresh metrics" and before the Netlify build. The recipes are **not** the same checks the "Validate repository" and "Lint repository" PR gates run, and a green `just build`/`just refresh` locally does **not** imply a green PR check. Before -opening a PR, also run `npm run test:unit:coverage:check` and the local check -scripts that match "Lint repository": `npm run check:format`, -`npm run check:spelling`, `npm run check:markdown`, `npm run check:duplication`, -`npm run check:lint` and `npm run check:unused`. The "Validate repository" gate -additionally runs `npm run check:audit` (dependency advisory allowlist) — see -step 5 below. `npm run check` also runs the lint checks, but it additionally -runs `check:audit`, `check:links` (network-dependent) and -`check:community-group-links` (needs `GH_TOKEN` and rewrites -`data/community-groups.json`) — see step 5 below before running the full -`npm run check`. `just` is optional — the npm scripts remain the canonical -interface and work without it. +For PRs that use the full-validation path, also run +`npm run test:unit:coverage:check` and the local check scripts that match "Lint +repository": `npm run check:format`, `npm run check:spelling`, +`npm run check:markdown`, `npm run check:duplication`, `npm run check:lint` and +`npm run check:unused`. Content-only PRs do not need to run the full unit +coverage or lint suites locally; CI runs focused checks on the changed documents +and builds the site. The full "Validate repository" gate also runs +`npm run check:audit` (dependency advisory allowlist) — see step 5 below. +`npm run check` runs the lint checks, but it additionally runs `check:audit`, +`check:links` (network-dependent) and `check:community-group-links` (needs +`GH_TOKEN` and rewrites `data/community-groups.json`) — see step 5 below before +running the full `npm run check`. `just` is optional — the npm scripts remain +the canonical interface and work without it. ## Content audience @@ -272,25 +274,34 @@ enough for a first contribution. ## Making changes +Pull requests that change only recognized, human-maintained Markdown or MDX +documents use focused CI: formatting, spelling, and Markdown checks run on the +changed documents, document-contract tests run, and the production site is +built. Generated or data-backed pages, code, configuration, assets, mixed +changes, and unrecognized paths keep the full validation, lint, and browser +suites. If CI cannot confidently classify a change, it uses the full path. + 1. Fork the repository and create a branch from `main`. 2. Make your change and run the relevant validation script. -3. Run `npm run test:unit:coverage:check` — the required "Validate repository" - check runs this on every PR, and it fails the build if unit-test coverage +3. For PRs that use the full-validation path, run + `npm run test:unit:coverage:check`. This check fails if unit-test coverage drops below its thresholds (99% lines / 100% source / 96% regions / 100% source regions). The harness under `tests/tools/` is floored separately (100% lines / 99% regions, and 98% regions per harness file): it is measured like source but reported in its own `harness files` row, so a gap in the test apparatus cannot be paid for out of `src/`'s slack. `npm run test:unit` alone - skips the coverage gate, so a green `test:unit` locally does not guarantee a - green PR check. If you changed a file under `src/` or `scripts/lib/`, also - run its mutation tests — see [Mutation testing](#mutation-testing) below. + skips the coverage gate. Content-only PRs use focused document checks + instead. If you changed a file under `src/` or `scripts/lib/`, also run its + mutation tests — see [Mutation testing](#mutation-testing) below. 4. Verify with `npm run build` before opening a PR. If your change touches pages, components, or navigation, also run the end-to-end suite — see [End-to-end tests](#end-to-end-tests) below. -5. Run `npm run check:format`, `npm run check:spelling`, - `npm run check:markdown`, `npm run check:duplication`, `npm run check:lint` - and `npm run check:unused` — the required "Lint repository" check runs these - six on every PR. If a check fails: +5. For PRs that use the full-validation path, run `npm run check:format`, + `npm run check:spelling`, `npm run check:markdown`, + `npm run check:duplication`, `npm run check:lint` and `npm run check:unused`. + The "Lint repository" check runs these six on that path. Content-only PRs use + focused formatting, spelling and Markdown checks on changed documents. If a + full-path check fails: - `check:format` — run `npm run fix:format` to auto-format. - `check:spelling` — add the flagged term to `.cspell.yml` if it's a legitimate project word, or fix the typo. diff --git a/scripts/lib/content-only-ci.mjs b/scripts/lib/content-only-ci.mjs new file mode 100644 index 00000000..1193c002 --- /dev/null +++ b/scripts/lib/content-only-ci.mjs @@ -0,0 +1,245 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { appendFileSync, existsSync, lstatSync } from 'node:fs'; +import { isAbsolute, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const REPOSITORY_ROOT = fileURLToPath(new URL('../..', import.meta.url)); + +const ROOT_DOCUMENTS = new Set([ + 'CODE_OF_CONDUCT.md', + 'CONTENT-SECURITY.md', + 'CONTRIBUTING.md', + 'GOVERNANCE.md', + 'MAINTAINERS.md', + 'README.md', + 'SECURITY.md', +]); + +const DATA_BACKED_DOCUMENTS = new Set([ + 'docs/community/awards.md', + 'docs/community/members.md', + 'docs/community/projects-born.md', + 'docs/events/index.md', + 'docs/resources/case-studies.md', + 'docs/resources/changelog.md', + 'docs/resources/metrics.md', + 'docs/resources/radar-reports.md', +]); + +function isRegularDocumentFile(path) { + const absolutePath = resolve(REPOSITORY_ROOT, path); + return existsSync(absolutePath) && lstatSync(absolutePath).isFile(); +} + +export function isAuthoredDocumentPath(path) { + if ( + typeof path !== 'string' || + path.includes('\\') || + isAbsolute(path) || + path.split('/').some((segment) => segment === '..' || segment === '.') + ) { + return false; + } + + if (!/\.mdx?$/.test(path) || DATA_BACKED_DOCUMENTS.has(path)) return false; + + if (path.startsWith('docs/architectures/')) { + return ( + path === 'docs/architectures/index.md' || + path === 'docs/architectures/submit.md' + ); + } + + if ( + path.startsWith('docs/') || + path.startsWith('blog/') || + path.startsWith('adr/') + ) { + return true; + } + + return ROOT_DOCUMENTS.has(path); +} + +export function parseNameStatus(output) { + if (typeof output !== 'string') return null; + const fields = output.split('\0'); + if (fields.at(-1) === '') fields.pop(); + + const changes = []; + for (let index = 0; index < fields.length;) { + const status = fields[index]; + const path = fields[index + 1]; + if ( + !/^(?:A|M|D|T|U|X|B)$/.test(status) || + typeof path !== 'string' || + path.length === 0 + ) { + return null; + } + changes.push({ status, path }); + index += 2; + } + return changes; +} + +export function classifyChanges(eventName, baseSha, changes) { + if ( + !['pull_request', 'merge_group'].includes(eventName) || + !/^[0-9a-f]{40}$/i.test(baseSha) || + !Array.isArray(changes) || + changes.length === 0 + ) { + return { documentOnly: false, documents: [] }; + } + + const documents = new Set(); + for (const change of changes) { + if ( + !change || + !['A', 'M', 'D'].includes(change.status) || + !isAuthoredDocumentPath(change.path) + ) { + return { documentOnly: false, documents: [] }; + } + if (change.status === 'D') continue; + if (!isRegularDocumentFile(change.path)) { + return { documentOnly: false, documents: [] }; + } + documents.add(change.path); + } + + return { + documentOnly: true, + documents: [...documents], + }; +} + +export function baseShaForEvent(eventName, environment = process.env) { + if (eventName === 'pull_request') return environment.PR_BASE_SHA; + if (eventName === 'merge_group') return environment.MERGE_BASE_SHA; + return null; +} + +export function writeOutputs(result, outputPath = process.env.GITHUB_OUTPUT) { + if (!outputPath) throw new Error('GITHUB_OUTPUT is required'); + + appendFileSync( + outputPath, + [ + `document_only=${result.documentOnly}`, + `documents_b64=${Buffer.from(JSON.stringify(result.documents)).toString('base64')}`, + '', + ].join('\n'), + ); +} + +export function classifyCurrentEvent( + environment = process.env, + readDiff = execFileSync, + warn = console.warn, +) { + const eventName = environment.GITHUB_EVENT_NAME; + const baseSha = baseShaForEvent(eventName, environment); + + if (!/^[0-9a-f]{40}$/i.test(baseSha)) { + return { documentOnly: false, documents: [] }; + } + + let output; + try { + output = readDiff( + 'git', + ['diff', '--name-status', '-z', '--no-renames', baseSha, 'HEAD', '--'], + { cwd: REPOSITORY_ROOT, encoding: 'utf8', maxBuffer: 50 * 1024 * 1024 }, + ); + } catch (error) { + warn( + `Could not classify changed paths; using full validation: ${error.message}`, + ); + return { documentOnly: false, documents: [] }; + } + + const changes = parseNameStatus(output); + return classifyChanges(eventName, baseSha, changes); +} + +export function runClassifier( + environment = process.env, + classify = classifyCurrentEvent, + log = console.log, +) { + const result = classify(environment); + writeOutputs(result, environment.GITHUB_OUTPUT); + log( + result.documentOnly + ? `Document-only change: ${result.documents.length} changed document(s)` + : 'Using full validation', + ); + return result; +} + +export function decodeChangedDocuments(encoded) { + if ( + typeof encoded !== 'string' || + encoded.length === 0 || + !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded) + ) { + throw new Error('CHANGED_DOCUMENTS_B64 must be a non-empty base64 string'); + } + + let documents; + try { + documents = JSON.parse(Buffer.from(encoded, 'base64').toString('utf8')); + } catch (error) { + throw new Error('CHANGED_DOCUMENTS_B64 must encode a JSON document list', { + cause: error, + }); + } + if (!Array.isArray(documents)) { + throw new Error('CHANGED_DOCUMENTS_B64 must encode a JSON document list'); + } + + const uniqueDocuments = new Set(); + for (const path of documents) { + if (!isAuthoredDocumentPath(path)) { + throw new Error(`Not an authored document path: ${String(path)}`); + } + if (!isRegularDocumentFile(path)) { + throw new Error(`Changed document does not exist: ${path}`); + } + uniqueDocuments.add(path); + } + return [...uniqueDocuments]; +} + +export function runDocumentLint(encoded, runCommand = spawnSync) { + const documents = decodeChangedDocuments(encoded); + if (documents.length === 0) return { documents, checked: [] }; + + const markdown = documents.filter((path) => path.endsWith('.md')); + const commands = [ + ['npx', ['prettier', '--check', '--ignore-path', '', ...documents]], + ['npx', ['cspell', '--no-progress', '-c', '.cspell.yml', ...documents]], + ]; + if (markdown.length > 0) { + commands.push([ + 'npx', + ['--no-install', 'markdownlint', '-c', '.markdownlint.yaml', ...markdown], + ]); + } + + for (const [command, args] of commands) { + const result = runCommand(command, args, { + cwd: REPOSITORY_ROOT, + stdio: 'inherit', + }); + if (result.error) throw result.error; + if (result.status !== 0) { + throw new Error( + `${command} ${args[0]} failed with status ${result.status}`, + ); + } + } + return { documents, checked: commands.map(([, args]) => args[0]) }; +} diff --git a/tests/content-only-ci-workflow.test.mjs b/tests/content-only-ci-workflow.test.mjs new file mode 100644 index 00000000..5e8b80b5 --- /dev/null +++ b/tests/content-only-ci-workflow.test.mjs @@ -0,0 +1,97 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { readWorkflow } from './helpers-workflows.mjs'; + +const workflow = readWorkflow('ci.yml').doc; + +function step(jobName, name) { + const found = workflow.jobs[jobName]?.steps.find( + (item) => item.name === name, + ); + assert.ok(found, `missing ${jobName} step "${name}"`); + return found; +} + +test('classifies PR and merge-queue changes with a complete diff base', () => { + const triggers = workflow.on ?? workflow.true; + assert.ok(Object.hasOwn(triggers, 'pull_request')); + assert.ok(Object.hasOwn(triggers, 'merge_group')); + + const classify = workflow.jobs.classify; + assert.equal( + classify.outputs.document_only, + '${{ steps.classify.outputs.document_only }}', + ); + assert.equal( + classify.outputs.documents_b64, + '${{ steps.classify.outputs.documents_b64 }}', + ); + assert.equal(classify.steps[0].with['fetch-depth'], 0); + assert.match(classify.steps[1].run, /runClassifier\(\)/u); +}); + +test('keeps content contracts and the production build on the focused path', () => { + assert.equal(workflow.jobs.validate.needs, 'classify'); + assert.match(workflow.jobs.validate.if, /!cancelled/u); + assert.match( + workflow.jobs.validate.outputs.document_only, + /needs\.classify\.result/u, + ); + assert.match( + workflow.jobs.validate.env.FULL_VALIDATION, + /needs\.classify\.result/u, + ); + assert.match( + workflow.jobs.validate.env.CONTENT_ONLY, + /document_only == 'true'/u, + ); + assert.equal( + step('validate', 'Check content contracts').if, + "${{ !cancelled() && env.CONTENT_ONLY == 'true' }}", + ); + assert.equal( + step('validate', 'Build production site').run, + 'npm run build:production', + ); + assert.match( + step('validate', 'Run unit tests with coverage').if, + /env\.FULL_VALIDATION/u, + ); +}); + +test('limits focused lint to changed documents and skips browsers only after validation', () => { + assert.equal( + step('lint', 'Check changed-document formatting, spelling, and Markdown') + .if, + "${{ !cancelled() && needs.classify.result == 'success' && needs.classify.outputs.document_only == 'true' }}", + ); + const fullLint = step('lint', 'Run full repository lint'); + assert.match(fullLint.if, /!cancelled\(\)/u); + assert.match(fullLint.if, /needs\.classify\.result != 'success'/u); + for (const command of [ + 'npm run check:format', + 'npm run check:spelling', + 'npm run check:markdown', + 'npm run check:duplication', + 'npm run check:lint', + 'npm run check:unused', + ]) { + assert.ok( + fullLint.run.includes(command), + `missing full lint command ${command}`, + ); + } + + const e2e = workflow.jobs.e2e; + assert.deepEqual(e2e.needs, ['validate', 'lint']); + assert.match(e2e.if, /needs\.validate\.result != 'success'/u); + assert.match(e2e.if, /needs\.lint\.result != 'success'/u); + assert.match(e2e.if, /needs\.validate\.outputs\.document_only != 'true'/u); + + const coverage = workflow.jobs['e2e-coverage']; + assert.deepEqual(coverage.needs, ['classify', 'validate', 'lint']); + assert.match(coverage.if, /needs\.classify\.result != 'success'/u); + assert.match(coverage.if, /needs\.validate\.result != 'success'/u); + assert.match(coverage.if, /needs\.lint\.result != 'success'/u); +}); diff --git a/tests/content-only-ci.test.mjs b/tests/content-only-ci.test.mjs new file mode 100644 index 00000000..243caa4a --- /dev/null +++ b/tests/content-only-ci.test.mjs @@ -0,0 +1,472 @@ +import assert from 'node:assert/strict'; +import { + appendFileSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; + +import { + baseShaForEvent, + classifyChanges, + classifyCurrentEvent, + decodeChangedDocuments, + isAuthoredDocumentPath, + parseNameStatus, + runDocumentLint, + runClassifier, + writeOutputs, +} from '../scripts/lib/content-only-ci.mjs'; + +const BASE_SHA = 'a'.repeat(40); + +test('accepts authored documents in supported content locations', () => { + for (const path of [ + 'README.md', + 'CODE_OF_CONDUCT.md', + 'CONTENT-SECURITY.md', + 'CONTRIBUTING.md', + 'GOVERNANCE.md', + 'MAINTAINERS.md', + 'SECURITY.md', + 'adr/0011-content-focused-validation-for-documentation.md', + 'blog/2026-10-29-post.md', + 'docs/community/user-groups/research/index.md', + 'docs/architectures/index.md', + 'docs/architectures/submit.md', + ]) { + assert.equal(isAuthoredDocumentPath(path), true, path); + } +}); + +test('rejects generated and data-backed documents', () => { + for (const path of [ + 'docs/architectures/zeiss/index.md', + 'docs/community/awards.md', + 'docs/community/members.md', + 'docs/community/projects-born.md', + 'docs/events/index.md', + 'docs/resources/case-studies.md', + 'docs/resources/changelog.md', + 'docs/resources/metrics.md', + 'docs/resources/radar-reports.md', + ]) { + assert.equal(isAuthoredDocumentPath(path), false, path); + } +}); + +test('rejects code, data, config, assets, and internal agent instructions', () => { + for (const path of [ + '.agents/skills/doc-authoring/SKILL.md', + '.github/PULL_REQUEST_TEMPLATE.md', + 'AGENTS.md', + 'data/awards.json', + 'docs/community/index.js', + 'docs/community/logo.svg', + 'docs/example.md.backup', + 'docs/example.mdx.backup', + 'scripts/validate-community-groups.mjs', + 'tests/content-only-ci.test.mjs', + ]) { + assert.equal(isAuthoredDocumentPath(path), false, path); + } +}); + +test('rejects unsafe and malformed document paths', () => { + for (const path of [ + '', + '/docs/example.md', + 'docs/../README.md', + 'docs/./example.md', + 'docs\\community\\index.md', + null, + ]) { + assert.equal(isAuthoredDocumentPath(path), false); + } +}); + +test('parses NUL-delimited git name-status output without losing spaces', () => { + assert.deepEqual( + parseNameStatus('M\0docs/a file.md\0A\0docs/new.md\0D\0docs/old.md\0'), + [ + { status: 'M', path: 'docs/a file.md' }, + { status: 'A', path: 'docs/new.md' }, + { status: 'D', path: 'docs/old.md' }, + ], + ); +}); + +test('parses empty git diff output as no changes', () => { + assert.deepEqual(parseNameStatus(''), []); + assert.deepEqual(parseNameStatus('M\0docs/file.md'), [ + { status: 'M', path: 'docs/file.md' }, + ]); + assert.equal(parseNameStatus('M\0\0'), null); +}); + +test('treats malformed git diff output as unclassifiable', () => { + for (const output of [ + null, + 'M\0', + 'M\0docs/file.md\0R100\0old.md\0', + 'MA\0docs/file.md\0', + 'Mfoo\0docs/file.md\0', + ]) { + assert.equal(parseNameStatus(output), null); + } +}); + +test('classifies a docs-only change and excludes deleted files from lint input', () => { + const result = classifyChanges('pull_request', BASE_SHA, [ + { status: 'M', path: 'docs/community/user-groups/research/index.md' }, + { status: 'A', path: 'docs/community/index.md' }, + { status: 'D', path: 'docs/community/user-groups/research/old.md' }, + ]); + + assert.equal(result.documentOnly, true); + assert.deepEqual(result.documents, [ + 'docs/community/user-groups/research/index.md', + 'docs/community/index.md', + ]); +}); + +test('accepts merge-group content diffs', () => { + const result = classifyChanges('merge_group', BASE_SHA, [ + { status: 'M', path: 'docs/community/index.md' }, + ]); + + assert.equal(result.documentOnly, true); +}); + +test('uses full validation for empty, unknown, mixed, or uncertain changes', () => { + const cases = [ + ['pull_request', BASE_SHA, []], + [ + 'workflow_dispatch', + BASE_SHA, + [{ status: 'M', path: 'docs/community/index.md' }], + ], + [ + 'pull_request', + `${BASE_SHA}x`, + [{ status: 'M', path: 'docs/community/index.md' }], + ], + [ + 'pull_request', + `x${BASE_SHA}`, + [{ status: 'M', path: 'docs/community/index.md' }], + ], + [ + 'pull_request', + BASE_SHA.slice(1), + [{ status: 'M', path: 'docs/community/index.md' }], + ], + ['pull_request', BASE_SHA, null], + ['pull_request', BASE_SHA, [null]], + [ + 'pull_request', + BASE_SHA, + [ + { status: 'M', path: 'docs/example.md' }, + { status: 'M', path: 'src/component.js' }, + ], + ], + ['pull_request', BASE_SHA, [{ status: 'R100', path: 'docs/new.md' }]], + ['pull_request', BASE_SHA, [{ status: 'M', path: 'unknown/file.md' }]], + ]; + + for (const [event, base, changes] of cases) { + assert.deepEqual(classifyChanges(event, base, changes), { + documentOnly: false, + documents: [], + }); + } +}); + +test('does not pass through a generated or data-backed document change', () => { + for (const path of [ + 'docs/resources/case-studies.md', + 'docs/architectures/zeiss/index.md', + ]) { + assert.deepEqual( + classifyChanges('pull_request', BASE_SHA, [{ status: 'M', path }]), + { documentOnly: false, documents: [] }, + ); + } +}); + +test('uses full validation for document paths that are not regular files', () => { + const path = 'docs/__content-only-ci-test__.md'; + symlinkSync('../README.md', path); + try { + assert.deepEqual( + classifyChanges('pull_request', BASE_SHA, [{ status: 'A', path }]), + { documentOnly: false, documents: [] }, + ); + } finally { + rmSync(path, { force: true }); + } +}); + +test('uses supported event-specific base SHAs', () => { + const environment = { PR_BASE_SHA: 'pr', MERGE_BASE_SHA: 'merge' }; + assert.equal(baseShaForEvent('pull_request', environment), 'pr'); + assert.equal(baseShaForEvent('merge_group', environment), 'merge'); + assert.equal(baseShaForEvent('workflow_dispatch', environment), null); +}); + +test('classifies invalid or unsupported events as full validation', () => { + const environments = [ + { GITHUB_EVENT_NAME: 'pull_request', PR_BASE_SHA: 'bad' }, + { GITHUB_EVENT_NAME: 'pull_request', PR_BASE_SHA: `${BASE_SHA}x` }, + { GITHUB_EVENT_NAME: 'pull_request', PR_BASE_SHA: `x${BASE_SHA}` }, + { GITHUB_EVENT_NAME: 'pull_request', PR_BASE_SHA: BASE_SHA.slice(1) }, + { GITHUB_EVENT_NAME: 'workflow_dispatch', PR_BASE_SHA: BASE_SHA }, + ]; + for (const environment of environments) { + let readDiffCalled = false; + const readDiff = () => { + readDiffCalled = true; + return 'M\0docs/community/index.md\0'; + }; + assert.deepEqual(classifyCurrentEvent(environment, readDiff), { + documentOnly: false, + documents: [], + }); + assert.equal(readDiffCalled, false); + } +}); + +test('reads the event diff and fails closed when git cannot read it', () => { + let gitCall; + assert.deepEqual( + classifyCurrentEvent( + { GITHUB_EVENT_NAME: 'pull_request', PR_BASE_SHA: BASE_SHA }, + (...args) => { + gitCall = args; + return 'M\0docs/community/index.md\0'; + }, + ), + { documentOnly: true, documents: ['docs/community/index.md'] }, + ); + assert.deepEqual(gitCall, [ + 'git', + ['diff', '--name-status', '-z', '--no-renames', BASE_SHA, 'HEAD', '--'], + { + cwd: fileURLToPath(new URL('../', import.meta.url)), + encoding: 'utf8', + maxBuffer: 50 * 1024 * 1024, + }, + ]); + + const warnings = []; + assert.deepEqual( + classifyCurrentEvent( + { GITHUB_EVENT_NAME: 'merge_group', MERGE_BASE_SHA: BASE_SHA }, + () => { + throw new Error('git unavailable'); + }, + (warning) => warnings.push(warning), + ), + { documentOnly: false, documents: [] }, + ); + assert.deepEqual(warnings, [ + 'Could not classify changed paths; using full validation: git unavailable', + ]); +}); + +test('writes safely encoded GitHub Actions outputs', () => { + const directory = mkdtempSync(join(tmpdir(), 'content-only-ci-')); + const outputPath = join(directory, 'output'); + try { + appendFileSync(outputPath, 'existing=value\n'); + const result = { + documentOnly: true, + documents: ['docs/community/user groups/research.md'], + }; + writeOutputs(result, outputPath); + assert.equal( + readFileSync(outputPath, 'utf8'), + `existing=value\ndocument_only=true\ndocuments_b64=${Buffer.from( + JSON.stringify(result.documents), + ).toString('base64')}\n`, + ); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test('requires a GitHub Actions output file', () => { + assert.throws( + () => writeOutputs({ documentOnly: false, documents: [] }, ''), + { + message: 'GITHUB_OUTPUT is required', + }, + ); +}); + +test('runs the classifier and writes full-validation outputs', () => { + const directory = mkdtempSync(join(tmpdir(), 'content-only-ci-')); + const outputPath = join(directory, 'output'); + const logs = []; + try { + const result = runClassifier( + { + GITHUB_EVENT_NAME: 'workflow_dispatch', + GITHUB_OUTPUT: outputPath, + }, + undefined, + (message) => logs.push(message), + ); + assert.deepEqual(result, { documentOnly: false, documents: [] }); + assert.match(readFileSync(outputPath, 'utf8'), /^document_only=false\n/); + assert.deepEqual(logs, ['Using full validation']); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test('reports the document-only classifier result', () => { + const directory = mkdtempSync(join(tmpdir(), 'content-only-ci-')); + const outputPath = join(directory, 'output'); + const logs = []; + try { + const result = runClassifier( + { GITHUB_OUTPUT: outputPath }, + () => ({ + documentOnly: true, + documents: ['README.md'], + }), + (message) => logs.push(message), + ); + assert.deepEqual(result, { documentOnly: true, documents: ['README.md'] }); + assert.match(readFileSync(outputPath, 'utf8'), /^document_only=true\n/); + assert.deepEqual(logs, ['Document-only change: 1 changed document(s)']); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test('decodes and validates the changed-document list', () => { + const encoded = (documents) => + Buffer.from(JSON.stringify(documents)).toString('base64'); + assert.deepEqual( + decodeChangedDocuments(encoded(['README.md', 'README.md'])), + ['README.md'], + ); + assert.deepEqual(decodeChangedDocuments(encoded([])), []); + assert.deepEqual(runDocumentLint(encoded([])), { + documents: [], + checked: [], + }); + + for (const invalid of [undefined, '', '!', '!W10=', 'W10=!']) { + assert.throws(() => decodeChangedDocuments(invalid), { + message: 'CHANGED_DOCUMENTS_B64 must be a non-empty base64 string', + }); + } + assert.throws( + () => decodeChangedDocuments(Buffer.from('not json').toString('base64')), + (error) => + error.message === + 'CHANGED_DOCUMENTS_B64 must encode a JSON document list' && + error.cause instanceof SyntaxError, + ); + assert.throws( + () => decodeChangedDocuments(Buffer.from('"README.md"').toString('base64')), + { + message: 'CHANGED_DOCUMENTS_B64 must encode a JSON document list', + }, + ); + assert.throws(() => decodeChangedDocuments(encoded(['src/app.js'])), { + message: 'Not an authored document path: src/app.js', + }); + assert.throws( + () => decodeChangedDocuments(encoded(['docs/not-a-real-page.md'])), + { message: 'Changed document does not exist: docs/not-a-real-page.md' }, + ); +}); + +test('runs changed-document formatting, spelling, and Markdown checks', () => { + const calls = []; + const result = runDocumentLint( + Buffer.from(JSON.stringify(['README.md'])).toString('base64'), + (command, args, options) => { + calls.push({ command, args, options }); + return { status: 0 }; + }, + ); + + assert.deepEqual(result, { + documents: ['README.md'], + checked: ['prettier', 'cspell', '--no-install'], + }); + const options = { + cwd: fileURLToPath(new URL('../', import.meta.url)), + stdio: 'inherit', + }; + assert.deepEqual(calls, [ + { + command: 'npx', + args: ['prettier', '--check', '--ignore-path', '', 'README.md'], + options, + }, + { + command: 'npx', + args: ['cspell', '--no-progress', '-c', '.cspell.yml', 'README.md'], + options, + }, + { + command: 'npx', + args: [ + '--no-install', + 'markdownlint', + '-c', + '.markdownlint.yaml', + 'README.md', + ], + options, + }, + ]); +}); + +test('runs only formatting and spelling for MDX documents', () => { + const path = 'docs/__content-only-ci-test__.mdx'; + writeFileSync(path, '# Test\n'); + try { + const calls = []; + const result = runDocumentLint( + Buffer.from(JSON.stringify([path])).toString('base64'), + (command, args) => { + calls.push({ command, args }); + return { status: 0 }; + }, + ); + assert.deepEqual(result.checked, ['prettier', 'cspell']); + assert.deepEqual( + calls.map(({ args }) => args[0]), + ['prettier', 'cspell'], + ); + } finally { + rmSync(path, { force: true }); + } +}); + +test('fails changed-document lint when a command fails or cannot start', () => { + const encoded = Buffer.from(JSON.stringify(['README.md'])).toString('base64'); + assert.throws( + () => runDocumentLint(encoded, () => ({ status: 1 })), + /npx prettier failed with status 1/u, + ); + const failure = new Error('command unavailable'); + assert.throws( + () => runDocumentLint(encoded, () => ({ error: failure })), + (error) => error === failure, + ); +}); diff --git a/tests/contributing-just-build-parity.test.mjs b/tests/contributing-just-build-parity.test.mjs index 4c3fa45f..501feddc 100644 --- a/tests/contributing-just-build-parity.test.mjs +++ b/tests/contributing-just-build-parity.test.mjs @@ -276,8 +276,8 @@ test('a gating check dropped from the qualifying paragraph is caught', () => { test('a check added to a gating job and nowhere else is caught', () => { const drifted = WORKFLOW.replace( - ' run: npm run check:spelling\n', - ' run: npm run check:spelling\n - name: Newly added gate\n run: npm run check:newly-added\n', + ' npm run check:spelling\n', + ' npm run check:spelling\n npm run check:newly-added\n', ); assert.notEqual(drifted, WORKFLOW, 'the fixture edit matched nothing'); const { missing } = uncompensatedGates(CONTRIBUTING, JUSTFILE, drifted); From 6e2b536becac780720e729cb3f95c8b5f6f0e461 Mon Sep 17 00:00:00 2001 From: mrbobbytables Date: Thu, 29 Oct 2026 19:27:38 -0500 Subject: [PATCH 3/3] fix: respect prettier ignores in focused document lint Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: mrbobbytables --- scripts/lib/content-only-ci.mjs | 2 +- tests/content-only-ci.test.mjs | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/scripts/lib/content-only-ci.mjs b/scripts/lib/content-only-ci.mjs index 1193c002..0ed96d03 100644 --- a/scripts/lib/content-only-ci.mjs +++ b/scripts/lib/content-only-ci.mjs @@ -219,7 +219,7 @@ export function runDocumentLint(encoded, runCommand = spawnSync) { const markdown = documents.filter((path) => path.endsWith('.md')); const commands = [ - ['npx', ['prettier', '--check', '--ignore-path', '', ...documents]], + ['npx', ['prettier', '--check', ...documents]], ['npx', ['cspell', '--no-progress', '-c', '.cspell.yml', ...documents]], ]; if (markdown.length > 0) { diff --git a/tests/content-only-ci.test.mjs b/tests/content-only-ci.test.mjs index 243caa4a..fdc421a8 100644 --- a/tests/content-only-ci.test.mjs +++ b/tests/content-only-ci.test.mjs @@ -414,7 +414,7 @@ test('runs changed-document formatting, spelling, and Markdown checks', () => { assert.deepEqual(calls, [ { command: 'npx', - args: ['prettier', '--check', '--ignore-path', '', 'README.md'], + args: ['prettier', '--check', 'README.md'], options, }, { @@ -436,6 +436,28 @@ test('runs changed-document formatting, spelling, and Markdown checks', () => { ]); }); +test('preserves repository Prettier ignores for blog documents', () => { + const calls = []; + runDocumentLint( + Buffer.from( + JSON.stringify(['blog/2026-07-22-welcome-to-endusers-cncf-io.md']), + ).toString('base64'), + (command, args) => { + calls.push({ command, args }); + return { status: 0 }; + }, + ); + + assert.deepEqual(calls[0], { + command: 'npx', + args: [ + 'prettier', + '--check', + 'blog/2026-07-22-welcome-to-endusers-cncf-io.md', + ], + }); +}); + test('runs only formatting and spelling for MDX documents', () => { const path = 'docs/__content-only-ci-test__.mdx'; writeFileSync(path, '# Test\n');