diff --git a/.gitignore b/.gitignore index b73add43a..9143d8c98 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ # Dependencies /node_modules -/scripts/site-build-gate/node_modules/ # Production /build diff --git a/AGENTS.md b/AGENTS.md index 28fe50452..bbd770c1b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,8 +49,10 @@ Netlify build or deployment responsible for merge eligibility: ## Static build gate contract -- Learn commits the complete checksum-bound Node.js 22 gate under `scripts/site-build-gate/` and - runs it locally after Docusaurus. CI and deployment do not fetch or execute the SEO repository. +- Learn installs the SEO-owned Node.js 22 gate as the published npm package + `@netdata/site-build-gate` and runs it locally after Docusaurus. Its major version is the + ruleset version and must match `config/site-build-gate-baseline.json`. CI and deployment do not + fetch or execute the SEO repository. - Learn declares `/docs/collecting-metrics/collectors` as its segment-safe integration route prefix. The generic page-weight diagnostic excludes that route family; integration pages remain subject to every non-weight rule, so this is not a whole-page validation bypass. diff --git a/README.md b/README.md index e027d9e5b..9cda3c715 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,7 @@ Please also look at the [style guide](https://github.com/netdata/netdata/blob/ma ``` 3. Install Yarn Classic 1.22.22 and the locked dependencies. Netlify selects Yarn because - this repository tracks `yarn.lock`; npm 10.9.9 remains pinned for the nested build-gate - install run by `build:netlify`. + this repository tracks `yarn.lock`. ```bash npm install --global yarn@1.22.22 @@ -74,8 +73,8 @@ the plugin's asset list; the site build gate requires the page to have a title. ### Dependency update authority The root JavaScript dependency authority is `package.json` plus Yarn Classic `yarn.lock`. Do not -add a root `package-lock.json`. The npm lock below `scripts/site-build-gate/` is a separate, -website-owned vendor contract and remains npm-managed. Dependabot watches the root Yarn graph and +add a root `package-lock.json`. The site build gate is the published `@netdata/site-build-gate` +package, so its parser dependencies are part of the root Yarn graph. Dependabot watches that graph and GitHub Actions; owner-controlled vendor packages are intentionally excluded from automatic updates. Root Yarn resolutions are only used for security releases that are compatible with every selected diff --git a/package.json b/package.json index 98b3dcaca..1464f608c 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "docusaurus": "docusaurus", "start": "docusaurus start", "build": "docusaurus build", - "build:netlify": "node scripts/verify-git-history.js && node scripts/verify-indexnow-vendor.js && npm ci --prefix scripts/site-build-gate --ignore-scripts --no-audit && docusaurus build && node scripts/run-post-build-gates.mjs", + "build:netlify": "node scripts/verify-git-history.js && node scripts/verify-indexnow-vendor.js && docusaurus build && node scripts/run-post-build-gates.mjs", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", "clear": "docusaurus clear", @@ -73,6 +73,7 @@ }, "devDependencies": { "@docusaurus/faster": "3.10.2", + "@netdata/site-build-gate": "^10.1.0", "@tailwindcss/postcss": "^4.3.3", "@testing-library/dom": "^10.4.2", "@testing-library/jest-dom": "^7.0.1", diff --git a/scripts/run-post-build-gates.mjs b/scripts/run-post-build-gates.mjs index 64f32952f..abd1fca51 100644 --- a/scripts/run-post-build-gates.mjs +++ b/scripts/run-post-build-gates.mjs @@ -21,7 +21,7 @@ const strictGeneratedCorpusGates = [ ['scripts/verify-rendered-indexability.js'], ['scripts/verify-cloudflare-rum.js', 'build'], [ - 'scripts/site-build-gate/site_build_gate.mjs', + 'node_modules/@netdata/site-build-gate/site_build_gate.mjs', '--build-dir', 'build', '--site-origin', diff --git a/scripts/site-build-gate/manifest.json b/scripts/site-build-gate/manifest.json deleted file mode 100644 index 28b9a9515..000000000 --- a/scripts/site-build-gate/manifest.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "schema": "netdata-site-build-gate-vendor-v2", - "contract": "netdata-site-build-gate-v1", - "ruleset_version": 10, - "node_major": 22, - "artifacts": [ - { - "path": "package-lock.json", - "sha256": "a3cac54442c679c4888131038520a50e37ea092c1c7ea378c88b546e483e0c35" - }, - { - "path": "package.json", - "sha256": "5cd23bae2605a5fcd2bed67b547ae7a4be0f9102dde6265eb4a770c59dce9fe4" - }, - { - "path": "site_build_gate.mjs", - "sha256": "171cc249076efa1e9e1edb5cecdbd83b5487f9ea6c7e5b94caa73a1be2f44d4f" - } - ] -} diff --git a/scripts/site-build-gate/package-lock.json b/scripts/site-build-gate/package-lock.json deleted file mode 100644 index 656282818..000000000 --- a/scripts/site-build-gate/package-lock.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "name": "@netdata/site-build-gate", - "version": "10.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@netdata/site-build-gate", - "version": "10.0.0", - "dependencies": { - "css-tree": "3.2.1", - "parse5": "8.0.1", - "saxes": "6.0.0" - }, - "engines": { - "node": "22.x" - } - }, - "node_modules/css-tree": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz", - "integrity": "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==", - "license": "MIT", - "dependencies": { - "mdn-data": "2.27.1", - "source-map-js": "^1.2.1" - }, - "engines": { - "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" - } - }, - "node_modules/entities": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-8.1.0.tgz", - "integrity": "sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=20.19.0" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/mdn-data": { - "version": "2.27.1", - "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz", - "integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==", - "license": "CC0-1.0" - }, - "node_modules/parse5": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz", - "integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==", - "license": "MIT", - "dependencies": { - "entities": "^8.0.0" - }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" - } - }, - "node_modules/saxes": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", - "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", - "license": "ISC", - "dependencies": { - "xmlchars": "^2.2.0" - }, - "engines": { - "node": ">=v12.22.7" - } - }, - "node_modules/source-map-js": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", - "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/xmlchars": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", - "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", - "license": "MIT" - } - } -} diff --git a/scripts/site-build-gate/package.json b/scripts/site-build-gate/package.json deleted file mode 100644 index 11054927e..000000000 --- a/scripts/site-build-gate/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "@netdata/site-build-gate", - "version": "10.0.0", - "private": true, - "type": "module", - "engines": { - "node": "22.x" - }, - "dependencies": { - "css-tree": "3.2.1", - "parse5": "8.0.1", - "saxes": "6.0.0" - } -} diff --git a/scripts/site-build-gate/site_build_gate.mjs b/scripts/site-build-gate/site_build_gate.mjs deleted file mode 100644 index 7776f7afe..000000000 --- a/scripts/site-build-gate/site_build_gate.mjs +++ /dev/null @@ -1,1104 +0,0 @@ -#!/usr/bin/env node -/** Fail a static site build when accepted technical defects regress. */ - -import { createHash } from "node:crypto"; -import { - existsSync, - lstatSync, - readFileSync, - readdirSync, - realpathSync, - statSync, -} from "node:fs"; -import { basename, dirname, extname, join, relative, resolve, sep } from "node:path"; -import { fileURLToPath } from "node:url"; - -import * as csstree from "css-tree"; -import { parse as parseHtml } from "parse5"; -import { SaxesParser } from "saxes"; - -export const CONTRACT_SCHEMA = "netdata-site-build-gate-v1"; -export const MANIFEST_SCHEMA = "netdata-site-build-gate-vendor-v2"; -export const BASELINE_SCHEMA = "netdata-site-build-gate-baseline-v1"; -export const REPORT_SCHEMA = "netdata-site-build-gate-report-v1"; -export const RULESET_VERSION = 10; -export const REQUIRED_NODE_MAJOR = 22; -export const HEAVY_BYTES = 500_000; - -const HTML_NAMESPACE = "http://www.w3.org/1999/xhtml"; -const SITEMAP_NAMESPACE = "http://www.sitemaps.org/schemas/sitemap/0.9"; -const HTML_SUFFIXES = new Set([".html", ".htm", ".xhtml"]); -const TRACKED_CSS_PROPERTIES = ["content-visibility", "display", "visibility"]; -const UA_DISPLAY_NONE_TAGS = new Set(["datalist", "rp"]); -const SEMANTICALLY_INERT = new Set([ - "iframe", - "noembed", - "noframes", - "noscript", - "script", - "style", - "template", - "xmp", -]); - -export const RULES = Object.freeze({ - "missing-title": { - classification: "accessibility", - basis: "WCAG 2.4.2 Page Titled, Level A", - }, - "images-missing-alt": { - classification: "accessibility", - basis: "WCAG 1.1.1 Non-text Content, Level A", - }, - "missing-description": { - classification: "presentation", - basis: "Google may compose a snippet; an authored description is a presentation choice", - }, - "missing-h1": { - classification: "presentation", - basis: "Neither WCAG nor Google requires one h1; the site contract requires a semantic heading", - }, - "multiple-h1": { - classification: "presentation", - basis: "Neither WCAG nor Google requires one h1; the site contract keeps one document heading", - }, - "duplicate-title": { - classification: "presentation", - basis: "Identical result titles are ambiguous to readers, not a Google penalty", - }, - "duplicate-description": { - classification: "presentation", - basis: "Identical descriptions make distinct pages read as interchangeable", - }, - "heavy-page": { - classification: "diagnostic", - basis: "500000 bytes is an estate performance guardrail, not a Google threshold", - }, -}); - -export class GateError extends Error { - constructor(message) { - super(message); - this.name = "GateError"; - } -} - -class MissingBuiltHtml extends GateError {} - -function compareText(left, right) { - return left < right ? -1 : left > right ? 1 : 0; -} - -function isPlainObject(value) { - return value !== null && typeof value === "object" && !Array.isArray(value); -} - -function exactKeys(value, expected, label) { - if (!isPlainObject(value)) throw new GateError(`${label} must be a JSON object`); - const actual = Object.keys(value).sort(compareText); - const wanted = [...expected].sort(compareText); - if (JSON.stringify(actual) !== JSON.stringify(wanted)) { - throw new GateError(`${label} fields must be exactly: ${wanted.join(", ")}`); - } -} - -function loadJson(path, label) { - try { - const value = JSON.parse(readFileSync(path, "utf8")); - if (!isPlainObject(value)) throw new GateError(`${label} must be a JSON object: ${path}`); - return value; - } catch (error) { - if (error instanceof GateError) throw error; - throw new GateError(`cannot read ${label} ${path}: ${error.message}`); - } -} - -function sha256Bytes(payload) { - return createHash("sha256").update(payload).digest("hex"); -} - -function requireNode22() { - const major = Number.parseInt(process.versions.node.split(".", 1)[0], 10); - if (major !== REQUIRED_NODE_MAJOR) { - throw new GateError(`Node.js ${REQUIRED_NODE_MAJOR}.x is required; running ${process.version}`); - } -} - -export function verifyContract(artifactPath, manifestPath) { - requireNode22(); - let manifestFile; - let runningArtifact; - try { - manifestFile = realpathSync(resolve(manifestPath)); - runningArtifact = realpathSync(resolve(artifactPath)); - } catch (error) { - throw new GateError(`cannot resolve vendor contract paths: ${error.message}`); - } - const manifest = loadJson(manifestFile, "vendor manifest"); - exactKeys(manifest, ["schema", "contract", "ruleset_version", "node_major", "artifacts"], "vendor manifest"); - if (manifest.schema !== MANIFEST_SCHEMA || manifest.contract !== CONTRACT_SCHEMA) { - throw new GateError("vendor manifest schema or contract is not supported"); - } - if (manifest.ruleset_version !== RULESET_VERSION || manifest.node_major !== REQUIRED_NODE_MAJOR) { - throw new GateError("vendor manifest ruleset or Node.js requirement does not match the checker"); - } - if (!Array.isArray(manifest.artifacts) || manifest.artifacts.length === 0) { - throw new GateError("vendor manifest artifacts must be a non-empty list"); - } - const root = dirname(manifestFile); - const names = new Set(); - for (const entry of manifest.artifacts) { - exactKeys(entry, ["path", "sha256"], "vendor artifact"); - if (typeof entry.path !== "string" || basename(entry.path) !== entry.path || entry.path === "manifest.json") { - throw new GateError(`unsafe vendor artifact path: ${JSON.stringify(entry.path)}`); - } - if (names.has(entry.path)) throw new GateError(`duplicate vendor artifact: ${entry.path}`); - names.add(entry.path); - let payload; - try { - const candidate = join(root, entry.path); - if (!lstatSync(candidate).isFile()) throw new GateError(`vendor artifact must be a regular file: ${entry.path}`); - payload = readFileSync(candidate); - } catch (error) { - throw new GateError(`cannot read vendor artifact ${entry.path}: ${error.message}`); - } - const actual = sha256Bytes(payload); - if (typeof entry.sha256 !== "string" || entry.sha256 !== actual) { - throw new GateError(`vendor checksum mismatch for ${entry.path}: expected ${entry.sha256}, got ${actual}`); - } - } - const boundArtifact = join(root, basename(runningArtifact)); - if (!names.has(basename(runningArtifact)) || realpathSync(boundArtifact) !== runningArtifact) { - throw new GateError(`vendor manifest does not bind the running artifact: ${basename(runningArtifact)}`); - } - return manifest; -} - -function preNormalizationPath(value, label) { - if (/[\\\u0000-\u0020\u007f]/u.test(value)) { - throw new GateError(`unsafe ${label}: raw controls, whitespace, and backslashes are forbidden`); - } - const match = /^[A-Za-z][A-Za-z0-9+.-]*:\/\/([^/?#]*)(\/[^?#]*)?(\?[^#]*)?(#.*)?$/su.exec(value); - if (!match) throw new GateError(`${label} must be an absolute HTTP(S) URL: ${JSON.stringify(value)}`); - if (match[1].includes("@")) throw new GateError(`${label} cannot contain user information: ${JSON.stringify(value)}`); - if (match[3] !== undefined || match[4] !== undefined) { - throw new GateError(`${label} cannot carry a query or fragment: ${JSON.stringify(value)}`); - } - const rawPath = match[2] || "/"; - if (rawPath.includes("//") || rawPath.includes("\\") || rawPath.includes("\0") || /%(?:2f|5c)/iu.test(rawPath)) { - throw new GateError(`unsafe ${label} path: ${JSON.stringify(rawPath)}`); - } - for (const rawPart of rawPath.split("/")) { - let decoded; - try { - decoded = decodeURIComponent(rawPart); - } catch { - throw new GateError(`${label} path has malformed or non-UTF-8 percent encoding: ${JSON.stringify(rawPath)}`); - } - if (decoded === "." || decoded === ".." || decoded.includes("\\") || decoded.includes("\0")) { - throw new GateError(`unsafe ${label} path: ${JSON.stringify(rawPath)}`); - } - } - return rawPath; -} - -function checkedUrl(value, expectedOrigin, label) { - preNormalizationPath(value, label); - let parsed; - try { - parsed = new URL(value); - } catch (error) { - throw new GateError(`${label} is not a valid URL: ${JSON.stringify(value)} (${error.message})`); - } - if (!new Set(["http:", "https:"]).has(parsed.protocol) || !parsed.hostname) { - throw new GateError(`${label} must be an absolute HTTP(S) URL: ${JSON.stringify(value)}`); - } - if (parsed.username || parsed.password) { - throw new GateError(`${label} cannot contain user information: ${JSON.stringify(value)}`); - } - if (expectedOrigin !== null && parsed.origin !== expectedOrigin) { - throw new GateError(`${label} origin ${JSON.stringify(parsed.origin)} does not match site origin ${JSON.stringify(expectedOrigin)}: ${JSON.stringify(value)}`); - } - if (parsed.search || parsed.hash) { - throw new GateError(`${label} cannot carry a query or fragment: ${JSON.stringify(value)}`); - } - return parsed; -} - -export function normalizeSiteOrigin(value) { - const parsed = checkedUrl(value, null, "site origin"); - if (parsed.pathname !== "/") { - throw new GateError(`site origin must not contain a path, query, or fragment: ${JSON.stringify(value)}`); - } - return parsed.origin; -} - -export function normalizeIntegrationRoutePrefixes(values = []) { - if (!Array.isArray(values)) throw new GateError("integration route prefixes must be a list"); - const policyOrigin = "https://integration-policy.invalid"; - const normalized = values.map((value) => { - if (typeof value !== "string" || !value.startsWith("/")) { - throw new GateError(`integration route prefix must be an absolute path: ${JSON.stringify(value)}`); - } - const parsed = checkedUrl(`${policyOrigin}${value}`, policyOrigin, "integration route prefix"); - const decoded = safeUrlPath(parsed, "integration route prefix").decodedPath; - const prefix = decoded.length > 1 && decoded.endsWith("/") ? decoded.slice(0, -1) : decoded; - if (prefix === "/") throw new GateError("integration route prefix cannot cover the site root"); - return prefix; - }).sort(compareText); - if (new Set(normalized).size !== normalized.length) { - throw new GateError("integration route prefixes must be unique after normalization"); - } - return normalized; -} - -function safeUrlPath(parsed, label) { - const rawPath = parsed.pathname || "/"; - const parts = []; - for (const rawPart of rawPath.split("/")) { - if (!rawPart) continue; - let decoded; - try { - decoded = decodeURIComponent(rawPart); - } catch { - throw new GateError(`${label} path is not UTF-8: ${JSON.stringify(rawPath)}`); - } - if (decoded === "." || decoded === ".." || decoded.includes("/") || decoded.includes("\\") || decoded.includes("\0")) { - throw new GateError(`unsafe ${label} path: ${JSON.stringify(rawPath)}`); - } - parts.push(decoded); - } - const trailingSlash = rawPath.endsWith("/") && parts.length > 0; - return { parts, decodedPath: parts.length ? `/${parts.join("/")}${trailingSlash ? "/" : ""}` : "/" }; -} - -function insideRoot(root, candidate) { - const delta = relative(root, candidate); - return delta === "" || (!delta.startsWith(`..${sep}`) && delta !== ".." && !delta.startsWith(sep)); -} - -function exactLexicalPath(buildRoot, parts, location) { - let lexical = buildRoot; - for (const part of parts) { - let names; - try { - if (!statSync(lexical).isDirectory()) return null; - names = readdirSync(lexical); - } catch (error) { - if (error?.code === "ENOENT" || error?.code === "ENOTDIR") return null; - throw new GateError(`cannot enumerate built path for ${JSON.stringify(location)}: ${error.message}`); - } - if (!names.includes(part)) return null; - lexical = join(lexical, part); - const actual = realpathSync(lexical); - if (!insideRoot(buildRoot, actual)) throw new GateError(`path symlink escapes build directory: ${JSON.stringify(location)}`); - } - return lexical; -} - -function safeExistingArtifact(buildRoot, parts, location) { - const lexical = exactLexicalPath(buildRoot, parts, location); - if (lexical === null) return null; - if (!insideRoot(buildRoot, lexical)) throw new GateError(`path escapes build directory: ${JSON.stringify(location)}`); - const actual = realpathSync(lexical); - if (!insideRoot(buildRoot, actual)) throw new GateError(`path symlink escapes build directory: ${JSON.stringify(location)}`); - return statSync(actual).isFile() ? { publicPath: lexical, file: actual } : null; -} - -function safeExistingPath(buildRoot, parts, location) { - return safeExistingArtifact(buildRoot, parts, location)?.file || null; -} - -function safeExistingDirectory(buildRoot, parts, location) { - const lexical = exactLexicalPath(buildRoot, parts, location); - if (lexical === null) return null; - const actual = realpathSync(lexical); - if (!insideRoot(buildRoot, actual)) throw new GateError(`path symlink escapes build directory: ${JSON.stringify(location)}`); - return statSync(actual).isDirectory() ? actual : null; -} - -function parseStrictSitemap(xml, label) { - const stack = []; - let root = null; - let failure = null; - const parser = new SaxesParser({ xmlns: true }); - parser.on("error", (error) => { - failure ||= new GateError(`cannot parse local sitemap ${label}: ${error.message}`); - }); - parser.on("opentag", (tag) => { - const node = { local: tag.local, uri: tag.uri, text: "", directLocs: [] }; - if (stack.length === 0) root = node; - stack.push(node); - }); - parser.on("text", (value) => { - if (stack.length) stack.at(-1).text += value; - }); - parser.on("cdata", (value) => { - if (stack.length) stack.at(-1).text += value; - }); - parser.on("closetag", () => { - const node = stack.pop(); - const parent = stack.at(-1); - if (node?.local === "loc" && node.uri === SITEMAP_NAMESPACE && parent) { - parent.directLocs.push(node.text.trim()); - } - }); - try { - parser.write(xml).close(); - } catch (error) { - failure ||= new GateError(`cannot parse local sitemap ${label}: ${error.message}`); - } - if (failure) throw failure; - if (!root || root.uri !== SITEMAP_NAMESPACE || !new Set(["urlset", "sitemapindex"]).has(root.local)) { - throw new GateError(`unsupported sitemap root or namespace: ${label}`); - } - - // Parse again to retain only direct root/entry/loc elements and reject malformed entries. - const entries = []; - const frames = []; - const direct = new SaxesParser({ xmlns: true }); - direct.on("error", (error) => { throw error; }); - direct.on("opentag", (tag) => frames.push({ local: tag.local, uri: tag.uri, text: "", locs: [] })); - direct.on("text", (value) => { if (frames.length) frames.at(-1).text += value; }); - direct.on("cdata", (value) => { if (frames.length) frames.at(-1).text += value; }); - direct.on("closetag", () => { - const node = frames.pop(); - const parent = frames.at(-1); - if (node.local === "loc" && node.uri === SITEMAP_NAMESPACE && frames.length === 2) { - parent.locs.push(node.text.trim()); - } - const expectedEntry = root.local === "urlset" ? "url" : "sitemap"; - if (node.local === expectedEntry && node.uri === SITEMAP_NAMESPACE && frames.length === 1) { - if (node.locs.length !== 1 || !node.locs[0]) { - throw new GateError(`sitemap ${expectedEntry} must contain exactly one direct loc element`); - } - entries.push(node.locs[0]); - } - }); - try { - direct.write(xml).close(); - } catch (error) { - if (error instanceof GateError) throw error; - throw new GateError(`cannot parse local sitemap ${label}: ${error.message}`); - } - if (entries.length === 0) throw new GateError(`sitemap has no usable locations: ${label}`); - return { kind: root.local, locations: entries }; -} - -function localSitemapPath(buildRoot, location, siteOrigin) { - const parsed = checkedUrl(location, siteOrigin, "sitemap location"); - const safe = safeUrlPath(parsed, "sitemap URL"); - const path = safeExistingPath(buildRoot, safe.parts, location); - if (path === null) throw new GateError(`local sitemap does not exist: ${location}`); - return path; -} - -export function sitemapUrls(buildDir, siteOrigin) { - const buildRoot = realpathSync(resolve(buildDir)); - const origin = typeof siteOrigin === "string" ? normalizeSiteOrigin(siteOrigin) : siteOrigin; - const rootSitemap = safeExistingPath(buildRoot, ["sitemap.xml"], "sitemap.xml"); - if (rootSitemap === null) throw new GateError("local root sitemap does not exist: sitemap.xml"); - const pending = [rootSitemap]; - const visited = new Set(); - const urls = []; - while (pending.length) { - const sitemap = realpathSync(pending.pop()); - if (!insideRoot(buildRoot, sitemap)) throw new GateError(`sitemap escapes build directory: ${sitemap}`); - if (visited.has(sitemap)) throw new GateError(`sitemap index cycle or duplicate: ${sitemap}`); - visited.add(sitemap); - let parsed; - try { - parsed = parseStrictSitemap(readFileSync(sitemap, "utf8"), sitemap); - } catch (error) { - if (error instanceof GateError) throw error; - throw new GateError(`cannot read local sitemap ${sitemap}: ${error.message}`); - } - if (parsed.kind === "sitemapindex") { - for (const value of parsed.locations) pending.push(localSitemapPath(buildRoot, value, origin)); - } else { - for (const value of parsed.locations) { - checkedUrl(value, origin, "page location"); - urls.push(value); - } - } - } - const duplicates = urls.filter((value, index) => urls.indexOf(value) !== index); - if (duplicates.length) throw new GateError(`duplicate sitemap URL: ${[...new Set(duplicates)].sort(compareText).slice(0, 5).join(", ")}`); - return urls.sort(compareText); -} - -function htmlFilesWithStem(buildRoot, directoryParts, stem, location) { - const directory = safeExistingDirectory(buildRoot, directoryParts, location); - if (directory === null) return []; - let children; - try { - children = readdirSync(directory, { withFileTypes: true }); - } catch (error) { - throw new GateError(`cannot enumerate built HTML directory ${directory}: ${error.message}`); - } - return children - .filter((entry) => entry.isFile() || entry.isSymbolicLink()) - .filter((entry) => basename(entry.name, extname(entry.name)) === stem && HTML_SUFFIXES.has(extname(entry.name).toLowerCase())) - .map((entry) => safeExistingPath(buildRoot, [...directoryParts, entry.name], location)) - .filter((path) => path !== null) - .sort(compareText); -} - -function pageFile(buildRoot, location, siteOrigin) { - const parsed = checkedUrl(location, siteOrigin, "page location"); - const route = parsed.pathname || "/"; - const safe = safeUrlPath(parsed, "page URL"); - const parts = safe.parts; - let candidates; - if (route.endsWith("/")) { - candidates = htmlFilesWithStem(buildRoot, parts, "index", location); - } else if (parts.length && HTML_SUFFIXES.has(extname(parts.at(-1)).toLowerCase())) { - candidates = [join(buildRoot, ...parts)]; - } else { - const nested = htmlFilesWithStem(buildRoot, parts, "index", location); - const flat = htmlFilesWithStem(buildRoot, parts.slice(0, -1), parts.at(-1) || "index", location); - candidates = [...nested, ...flat]; - } - const existing = []; - for (const candidate of candidates) { - const actual = safeExistingPath(buildRoot, relative(buildRoot, candidate).split(sep), location); - if (actual !== null && !existing.includes(actual)) existing.push(actual); - } - if (existing.length === 0) throw new MissingBuiltHtml(`missing built HTML for ${JSON.stringify(location)}`); - if (existing.length !== 1) throw new GateError(`ambiguous built HTML for ${JSON.stringify(location)}`); - return { route, policyPath: safe.decodedPath, file: existing[0] }; -} - -function exactBuiltArtifact(buildRoot, location, siteOrigin) { - const parsed = checkedUrl(location, siteOrigin, "page location"); - if (parsed.pathname.endsWith("/")) return null; - return safeExistingArtifact(buildRoot, safeUrlPath(parsed, "page URL").parts, location); -} - -function encodedRelativeRoute(relativePath) { - return relativePath.split(sep).map((part) => encodeURIComponent(part)).join("/"); -} - -function inferredRoute(buildRoot, publicFile) { - const relativePath = relative(buildRoot, publicFile); - const sourceSuffix = extname(relativePath); - const suffix = sourceSuffix.toLowerCase(); - const stem = basename(relativePath, sourceSuffix); - if (HTML_SUFFIXES.has(suffix) && stem === "index") { - const parent = dirname(relativePath); - return { - route: parent === "." ? "/" : `/${encodedRelativeRoute(parent)}/`, - policyPath: parent === "." ? "/" : `/${parent.split(sep).join("/")}/`, - }; - } - return { route: `/${encodedRelativeRoute(relativePath)}`, policyPath: `/${relativePath.split(sep).join("/")}` }; -} - -function allBuiltHtml(buildRoot) { - const output = []; - const seenFiles = new Set(); - const seenDirectories = new Set(); - function visit(publicDirectory) { - const actualDirectory = realpathSync(publicDirectory); - if (!insideRoot(buildRoot, actualDirectory)) throw new GateError(`built HTML symlink escapes build directory: ${publicDirectory}`); - if (seenDirectories.has(actualDirectory)) throw new GateError(`built directory symlink creates a cycle or duplicate: ${publicDirectory}`); - seenDirectories.add(actualDirectory); - const entries = readdirSync(publicDirectory, { withFileTypes: true }).sort((a, b) => compareText(a.name, b.name)); - for (const entry of entries) { - const publicPath = join(publicDirectory, entry.name); - const info = entry.isSymbolicLink() ? statSync(publicPath) : null; - if (entry.isDirectory() || info?.isDirectory()) { - visit(publicPath); - } else if ((entry.isFile() || info?.isFile()) && HTML_SUFFIXES.has(extname(entry.name).toLowerCase())) { - const actual = realpathSync(publicPath); - if (!insideRoot(buildRoot, actual)) throw new GateError(`built HTML symlink escapes build directory: ${publicPath}`); - if (seenFiles.has(actual)) throw new GateError(`more than one built path maps to the same HTML file: ${publicPath}`); - seenFiles.add(actual); - output.push({ file: actual, publicFile: publicPath }); - } - } - } - visit(buildRoot); - if (output.length === 0) throw new GateError(`build directory contains no HTML files: ${buildRoot}`); - return output; -} - -function attrsOf(node) { - const attrs = new Map(); - for (const attr of node.attrs || []) if (!attrs.has(attr.name.toLowerCase())) attrs.set(attr.name.toLowerCase(), attr.value); - return attrs; -} - -function normalizedWhitespace(value) { - return value.replace(/[\t\n\f\r ]+/gu, " ").trim(); -} - -function normalizedTitleWhitespace(value) { - return value.replace(/[\t\n\f\r ]+/gu, " ").replace(/^ /u, "").replace(/ $/u, ""); -} - -function textContent(node) { - if (node.nodeName === "#text") return node.value || ""; - return (node.childNodes || []).map(textContent).join(""); -} - -function htmlElements(root) { - const result = []; - function visit(node) { - if (node.tagName && node.namespaceURI === HTML_NAMESPACE) result.push(node); - for (const child of node.childNodes || []) visit(child); - } - visit(root); - return result; -} - -function cssValueTokens(value) { - const tokens = []; - csstree.tokenize(value, (type, start, end) => tokens.push({ type, text: value.slice(start, end) })); - return tokens; -} - -function decodedFunctionName(token) { - return csstree.ident.decode(token.text.slice(0, -1)).toLowerCase(); -} - -function matchingFunctionEnd(tokens, start) { - let depth = 1; - for (let index = start + 1; index < tokens.length; index += 1) { - if (tokens[index].type === csstree.tokenTypes.Function || tokens[index].type === csstree.tokenTypes.LeftParenthesis) depth += 1; - else if (tokens[index].type === csstree.tokenTypes.RightParenthesis) { - depth -= 1; - if (depth === 0) return index; - } - } - return -1; -} - -function significantCssTokens(tokens) { - return tokens.filter((token) => token.type !== csstree.tokenTypes.WhiteSpace && token.type !== csstree.tokenTypes.Comment); -} - -function topLevelComma(tokens, start, end) { - let depth = 0; - for (let index = start; index < end; index += 1) { - const type = tokens[index].type; - if (type === csstree.tokenTypes.Function || type === csstree.tokenTypes.LeftParenthesis) depth += 1; - else if (type === csstree.tokenTypes.RightParenthesis) depth -= 1; - else if (type === csstree.tokenTypes.Comma && depth === 0) return index; - } - return end; -} - -function validDynamicFunction(tokens, start, end, name) { - const comma = topLevelComma(tokens, start + 1, end); - const head = significantCssTokens(tokens.slice(start + 1, comma)); - if (name === "var") { - return head.length === 1 - && head[0].type === csstree.tokenTypes.Ident - && csstree.ident.decode(head[0].text).startsWith("--") - && csstree.ident.decode(head[0].text).length > 2; - } - if (name === "env") { - if (!head.length || head[0].type !== csstree.tokenTypes.Ident) return false; - return head.slice(1).every((token) => token.type === csstree.tokenTypes.Number && /^\+?[0-9]+$/u.test(token.text)); - } - if (!head.length || head[0].type !== csstree.tokenTypes.Ident) return false; - const type = head.slice(1); - if (type.length === 0) return true; - if (type.length === 1) { - return type[0].type === csstree.tokenTypes.Ident - || (type[0].type === csstree.tokenTypes.Delim && type[0].text === "%"); - } - return type[0].type === csstree.tokenTypes.Function - && decodedFunctionName(type[0]) === "type" - && matchingFunctionEnd(type, 0) === type.length - 1 - && significantCssTokens(type.slice(1, -1)).length > 0; -} - -function dynamicCssSyntax(value) { - const tokens = cssValueTokens(value); - let contains = false; - for (let index = 0; index < tokens.length; index += 1) { - const token = tokens[index]; - if (token.type !== csstree.tokenTypes.Function) continue; - const name = decodedFunctionName(token); - if (!new Set(["var", "env", "attr"]).has(name)) continue; - contains = true; - const end = matchingFunctionEnd(tokens, index); - if (end < 0 || !validDynamicFunction(tokens, index, end, name)) return { contains, valid: false }; - } - return { contains, valid: true }; -} - -function inlineDeclarations(style) { - if (!style) return { display: null, visibility: null, contentVisibility: null }; - let ast; - try { - ast = csstree.parse(style, { context: "declarationList", positions: false }); - } catch (error) { - throw new GateError(`cannot parse inline style: ${JSON.stringify(style)} (${error.message})`); - } - const winners = new Map(); - for (const declaration of ast.children.toArray()) { - if (declaration.type !== "Declaration") continue; - const property = csstree.ident.decode(declaration.property).toLowerCase(); - if (property !== "all" && !TRACKED_CSS_PROPERTIES.includes(property)) continue; - const decodedValue = csstree.clone(declaration.value); - csstree.walk(decodedValue, (node) => { - if (node.type === "Identifier" || node.type === "Function") node.name = csstree.ident.decode(node.name); - }); - let valid = false; - try { - valid = csstree.lexer.matchProperty(property, decodedValue).matched !== null; - } catch { - valid = false; - } - const dynamicSyntax = dynamicCssSyntax(csstree.generate(decodedValue)); - if (!dynamicSyntax.valid || (!valid && !dynamicSyntax.contains)) continue; - const dynamic = dynamicSyntax.contains; - const targets = property === "all" ? TRACKED_CSS_PROPERTIES : [property]; - for (const target of targets) { - const prior = winners.get(target); - if (!prior || declaration.important || !prior.important) { - winners.set(target, { decodedValue, important: Boolean(declaration.important), dynamic, sourceProperty: property }); - } - } - } - const output = { display: null, visibility: null, contentVisibility: null }; - for (const [property, winner] of winners) { - if (winner.dynamic) throw new GateError(`cannot resolve dynamic inline ${winner.sourceProperty} declaration affecting ${property}`); - const nodes = winner.decodedValue.children?.toArray?.() || []; - if (nodes.length !== 1 || nodes[0].type !== "Identifier") { - // Multi-keyword display values cannot hide an element. Visibility accepts one keyword only. - if (property !== "display") throw new GateError(`cannot resolve inline ${property} declaration: ${csstree.generate(winner.decodedValue)}`); - output.display = csstree.generate(winner.decodedValue).toLowerCase(); - continue; - } - const value = csstree.ident.decode(nodes[0].name).toLowerCase(); - if (value === "revert-layer") throw new GateError(`cannot resolve inline ${winner.sourceProperty} declaration using revert-layer`); - if (property === "content-visibility") output.contentVisibility = value; - else output[property] = value; - } - return output; -} - -function initialCssValue(property) { - if (property === "display") return "inline"; - return "visible"; -} - -function resolvedCssValue(property, authored, parent, userAgent) { - if (authored === null || authored === "revert") return userAgent; - if (authored === "initial") return initialCssValue(property); - if (authored === "inherit") return parent; - if (authored === "unset") return property === "visibility" ? parent : initialCssValue(property); - return authored; -} - -function userAgentDefaults(node, attrs, isHtml, parent) { - const defaults = { display: "inline", visibility: parent.visibility, contentVisibility: "visible" }; - if (!isHtml) return defaults; - if (attrs.has("hidden")) { - if ((attrs.get("hidden") || "").toLowerCase() === "until-found") defaults.contentVisibility = "hidden"; - else defaults.display = "none"; - } - if ((node.tagName === "dialog" && !attrs.has("open")) || attrs.has("popover") || UA_DISPLAY_NONE_TAGS.has(node.tagName)) { - defaults.display = "none"; - } - return defaults; -} - -function elementState(node, attrs, parent) { - const isHtml = node.namespaceURI === HTML_NAMESPACE; - const inline = inlineDeclarations(attrs.get("style") || ""); - const userAgent = userAgentDefaults(node, attrs, isHtml, parent); - const display = resolvedCssValue("display", inline.display, parent.display, userAgent.display); - const contentVisibility = resolvedCssValue( - "content-visibility", - inline.contentVisibility, - parent.contentVisibility, - userAgent.contentVisibility, - ); - const visibility = resolvedCssValue("visibility", inline.visibility, parent.visibility, userAgent.visibility); - const hard = parent.hard - || (isHtml && attrs.has("inert")) - || (attrs.get("aria-hidden") || "").trim().toLowerCase() === "true" - || display === "none" - || contentVisibility === "hidden"; - if (!new Set(["hidden", "collapse", "visible"]).has(visibility)) { - throw new GateError(`cannot resolve inline visibility declaration: ${visibility}`); - } - return { hard, display, visibility, contentVisibility }; -} - -function htmlNonNegativeInteger(value) { - if (value === undefined) return null; - const match = /^[\t\n\f\r ]*\+?([0-9]+)/u.exec(value); - if (!match) return null; - const parsed = Number(match[1]); - return Number.isSafeInteger(parsed) ? parsed : Number.MAX_SAFE_INTEGER; -} - -function isTrackingImage(attrs) { - const width = htmlNonNegativeInteger(attrs.get("width")); - const height = htmlNonNegativeInteger(attrs.get("height")); - return width !== null && height !== null && width <= 2 && height <= 2; -} - -export function robotsDisallowIndexing(values) { - const tokens = new Set(values.flatMap((value) => value.trim().toLowerCase().split(/[,\s]+/u).filter(Boolean))); - return tokens.has("noindex") || tokens.has("none"); -} - -function normalizedOpenDetails(document) { - const open = new Set(); - const named = new Set(); - function visit(node) { - if (node.tagName === "details" && node.namespaceURI === HTML_NAMESPACE) { - const attrs = attrsOf(node); - if (attrs.has("open")) { - const name = attrs.get("name") || ""; - if (!name || !named.has(name)) { - open.add(node); - if (name) named.add(name); - } - } - } - for (const child of node.childNodes || []) visit(child); - } - visit(document); - return open; -} - -function firstDirectSummary(details) { - return (details.childNodes || []).find((node) => node.tagName === "summary" && node.namespaceURI === HTML_NAMESPACE) || null; -} - -function originNeutralByteState(payload, siteOrigin) { - if (siteOrigin === null || siteOrigin === undefined) { - return { bytes_without_site_origin: payload.length, site_origin_occurrences: 0 }; - } - const originBytes = Buffer.from(siteOrigin, "utf8"); - let occurrences = 0; - let offset = 0; - while (offset <= payload.length - originBytes.length) { - const found = payload.indexOf(originBytes, offset); - if (found < 0) break; - occurrences += 1; - offset = found + originBytes.length; - } - return { - bytes_without_site_origin: payload.length - occurrences * originBytes.length, - site_origin_occurrences: occurrences, - }; -} - -export function parsePage(route, file, { inSitemap, policyPath = route, siteOrigin = null }) { - let payload; - try { - payload = readFileSync(file); - } catch (error) { - throw new GateError(`cannot read built HTML ${file}: ${error.message}`); - } - let document; - try { - document = parseHtml(payload.toString("utf8"), { scriptingEnabled: true }); - } catch (error) { - throw new GateError(`cannot parse built HTML ${file}: ${error.message}`); - } - const elements = htmlElements(document); - const titleNode = elements.find((node) => node.tagName === "title"); - const title = titleNode ? normalizedTitleWhitespace(textContent(titleNode)) : null; - let description = null; - const robots = []; - const head = elements.find((node) => node.tagName === "head"); - for (const node of head?.childNodes || []) { - if (node.namespaceURI !== HTML_NAMESPACE) continue; - if (node.tagName !== "meta") continue; - const attrs = attrsOf(node); - const name = (attrs.get("name") || "").toLowerCase(); - const content = (attrs.get("content") || "").trim(); - if (name === "description" && description === null) description = content; - if (name === "robots" || name === "googlebot") robots.push(content); - } - - const h1Records = []; - let imagesWithoutAlt = 0; - const activeH1s = []; - const openDetails = normalizedOpenDetails(document); - function visit(node, parentState, inertContext) { - if (node.nodeName === "#text") { - if (!inertContext && !parentState.hard && parentState.visibility === "visible") { - for (const record of activeH1s) record.parts.push(node.value || ""); - } - return; - } - if (!node.tagName) { - for (const child of node.childNodes || []) visit(child, parentState, inertContext); - return; - } - const attrs = attrsOf(node); - const isHtml = node.namespaceURI === HTML_NAMESPACE; - const state = elementState(node, attrs, parentState); - const nextInert = inertContext || (isHtml && SEMANTICALLY_INERT.has(node.tagName)); - const isH1 = isHtml && node.tagName === "h1" && !nextInert && !state.hard; - let record = null; - if (isH1) { - record = { parts: [], exposed: state.visibility === "visible" }; - h1Records.push(record); - activeH1s.push(record); - } - if (!nextInert && !state.hard && state.visibility === "visible") { - for (const active of activeH1s) active.exposed = true; - if (isHtml && node.tagName === "img" && !attrs.has("alt") && !isTrackingImage(attrs)) { - imagesWithoutAlt += 1; - } - } - const closedDetails = isHtml && node.tagName === "details" && !openDetails.has(node); - const summary = closedDetails ? firstDirectSummary(node) : null; - for (const child of node.childNodes || []) { - const childState = closedDetails && child !== summary ? { ...state, hard: true } : state; - visit(child, childState, nextInert); - } - if (record !== null) { - activeH1s.pop(); - } - } - visit(document, { hard: false, display: "block", visibility: "visible", contentVisibility: "visible" }, false); - const h1s = h1Records.filter((record) => record.exposed).map((record) => normalizedWhitespace(record.parts.join(""))); - const byteState = originNeutralByteState(payload, siteOrigin); - - return { - route, - policy_path: policyPath, - file, - in_sitemap: inSitemap, - html_bytes: payload.length, - html_bytes_without_site_origin: byteState.bytes_without_site_origin, - site_origin_occurrences: byteState.site_origin_occurrences, - title, - description, - noindex: robotsDisallowIndexing(robots), - h1s, - images_without_alt: imagesWithoutAlt, - }; -} - -export function titleIsOnlyTheSiteSuffix(title) { - const index = title.lastIndexOf("|"); - return index >= 0 && normalizedTitleWhitespace(title.slice(0, index)) === ""; -} - -function stableObject(value) { - if (Array.isArray(value)) return value.map(stableObject); - if (!isPlainObject(value)) return value; - return Object.fromEntries(Object.keys(value).sort(compareText).map((key) => [key, stableObject(value[key])])); -} - -function stateSignature(state) { - return JSON.stringify(stableObject(state)); -} - -function finding(rule, path, message, signature = null) { - if (!(rule in RULES)) throw new GateError(`unclassified gate rule: ${rule}`); - let identity = `${rule}|${path}`; - if (signature !== null) identity += `|${sha256Bytes(Buffer.from(signature, "utf8"))}`; - return { identity, rule, classification: RULES[rule].classification, path, message }; -} - -function sortedFindings(findings) { - return findings.sort((a, b) => compareText(a.rule, b.rule) || compareText(a.path, b.path) || compareText(a.identity, b.identity)); -} - -function isIntegrationPolicyPath(policyPath, integrationRoutePrefixes) { - return integrationRoutePrefixes.some((prefix) => - policyPath === prefix || policyPath.startsWith(`${prefix}/`), - ); -} - -export function scanBuiltHtml(buildDir, siteOrigin, { integrationRoutePrefixes = [] } = {}) { - const rootInput = resolve(buildDir); - if (!existsSync(rootInput) || !statSync(rootInput).isDirectory()) throw new GateError(`build directory does not exist: ${buildDir}`); - const root = realpathSync(rootInput); - const origin = typeof siteOrigin === "string" ? normalizeSiteOrigin(siteOrigin) : siteOrigin; - const integrationPrefixes = normalizeIntegrationRoutePrefixes(integrationRoutePrefixes); - const sitemapRoutes = new Map(); - const sitemapArtifacts = new Map(); - for (const location of sitemapUrls(root, origin)) { - const artifact = exactBuiltArtifact(root, location, origin); - if (artifact !== null) { - if (sitemapArtifacts.has(artifact.file)) { - throw new GateError(`more than one sitemap URL maps to ${artifact.file}: ${JSON.stringify(sitemapArtifacts.get(artifact.file))} and ${JSON.stringify(location)}`); - } - sitemapArtifacts.set(artifact.file, location); - if (!HTML_SUFFIXES.has(extname(artifact.publicPath).toLowerCase())) continue; - } - const mapped = pageFile(root, location, origin); - if (sitemapRoutes.has(mapped.file)) { - throw new GateError(`more than one sitemap URL maps to ${mapped.file}: ${JSON.stringify(sitemapRoutes.get(mapped.file))} and ${JSON.stringify(mapped.route)}`); - } - sitemapRoutes.set(mapped.file, mapped); - } - - const pages = []; - const seenRoutes = new Set(); - for (const item of allBuiltHtml(root)) { - const mapped = sitemapRoutes.get(item.file) || inferredRoute(root, item.publicFile); - if (seenRoutes.has(mapped.route)) throw new GateError(`more than one built HTML file maps to route ${JSON.stringify(mapped.route)}`); - seenRoutes.add(mapped.route); - pages.push(parsePage(mapped.route, item.file, { - inSitemap: sitemapRoutes.has(item.file), - policyPath: mapped.policyPath, - siteOrigin: origin, - })); - } - - const findings = []; - for (const page of pages) { - if (!page.title || titleIsOnlyTheSiteSuffix(page.title)) findings.push(finding("missing-title", page.route, "title is empty")); - if (page.images_without_alt) { - findings.push(finding("images-missing-alt", page.route, `${page.images_without_alt} statically exposed HTML image(s) lack an alt attribute`, stateSignature({ count: page.images_without_alt }))); - } - if (page.html_bytes > HEAVY_BYTES && !isIntegrationPolicyPath(page.policy_path, integrationPrefixes)) { - findings.push(finding("heavy-page", page.route, `${page.html_bytes} bytes exceeds the ${HEAVY_BYTES}-byte estate guardrail`, stateSignature({ - bytes_without_site_origin: page.html_bytes_without_site_origin, - site_origin_occurrences: page.site_origin_occurrences, - }))); - } - const guide = page.policy_path === "/guides" || page.policy_path === "/guides/" || page.policy_path.startsWith("/guides/"); - if ((!page.in_sitemap || page.noindex) && !guide) continue; - if (page.in_sitemap && !page.noindex && !page.description) findings.push(finding("missing-description", page.route, "meta description is empty")); - if (!page.h1s.length || !page.h1s[0]) { - const signature = stateSignature({ count: page.h1s.length, first: page.h1s.length ? "empty" : "absent" }); - findings.push(finding("missing-h1", page.route, "first h1 is absent or empty", signature)); - } - else if (page.h1s.length > 1) findings.push(finding("multiple-h1", page.route, `found ${page.h1s.length} h1 elements`, stateSignature({ count: page.h1s.length }))); - } - - for (const [field, rule] of [["title", "duplicate-title"], ["description", "duplicate-description"]]) { - const groups = new Map(); - for (const page of pages) { - if (!page.in_sitemap || page.noindex) continue; - const value = (page[field] || "").trim(); - if (!value) continue; - if (!groups.has(value)) groups.set(value, []); - groups.get(value).push(page); - } - for (const [value, group] of groups) { - if (group.length < 2) continue; - const paths = group.map((page) => page.route).sort(compareText); - const signature = stateSignature({ count: group.length, paths, value }); - for (const page of group) findings.push(finding(rule, page.route, `value is shared by ${group.length} pages`, signature)); - } - } - return { - pages: pages.sort((a, b) => compareText(a.route, b.route)), - findings: sortedFindings(findings), - integrationRoutePrefixes: integrationPrefixes, - }; -} - -export function loadBaseline(path) { - if (path === null || path === undefined) return new Map(); - const value = loadJson(path, "gate baseline"); - exactKeys(value, ["schema", "contract", "ruleset_version", "allowed"], "gate baseline"); - if (value.schema !== BASELINE_SCHEMA || value.contract !== CONTRACT_SCHEMA) throw new GateError("gate baseline schema or contract is not supported"); - if (value.ruleset_version !== RULESET_VERSION) throw new GateError(`baseline ruleset ${JSON.stringify(value.ruleset_version)} does not match ${RULESET_VERSION}`); - if (!Array.isArray(value.allowed)) throw new GateError("gate baseline allowed field must be a list"); - const allowed = new Map(); - for (const entry of value.allowed) { - exactKeys(entry, ["identity", "reason"], "baseline entry"); - if (typeof entry.identity !== "string" || !entry.identity || typeof entry.reason !== "string" || !entry.reason.trim()) { - throw new GateError("baseline identity and reason must be non-empty strings"); - } - if (!(entry.identity.split("|", 1)[0] in RULES)) throw new GateError(`baseline identity names an unknown rule: ${entry.identity}`); - if (allowed.has(entry.identity)) throw new GateError(`duplicate baseline identity: ${entry.identity}`); - allowed.set(entry.identity, entry.reason.trim()); - } - return allowed; -} - -export function runGate(buildDir, { siteOrigin, baselinePath = null, integrationRoutePrefixes = [] }) { - requireNode22(); - const origin = normalizeSiteOrigin(siteOrigin); - const { pages, findings, integrationRoutePrefixes: normalizedIntegrationPrefixes } = - scanBuiltHtml(buildDir, origin, { integrationRoutePrefixes }); - const allowed = loadBaseline(baselinePath); - const identities = new Set(findings.map((item) => item.identity)); - const regressions = findings.filter((item) => !allowed.has(item.identity)); - const accepted = findings.filter((item) => allowed.has(item.identity)).map((item) => ({ ...item, baseline_reason: allowed.get(item.identity) })); - const stale = [...allowed.entries()].filter(([identity]) => !identities.has(identity)).sort(([a], [b]) => compareText(a, b)).map(([identity, reason]) => ({ identity, reason })); - const counts = {}; - for (const item of findings) counts[item.rule] = (counts[item.rule] || 0) + 1; - return { - schema: REPORT_SCHEMA, - contract: CONTRACT_SCHEMA, - ruleset_version: RULESET_VERSION, - site_origin: origin, - integration_route_prefixes: normalizedIntegrationPrefixes, - checked_pages: pages.length, - checked_index_pages: pages.filter((page) => page.in_sitemap && !page.noindex).length, - counts: Object.fromEntries(Object.entries(counts).sort(([a], [b]) => compareText(a, b))), - findings, - accepted_findings: accepted, - regressions, - stale_baseline: stale, - }; -} - -function printText(report) { - console.log(`site build gate v${report.ruleset_version}: ${report.checked_pages} pages, ${report.regressions.length} regression(s), ${report.accepted_findings.length} accepted finding(s)`); - for (const item of report.regressions) console.log(`ERROR ${item.rule} [${item.classification}] ${item.path}: ${item.message}`); - for (const item of report.accepted_findings) console.log(`ACCEPTED ${item.rule} ${item.path}: ${item.baseline_reason}`); - for (const item of report.stale_baseline) console.log(`RESOLVED ${item.identity}: baseline entry can be removed`); -} - -function parseArgs(argv) { - const args = { buildDir: null, siteOrigin: null, baseline: null, manifest: join(dirname(fileURLToPath(import.meta.url)), "manifest.json"), format: "text", integrationRoutePrefixes: [] }; - for (let index = 0; index < argv.length; index += 2) { - const name = argv[index]; - const value = argv[index + 1]; - if (!value || !new Set(["--build-dir", "--site-origin", "--baseline", "--manifest", "--format", "--integration-route-prefix"]).has(name)) throw new GateError(`invalid or incomplete argument: ${name || ""}`); - if (name === "--build-dir") args.buildDir = value; - else if (name === "--site-origin") args.siteOrigin = value; - else if (name === "--baseline") args.baseline = value; - else if (name === "--manifest") args.manifest = value; - else if (name === "--format") args.format = value; - else args.integrationRoutePrefixes.push(value); - } - if (!args.buildDir || !args.siteOrigin) throw new GateError("--build-dir and --site-origin are required"); - if (!new Set(["text", "json"]).has(args.format)) throw new GateError("--format must be text or json"); - return args; -} - -export function main(argv = process.argv.slice(2)) { - try { - const args = parseArgs(argv); - const artifact = fileURLToPath(import.meta.url); - verifyContract(artifact, resolve(args.manifest)); - const report = runGate(args.buildDir, { - siteOrigin: args.siteOrigin, - baselinePath: args.baseline, - integrationRoutePrefixes: args.integrationRoutePrefixes, - }); - if (args.format === "json") console.log(JSON.stringify(report, null, 2)); - else printText(report); - return report.regressions.length ? 1 : 0; - } catch (error) { - console.error(`site build gate contract error: ${error.message}`); - return 2; - } -} - -function isDirectInvocation() { - if (!process.argv[1]) return false; - try { - return realpathSync(fileURLToPath(import.meta.url)) === realpathSync(resolve(process.argv[1])); - } catch { - return false; - } -} - -if (isDirectInvocation()) { - process.exitCode = main(); -} diff --git a/src/seo/deploymentIntegrations.test.js b/src/seo/deploymentIntegrations.test.js index e85d97803..2be80abb6 100644 --- a/src/seo/deploymentIntegrations.test.js +++ b/src/seo/deploymentIntegrations.test.js @@ -11,16 +11,15 @@ const packageJson = require('../../package.json'); const docusaurus = require('../../docusaurus.config.js'); describe('shared deployment integrations', () => { - it('runs the exact C8 bundle after generated output activates strict gates', () => { + it('runs the installed C8 gate after generated output activates strict gates', () => { const command = packageJson.scripts['build:netlify']; - const install = 'npm ci --prefix scripts/site-build-gate --ignore-scripts --no-audit'; const build = 'docusaurus build'; const gateRunner = 'node scripts/run-post-build-gates.mjs'; const scan = postBuildGateCommands({generatedOutputReady: true}).find( - ([script]) => script === 'scripts/site-build-gate/site_build_gate.mjs', + ([script]) => script === 'node_modules/@netdata/site-build-gate/site_build_gate.mjs', ); expect(scan).toEqual([ - 'scripts/site-build-gate/site_build_gate.mjs', + 'node_modules/@netdata/site-build-gate/site_build_gate.mjs', '--build-dir', 'build', '--site-origin', @@ -32,20 +31,8 @@ describe('shared deployment integrations', () => { '--format', 'json', ]); - expect(command).toContain(install); expect(command).toContain(gateRunner); - expect(command.indexOf(install)).toBeLessThan(command.indexOf(build)); expect(command.indexOf(build)).toBeLessThan(command.indexOf(gateRunner)); - - const manifest = JSON.parse( - fs.readFileSync(path.join(root, 'scripts/site-build-gate/manifest.json'), 'utf8'), - ); - expect(manifest).toMatchObject({ - schema: 'netdata-site-build-gate-vendor-v2', - contract: 'netdata-site-build-gate-v1', - ruleset_version: 10, - node_major: 22, - }); }); it('pins the exact Yarn runtime without the deprecated Netlify selector', () => { diff --git a/src/seo/postBuildGates.test.js b/src/seo/postBuildGates.test.js index 135f3db6d..bde14a4b3 100644 --- a/src/seo/postBuildGates.test.js +++ b/src/seo/postBuildGates.test.js @@ -32,7 +32,7 @@ describe('post-build gate rollout', () => { ['scripts/verify-rendered-indexability.js'], ['scripts/verify-cloudflare-rum.js', 'build'], [ - 'scripts/site-build-gate/site_build_gate.mjs', + 'node_modules/@netdata/site-build-gate/site_build_gate.mjs', '--build-dir', 'build', '--site-origin', diff --git a/tests/dependency_authority.test.js b/tests/dependency_authority.test.js index 6358d5207..9df80f2a6 100644 --- a/tests/dependency_authority.test.js +++ b/tests/dependency_authority.test.js @@ -126,19 +126,17 @@ function assertCompatibleVersionGroup(updateBlock, groupName) { assert.match(group, /^ {8}update-types:\n {10}- minor\n {10}- patch$/m, `${groupName} must leave majors separate`); } -test('uses one root Yarn authority and isolates owner-controlled npm vendors', () => { +test('uses one root Yarn authority', () => { const packageJson = require('../package.json'); assert.equal(packageJson.packageManager, 'yarn@1.22.22'); assert.deepEqual(packageJson.resolutions, expectedResolutions); assert.equal(fs.existsSync(path.join(root, 'yarn.lock')), true); assert.equal(fs.existsSync(path.join(root, 'package-lock.json')), false); - assert.equal(fs.existsSync(path.join(root, 'scripts/site-build-gate/package-lock.json')), true); const dependabot = fs.readFileSync(path.join(root, '.github/dependabot.yml'), 'utf8'); assertScopedUpdateBlock(dependabot, 'npm', '/'); assertScopedUpdateBlock(dependabot, 'github-actions', '/'); assertScopedUpdateBlock(dependabot, 'pip', '"/.learn_environment"'); - assert.doesNotMatch(dependabot, /scripts\/site-build-gate/); assert.throws( () => assertScopedUpdateBlock(dependabot.replace('directory: /', 'directory: /packages'), 'npm', '/'), diff --git a/tests/site_build_gate.test.js b/tests/site_build_gate.test.js index 2a0624f85..93fc63838 100644 --- a/tests/site_build_gate.test.js +++ b/tests/site_build_gate.test.js @@ -1,5 +1,4 @@ const assert = require('node:assert/strict'); -const crypto = require('node:crypto'); const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); @@ -7,67 +6,37 @@ const {spawnSync} = require('node:child_process'); const test = require('node:test'); const root = path.resolve(__dirname, '..'); -const vendor = path.join(root, 'scripts', 'site-build-gate'); -const gate = path.join(vendor, 'site_build_gate.mjs'); - -const expectedManifest = { - schema: 'netdata-site-build-gate-vendor-v2', - contract: 'netdata-site-build-gate-v1', - ruleset_version: 10, - node_major: 22, - artifacts: [ - { - path: 'package-lock.json', - sha256: 'a3cac54442c679c4888131038520a50e37ea092c1c7ea378c88b546e483e0c35', - }, - { - path: 'package.json', - sha256: '5cd23bae2605a5fcd2bed67b547ae7a4be0f9102dde6265eb4a770c59dce9fe4', - }, - { - path: 'site_build_gate.mjs', - sha256: '171cc249076efa1e9e1edb5cecdbd83b5487f9ea6c7e5b94caa73a1be2f44d4f', - }, - ], -}; - -function sha256(file) { - return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'); -} +const installed = path.join(root, 'node_modules', '@netdata', 'site-build-gate'); +const gate = path.join(installed, 'site_build_gate.mjs'); function writeFixture(directory, sitemap, html) { fs.writeFileSync(path.join(directory, 'sitemap.xml'), sitemap); fs.writeFileSync(path.join(directory, 'index.html'), html); } -test('the site vendors the exact accepted ruleset-v10 bundle', () => { - const manifest = JSON.parse(fs.readFileSync(path.join(vendor, 'manifest.json'), 'utf8')); - assert.deepEqual(manifest, expectedManifest); - for (const artifact of manifest.artifacts) { - assert.equal(sha256(path.join(vendor, artifact.path)), artifact.sha256); - } +test('the installed gate major version is the baseline ruleset', () => { + const {version} = JSON.parse(fs.readFileSync(path.join(installed, 'package.json'), 'utf8')); + const baseline = JSON.parse(fs.readFileSync(path.join(root, 'config', 'site-build-gate-baseline.json'), 'utf8')); + assert.equal(Number.parseInt(version, 10), baseline.ruleset_version); }); -test('every Netlify context stages the clean-installed gate after Docusaurus', async () => { +test('every Netlify context runs the installed gate after Docusaurus', async () => { const packageJson = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8')); const command = packageJson.scripts['build:netlify']; - const install = 'npm ci --prefix scripts/site-build-gate --ignore-scripts --no-audit'; const build = 'docusaurus build'; const gateRunner = 'node scripts/run-post-build-gates.mjs'; const {postBuildGateCommands} = await import('../scripts/run-post-build-gates.mjs'); const gateCommand = postBuildGateCommands({generatedOutputReady: true}).find( - ([script]) => script === 'scripts/site-build-gate/site_build_gate.mjs', + ([script]) => script === 'node_modules/@netdata/site-build-gate/site_build_gate.mjs', ); assert.deepEqual(gateCommand, [ - 'scripts/site-build-gate/site_build_gate.mjs', + 'node_modules/@netdata/site-build-gate/site_build_gate.mjs', '--build-dir', 'build', '--site-origin', 'https://learn.netdata.cloud', '--baseline', 'config/site-build-gate-baseline.json', '--integration-route-prefix', '/docs/collecting-metrics/collectors', '--format', 'json', ]); - assert.match(command, new RegExp(install)); - assert.ok(command.indexOf(install) < command.indexOf(build)); assert.ok(command.indexOf(build) < command.indexOf(gateRunner)); for (const filename of ['static.toml', 'netlify.toml']) { @@ -78,7 +47,7 @@ test('every Netlify context stages the clean-installed gate after Docusaurus', a } }); -test('the vendored gate fails closed on a rendered defect without changing output', (t) => { +test('the installed gate fails closed on a rendered defect without changing output', (t) => { const build = fs.mkdtempSync(path.join(os.tmpdir(), 'learn-site-gate-')); t.after(() => fs.rmSync(build, {recursive: true, force: true})); const sitemap = '' + @@ -103,7 +72,7 @@ test('the vendored gate fails closed on a rendered defect without changing outpu } }); -test('the vendored gate reports malformed build input as a contract error', (t) => { +test('the installed gate reports malformed build input as a contract error', (t) => { const build = fs.mkdtempSync(path.join(os.tmpdir(), 'learn-site-gate-')); t.after(() => fs.rmSync(build, {recursive: true, force: true})); writeFixture( diff --git a/yarn.lock b/yarn.lock index ecf28dba0..c9b254c99 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3176,6 +3176,15 @@ dependencies: "@tybys/wasm-util" "^0.10.3" +"@netdata/site-build-gate@^10.1.0": + version "10.1.0" + resolved "https://registry.npmjs.org/@netdata/site-build-gate/-/site-build-gate-10.1.0.tgz#08c5aa479c00652762a86670444c12a72945b372" + integrity sha512-MhzcxUyIHXNrP/CPWcRNrYanGMEvM4d77AFQJ6Ax3NyoS8Wh+5UOrtLTRGqGIfbLigqoNJc6/SW4ugAmHCGZJQ== + dependencies: + css-tree "^3.2.1" + parse5 "^8.0.1" + saxes "^6.0.0" + "@netlify/blobs@11.1.4": version "11.1.4" resolved "https://registry.npmjs.org/@netlify/blobs/-/blobs-11.1.4.tgz#74e4cd58b8f6a92c4b7254d644065d8610e8783d"