AR-20261104-2: canonical Link header + frontmatter on docs .md twins; drop /404/ from llms files

.github/workflows/ci.yml

@@ -141,6 +141,9 @@ jobs:
       - name: Test agent discovery catalog
         run: npm run test:agent-discovery
 
+      - name: Test Markdown twin canonical headers
+        run: npm run test:markdown-canonical
+
       - name: Build site
         run: npm run build
 
@@ -153,6 +156,12 @@ jobs:
       - name: Validate homepage JSON-LD
         run: npm run test:homepage-json-ld
 
+      - name: Validate llms files list only sitemap pages
+        run: npm run test:llms-txt
+
+      - name: Validate Markdown twin frontmatter
+        run: npm run test:markdown-twins
+
       - name: Validate page title order
         run: npm run test:page-title-order
 

astro.config.mjs

@@ -210,7 +210,9 @@ export default defineConfig({
 				// due to their size. The upstream plugin only applies `exclude` to
 				// llms-small.txt; our patch (patches/starlight-llms-txt+0.8.1.patch)
 				// extends it to llms-full.txt and custom sets as well.
-				exclude: ['support-and-community/community/open-source-licenses'],
+				// '404' is the not-found page; it is not documentation, and listing
+				// it sent agents to https://docs.warp.dev/404/.
+				exclude: ['support-and-community/community/open-source-licenses', '404'],
 					// This string is the first thing an AI engine reads about Warp, so it
 					// carries the product lineup. Kept in sync with the rename and with
 					// launches: it said "the Oz platform" until the 8/18 rename, which

package.json

@@ -7,7 +7,7 @@
     "dev": "astro dev",
     "predev": "node scripts/generate-openapi-json.mjs",
     "build": "astro build",
-    "postbuild": "node scripts/build-tracking-middleware.mjs",
+    "postbuild": "node scripts/build-tracking-middleware.mjs && node scripts/build-markdown-canonical-routes.mjs",
     "prebuild": "node scripts/generate-openapi-json.mjs",
     "preview": "astro preview",
     "astro": "astro",
@@ -22,10 +22,13 @@
     "test:docs-markdown": "node --test src/integrations/docs-markdown-integration.test.js",
     "test:tracking-redirect": "node --test scripts/tracking-redirect.test.mjs",
     "test:agent-discovery": "node --test scripts/agent-discovery.test.mjs",
+    "test:markdown-canonical": "node --test scripts/markdown-canonical-headers.test.mjs",
     "test:tracking-routes": "node scripts/validate-tracking-routes.mjs",
     "test:kapa-webmcp": "node --test src/lib/kapa-webmcp.test.mjs",
     "test:kapa-support-escalation": "node --test src/lib/kapa-support-escalation.test.mjs",
     "test:homepage-json-ld": "node scripts/validate-homepage-json-ld.mjs",
+    "test:llms-txt": "node scripts/validate-llms-txt.mjs",
+    "test:markdown-twins": "node scripts/validate-markdown-twins.mjs",
     "test:page-title-order": "node scripts/validate-page-title-order.mjs",
     "postinstall": "patch-package"
   },

scripts/build-markdown-canonical-routes.mjs

@@ -0,0 +1,63 @@
+// Adds canonical `Link` headers for the generated Markdown twins to the Vercel
+// build output. Runs from `postbuild`, after `astro build` writes
+// .vercel/output/config.json and the docs-markdown integration writes the twins.
+//
+// A twin is `<path>.md` (`index.md` for the homepage) whose HTML page,
+// `<path>/index.html`, advertises it with
+// `<link rel="alternate" type="text/markdown" href="/<path>.md">`. Only those
+// paths get the header, so 404 responses for other `.md` requests do not claim
+// a canonical page. Routes go before the filesystem phase, where vercel.json
+// headers also run, and use `continue` so the static file is still served.
+// See src/lib/markdown-canonical-routes.js.
+import { readdir, readFile, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { buildMarkdownCanonicalRoutes } from '../src/lib/markdown-canonical-routes.js';
+
+const output = fileURLToPath(new URL('../.vercel/output/', import.meta.url));
+const staticDir = path.join(output, 'static');
+const configPath = path.join(output, 'config.json');
+
+/** @param {string} dir */
+async function* markdownFiles(dir) {
+	for (const entry of await readdir(dir, { withFileTypes: true })) {
+		const entryPath = path.join(dir, entry.name);
+		if (entry.isDirectory()) yield* markdownFiles(entryPath);
+		else if (entry.isFile() && entry.name.endsWith('.md')) yield entryPath;
+	}
+}
+
+const twinPaths = [];
+for await (const file of markdownFiles(staticDir)) {
+	const twinPath = path.relative(staticDir, file).split(path.sep).join('/').slice(0, -'.md'.length);
+	const htmlFile = twinPath === 'index'
+		? path.join(staticDir, 'index.html')
+		: path.join(staticDir, twinPath, 'index.html');
+	let html;
+	try {
+		html = await readFile(htmlFile, 'utf8');
+	} catch {
+		continue;
+	}
+	if (html.includes(`<link rel="alternate" type="text/markdown" href="/${twinPath}.md"`)) {
+		twinPaths.push(twinPath);
+	}
+}
+if (twinPaths.length === 0) {
+	console.error('build-markdown-canonical-routes: no Markdown twins found in .vercel/output/static');
+	process.exit(1);
+}
+
+const config = JSON.parse(await readFile(configPath, 'utf8'));
+// Drop routes from an earlier run so rerunning the script does not duplicate them.
+const routes = config.routes.filter(
+	(route) => !(route.headers?.Link ?? '').endsWith('; rel="canonical"'),
+);
+const filesystemIndex = routes.findIndex((route) => route.handle === 'filesystem');
+const canonicalRoutes = buildMarkdownCanonicalRoutes(twinPaths);
+routes.splice(filesystemIndex === -1 ? routes.length : filesystemIndex, 0, ...canonicalRoutes);
+config.routes = routes;
+await writeFile(configPath, JSON.stringify(config, null, 2));
+console.log(
+	`build-markdown-canonical-routes: ${twinPaths.length} twins, ${canonicalRoutes.length} header routes`,
+);

scripts/markdown-canonical-headers.test.mjs

@@ -0,0 +1,82 @@
+import assert from 'node:assert/strict';
+import { readFile } from 'node:fs/promises';
+import { test } from 'node:test';
+import {
+	buildMarkdownCanonicalRoutes,
+	canonicalLinkValue,
+	linkHeadersFor,
+} from '../src/lib/markdown-canonical-routes.js';
+import { DOCS_ORIGIN } from '../src/lib/site.js';
+
+const twins = [
+	'index',
+	'agents',
+	'factories',
+	'terminal/blocks/block-basics',
+	'changelog/2026',
+	'factories/api-and-sdk/troubleshooting/errors/conflict',
+];
+const routes = buildMarkdownCanonicalRoutes(twins);
+
+test('every generated twin points to its HTML page', () => {
+	const cases = {
+		'/index.md': `${DOCS_ORIGIN}/`,
+		'/agents.md': `${DOCS_ORIGIN}/agents/`,
+		'/factories.md': `${DOCS_ORIGIN}/factories/`,
+		'/terminal/blocks/block-basics.md': `${DOCS_ORIGIN}/terminal/blocks/block-basics/`,
+		'/changelog/2026.md': `${DOCS_ORIGIN}/changelog/2026/`,
+		'/factories/api-and-sdk/troubleshooting/errors/conflict.md':
+			`${DOCS_ORIGIN}/factories/api-and-sdk/troubleshooting/errors/conflict/`,
+	};
+	for (const [pathname, htmlUrl] of Object.entries(cases)) {
+		assert.deepEqual(linkHeadersFor(routes, pathname), [canonicalLinkValue(htmlUrl)], pathname);
+	}
+});
+
+test('paths that are not generated twins get no canonical Link header', () => {
+	for (const pathname of [
+		// HTML pages and other generated files
+		'/', '/factories/', '/factories', '/llms.txt', '/openapi.yaml', '/sitemap-0.xml',
+		// reserved and missing paths, which return 404
+		'/assets/not-a-twin.md', '/_astro/not-a-twin.md', '/.well-known/not-a-twin.md',
+		'/robots.md', '/not-a-twin.md', '/factories/missing.md', '/terminal/index.md',
+		// near misses
+		'/factories.md/', '/factories.mdx', '/factories.md.html', '/xfactories.md', '/factories-x.md',
+		'/terminal/blocks.md', '/index.md/extra',
+		// case variants: static files are case-sensitive, so these 404
+		'/AGENTS.md', '/Factories.md', '/INDEX.md', '/factories.MD',
+	]) {
+		assert.deepEqual(linkHeadersFor(routes, pathname), [], pathname);
+	}
+});
+
+test('large twin sets are split into bounded routes that still match exactly', () => {
+	const many = Array.from({ length: 600 }, (_, index) => `section-${index}/page-with-a-long-name-${index}`);
+	const manyRoutes = buildMarkdownCanonicalRoutes(many);
+	assert.ok(manyRoutes.length > 1);
+	for (const route of manyRoutes) {
+		assert.ok(route.src.length < 4000, `src is ${route.src.length} characters`);
+		assert.equal(route.caseSensitive, true);
+		assert.equal(route.continue, true);
+	}
+	for (const twin of [many[0], many[299], many[599]]) {
+		assert.deepEqual(linkHeadersFor(manyRoutes, `/${twin}.md`), [canonicalLinkValue(`${DOCS_ORIGIN}/${twin}/`)]);
+	}
+	assert.deepEqual(linkHeadersFor(manyRoutes, '/section-0.md'), []);
+});
+
+test('regex characters in a path are matched literally', () => {
+	const special = buildMarkdownCanonicalRoutes(['guides/c++', 'a.b']);
+	assert.deepEqual(linkHeadersFor(special, '/guides/c++.md'), [canonicalLinkValue(`${DOCS_ORIGIN}/guides/c++/`)]);
+	assert.deepEqual(linkHeadersFor(special, '/guides/cc.md'), []);
+	assert.deepEqual(linkHeadersFor(special, '/aXb.md'), []);
+});
+
+test('vercel.json adds no canonical Link header (generated routes own it)', async () => {
+	const vercelConfig = JSON.parse(await readFile(new URL('../vercel.json', import.meta.url), 'utf8'));
+	for (const entry of vercelConfig.headers ?? []) {
+		for (const header of entry.headers) {
+			assert.notEqual(header.key.toLowerCase(), 'link', entry.source);
+		}
+	}
+});

scripts/validate-llms-txt.mjs

@@ -0,0 +1,53 @@
+// Checks the generated llms files after `npm run build`.
+//
+// Every page that llms-full.txt, llms-small.txt, or a /_llms-txt/*.txt set
+// includes carries a "Canonical page: [path](url)" line (see
+// patches/starlight-llms-txt+0.8.1.patch). Each of those URLs must be a page
+// listed in the sitemap, so the files never send agents to the 404 page or to a
+// page that search engines are told does not exist.
+//
+// Usage: node scripts/validate-llms-txt.mjs [outputDir]
+// outputDir defaults to .vercel/output/static.
+import { readdir, readFile } from 'node:fs/promises';
+import path from 'node:path';
+
+const outputDir = path.resolve(process.argv[2] ?? '.vercel/output/static');
+
+const sitemapFiles = (await readdir(outputDir)).filter((name) => /^sitemap-\d+\.xml$/.test(name));
+if (sitemapFiles.length === 0) {
+	console.error(`No sitemap-N.xml in ${outputDir}. Run npm run build first.`);
+	process.exit(1);
+}
+const sitemapUrls = new Set();
+for (const name of sitemapFiles) {
+	const xml = await readFile(path.join(outputDir, name), 'utf8');
+	for (const [, loc] of xml.matchAll(/<loc>([^<]+)<\/loc>/g)) sitemapUrls.add(loc.trim());
+}
+
+const llmsFiles = ['llms-full.txt', 'llms-small.txt'];
+try {
+	for (const name of await readdir(path.join(outputDir, '_llms-txt'))) {
+		if (name.endsWith('.txt')) llmsFiles.push(path.join('_llms-txt', name));
+	}
+} catch {
+	// No custom sets configured.
+}
+
+const problems = [];
+let checked = 0;
+for (const name of llmsFiles) {
+	const text = await readFile(path.join(outputDir, name), 'utf8');
+	for (const [, url] of text.matchAll(/Canonical page: \[[^\]]*\]\(([^)\s]+)\)/g)) {
+		checked += 1;
+		if (!sitemapUrls.has(url)) problems.push(`${name}: ${url} is not in the sitemap`);
+	}
+}
+
+if (checked === 0) problems.push('No "Canonical page:" lines found; the llms-txt patch may not have applied.');
+
+if (problems.length > 0) {
+	console.error(`llms files list ${problems.length} page(s) that are not in the sitemap:`);
+	for (const problem of [...new Set(problems)]) console.error(`  ${problem}`);
+	process.exit(1);
+}
+console.log(`OK: ${checked} canonical page entries across ${llmsFiles.length} llms files are all in the sitemap.`);

scripts/validate-markdown-twins.mjs

@@ -0,0 +1,124 @@
+// Checks the generated Markdown twins after `npm run build`.
+//
+// For every sitemap page that advertises a twin with
+// `<link rel="alternate" type="text/markdown">`:
+// - the advertised href is the expected twin, `/<path>.md` (`/` -> `/index.md`);
+// - the HTML page has a `<link rel="canonical">`;
+// - the twin exists and starts with YAML frontmatter whose `canonical` equals
+//   both the sitemap URL and the HTML canonical, and it has a `title`;
+// - the routes in .vercel/output/config.json (written by
+//   scripts/build-markdown-canonical-routes.mjs) give the twin exactly one
+//   `Link: <HTML canonical>; rel="canonical"` header.
+// Paths that are not twins, including reserved directories, missing files, and
+// case variants, must get no canonical `Link` header.
+//
+// Usage: node scripts/validate-markdown-twins.mjs [outputDir]
+// outputDir defaults to .vercel/output.
+import { readdir, readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { canonicalLinkValue, linkHeadersFor } from '../src/lib/markdown-canonical-routes.js';
+import { DOCS_ORIGIN } from '../src/lib/site.js';
+
+const outputRoot = path.resolve(process.argv[2] ?? '.vercel/output');
+const outputDir = path.join(outputRoot, 'static');
+
+const sitemapFiles = (await readdir(outputDir)).filter((name) => /^sitemap-\d+\.xml$/.test(name));
+if (sitemapFiles.length === 0) {
+	console.error(`No sitemap-N.xml in ${outputDir}. Run npm run build first.`);
+	process.exit(1);
+}
+const pageUrls = new Set();
+for (const name of sitemapFiles) {
+	const xml = await readFile(path.join(outputDir, name), 'utf8');
+	for (const [, loc] of xml.matchAll(/<loc>([^<]+)<\/loc>/g)) pageUrls.add(loc.trim());
+}
+const { routes } = JSON.parse(await readFile(path.join(outputRoot, 'config.json'), 'utf8'));
+
+const problems = [];
+let checked = 0;
+let skipped = 0;
+for (const url of pageUrls) {
+	if (!url.startsWith(`${DOCS_ORIGIN}/`)) continue;
+	const pagePath = new URL(url).pathname.replace(/\/$/, '');
+	const twinUrlPath = `${pagePath || '/index'}.md`;
+	const htmlFile = path.join(outputDir, pagePath, 'index.html');
+	const markdownFile = path.join(outputDir, twinUrlPath);
+
+	let html;
+	try {
+		html = await readFile(htmlFile, 'utf8');
+	} catch {
+		problems.push(`${url}: missing ${path.relative(outputDir, htmlFile)}`);
+		continue;
+	}
+	// Pages without a twin, such as the API reference at /api/, do not advertise one.
+	const alternate = html.match(/<link rel="alternate" type="text\/markdown" href="([^"]*)"/);
+	if (!alternate) {
+		skipped += 1;
+		continue;
+	}
+	checked += 1;
+	if (alternate[1] !== twinUrlPath) {
+		problems.push(`${url}: advertises Markdown at ${alternate[1]}, expected ${twinUrlPath}`);
+	}
+	const htmlCanonical = html.match(/<link rel="canonical" href="([^"]+)"/)?.[1];
+	if (!htmlCanonical) problems.push(`${url}: HTML page has no <link rel="canonical">`);
+
+	const headers = linkHeadersFor(routes, twinUrlPath);
+	const expectedHeader = canonicalLinkValue(htmlCanonical ?? url);
+	if (headers.length !== 1 || headers[0] !== expectedHeader) {
+		problems.push(`${url}: ${twinUrlPath} Link headers are ${JSON.stringify(headers)}, expected [${expectedHeader}]`);
+	}
+
+	let markdown;
+	try {
+		markdown = await readFile(markdownFile, 'utf8');
+	} catch {
+		problems.push(`${url}: advertises ${twinUrlPath}, which was not generated`);
+		continue;
+	}
+	const frontmatter = markdown.match(/^---\n([\s\S]*?)\n---\n/)?.[1];
+	if (!frontmatter) {
+		problems.push(`${url}: .md twin does not start with YAML frontmatter`);
+		continue;
+	}
+	const fields = {};
+	for (const line of frontmatter.split('\n')) {
+		const index = line.indexOf(': ');
+		try {
+			fields[line.slice(0, index)] = JSON.parse(line.slice(index + 2));
+		} catch {
+			problems.push(`${url}: frontmatter line is not a quoted value: ${line}`);
+		}
+	}
+	if (!fields.title) problems.push(`${url}: frontmatter has no title`);
+	if (fields.canonical !== url) {
+		problems.push(`${url}: frontmatter canonical is ${fields.canonical}`);
+	}
+	if (fields.canonical !== htmlCanonical) {
+		problems.push(`${url}: HTML canonical ${htmlCanonical} differs from twin canonical ${fields.canonical}`);
+	}
+}
+
+if (checked === 0) problems.push('No Markdown twins were checked.');
+
+const notTwins = [
+	'/', '/factories/', '/llms.txt', '/sitemap-0.xml', '/openapi.yaml',
+	'/assets/not-a-twin.md', '/_astro/not-a-twin.md', '/.well-known/not-a-twin.md',
+	'/robots.md', '/AGENTS.md', '/not-a-twin.md', '/Factories.md', '/INDEX.md', '/factories.MD',
+];
+for (const pathname of notTwins) {
+	const headers = linkHeadersFor(routes, pathname);
+	if (headers.length > 0) problems.push(`${pathname} is not a twin but gets Link ${JSON.stringify(headers)}`);
+}
+
+if (problems.length > 0) {
+	console.error(`${problems.length} Markdown twin problem(s):`);
+	for (const problem of problems.slice(0, 50)) console.error(`  ${problem}`);
+	process.exit(1);
+}
+console.log(
+	`OK: ${checked} Markdown twins have frontmatter and exactly one canonical Link header that match ` +
+		`their HTML page; ${notTwins.length} non-twin paths get none ` +
+		`(${skipped} sitemap page(s) advertise no twin).`,
+);