diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddca77de7..01f5f4c42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,6 +141,9 @@ jobs: - name: Test agent discovery catalog run: npm run test:agent-discovery + - name: Test Markdown twin canonical headers + run: npm run test:markdown-canonical + - name: Build site run: npm run build @@ -153,6 +156,12 @@ jobs: - name: Validate homepage JSON-LD run: npm run test:homepage-json-ld + - name: Validate llms files list only sitemap pages + run: npm run test:llms-txt + + - name: Validate Markdown twin frontmatter + run: npm run test:markdown-twins + - name: Validate page title order run: npm run test:page-title-order diff --git a/astro.config.mjs b/astro.config.mjs index deb71f7ca..077614928 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -210,7 +210,9 @@ export default defineConfig({ // due to their size. The upstream plugin only applies `exclude` to // llms-small.txt; our patch (patches/starlight-llms-txt+0.8.1.patch) // extends it to llms-full.txt and custom sets as well. - exclude: ['support-and-community/community/open-source-licenses'], + // '404' is the not-found page; it is not documentation, and listing + // it sent agents to https://docs.warp.dev/404/. + exclude: ['support-and-community/community/open-source-licenses', '404'], // This string is the first thing an AI engine reads about Warp, so it // carries the product lineup. Kept in sync with the rename and with // launches: it said "the Oz platform" until the 8/18 rename, which diff --git a/package.json b/package.json index 3938524e0..ee0b97c6a 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "dev": "astro dev", "predev": "node scripts/generate-openapi-json.mjs", "build": "astro build", - "postbuild": "node scripts/build-tracking-middleware.mjs", + "postbuild": "node scripts/build-tracking-middleware.mjs && node scripts/build-markdown-canonical-routes.mjs", "prebuild": "node scripts/generate-openapi-json.mjs", "preview": "astro preview", "astro": "astro", @@ -22,10 +22,13 @@ "test:docs-markdown": "node --test src/integrations/docs-markdown-integration.test.js", "test:tracking-redirect": "node --test scripts/tracking-redirect.test.mjs", "test:agent-discovery": "node --test scripts/agent-discovery.test.mjs", + "test:markdown-canonical": "node --test scripts/markdown-canonical-headers.test.mjs", "test:tracking-routes": "node scripts/validate-tracking-routes.mjs", "test:kapa-webmcp": "node --test src/lib/kapa-webmcp.test.mjs", "test:kapa-support-escalation": "node --test src/lib/kapa-support-escalation.test.mjs", "test:homepage-json-ld": "node scripts/validate-homepage-json-ld.mjs", + "test:llms-txt": "node scripts/validate-llms-txt.mjs", + "test:markdown-twins": "node scripts/validate-markdown-twins.mjs", "test:page-title-order": "node scripts/validate-page-title-order.mjs", "postinstall": "patch-package" }, diff --git a/scripts/build-markdown-canonical-routes.mjs b/scripts/build-markdown-canonical-routes.mjs new file mode 100644 index 000000000..524329e34 --- /dev/null +++ b/scripts/build-markdown-canonical-routes.mjs @@ -0,0 +1,63 @@ +// Adds canonical `Link` headers for the generated Markdown twins to the Vercel +// build output. Runs from `postbuild`, after `astro build` writes +// .vercel/output/config.json and the docs-markdown integration writes the twins. +// +// A twin is `.md` (`index.md` for the homepage) whose HTML page, +// `/index.html`, advertises it with +// ``. Only those +// paths get the header, so 404 responses for other `.md` requests do not claim +// a canonical page. Routes go before the filesystem phase, where vercel.json +// headers also run, and use `continue` so the static file is still served. +// See src/lib/markdown-canonical-routes.js. +import { readdir, readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { buildMarkdownCanonicalRoutes } from '../src/lib/markdown-canonical-routes.js'; + +const output = fileURLToPath(new URL('../.vercel/output/', import.meta.url)); +const staticDir = path.join(output, 'static'); +const configPath = path.join(output, 'config.json'); + +/** @param {string} dir */ +async function* markdownFiles(dir) { + for (const entry of await readdir(dir, { withFileTypes: true })) { + const entryPath = path.join(dir, entry.name); + if (entry.isDirectory()) yield* markdownFiles(entryPath); + else if (entry.isFile() && entry.name.endsWith('.md')) yield entryPath; + } +} + +const twinPaths = []; +for await (const file of markdownFiles(staticDir)) { + const twinPath = path.relative(staticDir, file).split(path.sep).join('/').slice(0, -'.md'.length); + const htmlFile = twinPath === 'index' + ? path.join(staticDir, 'index.html') + : path.join(staticDir, twinPath, 'index.html'); + let html; + try { + html = await readFile(htmlFile, 'utf8'); + } catch { + continue; + } + if (html.includes(` !(route.headers?.Link ?? '').endsWith('; rel="canonical"'), +); +const filesystemIndex = routes.findIndex((route) => route.handle === 'filesystem'); +const canonicalRoutes = buildMarkdownCanonicalRoutes(twinPaths); +routes.splice(filesystemIndex === -1 ? routes.length : filesystemIndex, 0, ...canonicalRoutes); +config.routes = routes; +await writeFile(configPath, JSON.stringify(config, null, 2)); +console.log( + `build-markdown-canonical-routes: ${twinPaths.length} twins, ${canonicalRoutes.length} header routes`, +); diff --git a/scripts/markdown-canonical-headers.test.mjs b/scripts/markdown-canonical-headers.test.mjs new file mode 100644 index 000000000..8fd4dfe03 --- /dev/null +++ b/scripts/markdown-canonical-headers.test.mjs @@ -0,0 +1,82 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { test } from 'node:test'; +import { + buildMarkdownCanonicalRoutes, + canonicalLinkValue, + linkHeadersFor, +} from '../src/lib/markdown-canonical-routes.js'; +import { DOCS_ORIGIN } from '../src/lib/site.js'; + +const twins = [ + 'index', + 'agents', + 'factories', + 'terminal/blocks/block-basics', + 'changelog/2026', + 'factories/api-and-sdk/troubleshooting/errors/conflict', +]; +const routes = buildMarkdownCanonicalRoutes(twins); + +test('every generated twin points to its HTML page', () => { + const cases = { + '/index.md': `${DOCS_ORIGIN}/`, + '/agents.md': `${DOCS_ORIGIN}/agents/`, + '/factories.md': `${DOCS_ORIGIN}/factories/`, + '/terminal/blocks/block-basics.md': `${DOCS_ORIGIN}/terminal/blocks/block-basics/`, + '/changelog/2026.md': `${DOCS_ORIGIN}/changelog/2026/`, + '/factories/api-and-sdk/troubleshooting/errors/conflict.md': + `${DOCS_ORIGIN}/factories/api-and-sdk/troubleshooting/errors/conflict/`, + }; + for (const [pathname, htmlUrl] of Object.entries(cases)) { + assert.deepEqual(linkHeadersFor(routes, pathname), [canonicalLinkValue(htmlUrl)], pathname); + } +}); + +test('paths that are not generated twins get no canonical Link header', () => { + for (const pathname of [ + // HTML pages and other generated files + '/', '/factories/', '/factories', '/llms.txt', '/openapi.yaml', '/sitemap-0.xml', + // reserved and missing paths, which return 404 + '/assets/not-a-twin.md', '/_astro/not-a-twin.md', '/.well-known/not-a-twin.md', + '/robots.md', '/not-a-twin.md', '/factories/missing.md', '/terminal/index.md', + // near misses + '/factories.md/', '/factories.mdx', '/factories.md.html', '/xfactories.md', '/factories-x.md', + '/terminal/blocks.md', '/index.md/extra', + // case variants: static files are case-sensitive, so these 404 + '/AGENTS.md', '/Factories.md', '/INDEX.md', '/factories.MD', + ]) { + assert.deepEqual(linkHeadersFor(routes, pathname), [], pathname); + } +}); + +test('large twin sets are split into bounded routes that still match exactly', () => { + const many = Array.from({ length: 600 }, (_, index) => `section-${index}/page-with-a-long-name-${index}`); + const manyRoutes = buildMarkdownCanonicalRoutes(many); + assert.ok(manyRoutes.length > 1); + for (const route of manyRoutes) { + assert.ok(route.src.length < 4000, `src is ${route.src.length} characters`); + assert.equal(route.caseSensitive, true); + assert.equal(route.continue, true); + } + for (const twin of [many[0], many[299], many[599]]) { + assert.deepEqual(linkHeadersFor(manyRoutes, `/${twin}.md`), [canonicalLinkValue(`${DOCS_ORIGIN}/${twin}/`)]); + } + assert.deepEqual(linkHeadersFor(manyRoutes, '/section-0.md'), []); +}); + +test('regex characters in a path are matched literally', () => { + const special = buildMarkdownCanonicalRoutes(['guides/c++', 'a.b']); + assert.deepEqual(linkHeadersFor(special, '/guides/c++.md'), [canonicalLinkValue(`${DOCS_ORIGIN}/guides/c++/`)]); + assert.deepEqual(linkHeadersFor(special, '/guides/cc.md'), []); + assert.deepEqual(linkHeadersFor(special, '/aXb.md'), []); +}); + +test('vercel.json adds no canonical Link header (generated routes own it)', async () => { + const vercelConfig = JSON.parse(await readFile(new URL('../vercel.json', import.meta.url), 'utf8')); + for (const entry of vercelConfig.headers ?? []) { + for (const header of entry.headers) { + assert.notEqual(header.key.toLowerCase(), 'link', entry.source); + } + } +}); diff --git a/scripts/validate-llms-txt.mjs b/scripts/validate-llms-txt.mjs new file mode 100644 index 000000000..75e7eb2b3 --- /dev/null +++ b/scripts/validate-llms-txt.mjs @@ -0,0 +1,53 @@ +// Checks the generated llms files after `npm run build`. +// +// Every page that llms-full.txt, llms-small.txt, or a /_llms-txt/*.txt set +// includes carries a "Canonical page: [path](url)" line (see +// patches/starlight-llms-txt+0.8.1.patch). Each of those URLs must be a page +// listed in the sitemap, so the files never send agents to the 404 page or to a +// page that search engines are told does not exist. +// +// Usage: node scripts/validate-llms-txt.mjs [outputDir] +// outputDir defaults to .vercel/output/static. +import { readdir, readFile } from 'node:fs/promises'; +import path from 'node:path'; + +const outputDir = path.resolve(process.argv[2] ?? '.vercel/output/static'); + +const sitemapFiles = (await readdir(outputDir)).filter((name) => /^sitemap-\d+\.xml$/.test(name)); +if (sitemapFiles.length === 0) { + console.error(`No sitemap-N.xml in ${outputDir}. Run npm run build first.`); + process.exit(1); +} +const sitemapUrls = new Set(); +for (const name of sitemapFiles) { + const xml = await readFile(path.join(outputDir, name), 'utf8'); + for (const [, loc] of xml.matchAll(/([^<]+)<\/loc>/g)) sitemapUrls.add(loc.trim()); +} + +const llmsFiles = ['llms-full.txt', 'llms-small.txt']; +try { + for (const name of await readdir(path.join(outputDir, '_llms-txt'))) { + if (name.endsWith('.txt')) llmsFiles.push(path.join('_llms-txt', name)); + } +} catch { + // No custom sets configured. +} + +const problems = []; +let checked = 0; +for (const name of llmsFiles) { + const text = await readFile(path.join(outputDir, name), 'utf8'); + for (const [, url] of text.matchAll(/Canonical page: \[[^\]]*\]\(([^)\s]+)\)/g)) { + checked += 1; + if (!sitemapUrls.has(url)) problems.push(`${name}: ${url} is not in the sitemap`); + } +} + +if (checked === 0) problems.push('No "Canonical page:" lines found; the llms-txt patch may not have applied.'); + +if (problems.length > 0) { + console.error(`llms files list ${problems.length} page(s) that are not in the sitemap:`); + for (const problem of [...new Set(problems)]) console.error(` ${problem}`); + process.exit(1); +} +console.log(`OK: ${checked} canonical page entries across ${llmsFiles.length} llms files are all in the sitemap.`); diff --git a/scripts/validate-markdown-twins.mjs b/scripts/validate-markdown-twins.mjs new file mode 100644 index 000000000..08fdddabd --- /dev/null +++ b/scripts/validate-markdown-twins.mjs @@ -0,0 +1,124 @@ +// Checks the generated Markdown twins after `npm run build`. +// +// For every sitemap page that advertises a twin with +// ``: +// - the advertised href is the expected twin, `/.md` (`/` -> `/index.md`); +// - the HTML page has a ``; +// - the twin exists and starts with YAML frontmatter whose `canonical` equals +// both the sitemap URL and the HTML canonical, and it has a `title`; +// - the routes in .vercel/output/config.json (written by +// scripts/build-markdown-canonical-routes.mjs) give the twin exactly one +// `Link: ; rel="canonical"` header. +// Paths that are not twins, including reserved directories, missing files, and +// case variants, must get no canonical `Link` header. +// +// Usage: node scripts/validate-markdown-twins.mjs [outputDir] +// outputDir defaults to .vercel/output. +import { readdir, readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { canonicalLinkValue, linkHeadersFor } from '../src/lib/markdown-canonical-routes.js'; +import { DOCS_ORIGIN } from '../src/lib/site.js'; + +const outputRoot = path.resolve(process.argv[2] ?? '.vercel/output'); +const outputDir = path.join(outputRoot, 'static'); + +const sitemapFiles = (await readdir(outputDir)).filter((name) => /^sitemap-\d+\.xml$/.test(name)); +if (sitemapFiles.length === 0) { + console.error(`No sitemap-N.xml in ${outputDir}. Run npm run build first.`); + process.exit(1); +} +const pageUrls = new Set(); +for (const name of sitemapFiles) { + const xml = await readFile(path.join(outputDir, name), 'utf8'); + for (const [, loc] of xml.matchAll(/([^<]+)<\/loc>/g)) pageUrls.add(loc.trim()); +} +const { routes } = JSON.parse(await readFile(path.join(outputRoot, 'config.json'), 'utf8')); + +const problems = []; +let checked = 0; +let skipped = 0; +for (const url of pageUrls) { + if (!url.startsWith(`${DOCS_ORIGIN}/`)) continue; + const pagePath = new URL(url).pathname.replace(/\/$/, ''); + const twinUrlPath = `${pagePath || '/index'}.md`; + const htmlFile = path.join(outputDir, pagePath, 'index.html'); + const markdownFile = path.join(outputDir, twinUrlPath); + + let html; + try { + html = await readFile(htmlFile, 'utf8'); + } catch { + problems.push(`${url}: missing ${path.relative(outputDir, htmlFile)}`); + continue; + } + // Pages without a twin, such as the API reference at /api/, do not advertise one. + const alternate = html.match(/`); + + const headers = linkHeadersFor(routes, twinUrlPath); + const expectedHeader = canonicalLinkValue(htmlCanonical ?? url); + if (headers.length !== 1 || headers[0] !== expectedHeader) { + problems.push(`${url}: ${twinUrlPath} Link headers are ${JSON.stringify(headers)}, expected [${expectedHeader}]`); + } + + let markdown; + try { + markdown = await readFile(markdownFile, 'utf8'); + } catch { + problems.push(`${url}: advertises ${twinUrlPath}, which was not generated`); + continue; + } + const frontmatter = markdown.match(/^---\n([\s\S]*?)\n---\n/)?.[1]; + if (!frontmatter) { + problems.push(`${url}: .md twin does not start with YAML frontmatter`); + continue; + } + const fields = {}; + for (const line of frontmatter.split('\n')) { + const index = line.indexOf(': '); + try { + fields[line.slice(0, index)] = JSON.parse(line.slice(index + 2)); + } catch { + problems.push(`${url}: frontmatter line is not a quoted value: ${line}`); + } + } + if (!fields.title) problems.push(`${url}: frontmatter has no title`); + if (fields.canonical !== url) { + problems.push(`${url}: frontmatter canonical is ${fields.canonical}`); + } + if (fields.canonical !== htmlCanonical) { + problems.push(`${url}: HTML canonical ${htmlCanonical} differs from twin canonical ${fields.canonical}`); + } +} + +if (checked === 0) problems.push('No Markdown twins were checked.'); + +const notTwins = [ + '/', '/factories/', '/llms.txt', '/sitemap-0.xml', '/openapi.yaml', + '/assets/not-a-twin.md', '/_astro/not-a-twin.md', '/.well-known/not-a-twin.md', + '/robots.md', '/AGENTS.md', '/not-a-twin.md', '/Factories.md', '/INDEX.md', '/factories.MD', +]; +for (const pathname of notTwins) { + const headers = linkHeadersFor(routes, pathname); + if (headers.length > 0) problems.push(`${pathname} is not a twin but gets Link ${JSON.stringify(headers)}`); +} + +if (problems.length > 0) { + console.error(`${problems.length} Markdown twin problem(s):`); + for (const problem of problems.slice(0, 50)) console.error(` ${problem}`); + process.exit(1); +} +console.log( + `OK: ${checked} Markdown twins have frontmatter and exactly one canonical Link header that match ` + + `their HTML page; ${notTwins.length} non-twin paths get none ` + + `(${skipped} sitemap page(s) advertise no twin).`, +); diff --git a/src/components/CopyPageButton.astro b/src/components/CopyPageButton.astro index 72cfde37f..99bcc00f7 100644 --- a/src/components/CopyPageButton.astro +++ b/src/components/CopyPageButton.astro @@ -140,6 +140,8 @@ const claudeUrl = `https://claude.ai/new?q=${prompt}`;