From 865c94d9eeb3ec5dd7ad15ea7a589df6245fe3fe Mon Sep 17 00:00:00 2001 From: "warp-agent-staging[bot]" <240773466+warp-agent-staging[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:22:11 +0000 Subject: [PATCH 1/2] AR-20261104-2: canonical Link header + frontmatter on docs .md twins; drop /404/ from llms files --- .github/workflows/ci.yml | 9 ++ astro.config.mjs | 4 +- package.json | 3 + scripts/markdown-canonical-headers.test.mjs | 48 ++++++++++ scripts/validate-llms-txt.mjs | 53 +++++++++++ scripts/validate-markdown-twins.mjs | 94 +++++++++++++++++++ src/components/CopyPageButton.astro | 7 +- src/integrations/docs-markdown-integration.js | 45 ++++++++- .../docs-markdown-integration.test.js | 72 ++++++++++++++ src/lib/docs-markdown.js | 12 +++ vercel.json | 18 ++++ 11 files changed, 360 insertions(+), 5 deletions(-) create mode 100644 scripts/markdown-canonical-headers.test.mjs create mode 100644 scripts/validate-llms-txt.mjs create mode 100644 scripts/validate-markdown-twins.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddca77de7..01f5f4c42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,6 +141,9 @@ jobs: - name: Test agent discovery catalog run: npm run test:agent-discovery + - name: Test Markdown twin canonical headers + run: npm run test:markdown-canonical + - name: Build site run: npm run build @@ -153,6 +156,12 @@ jobs: - name: Validate homepage JSON-LD run: npm run test:homepage-json-ld + - name: Validate llms files list only sitemap pages + run: npm run test:llms-txt + + - name: Validate Markdown twin frontmatter + run: npm run test:markdown-twins + - name: Validate page title order run: npm run test:page-title-order diff --git a/astro.config.mjs b/astro.config.mjs index deb71f7ca..077614928 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -210,7 +210,9 @@ export default defineConfig({ // due to their size. The upstream plugin only applies `exclude` to // llms-small.txt; our patch (patches/starlight-llms-txt+0.8.1.patch) // extends it to llms-full.txt and custom sets as well. - exclude: ['support-and-community/community/open-source-licenses'], + // '404' is the not-found page; it is not documentation, and listing + // it sent agents to https://docs.warp.dev/404/. + exclude: ['support-and-community/community/open-source-licenses', '404'], // This string is the first thing an AI engine reads about Warp, so it // carries the product lineup. Kept in sync with the rename and with // launches: it said "the Oz platform" until the 8/18 rename, which diff --git a/package.json b/package.json index 3938524e0..d96f3acff 100644 --- a/package.json +++ b/package.json @@ -22,10 +22,13 @@ "test:docs-markdown": "node --test src/integrations/docs-markdown-integration.test.js", "test:tracking-redirect": "node --test scripts/tracking-redirect.test.mjs", "test:agent-discovery": "node --test scripts/agent-discovery.test.mjs", + "test:markdown-canonical": "node --test scripts/markdown-canonical-headers.test.mjs", "test:tracking-routes": "node scripts/validate-tracking-routes.mjs", "test:kapa-webmcp": "node --test src/lib/kapa-webmcp.test.mjs", "test:kapa-support-escalation": "node --test src/lib/kapa-support-escalation.test.mjs", "test:homepage-json-ld": "node scripts/validate-homepage-json-ld.mjs", + "test:llms-txt": "node scripts/validate-llms-txt.mjs", + "test:markdown-twins": "node scripts/validate-markdown-twins.mjs", "test:page-title-order": "node scripts/validate-page-title-order.mjs", "postinstall": "patch-package" }, diff --git a/scripts/markdown-canonical-headers.test.mjs b/scripts/markdown-canonical-headers.test.mjs new file mode 100644 index 000000000..8b8cc3117 --- /dev/null +++ b/scripts/markdown-canonical-headers.test.mjs @@ -0,0 +1,48 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { test } from 'node:test'; +// Same converter Vercel uses to turn vercel.json `headers` into routes +// (installed through @astrojs/vercel), so the test exercises the real +// source pattern and `:path` substitution instead of a reimplementation. +import { getTransformedRoutes } from '@vercel/routing-utils'; +import { DOCS_ORIGIN } from '../src/lib/site.js'; + +const vercelConfig = JSON.parse(await readFile(new URL('../vercel.json', import.meta.url), 'utf8')); +const { routes, error } = getTransformedRoutes({ headers: vercelConfig.headers }); + +/** Returns every Link header value Vercel would add to a response for `pathname`. */ +function linkHeadersFor(pathname) { + const values = []; + for (const route of routes ?? []) { + const key = Object.keys(route.headers ?? {}).find((name) => name.toLowerCase() === 'link'); + if (!key) continue; + const match = pathname.match(new RegExp(route.src)); + if (!match) continue; + values.push(route.headers[key].replace(/\$(\d+)/g, (_, index) => match[Number(index)] ?? '')); + } + return values; +} + +const canonical = (url) => `<${url}>; rel="canonical"`; + +test('vercel.json headers compile', () => { + assert.equal(error, null); +}); + +test('every Markdown twin points to its HTML page with a canonical Link header', () => { + const cases = { + '/index.md': `${DOCS_ORIGIN}/`, + '/factories.md': `${DOCS_ORIGIN}/factories/`, + '/terminal/blocks/block-basics.md': `${DOCS_ORIGIN}/terminal/blocks/block-basics/`, + '/changelog/2026.md': `${DOCS_ORIGIN}/changelog/2026/`, + }; + for (const [pathname, htmlUrl] of Object.entries(cases)) { + assert.deepEqual(linkHeadersFor(pathname), [canonical(htmlUrl)], pathname); + } +}); + +test('HTML pages and other files get no canonical Link header', () => { + for (const pathname of ['/', '/factories/', '/factories', '/llms.txt', '/openapi.yaml', '/sitemap-0.xml']) { + assert.deepEqual(linkHeadersFor(pathname), [], pathname); + } +}); diff --git a/scripts/validate-llms-txt.mjs b/scripts/validate-llms-txt.mjs new file mode 100644 index 000000000..75e7eb2b3 --- /dev/null +++ b/scripts/validate-llms-txt.mjs @@ -0,0 +1,53 @@ +// Checks the generated llms files after `npm run build`. +// +// Every page that llms-full.txt, llms-small.txt, or a /_llms-txt/*.txt set +// includes carries a "Canonical page: [path](url)" line (see +// patches/starlight-llms-txt+0.8.1.patch). Each of those URLs must be a page +// listed in the sitemap, so the files never send agents to the 404 page or to a +// page that search engines are told does not exist. +// +// Usage: node scripts/validate-llms-txt.mjs [outputDir] +// outputDir defaults to .vercel/output/static. +import { readdir, readFile } from 'node:fs/promises'; +import path from 'node:path'; + +const outputDir = path.resolve(process.argv[2] ?? '.vercel/output/static'); + +const sitemapFiles = (await readdir(outputDir)).filter((name) => /^sitemap-\d+\.xml$/.test(name)); +if (sitemapFiles.length === 0) { + console.error(`No sitemap-N.xml in ${outputDir}. Run npm run build first.`); + process.exit(1); +} +const sitemapUrls = new Set(); +for (const name of sitemapFiles) { + const xml = await readFile(path.join(outputDir, name), 'utf8'); + for (const [, loc] of xml.matchAll(/([^<]+)<\/loc>/g)) sitemapUrls.add(loc.trim()); +} + +const llmsFiles = ['llms-full.txt', 'llms-small.txt']; +try { + for (const name of await readdir(path.join(outputDir, '_llms-txt'))) { + if (name.endsWith('.txt')) llmsFiles.push(path.join('_llms-txt', name)); + } +} catch { + // No custom sets configured. +} + +const problems = []; +let checked = 0; +for (const name of llmsFiles) { + const text = await readFile(path.join(outputDir, name), 'utf8'); + for (const [, url] of text.matchAll(/Canonical page: \[[^\]]*\]\(([^)\s]+)\)/g)) { + checked += 1; + if (!sitemapUrls.has(url)) problems.push(`${name}: ${url} is not in the sitemap`); + } +} + +if (checked === 0) problems.push('No "Canonical page:" lines found; the llms-txt patch may not have applied.'); + +if (problems.length > 0) { + console.error(`llms files list ${problems.length} page(s) that are not in the sitemap:`); + for (const problem of [...new Set(problems)]) console.error(` ${problem}`); + process.exit(1); +} +console.log(`OK: ${checked} canonical page entries across ${llmsFiles.length} llms files are all in the sitemap.`); diff --git a/scripts/validate-markdown-twins.mjs b/scripts/validate-markdown-twins.mjs new file mode 100644 index 000000000..7801dfb82 --- /dev/null +++ b/scripts/validate-markdown-twins.mjs @@ -0,0 +1,94 @@ +// Checks the generated Markdown twins after `npm run build`. +// +// Every sitemap page that advertises a twin with +// `` has one at `.md` +// (`/` -> `/index.md`), written by +// src/integrations/docs-markdown-integration.js. Each twin must exist and start +// with YAML frontmatter whose `canonical` is the HTML page's own +// ``, so the twin, its `Link` header (vercel.json), and +// the HTML page all name the same URL. +// +// Usage: node scripts/validate-markdown-twins.mjs [outputDir] +// outputDir defaults to .vercel/output/static. +import { readdir, readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { DOCS_ORIGIN } from '../src/lib/site.js'; + +const outputDir = path.resolve(process.argv[2] ?? '.vercel/output/static'); + +const sitemapFiles = (await readdir(outputDir)).filter((name) => /^sitemap-\d+\.xml$/.test(name)); +if (sitemapFiles.length === 0) { + console.error(`No sitemap-N.xml in ${outputDir}. Run npm run build first.`); + process.exit(1); +} +const pageUrls = new Set(); +for (const name of sitemapFiles) { + const xml = await readFile(path.join(outputDir, name), 'utf8'); + for (const [, loc] of xml.matchAll(/([^<]+)<\/loc>/g)) pageUrls.add(loc.trim()); +} + +const problems = []; +let checked = 0; +let skipped = 0; +for (const url of pageUrls) { + if (!url.startsWith(`${DOCS_ORIGIN}/`)) continue; + const pagePath = new URL(url).pathname.replace(/\/$/, ''); + const htmlFile = path.join(outputDir, pagePath, 'index.html'); + const markdownFile = path.join(outputDir, pagePath ? `${pagePath}.md` : 'index.md'); + + let html; + try { + html = await readFile(htmlFile, 'utf8'); + } catch { + problems.push(`${url}: missing ${path.relative(outputDir, htmlFile)}`); + continue; + } + // Pages without a twin, such as the API reference at /api/, do not advertise one. + if (!/ 0) { + console.error(`${problems.length} Markdown twin problem(s):`); + for (const problem of problems.slice(0, 50)) console.error(` ${problem}`); + process.exit(1); +} +console.log( + `OK: ${checked} Markdown twins start with frontmatter that points to their HTML page ` + + `(${skipped} sitemap page(s) advertise no twin).`, +); diff --git a/src/components/CopyPageButton.astro b/src/components/CopyPageButton.astro index 72cfde37f..99bcc00f7 100644 --- a/src/components/CopyPageButton.astro +++ b/src/components/CopyPageButton.astro @@ -140,6 +140,8 @@ const claudeUrl = `https://claude.ai/new?q=${prompt}`;