diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff229d761..3357dae8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,6 +108,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Set up Node.js uses: actions/setup-node@v4 @@ -129,6 +131,9 @@ jobs: - name: Test heading variables run: npm run test:heading-vars + - name: Test sitemap lastmod + run: npm run test:sitemap-lastmod + - name: Test tracking redirect middleware run: npm run test:tracking-redirect diff --git a/astro.config.mjs b/astro.config.mjs index deb71f7ca..0a9c57b38 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -4,7 +4,7 @@ import { defineConfig, envField } from 'astro/config'; import remarkGfm from 'remark-gfm'; import mermaid from 'astro-mermaid'; import react from '@astrojs/react'; -import sitemap from '@astrojs/sitemap'; +import sitemap from './src/integrations/sitemap-lastmod.js'; import starlight from '@astrojs/starlight'; import starlightLlmsTxt from 'starlight-llms-txt'; import starlightSidebarTopics from 'starlight-sidebar-topics'; diff --git a/package.json b/package.json index 18983cd39..8efe7d82d 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "predev": "node scripts/generate-openapi-json.mjs", "build": "astro build", "postbuild": "node scripts/build-tracking-middleware.mjs", - "prebuild": "node scripts/generate-openapi-json.mjs", + "prebuild": "node scripts/ensure-git-history.mjs && node scripts/generate-openapi-json.mjs", "preview": "astro preview", "astro": "astro", "typecheck": "astro check", @@ -24,6 +24,7 @@ "test:kapa-webmcp": "node --test src/lib/kapa-webmcp.test.mjs", "test:kapa-support-escalation": "node --test src/lib/kapa-support-escalation.test.mjs", "test:homepage-json-ld": "node scripts/validate-homepage-json-ld.mjs", + "test:sitemap-lastmod": "node --test src/integrations/sitemap-lastmod.test.mjs src/lib/git-history.test.mjs", "postinstall": "patch-package" }, "engines": { diff --git a/scripts/ensure-git-history.mjs b/scripts/ensure-git-history.mjs new file mode 100644 index 000000000..4ff811c4f --- /dev/null +++ b/scripts/ensure-git-history.mjs @@ -0,0 +1,23 @@ +import { spawnSync } from 'node:child_process'; +import { performance } from 'node:perf_hooks'; + +// A shallow boundary commit makes unchanged files look newly modified to +// Starlight. Fetch history before Astro inlines git dates for pages/sitemaps. +const shallow = spawnSync('git', ['rev-parse', '--is-shallow-repository'], { encoding: 'utf8' }); +if (shallow.status === 0 && shallow.stdout.trim() === 'true') { + const start = performance.now(); + const origin = spawnSync('git', ['remote', 'get-url', 'origin'], { encoding: 'utf8' }); + const { VERCEL_GIT_REPO_OWNER: owner, VERCEL_GIT_REPO_SLUG: slug } = process.env; + const publicRepo = owner && slug && /^[\w.-]+$/.test(owner) && /^[\w.-]+$/.test(slug) + ? `https://github.com/${owner}/${slug}.git` + : undefined; + const remote = origin.status === 0 ? 'origin' : publicRepo; + const fetch = remote + ? spawnSync('git', ['fetch', '--unshallow', '--no-tags', remote], { encoding: 'utf8', timeout: 120_000 }) + : undefined; + if (fetch?.status === 0) { + console.log(`Fetched full git history in ${((performance.now() - start) / 1000).toFixed(1)}s.`); + } else { + console.warn('Could not restore full git history; shallow-boundary page dates will be omitted.'); + } +} diff --git a/src/integrations/sitemap-lastmod.js b/src/integrations/sitemap-lastmod.js new file mode 100644 index 000000000..44549f2b7 --- /dev/null +++ b/src/integrations/sitemap-lastmod.js @@ -0,0 +1,40 @@ +import sitemap from '@astrojs/sitemap'; +import { readFile } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; +import { parseHTML } from 'linkedom'; + +export default function sitemapLastmod() { + let outputRoot; + const integration = sitemap({ + serialize: async (item) => { + const pathname = decodeURIComponent(new URL(item.url).pathname); + const htmlPath = path.join(outputRoot, pathname, 'index.html'); + try { + return withLastmod(item, await readFile(htmlPath, 'utf8')); + } catch (error) { + // Routes without generated HTML have no Starlight git date. + if (error.code === 'ENOENT' || error.code === 'ENOTDIR') return item; + throw error; + } + }, + }); + return { + ...integration, + hooks: { + ...integration.hooks, + // Runs before @astrojs/sitemap's build:done serialization, while + // the adapter still exposes the generated HTML directory. + 'astro:build:generated': ({ dir }) => { + outputRoot = fileURLToPath(dir); + }, + }, + }; +} + +export function withLastmod(item, html) { + const { document } = parseHTML(html); + const value = document.querySelector('meta[property="article:modified_time"]')?.getAttribute('content'); + if (!value || !Number.isFinite(Date.parse(value))) return item; + return { ...item, lastmod: new Date(value).toISOString() }; +} diff --git a/src/integrations/sitemap-lastmod.test.mjs b/src/integrations/sitemap-lastmod.test.mjs new file mode 100644 index 000000000..8ba4ca6b8 --- /dev/null +++ b/src/integrations/sitemap-lastmod.test.mjs @@ -0,0 +1,21 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { withLastmod } from './sitemap-lastmod.js'; + +test('sitemap uses the exact rendered Starlight date, not build time', () => { + const item = { url: 'https://docs.warp.dev/factories/quickstart/' }; + for (const date of ['2026-10-08T17:42:12.000Z', '2025-03-01T09:00:00.000Z']) { + assert.deepEqual( + withLastmod(item, ``), + { ...item, lastmod: date }, + ); + } +}); + +test('pages without a valid Starlight date do not invent lastmod', () => { + const item = { url: 'https://docs.warp.dev/api/' }; + for (const html of ['', '', + '']) { + assert.equal(withLastmod(item, html), item); + } +}); diff --git a/src/lib/git-history.js b/src/lib/git-history.js new file mode 100644 index 000000000..7ed5a8bfb --- /dev/null +++ b/src/lib/git-history.js @@ -0,0 +1,18 @@ +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; + +// Boundary commits in a shallow checkout synthesize modifications for all +// files at the clone horizon. Their timestamps are not content-change dates. +export function getShallowBoundaryDates() { + const result = spawnSync('git', ['rev-parse', '--git-path', 'shallow'], { encoding: 'utf8' }); + if (result.status !== 0) return new Set(); + let commits; + try { + commits = readFileSync(result.stdout.trim(), 'utf8').trim().split('\n'); + } catch { + return new Set(); + } + if (!commits.every((commit) => /^[a-f0-9]{40,64}$/.test(commit))) return new Set(); + const dates = spawnSync('git', ['show', '--no-patch', '--format=%ct', ...commits], { encoding: 'utf8' }); + return new Set(dates.stdout.trim().split('\n').map((timestamp) => Number(timestamp) * 1000)); +} diff --git a/src/lib/git-history.test.mjs b/src/lib/git-history.test.mjs new file mode 100644 index 000000000..77c2b4509 --- /dev/null +++ b/src/lib/git-history.test.mjs @@ -0,0 +1,41 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { getShallowBoundaryDates } from './git-history.js'; + +test('unavailable history is nonfatal and boundary dates are identifiable', () => { + const root = process.cwd(); + const repo = path.join(mkdtempSync(path.join(tmpdir(), 'docs-history-test-')), 'repo'); + const init = spawnSync('git', ['init', repo], { encoding: 'utf8' }); + assert.equal(init.status, 0, init.stderr); + const objects = spawnSync('git', ['rev-parse', '--git-path', 'objects'], { encoding: 'utf8' }); + const head = spawnSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).stdout.trim(); + writeFileSync(path.join(repo, '.git/objects/info/alternates'), `${path.resolve(root, objects.stdout.trim())}\n`); + const git = (...args) => { + const result = spawnSync('git', args, { cwd: repo, encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + return result.stdout.trim(); + }; + process.chdir(repo); + try { + git('update-ref', 'HEAD', head); + writeFileSync(path.join(repo, '.git/shallow'), `${head}\n`); + const timestamp = Number(git('show', '--no-patch', '--format=%ct', 'HEAD')) * 1000; + assert.deepEqual(getShallowBoundaryDates(), new Set([timestamp])); + const result = spawnSync('node', [path.join(root, 'scripts/ensure-git-history.mjs')], { + cwd: repo, + encoding: 'utf8', + env: { ...process.env, VERCEL_GIT_REPO_OWNER: '', VERCEL_GIT_REPO_SLUG: '' }, + }); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stderr, /shallow-boundary page dates will be omitted/); + assert.ok(readFileSync(path.join(repo, '.git/shallow'), 'utf8').trim()); + unlinkSync(path.join(repo, '.git/shallow')); + assert.deepEqual(getShallowBoundaryDates(), new Set()); + } finally { + process.chdir(root); + } +}); diff --git a/src/routeData.ts b/src/routeData.ts index f5fc833a9..673114315 100644 --- a/src/routeData.ts +++ b/src/routeData.ts @@ -1,6 +1,9 @@ import { defineRouteMiddleware } from '@astrojs/starlight/route-data'; import config from 'virtual:starlight/user-config'; import { getMarkdownPathFromHtmlPath } from './lib/docs-markdown.js'; +import { getShallowBoundaryDates } from './lib/git-history.js'; + +const shallowBoundaryDates = getShallowBoundaryDates(); /** * Per-page head augmentation. @@ -15,7 +18,20 @@ import { getMarkdownPathFromHtmlPath } from './lib/docs-markdown.js'; */ export const onRequest = defineRouteMiddleware((context) => { const route = context.locals.starlightRoute; + if (!(route.entry.data.lastUpdated instanceof Date) && + route.lastUpdated && shallowBoundaryDates.has(route.lastUpdated.getTime())) { + route.lastUpdated = undefined; + } const head = route.head; + // Reuse Starlight's git-derived date, including its frontmatter overrides. + // The sitemap serializer reads this from the generated page, so it never + // substitutes build time or guesses a source file from a URL. + if (route.lastUpdated) { + head.push({ + tag: 'meta', + attrs: { property: 'article:modified_time', content: route.lastUpdated.toISOString() }, + }); + } head.push({ tag: 'link',