From e5985792464870a08fda525355839c0d0c40f7e5 Mon Sep 17 00:00:00 2001 From: "warp-agent-staging[bot]" <240773466+warp-agent-staging[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:07:36 +0000 Subject: [PATCH 1/4] Emit sitemap lastmod from Starlight page dates --- .github/workflows/ci.yml | 5 +++ astro.config.mjs | 2 +- package.json | 1 + src/integrations/sitemap-lastmod.js | 40 +++++++++++++++++++++++ src/integrations/sitemap-lastmod.test.mjs | 21 ++++++++++++ src/routeData.ts | 9 +++++ 6 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 src/integrations/sitemap-lastmod.js create mode 100644 src/integrations/sitemap-lastmod.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11b7e03f0..73fb4b081 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,6 +107,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Set up Node.js uses: actions/setup-node@v4 @@ -128,6 +130,9 @@ jobs: - name: Test heading variables run: npm run test:heading-vars + - name: Test sitemap lastmod + run: npm run test:sitemap-lastmod + - name: Build site run: npm run build diff --git a/astro.config.mjs b/astro.config.mjs index f2a468a8c..3ef1216b6 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -3,7 +3,7 @@ import { defineConfig, envField } from 'astro/config'; import remarkGfm from 'remark-gfm'; import mermaid from 'astro-mermaid'; import react from '@astrojs/react'; -import sitemap from '@astrojs/sitemap'; +import sitemap from './src/integrations/sitemap-lastmod.js'; import starlight from '@astrojs/starlight'; import starlightLlmsTxt from 'starlight-llms-txt'; import starlightSidebarTopics from 'starlight-sidebar-topics'; diff --git a/package.json b/package.json index c213700c7..88bb58eae 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "test:heading-tocs": "node scripts/validate-heading-tocs.mjs", "test:heading-vars": "node --test src/plugins/heading-vars-core.test.mjs", "test:homepage-json-ld": "node scripts/validate-homepage-json-ld.mjs", + "test:sitemap-lastmod": "node --test src/integrations/sitemap-lastmod.test.mjs", "postinstall": "patch-package" }, "engines": { diff --git a/src/integrations/sitemap-lastmod.js b/src/integrations/sitemap-lastmod.js new file mode 100644 index 000000000..44549f2b7 --- /dev/null +++ b/src/integrations/sitemap-lastmod.js @@ -0,0 +1,40 @@ +import sitemap from '@astrojs/sitemap'; +import { readFile } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; +import { parseHTML } from 'linkedom'; + +export default function sitemapLastmod() { + let outputRoot; + const integration = sitemap({ + serialize: async (item) => { + const pathname = decodeURIComponent(new URL(item.url).pathname); + const htmlPath = path.join(outputRoot, pathname, 'index.html'); + try { + return withLastmod(item, await readFile(htmlPath, 'utf8')); + } catch (error) { + // Routes without generated HTML have no Starlight git date. + if (error.code === 'ENOENT' || error.code === 'ENOTDIR') return item; + throw error; + } + }, + }); + return { + ...integration, + hooks: { + ...integration.hooks, + // Runs before @astrojs/sitemap's build:done serialization, while + // the adapter still exposes the generated HTML directory. + 'astro:build:generated': ({ dir }) => { + outputRoot = fileURLToPath(dir); + }, + }, + }; +} + +export function withLastmod(item, html) { + const { document } = parseHTML(html); + const value = document.querySelector('meta[property="article:modified_time"]')?.getAttribute('content'); + if (!value || !Number.isFinite(Date.parse(value))) return item; + return { ...item, lastmod: new Date(value).toISOString() }; +} diff --git a/src/integrations/sitemap-lastmod.test.mjs b/src/integrations/sitemap-lastmod.test.mjs new file mode 100644 index 000000000..8ba4ca6b8 --- /dev/null +++ b/src/integrations/sitemap-lastmod.test.mjs @@ -0,0 +1,21 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { withLastmod } from './sitemap-lastmod.js'; + +test('sitemap uses the exact rendered Starlight date, not build time', () => { + const item = { url: 'https://docs.warp.dev/factories/quickstart/' }; + for (const date of ['2026-10-08T17:42:12.000Z', '2025-03-01T09:00:00.000Z']) { + assert.deepEqual( + withLastmod(item, ``), + { ...item, lastmod: date }, + ); + } +}); + +test('pages without a valid Starlight date do not invent lastmod', () => { + const item = { url: 'https://docs.warp.dev/api/' }; + for (const html of ['', '', + '']) { + assert.equal(withLastmod(item, html), item); + } +}); diff --git a/src/routeData.ts b/src/routeData.ts index f5fc833a9..926e2cdff 100644 --- a/src/routeData.ts +++ b/src/routeData.ts @@ -16,6 +16,15 @@ import { getMarkdownPathFromHtmlPath } from './lib/docs-markdown.js'; export const onRequest = defineRouteMiddleware((context) => { const route = context.locals.starlightRoute; const head = route.head; + // Reuse Starlight's git-derived date, including its frontmatter overrides. + // The sitemap serializer reads this from the generated page, so it never + // substitutes build time or guesses a source file from a URL. + if (route.lastUpdated) { + head.push({ + tag: 'meta', + attrs: { property: 'article:modified_time', content: route.lastUpdated.toISOString() }, + }); + } head.push({ tag: 'link', From ba6b9fe1a519277a5dc893d77cc6c113c977cc02 Mon Sep 17 00:00:00 2001 From: "warp-agent-staging[bot]" <240773466+warp-agent-staging[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:15:27 +0000 Subject: [PATCH 2/4] Restore git history before computing page dates --- package.json | 2 +- scripts/ensure-git-history.mjs | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 scripts/ensure-git-history.mjs diff --git a/package.json b/package.json index 88bb58eae..19290105c 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "dev": "astro dev", "predev": "node scripts/generate-openapi-json.mjs", "build": "astro build", - "prebuild": "node scripts/generate-openapi-json.mjs", + "prebuild": "node scripts/ensure-git-history.mjs && node scripts/generate-openapi-json.mjs", "preview": "astro preview", "astro": "astro", "typecheck": "astro check", diff --git a/scripts/ensure-git-history.mjs b/scripts/ensure-git-history.mjs new file mode 100644 index 000000000..84bb58045 --- /dev/null +++ b/scripts/ensure-git-history.mjs @@ -0,0 +1,12 @@ +import { spawnSync } from 'node:child_process'; +import { performance } from 'node:perf_hooks'; + +// A shallow boundary commit makes unchanged files look newly modified to +// Starlight. Fetch history before Astro inlines git dates for pages/sitemaps. +const shallow = spawnSync('git', ['rev-parse', '--is-shallow-repository'], { encoding: 'utf8' }); +if (shallow.status === 0 && shallow.stdout.trim() === 'true') { + const start = performance.now(); + const fetch = spawnSync('git', ['fetch', '--unshallow', '--no-tags', 'origin'], { stdio: 'inherit' }); + if (fetch.status !== 0) throw new Error('Full git history is required for truthful page dates.'); + console.log(`Fetched full git history in ${((performance.now() - start) / 1000).toFixed(1)}s.`); +} From 571f5121b0dba3b091a3d9426af4b09f1f710e66 Mon Sep 17 00:00:00 2001 From: "warp-agent-staging[bot]" <240773466+warp-agent-staging[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:24:10 +0000 Subject: [PATCH 3/4] Omit clone-horizon dates when history recovery is unavailable --- package.json | 2 +- scripts/ensure-git-history.mjs | 17 +++++++++++++--- src/lib/git-history.js | 18 +++++++++++++++++ src/lib/git-history.test.mjs | 36 ++++++++++++++++++++++++++++++++++ src/routeData.ts | 7 +++++++ 5 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 src/lib/git-history.js create mode 100644 src/lib/git-history.test.mjs diff --git a/package.json b/package.json index 19290105c..a9b0f3377 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "test:heading-tocs": "node scripts/validate-heading-tocs.mjs", "test:heading-vars": "node --test src/plugins/heading-vars-core.test.mjs", "test:homepage-json-ld": "node scripts/validate-homepage-json-ld.mjs", - "test:sitemap-lastmod": "node --test src/integrations/sitemap-lastmod.test.mjs", + "test:sitemap-lastmod": "node --test src/integrations/sitemap-lastmod.test.mjs src/lib/git-history.test.mjs", "postinstall": "patch-package" }, "engines": { diff --git a/scripts/ensure-git-history.mjs b/scripts/ensure-git-history.mjs index 84bb58045..4ff811c4f 100644 --- a/scripts/ensure-git-history.mjs +++ b/scripts/ensure-git-history.mjs @@ -6,7 +6,18 @@ import { performance } from 'node:perf_hooks'; const shallow = spawnSync('git', ['rev-parse', '--is-shallow-repository'], { encoding: 'utf8' }); if (shallow.status === 0 && shallow.stdout.trim() === 'true') { const start = performance.now(); - const fetch = spawnSync('git', ['fetch', '--unshallow', '--no-tags', 'origin'], { stdio: 'inherit' }); - if (fetch.status !== 0) throw new Error('Full git history is required for truthful page dates.'); - console.log(`Fetched full git history in ${((performance.now() - start) / 1000).toFixed(1)}s.`); + const origin = spawnSync('git', ['remote', 'get-url', 'origin'], { encoding: 'utf8' }); + const { VERCEL_GIT_REPO_OWNER: owner, VERCEL_GIT_REPO_SLUG: slug } = process.env; + const publicRepo = owner && slug && /^[\w.-]+$/.test(owner) && /^[\w.-]+$/.test(slug) + ? `https://github.com/${owner}/${slug}.git` + : undefined; + const remote = origin.status === 0 ? 'origin' : publicRepo; + const fetch = remote + ? spawnSync('git', ['fetch', '--unshallow', '--no-tags', remote], { encoding: 'utf8', timeout: 120_000 }) + : undefined; + if (fetch?.status === 0) { + console.log(`Fetched full git history in ${((performance.now() - start) / 1000).toFixed(1)}s.`); + } else { + console.warn('Could not restore full git history; shallow-boundary page dates will be omitted.'); + } } diff --git a/src/lib/git-history.js b/src/lib/git-history.js new file mode 100644 index 000000000..7ed5a8bfb --- /dev/null +++ b/src/lib/git-history.js @@ -0,0 +1,18 @@ +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; + +// Boundary commits in a shallow checkout synthesize modifications for all +// files at the clone horizon. Their timestamps are not content-change dates. +export function getShallowBoundaryDates() { + const result = spawnSync('git', ['rev-parse', '--git-path', 'shallow'], { encoding: 'utf8' }); + if (result.status !== 0) return new Set(); + let commits; + try { + commits = readFileSync(result.stdout.trim(), 'utf8').trim().split('\n'); + } catch { + return new Set(); + } + if (!commits.every((commit) => /^[a-f0-9]{40,64}$/.test(commit))) return new Set(); + const dates = spawnSync('git', ['show', '--no-patch', '--format=%ct', ...commits], { encoding: 'utf8' }); + return new Set(dates.stdout.trim().split('\n').map((timestamp) => Number(timestamp) * 1000)); +} diff --git a/src/lib/git-history.test.mjs b/src/lib/git-history.test.mjs new file mode 100644 index 000000000..444c622c5 --- /dev/null +++ b/src/lib/git-history.test.mjs @@ -0,0 +1,36 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { getShallowBoundaryDates } from './git-history.js'; + +test('unavailable history is nonfatal and boundary dates are identifiable', () => { + const root = process.cwd(); + const repo = mkdtempSync(path.join(tmpdir(), 'docs-history-test-')); + const git = (...args) => { + const result = spawnSync('git', args, { cwd: repo, encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + return result.stdout.trim(); + }; + git('init'); + process.chdir(repo); + try { + assert.deepEqual(getShallowBoundaryDates(), new Set()); + git('commit', '--allow-empty', '-m', 'History fixture'); + const timestamp = Number(git('show', '--no-patch', '--format=%ct', 'HEAD')) * 1000; + writeFileSync(path.join(repo, '.git/shallow'), `${git('rev-parse', 'HEAD')}\n`); + assert.deepEqual(getShallowBoundaryDates(), new Set([timestamp])); + const result = spawnSync('node', [path.join(root, 'scripts/ensure-git-history.mjs')], { + cwd: repo, + encoding: 'utf8', + env: { ...process.env, VERCEL_GIT_REPO_OWNER: '', VERCEL_GIT_REPO_SLUG: '' }, + }); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stderr, /shallow-boundary page dates will be omitted/); + assert.ok(readFileSync(path.join(repo, '.git/shallow'), 'utf8').trim()); + } finally { + process.chdir(root); + } +}); diff --git a/src/routeData.ts b/src/routeData.ts index 926e2cdff..673114315 100644 --- a/src/routeData.ts +++ b/src/routeData.ts @@ -1,6 +1,9 @@ import { defineRouteMiddleware } from '@astrojs/starlight/route-data'; import config from 'virtual:starlight/user-config'; import { getMarkdownPathFromHtmlPath } from './lib/docs-markdown.js'; +import { getShallowBoundaryDates } from './lib/git-history.js'; + +const shallowBoundaryDates = getShallowBoundaryDates(); /** * Per-page head augmentation. @@ -15,6 +18,10 @@ import { getMarkdownPathFromHtmlPath } from './lib/docs-markdown.js'; */ export const onRequest = defineRouteMiddleware((context) => { const route = context.locals.starlightRoute; + if (!(route.entry.data.lastUpdated instanceof Date) && + route.lastUpdated && shallowBoundaryDates.has(route.lastUpdated.getTime())) { + route.lastUpdated = undefined; + } const head = route.head; // Reuse Starlight's git-derived date, including its frontmatter overrides. // The sitemap serializer reads this from the generated page, so it never From d7d98b910f60f24e49a97171badb09624c879b42 Mon Sep 17 00:00:00 2001 From: "warp-agent-staging[bot]" <240773466+warp-agent-staging[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:29:30 +0000 Subject: [PATCH 4/4] Reuse git objects in history test without creating commits --- src/lib/git-history.test.mjs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/lib/git-history.test.mjs b/src/lib/git-history.test.mjs index 444c622c5..77c2b4509 100644 --- a/src/lib/git-history.test.mjs +++ b/src/lib/git-history.test.mjs @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; -import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { mkdtempSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import test from 'node:test'; @@ -8,19 +8,22 @@ import { getShallowBoundaryDates } from './git-history.js'; test('unavailable history is nonfatal and boundary dates are identifiable', () => { const root = process.cwd(); - const repo = mkdtempSync(path.join(tmpdir(), 'docs-history-test-')); + const repo = path.join(mkdtempSync(path.join(tmpdir(), 'docs-history-test-')), 'repo'); + const init = spawnSync('git', ['init', repo], { encoding: 'utf8' }); + assert.equal(init.status, 0, init.stderr); + const objects = spawnSync('git', ['rev-parse', '--git-path', 'objects'], { encoding: 'utf8' }); + const head = spawnSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).stdout.trim(); + writeFileSync(path.join(repo, '.git/objects/info/alternates'), `${path.resolve(root, objects.stdout.trim())}\n`); const git = (...args) => { const result = spawnSync('git', args, { cwd: repo, encoding: 'utf8' }); assert.equal(result.status, 0, result.stderr); return result.stdout.trim(); }; - git('init'); process.chdir(repo); try { - assert.deepEqual(getShallowBoundaryDates(), new Set()); - git('commit', '--allow-empty', '-m', 'History fixture'); + git('update-ref', 'HEAD', head); + writeFileSync(path.join(repo, '.git/shallow'), `${head}\n`); const timestamp = Number(git('show', '--no-patch', '--format=%ct', 'HEAD')) * 1000; - writeFileSync(path.join(repo, '.git/shallow'), `${git('rev-parse', 'HEAD')}\n`); assert.deepEqual(getShallowBoundaryDates(), new Set([timestamp])); const result = spawnSync('node', [path.join(root, 'scripts/ensure-git-history.mjs')], { cwd: repo, @@ -30,6 +33,8 @@ test('unavailable history is nonfatal and boundary dates are identifiable', () = assert.equal(result.status, 0, result.stderr); assert.match(result.stderr, /shallow-boundary page dates will be omitted/); assert.ok(readFileSync(path.join(repo, '.git/shallow'), 'utf8').trim()); + unlinkSync(path.join(repo, '.git/shallow')); + assert.deepEqual(getShallowBoundaryDates(), new Set()); } finally { process.chdir(root); }