diff --git a/app/llms.mdx/[[...slug]]/route.ts b/app/llms.mdx/[[...slug]]/route.ts index c7d923bf..901bd14c 100644 --- a/app/llms.mdx/[[...slug]]/route.ts +++ b/app/llms.mdx/[[...slug]]/route.ts @@ -1,4 +1,3 @@ -import { notFound } from 'next/navigation'; import { type NextRequest, NextResponse } from 'next/server'; import { getLLMText, shouldIncludeLLMPage } from '@/lib/get-llm-text'; import { appendMarkdownVaryHeader } from '@/lib/markdown-negotiation'; @@ -6,6 +5,16 @@ import { source } from '@/lib/source'; export const revalidate = false; +const MARKDOWN_NOT_FOUND = `# Page not found + +The requested Steel documentation page does not exist or is unavailable in this representation. + +- [Documentation home](/) +- [Documentation index](/llms.txt) +- [Sitemap](/sitemap.xml) +- [API catalog](/.well-known/api-catalog) +`; + function getPage(slug?: string[]) { let page = source.getPage(slug); if (!page && slug?.[0] !== 'en') { @@ -15,24 +24,34 @@ function getPage(slug?: string[]) { return page; } +function getMarkdownHeaders(): Headers { + const headers = new Headers({ + 'Content-Type': 'text/markdown; charset=utf-8', + 'X-Robots-Tag': 'noindex', + }); + appendMarkdownVaryHeader(headers); + return headers; +} + +function getNotFoundResponse(): NextResponse { + return new NextResponse(MARKDOWN_NOT_FOUND, { + status: 404, + headers: getMarkdownHeaders(), + }); +} + export async function GET(_req: NextRequest, { params }: { params: Promise<{ slug?: string[] }> }) { const { slug } = await params; const page = getPage(slug); - if (!page) notFound(); + if (!page) return getNotFoundResponse(); // Pages opted out of LLM surfaces (llm: false) are not served as markdown. - if (!shouldIncludeLLMPage(page)) notFound(); + if (!shouldIncludeLLMPage(page)) return getNotFoundResponse(); // This markdown duplicates the canonical HTML page, so keep it out of search // results. Crawlers may still fetch it: noindex only suppresses indexing. - const headers = new Headers({ - 'Content-Type': 'text/markdown; charset=utf-8', - 'X-Robots-Tag': 'noindex', - }); - appendMarkdownVaryHeader(headers); - return new NextResponse(await getLLMText(page, { indexPointer: true }), { - headers, + headers: getMarkdownHeaders(), }); } diff --git a/plans/009-make-markdown-404s-agent-recoverable.md b/plans/009-make-markdown-404s-agent-recoverable.md new file mode 100644 index 00000000..45252a1f --- /dev/null +++ b/plans/009-make-markdown-404s-agent-recoverable.md @@ -0,0 +1,315 @@ +# Plan 009: Make Markdown 404 responses agent-recoverable + +> **Executor instructions**: Follow this plan step by step. Run every +> verification command and confirm the expected result before moving to the +> next step. If anything in the "STOP conditions" section occurs, stop and +> report—do not improvise. When done, update the status row for this plan in +> `plans/README.md` unless a reviewer told you they maintain the index. +> +> **Drift check (run first)**: +> `git diff --stat a5d319f9..HEAD -- 'app/llms.mdx/[[...slug]]/route.ts' tests/e2e/llm-endpoints.test.ts` +> Stop if any other change altered missing-page handling, the `llm: false` +> exclusion, or the existing 404 tests. + +## Status + +- **Priority**: P2 +- **Effort**: S +- **Risk**: LOW +- **Depends on**: 002 +- **Category**: bug +- **Planned at**: commit `a5d319f9`, 2026-08-22 +- **Execution status**: DONE + +## Execution evidence (2026-08-22; clean branch reverified 2026-08-24) + +- `bun run test` passed: 335 tests, 0 failures. +- `bun run typecheck`, `bun run check`, `bun run validate-links`, + `bun run build`, and `git diff --check` passed. +- Local production probes confirmed recovery bodies for negotiated Markdown, + explicit `.md`, and `llm: false` paths; the branded browser HTML 404 remained + unchanged; HEAD returned status and headers without a body. +- Generated `/llms.txt`, `/llms-full.txt`, `/AGENTS.md`, `/DESIGN.md`, + `/.well-known/api-catalog`, `/.well-known/agent-skills/index.json`, + `/sitemap.xml`, `/robots.txt`, and `/openapi.json` were verified against the + built local server. +- Preview deployment `dpl_6BFhKs1vSKpF8BCsgsXaimfhiTT9` built successfully at + `https://docs-j2duyxy0g-nen-labs.vercel.app` in the `nen-labs/docs` project. +- Authenticated browser probes against the preview passed explicit and + negotiated Markdown 404s. Each returned status 404, a 244-character recovery + body, `text/markdown`, and `Vary` containing `Accept` and `User-Agent`. +- The preview HTML 404 remained HTML, and the Markdown `HEAD` response returned + status 404, the negotiated headers, and a zero-byte body. The manual probe's + displayed failures for those two rows came from applying recovery-body and + negotiation-header assertions to representations where they do not apply. + +## Why this matters + +The site already returns the correct HTTP 404 status for nonexistent HTML and +Markdown paths. The HTML response is useful, but the Markdown representation +has a zero-byte body. An agent that follows a stale or mistyped docs link learns +only that retrieval failed; it receives no index or sitemap from which to +recover. + +A live probe on 2026-08-22 confirmed the defect: + +```text +$ curl -sS -D - -H 'Accept: text/markdown' \ + https://docs.steel.dev/this-page-does-not-exist-agent-audit-20260822 +HTTP/2 404 +content-length: 0 +x-matched-path: /llms.mdx/[[...slug]] +``` + +The fix must preserve the real 404 status and the existing branded HTML 404. +Only the Markdown-backed representation should gain a short recovery body. + +## Current state + +- `app/llms.mdx/[[...slug]]/route.ts` serves Markdown renditions. Both a missing + source page and a page with `llm: false` call Next.js `notFound()`. +- `app/not-found.tsx` already gives browser users a branded HTML 404 with links + to the quickstart, Steel Skills, and `llms.txt`. It is out of scope. +- `tests/e2e/llm-endpoints.test.ts` boots a Next.js dev server and already tests + explicit `.md` URLs. Its two 404 cases assert only the status code. +- The successful Markdown response already defines the representation headers. + The 404 response must reuse that policy instead of defining a second one. + +Current missing-page control flow in +`app/llms.mdx/[[...slug]]/route.ts:18-24`: + +```ts +export async function GET(_req: NextRequest, { params }: { params: Promise<{ slug?: string[] }> }) { + const { slug } = await params; + const page = getPage(slug); + if (!page) notFound(); + + // Pages opted out of LLM surfaces (llm: false) are not served as markdown. + if (!shouldIncludeLLMPage(page)) notFound(); +``` + +Current regression tests in `tests/e2e/llm-endpoints.test.ts:480-492`: + +```ts +test('returns 404 for a .md URL with no matching page', async () => { + const response = await fetch(`${BASE_URL}/nonexistent-page.md`, { + headers: BROWSER_HEADERS, + }); + expect(response.status).toBe(404); +}); + +test('returns 404 for an llm: false page at its .md URL', async () => { + const response = await fetch(`${BASE_URL}/cookbook/authors/hussufo.md`, { + headers: BROWSER_HEADERS, + }); + expect(response.status).toBe(404); +}); +``` + +## Target behavior + +| Request | Status | Content type | Body | +|---------|--------|--------------|------| +| Missing path with `Accept: text/markdown` | 404 | `text/markdown` | Recovery Markdown | +| Missing explicit `.md` path | 404 | `text/markdown` | Recovery Markdown | +| Existing `llm: false` page requested as `.md` | 404 | `text/markdown` | Same generic body; no page details | +| Missing browser HTML path | 404 | `text/html` | Existing branded HTML page unchanged | +| `HEAD` for a missing negotiated path | 404 | Negotiated type | Empty body, normal HEAD semantics | + +The recovery body must be concise and contain all of these relative links: + +- `/` — documentation home +- `/llms.txt` — agent-readable documentation index +- `/sitemap.xml` — complete URL inventory +- `/.well-known/api-catalog` — API description and documentation discovery + +It must not echo the requested path, reveal whether an `llm: false` page exists, +or contain an HTML document shell. + +## Commands you will need + +| Purpose | Command | Expected on success | +|---------|---------|---------------------| +| Install | `bun install --frozen-lockfile` | exit 0 | +| Runtime tests | `bun test tests/e2e/llm-endpoints.test.ts` | all tests pass | +| Full tests | `bun run test` | all tests pass | +| Typecheck | `bun run typecheck` | exit 0, no errors | +| Lint/format check | `bun run check` | exit 0, no warnings | +| Link validation | `bun run validate-links` | exit 0 | +| Production build | `bun run build` | exit 0 | +| Diff hygiene | `git diff --check` | no output | + +## Scope + +**In scope** (the only implementation and test files to modify): + +- `app/llms.mdx/[[...slug]]/route.ts` +- `tests/e2e/llm-endpoints.test.ts` +- `plans/README.md` for the final status update + +**Out of scope**: + +- `app/not-found.tsx` and the visual HTML 404 design +- `middleware.ts` and the set of negotiable paths +- `lib/markdown-negotiation.ts` and Accept ranking +- New `/about`, `/contact`, `/developers`, or `/privacy` pages +- API behavior on `api.steel.dev`, including JSON errors, rate-limit headers, + versioning, deprecation, and sunset policy +- Organization address, telephone, or contact schema +- Cache-policy changes or redirects + +## Git workflow + +- Branch: `plan/agent-readiness-audit` +- One logical Conventional Commit: + `fix(llms): add recovery links to markdown 404s` +- Do not push or open a PR unless the operator instructs it. + +## Steps + +### Step 1: Expand the existing 404 tests before changing the route + +In `tests/e2e/llm-endpoints.test.ts`, extend the two existing `.md` 404 cases +and add negotiated canonical-path, browser HTML, and `HEAD` cases from the +target matrix. + +For the Markdown responses, assert: + +- status is exactly 404; +- `Content-Type` matches the negotiated representation; +- `X-Robots-Tag` contains `noindex`; +- the body starts with `# Page not found`; +- all four recovery links are present; +- `` below and run: + +```bash +curl -si -H 'Accept: text/markdown' https:///nonexistent-agent-page +curl -si https:///nonexistent-agent-page.md +curl -si -H 'Accept: text/html' https:///nonexistent-agent-page +curl -sI -H 'Accept: text/markdown' https:///nonexistent-agent-page +``` + +Confirm the status, content type, body, and HEAD behavior against the target +matrix. The Markdown response's delivered `Vary` must contain `Accept` and +`User-Agent`. Repeat the first and third probes once to exercise a potential CDN +cache hit; each must retain its requested representation. + +## Test plan + +- Extend `tests/e2e/llm-endpoints.test.ts`; do not introduce a second test + harness for the route. +- Cover explicit `.md`, canonical Accept negotiation, browser HTML, `HEAD`, and + the existing `llm: false` exclusion. +- Assert semantics and headers rather than the complete error string so minor + copy edits do not make the test brittle. +- Retain every existing test and assertion. + +## Done criteria + +- [x] Missing negotiated and explicit `.md` paths return status 404 with a + non-empty recovery body and the correct content type. +- [x] The recovery body contains `/`, `/llms.txt`, `/sitemap.xml`, and + `/.well-known/api-catalog` and contains no HTML shell. +- [x] An `llm: false` page returns the same generic 404 without disclosing its + title or content. +- [x] Browser HTML 404 appearance and behavior are unchanged. +- [x] `HEAD` returns 404 with the negotiated headers and no body. +- [x] Preview/CDN probes preserve the correct representation on repeated calls. +- [x] `bun run test`, `bun run typecheck`, `bun run check`, + `bun run validate-links`, and `bun run build` all pass. +- [x] `git diff --check` emits no output. +- [x] No files outside the in-scope list are modified. +- [x] `plans/README.md` marks Plan 009 DONE. + +## STOP conditions + +Stop and report rather than improvising if any of these occur: + +- The route no longer handles missing pages through the two branches shown in + "Current state". +- A custom 404 response changes the status from 404, loses `X-Robots-Tag`, or + causes a missing browser path to receive Markdown without requesting it. +- Returning the local 404 response causes `generateStaticParams` or the build + route count to change materially. +- The fix requires modifying middleware, the HTML 404, or another out-of-scope + file. +- A verification command fails twice after one reasonable correction. + +## Maintenance notes + +- Keep the recovery links stable and machine-readable. If the canonical agent + index or API catalog URL changes, update this body and its E2E assertions in + the same change. +- Reviewers should confirm that the `llm: false` response remains + indistinguishable from a missing page. +- This plan does not establish API-wide error conventions. JSON API errors and + rate-limit metadata belong to the API service and require a separate owner. diff --git a/plans/README.md b/plans/README.md index 79b9a350..1c75b6f8 100644 --- a/plans/README.md +++ b/plans/README.md @@ -10,6 +10,9 @@ established the homepage's visible semantics and answerable introduction. Plan 008 was added from commit `92581223` after live probes of `https://docs.steel.dev` confirmed three negotiation defects and disproved two findings from an external agent-readiness scan. +Plan 009 was added from commit `a5d319f9` after an agent-readiness review +confirmed that missing Markdown pages returned real 404 statuses with empty +bodies, leaving agents without recovery links. ## Execution order & status @@ -21,6 +24,7 @@ findings from an external agent-readiness scan. | 006 | Make `/` the canonical docs homepage | P2 | S | 002 | DONE | | 007 | Complete the structured-data entity graph | P2 | M | 006, PR #98 | DONE | | 008 | Deliver a correct content-negotiation contract | P1 | M | 002, 006 | TODO | +| 009 | Make Markdown 404 responses agent-recoverable | P2 | S | 002 | DONE | | 004 | Make the robots test parser faithful | P3 | S | — | DONE | | 005 | Bound CI runtime and preserve the spawn flake investigation | P3 | S | — | IN PROGRESS | @@ -40,6 +44,8 @@ REJECTED (with one-line rationale) those plans settled. It touches `lib/markdown-negotiation.ts`, which both earlier plans deliberately left alone, and it must not reopen the crawler-gets-HTML decision they made. +- Plan 009 follows Plan 002's existing Markdown route and negotiation policy but + does not depend on the broader refinements proposed by Plan 008. ## Findings considered and rejected diff --git a/tests/e2e/llm-endpoints.test.ts b/tests/e2e/llm-endpoints.test.ts index 59b0e20c..878dcde7 100644 --- a/tests/e2e/llm-endpoints.test.ts +++ b/tests/e2e/llm-endpoints.test.ts @@ -41,6 +41,23 @@ function varyTokens(response: Response): string[] { .filter(Boolean); } +async function expectAgentRecoverable404(response: Response): Promise { + expect(response.status).toBe(404); + expect(response.headers.get('content-type')).toStartWith('text/markdown'); + expect(response.headers.get('x-robots-tag')).toContain('noindex'); + expect(varyTokens(response)).toEqual(expect.arrayContaining(['accept', 'user-agent'])); + + const body = await response.text(); + expect(body).toStartWith('# Page not found'); + expect(body).toContain('[Documentation home](/)'); + expect(body).toContain('[Documentation index](/llms.txt)'); + expect(body).toContain('[Sitemap](/sitemap.xml)'); + expect(body).toContain('[API catalog](/.well-known/api-catalog)'); + expect(body).not.toContain(' { expect(response.headers.get('location')).toBe('https://api.steel.dev/sdk-openapi.json'); }); - test('returns 404 for a .md URL with no matching page', async () => { + test('returns an agent-recoverable 404 for a .md URL with no matching page', async () => { const response = await fetch(`${BASE_URL}/nonexistent-page.md`, { headers: BROWSER_HEADERS, }); - expect(response.status).toBe(404); + await expectAgentRecoverable404(response); }); - test('returns 404 for an llm: false page at its .md URL', async () => { + test('returns an agent-recoverable 404 for negotiated missing pages', async () => { + const markdown = await fetch(`${BASE_URL}/nonexistent-negotiated-page`, { + headers: { accept: 'text/markdown', 'user-agent': 'curl/8.7.1' }, + }); + await expectAgentRecoverable404(markdown); + }); + + test('returns the generic recovery 404 for an llm: false page at its .md URL', async () => { const response = await fetch(`${BASE_URL}/cookbook/authors/hussufo.md`, { headers: BROWSER_HEADERS, }); + const body = await expectAgentRecoverable404(response); + expect(body).not.toContain('Hussien Hussien'); + expect(body).not.toContain('4 recipes contributed'); + }); + + test('keeps the branded HTML 404 for browsers', async () => { + const response = await fetch(`${BASE_URL}/nonexistent-browser-page`, { + headers: BROWSER_HEADERS, + }); + expect(response.status).toBe(404); + expect(response.headers.get('content-type')).toStartWith('text/html'); + expect(await response.text()).toContain('Page not found'); + }); + + test('returns negotiated 404 headers without a body on HEAD', async () => { + const response = await fetch(`${BASE_URL}/nonexistent-head-page`, { + method: 'HEAD', + headers: { accept: 'text/markdown', 'user-agent': 'curl/8.7.1' }, + }); expect(response.status).toBe(404); + expect(response.headers.get('content-type')).toStartWith('text/markdown'); + expect(response.headers.get('x-robots-tag')).toContain('noindex'); + expect(await response.text()).toBe(''); }); test('still serves HTML at the canonical URL for browsers', async () => {