diff --git a/.github/scripts/live-route-compare.mjs b/.github/scripts/live-route-compare.mjs new file mode 100644 index 0000000..171062d --- /dev/null +++ b/.github/scripts/live-route-compare.mjs @@ -0,0 +1,276 @@ +#!/usr/bin/env node +/** + * live-route-compare.mjs — the pure three-way comparison. No network, no filesystem, no + * process.exit. Probe results are an INPUT here, not a side effect, so the entire verdict is + * testable offline against hand-built fixtures. + * + * ── THE TWO FINDINGS ──────────────────────────────────────────────────────────────────────────── + * live-undeclared The probe says the route is MAPPED, and it appears in NEITHER openapi.yaml + * nor the published contract. This is the direction `published-drift.mjs` + * structurally cannot see, because that gate compares those two documents to + * each other: public surface that no artifact describes and nobody reviewed as + * public surface. + * declared-not-live Declared in openapi.yaml WITHOUT `x-schema-status: draft` — stated as a + * promise to consumers — but the gateway answers ROUTE_NOT_MAPPED. The two + * documents can agree with each other and both be wrong; only the live probe + * can say so. + * + * `draft` suppresses the second direction for the same reason it does in `published-drift-compare.mjs`: + * it is the spec's own statement that a shape is a placeholder rather than a promise. It does NOT + * suppress the first — an undeclared live route is a finding no matter what the spec says about + * anything else, because the spec is not what is serving traffic. + */ +import { ABSENT, INDETERMINATE, MAPPED } from './live-route-probe.mjs'; + +const HTTP_METHODS = ['get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace']; + +export const DIRECTIONS = ['live-undeclared', 'declared-not-live']; + +/** `/clips` + server base `/v1` -> `/v1/clips`. Both specs declare paths relative to servers[0]. */ +export function basePath(doc) { + const url = doc?.servers?.[0]?.url; + if (!url) return ''; + try { + // A relative server url such as `/v1` is legal in OpenAPI 3; resolve it against a dummy base + // instead of letting `new URL('/v1')` throw and silently fall back to '' (which would drop the + // base off every candidate path and every probed path). + return new URL(url, 'https://base.invalid').pathname.replace(/\/$/, ''); + } catch { + return ''; + } +} + +/** Templated paths cannot be probed: a made-up id would test the id, not the route. */ +export function isProbeable(path) { + return !path.includes('{'); +} + +/** `/v1/render/{jobId}` -> `v1/render`, the product-level key the live enumerators are keyed at. */ +export function segmentKey(path) { + return path.split('/').filter(Boolean).slice(0, 2).join('/'); +} + +/** + * True when EVERY operation on this path item declares its own `servers` override — meaning the + * path is not actually served at the document's own base and must never be probed or compared + * against it. openapi.yaml uses this for the Realtime API: `/realtime/connect` and friends are + * documented under the main document (base `/v1`) but are annotated `servers: [{url: + * https://realtime.wave.online}]` and served there, not at `api.wave.online/v1/realtime/connect`. + * Without this exclusion the live prober — which only ever queries the document's own origin — + * would probe the wrong host, get back 403 ROUTE_NOT_MAPPED, and file a false `declared-not-live` + * finding against an endpoint that is live, just live somewhere else. + */ +export function hasOwnServerOverride(item) { + const ops = Object.entries(item ?? {}).filter(([m]) => HTTP_METHODS.includes(m.toLowerCase())); + return ops.length > 0 && ops.every(([, op]) => Array.isArray(op?.servers) && op.servers.length > 0); +} + +/** + * Is this path inside the CONTRACT'S DOMAIN OF DISCOURSE? + * + * openapi.yaml declares a contract for `https://api.wave.online/v1`. The gateway also serves + * `/robots.txt`, `/favicon.svg`, `/llms.txt`, `/health` and friends, which are live, public, and + * correctly absent from an API contract — no OpenAPI document would ever declare them. Reporting + * them as "undeclared live routes" would be 7 false findings sitting on top of the real ones, and a + * gate people learn to skim is a gate that stops working. + * + * This is a SCOPE rule, not a suppression: it is keyed on the spec's own `servers[0]` base, so any + * route under `/v1/` — the entire API surface, including anything new — is always in scope and can + * never be excluded by it. If the base is empty (a spec with no servers block) everything is in + * scope, which is the fail-closed direction. + */ +export function withinSpecBase(path, base) { + if (!base) return true; + return path === base || path.startsWith(`${base}/`); +} + +/** Union of every candidate path worth probing, from all five enumerators. */ +export function candidatePaths({ repoDoc, publishedDoc, scopeCatalog, capabilityIndex, seeds }) { + const out = new Set(); + const add = (p) => { + if (typeof p === 'string' && p.startsWith('/') && isProbeable(p)) out.add(p); + }; + const repoBase = basePath(repoDoc); + for (const [p, item] of Object.entries(repoDoc?.paths ?? {})) { + if (hasOwnServerOverride(item)) continue; // served at a different host — see hasOwnServerOverride + add(`${repoBase}${p}`); + } + const pubBase = basePath(publishedDoc); + for (const [p, item] of Object.entries(publishedDoc?.paths ?? {})) { + if (hasOwnServerOverride(item)) continue; + add(`${pubBase}${p}`); + } + for (const r of scopeCatalog?.routes ?? []) add(r?.path); + for (const p of scopeCatalog?.no_scope_required?.paths ?? []) add(p); + for (const s of Object.values(capabilityIndex ?? {})) add(s?.path); + for (const s of seeds ?? []) add(s?.path); + return [...out].sort(); +} + +/** + * What `published-drift.mjs` can see, reproduced in four lines for ONE purpose: the test suite + * asserts that this returns ZERO findings on an input where `compareAgainstLive` returns one. That + * is the mutation proof for this whole feature — remove the live probe and the answer silently + * becomes "no drift". Exported so the claim is executable rather than a comment someone can drift + * away from. + */ +export function twoArtifactDriftOnly({ repoDoc, publishedDoc }) { + const repoBase = basePath(repoDoc); + const pubBase = basePath(publishedDoc); + const repo = new Set(Object.keys(repoDoc?.paths ?? {}).map((p) => `${repoBase}${p}`)); + const pub = new Set(Object.keys(publishedDoc?.paths ?? {}).map((p) => `${pubBase}${p}`)); + return [...pub].filter((p) => !repo.has(p)).concat([...repo].filter((p) => !pub.has(p))); +} + +export function compareAgainstLive({ repoDoc, publishedDoc, probes, allowlist = [] }) { + const repoBase = basePath(repoDoc); + const pubBase = basePath(publishedDoc); + // Paths served at their own `servers` override (e.g. the Realtime API at realtime.wave.online) are + // excluded here too: they were never added as candidates against THIS host (see candidatePaths), + // and they must never be treated as "declared" for a probe that landed on this host by coincidence, + // nor compared against a probe result that could only ever come from the wrong origin. + const repoPaths = new Map( + Object.entries(repoDoc?.paths ?? {}) + .filter(([, item]) => !hasOwnServerOverride(item)) + .map(([p, item]) => [`${repoBase}${p}`, item]), + ); + const pubPaths = new Set( + Object.entries(publishedDoc?.paths ?? {}) + .filter(([, item]) => !hasOwnServerOverride(item)) + .map(([p]) => `${pubBase}${p}`), + ); + + // Segment-level coverage, because the live enumerators are product-granular: the capability index + // lists `/v1/voice` while the spec documents `/voice/voices` and `/voice/generate`. Calling the + // product root undeclared would bury the one real finding under a dozen false ones, and a gate + // nobody can read is a gate nobody acts on. + const repoSegs = new Set([...repoPaths.keys()].map(segmentKey)); + const pubSegs = new Set([...pubPaths].map(segmentKey)); + + const allowByKey = new Map(allowlist.map((e) => [`${e.direction} ${e.path}`, e])); + const usedAllowKeys = new Set(); + const findings = []; + const allowlisted = []; + const indeterminate = []; + const outOfScope = []; + + const record = (direction, path, detail) => { + const key = `${direction} ${path}`; + const allow = allowByKey.get(key); + if (allow) { + usedAllowKeys.add(key); + allowlisted.push({ direction, path, ...detail, justification: allow.justification }); + return; + } + findings.push({ direction, path, ...detail }); + }; + + for (const [path, probe] of probes) { + if (probe.state === INDETERMINATE) { + // Neither a pass nor a finding. Surfaced so a run that could not read half the surface can + // never masquerade as a clean one. + indeterminate.push({ path, reason: probe.error ?? `HTTP ${probe.status}` }); + continue; + } + if (!withinSpecBase(path, repoBase)) { + outOfScope.push({ path, reason: `outside the spec's server base ${repoBase}` }); + continue; + } + // The segment fallback covers ONLY the product-root case: the probed path IS its own two-segment + // key (e.g. `/v1/voice` probed against declarations at `/v1/voice/voices`). It must never cover a + // DEEPER live path that merely shares a declared prefix (`/v1/clips/export-all` sharing `v1/clips` + // with the declared `/v1/clips`) — that direction is exactly the undeclared-sub-route drift this + // gate exists to catch, and truncating it away would make it invisible again. + const key = segmentKey(path); + const isProductRoot = path === `/${key}`; + const declaredRepo = repoPaths.has(path) || (isProductRoot && repoSegs.has(key)); + const declaredPub = pubPaths.has(path) || (isProductRoot && pubSegs.has(key)); + + if (probe.state === MAPPED && !declaredRepo && !declaredPub) { + record('live-undeclared', path, { + severity: 'security-relevant', + status: probe.status, + note: + 'live on the gateway and absent from BOTH openapi.yaml and the published contract — public surface no artifact ' + + 'describes. published-drift.mjs cannot see this: it compares those two documents to each other.', + }); + continue; + } + + if (probe.state === ABSENT) { + const item = repoPaths.get(path); + if (!item) continue; // not declared here and not live: nothing to say + const ops = Object.entries(item).filter(([m]) => HTTP_METHODS.includes(m.toLowerCase())); + const promised = ops.filter(([, op]) => op?.['x-schema-status'] !== 'draft'); + if (!promised.length) continue; + + // THE PROBE IS A GET, SO IT CAN ONLY JUDGE A GET. The gateway's scope map is keyed by route + // AND method, so a POST-only route answers ROUTE_NOT_MAPPED to a GET while its POST is + // perfectly live. Reporting that as "declared but not served" would be this gate committing + // the very error it exists to prevent — asserting a fact about production that its evidence + // does not support. MEASURED: /v1/agent/auth/device and /v1/agent/auth/token are POST-only + // OAuth device-grant routes; a GET to each returns 403 ROUTE_NOT_MAPPED, and an earlier draft + // of this file reported both as findings. They are not findings. + // + // This does NOT quietly pass them. An unverifiable claim is INDETERMINATE and is surfaced as + // such — unknown is not a pass. Closing the gap properly means probing the declared method, + // which for a POST means a write, which this gate will not do. + if (!promised.some(([m]) => m.toLowerCase() === 'get')) { + indeterminate.push({ + path, + reason: `declares only ${promised.map(([m]) => m.toUpperCase()).sort().join('/')} — a GET probe cannot establish whether that method is served`, + }); + continue; + } + record('declared-not-live', path, { + severity: 'contract-ahead', + status: probe.status, + methods: promised.map(([m]) => m.toUpperCase()).sort(), + note: + 'declared in openapi.yaml without x-schema-status: draft, but the gateway answers ROUTE_NOT_MAPPED — the two ' + + 'documents may agree with each other and both still be wrong; only the live probe can tell.', + }); + } + } + + const unmatchedAllowlist = [...allowByKey.keys()].filter((k) => !usedAllowKeys.has(k)).sort(); + findings.sort((a, b) => `${a.direction} ${a.path}`.localeCompare(`${b.direction} ${b.path}`)); + + const count = (d) => findings.filter((f) => f.direction === d).length; + return { + headline: { + probed: probes.size, + mapped: [...probes.values()].filter((p) => p.state === MAPPED).length, + absent: [...probes.values()].filter((p) => p.state === ABSENT).length, + indeterminate: indeterminate.length, + outOfScope: outOfScope.length, + repoDeclaredPaths: repoPaths.size, + publishedPaths: pubPaths.size, + liveUndeclared: count('live-undeclared'), + declaredNotLive: count('declared-not-live'), + allowlisted: allowlisted.length, + unmatchedAllowlistEntries: unmatchedAllowlist.length, + }, + findings, + allowlisted, + indeterminate, + outOfScope, + unmatchedAllowlist, + }; +} + +/** Returns an error string, or null when the allowlist is well-formed. */ +export function validateAllowlist(allowlist) { + if (!Array.isArray(allowlist)) return `allowlist is not an array (got ${typeof allowlist})`; + const seen = new Set(); + for (const e of allowlist) { + if (!e || typeof e.path !== 'string') return `allowlist entry needs a string path: ${JSON.stringify(e)}`; + if (!DIRECTIONS.includes(e.direction)) return `allowlist entry ${e.path} has an unknown direction ${JSON.stringify(e.direction)}`; + if (typeof e.justification !== 'string' || e.justification.trim().length < 20) + return `allowlist entry ${e.path} needs a real justification (>=20 chars)`; + const key = `${e.direction} ${e.path}`; + if (seen.has(key)) return `duplicate allowlist entry for ${key}`; + seen.add(key); + } + return null; +} diff --git a/.github/scripts/live-route-drift-allowlist.json b/.github/scripts/live-route-drift-allowlist.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/.github/scripts/live-route-drift-allowlist.json @@ -0,0 +1 @@ +[] diff --git a/.github/scripts/live-route-drift-regressions.test.mjs b/.github/scripts/live-route-drift-regressions.test.mjs new file mode 100644 index 0000000..b0633dd --- /dev/null +++ b/.github/scripts/live-route-drift-regressions.test.mjs @@ -0,0 +1,126 @@ +// live-route-drift-regressions.test.mjs — offline. No network. Split out of +// live-route-drift.test.mjs (a real seam: that file is the feature's original test suite; this one +// is regressions for findings fixed after the fact) so neither file grows unbounded. +// +// Covers, in order: +// 1. basePath must resolve a relative OpenAPI `servers[0].url` instead of throwing. +// 2. The segment fallback must cover ONLY the product-root case, never a deeper undeclared +// sub-route that happens to share a declared prefix. +// 3. An operation with its own `servers` override (e.g. the Realtime API at +// realtime.wave.online) must never be probed or compared against this document's own base. +// 4. decideExit must not report EXIT_OK when the live surface could not actually be observed. +// 5. enumeratorShapeError must reject a malformed 200 rather than silently enumerating zero +// routes from it. +// 6. parseArgs must not mistake --out's value for the spec when the caller omits the spec. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { ABSENT, MAPPED } from './live-route-probe.mjs'; +import { basePath, candidatePaths, compareAgainstLive, hasOwnServerOverride } from './live-route-compare.mjs'; +import { decideExit, enumeratorShapeError, parseArgs, EXIT_OK, EXIT_UNKNOWN, EXIT_DRIFT } from './live-route-drift.mjs'; + +const SERVERS = [{ url: 'https://api.wave.online/v1' }]; +const probeMap = (entries) => new Map(entries.map((e) => [e.path, e])); + +// ─── basePath / relative server urls. ───────────────────────────────────────────────────────────── + +test('basePath resolves a relative server url instead of throwing and silently returning empty', () => { + assert.equal(basePath({ servers: [{ url: '/v1' }] }), '/v1', "a relative server url is legal OpenAPI 3 and must not collapse to ''"); + assert.equal(basePath({ servers: [{ url: 'https://api.wave.online/v1' }] }), '/v1'); +}); + +// ─── segment fallback scope. ─────────────────────────────────────────────────────────────────────── + +test('the segment fallback covers only the product-root case, never a deeper undeclared sub-route', () => { + const repo = { servers: SERVERS, paths: { '/clips': { get: {} } } }; + const r = compareAgainstLive({ + repoDoc: repo, + publishedDoc: repo, + probes: probeMap([ + { path: '/v1/clips', state: MAPPED, status: 402 }, + { path: '/v1/clips/export-all', state: MAPPED, status: 200 }, + ]), + }); + assert.equal(r.findings.length, 1, 'a deeper undeclared sub-route under a declared product root must still be a finding'); + assert.equal(r.findings[0].path, '/v1/clips/export-all'); + assert.equal(r.findings[0].direction, 'live-undeclared'); +}); + +// ─── cross-host operations (per-operation `servers` override). ──────────────────────────────────── + +test('hasOwnServerOverride is true only when every operation on the path declares its own servers', () => { + assert.equal(hasOwnServerOverride({ get: { operationId: 'a' } }), false); + assert.equal(hasOwnServerOverride({ get: { servers: [{ url: 'https://realtime.wave.online' }] } }), true); + assert.equal( + hasOwnServerOverride({ get: { servers: [{ url: 'https://realtime.wave.online' }] }, post: { operationId: 'b' } }), + false, + 'a mixed path item (one overridden method, one not) is still served at the document base for the other method', + ); + assert.equal(hasOwnServerOverride({}), false); +}); + +test('an operation-level servers override is excluded from candidates and from the API-host comparison', () => { + // openapi.yaml declares /realtime/connect (base /v1) but annotates it `servers: [{url: + // https://realtime.wave.online}]` — served at a different host entirely, never at + // api.wave.online/v1/realtime/connect. Probing or comparing it as if it used the document base + // would file a false declared-not-live finding the moment the wrong-host probe comes back absent. + const repo = { + servers: SERVERS, + paths: { + '/clips': { get: { operationId: 'listClips' } }, + '/realtime/connect': { get: { operationId: 'realtimeConnect', servers: [{ url: 'https://realtime.wave.online' }] } }, + }, + }; + const c = candidatePaths({ repoDoc: repo, publishedDoc: { servers: SERVERS, paths: {} } }); + assert.ok(!c.includes('/v1/realtime/connect'), "a cross-host operation must never be probed against this document's base"); + + const r = compareAgainstLive({ + repoDoc: repo, + publishedDoc: { servers: SERVERS, paths: {} }, + probes: probeMap([{ path: '/v1/realtime/connect', state: ABSENT, status: 403 }]), + }); + assert.equal(r.findings.length, 0, 'a cross-host operation must never be reported declared-not-live against the wrong origin'); +}); + +// ─── exit-code decision. ──────────────────────────────────────────────────────────────────────────── + +test('decideExit: findings drift, probe-level indeterminates are UNKNOWN, method-only indeterminates are OK', () => { + assert.equal(decideExit({ findings: [{}], indeterminate: [] }), EXIT_DRIFT); + assert.equal(decideExit({ findings: [], indeterminate: [{ reason: 'timed out after 20000ms' }] }), EXIT_UNKNOWN); + assert.equal(decideExit({ findings: [], indeterminate: [{ reason: 'HTTP 503' }] }), EXIT_UNKNOWN); + assert.equal( + decideExit({ findings: [], indeterminate: [{ reason: 'declares only POST — a GET probe cannot establish whether that method is served' }] }), + EXIT_OK, + 'a method-based indeterminate (a POST-only route probed with GET) is expected and must not report UNKNOWN', + ); + assert.equal(decideExit({ findings: [], indeterminate: [] }), EXIT_OK); +}); + +// ─── enumerator shape validation. ─────────────────────────────────────────────────────────────────── + +test('enumeratorShapeError rejects a malformed 200 instead of silently enumerating zero routes', () => { + assert.equal(enumeratorShapeError('published contract', { paths: {} }), null); + assert.match(enumeratorShapeError('published contract', { notPaths: {} }), /no usable "paths"/); + assert.match(enumeratorShapeError('published contract', []), /no usable "paths"/, 'the published contract must be an object, unlike the capability index'); + assert.match(enumeratorShapeError('published contract', null), /not a JSON object or array/); + assert.equal(enumeratorShapeError('scope catalog', { routes: [] }), null, 'a missing routes field is fine — candidatePaths defaults it'); + assert.match(enumeratorShapeError('scope catalog', { routes: 'not-an-array' }), /non-array "routes"/); + assert.match(enumeratorShapeError('scope catalog', []), /must be an object/, 'the scope catalog is an object, unlike the capability index'); + // MEASURED: the real capability index (.well-known/wave-skills.json) is a bare JSON ARRAY, and + // candidatePaths reads it with Object.values(), which is array-safe. A bare array must stay valid. + assert.equal(enumeratorShapeError('capability index', [{ path: '/v1/gpu' }]), null); + assert.match(enumeratorShapeError('capability index', 'not even json-object-shaped'), /not a JSON object or array/); +}); + +// ─── --out arg parsing. ───────────────────────────────────────────────────────────────────────────── + +test('parseArgs skips the value consumed by --out when picking the spec, including the default-spec case', () => { + assert.deepEqual(parseArgs(['openapi.yaml']), { spec: 'openapi.yaml', out: null }); + assert.deepEqual(parseArgs(['openapi.yaml', '--out', 'live-route-drift.json']), { spec: 'openapi.yaml', out: 'live-route-drift.json' }); + assert.deepEqual( + parseArgs(['--out', 'live-route-drift.json']), + { spec: 'openapi.yaml', out: 'live-route-drift.json' }, + "--out's value must never be mistaken for the spec when the caller uses the default spec", + ); + assert.match(parseArgs(['--out']).error, /needs a value/); + assert.match(parseArgs(['--out', '--out']).error, /needs a value/); +}); diff --git a/.github/scripts/live-route-drift.mjs b/.github/scripts/live-route-drift.mjs new file mode 100644 index 0000000..0cd8a86 --- /dev/null +++ b/.github/scripts/live-route-drift.mjs @@ -0,0 +1,281 @@ +#!/usr/bin/env node +/** + * live-route-drift.mjs — the CLI. Reads the documents, enumerates candidates, probes the LIVE + * surface, turns the verdict into an exit code. The probe rules live in `live-route-probe.mjs` and + * the comparison in `live-route-compare.mjs`. + * + * ── WHY A THIRD SOURCE ────────────────────────────────────────────────────────────────────────── + * `published-drift.mjs`, next to this file, compares two DOCUMENTS: the contract this repo declares + * (`openapi.yaml`) against the contract the gateway publishes (`https://api.wave.online/openapi.json`). + * That comparison is worth having, but it cannot close its own hole: NEITHER SIDE IS THE LIVE ROUTE + * TABLE. Both are artifacts, both are written by us, and both can be wrong in the SAME direction at + * the same time. A route that is live in production and absent from BOTH documents is invisible to + * that gate by construction — it can serve traffic indefinitely while the check stays green, + * because green there means "the two documents agree", not "the documents describe reality". + * + * A gate must be able to observe the thing it gates. This one observes the live surface directly. + * + * MEASURED 2026-09-05, which is why this is a script and not a doc comment: `GET + * https://api.wave.online/v1/samples/clips` answers HTTP 200 with a real body. It is absent from + * `openapi.yaml`, absent from the published `openapi.json`, and absent from the gateway's own + * capability index — three artifacts, three misses. The cause is structural, and it is a CLASS + * rather than an oversight: that route is dispatched PRE-AUTH, and the capability index is derived + * from the route->scope map, so a route that never consults a scope cannot appear in a + * scope-derived index. Every pre-auth route is invisible to every artifact-based check we have. + * Only a probe sees it. Positive control run the same way at the same time: `/v1/clips` and + * `/v1/render` are present in all three and answer 402, and a path that does not exist answers 403 + * ROUTE_NOT_MAPPED — so the method discriminates rather than reporting everything as missing. + * + * ── ENUMERATION ───────────────────────────────────────────────────────────────────────────────── + * Candidates come from five public sources, unioned, then every one is probed: + * 1. openapi.yaml (this repo) — parameterless paths only; see isProbeable(). + * 2. the published openapi.json — same treatment. + * 3. the gateway route->scope catalog — .well-known/wave-scopes.json, derived at request time + * from the gateway's own route map. + * 4. the gateway capability index — .well-known/wave-skills.json. + * 5. live-route-seeds.json (committed) — routes OBSERVED live that no machine-readable artifact + * enumerates. Without it the pre-auth class above could + * never even become a candidate, and this gate would inherit + * the blind spot it exists to remove. + * + * Sources 3 and 4 are ADVISORY ENUMERATORS, NOT AUTHORITIES. The scope catalog says so itself + * ("the live response to your request is always authoritative"), and it demonstrably misses the + * pre-auth class. The PROBE is the authority. That distinction is the same one the fleet has paid + * for elsewhere: a Worker's runtime environment is not its committed `wrangler.toml`, so repo-only + * verification of anything env-keyed is unsound. If we make a claim about production, we probe it. + * + * COST: every probe is an unauthenticated GET and is free; a 402 is the response, not a purchase. + * See live-route-probe.mjs. + * + * EXIT CODES — the same contract `published-drift.mjs` uses, so one workflow can grade both the + * same way. A FAILED READ IS NEVER REPORTED AS "NO DRIFT". + * 0 no drift — every live route is declared; every non-draft declaration is live (or allowlisted). + * 1 UNKNOWN — could not read the spec, an enumerator, the seeds or the allowlist. Red, files nothing. + * 2 DRIFT — at least one unexplained difference against the LIVE surface. + * + * USAGE + * node .github/scripts/live-route-drift.mjs openapi.yaml + * node .github/scripts/live-route-drift.mjs openapi.yaml --out live-route-drift.json + */ +import { readFileSync, writeFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join, resolve } from 'node:path'; +import { probeAll } from './live-route-probe.mjs'; +import { candidatePaths, compareAgainstLive, validateAllowlist } from './live-route-compare.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +export const PUBLISHED_SPEC_URL = 'https://api.wave.online/openapi.json'; +export const SCOPE_CATALOG_URL = 'https://gateway.wave.online/.well-known/wave-scopes.json'; +export const CAPABILITY_INDEX_URL = 'https://gateway.wave.online/.well-known/wave-skills.json'; +export const SEEDS_PATH = join(__dirname, 'live-route-seeds.json'); +export const ALLOWLIST_PATH = join(__dirname, 'live-route-drift-allowlist.json'); +export const FETCH_TIMEOUT_MS = 20_000; + +export const EXIT_OK = 0; +export const EXIT_UNKNOWN = 1; +export const EXIT_DRIFT = 2; + +/** + * A 200 with valid JSON in the WRONG shape is not a readable enumerator: `candidatePaths` reads + * `scopeCatalog?.routes`, `capabilityIndex` entries, etc with an optional-chaining `?? []` fallback + * that is silent by design for a MISSING field, but that same silence means a malformed successful + * response (an HTML error page's JSON wrapper, a truncated body, a shape change upstream) is read as + * "this enumerator has zero routes today" rather than "this enumerator could not be read" — and a + * gate that loses candidates silently can report no drift after losing the very routes it exists to + * catch. Each enumerator's minimum required shape is checked explicitly here, before it ever reaches + * candidatePaths. + */ +export function enumeratorShapeError(name, doc) { + // MEASURED 2026-09-05 against the live endpoints: the published contract and the scope catalog are + // both plain objects; the capability index is a bare JSON ARRAY (`candidatePaths` reads it with + // `Object.values(capabilityIndex ?? {})`, which is array-safe by design). So "is a JSON object" is + // not itself the bar — a bare array is a valid, readable shape for that one source. + if (doc === null || typeof doc !== 'object') return `${name} response is not a JSON object or array`; + if (name === 'published contract') { + if (Array.isArray(doc) || !doc.paths || typeof doc.paths !== 'object' || Array.isArray(doc.paths)) { + return `${name} response has no usable "paths" object`; + } + } + if (name === 'scope catalog') { + if (Array.isArray(doc)) return `${name} response must be an object with a "routes" array, not a bare array`; + if (doc.routes !== undefined && !Array.isArray(doc.routes)) return `${name} response has a non-array "routes" field`; + } + return null; +} + +/** + * Turn a comparison result into an exit code. A run that could not fully READ the live surface must + * never look clean: method-based indeterminates (a POST-only declaration probed with a GET, which + * cannot establish anything) are expected and excluded, but a probe-level failure — a 5xx, a + * timeout, a transport error — means the surface was not actually observed and must not report + * EXIT_OK just because it produced zero findings. Exported so this decision is testable offline + * rather than living only inside `main`'s side effects. + */ +export function decideExit(result) { + if (result.findings.length) return EXIT_DRIFT; + const unreadable = result.indeterminate.filter((i) => !String(i.reason).includes('declares only')).length; + if (unreadable > 0) return EXIT_UNKNOWN; + return EXIT_OK; +} + +/** Fetch one JSON enumerator. Returns a result, never throws, never defaults to "no drift". */ +export async function fetchJson(url, doFetch = fetch) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + try { + // Redirects are not followed, for the reason published-drift.mjs gives: this runs on a CI runner + // and the URL must not be able to choose the job's next destination. + const res = await doFetch(url, { signal: controller.signal, redirect: 'manual' }); + if (res.status >= 300 && res.status < 400) return { ok: false, error: `${url} redirected (HTTP ${res.status}) — refusing to follow` }; + if (!res.ok) return { ok: false, error: `HTTP ${res.status} from ${url}` }; + return { ok: true, doc: await res.json() }; + } catch (err) { + const reason = err?.name === 'AbortError' ? `timed out after ${FETCH_TIMEOUT_MS}ms` : (err?.message ?? String(err)); + return { ok: false, error: `${url}: ${reason}` }; + } finally { + clearTimeout(timer); + } +} + +/** + * Pull `spec` and `--out ` apart from argv. Skips the value CONSUMED BY `--out` when picking + * the spec: without this, `node live-route-drift.mjs --out live-route-drift.json` (the default spec, + * explicit output) reads the output filename as the spec and exits UNKNOWN before ever probing. + * Returns `{ error }` when `--out` is present with no value (or a value that is itself a flag). + */ +export function parseArgs(argv) { + const spec = argv.find((a, i) => !a.startsWith('--') && argv[i - 1] !== '--out') ?? 'openapi.yaml'; + const outIdx = argv.indexOf('--out'); + const out = outIdx === -1 ? null : argv[outIdx + 1]; + if (outIdx !== -1 && (!out || out.startsWith('--'))) { + return { error: 'live-route-drift: --out needs a value' }; + } + return { spec, out }; +} + +export async function main(argv = process.argv.slice(2)) { + const parsed = parseArgs(argv); + if (parsed.error) { + console.error(parsed.error); + return EXIT_UNKNOWN; + } + const { spec, out } = parsed; + + let repoDoc; + try { + const yaml = await import('js-yaml'); + repoDoc = (yaml.default ?? yaml).load(readFileSync(spec, 'utf8')); + } catch (err) { + console.error(`live-route-drift: could not read/parse ${spec}: ${err.message}`); + return EXIT_UNKNOWN; + } + if (!repoDoc?.paths || typeof repoDoc.paths !== 'object') { + console.error(`live-route-drift: ${spec} has no usable "paths" object`); + return EXIT_UNKNOWN; + } + + let seeds; + let allowlist; + try { + seeds = JSON.parse(readFileSync(SEEDS_PATH, 'utf8')); + allowlist = JSON.parse(readFileSync(ALLOWLIST_PATH, 'utf8')); + } catch (err) { + console.error(`live-route-drift: could not read/parse the seeds or the allowlist: ${err.message}`); + return EXIT_UNKNOWN; + } + const allowlistError = validateAllowlist(allowlist); + if (allowlistError) { + console.error(`live-route-drift: ${allowlistError}`); + return EXIT_UNKNOWN; + } + + const [published, scopes, skills] = await Promise.all([ + fetchJson(PUBLISHED_SPEC_URL), + fetchJson(SCOPE_CATALOG_URL), + fetchJson(CAPABILITY_INDEX_URL), + ]); + for (const [name, r] of [ + ['published contract', published], + ['scope catalog', scopes], + ['capability index', skills], + ]) { + if (!r.ok) { + // FAIL LOUD. An unreachable enumerator says nothing about drift and is not "no drift". + console.error(`live-route-drift: could not read the ${name}: ${r.error}`); + return EXIT_UNKNOWN; + } + const shapeError = enumeratorShapeError(name, r.doc); + if (shapeError) { + // FAIL LOUD here too: a malformed 200 is not "zero routes", it is an unread enumerator, and + // silently dropping its candidates would let this gate report clean after losing them. + // (shapeError already names the source; do not prefix it again.) + console.error(`live-route-drift: ${shapeError} — refusing to enumerate from it`); + return EXIT_UNKNOWN; + } + } + + const candidates = candidatePaths({ + repoDoc, + publishedDoc: published.doc, + scopeCatalog: scopes.doc, + capabilityIndex: skills.doc, + seeds, + }); + if (!candidates.length) { + console.error('live-route-drift: zero candidate paths — refusing to call that "no drift"'); + return EXIT_UNKNOWN; + } + + const probes = await probeAll(candidates); + const result = compareAgainstLive({ repoDoc, publishedDoc: published.doc, probes, allowlist }); + const h = result.headline; + + console.log(`live-route-drift: probed ${h.probed} candidate routes — mapped ${h.mapped}, absent ${h.absent}, indeterminate ${h.indeterminate}`); + console.log(`live-route-drift: findings — live-undeclared ${h.liveUndeclared}, declared-not-live ${h.declaredNotLive}; allowlisted ${h.allowlisted}`); + for (const i of result.indeterminate) console.log(`::warning::could not classify ${i.path}: ${i.reason}`); + for (const k of result.unmatchedAllowlist) console.log(`::warning::allowlist entry ${k} matched nothing — the exemption is dead and should be deleted rather than left standing`); + for (const f of result.findings) console.error(`::error::[${f.direction}] ${f.path} (HTTP ${f.status}) — ${f.note}`); + + if (out) { + writeFileSync( + out, + `${JSON.stringify( + { + about: + "Point-in-time diff between this repo's openapi.yaml, the gateway's published contract, and the LIVE route " + + 'surface established by unauthenticated GET probes. The live half is the one neither document can supply.', + generatedAt: new Date().toISOString(), + criterion: ['CONTRACT-001', 'API-001'], + sources: { + repoSpec: spec, + publishedSpec: PUBLISHED_SPEC_URL, + scopeCatalog: SCOPE_CATALOG_URL, + capabilityIndex: CAPABILITY_INDEX_URL, + seeds: 'live-route-seeds.json', + }, + ...result, + }, + null, + 2, + )}\n`, + ); + } + + const exit = decideExit(result); + if (exit === EXIT_DRIFT) { + console.error(`live-route-drift: DRIFT — ${result.findings.length} route(s) disagree with the live surface.`); + return EXIT_DRIFT; + } + if (exit === EXIT_UNKNOWN) { + const unreadable = result.indeterminate.filter((i) => !String(i.reason).includes('declares only')).length; + console.error(`live-route-drift: UNKNOWN — could not classify ${unreadable} probe(s); the live surface was not fully observed.`); + return EXIT_UNKNOWN; + } + console.log('live-route-drift: OK — every live route is declared, and every promised declaration is live.'); + return EXIT_OK; +} + +if (process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url))) { + process.exitCode = await main(); +} diff --git a/.github/scripts/live-route-drift.test.mjs b/.github/scripts/live-route-drift.test.mjs new file mode 100644 index 0000000..8052255 --- /dev/null +++ b/.github/scripts/live-route-drift.test.mjs @@ -0,0 +1,321 @@ +// live-route-drift.test.mjs — offline. No network: probe results are inputs, and the two tests that +// exercise the prober inject a fake `fetch`. +// +// Every test here is written against the specific false-green this feature closes +// (WAVE-GA-VERDICT #11): `published-contract-drift` compares repo-declared against +// gateway-published, so a route that is LIVE and absent from BOTH is invisible to it by +// construction. The suite is built so it cannot pass vacuously: +// (a) the previously-invisible condition now FAILS, +// (b) a POSITIVE CONTROL — a genuinely compliant state still passes, so the gate discriminates +// rather than blanket-failing, +// (c) the probe semantics are pinned by name, especially that 402 is PRESENCE and not absence, +// (d) a MUTATION PROOF — `twoArtifactDriftOnly` (what the old gate could see) returns ZERO on the +// same input where the three-way comparison returns a finding. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { classifyProbe, probePath, probeAll, ABSENT, MAPPED, INDETERMINATE } from './live-route-probe.mjs'; +import { + basePath, + candidatePaths, + compareAgainstLive, + isProbeable, + segmentKey, + twoArtifactDriftOnly, + withinSpecBase, + validateAllowlist, +} from './live-route-compare.mjs'; + +const SERVERS = [{ url: 'https://api.wave.online/v1' }]; + +/** A spec that declares /clips and /render, exactly as the real one does (relative to the /v1 base). */ +const REPO_DOC = { + servers: SERVERS, + paths: { + '/clips': { get: { operationId: 'listClips' } }, + '/render': { get: { operationId: 'getRender' } }, + }, +}; +const PUBLISHED_DOC = { + servers: SERVERS, + paths: { + '/clips': { get: { operationId: 'listClips' } }, + '/render': { get: { operationId: 'getRender' } }, + }, +}; + +const probeMap = (entries) => new Map(entries.map((e) => [e.path, e])); + +// ─── (c) Probe semantics. A paywall is not an absence. ─────────────────────────────────────────── + +test('402 is MAPPED — a paywall proves the route EXISTS and is priced, it is never an absence', () => { + // This is the single most consequential rule in the feature. If 402 ever read as "absent", the + // gate would go blind to exactly the routes that charge customers money. + assert.equal(classifyProbe({ status: 402, body: { x402Version: 1, error: 'payment required' } }), MAPPED); +}); + +test('only an explicit ROUTE_NOT_MAPPED code is ABSENT; a bare 403 is MAPPED', () => { + assert.equal(classifyProbe({ status: 403, body: { error: { code: 'ROUTE_NOT_MAPPED' } } }), ABSENT); + // A plain authorization failure PROVES the route exists — there was something to be unauthorized + // for. Inferring absence from the status number alone would delete real findings. + assert.equal(classifyProbe({ status: 403, body: { error: { code: 'FORBIDDEN' } } }), MAPPED); + assert.equal(classifyProbe({ status: 401, body: { error: { code: 'UNAUTHENTICATED' } } }), MAPPED); +}); + +test('200 is MAPPED and 5xx is INDETERMINATE — an origin having a bad minute is not an absence', () => { + assert.equal(classifyProbe({ status: 200, body: { source: 'sample' } }), MAPPED); + assert.equal(classifyProbe({ status: 503, body: null }), INDETERMINATE); + assert.equal(classifyProbe({ status: 500, body: { error: { code: 'ROUTE_NOT_MAPPED' } } }), INDETERMINATE); +}); + +test('probePath sends an unauthenticated GET and never throws on a transport failure', async () => { + const seen = []; + const fakeFetch = async (url, init) => { + seen.push({ url, method: init.method, hasAuth: Boolean(init.headers?.authorization ?? init.headers?.Authorization) }); + return { status: 402, json: async () => ({ error: 'payment required' }) }; + }; + const r = await probePath('/v1/render', fakeFetch); + assert.equal(r.state, MAPPED); + assert.equal(seen[0].method, 'GET', 'probes must be GET — never a POST that could do billable work'); + assert.equal(seen[0].hasAuth, false, 'probes must be unauthenticated — no tenant, meter or balance is touched'); + + const boom = await probePath('/v1/render', async () => { + throw new Error('ECONNRESET'); + }); + assert.equal(boom.state, INDETERMINATE, 'a failed probe is INDETERMINATE, never ABSENT'); +}); + +test('probeAll probes every candidate exactly once', async () => { + const calls = []; + const fakeFetch = async (url) => { + calls.push(url); + return { status: 402, json: async () => ({}) }; + }; + const out = await probeAll(['/v1/a', '/v1/b', '/v1/c'], fakeFetch, 'https://x.test', 2); + assert.equal(out.size, 3); + assert.equal(calls.length, 3); +}); + +// ─── (a) The previously-invisible condition now FAILS. ─────────────────────────────────────────── + +test('SEEDED VIOLATION — a route that is LIVE and absent from BOTH artifacts is a finding', () => { + // This is the exact shape measured against production on 2026-09-05: GET /v1/samples/clips + // answers 200, and it appears in neither openapi.yaml nor the published contract. + const probes = probeMap([ + { path: '/v1/clips', state: MAPPED, status: 402 }, + { path: '/v1/render', state: MAPPED, status: 402 }, + { path: '/v1/samples/clips', state: MAPPED, status: 200 }, + ]); + const r = compareAgainstLive({ repoDoc: REPO_DOC, publishedDoc: PUBLISHED_DOC, probes }); + assert.equal(r.headline.liveUndeclared, 1); + assert.equal(r.findings.length, 1); + assert.equal(r.findings[0].path, '/v1/samples/clips'); + assert.equal(r.findings[0].direction, 'live-undeclared'); + assert.equal(r.findings[0].severity, 'security-relevant'); +}); + +test('SEEDED VIOLATION — a 402-only route absent from both artifacts is a finding too', () => { + // Guards the money-relevant case specifically: a route can be live, PRICED, billable, and + // undocumented. If 402 were ever mistaken for an absence this finding would silently vanish. + const probes = probeMap([ + { path: '/v1/clips', state: MAPPED, status: 402 }, + { path: '/v1/undocumented-paid-thing', state: MAPPED, status: 402 }, + ]); + const r = compareAgainstLive({ repoDoc: REPO_DOC, publishedDoc: PUBLISHED_DOC, probes }); + assert.equal(r.headline.liveUndeclared, 1); + assert.equal(r.findings[0].path, '/v1/undocumented-paid-thing'); +}); + +test('SEEDED VIOLATION — a non-draft declaration the gateway does not serve is a finding', () => { + // The other direction only a probe can see: both documents can agree and both be wrong. + const probes = probeMap([ + { path: '/v1/clips', state: MAPPED, status: 402 }, + { path: '/v1/render', state: ABSENT, status: 403 }, + ]); + const r = compareAgainstLive({ repoDoc: REPO_DOC, publishedDoc: PUBLISHED_DOC, probes }); + assert.equal(r.headline.declaredNotLive, 1); + assert.equal(r.findings[0].path, '/v1/render'); + assert.deepEqual(r.findings[0].methods, ['GET']); +}); + +test('x-schema-status: draft suppresses declared-not-live, but never live-undeclared', () => { + const draftRepo = { servers: SERVERS, paths: { '/clips': { get: { 'x-schema-status': 'draft' } } } }; + const r = compareAgainstLive({ + repoDoc: draftRepo, + publishedDoc: { servers: SERVERS, paths: {} }, + probes: probeMap([{ path: '/v1/clips', state: ABSENT, status: 403 }]), + }); + assert.equal(r.findings.length, 0, 'a draft that is not yet served is not a finding'); + + const r2 = compareAgainstLive({ + repoDoc: draftRepo, + publishedDoc: { servers: SERVERS, paths: {} }, + probes: probeMap([{ path: '/v1/other', state: MAPPED, status: 200 }]), + }); + assert.equal(r2.headline.liveUndeclared, 1, 'draft never excuses an UNDECLARED LIVE route'); +}); + +// ─── (b) POSITIVE CONTROL. A compliant state must still pass. ──────────────────────────────────── + +test('POSITIVE CONTROL — a fully compliant surface produces ZERO findings', () => { + // Without this the gate could satisfy every test above by simply always failing, which is a + // different broken gate rather than a fixed one. These are the real /v1/clips and /v1/render, + // present in all three sources, classified by the same code path as the violations. + const probes = probeMap([ + { path: '/v1/clips', state: MAPPED, status: 402 }, + { path: '/v1/render', state: MAPPED, status: 402 }, + { path: '/v1/not-a-route', state: ABSENT, status: 403 }, + ]); + const r = compareAgainstLive({ repoDoc: REPO_DOC, publishedDoc: PUBLISHED_DOC, probes }); + assert.equal(r.findings.length, 0); + assert.equal(r.headline.liveUndeclared, 0); + assert.equal(r.headline.declaredNotLive, 0); + assert.equal(r.headline.mapped, 2); + assert.equal(r.headline.absent, 1); +}); + +test('POSITIVE CONTROL — a product-root live entry is covered by the segment its spec documents', () => { + // The live enumerators are product-granular (`/v1/voice`) while the spec documents `/voice/voices`. + // Reporting the product root as undeclared would bury real findings under a dozen false ones. + const repo = { servers: SERVERS, paths: { '/voice/voices': { get: {} }, '/voice/generate': { post: {} } } }; + const r = compareAgainstLive({ + repoDoc: repo, + publishedDoc: repo, + probes: probeMap([{ path: '/v1/voice', state: MAPPED, status: 402 }]), + }); + assert.equal(r.findings.length, 0); +}); + +test('an INDETERMINATE probe is neither a pass nor a finding — it is surfaced', () => { + const r = compareAgainstLive({ + repoDoc: REPO_DOC, + publishedDoc: PUBLISHED_DOC, + probes: probeMap([{ path: '/v1/mystery', state: INDETERMINATE, error: 'timed out' }]), + }); + assert.equal(r.findings.length, 0, 'an unreadable probe must not manufacture a finding'); + assert.equal(r.headline.indeterminate, 1, 'nor may it disappear into a clean-looking run'); +}); + +// ─── (d) MUTATION PROOF. Remove the third source and the finding silently vanishes. ────────────── + +test('MUTATION PROOF — the two-artifact comparison reports ZERO on the input the live probe catches', () => { + // `twoArtifactDriftOnly` is what published-drift.mjs can see: repo-declared vs gateway-published. + // On the live-undeclared input it finds NOTHING, because the route is missing from both documents + // and it only ever compares those two to each other. That is the entire false-green, executable. + const probes = probeMap([ + { path: '/v1/clips', state: MAPPED, status: 402 }, + { path: '/v1/render', state: MAPPED, status: 402 }, + { path: '/v1/samples/clips', state: MAPPED, status: 200 }, + ]); + const threeWay = compareAgainstLive({ repoDoc: REPO_DOC, publishedDoc: PUBLISHED_DOC, probes }); + const twoWay = twoArtifactDriftOnly({ repoDoc: REPO_DOC, publishedDoc: PUBLISHED_DOC }); + + assert.equal(twoWay.length, 0, 'the two documents agree perfectly — the old gate is green here'); + assert.equal(threeWay.findings.length, 1, 'the live surface disagrees with both of them'); + assert.equal(threeWay.findings[0].path, '/v1/samples/clips'); + // Stated as one assertion so the delta itself is the thing under test: deleting the probe from + // this feature reduces it to `twoWay`, and this line fails. + assert.ok(threeWay.findings.length > twoWay.length, 'the third source must find what two artifacts cannot'); +}); + +// ─── Scoping rules. Each must exclude only what it claims to, proven with an in-scope control. ─── + +test('paths outside the spec server base are out of scope, but every /v1 route stays in scope', () => { + // Measured against production: /robots.txt, /favicon.svg, /llms.txt, /health and friends are live + // and correctly absent from an API contract. Reporting them would be 7 false findings stacked on + // the real ones. The control below is what stops this rule from becoming a suppression. + const probes = probeMap([ + { path: '/robots.txt', state: MAPPED, status: 200 }, + { path: '/health', state: MAPPED, status: 200 }, + { path: '/v1/samples/clips', state: MAPPED, status: 200 }, + ]); + const r = compareAgainstLive({ repoDoc: REPO_DOC, publishedDoc: PUBLISHED_DOC, probes }); + assert.equal(r.headline.outOfScope, 2); + assert.equal(r.findings.length, 1, 'CONTROL: an undeclared route UNDER /v1 is still a finding'); + assert.equal(r.findings[0].path, '/v1/samples/clips'); +}); + +test('withinSpecBase does not exclude by prefix accident, and an absent base excludes nothing', () => { + assert.equal(withinSpecBase('/v1/clips', '/v1'), true); + assert.equal(withinSpecBase('/v1', '/v1'), true); + assert.equal(withinSpecBase('/v10/clips', '/v1'), false, '/v10 is a different base, not a child of /v1'); + assert.equal(withinSpecBase('/robots.txt', '/v1'), false); + assert.equal(withinSpecBase('/anything', ''), true, 'no server base means fail-closed: everything is in scope'); +}); + +test('a POST-only declaration answering ROUTE_NOT_MAPPED to a GET is INDETERMINATE, not a finding', () => { + // MEASURED: /v1/agent/auth/device and /v1/agent/auth/token are POST-only OAuth device-grant + // routes. A GET to each returns 403 ROUTE_NOT_MAPPED because the gateway's scope map is keyed by + // route AND method — which says nothing about whether their POST is served. An earlier draft of + // this gate reported both as findings; that was the gate asserting a fact its evidence did not + // support. Unknown is not a pass either: it is surfaced. + const repo = { + servers: SERVERS, + paths: { + '/agent/auth/device': { post: { operationId: 'deviceAuth' } }, + '/render': { get: { operationId: 'getRender' } }, + }, + }; + const r = compareAgainstLive({ + repoDoc: repo, + publishedDoc: repo, + probes: probeMap([ + { path: '/v1/agent/auth/device', state: ABSENT, status: 403 }, + { path: '/v1/render', state: ABSENT, status: 403 }, + ]), + }); + assert.equal(r.headline.declaredNotLive, 1, 'CONTROL: the GET-declaring path IS still reported'); + assert.equal(r.findings[0].path, '/v1/render'); + assert.equal(r.headline.indeterminate, 1); + assert.match(r.indeterminate[0].reason, /declares only POST/); +}); + +// ─── Enumeration and allowlist plumbing. ───────────────────────────────────────────────────────── + +test('candidatePaths unions all five enumerators and drops templated paths', () => { + const c = candidatePaths({ + repoDoc: { servers: SERVERS, paths: { '/clips': {}, '/clips/{clipId}': {} } }, + publishedDoc: { servers: SERVERS, paths: { '/render': {} } }, + scopeCatalog: { routes: [{ path: '/v1/voice' }], no_scope_required: { paths: ['/health'] } }, + capabilityIndex: { 0: { path: '/v1/gpu' } }, + seeds: [{ path: '/v1/samples/clips' }], + }); + assert.deepEqual(c, ['/health', '/v1/clips', '/v1/gpu', '/v1/render', '/v1/samples/clips', '/v1/voice']); + assert.ok(!c.includes('/v1/clips/{clipId}'), 'a templated path has no probe-able concrete form'); +}); + +test('basePath / isProbeable / segmentKey', () => { + assert.equal(basePath({ servers: SERVERS }), '/v1'); + assert.equal(basePath({}), ''); + assert.equal(isProbeable('/v1/clips'), true); + assert.equal(isProbeable('/v1/clips/{clipId}'), false); + assert.equal(segmentKey('/v1/render/{jobId}'), 'v1/render'); +}); + +test('an allowlist entry suppresses its finding, and a dead entry is surfaced not honoured silently', () => { + const probes = probeMap([{ path: '/v1/samples/clips', state: MAPPED, status: 200 }]); + const allowlist = [ + { + direction: 'live-undeclared', + path: '/v1/samples/clips', + justification: 'documented separately as a free synthetic sample surface, tracked for promotion into the spec', + }, + { + direction: 'live-undeclared', + path: '/v1/gone', + justification: 'this exemption no longer matches anything and must be reported as dead rather than left standing', + }, + ]; + const r = compareAgainstLive({ repoDoc: REPO_DOC, publishedDoc: PUBLISHED_DOC, probes, allowlist }); + assert.equal(r.findings.length, 0); + assert.equal(r.allowlisted.length, 1); + assert.deepEqual(r.unmatchedAllowlist, ['live-undeclared /v1/gone']); +}); + +test('validateAllowlist rejects a missing justification, an unknown direction and a duplicate', () => { + assert.equal(validateAllowlist([]), null); + assert.match(validateAllowlist([{ direction: 'live-undeclared', path: '/v1/x', justification: 'too short' }]), /justification/); + assert.match(validateAllowlist([{ direction: 'made-up', path: '/v1/x', justification: 'a perfectly long justification string here' }]), /unknown direction/); + const dup = { direction: 'live-undeclared', path: '/v1/x', justification: 'a perfectly long justification string here' }; + assert.match(validateAllowlist([dup, { ...dup }]), /duplicate/); + assert.match(validateAllowlist({}), /not an array/); +}); diff --git a/.github/scripts/live-route-probe.mjs b/.github/scripts/live-route-probe.mjs new file mode 100644 index 0000000..1ee0023 --- /dev/null +++ b/.github/scripts/live-route-probe.mjs @@ -0,0 +1,93 @@ +#!/usr/bin/env node +/** + * live-route-probe.mjs — how this repo asks the gateway whether a route EXISTS, and nothing else. + * Pure I/O plus one classification rule. The comparison lives in `live-route-compare.mjs` and the + * CLI in `live-route-drift.mjs`; see that file's header for why a third source is needed at all. + * + * ── THE PROBE SEMANTICS ARE LOAD-BEARING ──────────────────────────────────────────────────────── + * On this gateway an unmapped path answers HTTP 403 with `error.code === "ROUTE_NOT_MAPPED"` + * ("no scope rule for this route (fail-closed)"). Anything else — INCLUDING 402 — means the route + * exists. + * + * 402 IS NOT AN ABSENCE. It is the strongest available evidence of PRESENCE: the route is mapped + * and it is PRICED. Reading a paywall as "route not found" would make this gate blind to exactly + * the routes that charge customers money, which inverts its purpose. Do not ever quiet a noisy + * run by treating 402 as absent. + * + * ONLY an explicit `ROUTE_NOT_MAPPED` counts as absence. A bare 403 does not: 403 is also what an + * authorization failure looks like, and an authorization failure PROVES the route exists — there + * was something there to be unauthorized for. Requiring the code keeps "absent" a positive claim + * read off the body rather than an inference from a status number. + * + * A 5xx, a timeout or a transport error is INDETERMINATE, never absent. An origin having a bad + * minute must not be recorded as "this route does not exist", because that would silently clear a + * real finding and leave the gate greener than the evidence supports. + * + * ── COST ──────────────────────────────────────────────────────────────────────────────────────── + * Every probe is an unauthenticated GET. No credential is sent, so no tenant, meter or balance is + * touched, and a 402 is returned BEFORE any work is performed — the challenge IS the response. + * These probes are free. Never add a paid call, a POST, an authenticated request, or a retry storm + * to this file; concurrency is deliberately tiny because this is a correctness gate, not a load + * test. + */ +export const ORIGIN = 'https://api.wave.online'; +export const FETCH_TIMEOUT_MS = 20_000; +/** Deliberately tiny. This is a correctness gate, not a load test — never raise it. */ +export const PROBE_CONCURRENCY = 4; + +export const MAPPED = 'mapped'; +export const ABSENT = 'absent'; +export const INDETERMINATE = 'indeterminate'; + +/** Classify one probe response. See the header — 402 is MAPPED, and only ROUTE_NOT_MAPPED is ABSENT. */ +export function classifyProbe({ status, body }) { + if (status >= 500) return INDETERMINATE; + // A redirect conveys nothing about whether a route exists: `probePath` uses `redirect: 'manual'`, + // so a 3xx arrives with a non-JSON body and would otherwise fall through to MAPPED, which is wrong + // in both directions — it can hide a genuinely withdrawn/redirected route (false green) and it can + // fabricate a live-undeclared finding for a redirecting undeclared path (false red). + if (status >= 300 && status < 400) return INDETERMINATE; + // Require the 403 the documented contract specifies. Checking the body code alone would let a + // non-403 gateway error that happens to carry the same code hide a real live-route finding. + if (status === 403 && body?.error?.code === 'ROUTE_NOT_MAPPED') return ABSENT; + return MAPPED; +} + +/** GET one path, unauthenticated, bounded. Returns a result; never throws. */ +export async function probePath(path, doFetch = fetch, origin = ORIGIN) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + try { + const res = await doFetch(`${origin}${path}`, { + method: 'GET', + signal: controller.signal, + redirect: 'manual', + headers: { accept: 'application/json' }, + }); + let body = null; + try { + body = await res.json(); + } catch { + body = null; // a non-JSON body is fine; classification falls back to the status + } + return { path, ok: true, status: res.status, body, state: classifyProbe({ status: res.status, body }) }; + } catch (err) { + const reason = err?.name === 'AbortError' ? `timed out after ${FETCH_TIMEOUT_MS}ms` : (err?.message ?? String(err)); + return { path, ok: false, error: reason, state: INDETERMINATE }; + } finally { + clearTimeout(timer); + } +} + +/** Probe many paths with a small fixed concurrency. Returns `Map`. */ +export async function probeAll(paths, doFetch = fetch, origin = ORIGIN, concurrency = PROBE_CONCURRENCY) { + const queue = [...paths]; + const out = new Map(); + const workers = Array.from({ length: Math.min(concurrency, queue.length) }, async () => { + for (let p = queue.shift(); p !== undefined; p = queue.shift()) { + out.set(p, await probePath(p, doFetch, origin)); + } + }); + await Promise.all(workers); + return out; +} diff --git a/.github/scripts/live-route-seeds.json b/.github/scripts/live-route-seeds.json new file mode 100644 index 0000000..27870dd --- /dev/null +++ b/.github/scripts/live-route-seeds.json @@ -0,0 +1,7 @@ +[ + { + "path": "/v1/samples/clips", + "observed": "2026-09-05", + "why": "Routes OBSERVED live that no machine-readable artifact enumerates. This one is dispatched pre-auth, and the gateway's capability index is derived from the route->scope map, so a route that never consults a scope cannot appear in a scope-derived index. It is absent from openapi.yaml, from the published openapi.json, and from that index; GET returns HTTP 200. Without this seed it could never become a probe candidate and the new gate would inherit the exact blind spot it was built to remove. Delete this entry only when the route is documented or withdrawn — not to quiet a finding." + } +] diff --git a/.github/workflows/live-route-drift.yml b/.github/workflows/live-route-drift.yml new file mode 100644 index 0000000..01a8f8c --- /dev/null +++ b/.github/workflows/live-route-drift.yml @@ -0,0 +1,202 @@ +# live-route-drift.yml — the THIRD SOURCE for CONTRACT-001. +# +# published-contract-drift.yml, next to this file, compares repo-declared against gateway-published. +# Both are DOCUMENTS we write, and both can be wrong in the same direction at once, so a route that +# is live in production and absent from BOTH is invisible to it by construction. This workflow asks +# the question that comparison cannot: does the LIVE surface agree with either document? +# +# TWO JOBS, TWO TRIGGERS, for the same reason published-contract-drift.yml splits its own: +# +# `unit` runs on PULL REQUESTS. Offline — node --test over hand-built fixtures with an injected +# fetch, no network at all — so it is a property of the diff and belongs on the PR path. It is +# what keeps the probe semantics honest: it asserts by name that a 402 is PRESENCE, that only an +# explicit ROUTE_NOT_MAPPED is absence, and that the two-artifact comparison returns ZERO on the +# input the live probe catches. Anyone who softens the classifier fails a test that says so. +# +# `drift` runs on a SCHEDULE and on demand, never on a pull request. Whether the live surface has +# drifted is not a property of any given PR — it is a fact about the world that changes on the +# serving deployment's schedule, not this repo's. Asking it on every PR would make every author +# here depend on an unauthenticated network fetch, so an outage could red the whole repo for a +# reason no author could act on. +# +# COST: every probe is an unauthenticated GET. No credential is sent, so no tenant, meter or balance +# is touched, and a 402 is returned BEFORE any work is performed — the challenge IS the response. +# These probes are free. Concurrency is fixed at 4 in the script: this is a correctness gate, not a +# load test. Never add a paid call, a POST, an authenticated request or a retry storm. +# +# ON EXIT 2 (drift) the job files or updates ONE tracking issue, matched by exact title so repeated +# runs comment instead of opening a new issue every morning, and CLOSES it on exit 0 — an open issue +# is a claim about the current state of the default branch, and a claim nothing retracts becomes +# false the moment it is fixed. On exit 1 (UNKNOWN — a broken read) it goes red and files NOTHING: a +# failed read says nothing about drift. +# +# ON ANY OTHER EXIT CODE the job goes red. The script defines exactly three; a fourth means a crash. +# The per-code steps are all `if: code == ...`, so an unhandled code would match none of them and +# leave the job GREEN. A gate that fails open is worse than no gate, because it is trusted. +# +# THE DIFF ARTIFACT IS NOT COMMITTED. It enumerates the live route surface, which belongs in a run +# artifact rather than in the tree of a public repository. + +name: live-route-drift + +on: + pull_request: + paths: + - '.github/scripts/live-route-*' + - '.github/workflows/live-route-drift.yml' + - 'openapi.yaml' + schedule: + # 07:25 UTC daily — after published-contract-drift's own 07:10 run, so the two-document picture + # is already graded when this asks the harder question. + - cron: '25 7 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + unit: + name: unit tests (offline) + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '22' + # No install here: live-route-drift.test.mjs imports only live-route-probe.mjs and + # live-route-compare.mjs, neither of which touches js-yaml — that dependency belongs to the CLI + # (live-route-drift.mjs), not the offline unit suite. Reaching the npm registry from a PR-path + # job would let a registry outage fail an otherwise fully offline check. + - name: node --test + run: node --test .github/scripts/live-route-drift.test.mjs .github/scripts/live-route-drift-regressions.test.mjs + + drift: + name: live route drift + # workflow_dispatch is restricted to the default branch: this job evaluates whatever branch it + # checks out (implicitly the triggering ref) but files/closes ONE repository-wide tracking issue. + # Dispatching from a feature branch would grade that branch's openapi.yaml while mutating an issue + # that describes the state of the default branch. + if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '22' + - name: Install tooling + run: npm install --no-save --no-audit --no-fund --ignore-scripts js-yaml@4.1.0 + + - name: Compare openapi.yaml and the published contract against the LIVE route surface + id: drift + run: | + set +e + node .github/scripts/live-route-drift.mjs openapi.yaml --out /tmp/live-route-drift.json | tee /tmp/drift.log + code=${PIPESTATUS[0]} + set -e + echo "code=$code" >> "$GITHUB_OUTPUT" + echo "exit code: $code" + + - name: Upload the diff artifact + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: live-route-drift + path: /tmp/live-route-drift.json + if-no-files-found: warn + + # The inverse default case. See the header: without it a crash would match no step and leave a + # networked gate green while saying nothing at all. + - name: Fail on an unrecognized exit code + if: steps.drift.outputs.code != '0' && steps.drift.outputs.code != '1' && steps.drift.outputs.code != '2' + env: + CODE: ${{ steps.drift.outputs.code }} + run: | + echo "::error::live-route-drift exited $CODE, which is not one of its three documented codes (0 OK / 1 UNKNOWN / 2 DRIFT)." + echo "Treating an unrecognized exit as a failure: it is a crash, not a verdict, and must never read as a pass." + exit 1 + + - name: Fail loudly on a broken read + if: steps.drift.outputs.code == '1' + run: | + echo "::error::live-route-drift could not read the spec or one of its live enumerators." + echo "This says NOTHING about drift and no issue was filed. Fix the read, then re-run." + exit 1 + + # Nothing PR- or issue-controlled is interpolated into a shell command: the title is a fixed + # literal, the body is written to a file and passed with --body-file, and the issue number + # comes from a --jq filter over `gh search issues` output. + - name: File or update the tracking issue + if: steps.drift.outputs.code == '2' + env: + GH_TOKEN: ${{ github.token }} + TITLE: 'Live route surface disagrees with openapi.yaml and the published contract' + run: | + { + echo "The gateway is serving routes that neither \`openapi.yaml\` nor the published contract describes, and/or documenting non-draft operations it does not serve." + echo + echo "Run: [\`$GITHUB_RUN_ID\`](https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID) · commit \`$GITHUB_SHA\`" + echo + echo "The machine-readable diff is attached to the run as the \`live-route-drift\` artifact. It is deliberately not committed: it enumerates the live route surface." + echo + echo "Each finding is cleared by documenting the operation in \`openapi.yaml\`, withdrawing the route, or adding an allowlist entry with a real justification in \`.github/scripts/live-route-drift-allowlist.json\`." + echo "See \`.github/scripts/live-route-drift.mjs\` for what each direction means and why a 402 is presence rather than absence." + } > /tmp/issue-body.md + + # `gh issue list` is a date-ordered LIST capped at --limit: once 100 newer open issues + # exist, this tracking issue falls off the page and becomes invisible, so a drift run would + # create a duplicate and a clean run would leave the original open forever. `gh search + # issues` is a SEARCH, not a page, so it finds the issue by title regardless of how many + # newer issues exist; the jq filter still requires an EXACT title match, so a loosely + # matching search hit can never be treated as the tracking issue. + existing=$(gh search issues "$TITLE" --match title --repo "$GITHUB_REPOSITORY" --state open \ + --json number,title --jq "map(select(.title == \$ENV.TITLE)) | .[0].number // empty") + + if [ -n "$existing" ]; then + gh issue comment "$existing" --repo "$GITHUB_REPOSITORY" --body-file /tmp/issue-body.md + echo "Commented on existing issue #$existing" + else + gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body-file /tmp/issue-body.md + fi + + # Red on drift too, so a silenced or rate-limited issue write can never make it look green. + - name: Fail on drift + if: steps.drift.outputs.code == '2' + run: | + echo "::error::The live route surface disagrees with the declared and published contracts. A tracking issue was filed or updated." + exit 1 + + - name: Close the tracking issue once the live surface agrees again + if: steps.drift.outputs.code == '0' + env: + GH_TOKEN: ${{ github.token }} + TITLE: 'Live route surface disagrees with openapi.yaml and the published contract' + run: | + # See the matching comment above: search, not a capped list, so the issue is found + # regardless of how many newer open issues exist. + existing=$(gh search issues "$TITLE" --match title --repo "$GITHUB_REPOSITORY" --state open \ + --json number,title --jq "map(select(.title == \$ENV.TITLE)) | .[0].number // empty") + + if [ -n "$existing" ]; then + { + echo "Every live route is declared and every promised declaration is live again — this is resolved." + echo + echo "Run: [\`$GITHUB_RUN_ID\`](https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID) · commit \`$GITHUB_SHA\`" + } > /tmp/close-body.md + gh issue close "$existing" --repo "$GITHUB_REPOSITORY" --comment "$(cat /tmp/close-body.md)" + echo "Closed issue #$existing — the live surface agrees." + else + echo "No drift and no tracking issue is open. Nothing to close." + fi