From b188764c3b7169137a3de5998dd4ef667dba44ae Mon Sep 17 00:00:00 2001 From: Jake Fineman Date: Sat, 5 Sep 2026 14:47:53 -0400 Subject: [PATCH 1/3] ci(ga): add GA evidence producer for CONTRACT-001 + COMPAT-001 wave-av/sdks is currently the only repo emitting a schema-shaped ga-evidence.json for the WAVE GA readiness gate. This adds api-spec's own producer for the two platform criteria this repo owns: CONTRACT-001 (does the declared contract match what the gateway actually serves, reusing this repo's own published-drift comparator) and COMPAT-001 (zero breaking changes vs the last release tag, via oasdiff). Status is always computed from a real run, never hardcoded; COMPAT-001 never claims the unverified deprecation-notice half of its pass condition. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ga-evidence.yml | 120 ++++++++++++++ .gitignore | 3 + scripts/ga/check-COMPAT-001.sh | 12 ++ scripts/ga/check-CONTRACT-001.sh | 11 ++ scripts/ga/compat-001-check.mjs | 182 ++++++++++++++++++++ scripts/ga/contract-001-check.mjs | 229 +++++++++++++++++++++++++ scripts/ga/ga-evidence.mjs | 266 ++++++++++++++++++++++++++++++ 7 files changed, 823 insertions(+) create mode 100644 .github/workflows/ga-evidence.yml create mode 100755 scripts/ga/check-COMPAT-001.sh create mode 100755 scripts/ga/check-CONTRACT-001.sh create mode 100644 scripts/ga/compat-001-check.mjs create mode 100644 scripts/ga/contract-001-check.mjs create mode 100644 scripts/ga/ga-evidence.mjs diff --git a/.github/workflows/ga-evidence.yml b/.github/workflows/ga-evidence.yml new file mode 100644 index 0000000..91a99a1 --- /dev/null +++ b/.github/workflows/ga-evidence.yml @@ -0,0 +1,120 @@ +# ga-evidence.yml — GA-evidence PRODUCER for wave-av/api-spec (CONTRACT-001 + COMPAT-001). +# +# Part of the Instinct external GA validator epic (control repo wave-av/claude-workstation, +# governance/plans/instinct-ga-validator/E1-HANDSHAKE.md, P2 row 3). wave-av/sdks is, as of this +# workflow, the only repo that emits a schema-shaped ga-evidence.json for the WAVE GA readiness +# gate — see its scripts/ga/registry-cleanroom.mjs and .github/workflows/registry-cleanroom.yml, +# which this workflow mirrors action-for-action (same checkout / setup-node / upload-artifact +# pins, same fail-loud Enforce step). This repo owns CONTRACT-001 and COMPAT-001, the two +# platform-scoped criteria in governance/ga-gate/spec/WAVE-GA-gate-spec-v1.0.0.json that name +# `api-spec` in owning_surface_or_repo_class. +# +# WHAT THIS DOES NOT DO: it uploads the evidence document as a build artifact +# (`ga-evidence-api-spec`). It does NOT open a PR into claude-workstation's +# governance/ga-gate/evidence/incoming/ — that intake is a separate, credential-gated cross-repo +# step (see that directory's README: "the owning repo's own CI ... opens a PR to this repo"), and +# this repo carries no write credential to claude-workstation. Wiring that hand-off is a distinct, +# future change. +# +# FAIL-LOUD POSTURE (mirrors registry-cleanroom.yml exactly): no `|| true`, no +# `continue-on-error` anywhere in this file, on any trigger. Exit 1 (a criterion genuinely fails) +# and exit 2 (a gate could not run at all) both fail the job. A `pass` in the uploaded evidence +# means the producer actually verified the criterion's condition; neither an `unknown` nor a +# tooling failure is ever allowed to read as green here. + +name: ga-evidence + +on: + pull_request: + workflow_dispatch: {} + schedule: + # 09:23 UTC daily — offset from this repo's other scheduled gates (published-contract-drift's + # 07:10 UTC `drift` job, foundation-gate's own schedule) so a shared platform outage window + # does not take every scheduled GA signal out at once. + - cron: '23 9 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + ga-evidence: + name: GA evidence (CONTRACT-001 + COMPAT-001) + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + # COMPAT-001 resolves the highest `v*` tag and reads it with `git show :openapi.yaml`. + # actions/checkout's default shallow clone does not fetch tags reachable only from other + # history, which would make that read fail CLOSED (exit 2, could-not-run) rather than + # fail open — correct, but noisy for no reason on every single run. Full history avoids it. + fetch-depth: 0 + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '22' + + - name: Install tooling + # js-yaml is installed explicitly rather than leaned on transitively, matching the pattern + # published-contract-drift.yml and foundation-gate.yml already use in this repo: no + # node_modules is committed, so every workflow that needs a parser installs it itself. + run: npm install --no-save --no-audit --no-fund js-yaml@4.1.0 + + - name: Verify a Go toolchain is available for the oasdiff fallback + # ubuntu-latest ships Go preinstalled, which is what lets compat-001-check.mjs's + # `go run github.com/oasdiff/oasdiff@` fallback work with no extra + # toolchain-install step or third-party action. This step installs nothing; it only turns + # a silent runner-image change into a clear log line instead of a mysterious exit 2 three + # steps later. If oasdiff is ever preinstalled on these runners instead, the check module + # prefers it automatically (see compat-001-check.mjs's resolution order). + run: go version + + - name: Run the GA evidence producer + id: ga + run: | + set +e + node scripts/ga/ga-evidence.mjs --out-dir "$GITHUB_WORKSPACE/ga-out" 2>&1 | tee "$RUNNER_TEMP/ga-evidence.log" + code=${PIPESTATUS[0]} + set -e + echo "exit_code=$code" >> "$GITHUB_OUTPUT" + { + echo "## GA evidence — CONTRACT-001 / COMPAT-001" + echo + echo "Exit code \`$code\` (0 = every emitted result is \`pass\`, 1 = a criterion is \`fail\`, 2 = a gate could not run — never read as a pass)." + echo + echo '```' + cat "$RUNNER_TEMP/ga-evidence.log" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + + - name: Upload GA evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ga-evidence-api-spec + path: ga-out/ + if-no-files-found: warn + retention-days: 90 + + - name: Enforce + # A gate that cannot fail is not a gate — see registry-cleanroom.yml in wave-av/sdks for + # the exact failure mode this step exists to prevent (a PR merged on a green rollup while + # its own log read "FAILED: 8 check(s)"). No `|| true`, no continue-on-error, on any + # trigger, including pull_request. + env: + CODE: ${{ steps.ga.outputs.exit_code }} + run: | + if [ "$CODE" = "0" ]; then + echo "ga-evidence: every emitted criterion passed" + exit 0 + fi + echo "::error title=ga-evidence::producer exited $CODE (1 = a criterion failed, 2 = a gate could not run) — see the job summary and the ga-evidence-api-spec artifact" + exit 1 diff --git a/.gitignore b/.gitignore index cf5aa0b..cd44e1a 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ node_modules # macOS .DS_Store + +# scripts/ga/ga-evidence.mjs output — regenerated on every run, never committed. +ga-out/ diff --git a/scripts/ga/check-COMPAT-001.sh b/scripts/ga/check-COMPAT-001.sh new file mode 100755 index 0000000..2545339 --- /dev/null +++ b/scripts/ga/check-COMPAT-001.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# check-COMPAT-001.sh — GA evidence check for COMPAT-001. Thin wrapper: the actual diff lives in +# compat-001-check.mjs (which shells out to oasdiff — see that file for how it is resolved) so the +# shell entrypoint, the ga-evidence.mjs producer, and a human running this by hand all exercise the +# identical code path. +# +# OUTPUT: one `PASS|FAIL|UNKNOWN : ` line per sub-check on stdout. +# EXIT CODES: 0 no breaking changes / 1 a breaking change was found / 2 could not run (never a +# pass) — for example no v* tag or no oasdiff/go toolchain available. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec node "$HERE/compat-001-check.mjs" diff --git a/scripts/ga/check-CONTRACT-001.sh b/scripts/ga/check-CONTRACT-001.sh new file mode 100755 index 0000000..513f9e6 --- /dev/null +++ b/scripts/ga/check-CONTRACT-001.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# check-CONTRACT-001.sh — GA evidence check for CONTRACT-001. Thin wrapper: the actual comparison +# lives in contract-001-check.mjs (which reuses this repo's own published-contract-drift +# comparator) so the shell entrypoint, the ga-evidence.mjs producer, and a human running this by +# hand all exercise the identical code path. +# +# OUTPUT: one `PASS|FAIL|UNKNOWN : ` line per sub-check on stdout. +# EXIT CODES: 0 all sub-checks passed / 1 a sub-check failed / 2 could not run (never a pass). +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec node "$HERE/contract-001-check.mjs" diff --git a/scripts/ga/compat-001-check.mjs b/scripts/ga/compat-001-check.mjs new file mode 100644 index 0000000..8efda0e --- /dev/null +++ b/scripts/ga/compat-001-check.mjs @@ -0,0 +1,182 @@ +#!/usr/bin/env node +/** + * compat-001-check.mjs — GA evidence check for COMPAT-001 ("backward compatibility, versioning, + * deprecation, and sunset policy are enforced"). + * + * WHAT THIS CHECKS: does `oasdiff breaking` report zero ERR-level (breaking) changes between the + * last GA release tag's openapi.yaml (the baseline) and HEAD's openapi.yaml (the candidate)? This + * is exactly the gate spec's own `runnable_command` for COMPAT-001: `oasdiff breaking + * baseline/openapi.json candidate/openapi.json`. + * + * WHAT THIS DOES NOT CHECK, ON PURPOSE — see the HONESTY note at the bottom of run(). COMPAT-001's + * full pass_condition is "No unapproved breaking change; versions follow the published policy; + * deprecated features carry notice, migration path and support window; removals expose + * Deprecation/Sunset metadata where applicable." This repo has no machine check today for the + * deprecation-notice / migration-path / support-window half, so that half is always reported + * `unknown` with an explicit failing_checks entry — never folded into a claimed pass. + * + * BASELINE SELECTION: the highest `v*` tag on `origin` by semver sort (`git tag -l 'v*' + * --sort=-v:refname`). Override with GA_COMPAT_BASE_TAG for local reproduction and the + * deliberately-broken-input drill (an unknown ref makes the git read fail honestly, exit 2). + * + * OASDIFF RESOLUTION (this repo's own contract for a diff tool that a CI runner may not have + * preinstalled): + * 1. GA_OASDIFF_CMD, a full shell command line, if set (used by CI and by manual overrides). + * 2. an `oasdiff` binary already on PATH, if present (the common local-dev case). + * 3. `go run github.com/oasdiff/oasdiff@` if a `go` toolchain is present — the + * exact fallback this repo's brief names for CI. Pinned in OASDIFF_GO_MODULE below so a run + * today and a run next year resolve the identical tool. + * 4. none of the above: the check could not run. Never treated as a pass. + * + * EXIT CODES (shared contract with check-CONTRACT-001.sh): + * 0 ran, and zero breaking changes were found + * 1 ran, and at least one breaking change was found + * 2 could not run (no tag, no oasdiff, a git or process failure) — never read as a pass + */ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +export const REPO_ROOT = resolve(HERE, '..', '..'); + +// Pinned so `go run` resolves the identical tool on every run, the same reasoning +// registry-cleanroom.yml gives for SHA-pinning every action: an unpinned `@latest` would let the +// gate's behavior change under a PR nobody touched. Matches the oasdiff release this check was +// authored and verified against. +export const OASDIFF_GO_MODULE = 'github.com/oasdiff/oasdiff@v1.29.1'; + +function git(args) { + return spawnSync('git', args, { cwd: REPO_ROOT, encoding: 'utf8' }); +} + +export function resolveBaseTag(override) { + if (override) return override; + const r = git(['tag', '-l', 'v*', '--sort=-v:refname']); + if (r.status !== 0) return null; + const tags = (r.stdout || '').split('\n').map((s) => s.trim()).filter(Boolean); + return tags[0] ?? null; +} + +function resolveOasdiffCmd() { + if (process.env.GA_OASDIFF_CMD) { + const parts = process.env.GA_OASDIFF_CMD.split(' ').filter(Boolean); + return { bin: parts[0], args: parts.slice(1) }; + } + const which = spawnSync('sh', ['-c', 'command -v oasdiff'], { encoding: 'utf8' }); + if (which.status === 0 && which.stdout.trim()) { + return { bin: which.stdout.trim(), args: [] }; + } + const goWhich = spawnSync('sh', ['-c', 'command -v go'], { encoding: 'utf8' }); + if (goWhich.status === 0 && goWhich.stdout.trim()) { + return { bin: 'go', args: ['run', process.env.GA_OASDIFF_GO_MODULE || OASDIFF_GO_MODULE] }; + } + return null; +} + +function runOasdiff(basePath, candidatePath) { + const cmd = resolveOasdiffCmd(); + if (!cmd) { + return { couldNotRun: true, detail: 'oasdiff is not installed and no go toolchain is available to run it via GA_OASDIFF_CMD / go run fallback' }; + } + const args = [...cmd.args, 'breaking', '-o', 'ERR', '-f', 'json', basePath, candidatePath]; + const res = spawnSync(cmd.bin, args, { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: 180_000 }); + if (res.error) { + return { couldNotRun: true, detail: `could not execute "${cmd.bin} ${args.join(' ')}": ${res.error.message}` }; + } + // oasdiff's own contract for `breaking -o ERR`: 0 = no ERR-level finding, 1 = at least one. Any + // other code (a crash, a bad invocation, a network failure inside `go run`) is a tooling failure. + if (res.status !== 0 && res.status !== 1) { + return { + couldNotRun: true, + detail: `oasdiff exited ${res.status} (expected 0 or 1): ${(res.stderr || res.stdout || 'no output').slice(0, 500)}`, + }; + } + let parsed; + try { + parsed = JSON.parse(res.stdout || '[]'); + } catch (err) { + return { couldNotRun: true, detail: `could not parse oasdiff JSON output: ${err.message}` }; + } + const breaking = parsed.filter((f) => f.level === 3); + return { couldNotRun: false, breaking, exitStatus: res.status }; +} + +export async function run(opts = {}) { + const baseTagOverride = opts.baseTag ?? process.env.GA_COMPAT_BASE_TAG ?? null; + const tag = resolveBaseTag(baseTagOverride); + if (!tag) { + return fail('baseline-tag', 'no v* tag found on origin (git tag -l "v*" --sort=-v:refname returned nothing) and no GA_COMPAT_BASE_TAG override was set'); + } + + const headRev = git(['rev-parse', 'HEAD']); + if (headRev.status !== 0) return fail('candidate-read', `git rev-parse HEAD failed: ${headRev.stderr}`); + const head = headRev.stdout.trim(); + + const showBase = git(['show', `${tag}:openapi.yaml`]); + if (showBase.status !== 0) return fail('baseline-tag', `git show ${tag}:openapi.yaml failed: ${(showBase.stderr || '').trim().slice(0, 300)}`); + + const showHead = git(['show', `${head}:openapi.yaml`]); + if (showHead.status !== 0) return fail('candidate-read', `git show ${head}:openapi.yaml failed: ${(showHead.stderr || '').trim().slice(0, 300)}`); + + const dir = mkdtempSync(join(tmpdir(), 'ga-compat-')); + const basePath = join(dir, 'base.yaml'); + const candidatePath = join(dir, 'candidate.yaml'); + let oasdiffResult; + try { + writeFileSync(basePath, showBase.stdout); + writeFileSync(candidatePath, showHead.stdout); + oasdiffResult = runOasdiff(basePath, candidatePath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + if (oasdiffResult.couldNotRun) return fail('oasdiff-availability', oasdiffResult.detail, tag); + + const { breaking } = oasdiffResult; + const checks = [ + { + name: 'breaking-changes', + ok: breaking.length === 0, + detail: breaking.length === 0 + ? `zero breaking (ERR-level) changes between ${tag} and HEAD (${head.slice(0, 12)})` + : `${breaking.length} breaking (ERR-level) change(s) between ${tag} and HEAD: ${breaking.slice(0, 8).map((b) => `${b.id}@${b.operation} ${b.path}`).join('; ')}${breaking.length > 8 ? '; …' : ''}`, + }, + // HONESTY: never claim the deprecation-notice/migration-path half of COMPAT-001's + // pass_condition. This repo has no machine check for it today. + { + name: 'deprecation-notice', + ok: 'unknown', + detail: 'deprecation notice / migration path / support window not machine-verified by this repo', + }, + ]; + + return { couldNotRun: false, checks, tag, head, breakingCount: breaking.length }; +} + +function fail(name, detail, tag) { + return { couldNotRun: true, checks: [{ name, ok: null, detail }], tag }; +} + +async function cli() { + const result = await run(); + for (const c of result.checks) { + const label = c.ok === null || c.ok === 'unknown' ? 'UNKNOWN' : c.ok ? 'PASS' : 'FAIL'; + process.stdout.write(`${label} COMPAT-001/${c.name}: ${c.detail}\n`); + } + if (result.couldNotRun) { + process.exitCode = 2; + return; + } + const hasFail = result.checks.some((c) => c.ok === false); + process.exitCode = hasFail ? 1 : 0; +} + +const isMain = process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)); +if (isMain) { + cli().catch((err) => { + process.stderr.write(`compat-001-check could not run: ${err?.stack || err}\n`); + process.exit(2); + }); +} diff --git a/scripts/ga/contract-001-check.mjs b/scripts/ga/contract-001-check.mjs new file mode 100644 index 0000000..4b154fc --- /dev/null +++ b/scripts/ga/contract-001-check.mjs @@ -0,0 +1,229 @@ +#!/usr/bin/env node +/** + * contract-001-check.mjs — GA evidence check for CONTRACT-001 ("one promoted contract is the + * source of truth across spec, gateway, registry, MCP, SDK and CLI"). + * + * REUSE, NOT REIMPLEMENTATION. This repo already owns the comparison between the declared spec + * and the gateway's live published contract — see .github/workflows/published-contract-drift.yml + * and .github/scripts/published-drift*.mjs. That normalizer exists for a measured reason (its own + * header: "all 72 shared operations report a difference for enrichment reasons alone" without it), + * so this check imports those modules directly rather than re-deriving normalization rules that + * would silently drift out of sync with the ones the `drift` job actually uses. + * + * WHAT THIS CHECKS, PRECISELY (narrower than the informational `published-contract-drift` job, + * which is scheduled/advisory by design — see that workflow's header): + * + * 1. operation-parity — every operation this repo declares in openapi.yaml at HEAD that is not + * `x-schema-status: draft` is served live, and every operation the gateway serves is either + * declared here or explicitly allowlisted (published-drift-allowlist.json, with an owner and + * a lapsing predicate). This is CONTRACT-001's own text: "repo-only and live-only operations + * are zero unless explicitly allowlisted." + * 2. content-digest — for every operation declared on BOTH sides, this builds two independent + * sha256 digests: one walking the repo's copy of each shared operation (after stripping the + * gateway's serve-time enrichment via the shared normalizePair()), one walking the live + * document's copy the same way. These two digests must be byte-identical. Two independent + * walks are used deliberately, rather than one shared list, so a bug that fabricated one side + * from the other could not produce a false match. + * + * Both conditions must hold; neither substitutes for the other. (1) alone would miss in-place + * content drift on an operation whose path+method did not move. (2) alone would miss an entire + * operation appearing or disappearing. + * + * NOT CHECKED HERE: the registry/MCP/SDK/CLI surfaces CONTRACT-001 also names. Those are owned by + * wave-av/sdks (see its registry-cleanroom producer) and are out of this repo's scope. + * + * EXIT CODES (this repo's GA-evidence contract, shared with check-COMPAT-001.sh): + * 0 ran, and the criterion holds (both checks pass) + * 1 ran, and the criterion does not hold (a genuine finding) + * 2 could not run (fetch/parse failure, missing input) — never to be read as a pass + * + * OVERRIDES (for local reproduction and the deliberately-broken-input drill; never used in the + * scheduled/PR workflow, which always reads the real live URL with no auth): + * GA_CONTRACT_LIVE_URL fetch this URL instead of the default published contract + * GA_CONTRACT_LIVE_FILE read this local JSON file instead of fetching anything (fully offline) + */ +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { pathToFileURL } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +export const REPO_ROOT = resolve(HERE, '..', '..'); + +export const DEFAULT_LIVE_URL = 'https://api.wave.online/openapi.json'; +const FETCH_TIMEOUT_MS = 20_000; + +async function loadYaml(path) { + const yaml = await import('js-yaml'); + return (yaml.default ?? yaml).load(readFileSync(path, 'utf8')); +} + +export async function fetchLive(url, doFetch = fetch) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + try { + // redirect: 'manual' — a single hardcoded HTTPS URL has no legitimate reason to redirect a CI + // job with no credentials to leak; refusing to follow is the honest "could not run", never a + // grade of whatever the redirect target happened to serve. Same posture as published-drift.mjs. + const res = await doFetch(url, { signal: controller.signal, redirect: 'manual' }); + if (res.status >= 300 && res.status < 400) { + return { ok: false, error: `${url} redirected (HTTP ${res.status}) — refusing to follow` }; + } + if (!res.ok) return { ok: false, error: `HTTP ${res.status} from ${url}` }; + return { ok: true, doc: await res.json() }; + } catch (err) { + const reason = err?.name === 'AbortError' ? `timed out after ${FETCH_TIMEOUT_MS}ms` : (err?.message ?? String(err)); + return { ok: false, error: `${url}: ${reason}` }; + } finally { + clearTimeout(timer); + } +} + +function sha256(s) { + return createHash('sha256').update(s).digest('hex'); +} + +function sortKeysDeep(v) { + if (Array.isArray(v)) return v.map(sortKeysDeep); + if (v && typeof v === 'object') { + return Object.keys(v).sort().reduce((o, k) => { + o[k] = sortKeysDeep(v[k]); + return o; + }, {}); + } + return v; +} + +function canonicalJson(value) { + return JSON.stringify(sortKeysDeep(value ?? null)); +} + +/** + * Run the CONTRACT-001 check. Returns `{ couldNotRun, checks, ... }`. Never throws for an + * ordinary tooling failure — those come back as `couldNotRun: true` with a check named for what + * failed, so the caller can render `UNKNOWN : ` and exit 2. + */ +export async function run(opts = {}) { + const repoSpecPath = opts.repoSpecPath ?? join(REPO_ROOT, 'openapi.yaml'); + const liveFile = opts.liveFile ?? process.env.GA_CONTRACT_LIVE_FILE ?? null; + const liveUrl = opts.liveUrl ?? process.env.GA_CONTRACT_LIVE_URL ?? DEFAULT_LIVE_URL; + + const compareMod = await import(pathToFileURL(join(REPO_ROOT, '.github/scripts/published-drift-compare.mjs'))); + const normalizeMod = await import(pathToFileURL(join(REPO_ROOT, '.github/scripts/published-drift-normalize.mjs'))); + const { indexOperations, compare, validateAllowlist } = compareMod; + const { normalizePair } = normalizeMod; + const allowlistPath = join(REPO_ROOT, '.github/scripts/published-drift-allowlist.json'); + + let repoDoc; + try { + repoDoc = await loadYaml(repoSpecPath); + } catch (err) { + return fail('repo-spec-read', `could not read/parse ${repoSpecPath}: ${err.message}`); + } + if (!repoDoc?.paths || typeof repoDoc.paths !== 'object') { + return fail('repo-spec-read', `${repoSpecPath} has no usable "paths" object`); + } + + let allowlist; + try { + allowlist = JSON.parse(readFileSync(allowlistPath, 'utf8')); + } catch (err) { + return fail('allowlist-read', `could not read/parse ${allowlistPath}: ${err.message}`); + } + const allowlistErr = validateAllowlist(allowlist); + if (allowlistErr) return fail('allowlist-read', allowlistErr); + + let liveDoc; + let source; + if (liveFile) { + try { + liveDoc = JSON.parse(readFileSync(liveFile, 'utf8')); + source = `file:${liveFile}`; + } catch (err) { + return fail('live-fetch', `could not read/parse snapshot ${liveFile}: ${err.message}`); + } + } else { + const fetched = await fetchLive(liveUrl); + if (!fetched.ok) return fail('live-fetch', `could not fetch ${liveUrl}: ${fetched.error}`); + liveDoc = fetched.doc; + source = liveUrl; + } + if (!liveDoc?.paths || typeof liveDoc.paths !== 'object' || Object.keys(liveDoc.paths).length === 0) { + return fail('live-fetch', `${source} has no usable "paths" object`); + } + + const result = compare({ repoDoc, liveDoc, allowlist, normalize: true }); + const repoOps = indexOperations(repoDoc); + const liveOps = indexOperations(liveDoc); + const sharedKeys = [...repoOps.keys()].filter((k) => liveOps.has(k)).sort(); + + const rowsRepo = []; + const rowsLive = []; + for (const key of sharedKeys) { + const { path, method, op: repoOp } = repoOps.get(key); + const { op: liveOp } = liveOps.get(key); + const { repo, live } = normalizePair(repoOp, liveOp, path, method); + rowsRepo.push(`${key}\t${sha256(canonicalJson(repo))}`); + rowsLive.push(`${key}\t${sha256(canonicalJson(live))}`); + } + const localDigest = sha256([...rowsRepo].sort().join('\n')); + const liveDigest = sha256([...rowsLive].sort().join('\n')); + + const findings = result.findings ?? []; + const findingLabels = findings.slice(0, 10).map((f) => `${f.direction} ${f.method} ${f.path}`); + + const checks = [ + { + name: 'operation-parity', + ok: findings.length === 0, + detail: findings.length === 0 + ? `zero unexplained repo-only/live-only operations (${repoOps.size} declared, ${liveOps.size} live, ${sharedKeys.length} shared, ${result.allowlisted?.length ?? 0} allowlisted, ${result.draftNotYetPublished?.length ?? 0} draft)` + : `${findings.length} unexplained operation-level finding(s): ${findingLabels.join('; ')}${findings.length > findingLabels.length ? '; …' : ''}`, + }, + { + name: 'content-digest', + ok: localDigest === liveDigest, + detail: localDigest === liveDigest + ? `repo and live digests match over ${sharedKeys.length} shared operation(s) (${localDigest.slice(0, 12)})` + : `repo digest ${localDigest.slice(0, 12)} != live digest ${liveDigest.slice(0, 12)} over ${sharedKeys.length} shared operation(s)`, + }, + ]; + + return { + couldNotRun: false, + checks, + localDigest, + liveDigest, + source, + repoOpCount: repoOps.size, + liveOpCount: liveOps.size, + sharedCount: sharedKeys.length, + }; +} + +function fail(name, detail) { + return { couldNotRun: true, checks: [{ name, ok: null, detail }] }; +} + +async function cli() { + const result = await run(); + for (const c of result.checks) { + const label = c.ok === null ? 'UNKNOWN' : c.ok ? 'PASS' : 'FAIL'; + process.stdout.write(`${label} CONTRACT-001/${c.name}: ${c.detail}\n`); + } + if (result.couldNotRun) { + process.exitCode = 2; + return; + } + const allOk = result.checks.every((c) => c.ok === true); + process.exitCode = allOk ? 0 : 1; +} + +const isMain = process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)); +if (isMain) { + cli().catch((err) => { + process.stderr.write(`contract-001-check could not run: ${err?.stack || err}\n`); + process.exit(2); + }); +} diff --git a/scripts/ga/ga-evidence.mjs b/scripts/ga/ga-evidence.mjs new file mode 100644 index 0000000..6bf0ce7 --- /dev/null +++ b/scripts/ga/ga-evidence.mjs @@ -0,0 +1,266 @@ +#!/usr/bin/env node +/** + * ga-evidence.mjs — GA-evidence PRODUCER for wave-av/api-spec, on the wave-av/sdks pattern (see + * that repo's scripts/ga/registry-cleanroom.mjs and .github/workflows/registry-cleanroom.yml). + * + * Runs this repo's two GA-gate criteria — CONTRACT-001 (contract-001-check.mjs) and COMPAT-001 + * (compat-001-check.mjs) — and writes: + * ga-out/ga-report.json full detail: every sub-check, both raw runs + * ga-out/wave-av__api-spec.ga-evidence.json the schema-shaped document, ready to drop into + * governance/ga-gate/evidence/incoming/ in + * claude-workstation (a separate, credential- + * gated cross-repo step — not done by this repo) + * + * STATUS RULES — computed, never hardcoded (see each check module for what it actually verifies): + * CONTRACT-001 pass both operation-parity and content-digest checks passed + * fail either check found a genuine difference + * unknown the gate could not run (fetch/parse/allowlist failure) — a could-not-run + * is never reported as a pass + * COMPAT-001 fail oasdiff found at least one ERR-level (breaking) change + * unknown the breaking-change check ran clean OR the gate could not run — EITHER + * way this criterion's `deprecation notice / migration path / support + * window` half is never machine-verified by this repo, so a clean breaking- + * change run can still never be reported as a full `pass`. failing_checks + * always names exactly what stays unverified. + * + * EXIT CODE: mirrors the two checks' own contract — 0 if every emitted result is `pass`, 1 if any + * result is `fail`, 2 if any result is `unknown` because its gate could not run at all (as + * distinct from "ran, and one half is honestly unverifiable", which is a normal `unknown` and does + * not itself force a non-zero producer exit — see COULD_NOT_RUN below). + */ +import { createHash } from 'node:crypto'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +import { run as runContract001 } from './contract-001-check.mjs'; +import { run as runCompat001 } from './compat-001-check.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(HERE, '..', '..'); +const REPOSITORY = 'wave-av/api-spec'; +const SPEC_VERSION = '1.0.0'; +const EVIDENCE_URI = 'ci://wave-av/api-spec/.github/workflows/ga-evidence.yml#ga-report.json'; + +function parseArgs(argv) { + const out = { outDir: join(REPO_ROOT, 'ga-out') }; + for (let i = 0; i < argv.length; i += 1) { + if (argv[i] === '--out-dir') out.outDir = resolve(argv[++i]); + } + return out; +} + +function sha256Hex(s) { + return createHash('sha256').update(s).digest('hex'); +} + +function sortKeysDeep(v) { + if (Array.isArray(v)) return v.map(sortKeysDeep); + if (v && typeof v === 'object') { + return Object.keys(v).sort().reduce((o, k) => { + o[k] = sortKeysDeep(v[k]); + return o; + }, {}); + } + return v; +} + +function canonicalJson(value) { + return JSON.stringify(sortKeysDeep(value)); +} + +function gitRevParseHead() { + return execFileSync('git', ['rev-parse', 'HEAD'], { cwd: REPO_ROOT, encoding: 'utf8' }).trim(); +} + +/** ISO 8601 UTC, truncated to whole seconds — no milliseconds, per the evidence schema's utcTimestamp. */ +function nowIsoSeconds() { + return new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'); +} + +/** + * Build the CONTRACT-001 result row from contract-001-check.mjs's raw output. + * `couldNotRun` -> status 'unknown' (a failed read says nothing about the criterion and must + * never be graded as a pass); otherwise 'pass' only when every sub-check passed, else 'fail'. + */ +function buildContractRow(raw) { + const command = 'scripts/ga/check-CONTRACT-001.sh'; + if (raw.couldNotRun) { + const detail = raw.checks[0]?.detail ?? 'unknown failure'; + return { + criterion_id: 'CONTRACT-001', + status: 'unknown', + command, + failing_checks: [`gate could not run — ${raw.checks[0]?.name}: ${detail}`.slice(0, 500)], + targets_observed: [], + fingerprintPayload: { criterion_id: 'CONTRACT-001', couldNotRun: true, checks: raw.checks.map((c) => [c.name, c.ok]) }, + }; + } + const allPass = raw.checks.every((c) => c.ok === true); + const failing = raw.checks.filter((c) => c.ok !== true).map((c) => `${c.name}: ${c.detail}`.slice(0, 500)); + const targets = [ + `openapi.yaml@${raw.localDigest.slice(0, 12)}`, + `live-openapi.json@${raw.liveDigest.slice(0, 12)}`, + ]; + return { + criterion_id: 'CONTRACT-001', + status: allPass ? 'pass' : 'fail', + command, + failing_checks: allPass ? undefined : failing, + targets_observed: targets, + fingerprintPayload: { + criterion_id: 'CONTRACT-001', + checks: raw.checks.map((c) => [c.name, c.ok]).sort(), + targets: [...targets].sort(), + }, + }; +} + +/** + * Build the COMPAT-001 result row. This criterion's status is NEVER 'pass' from this producer — + * see the module header. A clean breaking-change run still reports 'unknown' because the + * deprecation-notice half is unverified; a dirty run reports 'fail'; a gate that could not run + * also reports 'unknown', distinguished in failing_checks. + */ +function buildCompatRow(raw) { + const command = 'scripts/ga/check-COMPAT-001.sh'; + if (raw.couldNotRun) { + const detail = raw.checks[0]?.detail ?? 'unknown failure'; + return { + criterion_id: 'COMPAT-001', + status: 'unknown', + command, + failing_checks: [`gate could not run — ${raw.checks[0]?.name}: ${detail}`.slice(0, 500)], + targets_observed: [], + fingerprintPayload: { criterion_id: 'COMPAT-001', couldNotRun: true, checks: raw.checks.map((c) => [c.name, c.ok]) }, + }; + } + const breakingCheck = raw.checks.find((c) => c.name === 'breaking-changes'); + const targets = [`openapi.yaml@${raw.tag}`, `openapi.yaml@${raw.head.slice(0, 12)}`]; + const failing = []; + let status; + if (breakingCheck.ok === false) { + status = 'fail'; + failing.push(`breaking-changes: ${breakingCheck.detail}`.slice(0, 500)); + } else { + status = 'unknown'; + } + // Always named, honestly, per the HONESTY RULES in this producer's brief: never claim the + // deprecation-notice half of COMPAT-001's pass_condition. + failing.push('deprecation notice/migration path not machine-verified'); + return { + criterion_id: 'COMPAT-001', + status, + command, + failing_checks: failing, + targets_observed: targets, + fingerprintPayload: { + criterion_id: 'COMPAT-001', + checks: raw.checks.map((c) => [c.name, c.ok]).sort(), + targets: [...targets].sort(), + }, + }; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const revision = gitRevParseHead(); + const verifiedAt = nowIsoSeconds(); + + process.stdout.write(`ga-evidence — wave-av/api-spec @ ${revision.slice(0, 12)}\n\n`); + + const [contractRaw, compatRaw] = await Promise.all([runContract001(), runCompat001()]); + + process.stdout.write('-- CONTRACT-001 --\n'); + for (const c of contractRaw.checks) { + const label = c.ok === null ? 'UNKNOWN' : c.ok ? 'PASS' : 'FAIL'; + process.stdout.write(`${label} ${c.name}: ${c.detail}\n`); + } + process.stdout.write('\n-- COMPAT-001 --\n'); + for (const c of compatRaw.checks) { + const label = c.ok === null || c.ok === 'unknown' ? 'UNKNOWN' : c.ok ? 'PASS' : 'FAIL'; + process.stdout.write(`${label} ${c.name}: ${c.detail}\n`); + } + process.stdout.write('\n'); + + const contractRow = buildContractRow(contractRaw); + const compatRow = buildCompatRow(compatRaw); + + // ONE fingerprint over BOTH rows, sorted by criterion id before hashing — same idempotency + // rule the gate spec states and the sdks producer follows (registry-cleanroom.mjs buildEvidence + // computes one `fingerprint` shared by every row). Deliberately excludes timestamps, temp paths + // and durations: two runs observing the same artifacts must produce the same digest. + const fingerprintPayload = [contractRow.fingerprintPayload, compatRow.fingerprintPayload] + .sort((a, b) => a.criterion_id.localeCompare(b.criterion_id)); + const fingerprint = sha256Hex(canonicalJson(fingerprintPayload)); + + const results = [contractRow, compatRow] + .sort((a, b) => a.criterion_id.localeCompare(b.criterion_id)) + .map((r) => { + const { fingerprintPayload: _drop, failing_checks, ...rest } = r; + const result = { + ...rest, + evidence_sha256: fingerprint, + evidence_uri: EVIDENCE_URI, + verified_at: verifiedAt, + }; + if (failing_checks && failing_checks.length > 0) result.failing_checks = failing_checks; + if (result.targets_observed && result.targets_observed.length === 0) delete result.targets_observed; + return result; + }); + + const evidence = { + spec_version: SPEC_VERSION, + repository: REPOSITORY, + revision, + generated_at: verifiedAt, + results, + }; + + const report = { + schema: 'wave-api-spec-ga-evidence/1', + spec_version: SPEC_VERSION, + repository: REPOSITORY, + revision, + generated_at: verifiedAt, + evidence_sha256: fingerprint, + runner: { node: process.version, platform: process.platform }, + checks: { + 'CONTRACT-001': contractRaw, + 'COMPAT-001': compatRaw, + }, + evidence, + }; + + mkdirSync(args.outDir, { recursive: true }); + writeFileSync(join(args.outDir, 'ga-report.json'), `${JSON.stringify(report, null, 2)}\n`); + // additionalProperties:false at every level of the evidence schema — evidence.json carries + // ONLY the schema's own shape, never the raw check detail (that lives in ga-report.json). + const evidenceOnly = { spec_version: SPEC_VERSION, repository: REPOSITORY, revision, generated_at: verifiedAt, results }; + writeFileSync(join(args.outDir, 'wave-av__api-spec.ga-evidence.json'), `${JSON.stringify(evidenceOnly, null, 2)}\n`); + + process.stdout.write(`${'-'.repeat(78)}\n`); + for (const r of results) process.stdout.write(`${r.criterion_id}: ${r.status.toUpperCase()}\n`); + process.stdout.write(`\nevidence fingerprint: ${fingerprint}\n`); + process.stdout.write(`wrote ${join(args.outDir, 'ga-report.json')} and ${join(args.outDir, 'wave-av__api-spec.ga-evidence.json')}\n`); + + const anyCouldNotRun = contractRaw.couldNotRun || compatRaw.couldNotRun; + const anyFail = results.some((r) => r.status === 'fail'); + if (anyCouldNotRun) { + process.stdout.write('\nGA-EVIDENCE COULD NOT FULLY RUN\n'); + process.exitCode = 2; + return; + } + if (anyFail) { + process.stdout.write('\nGA-EVIDENCE FOUND FAILING CRITERIA\n'); + process.exitCode = 1; + return; + } + process.stdout.write('\nga-evidence: ran cleanly (see per-criterion status above — unknown is not a failure but is not a pass either)\n'); +} + +main().catch((e) => { + process.stderr.write(`ga-evidence could not run: ${e?.stack || e}\n`); + process.exit(2); +}); From 92a9f0f5194e0a713ec41618e4f5ebeeb57f95c1 Mon Sep 17 00:00:00 2001 From: Jake Fineman Date: Sat, 5 Sep 2026 17:52:58 -0400 Subject: [PATCH 2/3] ci(ga-evidence): PR job warns on live criterion failure, fails only when the producer cannot run Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ga-evidence.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ga-evidence.yml b/.github/workflows/ga-evidence.yml index 91a99a1..5155578 100644 --- a/.github/workflows/ga-evidence.yml +++ b/.github/workflows/ga-evidence.yml @@ -16,11 +16,10 @@ # this repo carries no write credential to claude-workstation. Wiring that hand-off is a distinct, # future change. # -# FAIL-LOUD POSTURE (mirrors registry-cleanroom.yml exactly): no `|| true`, no -# `continue-on-error` anywhere in this file, on any trigger. Exit 1 (a criterion genuinely fails) -# and exit 2 (a gate could not run at all) both fail the job. A `pass` in the uploaded evidence -# means the producer actually verified the criterion's condition; neither an `unknown` nor a -# tooling failure is ever allowed to read as green here. +# PR CONTRACT: on `pull_request`, exit 1 (a live criterion failed) is a `::warning`, not a job +# failure — a failing live criterion is a property of the live surface, not of the PR's diff. +# Exit 2 (the producer could not run) always fails the job, on every trigger, including +# `pull_request` — and on schedule/workflow_dispatch/push, exit 1 fails the job too. name: ga-evidence @@ -108,13 +107,19 @@ jobs: # A gate that cannot fail is not a gate — see registry-cleanroom.yml in wave-av/sdks for # the exact failure mode this step exists to prevent (a PR merged on a green rollup while # its own log read "FAILED: 8 check(s)"). No `|| true`, no continue-on-error, on any - # trigger, including pull_request. + # trigger. See the PR CONTRACT note in the header: a live-criterion failure (exit 1) on + # `pull_request` is the one case that logs and exits 0 instead of failing the job. env: CODE: ${{ steps.ga.outputs.exit_code }} + EVENT: ${{ github.event_name }} run: | if [ "$CODE" = "0" ]; then echo "ga-evidence: every emitted criterion passed" exit 0 fi + if [ "$CODE" = "1" ] && [ "$EVENT" = "pull_request" ]; then + echo "::warning title=ga-evidence::api-spec GA evidence producer reports a failing live criterion (exit 1); evidence is in the job summary and artifact; this does not fail the PR because the criterion is a property of the live surface, not of this change" + exit 0 + fi echo "::error title=ga-evidence::producer exited $CODE (1 = a criterion failed, 2 = a gate could not run) — see the job summary and the ga-evidence-api-spec artifact" exit 1 From 03d02006baf29506d4954756b55f5001730816b9 Mon Sep 17 00:00:00 2001 From: Jake Fineman Date: Sat, 5 Sep 2026 23:17:15 -0400 Subject: [PATCH 3/3] fix(ga-evidence): close SSRF gap, fix ref-blind digest and parity/fingerprint gaps, add tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses review threads on the GA evidence producer (PR #86): - compat-001-check.mjs: pass --allow-external-refs=false to `oasdiff breaking` (coderabbit Security/Major + cubic P1) so a PR-supplied openapi.yaml with an external $ref cannot make the CI runner fetch an attacker-controlled URL. - compat-001-check.mjs: replace GA_OASDIFF_CMD.split(' ') with a real splitShellCommand() argv tokenizer (quotes + backslash escapes), fixing the quoted-path corruption gitar and cubic both flagged. - contract-001-check.mjs: fold reachable $ref content (resolveJsonPointer + collectReachableRefs) into the per-operation content digest, so a component schema change behind an unchanged $ref is no longer invisible to CONTRACT-001 (cubic P1). - contract-001-check.mjs: operation-parity now only counts undocumented-live/unpublished-repo findings, not shared-drift (already content-digest's job), so the two sub-checks name distinct failures (cubic P2). - ga-evidence.mjs: fingerprintPayload now includes each check's `detail` text, not just `ok`, so an evidence-relevant change that doesn't flip a boolean no longer gets deduplicated as stale evidence (cubic P2). Also adds the isMain guard this file was missing, so importing it for buildContractRow/ buildCompatRow no longer runs the live producer as a side effect. - ga-evidence.yml: reword the job-summary and Enforce success text so exit 0 is never read as COMPAT-001 == pass (cubic P3). - Add scripts/ga/{compat-001-check,contract-001-check,ga-evidence}.test.mjs (35 assertions, hermetic/offline/no shared-tag mutation) plus a `test:ga` npm script, closing the no-test-coverage asks from gitar and cubic. Declined: cubic's ga-evidence.yml:27 suggestion to path-filter the workflow and make producer exit 2 advisory on pull_request — the PR-job contract (exit 2 always fails, exit 1 warns on PR) is a deliberate, already-reviewed design documented in this same file's header; changing trigger/enforcement semantics is out of scope here. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ga-evidence.yml | 4 +- package.json | 3 +- scripts/ga/compat-001-check.mjs | 86 ++++++++-- scripts/ga/compat-001-check.test.mjs | 180 +++++++++++++++++++++ scripts/ga/contract-001-check.mjs | 77 ++++++++- scripts/ga/contract-001-check.test.mjs | 214 +++++++++++++++++++++++++ scripts/ga/ga-evidence.mjs | 31 ++-- scripts/ga/ga-evidence.test.mjs | 107 +++++++++++++ 8 files changed, 672 insertions(+), 30 deletions(-) create mode 100644 scripts/ga/compat-001-check.test.mjs create mode 100644 scripts/ga/contract-001-check.test.mjs create mode 100644 scripts/ga/ga-evidence.test.mjs diff --git a/.github/workflows/ga-evidence.yml b/.github/workflows/ga-evidence.yml index 5155578..afa2f71 100644 --- a/.github/workflows/ga-evidence.yml +++ b/.github/workflows/ga-evidence.yml @@ -86,7 +86,7 @@ jobs: { echo "## GA evidence — CONTRACT-001 / COMPAT-001" echo - echo "Exit code \`$code\` (0 = every emitted result is \`pass\`, 1 = a criterion is \`fail\`, 2 = a gate could not run — never read as a pass)." + echo "Exit code \`$code\` (0 = every runnable check passed — COMPAT-001 still reports \`unknown\` unless it found a breaking change, per its own honesty rule, not \`pass\`; 1 = a criterion is \`fail\`; 2 = a gate could not run — never read as a pass)." echo echo '```' cat "$RUNNER_TEMP/ga-evidence.log" @@ -114,7 +114,7 @@ jobs: EVENT: ${{ github.event_name }} run: | if [ "$CODE" = "0" ]; then - echo "ga-evidence: every emitted criterion passed" + echo "ga-evidence: all runnable checks passed — COMPAT-001 remains unknown unless it found a breaking change (see the job summary; never read exit 0 as COMPAT-001 == pass)" exit 0 fi if [ "$CODE" = "1" ] && [ "$EVENT" = "pull_request" ]; then diff --git a/package.json b/package.json index 5fcf127..1bb519a 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,8 @@ "description": "OpenAPI 3.1 specification for WAVE, media infrastructure for the agentic internet: openapi.yaml plus the tooling that validates it and generates SDK types from it.", "scripts": { "lint": "redocly lint openapi.yaml", - "gen:types": "openapi-typescript openapi.yaml -o generated/api-types.d.ts" + "gen:types": "openapi-typescript openapi.yaml -o generated/api-types.d.ts", + "test:ga": "node --test \"scripts/ga/**/*.test.mjs\"" }, "devDependencies": { "@redocly/cli": "2.40.0", diff --git a/scripts/ga/compat-001-check.mjs b/scripts/ga/compat-001-check.mjs index 8efda0e..a3533e7 100644 --- a/scripts/ga/compat-001-check.mjs +++ b/scripts/ga/compat-001-check.mjs @@ -48,30 +48,90 @@ export const REPO_ROOT = resolve(HERE, '..', '..'); // authored and verified against. export const OASDIFF_GO_MODULE = 'github.com/oasdiff/oasdiff@v1.29.1'; -function git(args) { - return spawnSync('git', args, { cwd: REPO_ROOT, encoding: 'utf8' }); +function git(args, cwd = REPO_ROOT) { + return spawnSync('git', args, { cwd, encoding: 'utf8' }); } -export function resolveBaseTag(override) { +// `cwd` defaults to this repo's root for every real caller; it exists as a parameter (rather than +// only reading the module-level REPO_ROOT) so tests can point it at a disposable scratch git repo +// and exercise "no v* tag" / "one tag" / "highest of several tags" without touching this repo's own +// tags (creating a real tag here would be a destructive, shared-state side effect of a test run). +export function resolveBaseTag(override, cwd = REPO_ROOT) { if (override) return override; - const r = git(['tag', '-l', 'v*', '--sort=-v:refname']); + const r = git(['tag', '-l', 'v*', '--sort=-v:refname'], cwd); if (r.status !== 0) return null; const tags = (r.stdout || '').split('\n').map((s) => s.trim()).filter(Boolean); return tags[0] ?? null; } -function resolveOasdiffCmd() { - if (process.env.GA_OASDIFF_CMD) { - const parts = process.env.GA_OASDIFF_CMD.split(' ').filter(Boolean); +/** + * Minimal argv tokenizer for GA_OASDIFF_CMD ("a full shell command line" per this override's own + * doc comment above). Handles single/double quotes and backslash escapes so a quoted path with + * spaces survives — naive `split(' ')` corrupted exactly that case. This is NOT a shell: no + * globbing, no variable expansion, no pipes/redirection, and the result is passed to `spawnSync` + * as an argv array (never through a shell string), so there is no command-injection surface here + * either way — this only fixes correctness of the split, not a security property. + */ +export function splitShellCommand(input) { + const parts = []; + let cur = ''; + let hasCur = false; + let quote = null; + for (let i = 0; i < input.length; i += 1) { + const ch = input[i]; + if (quote) { + if (ch === quote) { + quote = null; + } else if (ch === '\\' && quote === '"' && (input[i + 1] === '"' || input[i + 1] === '\\')) { + cur += input[i + 1]; + i += 1; + } else { + cur += ch; + } + hasCur = true; + continue; + } + if (ch === "'" || ch === '"') { + quote = ch; + hasCur = true; + continue; + } + if (ch === '\\' && i + 1 < input.length) { + cur += input[i + 1]; + i += 1; + hasCur = true; + continue; + } + if (/\s/.test(ch)) { + if (hasCur) { + parts.push(cur); + cur = ''; + hasCur = false; + } + continue; + } + cur += ch; + hasCur = true; + } + if (hasCur) parts.push(cur); + return parts; +} + +// Exported for tests, which override PATH (via `env`) to exercise "GA_OASDIFF_CMD set", "oasdiff on +// PATH", "no oasdiff but go on PATH", and "neither" without depending on what happens to be +// installed on the machine actually running the suite. +export function resolveOasdiffCmd(env = process.env) { + if (env.GA_OASDIFF_CMD) { + const parts = splitShellCommand(env.GA_OASDIFF_CMD).filter(Boolean); return { bin: parts[0], args: parts.slice(1) }; } - const which = spawnSync('sh', ['-c', 'command -v oasdiff'], { encoding: 'utf8' }); + const which = spawnSync('sh', ['-c', 'command -v oasdiff'], { encoding: 'utf8', env }); if (which.status === 0 && which.stdout.trim()) { return { bin: which.stdout.trim(), args: [] }; } - const goWhich = spawnSync('sh', ['-c', 'command -v go'], { encoding: 'utf8' }); + const goWhich = spawnSync('sh', ['-c', 'command -v go'], { encoding: 'utf8', env }); if (goWhich.status === 0 && goWhich.stdout.trim()) { - return { bin: 'go', args: ['run', process.env.GA_OASDIFF_GO_MODULE || OASDIFF_GO_MODULE] }; + return { bin: 'go', args: ['run', env.GA_OASDIFF_GO_MODULE || OASDIFF_GO_MODULE] }; } return null; } @@ -81,7 +141,11 @@ function runOasdiff(basePath, candidatePath) { if (!cmd) { return { couldNotRun: true, detail: 'oasdiff is not installed and no go toolchain is available to run it via GA_OASDIFF_CMD / go run fallback' }; } - const args = [...cmd.args, 'breaking', '-o', 'ERR', '-f', 'json', basePath, candidatePath]; + // --allow-external-refs=false: oasdiff resolves external $ref values by default. This runs on + // pull_request against a PR-supplied openapi.yaml, so an unreviewed external $ref must never let + // the CI runner fetch an attacker-chosen URL (SSRF). Both basePath/candidatePath are local files + // this process wrote itself; disabling external refs only affects following $refs OUT of them. + const args = [...cmd.args, 'breaking', '--allow-external-refs=false', '-o', 'ERR', '-f', 'json', basePath, candidatePath]; const res = spawnSync(cmd.bin, args, { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: 180_000 }); if (res.error) { return { couldNotRun: true, detail: `could not execute "${cmd.bin} ${args.join(' ')}": ${res.error.message}` }; diff --git a/scripts/ga/compat-001-check.test.mjs b/scripts/ga/compat-001-check.test.mjs new file mode 100644 index 0000000..70bcfb6 --- /dev/null +++ b/scripts/ga/compat-001-check.test.mjs @@ -0,0 +1,180 @@ +#!/usr/bin/env node +/** + * compat-001-check.test.mjs — hermetic, offline, zero network, zero mutation of this repo's own + * git tags. Covers the pure/parameterized surfaces named by review: splitShellCommand's quoting + * (gitar, cubic P3 x2), resolveBaseTag's tag-selection edge cases against a disposable scratch git + * repo (cubic P3), and resolveOasdiffCmd's resolution order via a PATH-scoped env override + * (gitar/cubic quality asks for unit coverage on these scripts). + * + * Run: node --test scripts/ga/ + */ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, writeFileSync, chmodSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; + +import { resolveBaseTag, resolveOasdiffCmd, splitShellCommand } from './compat-001-check.mjs'; + +// ── splitShellCommand ─────────────────────────────────────────────────────────────────────────── + +test('splitShellCommand: plain space-separated words', () => { + assert.deepEqual(splitShellCommand('oasdiff breaking -o ERR'), ['oasdiff', 'breaking', '-o', 'ERR']); +}); + +test('splitShellCommand: double-quoted argument with a space survives as one token', () => { + assert.deepEqual( + splitShellCommand('docker run --rm -v "/path with spaces:/data" oasdiff'), + ['docker', 'run', '--rm', '-v', '/path with spaces:/data', 'oasdiff'], + ); +}); + +test('splitShellCommand: single-quoted argument with a space survives as one token', () => { + assert.deepEqual(splitShellCommand("bin '/a b/c'"), ['bin', '/a b/c']); +}); + +test('splitShellCommand: escaped quote and backslash inside double quotes', () => { + assert.deepEqual(splitShellCommand('bin "a\\"b" "c\\\\d"'), ['bin', 'a"b', 'c\\d']); +}); + +test('splitShellCommand: backslash-escaped space outside quotes joins into one token', () => { + assert.deepEqual(splitShellCommand('bin /path\\ with\\ space'), ['bin', '/path with space']); +}); + +test('splitShellCommand: collapses repeated whitespace and trims', () => { + assert.deepEqual(splitShellCommand(' bin arg1 arg2 '), ['bin', 'arg1', 'arg2']); +}); + +test('splitShellCommand: empty string yields no tokens', () => { + assert.deepEqual(splitShellCommand(''), []); +}); + +// ── resolveBaseTag ────────────────────────────────────────────────────────────────────────────── + +function scratchGitRepo() { + const dir = mkdtempSync(join(tmpdir(), 'ga-compat-test-')); + const run = (args) => spawnSync('git', args, { cwd: dir, encoding: 'utf8' }); + run(['init', '-q']); + run(['config', 'user.email', 'test@example.com']); + run(['config', 'user.name', 'test']); + writeFileSync(join(dir, 'f.txt'), 'x'); + run(['add', '.']); + run(['commit', '-q', '-m', 'init']); + return dir; +} + +test('resolveBaseTag: an explicit override short-circuits without touching git', () => { + // A cwd that does not exist would make any git call fail; the override path must never reach it. + assert.equal(resolveBaseTag('v9.9.9', '/nonexistent/path/for/this/test'), 'v9.9.9'); +}); + +test('resolveBaseTag: no v* tags on the repo returns null', () => { + const dir = scratchGitRepo(); + try { + assert.equal(resolveBaseTag(null, dir), null); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('resolveBaseTag: a single v* tag is returned', () => { + const dir = scratchGitRepo(); + try { + spawnSync('git', ['tag', 'v1.0.0'], { cwd: dir }); + assert.equal(resolveBaseTag(null, dir), 'v1.0.0'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('resolveBaseTag: the highest semver v* tag wins over lexical order', () => { + const dir = scratchGitRepo(); + try { + for (const t of ['v1.2.0', 'v1.10.0', 'v1.9.0']) spawnSync('git', ['tag', t], { cwd: dir }); + assert.equal(resolveBaseTag(null, dir), 'v1.10.0', 'v1.10.0 > v1.9.0 in semver even though "1.10" < "1.9" lexically'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('resolveBaseTag: non-v-prefixed tags are ignored', () => { + const dir = scratchGitRepo(); + try { + spawnSync('git', ['tag', 'release-1'], { cwd: dir }); + assert.equal(resolveBaseTag(null, dir), null); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +// ── resolveOasdiffCmd ─────────────────────────────────────────────────────────────────────────── + +function scratchBinDir(names) { + const dir = mkdtempSync(join(tmpdir(), 'ga-compat-bin-')); + for (const name of names) { + const p = join(dir, name); + writeFileSync(p, '#!/bin/sh\necho fake\n'); + chmodSync(p, 0o755); + } + return dir; +} + +test('resolveOasdiffCmd: GA_OASDIFF_CMD override wins even when PATH has neither tool, and is parsed with quoting', () => { + const emptyBin = scratchBinDir([]); + try { + const env = { PATH: emptyBin, GA_OASDIFF_CMD: 'docker run --rm -v "/p a/x:/data" oasdiff' }; + assert.deepEqual(resolveOasdiffCmd(env), { bin: 'docker', args: ['run', '--rm', '-v', '/p a/x:/data', 'oasdiff'] }); + } finally { + rmSync(emptyBin, { recursive: true, force: true }); + } +}); + +// `resolveOasdiffCmd` shells out via `sh -c 'command -v ...'`, and spawnSync's `env` option +// REPLACES the child's whole environment — so `sh` itself must still be resolvable. Every PATH +// below appends the real `/bin` (where `sh` lives on both macOS and the ubuntu-latest runner) AFTER +// the scratch bin dir, so the scratch dir takes precedence for oasdiff/go while `sh` keeps working. +// `/bin` alone (no /usr/local/bin, no /opt/homebrew/bin) is deliberately narrow so a real oasdiff or +// go installed elsewhere on the dev/CI machine cannot leak into the "neither present" case. +const REAL_SH_DIR = '/bin'; + +test('resolveOasdiffCmd: neither GA_OASDIFF_CMD nor oasdiff nor go on PATH returns null', () => { + const emptyBin = scratchBinDir([]); + try { + assert.equal(resolveOasdiffCmd({ PATH: `${emptyBin}:${REAL_SH_DIR}` }), null); + } finally { + rmSync(emptyBin, { recursive: true, force: true }); + } +}); + +test('resolveOasdiffCmd: oasdiff present on PATH is preferred with no extra args', () => { + const bin = scratchBinDir(['oasdiff']); + try { + const result = resolveOasdiffCmd({ PATH: `${bin}:${REAL_SH_DIR}` }); + assert.equal(result.bin, join(bin, 'oasdiff')); + assert.deepEqual(result.args, []); + } finally { + rmSync(bin, { recursive: true, force: true }); + } +}); + +test('resolveOasdiffCmd: falls back to "go run " when only go is on PATH', () => { + const bin = scratchBinDir(['go']); + try { + const result = resolveOasdiffCmd({ PATH: `${bin}:${REAL_SH_DIR}` }); + assert.equal(result.bin, 'go'); + assert.deepEqual(result.args, ['run', 'github.com/oasdiff/oasdiff@v1.29.1']); + } finally { + rmSync(bin, { recursive: true, force: true }); + } +}); + +test('resolveOasdiffCmd: GA_OASDIFF_GO_MODULE overrides the pinned go module', () => { + const bin = scratchBinDir(['go']); + try { + const result = resolveOasdiffCmd({ PATH: `${bin}:${REAL_SH_DIR}`, GA_OASDIFF_GO_MODULE: 'example.com/other@v9' }); + assert.deepEqual(result.args, ['run', 'example.com/other@v9']); + } finally { + rmSync(bin, { recursive: true, force: true }); + } +}); diff --git a/scripts/ga/contract-001-check.mjs b/scripts/ga/contract-001-check.mjs index 4b154fc..3cc88bb 100644 --- a/scripts/ga/contract-001-check.mjs +++ b/scripts/ga/contract-001-check.mjs @@ -99,6 +99,60 @@ function canonicalJson(value) { return JSON.stringify(sortKeysDeep(value ?? null)); } +/** RFC 6901 JSON Pointer resolution against a document already fully loaded in memory. */ +export function resolveJsonPointer(doc, ref) { + if (typeof ref !== 'string' || !ref.startsWith('#/')) return undefined; + const parts = ref + .slice(2) + .split('/') + .map((p) => p.replace(/~1/g, '/').replace(/~0/g, '~')); + let cur = doc; + for (const part of parts) { + if (cur === null || typeof cur !== 'object') return undefined; + cur = cur[part]; + } + return cur; +} + +/** + * Walk `node` and resolve every internal `$ref` reachable from it against `doc`, recursively (a + * referenced schema may itself `$ref` another). Returns a Map of ref-string -> resolved value. + * + * WHY THIS EXISTS (cubic P1): the content digest below hashes only the operation object itself. An + * operation that still points at the same `$ref` while the REFERENCED component schema changed + * would report matching digests and miss real contract drift. Folding the reachable referenced + * content into the digest input closes that gap. Pure/offline: only reads objects already in + * memory, no I/O. + */ +export function collectReachableRefs(node, doc, out = new Map(), seen = new Set()) { + if (Array.isArray(node)) { + for (const item of node) collectReachableRefs(item, doc, out, seen); + return out; + } + if (!node || typeof node !== 'object') return out; + if (typeof node.$ref === 'string' && node.$ref.startsWith('#/')) { + if (!seen.has(node.$ref)) { + seen.add(node.$ref); + const resolved = resolveJsonPointer(doc, node.$ref); + if (resolved !== undefined) { + out.set(node.$ref, resolved); + collectReachableRefs(resolved, doc, out, seen); + } + } + } + for (const [key, value] of Object.entries(node)) { + if (key === '$ref') continue; + collectReachableRefs(value, doc, out, seen); + } + return out; +} + +/** Canonical JSON of every ref this operation reaches, sorted by ref string for determinism. */ +function canonicalReachableRefs(op, doc) { + const refs = collectReachableRefs(op, doc); + return canonicalJson(Object.fromEntries([...refs.entries()].sort(([a], [b]) => a.localeCompare(b)))); +} + /** * Run the CONTRACT-001 check. Returns `{ couldNotRun, checks, ... }`. Never throws for an * ordinary tooling failure — those come back as `couldNotRun: true` with a check named for what @@ -164,22 +218,33 @@ export async function run(opts = {}) { const { path, method, op: repoOp } = repoOps.get(key); const { op: liveOp } = liveOps.get(key); const { repo, live } = normalizePair(repoOp, liveOp, path, method); - rowsRepo.push(`${key}\t${sha256(canonicalJson(repo))}`); - rowsLive.push(`${key}\t${sha256(canonicalJson(live))}`); + // Reachable $ref content is resolved from the RAW (pre-normalize) operation against its own + // document — normalizePair only touches operation-level enrichment fields, never $ref targets, + // so this sees exactly what each side's components actually publish today. + const repoRefs = canonicalReachableRefs(repoOp, repoDoc); + const liveRefs = canonicalReachableRefs(liveOp, liveDoc); + rowsRepo.push(`${key}\t${sha256(`${canonicalJson(repo)}\n${repoRefs}`)}`); + rowsLive.push(`${key}\t${sha256(`${canonicalJson(live)}\n${liveRefs}`)}`); } const localDigest = sha256([...rowsRepo].sort().join('\n')); const liveDigest = sha256([...rowsLive].sort().join('\n')); const findings = result.findings ?? []; - const findingLabels = findings.slice(0, 10).map((f) => `${f.direction} ${f.method} ${f.path}`); + // operation-parity is CONTRACT-001's "repo-only and live-only operations are zero" text — it must + // NOT also fail on `shared-drift` findings (an operation present on both sides with different + // content), because that condition is already reported, more precisely, by content-digest below. + // Counting shared-drift here too would make both sub-checks fail for the same underlying cause + // and mislabel which condition actually broke (cubic P2). + const parityFindings = findings.filter((f) => f.direction === 'undocumented-live' || f.direction === 'unpublished-repo'); + const parityLabels = parityFindings.slice(0, 10).map((f) => `${f.direction} ${f.method} ${f.path}`); const checks = [ { name: 'operation-parity', - ok: findings.length === 0, - detail: findings.length === 0 + ok: parityFindings.length === 0, + detail: parityFindings.length === 0 ? `zero unexplained repo-only/live-only operations (${repoOps.size} declared, ${liveOps.size} live, ${sharedKeys.length} shared, ${result.allowlisted?.length ?? 0} allowlisted, ${result.draftNotYetPublished?.length ?? 0} draft)` - : `${findings.length} unexplained operation-level finding(s): ${findingLabels.join('; ')}${findings.length > findingLabels.length ? '; …' : ''}`, + : `${parityFindings.length} unexplained repo-only/live-only operation finding(s): ${parityLabels.join('; ')}${parityFindings.length > parityLabels.length ? '; …' : ''}`, }, { name: 'content-digest', diff --git a/scripts/ga/contract-001-check.test.mjs b/scripts/ga/contract-001-check.test.mjs new file mode 100644 index 0000000..8f3a391 --- /dev/null +++ b/scripts/ga/contract-001-check.test.mjs @@ -0,0 +1,214 @@ +#!/usr/bin/env node +/** + * contract-001-check.test.mjs — hermetic, offline, zero network (uses opts.repoSpecPath and + * opts.liveFile, never fetch). Covers: + * - resolveJsonPointer / collectReachableRefs (pure, cubic P1's reachable-ref digest fix) + * - operation-parity vs content-digest independence (cubic P2: shared-drift must not also fail + * operation-parity) + * - a content-digest catch that only the reachable-ref fold makes possible: an operation whose + * own object is byte-identical on both sides, but whose $ref-referenced component schema + * changed, must be reported as a content-digest mismatch (the exact gap cubic P1 named) + * + * Run: node --test scripts/ga/ + */ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { collectReachableRefs, resolveJsonPointer, run } from './contract-001-check.mjs'; + +// ── resolveJsonPointer / collectReachableRefs ────────────────────────────────────────────────── + +test('resolveJsonPointer: resolves a simple internal pointer', () => { + const doc = { components: { schemas: { Widget: { type: 'object' } } } }; + assert.deepEqual(resolveJsonPointer(doc, '#/components/schemas/Widget'), { type: 'object' }); +}); + +test('resolveJsonPointer: unescapes ~0 and ~1', () => { + const doc = { components: { schemas: { 'a/b~c': { type: 'string' } } } }; + assert.deepEqual(resolveJsonPointer(doc, '#/components/schemas/a~1b~0c'), { type: 'string' }); +}); + +test('resolveJsonPointer: a dangling pointer resolves to undefined, never throws', () => { + const doc = { components: { schemas: {} } }; + assert.equal(resolveJsonPointer(doc, '#/components/schemas/Missing'), undefined); +}); + +test('resolveJsonPointer: a non-internal ($ref does not start with #/) pointer is ignored', () => { + assert.equal(resolveJsonPointer({}, 'https://example.com/schema.json'), undefined); +}); + +test('collectReachableRefs: resolves a $ref reachable from a nested node', () => { + const doc = { + components: { schemas: { Widget: { type: 'object', properties: { id: { type: 'string' } } } } }, + }; + const op = { responses: { 200: { content: { 'application/json': { schema: { $ref: '#/components/schemas/Widget' } } } } } }; + const refs = collectReachableRefs(op, doc); + assert.equal(refs.size, 1); + assert.deepEqual(refs.get('#/components/schemas/Widget'), doc.components.schemas.Widget); +}); + +test('collectReachableRefs: chases a ref through another ref (transitive)', () => { + const doc = { + components: { + schemas: { + Widget: { properties: { owner: { $ref: '#/components/schemas/Owner' } } }, + Owner: { type: 'string' }, + }, + }, + }; + const op = { schema: { $ref: '#/components/schemas/Widget' } }; + const refs = collectReachableRefs(op, doc); + assert.equal(refs.size, 2); + assert.ok(refs.has('#/components/schemas/Widget')); + assert.ok(refs.has('#/components/schemas/Owner')); +}); + +test('collectReachableRefs: an external (non "#/") $ref is left unresolved, not thrown on', () => { + const op = { schema: { $ref: 'external.json#/Thing' } }; + const refs = collectReachableRefs(op, {}); + assert.equal(refs.size, 0); +}); + +test('collectReachableRefs: a self-referential (cyclic) ref does not infinite-loop', () => { + const doc = { components: { schemas: { Node: { properties: { next: { $ref: '#/components/schemas/Node' } } } } } }; + const op = { schema: { $ref: '#/components/schemas/Node' } }; + const refs = collectReachableRefs(op, doc); + assert.equal(refs.size, 1); +}); + +// ── run() end-to-end against hermetic fixtures ───────────────────────────────────────────────── + +function writeFixture(name, repoYaml, liveDoc) { + const dir = mkdtempSync(join(tmpdir(), `ga-contract-test-${name}-`)); + mkdirSync(dir, { recursive: true }); + const repoSpecPath = join(dir, 'openapi.yaml'); + const liveFile = join(dir, 'live.json'); + writeFileSync(repoSpecPath, repoYaml); + writeFileSync(liveFile, JSON.stringify(liveDoc, null, 2)); + return { dir, repoSpecPath, liveFile }; +} + +function checkByName(result, name) { + return result.checks.find((c) => c.name === name); +} + +test('run(): a content change reachable only via $ref is caught by content-digest, with byte-identical operations and zero shared-drift', async () => { + const repoYaml = ` +openapi: 3.1.0 +info: {title: t, version: 1.0.0} +paths: + /widgets: + get: + operationId: listWidgets + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/WidgetList' +components: + schemas: + WidgetList: + type: object + properties: + items: + type: array +`; + const sharedOp = { + operationId: 'listWidgets', + responses: { + 200: { + description: 'OK', + content: { 'application/json': { schema: { $ref: '#/components/schemas/WidgetList' } } }, + }, + }, + }; + const liveDoc = { + openapi: '3.1.0', + info: { title: 't', version: '1.0.0' }, + paths: { '/widgets': { get: sharedOp } }, + // The live component schema changed (an extra "total" property) while the operation still + // points at the identical $ref — the exact gap cubic P1 named. + components: { schemas: { WidgetList: { type: 'object', properties: { items: { type: 'array' }, total: { type: 'integer' } } } } }, + }; + const { dir, repoSpecPath, liveFile } = writeFixture('refs', repoYaml, liveDoc); + try { + const result = await run({ repoSpecPath, liveFile }); + assert.equal(result.couldNotRun, false); + assert.equal(checkByName(result, 'operation-parity').ok, true, 'the operation itself is declared and served on both sides'); + assert.equal(checkByName(result, 'content-digest').ok, false, 'a referenced-schema change must flip the content digest even though the operation object is unchanged'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('run(): operation-parity does not fail on a shared-drift-only difference (that is content-digest\'s job)', async () => { + const repoYaml = ` +openapi: 3.1.0 +info: {title: t, version: 1.0.0} +paths: + /alpha: + get: + operationId: getAlpha + summary: Alpha v1 + responses: + '200': + description: OK +`; + const liveDoc = { + openapi: '3.1.0', + info: { title: 't', version: '1.0.0' }, + paths: { + '/alpha': { + get: { operationId: 'getAlpha', summary: 'Alpha v2', responses: { 200: { description: 'OK' } } }, + }, + }, + }; + const { dir, repoSpecPath, liveFile } = writeFixture('shared-drift', repoYaml, liveDoc); + try { + const result = await run({ repoSpecPath, liveFile }); + assert.equal(result.couldNotRun, false); + assert.equal(checkByName(result, 'operation-parity').ok, true, 'a shared, differently-worded operation is not a parity problem'); + assert.equal(checkByName(result, 'content-digest').ok, false, 'the summary difference must still be caught, just by content-digest, not operation-parity'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('run(): an undocumented-live operation fails operation-parity, independent of content-digest', async () => { + const repoYaml = ` +openapi: 3.1.0 +info: {title: t, version: 1.0.0} +paths: + /alpha: + get: + operationId: getAlpha + responses: + '200': + description: OK +`; + const sharedOp = { operationId: 'getAlpha', responses: { 200: { description: 'OK' } } }; + const liveDoc = { + openapi: '3.1.0', + info: { title: 't', version: '1.0.0' }, + paths: { + '/alpha': { get: sharedOp }, + // Served live, never declared in the repo spec. + '/beta': { get: { operationId: 'getBeta', responses: { 200: { description: 'OK' } } } }, + }, + }; + const { dir, repoSpecPath, liveFile } = writeFixture('undocumented-live', repoYaml, liveDoc); + try { + const result = await run({ repoSpecPath, liveFile }); + assert.equal(result.couldNotRun, false); + assert.equal(checkByName(result, 'operation-parity').ok, false); + assert.match(checkByName(result, 'operation-parity').detail, /undocumented-live/); + assert.equal(checkByName(result, 'content-digest').ok, true, '/beta is unmatched, so it never enters the shared-operation digest loop'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/scripts/ga/ga-evidence.mjs b/scripts/ga/ga-evidence.mjs index 6bf0ce7..9ca8c64 100644 --- a/scripts/ga/ga-evidence.mjs +++ b/scripts/ga/ga-evidence.mjs @@ -84,7 +84,7 @@ function nowIsoSeconds() { * `couldNotRun` -> status 'unknown' (a failed read says nothing about the criterion and must * never be graded as a pass); otherwise 'pass' only when every sub-check passed, else 'fail'. */ -function buildContractRow(raw) { +export function buildContractRow(raw) { const command = 'scripts/ga/check-CONTRACT-001.sh'; if (raw.couldNotRun) { const detail = raw.checks[0]?.detail ?? 'unknown failure'; @@ -94,7 +94,7 @@ function buildContractRow(raw) { command, failing_checks: [`gate could not run — ${raw.checks[0]?.name}: ${detail}`.slice(0, 500)], targets_observed: [], - fingerprintPayload: { criterion_id: 'CONTRACT-001', couldNotRun: true, checks: raw.checks.map((c) => [c.name, c.ok]) }, + fingerprintPayload: { criterion_id: 'CONTRACT-001', couldNotRun: true, checks: raw.checks.map((c) => [c.name, c.ok, c.detail]) }, }; } const allPass = raw.checks.every((c) => c.ok === true); @@ -109,9 +109,12 @@ function buildContractRow(raw) { command, failing_checks: allPass ? undefined : failing, targets_observed: targets, + // `detail` is included (not just `ok`) so evidence-relevant content changes that don't flip a + // boolean — e.g. which operations are undocumented-live — still change the fingerprint instead + // of being deduplicated against stale evidence (cubic P2). fingerprintPayload: { criterion_id: 'CONTRACT-001', - checks: raw.checks.map((c) => [c.name, c.ok]).sort(), + checks: raw.checks.map((c) => [c.name, c.ok, c.detail]).sort(), targets: [...targets].sort(), }, }; @@ -123,7 +126,7 @@ function buildContractRow(raw) { * deprecation-notice half is unverified; a dirty run reports 'fail'; a gate that could not run * also reports 'unknown', distinguished in failing_checks. */ -function buildCompatRow(raw) { +export function buildCompatRow(raw) { const command = 'scripts/ga/check-COMPAT-001.sh'; if (raw.couldNotRun) { const detail = raw.checks[0]?.detail ?? 'unknown failure'; @@ -133,7 +136,7 @@ function buildCompatRow(raw) { command, failing_checks: [`gate could not run — ${raw.checks[0]?.name}: ${detail}`.slice(0, 500)], targets_observed: [], - fingerprintPayload: { criterion_id: 'COMPAT-001', couldNotRun: true, checks: raw.checks.map((c) => [c.name, c.ok]) }, + fingerprintPayload: { criterion_id: 'COMPAT-001', couldNotRun: true, checks: raw.checks.map((c) => [c.name, c.ok, c.detail]) }, }; } const breakingCheck = raw.checks.find((c) => c.name === 'breaking-changes'); @@ -157,7 +160,7 @@ function buildCompatRow(raw) { targets_observed: targets, fingerprintPayload: { criterion_id: 'COMPAT-001', - checks: raw.checks.map((c) => [c.name, c.ok]).sort(), + checks: raw.checks.map((c) => [c.name, c.ok, c.detail]).sort(), targets: [...targets].sort(), }, }; @@ -260,7 +263,15 @@ async function main() { process.stdout.write('\nga-evidence: ran cleanly (see per-criterion status above — unknown is not a failure but is not a pass either)\n'); } -main().catch((e) => { - process.stderr.write(`ga-evidence could not run: ${e?.stack || e}\n`); - process.exit(2); -}); +// Guarded so importing this module for its exports (buildContractRow/buildCompatRow, tested in +// ga-evidence.test.mjs) does not run the full producer — contract-001-check.mjs and +// compat-001-check.mjs already follow this pattern; this file was missing it, which made `node +// --test` actually execute the live producer (network fetch, git tag read, oasdiff invocation) as +// an import side effect every time the test file loaded it. +const isMain = process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)); +if (isMain) { + main().catch((e) => { + process.stderr.write(`ga-evidence could not run: ${e?.stack || e}\n`); + process.exit(2); + }); +} diff --git a/scripts/ga/ga-evidence.test.mjs b/scripts/ga/ga-evidence.test.mjs new file mode 100644 index 0000000..e7b8322 --- /dev/null +++ b/scripts/ga/ga-evidence.test.mjs @@ -0,0 +1,107 @@ +#!/usr/bin/env node +/** + * ga-evidence.test.mjs — hermetic, offline, zero network. Covers buildContractRow/buildCompatRow + * status derivation directly against synthetic "raw" check results (gitar/cubic's explicit ask), + * and the fingerprint-payload fix (cubic P2): two raw results with the same booleans but different + * `detail` text must NOT collapse to the same fingerprint. + * + * Run: node --test scripts/ga/ + */ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { buildCompatRow, buildContractRow } from './ga-evidence.mjs'; + +// ── buildContractRow ─────────────────────────────────────────────────────────────────────────── + +test('buildContractRow: couldNotRun -> status unknown, empty targets, failing_checks names the failure', () => { + const row = buildContractRow({ couldNotRun: true, checks: [{ name: 'live-fetch', ok: null, detail: 'HTTP 503' }] }); + assert.equal(row.status, 'unknown'); + assert.deepEqual(row.targets_observed, []); + assert.match(row.failing_checks[0], /live-fetch: HTTP 503/); +}); + +test('buildContractRow: both sub-checks pass -> status pass, no failing_checks', () => { + const row = buildContractRow({ + couldNotRun: false, + checks: [ + { name: 'operation-parity', ok: true, detail: 'zero unexplained operations' }, + { name: 'content-digest', ok: true, detail: 'digests match' }, + ], + localDigest: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + liveDigest: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + }); + assert.equal(row.status, 'pass'); + assert.equal(row.failing_checks, undefined); +}); + +test('buildContractRow: one sub-check fails -> status fail, failing_checks names only the failing one', () => { + const row = buildContractRow({ + couldNotRun: false, + checks: [ + { name: 'operation-parity', ok: true, detail: 'zero unexplained operations' }, + { name: 'content-digest', ok: false, detail: 'digests differ' }, + ], + localDigest: 'a'.repeat(64), + liveDigest: 'b'.repeat(64), + }); + assert.equal(row.status, 'fail'); + assert.deepEqual(row.failing_checks, ['content-digest: digests differ']); +}); + +// ── buildCompatRow ───────────────────────────────────────────────────────────────────────────── + +test('buildCompatRow: couldNotRun -> status unknown', () => { + const row = buildCompatRow({ couldNotRun: true, checks: [{ name: 'baseline-tag', ok: null, detail: 'no v* tag found' }] }); + assert.equal(row.status, 'unknown'); +}); + +test('buildCompatRow: a clean breaking-changes run is still unknown, never pass (the module\'s own honesty rule)', () => { + const row = buildCompatRow({ + couldNotRun: false, + tag: 'v1.0.0', + head: 'deadbeef', + checks: [ + { name: 'breaking-changes', ok: true, detail: 'zero breaking changes' }, + { name: 'deprecation-notice', ok: 'unknown', detail: 'not machine-verified' }, + ], + }); + assert.equal(row.status, 'unknown'); + assert.ok(row.failing_checks.some((f) => f.includes('deprecation notice/migration path not machine-verified'))); +}); + +test('buildCompatRow: a breaking change found -> status fail', () => { + const row = buildCompatRow({ + couldNotRun: false, + tag: 'v1.0.0', + head: 'deadbeef', + checks: [ + { name: 'breaking-changes', ok: false, detail: '1 breaking change' }, + { name: 'deprecation-notice', ok: 'unknown', detail: 'not machine-verified' }, + ], + }); + assert.equal(row.status, 'fail'); + assert.ok(row.failing_checks.some((f) => f.startsWith('breaking-changes:'))); +}); + +// ── fingerprint sensitivity (cubic P2) ──────────────────────────────────────────────────────── + +test('buildContractRow: fingerprintPayload changes when a check\'s detail changes even though ok stays the same', () => { + const base = (detail) => ({ + couldNotRun: false, + checks: [ + { name: 'operation-parity', ok: false, detail }, + { name: 'content-digest', ok: true, detail: 'digests match' }, + ], + localDigest: 'a'.repeat(64), + liveDigest: 'a'.repeat(64), + }); + const rowA = buildContractRow(base('1 unexplained operation-level finding(s): undocumented-live GET /a')); + const rowB = buildContractRow(base('1 unexplained operation-level finding(s): undocumented-live GET /b')); + assert.equal(rowA.checks?.[0]?.ok, undefined, 'sanity: buildContractRow does not itself expose a top-level checks array'); + assert.notDeepEqual( + rowA.fingerprintPayload, + rowB.fingerprintPayload, + 'two different findings behind the same boolean must not produce the same fingerprint input, or a real evidence change gets deduplicated as stale', + ); +});