From ad9667698d6517791e2470bc3d2cc5bd86165239 Mon Sep 17 00:00:00 2001 From: Chisanan232 Date: Thu, 13 Aug 2026 18:53:02 +0800 Subject: [PATCH 1/9] =?UTF-8?q?=E2=9C=85=20(tests):=20Bind=20each=20docume?= =?UTF-8?q?nted=20quick-start=20claim=20to=20the=20control=20that=20proves?= =?UTF-8?q?=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs/02-quick-start/index.md §2-§4 carry the Node quick-start's enforcement claims, and nothing connected them to the controls that prove them. A claim could be added, reworded, or left standing after the behaviour beneath it changed, and no gate would notice. Neither side of the comparison is a transcribed constant checking another transcribed constant. The claim sentences are read from the document; the control ids are extracted from each control file's TypeScript AST as " :: > ". Ids are file-qualified because the claims are not all proved in one place — the deny claims come from the negative controls, the auditSink claim from the AAASM-5681 disposition suite. The error class is derived from constructor.name on the value a real deny actually throws, and is deliberately not imported. Importing it would make a rename a type error: red, but it aborts before the assertion meant to catch the rename can run. That is the inverted-order defect the round-1 review of this ticket found in all three SDKs. The unbound-sentence check found two claims on its first run that reading had missed — that initAssembly warns and reports auditSink on the context, and that a denied call throws instead of executing. Both are now bound. The gateway-reachability claim is registered as unproven, naming AAASM-5663. Refs AAASM-5529 --- tests/quickstart-claim-bindings.test.ts | 426 ++++++++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100644 tests/quickstart-claim-bindings.test.ts diff --git a/tests/quickstart-claim-bindings.test.ts b/tests/quickstart-claim-bindings.test.ts new file mode 100644 index 000000000..c530906c6 --- /dev/null +++ b/tests/quickstart-claim-bindings.test.ts @@ -0,0 +1,426 @@ +/** + * Drift gate binding the documented Node quick-start's enforcement claims to + * the controls that prove them (AAASM-5529, Epic AAASM-5526). + * + * `docs/02-quick-start/index.md` §3 and §4 are where the quick-start tells a + * reader what governance does for them: that policy is enforced before a tool + * runs, that a deny throws a `PolicyViolationError` with the tool body never + * running, and that the allow path's governance event is discarded rather than + * retained. Nothing connected those sentences to the controls in + * `quickstart-negative-control.test.ts`, so a claim could be added, reworded, + * or left standing after the behaviour beneath it changed, and no gate would + * notice. + * + * WHAT THIS GATE PROVES + * + * 1. Every enforcement sentence in the gated sections is bound to a named + * control. The sentences are read out of the document, so a new claim that + * no binding quotes fails here rather than shipping unbacked. + * 2. Every binding still describes the document. Rewording a claim breaks its + * quote and fails. + * 3. Every control a binding names still exists. Control ids are extracted from + * each control file's TypeScript AST as ` :: > `, not + * transcribed, so renaming or deleting one fails here. + * 4. The error class the document names is the one the SDK actually throws. + * Derived by driving a real deny through `withAssembly` and reading + * `constructor.name` off the thrown value. The class is deliberately NOT + * imported here: importing it would make a rename a *type* error, which is + * red but aborts before the assertion meant to catch it can run — the + * inverted-order defect the round-1 review of this ticket found in all three + * SDKs. + * + * WHAT THIS GATE DOES NOT PROVE + * + * It does not execute, type-check or lint a documented snippet. + * `metadata/quickstart-snippets/` is excluded from ESLint + * (`eslint.config.mjs:22`), from Prettier (`.prettierignore:10`) and from every + * tsconfig, and the snippets import `createPolicyGatewayClient` from + * `"./policy.js"` — a file the reader supplies from the examples repo, not an + * SDK export. The existing drift check round-trips them as *text*: it proves + * this page matches the vendored snippet and nothing more. This gate does not + * change that. + * + * Nor does binding a claim make the claim true. A binding records which control + * stands behind a sentence. + */ + +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import ts from "typescript"; +import { describe, expect, it } from "vitest"; +import { withAssembly } from "../src/wrappers/with-assembly.js"; +import { createFileSideEffect, createPolicyGatewayClient } from "./helpers/negative-control.js"; + +const QUICK_START = resolve(process.cwd(), "docs/02-quick-start/index.md"); + +/** + * The test files a binding may name. + * + * More than one, because the quick-start's claims are not all proved in the + * same place: the deny claims are proved by the negative controls, while the + * "reports auditSink" claim is proved by the AAASM-5681 disposition suite. + * Control ids are file-qualified so two suites cannot collide on a title. + */ +const CONTROL_FILES = [ + "tests/quickstart-negative-control.test.ts", + "tests/audit-sink-disposition.test.ts" +] as const; + +const ENFORCEMENT = "enforcement"; +const LIFECYCLE = "lifecycle"; + +interface ClaimBinding { + readonly id: string; + readonly kind: typeof ENFORCEMENT | typeof LIFECYCLE; + /** + * A verbatim fragment of the claim as it appears in the document, with + * Markdown's soft wrapping collapsed. Rewording the document breaks it. + */ + readonly quote: string; + /** ` :: > ` ids drawn from CONTROL_FILES. */ + readonly controls?: readonly string[]; + /** Set when no control proves the claim. Must name a ticket. */ + readonly unprovenReason?: string; +} + +const BINDINGS: readonly ClaimBinding[] = [ + { + id: "policy-enforced-before-each-tool-runs", + kind: ENFORCEMENT, + quote: "wraps a map of tools so the local policy is enforced before each one runs", + // Both halves are named. The negative controls prove the "before" by the + // absence of the side effect; the positive controls prove the probe would + // have seen that effect had it happened. Either alone is the vacuous + // evidence this Epic exists to remove. + controls: [ + "tests/quickstart-negative-control.test.ts :: quick-start negative control: filesystem side effect > POSITIVE CONTROL: an allowed write_file really creates the file on disk", + "tests/quickstart-negative-control.test.ts :: quick-start negative control: filesystem side effect > NEGATIVE CONTROL: a denied write_file leaves no file on disk", + "tests/quickstart-negative-control.test.ts :: quick-start negative control: network side effect > POSITIVE CONTROL: an allowed egress tool reaches the listener", + "tests/quickstart-negative-control.test.ts :: quick-start negative control: network side effect > NEGATIVE CONTROL: a denied egress tool never reaches the listener" + ] + }, + { + id: "allow-event-is-discarded-not-retained", + kind: ENFORCEMENT, + // A negative capability claim, and the honest one: it says the SDK retains + // nothing. Bound so that if someone deletes the caveat because a sink was + // wired (AAASM-5750), this gate makes them revisit the control rather than + // quietly dropping the sentence. + quote: "it is discarded, not retained, so there is no audit trail to read it back from", + controls: [ + "tests/quickstart-negative-control.test.ts :: quick-start negative control: a deny is handed to the gateway's record call > hands the gateway an audit event naming the denied tool and its run" + ] + }, + { + id: "init-warns-and-reports-the-discard", + kind: ENFORCEMENT, + // Found by this gate's own unbound-sentence check, not by reading: the + // sentence before it was bound, this one was not, and the two make + // different claims. The first says the event is dropped; this one says + // initAssembly tells you so at startup and on the context. A reader relying + // on the second without the first would not know to look. + quote: + 'warns about this at startup and reports `auditSink: "discarded"` on the returned context', + controls: [ + 'tests/audit-sink-disposition.test.ts :: AAASM-5681: initAssembly surfaces the drop without AA_DEBUG > WARNS on stderr and reports auditSink="discarded" with AA_DEBUG unset' + ] + }, + { + id: "deny-rejects-and-body-never-runs", + kind: ENFORCEMENT, + quote: "message includes the tool name and the gateway's reason — the tool body never runs", + controls: [ + "tests/quickstart-negative-control.test.ts :: quick-start negative control: filesystem side effect > NEGATIVE CONTROL: a denied write_file leaves no file on disk", + "tests/quickstart-negative-control.test.ts :: quick-start negative control: network side effect > NEGATIVE CONTROL: a denied egress tool never reaches the listener" + ] + }, + { + id: "pending-waits-then-proceeds-or-rejects", + kind: ENFORCEMENT, + quote: "for a human decision, then either proceeds or rejects", + controls: [ + "tests/quickstart-negative-control.test.ts :: quick-start negative control: a deny is handed to the gateway's record call > hands the gateway a distinct audit event when an approval is rejected" + ] + }, + { + id: "deny-throws-instead-of-executing", + kind: ENFORCEMENT, + // Also found by the unbound-sentence check. It restates the deny claim in + // the strongest terms the page uses ("the whole point"), and a restatement + // is exactly the kind of sentence that survives a behaviour change because + // nobody thinks of it as the claim. + quote: "a denied tool call throws instead of executing", + controls: [ + "tests/quickstart-negative-control.test.ts :: quick-start negative control: filesystem side effect > NEGATIVE CONTROL: a denied write_file leaves no file on disk", + "tests/quickstart-negative-control.test.ts :: quick-start negative control: filesystem side effect > FALSIFICATION: the same write, ungoverned, does create the file" + ] + }, + { + id: "observe-mode-does-not-block", + kind: ENFORCEMENT, + quote: "watch what *would* be blocked without actually blocking it", + controls: [ + "tests/quickstart-negative-control.test.ts :: quick-start negative control: the zero-config initAssembly path > BOUNDARY: enforcementMode observe inits and lets the tool body run" + ] + }, + { + id: "gateway-must-be-reachable", + kind: LIFECYCLE, + quote: "The SDK enforces policy by talking to an Agent Assembly **gateway**", + unprovenReason: + "AAASM-5663: both README entrypoints were executed and neither runs as written, " + + "so no control covers the documented zero-config path end to end. The control " + + "that exists asserts the documented config REFUSES to init; it does not prove a " + + "reader can reach a working gateway." + } +]; + +/** Drops fenced code and collapses soft wrapping. */ +function flattenMarkdown(text: string): string { + return text + .replace(/```[\s\S]*?```/g, " ") + .split(/\s+/) + .join(" ") + .trim(); +} + +/** + * The flattened text of the quick-start regions this gate is responsible for. + * + * Read from the document rather than transcribed, so a claim added to either + * region shows up here without anyone editing this file. + */ +function gatedDocumentRegions(): Record { + const document = readFileSync(QUICK_START, "utf-8"); + const openings: Record = { + "gateway-reachable": "## 2. Make sure a gateway is reachable", + "govern-your-first-agent": "## 3. Govern your first agent", + "what-to-expect": "## 4. What to expect" + }; + + const regions: Record = {}; + for (const [name, opening] of Object.entries(openings)) { + const start = document.indexOf(opening); + expect( + start, + `${QUICK_START} no longer contains the "${name}" region (looked for ${JSON.stringify(opening)}). ` + + "If the quick-start was restructured, re-point this gate at the section that now " + + "carries the enforcement claims — do not delete it." + ).toBeGreaterThan(-1); + + // Skip the region's own heading LINE before looking for the next heading. + const body = document.slice(start); + const afterHeading = body.indexOf("\n"); + const rest = afterHeading === -1 ? "" : body.slice(afterHeading); + const next = rest.search(/\n#{1,6} /); + regions[name] = flattenMarkdown(next === -1 ? body : body.slice(0, afterHeading + next)); + } + return regions; +} + +/** + * Extract ` :: > ` ids from each control file's AST. + * + * Derived from the source rather than transcribed, so this set changes when a + * control is renamed or removed and the bindings above then fail. + */ +function controlTitles(): Set { + const titles = new Set(); + + const literalTitle = (node: ts.CallExpression): string | undefined => { + const [first] = node.arguments; + return first !== undefined && ts.isStringLiteralLike(first) ? first.text : undefined; + }; + + for (const file of CONTROL_FILES) { + const absolute = resolve(process.cwd(), file); + const source = ts.createSourceFile( + absolute, + readFileSync(absolute, "utf-8"), + ts.ScriptTarget.Latest, + true + ); + + const walk = (node: ts.Node, prefix: string): void => { + let nextPrefix = prefix; + if (ts.isCallExpression(node) && ts.isIdentifier(node.expression)) { + const fn = node.expression.text; + const title = literalTitle(node); + if (title !== undefined) { + if (fn === "describe") { + nextPrefix = prefix === "" ? title : `${prefix} > ${title}`; + } else if (fn === "it" || fn === "test") { + titles.add(`${file} :: ${prefix === "" ? title : `${prefix} > ${title}`}`); + } + } + } + ts.forEachChild(node, (child) => walk(child, nextPrefix)); + }; + + walk(source, ""); + } + return titles; +} + +describe("claim gate: it can see what it gates", () => { + // Positive controls for the gate itself. Every check below reads a real + // artifact; these prove the reads arrived. An empty parse and a clean result + // are otherwise indistinguishable. + it("finds the gated document regions and they are non-empty", () => { + const regions = gatedDocumentRegions(); + expect(Object.keys(regions).sort()).toEqual([ + "gateway-reachable", + "govern-your-first-agent", + "what-to-expect" + ]); + for (const [name, text] of Object.entries(regions)) { + expect( + text.length, + `region "${name}" is too short to hold its claims: ${text}` + ).toBeGreaterThan(80); + } + }); + + it("extracts the negative-control titles from the AST", () => { + const titles = controlTitles(); + expect(titles.size).toBeGreaterThanOrEqual(12); + // A named one, so an extraction that returned an unrelated set of the right + // size cannot satisfy the count above. + expect([...titles]).toContain( + "tests/quickstart-negative-control.test.ts :: quick-start negative control: filesystem side effect > NEGATIVE CONTROL: a denied write_file leaves no file on disk" + ); + }); +}); + +describe("claim gate: every documented claim is bound", () => { + it.each(BINDINGS.map((binding) => [binding.id, binding] as const))( + "%s still quotes the document", + (_id, binding) => { + const regions = Object.values(gatedDocumentRegions()); + expect( + regions.some((text) => text.includes(binding.quote)), + `Claim binding "${binding.id}" quotes:\n ${JSON.stringify(binding.quote)}\n` + + "which no longer appears in the gated regions of the quick-start. The claim was " + + "reworded or removed. Update the quote and re-check that the named controls still " + + "prove the new wording." + ).toBe(true); + } + ); + + it("no enforcement sentence in the gated regions is unbound", () => { + // The check that makes this gate load-bearing rather than decorative: a new + // enforcement sentence cannot reach the published quick-start without + // someone naming the control behind it. + // + // Deliberately excludes a bare "policy": it appears in every snippet import + // line and tab label, so matching it would sweep in prose that makes no + // enforcement claim. + const enforcementLanguage = + /\bdenie[sd]\b|\bdeny\b|\bblocked\b|\bblocking\b|\bnever runs\b|\benforced?\b|\brejects\b|\bdiscarded\b/i; + + const unbound: string[] = []; + for (const [region, text] of Object.entries(gatedDocumentRegions())) { + for (const raw of text.split(/(?<=\.)\s+/)) { + const sentence = raw.trim(); + if (sentence === "" || !enforcementLanguage.test(sentence)) continue; + if (BINDINGS.some((binding) => sentence.includes(binding.quote))) continue; + unbound.push(`[${region}] ${sentence}`); + } + } + + expect( + unbound, + "These quick-start enforcement sentences have no ClaimBinding:\n" + + unbound.map((s) => ` ${s}`).join("\n") + + "\n\nAdd a ClaimBinding naming the control that proves each one. If no control " + + "does, set unprovenReason and name the ticket — do not delete the claim from " + + "this gate to make it pass." + ).toEqual([]); + }); +}); + +describe("claim gate: every binding names something real", () => { + it.each(BINDINGS.map((binding) => [binding.id, binding] as const))( + "%s names controls that exist", + (_id, binding) => { + const available = controlTitles(); + const missing = (binding.controls ?? []).filter((control) => !available.has(control)); + expect( + missing, + `Claim binding "${binding.id}" names controls that do not exist in ` + + "quickstart-negative-control.test.ts. The control was renamed or removed. " + + "Re-point the binding at the control that now proves the claim, or mark the " + + "claim unproven and name the ticket." + ).toEqual([]); + } + ); + + it.each(BINDINGS.map((binding) => [binding.id, binding] as const))( + "%s is either proven or openly unproven", + (_id, binding) => { + if ((binding.controls ?? []).length > 0) return; + expect( + binding.unprovenReason, + `Claim "${binding.id}" names no control and gives no unprovenReason. One or the ` + + "other is required: a documented enforcement claim with neither is exactly the " + + "unbacked assertion AAASM-5526 exists to eliminate." + ).toBeTruthy(); + expect( + binding.unprovenReason ?? "", + `Claim "${binding.id}" is unproven but its reason names no ticket. An unproven ` + + "claim must be traceable to the work that resolves it." + ).toMatch(/AAASM-\d+/); + } + ); +}); + +describe("claim gate: the documented error class is the one the SDK throws", () => { + it("a denied call rejects with the class the quick-start names", async () => { + const effect = createFileSideEffect(); + try { + const gateway = createPolicyGatewayClient({ denyTools: ["write_file"] }); + const tools = { + write_file: { execute: async (content: string) => effect.write(content) } + }; + + withAssembly(tools, { gatewayClient: gateway, agentId: "claim-binding-agent" }); + + const outcome = await tools.write_file.execute("denied-content").then( + (value: unknown) => value, + (error: unknown) => error + ); + + // Absence of the effect first, as in every control in this suite: an + // error assertion placed ahead of it aborts before the side effect is + // checked. + expect( + effect.occurred(), + "the governed call ran the tool body; this gate is measuring the wrong path" + ).toBe(false); + expect(outcome).toBeInstanceOf(Error); + + // The name as the running SDK reports it, derived not imported. + const thrown = (outcome as Error).constructor.name; + + // The names the document promises a reader, read out of the document. + const documented = [ + ...readFileSync(QUICK_START, "utf-8").matchAll(/`([A-Za-z0-9_]+Error)`/g) + ].map((match) => match[1]); + expect( + documented.length, + "The quick-start no longer names any `Error`. It used to promise a reader " + + "the concrete error a deny throws; if that promise was removed, this gate must " + + "be re-pointed rather than deleted." + ).toBeGreaterThan(0); + + expect( + documented, + `A denied call rejects with ${thrown}, which the quick-start does not name. ` + + "Either the class was renamed and the documentation now points at something a " + + "reader cannot import, or the deny path changed which error it throws." + ).toContain(thrown); + } finally { + effect.cleanup(); + } + }); +}); From 228811e5f1653dd38ab2eaf254bb2e4d8707ce95 Mon Sep 17 00:00:00 2001 From: Chisanan232 Date: Thu, 13 Aug 2026 18:53:02 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=F0=9F=94=A7=20(ci):=20Run=20the=20claim-bi?= =?UTF-8?q?nding=20gate=20where=20a=20docs-only=20PR=20can=20reach=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test-matrix.yml sets paths-ignore on docs/** and **/*.md, so a PR that only rewords a quick-start claim runs no tests — which is exactly the change the claim-binding gate exists to catch. publish-docs.yml already triggers on docs/**, so the gate runs there. The step's comment states the limit rather than leaving it implied: publish-docs is not one of the workflows ci-success.yml aggregates, and branch protection on main declares no required status checks at all (verified against the API). This step makes the drift visible on the PR; it does not by itself block a merge. Closing that gap is AAASM-5677's scope. Refs AAASM-5529 --- .github/workflows/publish-docs.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index ccb807fd1..5e071fdbe 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -74,6 +74,28 @@ jobs: pnpm run generate:docs-metadata git diff --exit-code -- README.md docs src/generated + # AAASM-5529. The step above round-trips the quickstart tab block as TEXT: + # it proves docs match the vendored snippets under metadata/ and nothing + # more. The snippets are never executed, type-checked or linted — they are + # excluded from ESLint (eslint.config.mjs), Prettier (.prettierignore) and + # every tsconfig, and they import "./policy.js", a file the reader supplies. + # + # This step gates a different surface: the prose in §2, §3 and §4, where + # the quick-start states what governance does. Each claim is bound to the + # control that proves it. + # + # It runs HERE because test-matrix.yml sets paths-ignore on docs/** and + # **/*.md, so a docs-only PR — precisely the change that rewords a claim — + # runs no tests at all. + # + # SCOPE OF ENFORCEMENT, so nobody reads this as more than it is: this + # workflow is NOT one of the workflows ci-success.yml aggregates, and + # branch protection on main currently declares no required status checks + # at all. This step therefore makes the drift VISIBLE on the PR; it does + # not by itself block a merge. Closing that gap is AAASM-5677's scope. + - name: Every documented enforcement claim names the control that proves it + run: pnpm exec vitest run tests/quickstart-claim-bindings.test.ts + - name: Install website dependencies # `--ignore-workspace` tells pnpm to skip the parent # `pnpm-workspace.yaml` (added under AAASM-1220) when installing From 6e983ad1b6afbec93a8b5d7d5e0982f6f56db369 Mon Sep 17 00:00:00 2001 From: Chisanan232 Date: Thu, 13 Aug 2026 19:36:08 +0800 Subject: [PATCH 3/9] =?UTF-8?q?=F0=9F=90=9B=20(tests):=20Gate=20whole=20se?= =?UTF-8?q?ntences,=20bind=20the=20audit=20claim=20to=20a=20control=20that?= =?UTF-8?q?=20can=20fail?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review broke the central promise in under a minute. Four defects, all here: F1 — the gate asked whether the region CONTAINED a binding's quote. The reviewer replaced the true claim with its opposite ("a denied call is recorded for review while the tool still runs to completion"), left the bound fragment at :105 intact, and the gate stayed green at 28 passed. Quotes are now whole sentences compared with ===, and exactly one binding may match a sentence. Re-run under the same attack: 2 failed. F2 — "with either gateway client this SDK ships it is discarded, not retained" was bound to a negative control that hands the event to a FIXTURE client, which retains it. Flipping both shipped clients to caller-supplied left that control green, so the binding could not fail when the claim became false. Repointed at the audit-sink suite, which drives the shipped client against a downstream boundary probe with a reachability positive control beside it. Under the same flip that control now fails 9 of 13. F4 — `kind` was written and never read: dead decoration a reader would assume was enforced. Removed; every claim now needs controls or a ticketed reason. F6 — only three regions were scanned. The scan now covers the whole document minus a named section allow-list whose entries must still resolve to real headings. Front matter is stripped and fenced blocks become paragraph breaks rather than spaces: a fence replaced by a space glued the sentences either side of it into one, which would have let a binding cover two claims at once. Two claims that surfaced once the scan widened are now bound: auto-start being opt-in, and the page's own "from nothing to a governed agent" promise. Refs AAASM-5529 --- tests/quickstart-claim-bindings.test.ts | 485 +++++++++++++----------- 1 file changed, 274 insertions(+), 211 deletions(-) diff --git a/tests/quickstart-claim-bindings.test.ts b/tests/quickstart-claim-bindings.test.ts index c530906c6..f9c1d3493 100644 --- a/tests/quickstart-claim-bindings.test.ts +++ b/tests/quickstart-claim-bindings.test.ts @@ -2,46 +2,43 @@ * Drift gate binding the documented Node quick-start's enforcement claims to * the controls that prove them (AAASM-5529, Epic AAASM-5526). * - * `docs/02-quick-start/index.md` §3 and §4 are where the quick-start tells a - * reader what governance does for them: that policy is enforced before a tool - * runs, that a deny throws a `PolicyViolationError` with the tool body never - * running, and that the allow path's governance event is discarded rather than - * retained. Nothing connected those sentences to the controls in - * `quickstart-negative-control.test.ts`, so a claim could be added, reworded, - * or left standing after the behaviour beneath it changed, and no gate would - * notice. + * `docs/02-quick-start/index.md` tells a reader what governance does for them. + * Those sentences are the product's load-bearing enforcement claims, and + * nothing connected them to the controls that prove them. * * WHAT THIS GATE PROVES * - * 1. Every enforcement sentence in the gated sections is bound to a named - * control. The sentences are read out of the document, so a new claim that - * no binding quotes fails here rather than shipping unbacked. - * 2. Every binding still describes the document. Rewording a claim breaks its - * quote and fails. - * 3. Every control a binding names still exists. Control ids are extracted from - * each control file's TypeScript AST as ` :: > `, not - * transcribed, so renaming or deleting one fails here. - * 4. The error class the document names is the one the SDK actually throws. - * Derived by driving a real deny through `withAssembly` and reading - * `constructor.name` off the thrown value. The class is deliberately NOT - * imported here: importing it would make a rename a *type* error, which is - * red but aborts before the assertion meant to catch it can run — the - * inverted-order defect the round-1 review of this ticket found in all three - * SDKs. + * 1. The WHOLE document is scanned, not opted-in regions. Every sentence using + * enforcement vocabulary must be bound. Sections and sentences are excluded + * only through the two named allow-lists, each entry carrying a reason and + * an exact sentence, so an entry cannot cover a reworded or new claim. + * 2. A binding must match a WHOLE sentence, exactly — compared with ===, never + * with `includes`. Containment let a sentence carry unlimited extra unbound + * claims as long as one bound fragment survived. The reviewer replaced the + * true claim with its opposite, left the bound fragment intact, and this + * gate stayed green; that is what this revision closes. + * 3. Exactly one binding may match a sentence. + * 4. Every control a binding names still exists, extracted from each control + * file's TypeScript AST as ` :: > `. + * 5. Every claim is proven or openly unproven, with no exempt category. There + * was a `kind` field; it was written and never read, which is worse than a + * bypass because a reader assumes it is enforced. Removed. + * 6. The error class the document names is the one the SDK actually throws, + * derived from `constructor.name` on a real deny rather than imported. * * WHAT THIS GATE DOES NOT PROVE * - * It does not execute, type-check or lint a documented snippet. - * `metadata/quickstart-snippets/` is excluded from ESLint - * (`eslint.config.mjs:22`), from Prettier (`.prettierignore:10`) and from every - * tsconfig, and the snippets import `createPolicyGatewayClient` from - * `"./policy.js"` — a file the reader supplies from the examples repo, not an - * SDK export. The existing drift check round-trips them as *text*: it proves - * this page matches the vendored snippet and nothing more. This gate does not + * It does not execute or lint a documented snippet. + * `metadata/quickstart-snippets/` is excluded from ESLint (`eslint.config.mjs`) + * and Prettier (`.prettierignore`), and the snippets import + * `createPolicyGatewayClient` from `"./policy.js"` — a file the reader supplies + * from the examples repo, not an SDK export. The base `tsconfig.json` DOES + * include the snippet files, but no CI job type-checks with that config + * (`pnpm typecheck` runs `tsconfig.test.json`), so nothing gates them. The + * existing drift step round-trips them as text only, and this gate does not * change that. * - * Nor does binding a claim make the claim true. A binding records which control - * stands behind a sentence. + * Binding a claim also does not make it true. */ import { readFileSync } from "node:fs"; @@ -57,25 +54,24 @@ const QUICK_START = resolve(process.cwd(), "docs/02-quick-start/index.md"); * The test files a binding may name. * * More than one, because the quick-start's claims are not all proved in the - * same place: the deny claims are proved by the negative controls, while the - * "reports auditSink" claim is proved by the AAASM-5681 disposition suite. - * Control ids are file-qualified so two suites cannot collide on a title. + * same place: the deny claims come from the negative controls, the audit-sink + * claims from the AAASM-5681 disposition suite, and the auto-start claim from + * the gateway resolver's suite. Control ids are file-qualified so two suites + * cannot collide on a title. */ const CONTROL_FILES = [ "tests/quickstart-negative-control.test.ts", - "tests/audit-sink-disposition.test.ts" + "tests/audit-sink-disposition.test.ts", + "tests/gateway-resolver.test.ts" ] as const; -const ENFORCEMENT = "enforcement"; -const LIFECYCLE = "lifecycle"; +const NEG = "tests/quickstart-negative-control.test.ts :: quick-start negative control: "; +const AUDIT = "tests/audit-sink-disposition.test.ts :: AAASM-5681: "; +const RESOLVER = "tests/gateway-resolver.test.ts :: resolveGatewayUrl > "; interface ClaimBinding { readonly id: string; - readonly kind: typeof ENFORCEMENT | typeof LIFECYCLE; - /** - * A verbatim fragment of the claim as it appears in the document, with - * Markdown's soft wrapping collapsed. Rewording the document breaks it. - */ + /** The claim as a WHOLE sentence, flattened. Compared with ===, not includes. */ readonly quote: string; /** ` :: > ` ids drawn from CONTROL_FILES. */ readonly controls?: readonly string[]; @@ -83,139 +79,188 @@ interface ClaimBinding { readonly unprovenReason?: string; } +const DENY_CONTROLS = [ + `${NEG}filesystem side effect > NEGATIVE CONTROL: a denied write_file leaves no file on disk`, + `${NEG}network side effect > NEGATIVE CONTROL: a denied egress tool never reaches the listener` +] as const; + +const ALLOW_AND_DENY_CONTROLS = [ + `${NEG}filesystem side effect > POSITIVE CONTROL: an allowed write_file really creates the file on disk`, + `${NEG}filesystem side effect > NEGATIVE CONTROL: a denied write_file leaves no file on disk`, + `${NEG}network side effect > POSITIVE CONTROL: an allowed egress tool reaches the listener`, + `${NEG}network side effect > NEGATIVE CONTROL: a denied egress tool never reaches the listener` +] as const; + const BINDINGS: readonly ClaimBinding[] = [ { - id: "policy-enforced-before-each-tool-runs", - kind: ENFORCEMENT, - quote: "wraps a map of tools so the local policy is enforced before each one runs", - // Both halves are named. The negative controls prove the "before" by the - // absence of the side effect; the positive controls prove the probe would - // have seen that effect had it happened. Either alone is the vacuous - // evidence this Epic exists to remove. + id: "page-takes-you-to-a-governed-agent", + quote: "This page takes you from nothing to a governed agent in a few minutes.", + controls: ALLOW_AND_DENY_CONTROLS + }, + { + id: "sdk-enforces-by-talking-to-a-gateway", + quote: "The SDK enforces policy by talking to an Agent Assembly **gateway**.", + unprovenReason: + "AAASM-5663: both README entrypoints were executed and neither runs as written, so no " + + "control covers a reader actually reaching a gateway. The control that exists asserts " + + "the documented config REFUSES to init; it does not prove this sentence." + }, + { + id: "auto-start-is-opt-in", + quote: + "Auto-start is opt-in — without it, a missing gateway throws a `ConfigurationError` " + + "instead of spawning anything.", controls: [ - "tests/quickstart-negative-control.test.ts :: quick-start negative control: filesystem side effect > POSITIVE CONTROL: an allowed write_file really creates the file on disk", - "tests/quickstart-negative-control.test.ts :: quick-start negative control: filesystem side effect > NEGATIVE CONTROL: a denied write_file leaves no file on disk", - "tests/quickstart-negative-control.test.ts :: quick-start negative control: network side effect > POSITIVE CONTROL: an allowed egress tool reaches the listener", - "tests/quickstart-negative-control.test.ts :: quick-start negative control: network side effect > NEGATIVE CONTROL: a denied egress tool never reaches the listener" + `${RESOLVER}throws ConfigurationError instead of auto-starting when AA_AUTO_START is unset` ] }, + { + id: "policy-enforced-before-each-tool-runs", + quote: + "`withAssembly` wraps a map of tools so the local policy is enforced before each one runs: " + + "an allowed call executes normally, while a denied call throws a `PolicyViolationError` and " + + "the tool body never runs.", + // Both halves. The negative controls prove the "before" by absence of the + // side effect; the positive controls prove the probe would have seen that + // effect had it happened. + controls: ALLOW_AND_DENY_CONTROLS + }, { id: "allow-event-is-discarded-not-retained", - kind: ENFORCEMENT, - // A negative capability claim, and the honest one: it says the SDK retains - // nothing. Bound so that if someone deletes the caveat because a sink was - // wired (AAASM-5750), this gate makes them revisit the control rather than - // quietly dropping the sentence. - quote: "it is discarded, not retained, so there is no audit trail to read it back from", + quote: + "A governance event is emitted — but with either gateway client this SDK ships it is " + + "discarded, not retained, so there is no audit trail to read it back from.", + // Repointed under review. This was bound to a negative control that hands + // the event to a FIXTURE client, which retains it — so flipping the shipped + // clients' disposition left that control green and the binding could not + // fail when the claim became false. The control named here drives the + // SHIPPED client against a downstream boundary probe, and its sibling at + // :164 is the reachability positive control for that same probe. controls: [ - "tests/quickstart-negative-control.test.ts :: quick-start negative control: a deny is handed to the gateway's record call > hands the gateway an audit event naming the denied tool and its run" + `${AUDIT}shipped clients declare what they do with audit events > a client declaring "discarded" reaches nothing with any audit method` ] }, { id: "init-warns-and-reports-the-discard", - kind: ENFORCEMENT, - // Found by this gate's own unbound-sentence check, not by reading: the - // sentence before it was bound, this one was not, and the two make - // different claims. The first says the event is dropped; this one says - // initAssembly tells you so at startup and on the context. A reader relying - // on the second without the first would not know to look. quote: - 'warns about this at startup and reports `auditSink: "discarded"` on the returned context', + '`initAssembly` warns about this at startup and reports `auditSink: "discarded"` on the ' + + "returned context; supply your own `gatewayClient` to retain the event (AAASM-5681).", controls: [ - 'tests/audit-sink-disposition.test.ts :: AAASM-5681: initAssembly surfaces the drop without AA_DEBUG > WARNS on stderr and reports auditSink="discarded" with AA_DEBUG unset' + `${AUDIT}initAssembly surfaces the drop without AA_DEBUG > WARNS on stderr and reports auditSink="discarded" with AA_DEBUG unset` ] }, { id: "deny-rejects-and-body-never-runs", - kind: ENFORCEMENT, - quote: "message includes the tool name and the gateway's reason — the tool body never runs", - controls: [ - "tests/quickstart-negative-control.test.ts :: quick-start negative control: filesystem side effect > NEGATIVE CONTROL: a denied write_file leaves no file on disk", - "tests/quickstart-negative-control.test.ts :: quick-start negative control: network side effect > NEGATIVE CONTROL: a denied egress tool never reaches the listener" - ] + quote: + "- **Deny.** The wrapped `invoke()` *rejects* with a `PolicyViolationError` whose message " + + "includes the tool name and the gateway's reason — the tool body never runs.", + controls: DENY_CONTROLS }, { id: "pending-waits-then-proceeds-or-rejects", - kind: ENFORCEMENT, - quote: "for a human decision, then either proceeds or rejects", + quote: + "- **Pending (needs approval).** The call waits up to `langchain.approvalTimeoutMs` (default " + + "applies if unset) for a human decision, then either proceeds or rejects.", controls: [ - "tests/quickstart-negative-control.test.ts :: quick-start negative control: a deny is handed to the gateway's record call > hands the gateway a distinct audit event when an approval is rejected" + `${NEG}a deny is handed to the gateway's record call > hands the gateway a distinct audit event when an approval is rejected` ] }, { id: "deny-throws-instead-of-executing", - kind: ENFORCEMENT, - // Also found by the unbound-sentence check. It restates the deny claim in - // the strongest terms the page uses ("the whole point"), and a restatement - // is exactly the kind of sentence that survives a behaviour change because - // nobody thinks of it as the claim. - quote: "a denied tool call throws instead of executing", + // A restatement, which is exactly the kind of sentence that survives a + // behaviour change because nobody thinks of it as the claim. + quote: + "That deny-on-policy behavior is the whole point: a denied tool call throws instead of executing.", controls: [ - "tests/quickstart-negative-control.test.ts :: quick-start negative control: filesystem side effect > NEGATIVE CONTROL: a denied write_file leaves no file on disk", - "tests/quickstart-negative-control.test.ts :: quick-start negative control: filesystem side effect > FALSIFICATION: the same write, ungoverned, does create the file" + `${NEG}filesystem side effect > NEGATIVE CONTROL: a denied write_file leaves no file on disk`, + `${NEG}filesystem side effect > FALSIFICATION: the same write, ungoverned, does create the file` ] }, { id: "observe-mode-does-not-block", - kind: ENFORCEMENT, - quote: "watch what *would* be blocked without actually blocking it", + quote: + "If you want to watch what *would* be blocked without actually blocking it while you tune " + + "policy, register the agent in observe mode:", controls: [ - "tests/quickstart-negative-control.test.ts :: quick-start negative control: the zero-config initAssembly path > BOUNDARY: enforcementMode observe inits and lets the tool body run" + `${NEG}the zero-config initAssembly path > BOUNDARY: enforcementMode observe inits and lets the tool body run` ] - }, - { - id: "gateway-must-be-reachable", - kind: LIFECYCLE, - quote: "The SDK enforces policy by talking to an Agent Assembly **gateway**", - unprovenReason: - "AAASM-5663: both README entrypoints were executed and neither runs as written, " + - "so no control covers the documented zero-config path end to end. The control " + - "that exists asserts the documented config REFUSES to init; it does not prove a " + - "reader can reach a working gateway." } ]; -/** Drops fenced code and collapses soft wrapping. */ +/** Whole sections excluded from the scan, each with a reason. */ +const EXCLUDED_SECTIONS: Record = { + "## Next steps": + "A link list. Every line is a cross-reference to another page; the claims live on the pages " + + "linked to and are gated there." +}; + +/** + * Individual sentences excluded from the scan, each with a reason. Exact + * flattened sentences, never patterns, so an entry cannot silently cover a + * reworded or newly added claim. + */ +const EXCLUDED_SENTENCES: Record = {}; + +/** Strips Docusaurus front matter, which otherwise flattens into sentence one. */ +function stripFrontMatter(text: string): string { + return text.replace(/^---\n[\s\S]*?\n---\n/, ""); +} + function flattenMarkdown(text: string): string { - return text - .replace(/```[\s\S]*?```/g, " ") - .split(/\s+/) - .join(" ") - .trim(); + return text.split(/\s+/).join(" ").trim(); +} + +function splitSentences(paragraph: string): string[] { + const parts = paragraph.split(/(?<=\.)\s/); + return parts; } /** - * The flattened text of the quick-start regions this gate is responsible for. + * Every sentence in the document, keyed to its section heading. * - * Read from the document rather than transcribed, so a claim added to either - * region shows up here without anyone editing this file. + * The gate opts sections OUT by name rather than opting them in, so a claim + * added to a section nobody thought about is still caught. A fenced block + * becomes a PARAGRAPH break, not a space: replacing it with a space glued the + * sentence before a code sample to the one after it, and a binding quoting the + * glued pair would cover two claims at once — fragment containment one level up. */ -function gatedDocumentRegions(): Record { - const document = readFileSync(QUICK_START, "utf-8"); - const openings: Record = { - "gateway-reachable": "## 2. Make sure a gateway is reachable", - "govern-your-first-agent": "## 3. Govern your first agent", - "what-to-expect": "## 4. What to expect" - }; +function scannedSentences(): Record { + const body = stripFrontMatter(readFileSync(QUICK_START, "utf-8")).replace( + /```[\s\S]*?```/g, + "\n\n" + ); - const regions: Record = {}; - for (const [name, opening] of Object.entries(openings)) { - const start = document.indexOf(opening); - expect( - start, - `${QUICK_START} no longer contains the "${name}" region (looked for ${JSON.stringify(opening)}). ` + - "If the quick-start was restructured, re-point this gate at the section that now " + - "carries the enforcement claims — do not delete it." - ).toBeGreaterThan(-1); - - // Skip the region's own heading LINE before looking for the next heading. - const body = document.slice(start); - const afterHeading = body.indexOf("\n"); - const rest = afterHeading === -1 ? "" : body.slice(afterHeading); - const next = rest.search(/\n#{1,6} /); - regions[name] = flattenMarkdown(next === -1 ? body : body.slice(0, afterHeading + next)); + const sentences: Record = {}; + let section = "(preamble)"; + for (const chunk of body.split(/^(#{2,6} .*)$/m)) { + if (chunk === undefined) continue; + if (/^#{2,6} /.test(chunk)) { + section = chunk.trim(); + continue; + } + if (section in EXCLUDED_SECTIONS) continue; + for (const paragraph of chunk.split("\n\n")) { + for (const raw of splitSentences(paragraph)) { + const flat = flattenMarkdown(raw); + if (flat !== "") sentences[flat] = section; + } + } } - return regions; + return sentences; +} + +const ENFORCEMENT_VOCABULARY = + /\bdenie[sd]\b|\bdeny\b|\bblocked\b|\bblocking\b|\bnever runs?\b|\bbefore execution\b|\bchecked against\b|\benforces?\b|\benforced\b|\bpassthrough\b|\bdiscards?\b|\bdiscarded\b|\bthrows?\b|\brejects?\b|\brouted\b|\bintercepts?\b|\binterception\b|\bgoverned\b|\bverified\b|\bprotection\b|\bunprotected\b|\bbypass(ed|es)?\b/i; + +/** The scanned sentences that make an enforcement claim. */ +function claimSentences(): Record { + const claims: Record = {}; + for (const [sentence, section] of Object.entries(scannedSentences())) { + if (!ENFORCEMENT_VOCABULARY.test(sentence)) continue; + if (sentence in EXCLUDED_SENTENCES) continue; + claims[sentence] = section; + } + return claims; } /** @@ -263,78 +308,103 @@ function controlTitles(): Set { } describe("claim gate: it can see what it gates", () => { - // Positive controls for the gate itself. Every check below reads a real - // artifact; these prove the reads arrived. An empty parse and a clean result + // Positive controls for the gate itself. An empty parse and a clean result // are otherwise indistinguishable. - it("finds the gated document regions and they are non-empty", () => { - const regions = gatedDocumentRegions(); - expect(Object.keys(regions).sort()).toEqual([ - "gateway-reachable", - "govern-your-first-agent", - "what-to-expect" - ]); - for (const [name, text] of Object.entries(regions)) { + it("reads the whole document and splits it into sentences", () => { + expect(Object.keys(scannedSentences()).length).toBeGreaterThan(30); + }); + + it("finds enforcement claims in more than one section", () => { + const claims = claimSentences(); + expect(Object.keys(claims).length).toBeGreaterThanOrEqual(8); + // Narrowing the scan back to one region would otherwise look identical to + // a clean pass. + expect(new Set(Object.values(claims)).size).toBeGreaterThanOrEqual(3); + }); + + it("extracts control titles from every control file", () => { + const titles = [...controlTitles()]; + for (const file of CONTROL_FILES) { expect( - text.length, - `region "${name}" is too short to hold its claims: ${text}` - ).toBeGreaterThan(80); + titles.some((title) => title.startsWith(`${file} :: `)), + `AST extraction found no controls in ${file}` + ).toBe(true); } + expect(titles).toContain( + `${NEG}filesystem side effect > NEGATIVE CONTROL: a denied write_file leaves no file on disk` + ); }); +}); - it("extracts the negative-control titles from the AST", () => { - const titles = controlTitles(); - expect(titles.size).toBeGreaterThanOrEqual(12); - // A named one, so an extraction that returned an unrelated set of the right - // size cannot satisfy the count above. - expect([...titles]).toContain( - "tests/quickstart-negative-control.test.ts :: quick-start negative control: filesystem side effect > NEGATIVE CONTROL: a denied write_file leaves no file on disk" - ); +describe("claim gate: the allow-list cannot become a bypass", () => { + it("every excluded section is still a real heading", () => { + const document = readFileSync(QUICK_START, "utf-8"); + for (const [heading, reason] of Object.entries(EXCLUDED_SECTIONS)) { + expect(document, `EXCLUDED_SECTIONS names "${heading}", no longer a heading`).toContain( + heading + ); + expect(reason.trim()).not.toBe(""); + } + }); + + it("every excluded sentence is still present verbatim", () => { + // An entry is a whole sentence, so rewording the claim makes the entry + // stale and fails here rather than silently exempting the new wording. + const scanned = scannedSentences(); + for (const [sentence, reason] of Object.entries(EXCLUDED_SENTENCES)) { + expect( + Object.hasOwn(scanned, sentence), + `EXCLUDED_SENTENCES contains a sentence no longer in the quick-start:\n ${sentence}\n` + + "Delete the stale entry, and if the replacement makes a claim, bind it." + ).toBe(true); + expect(reason.trim()).not.toBe(""); + } }); }); describe("claim gate: every documented claim is bound", () => { + it("no enforcement sentence is unbound", () => { + const quotes = new Set(BINDINGS.map((binding) => binding.quote)); + const unbound = Object.entries(claimSentences()) + .filter(([sentence]) => !quotes.has(sentence)) + .map(([sentence, section]) => `[${section}] ${sentence}`); + + expect( + unbound, + "These quick-start sentences make an enforcement claim and have no ClaimBinding:\n" + + unbound.map((s) => ` ${s}`).join("\n") + + "\n\nAdd a ClaimBinding whose quote is the WHOLE sentence, naming the control that proves " + + "it. If no control does, set unprovenReason and name the ticket. If the sentence makes no " + + "capability claim, add it to EXCLUDED_SENTENCES with a reason — do not delete the claim " + + "from this gate to make it pass." + ).toEqual([]); + }); + it.each(BINDINGS.map((binding) => [binding.id, binding] as const))( - "%s still quotes the document", + "%s matches exactly one whole sentence", (_id, binding) => { - const regions = Object.values(gatedDocumentRegions()); + // Whole-sentence equality, not containment. Containment allowed a + // sentence to carry extra unbound claims — up to and including its own + // negation — while one bound fragment kept the gate green. + const matches = Object.keys(scannedSentences()).filter( + (sentence) => sentence === binding.quote + ); expect( - regions.some((text) => text.includes(binding.quote)), - `Claim binding "${binding.id}" quotes:\n ${JSON.stringify(binding.quote)}\n` + - "which no longer appears in the gated regions of the quick-start. The claim was " + - "reworded or removed. Update the quote and re-check that the named controls still " + - "prove the new wording." - ).toBe(true); + matches.length, + `ClaimBinding "${binding.id}" must match exactly one whole sentence; it matched ` + + `${matches.length}.\nIts quote is:\n ${binding.quote}\n` + + "The claim was reworded, split, or merged. Update the quote to the new whole sentence " + + "and re-check that the named controls still prove it." + ).toBe(1); } ); - it("no enforcement sentence in the gated regions is unbound", () => { - // The check that makes this gate load-bearing rather than decorative: a new - // enforcement sentence cannot reach the published quick-start without - // someone naming the control behind it. - // - // Deliberately excludes a bare "policy": it appears in every snippet import - // line and tab label, so matching it would sweep in prose that makes no - // enforcement claim. - const enforcementLanguage = - /\bdenie[sd]\b|\bdeny\b|\bblocked\b|\bblocking\b|\bnever runs\b|\benforced?\b|\brejects\b|\bdiscarded\b/i; - - const unbound: string[] = []; - for (const [region, text] of Object.entries(gatedDocumentRegions())) { - for (const raw of text.split(/(?<=\.)\s+/)) { - const sentence = raw.trim(); - if (sentence === "" || !enforcementLanguage.test(sentence)) continue; - if (BINDINGS.some((binding) => sentence.includes(binding.quote))) continue; - unbound.push(`[${region}] ${sentence}`); - } - } - + it("no two bindings claim the same sentence", () => { + const quotes = BINDINGS.map((binding) => binding.quote); + const duplicates = quotes.filter((quote, index) => quotes.indexOf(quote) !== index); expect( - unbound, - "These quick-start enforcement sentences have no ClaimBinding:\n" + - unbound.map((s) => ` ${s}`).join("\n") + - "\n\nAdd a ClaimBinding naming the control that proves each one. If no control " + - "does, set unprovenReason and name the ticket — do not delete the claim from " + - "this gate to make it pass." + duplicates, + "Split responsibility like that and neither binding owns the claim." ).toEqual([]); }); }); @@ -347,10 +417,9 @@ describe("claim gate: every binding names something real", () => { const missing = (binding.controls ?? []).filter((control) => !available.has(control)); expect( missing, - `Claim binding "${binding.id}" names controls that do not exist in ` + - "quickstart-negative-control.test.ts. The control was renamed or removed. " + - "Re-point the binding at the control that now proves the claim, or mark the " + - "claim unproven and name the ticket." + `ClaimBinding "${binding.id}" names controls that do not exist. The control was renamed ` + + "or removed. Re-point the binding at the control that now proves the claim, or mark " + + "the claim unproven and name the ticket." ).toEqual([]); } ); @@ -358,18 +427,16 @@ describe("claim gate: every binding names something real", () => { it.each(BINDINGS.map((binding) => [binding.id, binding] as const))( "%s is either proven or openly unproven", (_id, binding) => { + // Every claim, with no exempt category. There used to be a `kind` field + // that was written and never read — dead decoration a reader would assume + // was enforced. Removed rather than wired up. if ((binding.controls ?? []).length > 0) return; expect( binding.unprovenReason, - `Claim "${binding.id}" names no control and gives no unprovenReason. One or the ` + - "other is required: a documented enforcement claim with neither is exactly the " + - "unbacked assertion AAASM-5526 exists to eliminate." + `Claim "${binding.id}" names no control and gives no unprovenReason. A documented claim ` + + "with neither is exactly the unbacked assertion AAASM-5526 exists to eliminate." ).toBeTruthy(); - expect( - binding.unprovenReason ?? "", - `Claim "${binding.id}" is unproven but its reason names no ticket. An unproven ` + - "claim must be traceable to the work that resolves it." - ).toMatch(/AAASM-\d+/); + expect(binding.unprovenReason ?? "").toMatch(/AAASM-\d+/); } ); }); @@ -390,34 +457,30 @@ describe("claim gate: the documented error class is the one the SDK throws", () (error: unknown) => error ); - // Absence of the effect first, as in every control in this suite: an - // error assertion placed ahead of it aborts before the side effect is - // checked. + // Absence of the effect first, as in every control in this suite. expect( effect.occurred(), - "the governed call ran the tool body; this gate is measuring the wrong path" + "the governed call ran the tool body; measuring the wrong path" ).toBe(false); expect(outcome).toBeInstanceOf(Error); // The name as the running SDK reports it, derived not imported. const thrown = (outcome as Error).constructor.name; - // The names the document promises a reader, read out of the document. const documented = [ ...readFileSync(QUICK_START, "utf-8").matchAll(/`([A-Za-z0-9_]+Error)`/g) ].map((match) => match[1]); expect( documented.length, - "The quick-start no longer names any `Error`. It used to promise a reader " + - "the concrete error a deny throws; if that promise was removed, this gate must " + - "be re-pointed rather than deleted." + "The quick-start no longer names any `Error`. If that promise was removed, this " + + "gate must be re-pointed rather than deleted." ).toBeGreaterThan(0); expect( documented, - `A denied call rejects with ${thrown}, which the quick-start does not name. ` + - "Either the class was renamed and the documentation now points at something a " + - "reader cannot import, or the deny path changed which error it throws." + `A denied call rejects with ${thrown}, which the quick-start does not name. Either the ` + + "class was renamed and the documentation now points at something a reader cannot " + + "import, or the deny path changed which error it throws." ).toContain(thrown); } finally { effect.cleanup(); From 1f27fc4b18ca40a15a431d00737e30ef0e506fc1 Mon Sep 17 00:00:00 2001 From: Chisanan232 Date: Thu, 13 Aug 2026 19:36:08 +0800 Subject: [PATCH 4/9] =?UTF-8?q?=F0=9F=94=A7=20(ci):=20Re-run=20the=20claim?= =?UTF-8?q?=20gate=20when=20the=20gate=20itself=20is=20weakened?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F8 — publish-docs.yml's paths omitted tests/**, so a PR that renamed a control or changed which error the deny throws would not re-run the gate that exists to notice. Added tests/**, src/gateway/** and src/wrappers/**. F9 — the step comment said the snippets are excluded from "every tsconfig". That is false: the base tsconfig.json does include them. The substantive point stands for a different reason — no CI job type-checks with that config, since pnpm typecheck runs tsconfig.test.json — so the comment now says that instead. Refs AAASM-5529 --- .github/workflows/publish-docs.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 5e071fdbe..8fbd76aad 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -17,6 +17,12 @@ on: - "src/generated/**" - "metadata/**" - "scripts/generate-docs-metadata.mjs" + # AAASM-5529: the claim-binding gate below reads the control files and the + # deny path. Without these, a PR that only WEAKENS the gate — renaming a + # control, or changing which error the deny throws — would not re-run it. + - "tests/**" + - "src/gateway/**" + - "src/wrappers/**" push: branches: - main @@ -76,9 +82,11 @@ jobs: # AAASM-5529. The step above round-trips the quickstart tab block as TEXT: # it proves docs match the vendored snippets under metadata/ and nothing - # more. The snippets are never executed, type-checked or linted — they are - # excluded from ESLint (eslint.config.mjs), Prettier (.prettierignore) and - # every tsconfig, and they import "./policy.js", a file the reader supplies. + # more. The snippets are never executed or linted in CI — they are excluded + # from ESLint (eslint.config.mjs) and Prettier (.prettierignore), and they + # import "./policy.js", a file the reader supplies. The base tsconfig.json + # does include them, but no CI job type-checks with it (pnpm typecheck runs + # tsconfig.test.json), so nothing gates them. # # This step gates a different surface: the prose in §2, §3 and §4, where # the quick-start states what governance does. Each claim is bound to the From ee1ea27bc5f40576c9d7db6338f6904d7f9b531d Mon Sep 17 00:00:00 2001 From: Chisanan232 Date: Thu, 13 Aug 2026 19:45:16 +0800 Subject: [PATCH 5/9] =?UTF-8?q?=F0=9F=90=9B=20(tests):=20Normalise=20line?= =?UTF-8?q?=20endings=20before=20splitting=20the=20document?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four Windows legs of test-matrix failed on "no enforcement sentence is unbound" while Linux and macOS passed. Cause: git checks the doc out with CRLF on Windows, so split("\n\n") finds no paragraph break and the whole section collapses into one "sentence" that matches no binding. Proven load-bearing rather than assumed: with the normalisation removed and a CRLF copy of the document, the suite fails 2 of 38 — the same signature CI reported. With it, CRLF and LF both pass. The same normalisation was added to the Python and Go gates, where the bug is latent because neither repo's CI runs Windows. Refs AAASM-5529 --- tests/quickstart-claim-bindings.test.ts | 27 +++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/tests/quickstart-claim-bindings.test.ts b/tests/quickstart-claim-bindings.test.ts index f9c1d3493..23701a5bc 100644 --- a/tests/quickstart-claim-bindings.test.ts +++ b/tests/quickstart-claim-bindings.test.ts @@ -201,6 +201,20 @@ const EXCLUDED_SECTIONS: Record = { */ const EXCLUDED_SENTENCES: Record = {}; +/** + * Read the quick-start with line endings normalised to LF. + * + * Without this the paragraph split below never fires on Windows: git checks the + * file out with CRLF, so `split("\n\n")` finds no `\n\n` and the whole section + * collapses into one "sentence" that matches no binding. The four Windows legs + * of test-matrix caught exactly that — the Linux and macOS legs stayed green, + * which is why it is worth stating that this gate reads a file whose bytes + * differ per platform. + */ +function readQuickStart(): string { + return readFileSync(QUICK_START, "utf-8").replace(/\r\n/g, "\n"); +} + /** Strips Docusaurus front matter, which otherwise flattens into sentence one. */ function stripFrontMatter(text: string): string { return text.replace(/^---\n[\s\S]*?\n---\n/, ""); @@ -225,10 +239,7 @@ function splitSentences(paragraph: string): string[] { * glued pair would cover two claims at once — fragment containment one level up. */ function scannedSentences(): Record { - const body = stripFrontMatter(readFileSync(QUICK_START, "utf-8")).replace( - /```[\s\S]*?```/g, - "\n\n" - ); + const body = stripFrontMatter(readQuickStart()).replace(/```[\s\S]*?```/g, "\n\n"); const sentences: Record = {}; let section = "(preamble)"; @@ -338,7 +349,7 @@ describe("claim gate: it can see what it gates", () => { describe("claim gate: the allow-list cannot become a bypass", () => { it("every excluded section is still a real heading", () => { - const document = readFileSync(QUICK_START, "utf-8"); + const document = readQuickStart(); for (const [heading, reason] of Object.entries(EXCLUDED_SECTIONS)) { expect(document, `EXCLUDED_SECTIONS names "${heading}", no longer a heading`).toContain( heading @@ -467,9 +478,9 @@ describe("claim gate: the documented error class is the one the SDK throws", () // The name as the running SDK reports it, derived not imported. const thrown = (outcome as Error).constructor.name; - const documented = [ - ...readFileSync(QUICK_START, "utf-8").matchAll(/`([A-Za-z0-9_]+Error)`/g) - ].map((match) => match[1]); + const documented = [...readQuickStart().matchAll(/`([A-Za-z0-9_]+Error)`/g)].map( + (match) => match[1] + ); expect( documented.length, "The quick-start no longer names any `Error`. If that promise was removed, this " + From 5a108c85ec291eb21c231e3cdeaa3f0d900b5b47 Mon Sep 17 00:00:00 2001 From: Chisanan232 Date: Thu, 13 Aug 2026 20:25:12 +0800 Subject: [PATCH 6/9] =?UTF-8?q?=F0=9F=90=9B=20(tests):=20Invert=20the=20de?= =?UTF-8?q?fault=20=E2=80=94=20every=20sentence=20bound=20or=20allow-liste?= =?UTF-8?q?d,=20no=20keyword=20filter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R1 — review appended three plain sentences to the quick-start, the last of them "Tool bodies always execute; the policy result is recorded alongside them" — the negation of the product — and this gate stayed green at 38 passed. None used any of the 21 vocabulary terms. Widening 3 -> 21 closed the instance, not the class: a keyword allow-list cannot be completed, because whoever adds the claim picks the words after reading the list. Every one of the document's 55 sentences must now be bound to a control or allow-listed by exact text. The vocabulary gates nothing; it is a severity hint in the failure message and the trigger for a stricter allow-list rule — waving through a sentence that reads like a claim costs a written justification, not a category. R3 — section exclusions are gone. "## Next steps" was a black hole: the guard checked the heading existed and said nothing about its contents, so a claim inserted there was never scanned. A positive control now asserts that section is in the scan. R4 — HTML and MDX comments are stripped, as fences already were, because a reader cannot see them. Leaving them in let a bound claim be commented out of the rendered page while the gate still counted it. Verified with the MDX {/* */} form specifically. R5 — an unprovenReason may not name AAASM-5529, the ticket this file implements; such a pointer resolves to a closed issue on merge. Guard proven able to fail by repointing a real binding at 5529 and watching it go red. R6 — the splitter drops bare list markers, treats "!" as text, handles "?", and splits list items and table rows as units. Refs AAASM-5529 --- tests/quickstart-claim-bindings.test.ts | 424 ++++++++++++++++++------ 1 file changed, 318 insertions(+), 106 deletions(-) diff --git a/tests/quickstart-claim-bindings.test.ts b/tests/quickstart-claim-bindings.test.ts index 23701a5bc..a43009734 100644 --- a/tests/quickstart-claim-bindings.test.ts +++ b/tests/quickstart-claim-bindings.test.ts @@ -1,44 +1,53 @@ /** - * Drift gate binding the documented Node quick-start's enforcement claims to - * the controls that prove them (AAASM-5529, Epic AAASM-5526). + * Drift gate binding the documented Node quick-start's claims to the controls + * that prove them (AAASM-5529, Epic AAASM-5526). * - * `docs/02-quick-start/index.md` tells a reader what governance does for them. - * Those sentences are the product's load-bearing enforcement claims, and - * nothing connected them to the controls that prove them. + * Every sentence in `docs/02-quick-start/index.md` must be either BOUND to a + * control that proves it, or explicitly ALLOW-LISTED as making no capability + * claim. There is no third state and no keyword filter. + * + * WHY THE DEFAULT IS INVERTED + * + * Earlier revisions only scanned sentences matching an enforcement vocabulary. + * Review appended three plain sentences using none of the 21 terms — the last, + * "Tool bodies always execute; the policy result is recorded alongside them", + * is the negation of the product — and every gate stayed green. Widening 3 → 21 + * closed the instance, not the class: a keyword allow-list cannot be completed, + * because whoever adds the claim picks the words after reading the list. + * + * The vocabulary now gates nothing. It survives as a severity hint in the + * failure message, and as the trigger for a stricter allow-list rule: an entry + * whose sentence reads like a claim needs a written justification, not a + * category. + * + * Section-level exclusions are gone. An excluded section was a black hole — the + * guard checked the heading still existed and said nothing about its contents — + * so a claim inserted into "## Next steps" was never scanned at all. * * WHAT THIS GATE PROVES * - * 1. The WHOLE document is scanned, not opted-in regions. Every sentence using - * enforcement vocabulary must be bound. Sections and sentences are excluded - * only through the two named allow-lists, each entry carrying a reason and - * an exact sentence, so an entry cannot cover a reworded or new claim. - * 2. A binding must match a WHOLE sentence, exactly — compared with ===, never - * with `includes`. Containment let a sentence carry unlimited extra unbound - * claims as long as one bound fragment survived. The reviewer replaced the - * true claim with its opposite, left the bound fragment intact, and this - * gate stayed green; that is what this revision closes. - * 3. Exactly one binding may match a sentence. - * 4. Every control a binding names still exists, extracted from each control - * file's TypeScript AST as ` :: > `. - * 5. Every claim is proven or openly unproven, with no exempt category. There - * was a `kind` field; it was written and never read, which is worse than a - * bypass because a reader assumes it is enforced. Removed. - * 6. The error class the document names is the one the SDK actually throws, - * derived from `constructor.name` on a real deny rather than imported. + * 1. Every sentence in the document is accounted for. + * 2. A binding matches a WHOLE sentence, exactly, and only one binding may. + * 3. Every control a binding names exists, extracted from the control files' + * TypeScript ASTs rather than transcribed. + * 4. Every claim is proven or openly unproven, and an unproven claim may not + * name the ticket this file implements — that pointer resolves to a closed + * issue the moment the work merges. + * 5. Comments are stripped before scanning, HTML and MDX alike, because a + * reader cannot see them. Leaving them in let a bound claim be commented out + * of the rendered page while the gate still counted it. * * WHAT THIS GATE DOES NOT PROVE * * It does not execute or lint a documented snippet. * `metadata/quickstart-snippets/` is excluded from ESLint (`eslint.config.mjs`) * and Prettier (`.prettierignore`), and the snippets import - * `createPolicyGatewayClient` from `"./policy.js"` — a file the reader supplies - * from the examples repo, not an SDK export. The base `tsconfig.json` DOES - * include the snippet files, but no CI job type-checks with that config - * (`pnpm typecheck` runs `tsconfig.test.json`), so nothing gates them. The - * existing drift step round-trips them as text only, and this gate does not - * change that. + * `createPolicyGatewayClient` from `"./policy.js"` — a file the reader supplies. + * The base `tsconfig.json` DOES include them, but no CI job type-checks with + * that config (`pnpm typecheck` runs `tsconfig.test.json`), so nothing gates + * them. The existing drift step round-trips them as text only. * - * Binding a claim also does not make it true. + * Binding a claim does not make it true. */ import { readFileSync } from "node:fs"; @@ -50,6 +59,12 @@ import { createFileSideEffect, createPolicyGatewayClient } from "./helpers/negat const QUICK_START = resolve(process.cwd(), "docs/02-quick-start/index.md"); +/** + * The ticket this file implements. An unproven claim may not name it: on merge + * that pointer resolves to a closed issue and nothing would notice. + */ +const IMPLEMENTING_TICKET = "AAASM-5529"; + /** * The test files a binding may name. * @@ -152,14 +167,14 @@ const BINDINGS: readonly ClaimBinding[] = [ { id: "deny-rejects-and-body-never-runs", quote: - "- **Deny.** The wrapped `invoke()` *rejects* with a `PolicyViolationError` whose message " + + "**Deny.** The wrapped `invoke()` *rejects* with a `PolicyViolationError` whose message " + "includes the tool name and the gateway's reason — the tool body never runs.", controls: DENY_CONTROLS }, { id: "pending-waits-then-proceeds-or-rejects", quote: - "- **Pending (needs approval).** The call waits up to `langchain.approvalTimeoutMs` (default " + + "**Pending (needs approval).** The call waits up to `langchain.approvalTimeoutMs` (default " + "applies if unset) for a human decision, then either proceeds or rejects.", controls: [ `${NEG}a deny is handed to the gateway's record call > hands the gateway a distinct audit event when an approval is rejected` @@ -187,19 +202,148 @@ const BINDINGS: readonly ClaimBinding[] = [ } ]; -/** Whole sections excluded from the scan, each with a reason. */ -const EXCLUDED_SECTIONS: Record = { - "## Next steps": - "A link list. Every line is a cross-reference to another page; the claims live on the pages " + - "linked to and are gated there." -}; +/** + * Allow-list categories. Permitted only for a sentence that does NOT match the + * vocabulary below; anything that does needs a written justification. + */ +const NOT_A_CLAIM = + "Descriptive or instructional prose. Says nothing about what governance does to a tool call."; +const NAVIGATION = + "A cross-reference. The claim, if any, lives on the page linked to and is gated there."; /** - * Individual sentences excluded from the scan, each with a reason. Exact - * flattened sentences, never patterns, so an entry cannot silently cover a - * reworded or newly added claim. + * Every sentence that makes no capability claim, keyed exactly. + * + * There is no section-level exclusion: an excluded section was a black hole, + * since the guard checked the heading still existed and said nothing about its + * contents. */ -const EXCLUDED_SENTENCES: Record = {}; +const ALLOWED = new Map([ + ['import Tabs from "@theme/Tabs"; import TabItem from "@theme/TabItem";', NOT_A_CLAIM], + [ + "Everything here is copy-paste; the snippets mirror the patterns the SDK's own test suite exercises.", + NOT_A_CLAIM + ], + [ + ' ', + NOT_A_CLAIM + ], + [' ', NOT_A_CLAIM], + [' ', NOT_A_CLAIM], + [' ', NOT_A_CLAIM], + [" ", NOT_A_CLAIM], + [ + "The package ships dual ESM/CJS entries and selects a prebuilt native binding for your platform during `postinstall`, so there is no extra build step for typical consumers.", + NOT_A_CLAIM + ], + [ + ":::note[Pre-1.0 / release candidate] The public surface (`initAssembly`, `withAssembly`) is stabilizing but may change between pre-releases.", + NOT_A_CLAIM + ], + ["Pin an exact version for reproducible installs:", NOT_A_CLAIM], + ["`npm install @agent-assembly/sdk@0.0.1-rc.6`", NOT_A_CLAIM], + [":::", NOT_A_CLAIM], + ["You have two options:", NOT_A_CLAIM], + [ + "**Let the SDK auto-start a local gateway.** If you have the `aasm` binary on your `PATH` (`brew install ai-agent-assembly/tap/aasm`, or `curl -fsSL https://agent-assembly.com/install.sh | sh`) and set `AA_AUTO_START=1`, a zero-config `initAssembly()` will probe `http://localhost:7391` and start a local gateway for you if nothing is running.", + NOT_A_CLAIM + ], + [ + "**Point at a gateway you already run.** Set `AA_GATEWAY_URL` (and `AA_API_KEY` if it requires auth), or pass `gatewayUrl` explicitly.", + NOT_A_CLAIM + ], + [ + ":::note[Local-mode transports: `:7391` REST + `:50051` gRPC] Starting the local gateway binds **two** loopback surfaces in one process:", + NOT_A_CLAIM + ], + [ + "This exposes the REST API on `http://localhost:7391` (what `gatewayUrl` points to, and what the SDK probes and, with `AA_AUTO_START=1`, auto-starts) **and** the gRPC `AgentLifecycleService` on `127.0.0.1:50051`, which is the endpoint the native `aa-sdk-client` binding dials to **register** your agent.", + NOT_A_CLAIM + ], + [ + "You don't configure `:50051` yourself — registration dials it automatically — so a no-argument `initAssembly()` both connects and shows the agent in the dashboard once a gateway is reachable.", + NOT_A_CLAIM + ], + [ + "To confirm both surfaces are actually up rather than guessing from the SDK's behavior, check them directly:", + NOT_A_CLAIM + ], + ["See [Configuration](../05-configuration/index.md) for the full resolution order.", NAVIGATION], + [ + "**LangChain.js** is the validated path; the remaining frameworks are experimental.", + NOT_A_CLAIM + ], + [ + 'The excerpts are ESM / TypeScript; under CommonJS, swap the import for `const { withAssembly } = require("@agent-assembly/sdk")`.', + NOT_A_CLAIM + ], + [ + ' ', + NOT_A_CLAIM + ], + [ + ":::note[Version compatibility] Base tool abstractions like `Tool` moved out of the `langchain` monolith into `@langchain/core` when LangChain split the package in [v0.1.0](https://www.langchain.com/blog/langchain-v0-1-0) (Jan 2024; `@langchain/core` first published to npm 2023-11-22).", + NOT_A_CLAIM + ], + ['**`langchain` < 0.1.0:** `import { Tool } from "langchain/tools";`', NOT_A_CLAIM], + [ + '**`langchain` / `@langchain/core` ≥ 0.1.0 (current):** `import { tool } from "@langchain/core/tools";` :::', + NOT_A_CLAIM + ], + [' ', NOT_A_CLAIM], + [ + ' ', + NOT_A_CLAIM + ], + [' ', NOT_A_CLAIM], + [ + ":::note[Version compatibility] The React UI hooks were extracted out of the core `ai` package into a dedicated package in [AI SDK 5.0](https://ai-sdk.dev/docs/migration-guides/migration-guide-5-0), which removed the deprecated `ai/react` export.", + NOT_A_CLAIM + ], + [ + 'The `tool()` factory used above is unaffected — it still imports from `"ai"` unchanged through the 7.x line this example pins.', + NOT_A_CLAIM + ], + ['**AI SDK 4.x:** `import { useChat } from "ai/react";`', NOT_A_CLAIM], + ['**AI SDK ≥ 5.0 (current):** `import { useChat } from "@ai-sdk/react";` :::', NOT_A_CLAIM], + [' ', NOT_A_CLAIM], + [' ', NOT_A_CLAIM], + [ + ":::note[Version compatibility] Mastra [v1](https://mastra.ai/guides/migrations/upgrade-to-v1/mastra) moved every export except `Mastra` itself off the `@mastra/core` root entry point onto subpaths (see the `npx @mastra/codemod@latest v1/mastra-core-imports` codemod).", + NOT_A_CLAIM + ], + ["This example's `@mastra/core` pin (`^1.50.1`) already uses the new layout.", NOT_A_CLAIM], + [ + '**`@mastra/core` 0.x:** `import { Agent, Workflow, createTool } from "@mastra/core";`', + NOT_A_CLAIM + ], + [ + '**`@mastra/core` ≥ 1.0 (current):** `import { createTool } from "@mastra/core/tools";` (similarly `Agent` from `@mastra/core/agent`, `Workflow` from `@mastra/core/workflows`) :::', + NOT_A_CLAIM + ], + ["**Allow.** The tool runs normally and returns its result.", NOT_A_CLAIM], + [ + "**[Configuration](../05-configuration/index.md)** — every `AssemblyConfig` field and the gateway/API-key resolution precedence.", + NAVIGATION + ], + [ + "**[Core Concepts](../03-core-concepts/index.md)** — what the native binding, the adapter registry, and the `initAssembly` lifecycle actually do.", + NAVIGATION + ], + // --- sentences matching the vocabulary: written justification required --- + [ + "Pick your framework below — each tab is the governance-wiring excerpt from that framework's runnable [example](https://github.com/ai-agent-assembly/examples/tree/HEAD/node), vendored into this repo and kept in lock-step with this page by a CI drift check — the check catches this page drifting from the vendored snippet, not the vendored snippet drifting from the upstream example.", + "Describes where the tab content comes from, and is unusually precise about the limit of the drift check. 'governance-wiring excerpt' names the excerpt's provenance, not an enforcement outcome." + ], + [ + "Copy the full, runnable script — imports, tools, and the agent run — from the linked example; the slice below is just the part that wires in governance.", + "An instruction about which lines to copy. 'wires in governance' identifies the excerpt and makes no claim about what that wiring then does to a tool call." + ], + [ + "**[Guides](../04-guides/index.md)** — the full LangChain walkthrough, the low-level `withAssembly` wrapper, experimental frameworks, and how to handle allow/deny decisions and errors.", + "A cross-reference. It matches the vocabulary only through the linked page's title ('allow/deny decisions'); the claims live on that page." + ] +]); /** * Read the quick-start with line endings normalised to LF. @@ -224,56 +368,73 @@ function flattenMarkdown(text: string): string { return text.split(/\s+/).join(" ").trim(); } -function splitSentences(paragraph: string): string[] { - const parts = paragraph.split(/(?<=\.)\s/); - return parts; +/** A line that opens a list item or a table row starts a new unit. */ +const UNIT_OPENER = /^\s*(?:[-*+]|\d+\.)\s|^\|/; +const LIST_MARKER = /^\s*(?:[-*+]|\d+\.)\s+/gm; + +function splitUnits(paragraph: string): string[] { + const units: string[] = []; + let current: string[] = []; + for (const line of paragraph.split("\n")) { + if (UNIT_OPENER.test(line) && current.length > 0) { + units.push(current.join("\n")); + current = []; + } + current.push(line); + } + if (current.length > 0) units.push(current.join("\n")); + return units; } +/** + * '.' and '?' only. '!' is not a terminator: emphatic prose and admonition + * markers would otherwise split into fragments. + */ +function splitSentences(unit: string): string[] { + return unit.split(/(?<=[.?])\s/); +} + +/** + * Gates NOTHING. A severity hint in the failure message, and the trigger for the + * stricter allow-list rule. See the header for why a keyword list is unsound. + */ +const ENFORCEMENT_VOCABULARY = + /\bdenie[sd]\b|\bdeny\b|\bblocked\b|\bblocking\b|\bnever runs?\b|\bbefore execution\b|\bchecked against\b|\benforces?\b|\benforced\b|\bpassthrough\b|\bdiscards?\b|\bdiscarded\b|\bthrows?\b|\brejects?\b|\brouted\b|\bintercepts?\b|\binterception\b|\bgovern(s|ed|ance)?\b|\bverified\b|\bprotection\b|\bunprotected\b|\bbypass(ed|es)?\b/i; + /** * Every sentence in the document, keyed to its section heading. * - * The gate opts sections OUT by name rather than opting them in, so a claim - * added to a section nobody thought about is still caught. A fenced block - * becomes a PARAGRAPH break, not a space: replacing it with a space glued the - * sentence before a code sample to the one after it, and a binding quoting the - * glued pair would cover two claims at once — fragment containment one level up. + * No section is skipped. Fenced code and comments become PARAGRAPH breaks: + * replacing a fence with a space glued the sentences either side into one, and + * a comment is invisible to a reader, so a bound claim commented out of the + * rendered page must not still satisfy this gate. */ function scannedSentences(): Record { - const body = stripFrontMatter(readQuickStart()).replace(/```[\s\S]*?```/g, "\n\n"); + let body = stripFrontMatter(readQuickStart()); + for (const pattern of [/```[\s\S]*?```/g, //g, /\{\/\*[\s\S]*?\*\/\}/g]) { + body = body.replace(pattern, "\n\n"); + } const sentences: Record = {}; let section = "(preamble)"; - for (const chunk of body.split(/^(#{2,6} .*)$/m)) { + for (const chunk of body.split(/^(#{1,6} .*)$/m)) { if (chunk === undefined) continue; - if (/^#{2,6} /.test(chunk)) { + if (/^#{1,6} /.test(chunk)) { section = chunk.trim(); continue; } - if (section in EXCLUDED_SECTIONS) continue; for (const paragraph of chunk.split("\n\n")) { - for (const raw of splitSentences(paragraph)) { - const flat = flattenMarkdown(raw); - if (flat !== "") sentences[flat] = section; + for (const unit of splitUnits(paragraph)) { + for (const raw of splitSentences(unit.replace(LIST_MARKER, ""))) { + const flat = flattenMarkdown(raw); + if (flat !== "") sentences[flat] = section; + } } } } return sentences; } -const ENFORCEMENT_VOCABULARY = - /\bdenie[sd]\b|\bdeny\b|\bblocked\b|\bblocking\b|\bnever runs?\b|\bbefore execution\b|\bchecked against\b|\benforces?\b|\benforced\b|\bpassthrough\b|\bdiscards?\b|\bdiscarded\b|\bthrows?\b|\brejects?\b|\brouted\b|\bintercepts?\b|\binterception\b|\bgoverned\b|\bverified\b|\bprotection\b|\bunprotected\b|\bbypass(ed|es)?\b/i; - -/** The scanned sentences that make an enforcement claim. */ -function claimSentences(): Record { - const claims: Record = {}; - for (const [sentence, section] of Object.entries(scannedSentences())) { - if (!ENFORCEMENT_VOCABULARY.test(sentence)) continue; - if (sentence in EXCLUDED_SENTENCES) continue; - claims[sentence] = section; - } - return claims; -} - /** * Extract ` :: > ` ids from each control file's AST. * @@ -325,12 +486,25 @@ describe("claim gate: it can see what it gates", () => { expect(Object.keys(scannedSentences()).length).toBeGreaterThan(30); }); - it("finds enforcement claims in more than one section", () => { - const claims = claimSentences(); - expect(Object.keys(claims).length).toBeGreaterThanOrEqual(8); - // Narrowing the scan back to one region would otherwise look identical to - // a clean pass. - expect(new Set(Object.values(claims)).size).toBeGreaterThanOrEqual(3); + it("reaches every section, including the last", () => { + const sections = new Set(Object.values(scannedSentences())); + expect(sections.size).toBeGreaterThanOrEqual(5); + // "## Next steps" used to be excluded by name, which made it a black hole: + // a claim inserted there was never seen. + expect([...sections], "'## Next steps' must be scanned, not excluded").toContain( + "## Next steps" + ); + }); + + it("strips comments before scanning", () => { + const document = readQuickStart(); + expect(document, "this control assumes the page still contains an MDX or HTML comment").toMatch( + //g, /\{\/\*[\s\S]*?\*\/\}/g]) { body = body.replace(pattern, "\n\n"); } - const sentences: Record = {}; + const occurrences: Occurrence[] = []; let section = "(preamble)"; for (const chunk of body.split(/^(#{1,6} .*)$/m)) { if (chunk === undefined) continue; @@ -427,12 +504,17 @@ function scannedSentences(): Record { for (const unit of splitUnits(paragraph)) { for (const raw of splitSentences(unit.replace(LIST_MARKER, ""))) { const flat = flattenMarkdown(raw); - if (flat !== "") sentences[flat] = section; + if (flat !== "") occurrences.push({ text: flat, section }); } } } } - return sentences; + return occurrences; +} + +/** The de-duplicated set, for membership questions where count does not matter. */ +function scannedTexts(): Set { + return new Set(scannedOccurrences().map((occ) => occ.text)); } /** @@ -483,11 +565,11 @@ describe("claim gate: it can see what it gates", () => { // Positive controls for the gate itself. An empty parse and a clean result // are otherwise indistinguishable. it("reads the whole document and splits it into sentences", () => { - expect(Object.keys(scannedSentences()).length).toBeGreaterThan(30); + expect(scannedOccurrences().length).toBeGreaterThan(30); }); it("reaches every section, including the last", () => { - const sections = new Set(Object.values(scannedSentences())); + const sections = new Set(scannedOccurrences().map((occ) => occ.section)); expect(sections.size).toBeGreaterThanOrEqual(5); // "## Next steps" used to be excluded by name, which made it a black hole: // a claim inserted there was never seen. @@ -501,7 +583,7 @@ describe("claim gate: it can see what it gates", () => { expect(document, "this control assumes the page still contains an MDX or HTML comment").toMatch( /