From 17cd061210f9b09e3045acb8e72a13e863aa5791 Mon Sep 17 00:00:00 2001 From: Bryant Liu Date: Thu, 6 Aug 2026 14:58:10 +0800 Subject: [PATCH 1/9] =?UTF-8?q?=E2=9C=85=20(tests):=20Add=20reusable=20enf?= =?UTF-8?q?orcement-truth=20negative-control=20fixture?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real, externally-observable side effects (a file on disk, a live loopback HTTP listener) plus a policy-driven GatewayClient that records the identity triple each decision was made against. Existing deny tests assert over vi.fn() spies, which prove the SDK did not call a reference it holds — not that the effect the tool exists to produce was prevented. Refs AAASM-5529, Epic AAASM-5526 --- tests/helpers/negative-control.ts | 190 ++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 tests/helpers/negative-control.ts diff --git a/tests/helpers/negative-control.ts b/tests/helpers/negative-control.ts new file mode 100644 index 000000000..35e9e683e --- /dev/null +++ b/tests/helpers/negative-control.ts @@ -0,0 +1,190 @@ +/** + * Reusable enforcement-truth negative-control fixture (AAASM-5529). + * + * A test that only asserts "a `PolicyViolationError` was thrown" proves the SDK + * printed a refusal, not that the refusal *prevented* anything: a tool whose + * body never had an observable effect in the first place would produce the same + * green result. These helpers give a denied tool a real, externally-observable + * side effect — a file written to disk, an HTTP request delivered to a live + * listener — so a deny can be asserted as an *absence of the effect*, and the + * matching allow can be asserted as its *presence*. + * + * Every control built on this fixture must be used as a pair: + * + * - **positive control** — policy allows, the side effect is observed. Without + * it, "no file on disk" is indistinguishable from "the tool was never called + * at all", and the negative control proves nothing. + * - **negative control** — policy denies, the same side effect is absent. + * + * The side effects are deliberately real (`node:fs`, `node:http`) rather than + * spies: a spy records an intent to act, and the whole point of this Epic + * (AAASM-5526) is that intent-level evidence is what over-claimed enforcement + * looks like. + */ + +import { createServer, type IncomingMessage, type Server } from "node:http"; +import { mkdtempSync, readFileSync, rmSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { writeFile } from "node:fs/promises"; +import { AddressInfo } from "node:net"; +import type { GatewayClient } from "../../src/gateway/client.js"; +import type { + GatewayCheckRequest, + GatewayDecision, + GatewayRecordEvent, + GatewayResultRecord +} from "../../src/types/gateway-governance.js"; + +/** + * A filesystem-backed side effect: `write` really creates a file, `occurred` + * really stats it. Nothing is mocked, so an assertion over `occurred()` is an + * assertion over the world, not over the SDK's own bookkeeping. + */ +export interface FileSideEffect { + /** Absolute path the governed tool would create. */ + readonly path: string; + /** Perform the side effect (what a denied tool must never reach). */ + write: (content: string) => Promise; + /** Whether the side effect is observable on disk right now. */ + occurred: () => boolean; + /** Content actually written, or `undefined` when the effect never occurred. */ + content: () => string | undefined; + cleanup: () => void; +} + +export function createFileSideEffect(name = "denied-write.txt"): FileSideEffect { + const dir = mkdtempSync(join(tmpdir(), "aaasm-5529-")); + const path = join(dir, name); + return { + path, + write: async (content: string) => { + await writeFile(path, content, "utf8"); + return path; + }, + occurred: () => existsSync(path), + content: () => (existsSync(path) ? readFileSync(path, "utf8") : undefined), + cleanup: () => rmSync(dir, { recursive: true, force: true }) + }; +} + +/** + * A network-backed side effect: a real loopback HTTP server that records every + * request it receives. A denied tool must leave `requests()` empty — the + * strongest available in-process evidence that the egress the tool would have + * performed never left the process. + */ +export interface NetworkSideEffect { + /** URL the governed tool would call. */ + readonly url: string; + /** Perform the side effect (what a denied tool must never reach). */ + call: (body: string) => Promise; + /** Requests the listener actually received, in arrival order. */ + requests: () => readonly { method: string; url: string; body: string }[]; + /** Whether any request reached the listener. */ + occurred: () => boolean; + close: () => Promise; +} + +export async function createNetworkSideEffect(): Promise { + const received: { method: string; url: string; body: string }[] = []; + const server: Server = createServer((req: IncomingMessage, res) => { + const chunks: Buffer[] = []; + req.on("data", (chunk: Buffer) => chunks.push(chunk)); + req.on("end", () => { + received.push({ + method: req.method ?? "", + url: req.url ?? "", + body: Buffer.concat(chunks).toString("utf8") + }); + res.writeHead(204); + res.end(); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + const url = `http://127.0.0.1:${port}/exfiltrate`; + + return { + url, + call: async (body: string) => { + const response = await fetch(url, { method: "POST", body }); + return response.status; + }, + requests: () => received, + occurred: () => received.length > 0, + close: async () => { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } + }; +} + +/** + * One governance decision as the fixture gateway recorded it, carrying the full + * identity triple (`agentId` / `toolName` / `runId`) the SDK presented at check + * time. Asserting over this is how a control shows the deny was attributed to + * the right agent and tool rather than being an anonymous refusal. + */ +export interface RecordedCheck { + readonly agentId: string; + readonly toolName: string | undefined; + readonly action: string; + readonly runId: string; + readonly denied: boolean; +} + +/** + * Policy-driven {@link GatewayClient} standing in for the quick-start's + * `createPolicyGatewayClient()` (docs/02-quick-start, `withAssembly(..., { + * gatewayClient })`). It denies exactly the named tools and records every + * decision plus every audit event, so a test can assert that the audit evidence + * carries the same agent/tool identity the decision was made against — the + * AAASM-5529 acceptance criterion that a deny is attributable, not anonymous. + */ +export interface PolicyGatewayClient extends GatewayClient { + readonly decisions: readonly RecordedCheck[]; + readonly auditEvents: readonly GatewayRecordEvent[]; + readonly auditResults: readonly GatewayResultRecord[]; +} + +export function createPolicyGatewayClient(options: { + agentId: string; + denyTools: readonly string[]; +}): PolicyGatewayClient { + const decisions: RecordedCheck[] = []; + const auditEvents: GatewayRecordEvent[] = []; + const auditResults: GatewayResultRecord[] = []; + const denied = new Set(options.denyTools); + + return { + mode: "sdk-only", + decisions, + auditEvents, + auditResults, + start: async () => undefined, + close: async () => undefined, + check: async (request: GatewayCheckRequest): Promise => { + const isDenied = request.toolName !== undefined && denied.has(request.toolName); + decisions.push({ + agentId: options.agentId, + toolName: request.toolName, + action: request.action, + runId: request.runId, + denied: isDenied + }); + return isDenied + ? { denied: true, pending: false, reason: `tool '${request.toolName}' is denied by policy` } + : { denied: false, pending: false }; + }, + waitForApproval: async () => ({ denied: false }), + record: async (event: GatewayRecordEvent) => { + auditEvents.push(event); + }, + recordResult: async (record: GatewayResultRecord) => { + auditResults.push(record); + }, + scanPrompts: async () => undefined + }; +} From cfb991e243eab7394e01953ef8508f5d14b81e63 Mon Sep 17 00:00:00 2001 From: Chisanan232 Date: Thu, 6 Aug 2026 14:59:03 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=E2=9C=85=20(tests):=20Prove=20a=20denied?= =?UTF-8?q?=20quick-start=20tool=20writes=20no=20file=20to=20disk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pairs a positive control (allow -> the file exists with the written content) with the negative control (deny -> nothing on disk) and a falsification case running the same write ungoverned. The side-effect assertion runs before the error assertion so removing the deny fails the suite on the absence check, not on "no error was thrown". Refs AAASM-5529, Epic AAASM-5526 --- tests/quickstart-negative-control.test.ts | 116 ++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 tests/quickstart-negative-control.test.ts diff --git a/tests/quickstart-negative-control.test.ts b/tests/quickstart-negative-control.test.ts new file mode 100644 index 000000000..1241df92e --- /dev/null +++ b/tests/quickstart-negative-control.test.ts @@ -0,0 +1,116 @@ +/** + * Enforcement-truth negative controls for the documented Node quick-start + * (AAASM-5529, Epic AAASM-5526). + * + * `docs/02-quick-start/index.md` §3 tells a reader that `withAssembly` wraps a + * tool map so "an allowed call executes normally, while a denied call throws a + * `PolicyViolationError` and the tool body never runs". Every existing test of + * that claim asserts it with a `vi.fn()` spy. A spy proves the SDK did not call + * a function it holds a reference to; it does not prove that the *effect the + * tool exists to produce* was prevented. This suite closes that gap: each tool + * here performs a real, externally-observable effect (a file on disk, an HTTP + * request delivered to a live loopback listener), and each deny is asserted as + * the absence of that effect. + * + * Every negative control is paired with a positive control over the same tool + * and the same fixture. Without the pair, "the file is absent" is satisfied + * equally well by enforcement working and by the tool being incapable of + * writing anything — which is precisely the class of vacuous evidence this + * Epic exists to eliminate. + * + * The `FALSIFICATION` cases run the identical tool with governance removed + * (calling the pre-wrap function directly). They must observe the side effect. + * If they ever stop observing it, every deny assertion in this file has become + * vacuous and the suite is no longer measuring enforcement. + */ + +import { afterEach, describe, expect, it } from "vitest"; +import { PolicyViolationError } from "../src/errors/policy-violation-error.js"; +import { withAssembly } from "../src/wrappers/with-assembly.js"; +import { + createFileSideEffect, + createPolicyGatewayClient, + type FileSideEffect +} from "./helpers/negative-control.js"; + +const AGENT_ID = "quickstart-negative-control-agent"; + +const cleanups: (() => void | Promise)[] = []; + +afterEach(async () => { + for (const cleanup of cleanups.splice(0)) { + await cleanup(); + } +}); + +function fileEffect(): FileSideEffect { + const effect = createFileSideEffect(); + cleanups.push(effect.cleanup); + return effect; +} + +/** + * Settle a governed call without letting its outcome abort the test. + * + * The side-effect assertion is the load-bearing one, so it must be reached and + * evaluated even when the call unexpectedly *succeeds*. Asserting `rejects` + * first would short-circuit there and leave the side-effect assertion + * unexercised — the falsification run would then only ever prove "no error was + * thrown", which is the weak evidence this suite exists to replace. + */ +async function settle(call: Promise): Promise { + return call.then( + (value) => value, + (error: unknown) => error + ); +} + +describe("quick-start negative control: filesystem side effect", () => { + it("POSITIVE CONTROL: an allowed write_file really creates the file on disk", async () => { + const effect = fileEffect(); + const gateway = createPolicyGatewayClient({ agentId: AGENT_ID, denyTools: [] }); + const tools = { + write_file: { execute: async (content: string) => effect.write(content) } + }; + + withAssembly(tools, { gatewayClient: gateway, agentId: AGENT_ID }); + await tools.write_file.execute("allowed-content"); + + expect(effect.occurred()).toBe(true); + expect(effect.content()).toBe("allowed-content"); + }); + + it("NEGATIVE CONTROL: a denied write_file leaves no file on disk", async () => { + const effect = fileEffect(); + const gateway = createPolicyGatewayClient({ + agentId: AGENT_ID, + denyTools: ["write_file"] + }); + const tools = { + write_file: { execute: async (content: string) => effect.write(content) } + }; + + withAssembly(tools, { gatewayClient: gateway, agentId: AGENT_ID }); + + const outcome = await settle(tools.write_file.execute("denied-content")); + + // The load-bearing assertion, asserted first: not "an error was raised", but + // "the effect the tool exists to produce is absent from the filesystem". + expect(effect.occurred()).toBe(false); + expect(effect.content()).toBeUndefined(); + // Secondary: the client also receives the documented error. + expect(outcome).toBeInstanceOf(PolicyViolationError); + expect((outcome as Error).message).toContain("Tool 'write_file' blocked"); + }); + + it("FALSIFICATION: the same write, ungoverned, does create the file", async () => { + const effect = fileEffect(); + + // No withAssembly, no gateway — enforcement removed. If this does not write, + // the negative control above is vacuous. + await effect.write("ungoverned-content"); + + expect(effect.occurred()).toBe(true); + expect(effect.content()).toBe("ungoverned-content"); + }); +}); From 033e7fdfa02695a8b6edacc33492af2c1e91ee7b Mon Sep 17 00:00:00 2001 From: Chisanan232 Date: Thu, 6 Aug 2026 14:59:31 +0800 Subject: [PATCH 3/9] =?UTF-8?q?=E2=9C=85=20(tests):=20Prove=20a=20denied?= =?UTF-8?q?=20quick-start=20tool=20sends=20no=20HTTP=20request?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A real loopback listener records every request it receives, so the deny is asserted as zero deliveries rather than as a raised exception. The positive control on the same fixture establishes the listener was reachable, which is what makes the empty request log evidence of prevention. Refs AAASM-5529, Epic AAASM-5526 --- tests/quickstart-negative-control.test.ts | 58 ++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/tests/quickstart-negative-control.test.ts b/tests/quickstart-negative-control.test.ts index 1241df92e..8421a9a53 100644 --- a/tests/quickstart-negative-control.test.ts +++ b/tests/quickstart-negative-control.test.ts @@ -29,8 +29,10 @@ import { PolicyViolationError } from "../src/errors/policy-violation-error.js"; import { withAssembly } from "../src/wrappers/with-assembly.js"; import { createFileSideEffect, + createNetworkSideEffect, createPolicyGatewayClient, - type FileSideEffect + type FileSideEffect, + type NetworkSideEffect } from "./helpers/negative-control.js"; const AGENT_ID = "quickstart-negative-control-agent"; @@ -49,6 +51,12 @@ function fileEffect(): FileSideEffect { return effect; } +async function networkEffect(): Promise { + const effect = await createNetworkSideEffect(); + cleanups.push(effect.close); + return effect; +} + /** * Settle a governed call without letting its outcome abort the test. * @@ -114,3 +122,51 @@ describe("quick-start negative control: filesystem side effect", () => { expect(effect.content()).toBe("ungoverned-content"); }); }); + +describe("quick-start negative control: network side effect", () => { + it("POSITIVE CONTROL: an allowed egress tool reaches the listener", async () => { + const effect = await networkEffect(); + const gateway = createPolicyGatewayClient({ agentId: AGENT_ID, denyTools: [] }); + const tools = { + post_report: { execute: async (body: string) => effect.call(body) } + }; + + withAssembly(tools, { gatewayClient: gateway, agentId: AGENT_ID }); + const status = await tools.post_report.execute("allowed-payload"); + + expect(status).toBe(204); + expect(effect.requests()).toHaveLength(1); + expect(effect.requests()[0]?.body).toBe("allowed-payload"); + }); + + it("NEGATIVE CONTROL: a denied egress tool never reaches the listener", async () => { + const effect = await networkEffect(); + const gateway = createPolicyGatewayClient({ + agentId: AGENT_ID, + denyTools: ["post_report"] + }); + const tools = { + post_report: { execute: async (body: string) => effect.call(body) } + }; + + withAssembly(tools, { gatewayClient: gateway, agentId: AGENT_ID }); + + const outcome = await settle(tools.post_report.execute("denied-payload")); + + // The listener is live and was reachable throughout (the positive control + // above proves that on the same fixture), so zero received requests is + // evidence the egress did not happen — not that it could not have. + expect(effect.occurred()).toBe(false); + expect(effect.requests()).toHaveLength(0); + expect(outcome).toBeInstanceOf(PolicyViolationError); + }); + + it("FALSIFICATION: the same egress, ungoverned, does reach the listener", async () => { + const effect = await networkEffect(); + + await effect.call("ungoverned-payload"); + + expect(effect.occurred()).toBe(true); + expect(effect.requests()[0]?.body).toBe("ungoverned-payload"); + }); +}); From 94e791e5f546ff1223d07ecbfed8a86197cc0193 Mon Sep 17 00:00:00 2001 From: Chisanan232 Date: Thu, 6 Aug 2026 14:59:48 +0800 Subject: [PATCH 4/9] =?UTF-8?q?=E2=9C=85=20(tests):=20Assert=20a=20deny=20?= =?UTF-8?q?carries=20agent,=20tool=20and=20run=20identity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AAASM-5529 requires deny evidence to be attributable: the fixture gateway records the identity triple it decided against, and the control checks the recorded agent id, tool name, action and run id alongside the absent side effect. An anonymous refusal is not usable audit evidence. Refs AAASM-5529, Epic AAASM-5526 --- tests/quickstart-negative-control.test.ts | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/quickstart-negative-control.test.ts b/tests/quickstart-negative-control.test.ts index 8421a9a53..f6824499b 100644 --- a/tests/quickstart-negative-control.test.ts +++ b/tests/quickstart-negative-control.test.ts @@ -170,3 +170,32 @@ describe("quick-start negative control: network side effect", () => { expect(effect.requests()[0]?.body).toBe("ungoverned-payload"); }); }); + +describe("quick-start negative control: deny is attributable in audit evidence", () => { + it("records the same agent id, tool name and run id the deny was decided against", async () => { + const effect = fileEffect(); + const gateway = createPolicyGatewayClient({ + agentId: AGENT_ID, + denyTools: ["write_file"] + }); + const tools = { + write_file: { execute: async (content: string) => effect.write(content) } + }; + + withAssembly(tools, { gatewayClient: gateway, agentId: AGENT_ID }); + + const outcome = await settle(tools.write_file.execute("denied-content")); + expect(outcome).toBeInstanceOf(PolicyViolationError); + + expect(gateway.decisions).toHaveLength(1); + const decision = gateway.decisions[0]; + expect(decision?.denied).toBe(true); + expect(decision?.agentId).toBe(AGENT_ID); + expect(decision?.toolName).toBe("write_file"); + expect(decision?.action).toBe("tool_call"); + // A run id must be present so the deny can be correlated with the rest of + // the trace; an anonymous deny is not usable evidence. + expect(decision?.runId).toMatch(/^run_/); + expect(effect.occurred()).toBe(false); + }); +}); From 83005acd92f6a7797704d98e7378af0a5555a59d Mon Sep 17 00:00:00 2001 From: Chisanan232 Date: Thu, 6 Aug 2026 15:00:04 +0800 Subject: [PATCH 5/9] =?UTF-8?q?=E2=9C=85=20(tests):=20Prove=20an=20unwrapp?= =?UTF-8?q?able=20tool=20cannot=20present=20as=20protected?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A tool exposing neither execute nor invoke has no seam for withAssembly to wrap (AAASM-4847). The control checks both halves of that: the SDK warns on stderr, and the tool's side effect really does occur under a deny policy with no decision ever recorded — so the warning is load-bearing, not cosmetic. Refs AAASM-5529, Epic AAASM-5526 --- tests/quickstart-negative-control.test.ts | 34 +++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/quickstart-negative-control.test.ts b/tests/quickstart-negative-control.test.ts index f6824499b..1b34cf52f 100644 --- a/tests/quickstart-negative-control.test.ts +++ b/tests/quickstart-negative-control.test.ts @@ -199,3 +199,37 @@ describe("quick-start negative control: deny is attributable in audit evidence", expect(effect.occurred()).toBe(false); }); }); + +describe("quick-start negative control: an ungoverned seam cannot look protected", () => { + it("a tool with no execute/invoke seam is warned about and still performs its effect", async () => { + const effect = fileEffect(); + const gateway = createPolicyGatewayClient({ + agentId: AGENT_ID, + denyTools: ["run_now"] + }); + // No `execute` / `invoke`: withAssembly has nothing to wrap (AAASM-4847). + const tools = { run_now: { call: async () => effect.write("unwrappable") } }; + + const warnings: string[] = []; + const originalWrite = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { + warnings.push(String(chunk)); + return true; + }) as typeof process.stderr.write; + try { + withAssembly(tools, { gatewayClient: gateway, agentId: AGENT_ID }); + } finally { + process.stderr.write = originalWrite; + } + + // The SDK must say so out loud rather than let the caller believe the tool + // is governed (AAASM-5526: a degraded path may not present as protected). + expect(warnings.join("")).toContain("will NOT be governed"); + + // And the negative control proves the warning is not cosmetic: the effect + // really does happen despite the deny policy, because nothing intercepts it. + await tools.run_now.call(); + expect(effect.occurred()).toBe(true); + expect(gateway.decisions).toHaveLength(0); + }); +}); From 36a467c4ab30101587f92c33d500bc6d7630398e Mon Sep 17 00:00:00 2001 From: Chisanan232 Date: Thu, 6 Aug 2026 15:19:19 +0800 Subject: [PATCH 6/9] =?UTF-8?q?=E2=9C=85=20(tests):=20Pin=20the=20zero-con?= =?UTF-8?q?fig=20initAssembly=20enforcement=20boundary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README quickstart config (no mode, no enforcementMode, langchain.tools) refuses to init rather than registering under an allow-all check — assert that, plus the observe opt-out really passing the tool body through. Without the second, the refusal is indistinguishable from "this path never works", and a reader cannot tell an advisory posture from an enforcing one. Refs AAASM-5529, Epic AAASM-5526, AAASM-4991 --- tests/quickstart-negative-control.test.ts | 59 +++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/quickstart-negative-control.test.ts b/tests/quickstart-negative-control.test.ts index 1b34cf52f..f0c4bd66d 100644 --- a/tests/quickstart-negative-control.test.ts +++ b/tests/quickstart-negative-control.test.ts @@ -25,7 +25,9 @@ */ import { afterEach, describe, expect, it } from "vitest"; +import { ConfigurationError } from "../src/errors/configuration-error.js"; import { PolicyViolationError } from "../src/errors/policy-violation-error.js"; +import { initAssembly } from "../src/core/init-assembly.js"; import { withAssembly } from "../src/wrappers/with-assembly.js"; import { createFileSideEffect, @@ -233,3 +235,60 @@ describe("quick-start negative control: an ungoverned seam cannot look protected expect(gateway.decisions).toHaveLength(0); }); }); + +describe("quick-start negative control: the zero-config initAssembly path", () => { + it("refuses to init when wrapped tools would route through the allow-all no-op client", async () => { + // The README quickstart config verbatim: gatewayUrl + agentId + langchain + // tools, no `mode`, no `enforcementMode`. An omitted posture resolves + // fail-closed, and mode "auto" is not check-capable, so the AAASM-4735 + // guard refuses rather than register under a check that cannot deny. + // Asserting the refusal is the negative control for this path: there is no + // configuration here under which a deny could be silently allowed, because + // there is no working configuration at all. + const effect = fileEffect(); + const tool = { + name: "write_file", + invoke: async () => effect.write("should-never-run") + }; + + const outcome = await settle( + initAssembly({ + gatewayUrl: "http://localhost:7391", + agentId: AGENT_ID, + langchain: { tools: { write_file: tool } } + }) + ); + + expect(effect.occurred()).toBe(false); + expect(outcome).toBeInstanceOf(ConfigurationError); + expect((outcome as Error).message).toContain("allow-all no-op"); + }); + + it("BOUNDARY: enforcementMode observe inits and lets the tool body run", async () => { + // The documented opt-out. It must really pass through, otherwise the + // refusal above would be indistinguishable from "this path never works" — + // and a reader would have no way to tell an advisory posture from an + // enforcing one. This posture is telemetry-only: tool checks route through + // the allow-all no-op client, so no policy decision can block a call here. + const effect = fileEffect(); + const tool = { + name: "write_file", + invoke: async () => effect.write("observe-posture") + }; + + const context = await initAssembly({ + gatewayUrl: "http://localhost:7391", + agentId: AGENT_ID, + enforcementMode: "observe", + langchain: { tools: { write_file: tool } } + }); + try { + await tool.invoke(); + } finally { + await context.shutdown(); + } + + expect(effect.occurred()).toBe(true); + expect(effect.content()).toBe("observe-posture"); + }); +}); From 3b08bdfd6d710985ab96ba235cbde1e6e5a18a4c Mon Sep 17 00:00:00 2001 From: Chisanan232 Date: Thu, 6 Aug 2026 19:49:53 +0800 Subject: [PATCH 7/9] =?UTF-8?q?=E2=9C=85=20(test):=20Assert=20the=20absent?= =?UTF-8?q?=20side=20effect=20before=20the=20policy=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit-evidence control asserted `toBeInstanceOf(PolicyViolationError)` before its `effect.occurred()` check. A failed assertion aborts the test, so under a mutation that neuters the deny this control failed on the missing exception and its load-bearing absence assertion was never exercised — the control had never been shown to bite. The two sibling controls in this file already asserted absence first; this one was missed. Refs AAASM-5529, Epic AAASM-5526 --- tests/quickstart-negative-control.test.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/quickstart-negative-control.test.ts b/tests/quickstart-negative-control.test.ts index f0c4bd66d..19dac9010 100644 --- a/tests/quickstart-negative-control.test.ts +++ b/tests/quickstart-negative-control.test.ts @@ -187,6 +187,14 @@ describe("quick-start negative control: deny is attributable in audit evidence", withAssembly(tools, { gatewayClient: gateway, agentId: AGENT_ID }); const outcome = await settle(tools.write_file.execute("denied-content")); + + // The load-bearing assertion, asserted first — the same shape the other + // negative controls in this file already use. Asserting the error first + // aborts the test before this line is ever reached, so under a mutation + // that neuters the deny this control failed on the missing exception and + // its absence check was never exercised at all. + expect(effect.occurred()).toBe(false); + // Secondary: the client also receives the documented error. expect(outcome).toBeInstanceOf(PolicyViolationError); expect(gateway.decisions).toHaveLength(1); @@ -198,7 +206,6 @@ describe("quick-start negative control: deny is attributable in audit evidence", // A run id must be present so the deny can be correlated with the rest of // the trace; an anonymous deny is not usable evidence. expect(decision?.runId).toMatch(/^run_/); - expect(effect.occurred()).toBe(false); }); }); From 7493c55508de08d7164563f2248854cd6a0033e3 Mon Sep 17 00:00:00 2001 From: Chisanan232 Date: Thu, 6 Aug 2026 19:53:43 +0800 Subject: [PATCH 8/9] =?UTF-8?q?=E2=9C=85=20(test):=20Replace=20the=20tauto?= =?UTF-8?q?logical=20agent-identity=20assertion=20with=20the=20real=20gap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `expect(decision.agentId).toBe(AGENT_ID)` could not fail. The fixture set that field from its own constructor argument, so it compared the test's constant with itself — a probe passing withAssembly `agentId: "TOTALLY-DIFFERENT-AGENT"` still observed `"FIXTURE-AGENT"`. The SDK supplies no agent identity at all on the check path: `WithAssemblyOptions.agentId` is declared and never read (the only `options.` reads in with-assembly.ts are gatewayClient, approvalTimeoutMs and opControl), and `GatewayCheckRequest` has no field to carry one. The outbound request keys are exactly action/args/runId/toolName. Drop the fixture's `agentId` option so it can no longer echo back a value the SDK never sent, record the verbatim outbound requests instead, and pin today's real behaviour over them. The pin fails if an agent identity is ever added, and says in its failure message that it must then be rewritten rather than deleted. Refs AAASM-5529, Epic AAASM-5526 --- tests/helpers/negative-control.ts | 37 ++++++++++----- tests/quickstart-negative-control.test.ts | 56 ++++++++++++++--------- 2 files changed, 61 insertions(+), 32 deletions(-) diff --git a/tests/helpers/negative-control.ts b/tests/helpers/negative-control.ts index 35e9e683e..38867ac40 100644 --- a/tests/helpers/negative-control.ts +++ b/tests/helpers/negative-control.ts @@ -122,13 +122,18 @@ export async function createNetworkSideEffect(): Promise { } /** - * One governance decision as the fixture gateway recorded it, carrying the full - * identity triple (`agentId` / `toolName` / `runId`) the SDK presented at check - * time. Asserting over this is how a control shows the deny was attributed to - * the right agent and tool rather than being an anonymous refusal. + * One governance decision as the fixture gateway recorded it, carrying only + * values the SDK itself supplied in its {@link GatewayCheckRequest}. + * + * Nothing here is echoed back from the fixture's own construction. A field the + * fixture populated from its own constructor argument would compare equal no + * matter what the SDK sent, so an assertion over it could never fail — the + * exact shape of vacuous evidence this Epic (AAASM-5526) exists to eliminate. + * That is why there is no `agentId`: the SDK does not send one on the check + * path, so the fixture cannot observe one. See the pinning test in + * `quickstart-negative-control.test.ts`. */ export interface RecordedCheck { - readonly agentId: string; readonly toolName: string | undefined; readonly action: string; readonly runId: string; @@ -138,22 +143,31 @@ export interface RecordedCheck { /** * Policy-driven {@link GatewayClient} standing in for the quick-start's * `createPolicyGatewayClient()` (docs/02-quick-start, `withAssembly(..., { - * gatewayClient })`). It denies exactly the named tools and records every - * decision plus every audit event, so a test can assert that the audit evidence - * carries the same agent/tool identity the decision was made against — the - * AAASM-5529 acceptance criterion that a deny is attributable, not anonymous. + * gatewayClient })`). It denies exactly the named tools and records the + * verbatim outbound requests, the resulting decisions, and every audit event, + * so a test can assert what a deny was actually attributed to. + * + * It deliberately accepts no `agentId`: the SDK puts no agent identity on the + * check path, so a fixture that took one could only hand it straight back. */ export interface PolicyGatewayClient extends GatewayClient { readonly decisions: readonly RecordedCheck[]; + /** + * Every {@link GatewayCheckRequest} the SDK passed to `check`, verbatim and + * unmodified. Asserting over this — rather than over anything the fixture + * derived — is the only way a control can state what identity the SDK does, + * and does not, attribute a policy check to. + */ + readonly checkRequests: readonly GatewayCheckRequest[]; readonly auditEvents: readonly GatewayRecordEvent[]; readonly auditResults: readonly GatewayResultRecord[]; } export function createPolicyGatewayClient(options: { - agentId: string; denyTools: readonly string[]; }): PolicyGatewayClient { const decisions: RecordedCheck[] = []; + const checkRequests: GatewayCheckRequest[] = []; const auditEvents: GatewayRecordEvent[] = []; const auditResults: GatewayResultRecord[] = []; const denied = new Set(options.denyTools); @@ -161,14 +175,15 @@ export function createPolicyGatewayClient(options: { return { mode: "sdk-only", decisions, + checkRequests, auditEvents, auditResults, start: async () => undefined, close: async () => undefined, check: async (request: GatewayCheckRequest): Promise => { const isDenied = request.toolName !== undefined && denied.has(request.toolName); + checkRequests.push(request); decisions.push({ - agentId: options.agentId, toolName: request.toolName, action: request.action, runId: request.runId, diff --git a/tests/quickstart-negative-control.test.ts b/tests/quickstart-negative-control.test.ts index 19dac9010..6b257de85 100644 --- a/tests/quickstart-negative-control.test.ts +++ b/tests/quickstart-negative-control.test.ts @@ -78,7 +78,7 @@ async function settle(call: Promise): Promise { describe("quick-start negative control: filesystem side effect", () => { it("POSITIVE CONTROL: an allowed write_file really creates the file on disk", async () => { const effect = fileEffect(); - const gateway = createPolicyGatewayClient({ agentId: AGENT_ID, denyTools: [] }); + const gateway = createPolicyGatewayClient({ denyTools: [] }); const tools = { write_file: { execute: async (content: string) => effect.write(content) } }; @@ -92,10 +92,7 @@ describe("quick-start negative control: filesystem side effect", () => { it("NEGATIVE CONTROL: a denied write_file leaves no file on disk", async () => { const effect = fileEffect(); - const gateway = createPolicyGatewayClient({ - agentId: AGENT_ID, - denyTools: ["write_file"] - }); + const gateway = createPolicyGatewayClient({ denyTools: ["write_file"] }); const tools = { write_file: { execute: async (content: string) => effect.write(content) } }; @@ -128,7 +125,7 @@ describe("quick-start negative control: filesystem side effect", () => { describe("quick-start negative control: network side effect", () => { it("POSITIVE CONTROL: an allowed egress tool reaches the listener", async () => { const effect = await networkEffect(); - const gateway = createPolicyGatewayClient({ agentId: AGENT_ID, denyTools: [] }); + const gateway = createPolicyGatewayClient({ denyTools: [] }); const tools = { post_report: { execute: async (body: string) => effect.call(body) } }; @@ -143,10 +140,7 @@ describe("quick-start negative control: network side effect", () => { it("NEGATIVE CONTROL: a denied egress tool never reaches the listener", async () => { const effect = await networkEffect(); - const gateway = createPolicyGatewayClient({ - agentId: AGENT_ID, - denyTools: ["post_report"] - }); + const gateway = createPolicyGatewayClient({ denyTools: ["post_report"] }); const tools = { post_report: { execute: async (body: string) => effect.call(body) } }; @@ -173,13 +167,10 @@ describe("quick-start negative control: network side effect", () => { }); }); -describe("quick-start negative control: deny is attributable in audit evidence", () => { - it("records the same agent id, tool name and run id the deny was decided against", async () => { +describe("quick-start negative control: what a deny is and is not attributed to", () => { + it("records the tool name and run id the deny was decided against, and no agent id", async () => { const effect = fileEffect(); - const gateway = createPolicyGatewayClient({ - agentId: AGENT_ID, - denyTools: ["write_file"] - }); + const gateway = createPolicyGatewayClient({ denyTools: ["write_file"] }); const tools = { write_file: { execute: async (content: string) => effect.write(content) } }; @@ -200,22 +191,45 @@ describe("quick-start negative control: deny is attributable in audit evidence", expect(gateway.decisions).toHaveLength(1); const decision = gateway.decisions[0]; expect(decision?.denied).toBe(true); - expect(decision?.agentId).toBe(AGENT_ID); expect(decision?.toolName).toBe("write_file"); expect(decision?.action).toBe("tool_call"); // A run id must be present so the deny can be correlated with the rest of // the trace; an anonymous deny is not usable evidence. expect(decision?.runId).toMatch(/^run_/); + + // And what the deny is NOT attributed to: an agent. + // + // This test used to assert `decision.agentId === AGENT_ID`, which could + // never fail. The fixture populated that field from its own constructor + // argument, so the assertion compared the constant this test passed with + // itself; it stayed green even when `withAssembly` was handed a completely + // different agent id. The SDK never supplied it and still does not: + // `WithAssemblyOptions` declares `agentId` (src/wrappers/with-assembly.ts) + // and no code path reads it, and `GatewayCheckRequest` + // (src/types/gateway-governance.ts) has no field to carry it. The + // documented quick-start passes an `agentId` + // (docs/02-quick-start/index.md) and the SDK discards it. + // + // Pinning the real shape of the outbound request is worth more than + // claiming an attribution the SDK does not make: this assertion fails the + // moment the gap is closed, which is exactly when someone needs to know. + expect(gateway.checkRequests).toHaveLength(1); + const requestKeys = Object.keys(gateway.checkRequests[0] ?? {}).sort(); + expect( + requestKeys, + "The outbound GatewayCheckRequest shape changed. If an agent identity is " + + "now sent on the tool-call check path then this gap is CLOSED, and this " + + "test must be REWRITTEN (not deleted) to assert the deny carries the " + + "correct agent id — passing withAssembly a different agentId than the " + + "fixture expects, so that the new assertion is able to fail." + ).toEqual(["action", "args", "runId", "toolName"]); }); }); describe("quick-start negative control: an ungoverned seam cannot look protected", () => { it("a tool with no execute/invoke seam is warned about and still performs its effect", async () => { const effect = fileEffect(); - const gateway = createPolicyGatewayClient({ - agentId: AGENT_ID, - denyTools: ["run_now"] - }); + const gateway = createPolicyGatewayClient({ denyTools: ["run_now"] }); // No `execute` / `invoke`: withAssembly has nothing to wrap (AAASM-4847). const tools = { run_now: { call: async () => effect.write("unwrappable") } }; From a1e9bc098f8cb752977fd778a43f0de324be9d81 Mon Sep 17 00:00:00 2001 From: Chisanan232 Date: Thu, 6 Aug 2026 19:54:34 +0800 Subject: [PATCH 9/9] =?UTF-8?q?=F0=9F=90=9B=20(test):=20Settle=20every=20c?= =?UTF-8?q?leanup=20so=20one=20failure=20cannot=20leak=20the=20rest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The afterEach drained the queue and awaited each entry in a bare loop, so the first throwing cleanup aborted the iteration and skipped every remaining one. The leaked temp dirs are merely untidy, but the leaked loopback HTTP listeners keep open handles that hang the vitest worker — turning one cleanup failure into a stalled run. Settle them all, then rethrow the first error. Refs AAASM-5529, Epic AAASM-5526 --- tests/quickstart-negative-control.test.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/quickstart-negative-control.test.ts b/tests/quickstart-negative-control.test.ts index 6b257de85..71a5ff616 100644 --- a/tests/quickstart-negative-control.test.ts +++ b/tests/quickstart-negative-control.test.ts @@ -42,8 +42,21 @@ const AGENT_ID = "quickstart-negative-control-agent"; const cleanups: (() => void | Promise)[] = []; afterEach(async () => { - for (const cleanup of cleanups.splice(0)) { - await cleanup(); + // Drain the queue first so nothing carries into the next test, then settle + // *every* cleanup before rethrowing. Awaiting them in a bare loop meant one + // throwing cleanup skipped all the rest, leaking the temp dirs and — worse — + // the still-listening HTTP servers, whose open handles hang the vitest worker. + const pending = cleanups.splice(0); + const failures: unknown[] = []; + for (const cleanup of pending) { + try { + await cleanup(); + } catch (error) { + failures.push(error); + } + } + if (failures.length > 0) { + throw failures[0]; } });