diff --git a/tests/helpers/negative-control.ts b/tests/helpers/negative-control.ts new file mode 100644 index 000000000..38867ac40 --- /dev/null +++ b/tests/helpers/negative-control.ts @@ -0,0 +1,205 @@ +/** + * Reusable enforcement-truth negative-control fixture (AAASM-5529). + * + * A test that only asserts "a `PolicyViolationError` was thrown" proves the SDK + * printed a refusal, not that the refusal *prevented* anything: a tool whose + * body never had an observable effect in the first place would produce the same + * green result. These helpers give a denied tool a real, externally-observable + * side effect — a file written to disk, an HTTP request delivered to a live + * listener — so a deny can be asserted as an *absence of the effect*, and the + * matching allow can be asserted as its *presence*. + * + * Every control built on this fixture must be used as a pair: + * + * - **positive control** — policy allows, the side effect is observed. Without + * it, "no file on disk" is indistinguishable from "the tool was never called + * at all", and the negative control proves nothing. + * - **negative control** — policy denies, the same side effect is absent. + * + * The side effects are deliberately real (`node:fs`, `node:http`) rather than + * spies: a spy records an intent to act, and the whole point of this Epic + * (AAASM-5526) is that intent-level evidence is what over-claimed enforcement + * looks like. + */ + +import { createServer, type IncomingMessage, type Server } from "node:http"; +import { mkdtempSync, readFileSync, rmSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { writeFile } from "node:fs/promises"; +import { AddressInfo } from "node:net"; +import type { GatewayClient } from "../../src/gateway/client.js"; +import type { + GatewayCheckRequest, + GatewayDecision, + GatewayRecordEvent, + GatewayResultRecord +} from "../../src/types/gateway-governance.js"; + +/** + * A filesystem-backed side effect: `write` really creates a file, `occurred` + * really stats it. Nothing is mocked, so an assertion over `occurred()` is an + * assertion over the world, not over the SDK's own bookkeeping. + */ +export interface FileSideEffect { + /** Absolute path the governed tool would create. */ + readonly path: string; + /** Perform the side effect (what a denied tool must never reach). */ + write: (content: string) => Promise; + /** Whether the side effect is observable on disk right now. */ + occurred: () => boolean; + /** Content actually written, or `undefined` when the effect never occurred. */ + content: () => string | undefined; + cleanup: () => void; +} + +export function createFileSideEffect(name = "denied-write.txt"): FileSideEffect { + const dir = mkdtempSync(join(tmpdir(), "aaasm-5529-")); + const path = join(dir, name); + return { + path, + write: async (content: string) => { + await writeFile(path, content, "utf8"); + return path; + }, + occurred: () => existsSync(path), + content: () => (existsSync(path) ? readFileSync(path, "utf8") : undefined), + cleanup: () => rmSync(dir, { recursive: true, force: true }) + }; +} + +/** + * A network-backed side effect: a real loopback HTTP server that records every + * request it receives. A denied tool must leave `requests()` empty — the + * strongest available in-process evidence that the egress the tool would have + * performed never left the process. + */ +export interface NetworkSideEffect { + /** URL the governed tool would call. */ + readonly url: string; + /** Perform the side effect (what a denied tool must never reach). */ + call: (body: string) => Promise; + /** Requests the listener actually received, in arrival order. */ + requests: () => readonly { method: string; url: string; body: string }[]; + /** Whether any request reached the listener. */ + occurred: () => boolean; + close: () => Promise; +} + +export async function createNetworkSideEffect(): Promise { + const received: { method: string; url: string; body: string }[] = []; + const server: Server = createServer((req: IncomingMessage, res) => { + const chunks: Buffer[] = []; + req.on("data", (chunk: Buffer) => chunks.push(chunk)); + req.on("end", () => { + received.push({ + method: req.method ?? "", + url: req.url ?? "", + body: Buffer.concat(chunks).toString("utf8") + }); + res.writeHead(204); + res.end(); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + const url = `http://127.0.0.1:${port}/exfiltrate`; + + return { + url, + call: async (body: string) => { + const response = await fetch(url, { method: "POST", body }); + return response.status; + }, + requests: () => received, + occurred: () => received.length > 0, + close: async () => { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } + }; +} + +/** + * One governance decision as the fixture gateway recorded it, carrying only + * values the SDK itself supplied in its {@link GatewayCheckRequest}. + * + * Nothing here is echoed back from the fixture's own construction. A field the + * fixture populated from its own constructor argument would compare equal no + * matter what the SDK sent, so an assertion over it could never fail — the + * exact shape of vacuous evidence this Epic (AAASM-5526) exists to eliminate. + * That is why there is no `agentId`: the SDK does not send one on the check + * path, so the fixture cannot observe one. See the pinning test in + * `quickstart-negative-control.test.ts`. + */ +export interface RecordedCheck { + readonly toolName: string | undefined; + readonly action: string; + readonly runId: string; + readonly denied: boolean; +} + +/** + * Policy-driven {@link GatewayClient} standing in for the quick-start's + * `createPolicyGatewayClient()` (docs/02-quick-start, `withAssembly(..., { + * gatewayClient })`). It denies exactly the named tools and records the + * verbatim outbound requests, the resulting decisions, and every audit event, + * so a test can assert what a deny was actually attributed to. + * + * It deliberately accepts no `agentId`: the SDK puts no agent identity on the + * check path, so a fixture that took one could only hand it straight back. + */ +export interface PolicyGatewayClient extends GatewayClient { + readonly decisions: readonly RecordedCheck[]; + /** + * Every {@link GatewayCheckRequest} the SDK passed to `check`, verbatim and + * unmodified. Asserting over this — rather than over anything the fixture + * derived — is the only way a control can state what identity the SDK does, + * and does not, attribute a policy check to. + */ + readonly checkRequests: readonly GatewayCheckRequest[]; + readonly auditEvents: readonly GatewayRecordEvent[]; + readonly auditResults: readonly GatewayResultRecord[]; +} + +export function createPolicyGatewayClient(options: { + denyTools: readonly string[]; +}): PolicyGatewayClient { + const decisions: RecordedCheck[] = []; + const checkRequests: GatewayCheckRequest[] = []; + const auditEvents: GatewayRecordEvent[] = []; + const auditResults: GatewayResultRecord[] = []; + const denied = new Set(options.denyTools); + + return { + mode: "sdk-only", + decisions, + checkRequests, + auditEvents, + auditResults, + start: async () => undefined, + close: async () => undefined, + check: async (request: GatewayCheckRequest): Promise => { + const isDenied = request.toolName !== undefined && denied.has(request.toolName); + checkRequests.push(request); + decisions.push({ + toolName: request.toolName, + action: request.action, + runId: request.runId, + denied: isDenied + }); + return isDenied + ? { denied: true, pending: false, reason: `tool '${request.toolName}' is denied by policy` } + : { denied: false, pending: false }; + }, + waitForApproval: async () => ({ denied: false }), + record: async (event: GatewayRecordEvent) => { + auditEvents.push(event); + }, + recordResult: async (record: GatewayResultRecord) => { + auditResults.push(record); + }, + scanPrompts: async () => undefined + }; +} diff --git a/tests/quickstart-negative-control.test.ts b/tests/quickstart-negative-control.test.ts new file mode 100644 index 000000000..71a5ff616 --- /dev/null +++ b/tests/quickstart-negative-control.test.ts @@ -0,0 +1,328 @@ +/** + * Enforcement-truth negative controls for the documented Node quick-start + * (AAASM-5529, Epic AAASM-5526). + * + * `docs/02-quick-start/index.md` §3 tells a reader that `withAssembly` wraps a + * tool map so "an allowed call executes normally, while a denied call throws a + * `PolicyViolationError` and the tool body never runs". Every existing test of + * that claim asserts it with a `vi.fn()` spy. A spy proves the SDK did not call + * a function it holds a reference to; it does not prove that the *effect the + * tool exists to produce* was prevented. This suite closes that gap: each tool + * here performs a real, externally-observable effect (a file on disk, an HTTP + * request delivered to a live loopback listener), and each deny is asserted as + * the absence of that effect. + * + * Every negative control is paired with a positive control over the same tool + * and the same fixture. Without the pair, "the file is absent" is satisfied + * equally well by enforcement working and by the tool being incapable of + * writing anything — which is precisely the class of vacuous evidence this + * Epic exists to eliminate. + * + * The `FALSIFICATION` cases run the identical tool with governance removed + * (calling the pre-wrap function directly). They must observe the side effect. + * If they ever stop observing it, every deny assertion in this file has become + * vacuous and the suite is no longer measuring enforcement. + */ + +import { afterEach, describe, expect, it } from "vitest"; +import { ConfigurationError } from "../src/errors/configuration-error.js"; +import { PolicyViolationError } from "../src/errors/policy-violation-error.js"; +import { initAssembly } from "../src/core/init-assembly.js"; +import { withAssembly } from "../src/wrappers/with-assembly.js"; +import { + createFileSideEffect, + createNetworkSideEffect, + createPolicyGatewayClient, + type FileSideEffect, + type NetworkSideEffect +} from "./helpers/negative-control.js"; + +const AGENT_ID = "quickstart-negative-control-agent"; + +const cleanups: (() => void | Promise)[] = []; + +afterEach(async () => { + // Drain the queue first so nothing carries into the next test, then settle + // *every* cleanup before rethrowing. Awaiting them in a bare loop meant one + // throwing cleanup skipped all the rest, leaking the temp dirs and — worse — + // the still-listening HTTP servers, whose open handles hang the vitest worker. + const pending = cleanups.splice(0); + const failures: unknown[] = []; + for (const cleanup of pending) { + try { + await cleanup(); + } catch (error) { + failures.push(error); + } + } + if (failures.length > 0) { + throw failures[0]; + } +}); + +function fileEffect(): FileSideEffect { + const effect = createFileSideEffect(); + cleanups.push(effect.cleanup); + return effect; +} + +async function networkEffect(): Promise { + const effect = await createNetworkSideEffect(); + cleanups.push(effect.close); + return effect; +} + +/** + * Settle a governed call without letting its outcome abort the test. + * + * The side-effect assertion is the load-bearing one, so it must be reached and + * evaluated even when the call unexpectedly *succeeds*. Asserting `rejects` + * first would short-circuit there and leave the side-effect assertion + * unexercised — the falsification run would then only ever prove "no error was + * thrown", which is the weak evidence this suite exists to replace. + */ +async function settle(call: Promise): Promise { + return call.then( + (value) => value, + (error: unknown) => error + ); +} + +describe("quick-start negative control: filesystem side effect", () => { + it("POSITIVE CONTROL: an allowed write_file really creates the file on disk", async () => { + const effect = fileEffect(); + const gateway = createPolicyGatewayClient({ denyTools: [] }); + const tools = { + write_file: { execute: async (content: string) => effect.write(content) } + }; + + withAssembly(tools, { gatewayClient: gateway, agentId: AGENT_ID }); + await tools.write_file.execute("allowed-content"); + + expect(effect.occurred()).toBe(true); + expect(effect.content()).toBe("allowed-content"); + }); + + it("NEGATIVE CONTROL: a denied write_file leaves no file on disk", async () => { + const effect = fileEffect(); + const gateway = createPolicyGatewayClient({ denyTools: ["write_file"] }); + const tools = { + write_file: { execute: async (content: string) => effect.write(content) } + }; + + withAssembly(tools, { gatewayClient: gateway, agentId: AGENT_ID }); + + const outcome = await settle(tools.write_file.execute("denied-content")); + + // The load-bearing assertion, asserted first: not "an error was raised", but + // "the effect the tool exists to produce is absent from the filesystem". + expect(effect.occurred()).toBe(false); + expect(effect.content()).toBeUndefined(); + // Secondary: the client also receives the documented error. + expect(outcome).toBeInstanceOf(PolicyViolationError); + expect((outcome as Error).message).toContain("Tool 'write_file' blocked"); + }); + + it("FALSIFICATION: the same write, ungoverned, does create the file", async () => { + const effect = fileEffect(); + + // No withAssembly, no gateway — enforcement removed. If this does not write, + // the negative control above is vacuous. + await effect.write("ungoverned-content"); + + expect(effect.occurred()).toBe(true); + expect(effect.content()).toBe("ungoverned-content"); + }); +}); + +describe("quick-start negative control: network side effect", () => { + it("POSITIVE CONTROL: an allowed egress tool reaches the listener", async () => { + const effect = await networkEffect(); + const gateway = createPolicyGatewayClient({ denyTools: [] }); + const tools = { + post_report: { execute: async (body: string) => effect.call(body) } + }; + + withAssembly(tools, { gatewayClient: gateway, agentId: AGENT_ID }); + const status = await tools.post_report.execute("allowed-payload"); + + expect(status).toBe(204); + expect(effect.requests()).toHaveLength(1); + expect(effect.requests()[0]?.body).toBe("allowed-payload"); + }); + + it("NEGATIVE CONTROL: a denied egress tool never reaches the listener", async () => { + const effect = await networkEffect(); + const gateway = createPolicyGatewayClient({ denyTools: ["post_report"] }); + const tools = { + post_report: { execute: async (body: string) => effect.call(body) } + }; + + withAssembly(tools, { gatewayClient: gateway, agentId: AGENT_ID }); + + const outcome = await settle(tools.post_report.execute("denied-payload")); + + // The listener is live and was reachable throughout (the positive control + // above proves that on the same fixture), so zero received requests is + // evidence the egress did not happen — not that it could not have. + expect(effect.occurred()).toBe(false); + expect(effect.requests()).toHaveLength(0); + expect(outcome).toBeInstanceOf(PolicyViolationError); + }); + + it("FALSIFICATION: the same egress, ungoverned, does reach the listener", async () => { + const effect = await networkEffect(); + + await effect.call("ungoverned-payload"); + + expect(effect.occurred()).toBe(true); + expect(effect.requests()[0]?.body).toBe("ungoverned-payload"); + }); +}); + +describe("quick-start negative control: what a deny is and is not attributed to", () => { + it("records the tool name and run id the deny was decided against, and no agent id", async () => { + const effect = fileEffect(); + const gateway = createPolicyGatewayClient({ denyTools: ["write_file"] }); + const tools = { + write_file: { execute: async (content: string) => effect.write(content) } + }; + + withAssembly(tools, { gatewayClient: gateway, agentId: AGENT_ID }); + + const outcome = await settle(tools.write_file.execute("denied-content")); + + // The load-bearing assertion, asserted first — the same shape the other + // negative controls in this file already use. Asserting the error first + // aborts the test before this line is ever reached, so under a mutation + // that neuters the deny this control failed on the missing exception and + // its absence check was never exercised at all. + expect(effect.occurred()).toBe(false); + // Secondary: the client also receives the documented error. + expect(outcome).toBeInstanceOf(PolicyViolationError); + + expect(gateway.decisions).toHaveLength(1); + const decision = gateway.decisions[0]; + expect(decision?.denied).toBe(true); + expect(decision?.toolName).toBe("write_file"); + expect(decision?.action).toBe("tool_call"); + // A run id must be present so the deny can be correlated with the rest of + // the trace; an anonymous deny is not usable evidence. + expect(decision?.runId).toMatch(/^run_/); + + // And what the deny is NOT attributed to: an agent. + // + // This test used to assert `decision.agentId === AGENT_ID`, which could + // never fail. The fixture populated that field from its own constructor + // argument, so the assertion compared the constant this test passed with + // itself; it stayed green even when `withAssembly` was handed a completely + // different agent id. The SDK never supplied it and still does not: + // `WithAssemblyOptions` declares `agentId` (src/wrappers/with-assembly.ts) + // and no code path reads it, and `GatewayCheckRequest` + // (src/types/gateway-governance.ts) has no field to carry it. The + // documented quick-start passes an `agentId` + // (docs/02-quick-start/index.md) and the SDK discards it. + // + // Pinning the real shape of the outbound request is worth more than + // claiming an attribution the SDK does not make: this assertion fails the + // moment the gap is closed, which is exactly when someone needs to know. + expect(gateway.checkRequests).toHaveLength(1); + const requestKeys = Object.keys(gateway.checkRequests[0] ?? {}).sort(); + expect( + requestKeys, + "The outbound GatewayCheckRequest shape changed. If an agent identity is " + + "now sent on the tool-call check path then this gap is CLOSED, and this " + + "test must be REWRITTEN (not deleted) to assert the deny carries the " + + "correct agent id — passing withAssembly a different agentId than the " + + "fixture expects, so that the new assertion is able to fail." + ).toEqual(["action", "args", "runId", "toolName"]); + }); +}); + +describe("quick-start negative control: an ungoverned seam cannot look protected", () => { + it("a tool with no execute/invoke seam is warned about and still performs its effect", async () => { + const effect = fileEffect(); + const gateway = createPolicyGatewayClient({ denyTools: ["run_now"] }); + // No `execute` / `invoke`: withAssembly has nothing to wrap (AAASM-4847). + const tools = { run_now: { call: async () => effect.write("unwrappable") } }; + + const warnings: string[] = []; + const originalWrite = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { + warnings.push(String(chunk)); + return true; + }) as typeof process.stderr.write; + try { + withAssembly(tools, { gatewayClient: gateway, agentId: AGENT_ID }); + } finally { + process.stderr.write = originalWrite; + } + + // The SDK must say so out loud rather than let the caller believe the tool + // is governed (AAASM-5526: a degraded path may not present as protected). + expect(warnings.join("")).toContain("will NOT be governed"); + + // And the negative control proves the warning is not cosmetic: the effect + // really does happen despite the deny policy, because nothing intercepts it. + await tools.run_now.call(); + expect(effect.occurred()).toBe(true); + expect(gateway.decisions).toHaveLength(0); + }); +}); + +describe("quick-start negative control: the zero-config initAssembly path", () => { + it("refuses to init when wrapped tools would route through the allow-all no-op client", async () => { + // The README quickstart config verbatim: gatewayUrl + agentId + langchain + // tools, no `mode`, no `enforcementMode`. An omitted posture resolves + // fail-closed, and mode "auto" is not check-capable, so the AAASM-4735 + // guard refuses rather than register under a check that cannot deny. + // Asserting the refusal is the negative control for this path: there is no + // configuration here under which a deny could be silently allowed, because + // there is no working configuration at all. + const effect = fileEffect(); + const tool = { + name: "write_file", + invoke: async () => effect.write("should-never-run") + }; + + const outcome = await settle( + initAssembly({ + gatewayUrl: "http://localhost:7391", + agentId: AGENT_ID, + langchain: { tools: { write_file: tool } } + }) + ); + + expect(effect.occurred()).toBe(false); + expect(outcome).toBeInstanceOf(ConfigurationError); + expect((outcome as Error).message).toContain("allow-all no-op"); + }); + + it("BOUNDARY: enforcementMode observe inits and lets the tool body run", async () => { + // The documented opt-out. It must really pass through, otherwise the + // refusal above would be indistinguishable from "this path never works" — + // and a reader would have no way to tell an advisory posture from an + // enforcing one. This posture is telemetry-only: tool checks route through + // the allow-all no-op client, so no policy decision can block a call here. + const effect = fileEffect(); + const tool = { + name: "write_file", + invoke: async () => effect.write("observe-posture") + }; + + const context = await initAssembly({ + gatewayUrl: "http://localhost:7391", + agentId: AGENT_ID, + enforcementMode: "observe", + langchain: { tools: { write_file: tool } } + }); + try { + await tool.invoke(); + } finally { + await context.shutdown(); + } + + expect(effect.occurred()).toBe(true); + expect(effect.content()).toBe("observe-posture"); + }); +});