diff --git a/evals/delivery-presentation.ts b/evals/delivery-presentation.ts index 054c1a3b..671af5f5 100644 --- a/evals/delivery-presentation.ts +++ b/evals/delivery-presentation.ts @@ -1,3 +1,5 @@ +import type { EvidencePlatform } from "../src/domain/session.js"; +import { EVIDENCE_PLATFORMS } from "../src/domain/validation.js"; import { canonicalJson } from "./canonical-json.js"; type Closure = "completed" | "deferred" | "abandoned"; @@ -9,11 +11,15 @@ type CommandIntegrity = | "not-claimed" | "script-unchanged" | "script-and-invocation-unchanged"; +type IndependentReviewClaim = + | { kind: "passed"; findings: "none" | "not-claimed" } + | { kind: "not-performed" }; type CommandObservation = { command: string; exitCode: number | null; integrity: CommandIntegrity; qualification: "observation" | "does-not-claim-pass" | "claimed-pass" | null; + hostPlatform?: EvidencePlatform; }; function commandIntegrityValue( clause: string, @@ -55,11 +61,7 @@ type CurrentHandoffFacts = { targetPlatform: string; hostPlatform: string; }[]; - independentReview: ( - | { kind: "passed"; findings: "none" | "not-claimed" } - | { kind: "not-performed" } - | null - )[]; + independentReview: (IndependentReviewClaim | null)[]; observations: CommandObservation[]; unsupported: string[]; }; @@ -307,11 +309,45 @@ function commandStatusAssertion(text: string): boolean { text, ); } +function commandContinuation(text: string): boolean { + return ( + commandStatusAssertion(text) || + /^(?:this (?:(?:command|observation)|does not claim the command passed)|it|its|the (?:script|invocation))\b/i.test( + text, + ) + ); +} +function commandQualifierAssertion(text: string): boolean { + return ( + commandStatusAssertion(text) || + /^(?:its (?:script|invocation|command)|the (?:script|invocation))\b[^.!?;]*\b(?:changed|unchanged|modified)\b/i.test( + text, + ) + ); +} +function independentReviewValue(value: string): IndependentReviewClaim | null { + return /^(?:was )?not performed$/i.test(value) + ? { kind: "not-performed" } + : /^(?:has |was )?(?:passed|passed with no findings)$/i.test(value) + ? { + kind: "passed", + findings: /with no findings$/i.test(value) ? "none" : "not-claimed", + } + : null; +} function platformValue(value: string) { const lower = value.toLowerCase(); return lower === "macos" ? "darwin" : lower === "windows" ? "win32" : lower; } -function sentenceBoundary(text: string, start = 0) { +function claimedHostPlatform(value: string): EvidencePlatform | null { + const normalized = platformValue(value); + return EVIDENCE_PLATFORMS.find((platform) => platform === normalized) ?? null; +} +function sentenceBoundary( + text: string, + start = 0, + mode: "sentence" | "command" | "metadata" = "sentence", +) { let quote: string | null = null; for (let index = start; index < text.length; index++) { const character = text[index]; @@ -333,10 +369,41 @@ function sentenceBoundary(text: string, start = 0) { quote = character; continue; } - if (character === "." && /^\s+[A-Za-z0-9]/.test(text.slice(index + 1))) - return { end: index + 1, unterminatedQuote: false }; + if ( + mode === "metadata" && + character === "," && + /^,\s*host\b/i.test(text.slice(index)) + ) + return { end: index, next: index, unterminatedQuote: false }; + if (mode === "command" && character === ",") { + const sibling = /^,\s+and\s+(?=Independent review(?::|\s))/i.exec( + text.slice(index), + ); + if (sibling) + return { + end: index, + next: index + sibling[0].length, + unterminatedQuote: false, + }; + } + if ( + mode === "command" && + character === ";" && + commandContinuation(text.slice(index + 1).trimStart()) + ) + return { end: index + 1, next: index + 1, unterminatedQuote: false }; + if ( + mode !== "metadata" && + character === "." && + /^\s+[A-Za-z0-9]/.test(text.slice(index + 1)) + ) + return { end: index + 1, next: index + 1, unterminatedQuote: false }; } - return { end: text.length, unterminatedQuote: quote !== null }; + return { + end: text.length, + next: text.length, + unterminatedQuote: quote !== null, + }; } function commandResultValue(line: string, commands: readonly string[]) { if (/^Example:/i.test(line)) return null; @@ -366,25 +433,51 @@ function commandResultValue(line: string, commands: readonly string[]) { remainder: "", source: line, }; - let boundary = sentenceBoundary(rawBody); + let boundary = sentenceBoundary(rawBody, 0, "command"); while ( boundary.end < rawBody.length && - (commandStatusAssertion(rawBody.slice(boundary.end).trimStart()) || - /^(?:this (?:(?:command|observation)|does not claim the command passed)|it|its|the (?:script|invocation))\b/i.test( - rawBody.slice(boundary.end).trimStart(), - )) + commandContinuation(rawBody.slice(boundary.end).trimStart()) ) { - boundary = sentenceBoundary(rawBody, boundary.end); + boundary = sentenceBoundary(rawBody, boundary.end, "command"); } const remainderStart = boundary.end; - const record = rawBody.slice(0, remainderStart); - const remainder = rawBody.slice(remainderStart).trim(); + let record = rawBody.slice(0, remainderStart); + let sourceEnd = remainderStart; + let remainder = rawBody.slice(boundary.next).trim(); + let sibling: + | { value: IndependentReviewClaim | null; source: string } + | undefined; + if (/^Independent review(?::|\s)/i.test(remainder)) { + const start = boundary.next + rawBody.slice(boundary.next).search(/\S/); + let reviewBoundary = sentenceBoundary(rawBody, start, "command"); + const reviewEnd = reviewBoundary.end; + while ( + reviewBoundary.end < rawBody.length && + commandQualifierAssertion(rawBody.slice(reviewBoundary.end).trimStart()) + ) + reviewBoundary = sentenceBoundary( + rawBody, + reviewBoundary.end, + "command", + ); + const source = rawBody.slice(start, reviewEnd).trim(); + const value = source + .replace(/^Independent review(?::|\s)\s*/i, "") + .replace(/[.;]$/, ""); + sibling = { value: independentReviewValue(value), source }; + if (reviewBoundary.end > reviewEnd) { + record = `${record.replace(/[.;]$/, "")}; ${rawBody.slice(reviewEnd, reviewBoundary.end).trim()}`; + sourceEnd = reviewBoundary.end; + } + remainder = rawBody.slice(reviewBoundary.next).trim(); + } if ( /^Outstanding proof:/i.test(line) && /^\s+on (?:macOS|darwin|Linux|Windows)\.?\s*$/i.test(record) ) return { observation: null, + ...(sibling ? { sibling } : {}), remainder, source: line .slice(0, line.length - rawBody.length + remainderStart) @@ -398,10 +491,9 @@ function commandResultValue(line: string, commands: readonly string[]) { ); return { observation: parsed, + ...(sibling ? { sibling } : {}), remainder, - source: line - .slice(0, line.length - rawBody.length + remainderStart) - .trim(), + source: line.slice(0, line.length - rawBody.length + sourceEnd).trim(), }; } return null; @@ -447,7 +539,7 @@ function parseCommandResult( command: string, commands: readonly string[], unterminatedQuote: boolean, -) { +): CommandObservation { const body = rawBody.trim().replace(/^(?::\s*|[—–]\s*|-\s+)/, ""); const invalid: CommandObservation = { command, @@ -465,7 +557,7 @@ function parseCommandResult( /^(?:(passed)(?:,\s*| with )|(recorded as an observation),\s*)?(?:exited|exit(?: code)?)\s+(-?\d+|unavailable)(.*)$/i.exec( status, ); - const barePass = /^passed( unchanged)?$/i.exec(status); + const barePass = /^passed( unchanged)?(?: on ([A-Za-z0-9]+))?$/i.exec(status); if (!value && !barePass) return invalid; const rawExit = barePass ? "0" : (value?.[3] ?? ""); const exitCode = @@ -478,6 +570,27 @@ function parseCommandResult( integrity: "not-claimed", }; const metadata = value?.[4] ?? ""; + const assertedHost = barePass?.[2]; + let host: EvidencePlatform | undefined; + if (assertedHost !== undefined) { + const parsed = claimedHostPlatform(assertedHost); + if (parsed === null) return malformed; + host = parsed; + } + let metadataStart = 0; + while (metadataStart < metadata.length) { + const boundary = sentenceBoundary(metadata, metadataStart, "metadata"); + if (boundary.end === metadata.length) break; + const field = /^,\s*host\s+([^,\s]+)\s*(?=,|$)/i.exec( + metadata.slice(boundary.end), + ); + if (!field) return malformed; + const claimed = claimedHostPlatform(field[1] ?? ""); + if (claimed === null || (host !== undefined && host !== claimed)) + return malformed; + host = claimed; + metadataStart = boundary.end + field[0].length; + } if ( metadata && !/^(?:, (?:host|source|output|report) .+|, reporting \d+ [A-Za-z ]+)$/i.test( @@ -530,7 +643,13 @@ function parseCommandResult( integrity = claim; } } - return { command, exitCode, qualification, integrity }; + return { + command, + exitCode, + qualification, + integrity, + ...(host !== undefined ? { hostPlatform: host } : {}), + }; } export function currentHandoffFacts( text: string, @@ -560,6 +679,11 @@ export function currentHandoffFacts( } const commandRecord = commandResultValue(line, observationCommands); if (commandRecord) { + if ("sibling" in commandRecord && commandRecord.sibling) { + facts.independentReview.push(commandRecord.sibling.value); + if (commandRecord.sibling.value === null) + facts.unsupported.push(commandRecord.sibling.source); + } if ("unavailable" in commandRecord && commandRecord.unavailable) facts.unavailableCommands.push(commandRecord.unavailable); if (commandRecord.observation) { @@ -593,7 +717,7 @@ export function currentHandoffFacts( (/^(?:[^:]+:\s*)?(?:node|bun) \S+[^;]*\bpassed(?:,\s*| with )exit(?: code)? -?\d+\b/i.test( line, ) || - /^(?:[^:]+:\s*)?(?:node|bun) \S+[^;]*\s+passed(?: unchanged)?(?:[.;]|$)/i.test( + /^(?:[^:]+:\s*)?(?:node|bun) \S+[^;]*\s+passed(?: unchanged)?(?: on [A-Za-z0-9]+)?(?:[,.;]|$)/i.test( line, )) ) { @@ -611,18 +735,7 @@ export function currentHandoffFacts( const review = /^Independent review(?::|\s)\s*(.*)$/i.exec(claim); if (review) { const value = review[1] ?? ""; - facts.independentReview.push( - /^(?:was )?not performed$/i.test(value) - ? { kind: "not-performed" } - : /^(?:has |was )?(?:passed|passed with no findings)$/i.test(value) - ? { - kind: "passed", - findings: /with no findings$/i.test(value) - ? "none" - : "not-claimed", - } - : null, - ); + facts.independentReview.push(independentReviewValue(value)); if (facts.independentReview.at(-1) === null) facts.unsupported.push(claim); continue; diff --git a/evals/delivery-scenario-checks.ts b/evals/delivery-scenario-checks.ts index 10dd3eb7..8c53fdb3 100644 --- a/evals/delivery-scenario-checks.ts +++ b/evals/delivery-scenario-checks.ts @@ -298,6 +298,7 @@ function capturedDeferredPass( archive: z.infer, close: ScenarioGradeInput["allCalls"][number], command: string, + desiredHost?: string, ): { hostPlatform: string; status: Record } | null { if (archive.closure.kind !== "deferred") return null; const candidates = archive.runs.flatMap((run) => @@ -315,7 +316,11 @@ function capturedDeferredPass( candidate.run, parsed.data, ); - if (proof) return proof; + if ( + proof && + (desiredHost === undefined || proof.hostPlatform === desiredHost) + ) + return proof; } return null; } @@ -783,7 +788,13 @@ export function deliveryIssues( for (const claim of facts.unavailableCommands) { const proof = conclusion === "completion-not-claimed" - ? capturedDeferredPass(input, archive, accepted, expected.gate) + ? capturedDeferredPass( + input, + archive, + accepted, + expected.gate, + claim.hostPlatform, + ) : null; const declared = (archive.plan.evidence ?? []).some( (entry) => @@ -887,10 +898,13 @@ export function deliveryIssues( ); issues.push(...missingAssuranceDisclosures(input.finalText)); for (const result of facts.observations.filter( - (record) => record.qualification === "claimed-pass", + (record) => + record.qualification === "claimed-pass" || + record.hostPlatform !== undefined, )) { - const acceptedPass = - result.exitCode === 0 && + const claimsPass = result.qualification === "claimed-pass"; + const acceptedProof = + (!claimsPass || result.exitCode === 0) && archive.runs.some((run) => run.reviews.some( (review) => @@ -899,21 +913,36 @@ export function deliveryIssues( (validation) => review.validationIds.includes(validation.id) && validation.command === result.command && + (result.hostPlatform === undefined || + result.hostPlatform === validation.hostPlatform) && validation.exitCode === result.exitCode && validation.outputComplete && - validation.intent !== "observe" && + (!claimsPass || validation.intent !== "observe") && validation.sourceDigest === review.sourceDigest, ), ), ); - const capturedPass = + const captured = expected.closure === "deferred" && conclusion === "completion-not-claimed" && - result.exitCode === 0 && - capturedDeferredPass(input, archive, accepted, result.command) !== null; - if (!acceptedPass && !capturedPass) + result.exitCode === 0 + ? capturedDeferredPass( + input, + archive, + accepted, + result.command, + result.hostPlatform, + ) + : null; + const capturedPass = + captured !== null && + (result.hostPlatform === undefined || + result.hostPlatform === captured.hostPlatform); + if (!acceptedProof && !capturedPass) issues.push( - "Claimed command pass lacks matching accepted complete source evidence.", + claimsPass + ? "Claimed command pass lacks matching accepted complete source evidence." + : "Claimed command host lacks matching accepted complete source evidence.", ); } for (const result of facts.observations) { @@ -1019,6 +1048,9 @@ export function deliveryIssues( if (expected.observed) { const observation = expected.observed; + const reported = facts.observations.filter( + (row) => row.command === observation.command, + ); if ( /audit (?:check )?(?:passed|succeeded)|all (?:checks|commands) passed/i.test( text, @@ -1034,6 +1066,11 @@ export function deliveryIssues( (validation) => review.validationIds.includes(validation.id) && validation.command === observation.command && + reported.every( + (row) => + row.hostPlatform === undefined || + row.hostPlatform === validation.hostPlatform, + ) && validation.exitCode === observation.exitCode && validation.intent === "observe" && validation.outputComplete && @@ -1043,9 +1080,6 @@ export function deliveryIssues( ) ) issues.push("Nonzero observation lacks accepted reviewed evidence."); - const reported = facts.observations.filter( - (row) => row.command === observation.command, - ); if ( !reported.length || reported.some( diff --git a/tests/delivery-platform-pass.test.ts b/tests/delivery-platform-pass.test.ts new file mode 100644 index 00000000..3635879b --- /dev/null +++ b/tests/delivery-platform-pass.test.ts @@ -0,0 +1,558 @@ +import { expect, test } from "bun:test"; +import { currentHandoffFacts } from "../evals/delivery-presentation.js"; +import { deliveryIssues } from "../evals/delivery-scenario-checks.js"; +import { DELIVERY_SCENARIOS } from "../evals/delivery-scenarios.js"; +import { autoQualifiedOutcome } from "./fixtures/auto-qualified-outcome.js"; +import { + validation as capturedValidation, + deferredCaptureOutcome, + externalCommand, + localCommand, + nativeTrace, +} from "./fixtures/deferred-capture-outcome.js"; +import saved from "./fixtures/delivery-platform-pass-answer.json" with { + type: "json", +}; + +const gate = "node scripts/verify.mjs"; +const phrase = "passed on Linux, and independent review passed."; +const scenario = DELIVERY_SCENARIOS.find( + (item) => item.id === "delivery-summary-observed-failure", +); +if (!scenario) throw new Error("Missing observed-failure scenario."); +function object(value: unknown): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) + throw new Error("Missing fixture object."); + return value as Record; +} +function fixture(status = phrase) { + const input = autoQualifiedOutcome("audit", { + goal: saved.goal, + featureId: saved.featureId, + }); + const close = input.allCalls.find( + (call) => call.tool === "flow_session_close", + ); + object(object(close?.output).workflowData).delivery = structuredClone( + saved.delivery, + ); + return { ...input, finalText: saved.answer.replace(phrase, status) }; +} +function run(input: ReturnType) { + const runs = object(input.archives[0]).runs; + if (!Array.isArray(runs)) throw new Error("Missing runs."); + return object(runs[0]); +} +function validation(input: ReturnType) { + const validations = run(input).validations; + if (!Array.isArray(validations)) throw new Error("Missing validations."); + const value = validations.map(object).find((item) => item.command === gate); + if (!value) throw new Error("Missing gate validation."); + return value; +} +for (const status of [ + phrase, + "passed on Linux. Independent review passed.", + "passed, and independent review passed.", + "passed. Independent review passed.", + "passed, exit 0, host Linux. Independent review passed.", +]) { + test(`organic host and review clauses succeed ${status}`, () => { + expect(scenario.check(fixture(status))).toEqual([]); + }); +} +test("host and joined review remain separate typed facts", () => { + const facts = currentHandoffFacts(`${gate} ${phrase}`, [gate]); + const expected = [ + { + command: gate, + exitCode: 0, + qualification: "claimed-pass" as const, + integrity: "not-claimed" as const, + hostPlatform: "linux" as const, + }, + ]; + expect(facts.observations).toEqual(expected); + expect(facts.independentReview).toEqual([ + { kind: "passed", findings: "not-claimed" }, + ]); +}); +for (const host of ["darwin", "win32", "other", null, "unknown"]) { + test(`asserted Linux host rejects mismatched native witness ${host}`, () => { + const input = fixture(); + validation(input).hostPlatform = host; + expect(scenario.check(input)).toContain( + "Claimed command pass lacks matching accepted complete source evidence.", + ); + }); +} +test("unclaimed native host metadata retains existing native protocol rejection", () => { + const input = fixture("passed. Independent review passed."); + validation(input).hostPlatform = "other"; + expect(scenario.check(input)).toEqual([ + "parser-null: Accepted current-source validation and submitted review are missing.", + "Independent review claim contradicts accepted native review evidence.", + ]); +}); +test("an unrelated matching host cannot rescue the bound validation", () => { + const input = fixture(); + validation(input).hostPlatform = "darwin"; + const validations = run(input).validations; + if (!Array.isArray(validations)) throw new Error("Missing validations."); + validations.push({ + ...validation(input), + id: "unaccepted-host-witness", + hostPlatform: "linux" as const, + }); + expect(scenario.check(input)).toContain( + "Claimed command pass lacks matching accepted complete source evidence.", + ); +}); +for (const change of [ + "nonzero", + "incomplete", + "source", + "unaccepted", + "observe", +]) { + test(`platform claim cannot rescue ${change} native proof`, () => { + const input = fixture(); + const value = validation(input); + if (change === "nonzero") value.exitCode = 1; + if (change === "incomplete") value.outputComplete = false; + if (change === "source") value.sourceDigest = `sha256:${"b".repeat(64)}`; + if (change === "observe") value.intent = "observe"; + if (change === "unaccepted") { + const reviews = run(input).reviews; + if (!Array.isArray(reviews)) throw new Error("Missing reviews."); + object(reviews[0]).validationIds = []; + } + expect(scenario.check(input)).toContain( + "Claimed command pass lacks matching accepted complete source evidence.", + ); + }); +} +for (const tail of [ + "This command failed.", + "This observation does not claim a pass.", + "Its script changed.", +]) { + test(`joined sibling preserves conflicting tail ${tail}`, () => { + expect(scenario.check(fixture(`${phrase} ${tail}`))).toContain( + "Unsupported or conflicting current handoff assertions.", + ); + }); +} +for (const change of ["native-host", "reported-host"]) { + test(`numeric audit host binds to its own reviewed witness ${change}`, () => { + const input = fixture("passed. Independent review passed."); + if (change === "native-host") { + const values = run(input).validations; + if (!Array.isArray(values)) throw new Error("Missing validations."); + const audit = values + .map(object) + .find((value) => value.command === "node scripts/audit.mjs"); + if (!audit) throw new Error("Missing audit validation."); + audit.hostPlatform = "win32"; + } else + input.finalText = input.finalText.replace( + "exit 12, host linux", + "exit 12, host Windows", + ); + expect(scenario.check(input)).toContain( + "Nonzero observation lacks accepted reviewed evidence.", + ); + }); +} +for (const tail of ["; its script changed.", "; this command exited 1."]) { + test(`semicolon sibling tail remains checked ${tail}`, () => { + expect(scenario.check(fixture(phrase.replace(/\.$/, tail)))).toContain( + "Unsupported or conflicting current handoff assertions.", + ); + }); +} +for (const status of [ + "passed on Mars. Independent review passed.", + "passed, exit 0, host Mars. Independent review passed.", + "passed, exit 0, host Windows. Independent review passed.", + "passed on Linux, and independent review passed and deployed.", +]) { + test(`invalid host or review suffix cannot pass ${status}`, () => { + expect(scenario.check(fixture(status)).length).toBeGreaterThan(0); + }); +} +test("unregistered compound command cannot borrow native proof", () => { + const input = fixture(); + input.finalText = input.finalText.replace( + `\`${gate}\` ${phrase}`, + `\`node scripts/other.mjs\` ${phrase}`, + ); + expect(scenario.check(input)).toContain( + "Unsupported or conflicting current handoff assertions.", + ); +}); +for (const text of [ + `Goal: Preserve '${gate} ${phrase}'`, + `Historical handoff\n${gate} ${phrase}`, + `Example: ${gate} ${phrase}`, +]) { + test(`compound proof retains scope ${text}`, () => { + const facts = currentHandoffFacts(text, [gate]); + expect(facts.observations).toEqual([]); + expect(facts.independentReview).toEqual([]); + }); +} +test("quoted command arguments cannot become review siblings", () => { + const command = `${gate} --label "retained, and independent review passed with no findings"`; + const facts = currentHandoffFacts(`${command} passed on Linux.`, [command]); + const expected = [ + { + command, + exitCode: 0, + qualification: "claimed-pass" as const, + integrity: "not-claimed" as const, + hostPlatform: "linux" as const, + }, + ]; + expect(facts.observations).toEqual(expected); + expect(facts.independentReview).toEqual([]); +}); +for (const prefix of [ + "Its script was modified to preserve trimming behavior.", + "The script was modified to handle null input.", +]) { + test(`unowned implementation prose remains context ${prefix}`, () => { + const input = fixture(); + input.finalText = `${prefix}\n${input.finalText}`; + expect(scenario.check(input)).toEqual([]); + }); +} +for (const separator of [". ", "; "]) { + for (const qualifier of [ + "Its script was unchanged.", + "Its script and invocation were unchanged.", + ]) { + test(`joined review keeps truthful command qualifier ${separator}${qualifier}`, () => { + const status = phrase.replace(/\.$/, `${separator}${qualifier}`); + expect(scenario.check(fixture(status))).toEqual([]); + }); + } +} +for (const metadata of [ + "host Linux, source current", + "host Linux, report retained", + "host Linux, output recorded", + "host Linux, source current, report retained", + 'host Linux, report "retained, host Windows"', +]) { + test(`numeric host preserves supported metadata ${metadata}`, () => { + expect( + scenario.check( + fixture(`passed, exit 0, ${metadata}. Independent review passed.`), + ), + ).toEqual([]); + }); +} +for (const host of ["Linux", "Windows"]) { + test(`nonpass observation host binds to genuine deferred capture ${host}`, () => { + const input = deferredCaptureOutcome(); + const finalText = `${input.finalText}\nObserved "${localCommand}": exit 0, host ${host}; this does not claim the command passed.`; + const issues = deliveryIssues( + { ...input, finalText }, + { + closure: "deferred", + presentation: "summary", + gate: localCommand, + missingEvidenceCommand: externalCommand, + allowedPaths: ["src/parser.mjs"], + }, + ); + if (host === "Linux") expect(issues).toEqual([]); + else + expect(issues).toContain( + "Claimed command host lacks matching accepted complete source evidence.", + ); + }); +} +for (const prose of [ + "It preserves trimming behavior.", + "Its implementation preserves trimmed string inputs.", + "The script handles null safely.", +]) { + test(`ordinary post-review prose remains intact ${prose}`, () => { + expect(scenario.check(fixture(`${phrase} ${prose}`))).toEqual([]); + }); +} +test("canonical win32 platform becomes a typed host claim", () => { + const expected = [ + { + command: gate, + exitCode: 0, + qualification: "claimed-pass" as const, + integrity: "not-claimed" as const, + hostPlatform: "win32" as const, + }, + ]; + expect( + currentHandoffFacts(`${gate} passed on win32.`, [gate]).observations, + ).toEqual(expected); +}); +test("unregistered win32 platform claim remains unsupported", () => { + const input = fixture(); + input.finalText = input.finalText.replace( + `\`${gate}\` ${phrase}`, + "`node scripts/other.mjs` passed on win32. Independent review passed.", + ); + expect(scenario.check(input)).toContain( + "Unsupported or conflicting current handoff assertions.", + ); +}); +test("recognized win32 claim still fails the Linux native witness", () => { + expect( + scenario.check(fixture("passed on win32. Independent review passed.")), + ).toContain( + "Claimed command pass lacks matching accepted complete source evidence.", + ); +}); +for (const change of ["nonzero", "incomplete", "source", "intent"]) { + test(`nonpass host cannot rescue corrupt deferred capture ${change}`, () => { + const input = deferredCaptureOutcome(); + const value = capturedValidation(input); + if (change === "nonzero") value.exitCode = 1; + if (change === "incomplete") value.outputComplete = false; + if (change === "source") value.sourceDigest = `sha256:${"b".repeat(64)}`; + if (change === "intent") value.intent = "observe"; + const finalText = `${input.finalText}\nObserved "${localCommand}": exit 0, host Linux; this does not claim the command passed.`; + expect( + deliveryIssues( + { ...input, finalText }, + { + closure: "deferred", + presentation: "summary", + gate: localCommand, + missingEvidenceCommand: externalCommand, + allowedPaths: ["src/parser.mjs"], + }, + ), + ).toContain( + "Claimed command host lacks matching accepted complete source evidence.", + ); + }); +} +for (const metadata of [ + "host Windows, source current", + "host Solaris, source current", + "host Linux, host Windows", +]) { + test(`numeric metadata cannot hide conflicting host ${metadata}`, () => { + expect( + scenario.check( + fixture(`passed, exit 0, ${metadata}. Independent review passed.`), + ).length, + ).toBeGreaterThan(0); + }); +} +function capturedHosts(first: string, second?: string, pinned = false) { + const input = deferredCaptureOutcome(); + const archive = object(input.archives[0]); + const plan = object(archive.plan); + if (!pinned) { + if (!Array.isArray(plan.evidence)) + throw new Error("Missing plan evidence."); + for (const entry of plan.evidence.map(object)) + if (entry.command === localCommand) delete entry.platform; + if (!Array.isArray(plan.features)) + throw new Error("Missing plan features."); + for (const feature of plan.features.map(object)) + if (Array.isArray(feature.checks)) + for (const check of feature.checks.map(object)) + if (check.command === localCommand) delete check.platform; + } + capturedValidation(input).hostPlatform = first; + const calls = [...input.allCalls]; + const arm = calls.find((call) => call.tool === "flow_validation_start"); + const bash = calls.find((call) => call.tool === "bash"); + const status = calls.find((call) => call.tool === "flow_status"); + const close = calls.find((call) => call.tool === "flow_session_close"); + if (!arm || !bash || !status || !close) + throw new Error("Missing capture calls."); + const firstProjection = object( + object(object(status.output).workflowData).projection, + ); + firstProjection.plan = structuredClone(plan); + const firstRuns = firstProjection.runs; + if (!Array.isArray(firstRuns)) throw new Error("Missing status runs."); + const firstValidations = object(firstRuns[0]).validations; + if (!Array.isArray(firstValidations)) + throw new Error("Missing status validation."); + object(firstValidations[0]).hostPlatform = first; + Object.assign(status, { rawOutput: JSON.stringify(status.output) }); + if (second !== undefined) { + const next = { + ...capturedValidation(input), + id: "capture-second", + recordedRevision: 5, + hostPlatform: second, + }; + const archiveRuns = archive.runs; + if (!Array.isArray(archiveRuns)) throw new Error("Missing archive runs."); + const values = object(archiveRuns[0]).validations; + if (!Array.isArray(values)) throw new Error("Missing archive validations."); + values.push(next); + const secondArm = structuredClone(arm); + const secondBash = structuredClone(bash); + const secondStatus = structuredClone(status); + for (const [index, call] of [secondArm, secondBash, secondStatus].entries()) + Object.assign(object(call.native), { + messageId: `msg_second${index}`, + partId: `prt_second${index}`, + callId: `call_second${index}`, + startedAt: 26 + index * 2, + completedAt: 27 + index * 2, + }); + object(secondArm.input.request).expectedRevision = 4; + object(object(object(secondArm.output).workflowData).capture).captureId = + next.id; + Object.assign(secondArm, { rawOutput: JSON.stringify(secondArm.output) }); + const marker = `[flow-validation] ${JSON.stringify({ id: next.id, scope: "broad", intent: "pass", passed: true, observed: false, recordedRevision: 5 })}`; + Object.assign(secondBash, { + output: `(no output)\n\n${marker}`, + rawOutput: `(no output)\n\n${marker}`, + }); + const projection = object( + object(object(secondStatus.output).workflowData).projection, + ); + projection.revision = 5; + projection.runs = [ + { ...structuredClone(object(archiveRuns[0])), state: "active" }, + ]; + Object.assign(secondStatus, { + rawOutput: JSON.stringify(secondStatus.output), + }); + archive.revision = 6; + object(archive.closure).recordedRevision = 6; + object(close.input.request).expectedRevision = 5; + object(object(object(close.output).workflowData).operation).revision = 6; + calls.splice(calls.indexOf(close), 0, secondArm, secondBash, secondStatus); + } + return { + ...input, + allCalls: calls, + flowCalls: calls.filter((call) => call.tool.startsWith("flow_")), + hostTrace: nativeTrace(calls), + }; +} +const deferredExpectation = { + closure: "deferred" as const, + presentation: "summary" as const, + gate: localCommand, + missingEvidenceCommand: externalCommand, + allowedPaths: ["src/parser.mjs"], +}; +test("canonical other category is retained as a host claim", () => { + const claim = currentHandoffFacts(`${gate} passed on Other.`, [gate]) + .observations[0]; + expect(String(claim?.hostPlatform)).toBe("other"); +}); +test("unconstrained genuinely captured Other supports an Other claim", () => { + const input = capturedHosts("other"); + const finalText = + input.finalText + .replace(/^- \*\*Linux validation:\*\*.*\n/m, "") + .replace(/^- \*\*Outstanding proof:\*\*.*\n/m, "") + + `\n${localCommand} passed, exit 0, host other.`; + expect(deliveryIssues({ ...input, finalText }, deferredExpectation)).toEqual( + [], + ); +}); +test("other never infers an unknown Solaris category", () => { + const input = capturedHosts("other"); + const finalText = + input.finalText + .replace(/^- \*\*Linux validation:\*\*.*\n/m, "") + .replace(/^- \*\*Outstanding proof:\*\*.*\n/m, "") + + `\n${localCommand} passed, exit 0, host Solaris.`; + expect( + deliveryIssues({ ...input, finalText }, deferredExpectation), + ).toContain("Unsupported or conflicting current handoff assertions."); +}); +for (const status of [ + `${localCommand} passed on Linux.`, + `Observed "${localCommand}": exit 0, host Linux; this does not claim the command passed.`, +]) { + test(`later fully attested Linux capture supports the requested host ${status}`, () => { + const input = capturedHosts("other", "linux"); + expect( + deliveryIssues( + { ...input, finalText: `${input.finalText}\n${status}` }, + deferredExpectation, + ), + ).toEqual([]); + }); +} +test("declared Linux platform still rejects a captured Other host", () => { + const input = capturedHosts("other", undefined, true); + const finalText = `${input.finalText}\nObserved "${localCommand}": exit 0, host Other; this does not claim the command passed.`; + expect( + deliveryIssues({ ...input, finalText }, deferredExpectation), + ).toContain( + "Claimed command host lacks matching accepted complete source evidence.", + ); +}); +for (const corruption of [ + "arm", + "marker", + "clock", + "status", + "source", + "intent", + "incomplete", +]) { + test(`invalid later matching-host capture cannot rescue ${corruption}`, () => { + const input = capturedHosts("other", "linux"); + const arm = input.allCalls.find( + (call) => call.native?.callId === "call_second0", + ); + const bash = input.allCalls.find( + (call) => call.native?.callId === "call_second1", + ); + const status = input.allCalls.find( + (call) => call.native?.callId === "call_second2", + ); + if (!arm || !bash || !status) + throw new Error("Missing second capture calls."); + if (corruption === "arm") object(arm.input.request).expectedRevision = 3; + if (corruption === "marker") + Object.assign(bash, { + rawOutput: bash.rawOutput.replace("capture-second", "unmatched-marker"), + output: bash.rawOutput.replace("capture-second", "unmatched-marker"), + }); + if (corruption === "clock") object(bash.native).startedAt = 18; + const runs = object(input.archives[0]).runs; + if (!Array.isArray(runs)) throw new Error("Missing archive runs."); + const values = object(runs[0]).validations; + if (!Array.isArray(values)) throw new Error("Missing archive validations."); + const capture = object(values[1]); + if (corruption === "source") + capture.sourceDigest = `sha256:${"b".repeat(64)}`; + if (corruption === "intent") capture.intent = "observe"; + if (corruption === "incomplete") capture.outputComplete = false; + if (corruption === "status") { + const projectedRuns = object( + object(object(status.output).workflowData).projection, + ).runs; + if (!Array.isArray(projectedRuns)) + throw new Error("Missing status runs."); + const projected = object(projectedRuns[0]).validations; + if (!Array.isArray(projected)) + throw new Error("Missing status validations."); + object(projected[1]).outputDigest = `sha256:${"b".repeat(64)}`; + } + const finalText = `${input.finalText}\n${localCommand} passed on Linux.`; + expect( + deliveryIssues({ ...input, finalText }, deferredExpectation), + ).toContain( + "Claimed command pass lacks matching accepted complete source evidence.", + ); + }); +} diff --git a/tests/fixtures/delivery-platform-pass-answer.json b/tests/fixtures/delivery-platform-pass-answer.json new file mode 100644 index 00000000..aae2699b --- /dev/null +++ b/tests/fixtures/delivery-platform-pass-answer.json @@ -0,0 +1,154 @@ +{ + "answer": "Updated `src/parser.mjs`:\n\n```js\nexport function parse(input) { return input === null ? '' : input.trim(); }\n```\n\n`node scripts/verify.mjs` passed on Linux, and independent review passed. Both scripts and advisory data were preserved.\n\n```text\nHandoff format: 1\nExternal action authority: not-granted\nGoal: Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs.\nClosure: completed\nProgress: 1 of 1 features complete\nUnfinished features: none\nNonblocking live findings: advisory 1. Historical findings: 0.\nObserved \"node scripts/audit.mjs\": exit 12, host linux; this does not claim the command passed.\nAssurance: completion supported\nAssurance checks: 4 satisfied, 0 not applicable, 0 unsatisfied.\n- Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.\n- Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments.\n- Freshness holds when review is accepted; an archive does not attest the current workspace.\nReported artifacts: 1 latest, 0 superseded only. Caller declarations, not an exact or exhaustive Git delta.\nFull report is included in this close response.\n```", + "goal": "Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs.", + "featureId": "parser-null", + "delivery": { + "assurance": { + "checks": [ + { + "explanation": "1/1 features and 1/1 independent reviews pass, including a final review with no terminal blocker.", + "id": "recorded-completion", + "label": "Recorded completion", + "status": "satisfied", + "tier": "ts-enforced" + }, + { + "explanation": "1/1 terminal runs have eligible host evidence accepted by review.", + "id": "accepted-validation", + "label": "Accepted validation", + "status": "satisfied", + "tier": "host-attested" + }, + { + "explanation": "\"node scripts/verify.mjs\" must have passing broad evidence accepted by review.", + "id": "canonical-gate", + "label": "Canonical gate", + "status": "satisfied", + "tier": "host-attested" + }, + { + "explanation": "1/1 declared obligations have accepted evidence on their declared host; observed gates do not claim a pass.", + "id": "declared-evidence", + "label": "Declared evidence", + "status": "satisfied", + "tier": "host-attested" + } + ], + "conclusion": "completion-supported", + "limitations": [ + "Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.", + "Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments.", + "Freshness holds when review is accepted; an archive does not attest the current workspace." + ] + }, + "closure": { + "kind": "completed", + "summary": "Implemented null guard only in src/parser.mjs; Linux verification passed, audit observed exit 12 retained as advisory, independent review passed." + }, + "features": [ + { + "attempts": 1, + "id": "parser-null", + "latestState": "completed", + "outcomeSummary": "Verified parser-null and all four approved requirements: strict null returns empty string, unchanged string trimming, parser-only scope with scripts/advisory data and preexisting work preserved, and separate Linux observe/pass checks. Inspected current source, downstream verification, README/package surfaces and complete host-bound base diff: no deletion, rename, type or mode change. Source-bound broad verification passed with complete output. Audit observation retained as advisory; no prior findings or blocking gaps.", + "terminalFindings": [ + { + "severity": "advisory", + "summary": "Audit observation reports 12 outstanding advisory items; retained without repair or blocking completion." + } + ], + "title": "Null-safe parser" + } + ], + "findingsDigest": [ + { + "attempt": 1, + "evidence": "scripts/audit.mjs:1 reports 12 outstanding items and exits 12; advisories.json:1 records outstanding=12. Source-bound Linux validation 102ba2f5-c44f-4f9e-b3cb-e6a004ca7c16 records intent observe, exitCode 12, complete output and output digest. Approved scope explicitly preserves these artifacts.", + "featureId": "parser-null", + "findingId": "parser-null.R6-01", + "live": true, + "severity": "advisory", + "summary": "Audit observation reports 12 outstanding advisory items; retained without repair or blocking completion.", + "verdict": "passed" + } + ], + "goal": "Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs.", + "handoff": { + "externalActionAuthority": "not-granted", + "formatVersion": 1 + }, + "observations": [ + { + "command": "node scripts/audit.mjs", + "exitCode": 12, + "featureId": "parser-null", + "hostPlatform": "linux", + "id": "102ba2f5-c44f-4f9e-b3cb-e6a004ca7c16", + "intent": "observe", + "outputComplete": true, + "outputDigest": "sha256:c6793caf5f3c67b895af8671e9d3705639e0d1e4590377e84b77db953ebe017d", + "recordedRevision": 4, + "runId": "run:44ec7349-601c-48cb-8c63-95ac0f2f58c9", + "scope": "focused", + "sourceDigest": "sha256:0c341eab346e3a201294327b2e2b5d500aa42cd700425dc491bdc64965a6ee0e" + } + ], + "progress": { + "completed": 1, + "total": 1 + }, + "report": [ + "Handoff format: 1", + "External action authority: not-granted", + "Goal: Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs.", + "Closure: completed \u2014 Implemented null guard only in src/parser.mjs; Linux verification passed, audit observed exit 12 retained as advisory, independent review passed.", + "Progress: 1 of 1 features complete", + "Features:", + "- parser-null \u2014 Null-safe parser", + " attempts: 1; latest state: completed", + " outcome: Verified parser-null and all four approved requirements: strict null returns empty string, unchanged string trimming, parser-only scope with scripts/advisory data and preexisting work preserved, and separate Linux observe/pass checks. Inspected current source, downstream verification, README/package surfaces and complete host-bound base diff: no deletion, rename, type or mode change. Source-bound broad verification passed with complete output. Audit observation retained as advisory; no prior findings or blocking gaps.", + " terminal findings:", + " - advisory: Audit observation reports 12 outstanding advisory items; retained without repair or blocking completion.", + "Findings digest:", + "- live parser-null parser-null.R6-01 advisory: Audit observation reports 12 outstanding advisory items; retained without repair or blocking completion.", + "Observed \"node scripts/audit.mjs\": exit 12, host linux, source sha256:0c341eab346e3a201294327b2e2b5d500aa42cd700425dc491bdc64965a6ee0e, output sha256:c6793caf5f3c67b895af8671e9d3705639e0d1e4590377e84b77db953ebe017d; this does not claim the command passed.", + "Assurance: completion supported", + "Assurance checks:", + "- satisfied [TS-enforced] Recorded completion: 1/1 features and 1/1 independent reviews pass, including a final review with no terminal blocker.", + "- satisfied [host-attested] Accepted validation: 1/1 terminal runs have eligible host evidence accepted by review.", + "- satisfied [host-attested] Canonical gate: \"node scripts/verify.mjs\" must have passing broad evidence accepted by review.", + "- satisfied [host-attested] Declared evidence: 1/1 declared obligations have accepted evidence on their declared host; observed gates do not claim a pass.", + "Assurance limitations:", + "- Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.", + "- Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments.", + "- Freshness holds when review is accepted; an archive does not attest the current workspace.", + "Artifacts as reported by Flow from caller declarations, not an exact or exhaustive Git delta:", + "- latest attempts: src/parser.mjs", + "- superseded attempts only: none reported" + ], + "reportedArtifacts": { + "latestAttempts": ["src/parser.mjs"], + "supersededAttemptsOnly": [] + }, + "summary": { + "fullReportAvailable": true, + "lines": [ + "Handoff format: 1", + "External action authority: not-granted", + "Goal: Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs.", + "Closure: completed", + "Progress: 1 of 1 features complete", + "Unfinished features: none", + "Nonblocking live findings: advisory 1. Historical findings: 0.", + "Observed \"node scripts/audit.mjs\": exit 12, host linux; this does not claim the command passed.", + "Assurance: completion supported", + "Assurance checks: 4 satisfied, 0 not applicable, 0 unsatisfied.", + "- Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.", + "- Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments.", + "- Freshness holds when review is accepted; an archive does not attest the current workspace.", + "Reported artifacts: 1 latest, 0 superseded only. Caller declarations, not an exact or exhaustive Git delta.", + "Full report is included in this close response." + ] + } + } +}