diff --git a/src/opencode/verifier.ts b/src/opencode/verifier.ts index 26e0dd9c..2608a5a4 100644 --- a/src/opencode/verifier.ts +++ b/src/opencode/verifier.ts @@ -1,5 +1,5 @@ -import { DEFAULT_VERIFIER_TIMEOUT_MS, SemanticVerifierUnavailableError, corroborateSemanticVerifierEvidence, semanticVerificationPrompt, semanticVerifierHostEvidence } from "../verification/verifier-evidence.js" -export { DEFAULT_VERIFIER_TIMEOUT_MS, SemanticVerifierUnavailableError, corroborateSemanticVerifierEvidence, semanticVerificationPrompt, semanticVerifierHostEvidence } from "../verification/verifier-evidence.js" +import { DEFAULT_VERIFIER_TIMEOUT_MS, SemanticVerifierUnavailableError, collectHostAbsenceEvidence, corroborateSemanticVerifierEvidence, semanticVerificationPrompt, semanticVerifierHostEvidence } from "../verification/verifier-evidence.js" +export { DEFAULT_VERIFIER_TIMEOUT_MS, SemanticVerifierUnavailableError, collectHostAbsenceEvidence, corroborateSemanticVerifierEvidence, semanticVerificationPrompt, semanticVerifierHostEvidence } from "../verification/verifier-evidence.js" import { randomUUID } from "node:crypto" import { tool } from "@opencode-ai/plugin/tool" import type { EvidenceRecord, GoalState } from "../domain/types.js" @@ -190,7 +190,10 @@ export function createSemanticVerifierRuntime(client: any, root: string, options : timeoutMs const allowTimeoutRetry = verifyOptions.allowTimeoutRetry !== false const auditToken = randomUUID() - const hostEvidenceRecords = semanticVerifierHostEvidence(goal, verifyOptions.currentMessageID) + const hostEvidenceRecords = [ + ...semanticVerifierHostEvidence(goal, verifyOptions.currentMessageID), + ...await collectHostAbsenceEvidence(root, goal), + ] let childID = "" let retryAfterTimeout = false try { diff --git a/src/opencode2/semantic-verifier.ts b/src/opencode2/semantic-verifier.ts index b7701c1b..4378113b 100644 --- a/src/opencode2/semantic-verifier.ts +++ b/src/opencode2/semantic-verifier.ts @@ -9,6 +9,7 @@ import { import { DEFAULT_VERIFIER_TIMEOUT_MS, SemanticVerifierUnavailableError, + collectHostAbsenceEvidence, corroborateSemanticVerifierEvidence, semanticVerificationPrompt, semanticVerifierHostEvidence, @@ -335,7 +336,10 @@ export function createOpenCode2SemanticVerifierRuntime( const allowTimeoutRetry = verifyOptions.allowTimeoutRetry !== false const auditToken = randomUUID() const root = await resolveRoot(parentSessionID) - const hostEvidenceRecords = semanticVerifierHostEvidence(goal, verifyOptions.currentMessageID) + const hostEvidenceRecords = [ + ...semanticVerifierHostEvidence(goal, verifyOptions.currentMessageID), + ...await collectHostAbsenceEvidence(root, goal), + ] let childID = "" let retryAfterTimeout = false let primaryTimeoutMessage: string | undefined diff --git a/src/verification/process.ts b/src/verification/process.ts index bd31a2f0..c465fdd5 100644 --- a/src/verification/process.ts +++ b/src/verification/process.ts @@ -1,5 +1,6 @@ import type { EvidenceRecord, GoalState } from "../domain/types.js" import type { SemanticRequirementResult } from "./semantic.js" +import { explicitFileTargets, requestsDeletion, needsConcreteDeliverable } from "./semantic-targets.js" export interface ProcessTurnExpectation { turns: number @@ -73,6 +74,97 @@ function runtimeMetric(records: EvidenceRecord[], key: "turns" | "mutations"): n return undefined } +/** + * Runtime turn/progress counters are authoritative for an explicit process + * contract (e.g. N distinct Goal turns), but cannot certify a generic + * deliverable such as "implement the feature" or "create the file". + * + * By this point verifier file quotes have been host-corroborated. Only + * literal corroborated files or an explicitly cited, current passing + * file/command host record may establish a non-process claim. The same + * verified file record can support both a file requirement and the broader + * semantic objective; requiring identical requirement IDs would break + * legitimate shared evidence. Runtime counters never suffice. + */ +function normalizedPath(value: string | undefined): string { + const normalized = String(value ?? "").replace(/\\/g, "/").replace(/^\.\/+/, "") + return process.platform === "win32" ? normalized.toLowerCase() : normalized +} + +function hasConcreteResultEvidence( + goal: GoalState, + requirementText: string, + result: SemanticRequirementResult, + records: EvidenceRecord[], +): boolean { + const targets = explicitFileTargets(requirementText).map(normalizedPath) + const deletionOnly = requestsDeletion(requirementText) + && !/\b(?:implement|create|update|write|build|fix|add|refactor)\b/i.test(requirementText) + const quotedPaths = new Set(result.evidence + .filter((item) => item.path?.trim() && item.quote?.trim()) + .map((item) => normalizedPath(item.path))) + const ids = new Set(result.hostEvidenceIDs) + const current = records.filter((item) => + ids.has(item.id) + && item.goalRevision === goal.revision + && item.trust === "host" + && item.passed === true, + ) + // Merely citing a passing host check is not enough. Bind it to its + // structured file/command contract; runtime counters have neither. + const provedFiles = current.filter((item) => { + if (item.kind !== "file" || !item.source || item.metadata?.absent === true) return false + return goal.requirements.some((requirement) => + requirement.verification === "file" + && requirement.file + && normalizedPath(requirement.file) === normalizedPath(item.source) + && item.requirementIDs.includes(requirement.id) + && requirement.status === "proven" + && item.metadata?.exists !== false + && item.metadata?.regularFile !== false, + ) + }) + const absence = current.filter((item) => + item.kind === "file" + && item.metadata?.absent === true + && item.requirementIDs.includes(result.requirementID) + && item.source, + ) + + // Named deliverables require proof of THOSE paths, not just any README + // contract, a generic npm test, or a host runtime/turn count. + if (targets.length > 0) { + return targets.every((target) => + deletionOnly + ? absence.some((item) => normalizedPath(item.source) === target) + : quotedPaths.has(target) || provedFiles.some((item) => normalizedPath(item.source) === target), + ) + } + + // Unnamed deletion has no inspectable negative target; do not infer it from + // a command or an unrelated file. Explicit named deletions use host absence. + if (deletionOnly) return false + if (quotedPaths.size > 0 || provedFiles.length > 0) return true + + // A direct check can prove a test/lint/build-only objective if the passed + // command actually implements the named verification. "true" cannot prove + // an implementation, and a test pass alone cannot prove a named artifact. + const testOnly = /\b(?:tests?|lint|typecheck|checks?|build|compile|ci)\b/i.test(requirementText) + && !/\b(?:implement|create|delete|remove|update|write|modify|fix|refactor)\b/i.test(requirementText) + if (!testOnly) return false + return current.some((item) => + item.kind === "command" + && item.source + && /\b(?:test|lint|check|build|compile|tsc)\b/i.test(item.source) + && goal.requirements.some((requirement) => + requirement.verification === "command" + && requirement.status === "proven" + && requirement.command === item.source + && item.requirementIDs.includes(requirement.id), + ), + ) +} + function guarded(result: SemanticRequirementResult, verdict: "failed" | "unknown", reason: string): SemanticRequirementResult { return { ...result, @@ -107,6 +199,12 @@ export function guardSemanticProcessResults( const requirement = requirements.get(result.requirementID) if (!requirement || requirement.verification !== "semantic") return result const expectation = inferProcessTurnExpectation(requirement.text) + // A turn requirement can coexist with an actual deliverable. Proving + // cadence alone must not silently prove an implementation or deleted file. + const needsArtifact = !expectation || needsConcreteDeliverable(requirement.text, expectation.requireMutationPerTurn) + if (needsArtifact && !hasConcreteResultEvidence(goal, requirement.text, result, hostEvidenceRecords)) { + return guarded(result, "unknown", "non-process deliverable has only runtime counters, unrelated checks or unsupported evidence; real target-file, host absence, or directly linked verification evidence is required.") + } if (!expectation) return result if (expectation.requireMutationPerTurn) { diff --git a/src/verification/semantic-targets.ts b/src/verification/semantic-targets.ts new file mode 100644 index 00000000..2bd74cff --- /dev/null +++ b/src/verification/semantic-targets.ts @@ -0,0 +1,27 @@ +// Conservative helpers for deciding what a *semantic* Goal actually asks to +// deliver. These are guardrails, not substitutes for the independent verifier. + +const FILE_TOKEN = /\b(?:[a-zA-Z0-9_@.-]+[\\/])*[a-zA-Z0-9_@.-]+\.[a-zA-Z][a-zA-Z0-9]{0,8}\b/g + +export function explicitFileTargets(text: string): string[] { + return [...new Set((text.match(FILE_TOKEN) ?? []).map((value) => value.replace(/\\/g, "/")))] +} + +export function requestsDeletion(text: string): boolean { + return /\b(?:delete|deleting|remove|removing|unlink|erase)\b|\bsil(?:mek)?\b|\bkald[ıi]r\b/i.test(text) +} + +/** + * "Exactly N Goal turns" is a process contract. + * "Implement X across N Goal turns" is a mixed contract: the count proves + * only the process part, and real project evidence must prove X separately. + * + * Pure per-turn mutation instructions may be proven by host-owned mutation + * fingerprints; otherwise an explicit create/write/update deliverable still + * needs corroborated artifacts. + */ +export function needsConcreteDeliverable(text: string, perTurnMutation: boolean): boolean { + if (/\b(?:implement|create|delete|deleting|remove|removing|ship|deliver|build|fix|refactor|publish|document)\b/i.test(text)) return true + if (!perTurnMutation && /\b(?:write|update|modify|edit|add)\b/i.test(text)) return true + return false +} diff --git a/src/verification/verifier-evidence.ts b/src/verification/verifier-evidence.ts index 2d02372b..87d8158c 100644 --- a/src/verification/verifier-evidence.ts +++ b/src/verification/verifier-evidence.ts @@ -3,6 +3,7 @@ import { promises as fs } from "node:fs" import path from "node:path" import type { EvidenceRecord, GoalState } from "../domain/types.js" import type { SemanticEvidenceRef, SemanticRequirementResult } from "./semantic.js" +import { explicitFileTargets, requestsDeletion } from "./semantic-targets.js" export const DEFAULT_VERIFIER_TIMEOUT_MS = 60_000 @@ -57,6 +58,60 @@ export function semanticVerifierHostEvidence(goal: GoalState, currentMessageID?: return [...runtime, ...persisted] } + +/** + * Verify a negative filesystem assertion without trusting a model or a stale + * diff claim. A symlink/junction in the path is never accepted as absence. + * Missing ancestors count as absence of the target; permissions/errors do not. + */ +async function hostVerifiedAbsence(root: string, candidate: string): Promise { + const absolute = resolveInside(root, candidate) + const relative = path.relative(path.resolve(root), absolute) + let cursor = path.resolve(root) + for (const segment of relative.split(path.sep).filter(Boolean)) { + cursor = path.join(cursor, segment) + try { + const info = await fs.lstat(cursor) + if (info.isSymbolicLink()) return false + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return true + throw error + } + } + return false +} + +/** + * The verifier cannot quote a file that was deleted. Supply explicit host- + * observed absence evidence only for named paths in delete/remove objectives. + * The evidence is rechecked after the verifier returns to close race windows. + */ +export async function collectHostAbsenceEvidence(root: string, goal: GoalState): Promise { + const observations: EvidenceRecord[] = [] + for (const requirement of goal.requirements) { + if (!requirement.required || requirement.verification !== "semantic" || !requestsDeletion(requirement.text)) continue + for (const file of explicitFileTargets(requirement.text)) { + let absent = false + try { absent = await hostVerifiedAbsence(root, file) } catch { /* fail closed */ } + if (!absent) continue + const key = createHash("sha256").update([goal.id, goal.revision, requirement.id, file].join("\0")).digest("hex").slice(0, 18) + observations.push({ + id: "goal-absent-r" + goal.revision + "-" + key, + kind: "file", + trust: "host", + summary: "Host verified the requested removal target " + file + " is absent from the current workspace (symlinks are not accepted).", + createdAt: Date.now(), + goalRevision: goal.revision, + requirementIDs: [requirement.id], + source: file, + passed: true, + metadata: { absent: true, exists: false }, + }) + } + } + return observations +} + export function semanticVerificationPrompt(goal: GoalState, auditToken: string, hostEvidenceRecords: EvidenceRecord[]): string { const semantic = goal.requirements.filter((item) => item.required && item.verification === "semantic") const hostEvidence = hostEvidenceRecords @@ -69,7 +124,7 @@ export function semanticVerificationPrompt(goal: GoalState, auditToken: string, objective: goal.objective, requirements: semantic.map((item) => ({ id: item.id, text: item.text })), }, null, 2) - return `Independently audit the semantic requirements for an OpenCode goal.\n\nThe goal executor's claims are not proof. Inspect the current workspace yourself using only read/search tools. Never edit files, run shell commands, delegate tasks, or mutate goal state. Preserve the full requested scope.\n\nVerdicts:\n- proven: current authoritative workspace evidence directly establishes the requirement. Support proven verdicts with exact file excerpts as {path, quote} and/or IDs of current passing host evidence. The host will independently re-read file quotes and validate every host-evidence ID.\n- failed: current evidence directly contradicts the requirement.\n- unknown: evidence is missing, indirect, ambiguous, external, or would require executing a command you cannot run.\n\nPrefer a current passing host-evidence ID when it directly establishes the requirement. When citing a file, use a path relative to the workspace root (for example docs/file.md) or an absolute path with its leading root/drive intact; never strip the leading slash or drive prefix. Quote only literal text that exists inside that file. Never copy read-tool line numbers, path headers, XML/Markdown wrappers, labels, or explanatory prose into the quote.\n\nDo not treat a vague statement, plan, TODO, changelog claim, or unverified test claim as proof. Host-run verification evidence, when present below, may be used only for what it actually establishes. A path without an exact quote is not proof, and an unknown host-evidence ID is not proof. Temporal/process requirements such as doing an action across N distinct turns are not proven by a final file value alone: use the host runtime turn/progress evidence below and return unknown or failed when the requested cadence/count is not established.\n\nHost evidence:\n${hostEvidence}\n\nVerification request:\n${request}\n\nCall opencode_goal_verifier_result exactly once with the auditToken and one result for every listed requirement. Do not return a success verdict outside that tool.` + return `Independently audit the semantic requirements for an OpenCode goal.\n\nThe goal executor's claims are not proof. Inspect the current workspace yourself using only read/search tools. Never edit files, run shell commands, delegate tasks, or mutate goal state. Preserve the full requested scope.\n\nVerdicts:\n- proven: current authoritative workspace evidence directly establishes the requirement. Support proven verdicts with exact file excerpts as {path, quote} and/or IDs of current passing host evidence. The host will independently re-read file quotes and validate every host-evidence ID.\n- failed: current evidence directly contradicts the requirement.\n- unknown: evidence is missing, indirect, ambiguous, external, or would require executing a command you cannot run.\n\nPrefer a current passing host-evidence ID when it directly establishes the requirement. Deletion-only objectives may cite a host-verified missing-path record when listed below; runtime mutation counters cannot prove that a named file is absent. Host-verified absence will be rechecked after your verdict. Never cite unrelated passing checks or files as proof of a different deliverable.\n\nWhen citing a file, use a path relative to the workspace root (for example docs/file.md) or an absolute path with its leading root/drive intact; never strip the leading slash or drive prefix. Quote only literal text that exists inside that file. Never copy read-tool line numbers, path headers, XML/Markdown wrappers, labels, or explanatory prose into the quote.\n\nDo not treat a vague statement, plan, TODO, changelog claim, or unverified test claim as proof. Host-run verification evidence, when present below, may be used only for what it actually establishes. A path without an exact quote is not proof, and an unknown host-evidence ID is not proof. Temporal/process requirements such as doing an action across N distinct turns are not proven by a final file value alone: use the host runtime turn/progress evidence below and return unknown or failed when the requested cadence/count is not established.\n\nHost evidence:\n${hostEvidence}\n\nVerification request:\n${request}\n\nCall opencode_goal_verifier_result exactly once with the auditToken and one result for every listed requirement. Do not return a success verdict outside that tool.` } function resolveInside(root: string, candidate: string): string { @@ -217,6 +272,14 @@ export async function corroborateSemanticVerifierEvidence( if (!hostEvidence || hostEvidence.goalRevision !== goal.revision || hostEvidence.trust !== "host" || hostEvidence.passed !== true) { throw new Error(`verifier referenced invalid or non-passing host evidence: ${id}`) } + + // Missing-path evidence is a snapshot fact. If the file is recreated + // during verification, no earlier absence observation can authorize done. + if (hostEvidence.metadata?.absent === true) { + if (hostEvidence.kind !== "file" || !hostEvidence.source || !await hostVerifiedAbsence(root, hostEvidence.source)) { + throw new Error("host-verified deletion target is no longer absent: " + (hostEvidence.source || id)) + } + } } if (result.verdict === "proven" && evidence.length === 0 && normalizedHostEvidenceIDs.length === 0) { throw new Error("proven semantic requirement lacks current corroborated evidence") diff --git a/test/opencode2-semantic-verifier.test.mjs b/test/opencode2-semantic-verifier.test.mjs index 77d80fc7..39b6eb7e 100644 --- a/test/opencode2-semantic-verifier.test.mjs +++ b/test/opencode2-semantic-verifier.test.mjs @@ -151,6 +151,49 @@ test("V2 semantic verifier is child-session bound, read-only, and host-corrobora } }) + +test("V2 named deletion uses current host absence and rejects recreation during independent verification", async () => { + for (const recreate of [false, true]) { + const root = await mkdtemp(path.join(os.tmpdir(), "goal-v2-delete-")) + const goal = createGoal({ sessionID: "parent", objective: "delete obsolete.txt" }) + let runtime + const session = { + async create() { return { id: "verifier-delete-child" } }, + async prompt(input) { + if (input.resume === false) return { id: "verifier-delete-message" } + const request = verificationRequest(input.text) + const absenceID = input.text.match(/- \[(goal-absent-r\d+-[a-f0-9]+)\] Host verified/)?.[1] + assert.ok(absenceID, "native verifier must see a host-observed named absence, not runtime counters") + const accepted = await runtime.resultTool.execute({ + auditToken: request.auditToken, + results: request.requirements.map((item) => ({ + requirementID: item.id, verdict: "proven", + reason: "Host confirms obsolete.txt is absent.", + evidence: [], hostEvidenceIDs: [absenceID], + })), + }, { sessionID: "verifier-delete-child" }) + assert.equal(accepted.content, "Semantic verifier result accepted.") + if (recreate) await writeFile(path.join(root, "obsolete.txt"), "recreated while verifying") + return { id: "verifier-delete-message" } + }, + async wait() { return { status: "idle" } }, + async delete() {}, + async interrupt() {}, + } + runtime = createOpenCode2SemanticVerifierRuntime(session, async () => root, { timeoutMs: 2_000 }) + try { + if (recreate) { + await assert.rejects(() => runtime.verify("parent", goal), /no longer absent/) + } else { + const verified = await runtime.verify("parent", goal) + assert.equal(verified.requirements.find((item) => item.verification === "semantic")?.status, "proven") + } + } finally { + await rm(root, { recursive: true, force: true }) + } + } +}) + test("V2 semantic verifier rejects hallucinated quotes and leaves completion unproven", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "opencode-goal-v2-semantic-bad-")) try { diff --git a/test/verifier-process-guard.test.mjs b/test/verifier-process-guard.test.mjs index ead73b41..53c97ca1 100644 --- a/test/verifier-process-guard.test.mjs +++ b/test/verifier-process-guard.test.mjs @@ -1,6 +1,13 @@ import test from "node:test" import assert from "node:assert/strict" import { guardSemanticProcessResults, inferProcessTurnExpectation } from "../dist/verification/process.js" +import { createGoal } from "../dist/domain/goal.js" +import { applySemanticVerifierResults } from "../dist/verification/semantic.js" +import { completeGoal } from "../dist/verification/audit.js" +import { collectHostAbsenceEvidence, corroborateSemanticVerifierEvidence, semanticVerifierHostEvidence } from "../dist/verification/verifier-evidence.js" +import { mkdtemp, rm, writeFile } from "node:fs/promises" +import os from "node:os" +import path from "node:path" function goal(text) { return { @@ -169,6 +176,184 @@ test("does not reinterpret non-process numeric requirements", () => { assert.equal(result.verdict, "proven") }) + +test("a verifier cannot mark generic deliverable complete using only runtime turn counters", () => { + const given = [{ + requirementID: "req-process", + verdict: "proven", + reason: "The assistant says all work is done.", + evidence: [], + hostEvidenceIDs: ["goal-runtime-turns-r1", "goal-runtime-progress-r1"], + }] + for (const counts of [ + { turns: 0, mutations: 0 }, + { turns: 7, mutations: 0 }, + { turns: 7, mutations: 30 }, + ]) { + const result = guardSemanticProcessResults( + goal("Objective achieved: implement src/slug.js with passing tests"), + given, + hostEvidence(counts), + )[0] + assert.equal(result.verdict, "unknown", "turns and mutation fingerprints cannot establish an implementation") + assert.match(result.reason, /real target-file, host absence, or directly linked verification evidence is required/) + } +}) + +test("explicit pure turn-count contract can still use host-observed runtime counters", () => { + const given = [{ + requirementID: "req-process", + verdict: "proven", + reason: "The host has counted the required turns.", + evidence: [], + hostEvidenceIDs: ["goal-runtime-turns-r1"], + }] + const passed = guardSemanticProcessResults( + goal("Objective achieved: at least 3 distinct Goal turns"), + given, + hostEvidence({ turns: 3, mutations: 0 }), + )[0] + assert.equal(passed.verdict, "proven") +}) + +test("host-corroborated file excerpt satisfies the non-process evidence guard", () => { + const given = [{ + requirementID: "req-process", + verdict: "proven", + reason: "The workspace file was checked by the host.", + evidence: [{ path: "src/slug.js", quote: "export function slugify" }], + hostEvidenceIDs: [], + }] + const guarded = guardSemanticProcessResults( + goal("Objective achieved: implement src/slug.js"), + given, + hostEvidence({ turns: 1, mutations: 1 }), + )[0] + assert.equal(guarded.verdict, "proven") +}) + +test("passing current host file/command checks are admissible but stale or runtime-only proof is not", () => { + const current = { id: "host-test-r1", kind: "command", trust: "host", + passed: true, goalRevision: 1, source: "npm test", requirementIDs: ["check-npm-test"] } + const given = [{ + requirementID: "req-process", verdict: "proven", + reason: "Verified host command passed.", + evidence: [], hostEvidenceIDs: [current.id], + }] + const target = goal("Objective achieved: ship passing tests") + target.requirements.push({ + id: "check-npm-test", text: "Verification command passes: npm test", + verification: "command", command: "npm test", required: true, + status: "proven", evidenceIDs: [current.id], updatedAt: Date.now(), + }) + assert.equal(guardSemanticProcessResults(target, given, [current])[0].verdict, "proven") + assert.equal(guardSemanticProcessResults(target, given, [{ ...current, goalRevision: 0 }])[0].verdict, "unknown") + assert.equal(guardSemanticProcessResults(target, given, [{ ...current, passed: false }])[0].verdict, "unknown") + assert.equal(guardSemanticProcessResults(target, given, [{ ...current, kind: "runtime" }])[0].verdict, "unknown") + assert.equal(guardSemanticProcessResults(target, given, [{ ...current, requirementIDs: ["unrelated-check"] }])[0].verdict, "unknown", "an unrelated passing command is not deliverable proof") + assert.equal(guardSemanticProcessResults(target, [{ ...given[0], hostEvidenceIDs: [] }], [current])[0].verdict, "unknown", "uncited host checks do not authorize completion") +}) + + +test("a false done verdict cannot persist completed Goal when no project artifact exists", () => { + const current = createGoal({ + sessionID: "false-complete-regression", + objective: "implement src/slug.js with tests", + }) + const requirement = current.requirements.find((item) => item.verification === "semantic") + assert.ok(requirement, "a generic Goal requires independent semantic proof") + const evidence = semanticVerifierHostEvidence(current, "current-model-turn") + const claims = [{ + requirementID: requirement.id, + verdict: "proven", + reason: "The assistant claims it has finished but no target file exists.", + evidence: [], + hostEvidenceIDs: evidence.map((item) => item.id), + }] + const checked = guardSemanticProcessResults(current, claims, evidence) + assert.equal(checked[0].verdict, "unknown") + const considered = applySemanticVerifierResults(current, checked) + const completed = completeGoal(considered, "All done!") + assert.equal(completed.audit.ok, false) + assert.notEqual(completed.goal.status, "completed") + assert.match(completed.audit.reasons.join(" "), /not proven/) +}) + + +test("mixed implementation-plus-process goal needs both actual deliverable and required turns", () => { + const objective = goal("Objective achieved: implement src/slug.js across at least 3 distinct Goal turns") + const records = hostEvidence({ turns: 3, mutations: 7 }) + const weak = [{ + requirementID: "req-process", verdict: "proven", + reason: "Three turns happened but the target file was never created.", + evidence: [], hostEvidenceIDs: records.map((item) => item.id), + }] + assert.equal(guardSemanticProcessResults(objective, weak, records)[0].verdict, "unknown") + const realFile = { + ...weak[0], + evidence: [{ path: "src/slug.js", quote: "export function slugify" }], + } + assert.equal(guardSemanticProcessResults(objective, [realFile], records)[0].verdict, "proven") + assert.equal(guardSemanticProcessResults(objective, [realFile], hostEvidence({ turns: 1, mutations: 7 }))[0].verdict, "unknown") +}) + +test("unrelated passing command or README contract cannot prove a named implementation target", () => { + const objective = goal("Objective achieved: implement src/slug.js") + objective.requirements.push({ + id: "check-true", text: "Verification command passes: true", + verification: "command", command: "true", required: true, status: "proven", + evidenceIDs: ["host-true"], updatedAt: Date.now(), + }) + objective.requirements.push({ + id: "file-readme", text: "File README.md exists", + verification: "file", file: "README.md", required: true, status: "proven", + evidenceIDs: ["host-readme"], updatedAt: Date.now(), + }) + const records = [ + { id: "host-true", kind: "command", trust: "host", passed: true, + goalRevision: 1, source: "true", requirementIDs: ["check-true"] }, + { id: "host-readme", kind: "file", trust: "host", passed: true, + goalRevision: 1, source: "README.md", requirementIDs: ["file-readme"], + metadata: { exists: true, regularFile: true } }, + ] + const claimed = [{ + requirementID: "req-process", verdict: "proven", reason: "unrelated checks passed", + evidence: [], hostEvidenceIDs: records.map((x) => x.id), + }] + assert.equal(guardSemanticProcessResults(objective, claimed, records)[0].verdict, "unknown") + assert.equal(guardSemanticProcessResults(objective, [{ ...claimed[0], + evidence: [{ path: "src/slug.js", quote: "export function slugify" }], + }], records)[0].verdict, "proven") +}) + +test("named deletion can use host-observed absence but not a stale or recreated file", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "goal-delete-proof-")) + const objective = createGoal({ sessionID: "deletion-regression", objective: "delete obsolete.txt" }) + const semantic = objective.requirements.find((x) => x.verification === "semantic") + assert.ok(semantic) + try { + const absent = await collectHostAbsenceEvidence(root, objective) + assert.equal(absent.length, 1) + assert.equal(absent[0].source, "obsolete.txt") + assert.equal(absent[0].metadata.absent, true) + const raw = [{ + requirementID: semantic.id, verdict: "proven", + reason: "Host independently checked the named file is absent.", + evidence: [], hostEvidenceIDs: [absent[0].id], + }] + const corroborated = await corroborateSemanticVerifierEvidence(root, objective, raw, absent) + assert.equal(guardSemanticProcessResults(objective, corroborated, absent)[0].verdict, "proven") + await writeFile(path.join(root, "obsolete.txt"), "recreated") + await assert.rejects( + () => corroborateSemanticVerifierEvidence(root, objective, raw, absent), + /no longer absent/, + ) + assert.deepEqual(await collectHostAbsenceEvidence(root, objective), []) + } finally { + await rm(root, { recursive: true, force: true }) + } +}) + test("does not reinterpret within-N-turn budget language as required cadence", () => { assert.equal(inferProcessTurnExpectation("Objective achieved: finish this within 10 Goal turns"), undefined) }) diff --git a/test/verifier-timeout.test.mjs b/test/verifier-timeout.test.mjs index b4590a9f..383e0a1f 100644 --- a/test/verifier-timeout.test.mjs +++ b/test/verifier-timeout.test.mjs @@ -74,7 +74,7 @@ test("hung semantic verifier stays bounded across one fresh timeout retry", asyn assert.equal(deleted, 2, "each verifier child must be cleaned up") }) -test("promptAsync dispatch waits for verifier tool submission without blocking on session.prompt", async () => { +test("promptAsync dispatch waits for verifier tool submission without treating runtime-only claims as completed work", async () => { let runtime let promptCalls = 0 let promptAsyncCalls = 0 @@ -97,7 +97,7 @@ test("promptAsync dispatch waits for verifier tool submission without blocking o results: request.requirements.map((requirement) => ({ requirementID: requirement.id, verdict: "proven", - reason: "Host runtime evidence establishes the requested test verdict.", + reason: "Uncorroborated runtime turn count does not establish implementation.", evidence: [], hostEvidenceIDs: ["goal-runtime-turns-r1"], })), @@ -117,7 +117,7 @@ test("promptAsync dispatch waits for verifier tool submission without blocking o runtime = createSemanticVerifierRuntime(client, process.cwd(), { timeoutMs: 250 }) const result = await runtime.verify("parent", semanticGoal(), { currentMessageID: "executor-current" }) - assert.equal(result.requirements[0].status, "proven") + assert.equal(result.requirements[0].status, "unknown", "transport success cannot authorize general completion using only runtime counters") assert.equal(promptAsyncCalls, 1) assert.equal(promptCalls, 0) assert.equal(deleted, 1) diff --git a/test/verifier.test.mjs b/test/verifier.test.mjs index 76e42fa4..b7b8a071 100644 --- a/test/verifier.test.mjs +++ b/test/verifier.test.mjs @@ -298,7 +298,12 @@ test("semantic verifier can reference fresh host file evidence", async () => { const text = body.parts[0].text const request = verificationRequest(text) const ids = hostEvidenceIDs(text) - assert.ok(ids.length > 0, "declared file contract should be host-verified before semantic audit") + // Runtime turn counters are also host-passing evidence, but they + // cannot prove that README contains the required product text. + // Select the current file/command contract evidence, not ids[0] + // (which is the synthetic goal-runtime-turns-rN record). + const fileContractID = ids.find((id) => !id.startsWith("goal-runtime-")) + assert.ok(fileContractID, "declared README content contract must have current host file proof") const result = await hooks.tool.opencode_goal_verifier_result.execute({ auditToken: request.auditToken, results: request.requirements.map((requirement) => ({ @@ -306,7 +311,7 @@ test("semantic verifier can reference fresh host file evidence", async () => { verdict: "proven", reason: "The host-verified README contract establishes the requested docs state.", evidence: [], - hostEvidenceIDs: [ids[0]], + hostEvidenceIDs: [fileContractID], })), }, { sessionID: childID, messageID: "verifier-message", agent: DEFAULT_VERIFIER_AGENT }) assert.equal(result, "Semantic verifier result accepted.")