fix(goal): reject runtime-only proof for ordinary completion claims (#281)

src/opencode/verifier.ts

@@ -1,5 +1,5 @@
-import { DEFAULT_VERIFIER_TIMEOUT_MS, SemanticVerifierUnavailableError, corroborateSemanticVerifierEvidence, semanticVerificationPrompt, semanticVerifierHostEvidence } from "../verification/verifier-evidence.js"
-export { DEFAULT_VERIFIER_TIMEOUT_MS, SemanticVerifierUnavailableError, corroborateSemanticVerifierEvidence, semanticVerificationPrompt, semanticVerifierHostEvidence } from "../verification/verifier-evidence.js"
+import { DEFAULT_VERIFIER_TIMEOUT_MS, SemanticVerifierUnavailableError, collectHostAbsenceEvidence, corroborateSemanticVerifierEvidence, semanticVerificationPrompt, semanticVerifierHostEvidence } from "../verification/verifier-evidence.js"
+export { DEFAULT_VERIFIER_TIMEOUT_MS, SemanticVerifierUnavailableError, collectHostAbsenceEvidence, corroborateSemanticVerifierEvidence, semanticVerificationPrompt, semanticVerifierHostEvidence } from "../verification/verifier-evidence.js"
 import { randomUUID } from "node:crypto"
 import { tool } from "@opencode-ai/plugin/tool"
 import type { EvidenceRecord, GoalState } from "../domain/types.js"
@@ -190,7 +190,10 @@ export function createSemanticVerifierRuntime(client: any, root: string, options
       : timeoutMs
     const allowTimeoutRetry = verifyOptions.allowTimeoutRetry !== false
     const auditToken = randomUUID()
-    const hostEvidenceRecords = semanticVerifierHostEvidence(goal, verifyOptions.currentMessageID)
+    const hostEvidenceRecords = [
+      ...semanticVerifierHostEvidence(goal, verifyOptions.currentMessageID),
+      ...await collectHostAbsenceEvidence(root, goal),
+    ]
     let childID = ""
     let retryAfterTimeout = false
     try {

src/opencode2/semantic-verifier.ts

@@ -9,6 +9,7 @@ import {
 import {
   DEFAULT_VERIFIER_TIMEOUT_MS,
   SemanticVerifierUnavailableError,
+  collectHostAbsenceEvidence,
   corroborateSemanticVerifierEvidence,
   semanticVerificationPrompt,
   semanticVerifierHostEvidence,
@@ -335,7 +336,10 @@ export function createOpenCode2SemanticVerifierRuntime(
     const allowTimeoutRetry = verifyOptions.allowTimeoutRetry !== false
     const auditToken = randomUUID()
     const root = await resolveRoot(parentSessionID)
-    const hostEvidenceRecords = semanticVerifierHostEvidence(goal, verifyOptions.currentMessageID)
+    const hostEvidenceRecords = [
+      ...semanticVerifierHostEvidence(goal, verifyOptions.currentMessageID),
+      ...await collectHostAbsenceEvidence(root, goal),
+    ]
     let childID = ""
     let retryAfterTimeout = false
     let primaryTimeoutMessage: string | undefined

src/verification/process.ts

@@ -1,5 +1,6 @@
 import type { EvidenceRecord, GoalState } from "../domain/types.js"
 import type { SemanticRequirementResult } from "./semantic.js"
+import { explicitFileTargets, requestsDeletion, needsConcreteDeliverable } from "./semantic-targets.js"
 
 export interface ProcessTurnExpectation {
   turns: number
@@ -73,6 +74,97 @@ function runtimeMetric(records: EvidenceRecord[], key: "turns" | "mutations"): n
   return undefined
 }
 
+/**
+ * Runtime turn/progress counters are authoritative for an explicit process
+ * contract (e.g. N distinct Goal turns), but cannot certify a generic
+ * deliverable such as "implement the feature" or "create the file".
+ *
+ * By this point verifier file quotes have been host-corroborated. Only
+ * literal corroborated files or an explicitly cited, current passing
+ * file/command host record may establish a non-process claim. The same
+ * verified file record can support both a file requirement and the broader
+ * semantic objective; requiring identical requirement IDs would break
+ * legitimate shared evidence. Runtime counters never suffice.
+ */
+function normalizedPath(value: string | undefined): string {
+  const normalized = String(value ?? "").replace(/\\/g, "/").replace(/^\.\/+/, "")
+  return process.platform === "win32" ? normalized.toLowerCase() : normalized
+}
+
+function hasConcreteResultEvidence(
+  goal: GoalState,
+  requirementText: string,
+  result: SemanticRequirementResult,
+  records: EvidenceRecord[],
+): boolean {
+  const targets = explicitFileTargets(requirementText).map(normalizedPath)
+  const deletionOnly = requestsDeletion(requirementText)
+    && !/\b(?:implement|create|update|write|build|fix|add|refactor)\b/i.test(requirementText)
+  const quotedPaths = new Set(result.evidence
+    .filter((item) => item.path?.trim() && item.quote?.trim())
+    .map((item) => normalizedPath(item.path)))
+  const ids = new Set(result.hostEvidenceIDs)
+  const current = records.filter((item) =>
+    ids.has(item.id)
+    && item.goalRevision === goal.revision
+    && item.trust === "host"
+    && item.passed === true,
+  )
+  // Merely citing a passing host check is not enough. Bind it to its
+  // structured file/command contract; runtime counters have neither.
+  const provedFiles = current.filter((item) => {
+    if (item.kind !== "file" || !item.source || item.metadata?.absent === true) return false
+    return goal.requirements.some((requirement) =>
+      requirement.verification === "file"
+      && requirement.file
+      && normalizedPath(requirement.file) === normalizedPath(item.source)
+      && item.requirementIDs.includes(requirement.id)
+      && requirement.status === "proven"
+      && item.metadata?.exists !== false
+      && item.metadata?.regularFile !== false,
+    )
+  })
+  const absence = current.filter((item) =>
+    item.kind === "file"
+    && item.metadata?.absent === true
+    && item.requirementIDs.includes(result.requirementID)
+    && item.source,
+  )
+
+  // Named deliverables require proof of THOSE paths, not just any README
+  // contract, a generic npm test, or a host runtime/turn count.
+  if (targets.length > 0) {
+    return targets.every((target) =>
+      deletionOnly
+        ? absence.some((item) => normalizedPath(item.source) === target)
+        : quotedPaths.has(target) || provedFiles.some((item) => normalizedPath(item.source) === target),
+    )
+  }
+
+  // Unnamed deletion has no inspectable negative target; do not infer it from
+  // a command or an unrelated file. Explicit named deletions use host absence.
+  if (deletionOnly) return false
+  if (quotedPaths.size > 0 || provedFiles.length > 0) return true
+
+  // A direct check can prove a test/lint/build-only objective if the passed
+  // command actually implements the named verification. "true" cannot prove
+  // an implementation, and a test pass alone cannot prove a named artifact.
+  const testOnly = /\b(?:tests?|lint|typecheck|checks?|build|compile|ci)\b/i.test(requirementText)
+    && !/\b(?:implement|create|delete|remove|update|write|modify|fix|refactor)\b/i.test(requirementText)
+  if (!testOnly) return false
+  return current.some((item) =>
+    item.kind === "command"
+    && item.source
+    && /\b(?:test|lint|check|build|compile|tsc)\b/i.test(item.source)
+    && goal.requirements.some((requirement) =>
+      requirement.verification === "command"
+      && requirement.status === "proven"
+      && requirement.command === item.source
+      && item.requirementIDs.includes(requirement.id),
+    ),
+  )
+}
+
 function guarded(result: SemanticRequirementResult, verdict: "failed" | "unknown", reason: string): SemanticRequirementResult {
   return {
     ...result,
@@ -107,6 +199,12 @@ export function guardSemanticProcessResults(
     const requirement = requirements.get(result.requirementID)
     if (!requirement || requirement.verification !== "semantic") return result
     const expectation = inferProcessTurnExpectation(requirement.text)
+    // A turn requirement can coexist with an actual deliverable. Proving
+    // cadence alone must not silently prove an implementation or deleted file.
+    const needsArtifact = !expectation || needsConcreteDeliverable(requirement.text, expectation.requireMutationPerTurn)
+    if (needsArtifact && !hasConcreteResultEvidence(goal, requirement.text, result, hostEvidenceRecords)) {
+      return guarded(result, "unknown", "non-process deliverable has only runtime counters, unrelated checks or unsupported evidence; real target-file, host absence, or directly linked verification evidence is required.")
+    }
     if (!expectation) return result
 
     if (expectation.requireMutationPerTurn) {

src/verification/semantic-targets.ts

@@ -0,0 +1,27 @@
+// Conservative helpers for deciding what a *semantic* Goal actually asks to
+// deliver. These are guardrails, not substitutes for the independent verifier.
+
+const FILE_TOKEN = /\b(?:[a-zA-Z0-9_@.-]+[\\/])*[a-zA-Z0-9_@.-]+\.[a-zA-Z][a-zA-Z0-9]{0,8}\b/g
+
+export function explicitFileTargets(text: string): string[] {
+  return [...new Set((text.match(FILE_TOKEN) ?? []).map((value) => value.replace(/\\/g, "/")))]
+}
+
+export function requestsDeletion(text: string): boolean {
+  return /\b(?:delete|deleting|remove|removing|unlink|erase)\b|\bsil(?:mek)?\b|\bkald[ıi]r\b/i.test(text)
+}
+
+/**
+ * "Exactly N Goal turns" is a process contract.
+ * "Implement X across N Goal turns" is a mixed contract: the count proves
+ * only the process part, and real project evidence must prove X separately.
+ *
+ * Pure per-turn mutation instructions may be proven by host-owned mutation
+ * fingerprints; otherwise an explicit create/write/update deliverable still
+ * needs corroborated artifacts.
+ */
+export function needsConcreteDeliverable(text: string, perTurnMutation: boolean): boolean {
+  if (/\b(?:implement|create|delete|deleting|remove|removing|ship|deliver|build|fix|refactor|publish|document)\b/i.test(text)) return true
+  if (!perTurnMutation && /\b(?:write|update|modify|edit|add)\b/i.test(text)) return true
+  return false
+}

src/verification/verifier-evidence.ts

@@ -3,6 +3,7 @@ import { promises as fs } from "node:fs"
 import path from "node:path"
 import type { EvidenceRecord, GoalState } from "../domain/types.js"
 import type { SemanticEvidenceRef, SemanticRequirementResult } from "./semantic.js"
+import { explicitFileTargets, requestsDeletion } from "./semantic-targets.js"
 
 export const DEFAULT_VERIFIER_TIMEOUT_MS = 60_000
 
@@ -57,6 +58,60 @@ export function semanticVerifierHostEvidence(goal: GoalState, currentMessageID?:
   return [...runtime, ...persisted]
 }
 
+
+/**
+ * Verify a negative filesystem assertion without trusting a model or a stale
+ * diff claim. A symlink/junction in the path is never accepted as absence.
+ * Missing ancestors count as absence of the target; permissions/errors do not.
+ */
+async function hostVerifiedAbsence(root: string, candidate: string): Promise<boolean> {
+  const absolute = resolveInside(root, candidate)
+  const relative = path.relative(path.resolve(root), absolute)
+  let cursor = path.resolve(root)
+  for (const segment of relative.split(path.sep).filter(Boolean)) {
+    cursor = path.join(cursor, segment)
+    try {
+      const info = await fs.lstat(cursor)
+      if (info.isSymbolicLink()) return false
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code === "ENOENT") return true
+      throw error
+    }
+  }
+  return false
+}
+
+/**
+ * The verifier cannot quote a file that was deleted. Supply explicit host-
+ * observed absence evidence only for named paths in delete/remove objectives.
+ * The evidence is rechecked after the verifier returns to close race windows.
+ */
+export async function collectHostAbsenceEvidence(root: string, goal: GoalState): Promise<EvidenceRecord[]> {
+  const observations: EvidenceRecord[] = []
+  for (const requirement of goal.requirements) {
+    if (!requirement.required || requirement.verification !== "semantic" || !requestsDeletion(requirement.text)) continue
+    for (const file of explicitFileTargets(requirement.text)) {
+      let absent = false
+      try { absent = await hostVerifiedAbsence(root, file) } catch { /* fail closed */ }
+      if (!absent) continue
+      const key = createHash("sha256").update([goal.id, goal.revision, requirement.id, file].join("\0")).digest("hex").slice(0, 18)
+      observations.push({
+        id: "goal-absent-r" + goal.revision + "-" + key,
+        kind: "file",
+        trust: "host",
+        summary: "Host verified the requested removal target " + file + " is absent from the current workspace (symlinks are not accepted).",
+        createdAt: Date.now(),
+        goalRevision: goal.revision,
+        requirementIDs: [requirement.id],
+        source: file,
+        passed: true,
+        metadata: { absent: true, exists: false },
+      })
+    }
+  }
+  return observations
+}
+
 export function semanticVerificationPrompt(goal: GoalState, auditToken: string, hostEvidenceRecords: EvidenceRecord[]): string {
   const semantic = goal.requirements.filter((item) => item.required && item.verification === "semantic")
   const hostEvidence = hostEvidenceRecords
@@ -69,7 +124,7 @@ export function semanticVerificationPrompt(goal: GoalState, auditToken: string,
     objective: goal.objective,
     requirements: semantic.map((item) => ({ id: item.id, text: item.text })),
   }, null, 2)
-  return `Independently audit the semantic requirements for an OpenCode goal.\n\nThe goal executor's claims are not proof. Inspect the current workspace yourself using only read/search tools. Never edit files, run shell commands, delegate tasks, or mutate goal state. Preserve the full requested scope.\n\nVerdicts:\n- proven: current authoritative workspace evidence directly establishes the requirement. Support proven verdicts with exact file excerpts as {path, quote} and/or IDs of current passing host evidence. The host will independently re-read file quotes and validate every host-evidence ID.\n- failed: current evidence directly contradicts the requirement.\n- unknown: evidence is missing, indirect, ambiguous, external, or would require executing a command you cannot run.\n\nPrefer a current passing host-evidence ID when it directly establishes the requirement. When citing a file, use a path relative to the workspace root (for example docs/file.md) or an absolute path with its leading root/drive intact; never strip the leading slash or drive prefix. Quote only literal text that exists inside that file. Never copy read-tool line numbers, path headers, XML/Markdown wrappers, labels, or explanatory prose into the quote.\n\nDo not treat a vague statement, plan, TODO, changelog claim, or unverified test claim as proof. Host-run verification evidence, when present below, may be used only for what it actually establishes. A path without an exact quote is not proof, and an unknown host-evidence ID is not proof. Temporal/process requirements such as doing an action across N distinct turns are not proven by a final file value alone: use the host runtime turn/progress evidence below and return unknown or failed when the requested cadence/count is not established.\n\nHost evidence:\n${hostEvidence}\n\nVerification request:\n${request}\n\nCall opencode_goal_verifier_result exactly once with the auditToken and one result for every listed requirement. Do not return a success verdict outside that tool.`
+  return `Independently audit the semantic requirements for an OpenCode goal.\n\nThe goal executor's claims are not proof. Inspect the current workspace yourself using only read/search tools. Never edit files, run shell commands, delegate tasks, or mutate goal state. Preserve the full requested scope.\n\nVerdicts:\n- proven: current authoritative workspace evidence directly establishes the requirement. Support proven verdicts with exact file excerpts as {path, quote} and/or IDs of current passing host evidence. The host will independently re-read file quotes and validate every host-evidence ID.\n- failed: current evidence directly contradicts the requirement.\n- unknown: evidence is missing, indirect, ambiguous, external, or would require executing a command you cannot run.\n\nPrefer a current passing host-evidence ID when it directly establishes the requirement. Deletion-only objectives may cite a host-verified missing-path record when listed below; runtime mutation counters cannot prove that a named file is absent. Host-verified absence will be rechecked after your verdict. Never cite unrelated passing checks or files as proof of a different deliverable.\n\nWhen citing a file, use a path relative to the workspace root (for example docs/file.md) or an absolute path with its leading root/drive intact; never strip the leading slash or drive prefix. Quote only literal text that exists inside that file. Never copy read-tool line numbers, path headers, XML/Markdown wrappers, labels, or explanatory prose into the quote.\n\nDo not treat a vague statement, plan, TODO, changelog claim, or unverified test claim as proof. Host-run verification evidence, when present below, may be used only for what it actually establishes. A path without an exact quote is not proof, and an unknown host-evidence ID is not proof. Temporal/process requirements such as doing an action across N distinct turns are not proven by a final file value alone: use the host runtime turn/progress evidence below and return unknown or failed when the requested cadence/count is not established.\n\nHost evidence:\n${hostEvidence}\n\nVerification request:\n${request}\n\nCall opencode_goal_verifier_result exactly once with the auditToken and one result for every listed requirement. Do not return a success verdict outside that tool.`
 }
 
 function resolveInside(root: string, candidate: string): string {
@@ -217,6 +272,14 @@ export async function corroborateSemanticVerifierEvidence(
       if (!hostEvidence || hostEvidence.goalRevision !== goal.revision || hostEvidence.trust !== "host" || hostEvidence.passed !== true) {
         throw new Error(`verifier referenced invalid or non-passing host evidence: ${id}`)
       }
+
+      // Missing-path evidence is a snapshot fact. If the file is recreated
+      // during verification, no earlier absence observation can authorize done.
+      if (hostEvidence.metadata?.absent === true) {
+        if (hostEvidence.kind !== "file" || !hostEvidence.source || !await hostVerifiedAbsence(root, hostEvidence.source)) {
+          throw new Error("host-verified deletion target is no longer absent: " + (hostEvidence.source || id))
+        }
+      }
     }
     if (result.verdict === "proven" && evidence.length === 0 && normalizedHostEvidenceIDs.length === 0) {
       throw new Error("proven semantic requirement lacks current corroborated evidence")