fix(evals): bind paid artifacts and preserve command explanations

evals/README.md

@@ -22,6 +22,20 @@ bun run eval -- --model openai/gpt-5.6-sol --model opencode/claude-opus-5
 bun run eval -- --scenario happy-path --repeat 3
 ```
 
+To bind a campaign to a reviewed package, supply both
+`--expected-tarball-sha256 sha256:<64 lowercase hex digits>` and
+`--expected-manifest-sha256 sha256:<64 lowercase hex digits>`. The runner compares
+these values against its actual packed archive before cache installation,
+credential copying, model probes or workflow dispatches. A raw archive mismatch
+fails even when the complete unpacked content manifest matches. Missing, repeated,
+malformed or unpaired hash options fail before building. Both options also accept
+`--option=value` syntax. Runs without either option retain their existing behavior.
+
+For a paid release qualification, use the reviewed raw archive and full manifest
+hashes in the launch command. Keep the build process's reviewed file mask. Protect
+private logs through individual exclusive files with mode `0600`, rather than
+changing the process file mask for both logging and package creation.
+
 Ids are `providerID/modelID` as the host resolves them, which depends on which
 providers you have authenticated — Opus 5 may be `opencode/claude-opus-5` rather
 than `anthropic/claude-opus-5`. Only the first slash separates the two halves, so

evals/delivery-presentation.ts

@@ -534,6 +534,64 @@ function proseGateClause(line: string) {
 	}
 	return null;
 }
+const BEHAVIOR_PROCESSING_HEADS: ReadonlySet<string> = new Set([
+	"handling",
+	"trimming",
+	"parsing",
+	"formatting",
+	"normalization",
+	"validation",
+]);
+function behaviorComplementValue(text: string): boolean {
+	return text.split(/\band\b/i).every((phrase) => {
+		const words = phrase.trim().split(/\s+/);
+		const head = words.at(-1)?.toLowerCase();
+		return head !== undefined && BEHAVIOR_PROCESSING_HEADS.has(head);
+	});
+}
+type CommandAdjunct =
+	| { kind: "explanation"; status: string; text: string }
+	| { kind: "qualifier"; text: string };
+function commandAdjunctValue(text: string): CommandAdjunct {
+	let quote: string | null = null;
+	for (let index = 0; index < text.length; index++) {
+		const character = text[index];
+		if (character === "\\") {
+			index++;
+			continue;
+		}
+		if (quote) {
+			if (character === quote) quote = null;
+			continue;
+		}
+		if (character === '"' || character === "'") {
+			quote = character;
+			continue;
+		}
+		if (character !== ",") continue;
+		const match = /^,\s+(?:confirming|verifying|demonstrating)\s+(.+)$/i.exec(
+			text.slice(index),
+		);
+		if (!match) continue;
+		const complement = match[1] ?? "";
+		if (
+			!behaviorComplementValue(complement) ||
+			!/^[\p{L}\p{N}]+(?:[-'][\p{L}\p{N}]+)*(?:\s+[\p{L}\p{N}]+(?:[-'][\p{L}\p{N}]+)*)*$/u.test(
+				complement,
+			) ||
+			/\b(?:commands?|observations?|scripts?|invocations?|hosts?|platforms?|Linux|Windows|macOS|darwin|win32|outputs?|sources?|reports?|reviews?|authorit(?:y|ies)|authoriz(?:e(?:d|s)?|ations?|ing)|completions?|complete|completed|assurances?|proofs?|proven|validated|verified|met|approv(?:e(?:d|s)?|als?|ing)|accept(?:ed|s|ing)?|permissions?|ready|evidences?|pass(?:ed|es|ing)?|succeed(?:ed|s|ing)?|success(?:es|ful(?:ly)?)?|fail(?:ed|s|ing|ures?)?|exit(?:ed|s)?|ran|finished|skipped|partial|truncated|unproven|unverified|unobserved|missing|rewritten|edit(?:ed|s|ing)?|replac(?:e(?:d|s)?|ing)|bypass(?:ed|es|ing)?|disabl(?:e(?:d|s)?|ing)|unavailable|incomplete|chang(?:e(?:d|s)?|ing)|unchanged|modif(?:y|ied|ies|ying)|grant(?:ed|s|ing)?|not|no|without|despite|but|if|unless|would|could|should|is|was|are|were|has|have|had|does|did)\b/i.test(
+				complement,
+			)
+		)
+			return { kind: "qualifier", text };
+		return {
+			kind: "explanation",
+			status: text.slice(0, index).trim(),
+			text: complement,
+		};
+	}
+	return { kind: "qualifier", text };
+}
 function parseCommandResult(
 	rawBody: string,
 	command: string,
@@ -552,7 +610,8 @@ function parseCommandResult(
 	const parts = body
 		.split(/;|\.\s+(?=[A-Za-z])/)
 		.map((part) => part.trim().replace(/\.$/, ""));
-	const status = parts.shift() ?? "";
+	const adjunct = commandAdjunctValue(parts.shift() ?? "");
+	const status = adjunct.kind === "explanation" ? adjunct.status : adjunct.text;
 	const value =
 		/^(?:(passed)(?:,\s*| with )|(recorded as an observation),\s*)?(?:exited|exit(?: code)?)\s+(-?\d+|unavailable)(.*)$/i.exec(
 			status,

evals/run.ts

@@ -94,6 +94,7 @@ import {
 	inspectArtifact,
 	instructionDelivery,
 	normalizeRequestedModel,
+	type PackedArtifactIdentity,
 	redactTranscript,
 	tarballSha256,
 } from "./provenance.js";
@@ -556,9 +557,15 @@ type Recorded = {
 	readonly cassette: Cassette | null;
 };
 
+type ExpectedArtifactIdentity = Pick<
+	PackedArtifactIdentity,
+	"tarballSha256" | "unpackedManifestSha256"
+>;
+
 function parseArgs(argv: string[]) {
 	const models: string[] = [];
 	const scenarios: string[] = [];
+	const expectedHashes: Partial<ExpectedArtifactIdentity> = {};
 	let repeat = 1;
 	const release = argv.includes("--release");
 	let concurrency = 0;
@@ -578,6 +585,35 @@ function parseArgs(argv: string[]) {
 	for (let index = 0; index < argv.length; index += 1) {
 		const flag = argv[index] ?? "";
 		const value = argv[index + 1];
+		const expectedFlag = flag.split("=", 1)[0];
+		if (
+			expectedFlag === "--expected-tarball-sha256" ||
+			expectedFlag === "--expected-manifest-sha256"
+		) {
+			const inline = flag.includes("=");
+			const digest = inline ? flag.slice(expectedFlag.length + 1) : value;
+			if (!digest || digest.startsWith("--")) {
+				console.error(`${expectedFlag} requires a value.`);
+				process.exit(2);
+			}
+			const key =
+				expectedFlag === "--expected-tarball-sha256"
+					? "tarballSha256"
+					: "unpackedManifestSha256";
+			if (expectedHashes[key] !== undefined) {
+				console.error(`${expectedFlag} may only be supplied once.`);
+				process.exit(2);
+			}
+			if (digest.length !== 71 || !/^sha256:[a-f0-9]{64}$/.test(digest)) {
+				console.error(
+					`${expectedFlag} requires a lowercase SHA-256 in sha256:<64 hex digits> form.`,
+				);
+				process.exit(2);
+			}
+			expectedHashes[key] = digest;
+			if (!inline) index += 1;
+			continue;
+		}
 		if (
 			["--model", "--scenario", "--repeat", "--concurrency"].includes(flag) &&
 			(!value || value.startsWith("--"))
@@ -599,11 +635,25 @@ function parseArgs(argv: string[]) {
 			index += 1;
 		} else if (flag === "--help" || flag === "-h") {
 			console.log(
-				"usage: bun run eval -- --model <provider/model> [--model ...] [--scenario <id> --repeat <n> | --release] [--concurrency <n>]",
+				"usage: bun run eval -- --model <provider/model> [--model ...] [--scenario <id> --repeat <n> | --release] [--concurrency <n>] [--expected-tarball-sha256 <sha256:hex> --expected-manifest-sha256 <sha256:hex>]",
 			);
 			process.exit(0);
 		}
 	}
+	const { tarballSha256, unpackedManifestSha256 } = expectedHashes;
+	if (
+		(tarballSha256 === undefined) !==
+		(unpackedManifestSha256 === undefined)
+	) {
+		console.error(
+			"--expected-tarball-sha256 and --expected-manifest-sha256 must be supplied together.",
+		);
+		process.exit(2);
+	}
+	const expectedArtifact: ExpectedArtifactIdentity | null =
+		tarballSha256 !== undefined && unpackedManifestSha256 !== undefined
+			? { tarballSha256, unpackedManifestSha256 }
+			: null;
 	if (models.length === 0) {
 		const fromEnv = process.env.FLOW_EVAL_MODEL?.trim();
 		if (fromEnv)
@@ -664,7 +714,13 @@ function parseArgs(argv: string[]) {
 	const sampling: EvalSampling = release
 		? { kind: "release" }
 		: { kind: "ordinary", repeat };
-	return { models, scenarios, sampling, concurrency: workers };
+	return {
+		models,
+		scenarios,
+		sampling,
+		concurrency: workers,
+		expectedArtifact,
+	};
 }
 
 /** Bytes of prompt text this build ships, per surface and in total. */
@@ -816,7 +872,8 @@ export async function runCampaign(
 	repositoryRoot = join(import.meta.dir, ".."),
 	beginFinalization: () => void = () => {},
 ): Promise<number> {
-	const { models, scenarios, sampling, concurrency } = parseArgs(args);
+	const { models, scenarios, sampling, concurrency, expectedArtifact } =
+		parseArgs(args);
 	if (import.meta.main) await requirePaidAuthorization();
 	const selected =
 		sampling.kind === "release"
@@ -905,6 +962,16 @@ export async function runCampaign(
 			repositoryRoot,
 			tarballPath: tarball,
 		});
+		if (
+			expectedArtifact &&
+			(artifact.tarballSha256 !== expectedArtifact.tarballSha256 ||
+				artifact.unpackedManifestSha256 !==
+					expectedArtifact.unpackedManifestSha256)
+		) {
+			throw new Error(
+				`Artifact identity mismatch. Expected tarball ${expectedArtifact.tarballSha256} and manifest ${expectedArtifact.unpackedManifestSha256}; observed tarball ${artifact.tarballSha256} and manifest ${artifact.unpackedManifestSha256}. No model probe or workflow was started.`,
+			);
+		}
 		await persistEvaluation("artifact", () =>
 			reportStore.writeArtifact(tarball),
 		);