From 7dbac923afa65aacf726847f07beaf7fbb3223c9 Mon Sep 17 00:00:00 2001 From: bigboateng Date: Wed, 22 Jul 2026 17:10:44 +0100 Subject: [PATCH 1/2] eval(deltawire): integrate v5 infrastructure and evidence lock --- .../environment/.deltawire/config.json | 12 +++ .../schemas/range-large.schema.json | 16 ++++ .../environment/.generated/.gitignore | 2 + .../environment/Dockerfile | 10 +++ .../deltawire-environment-expectations.json | 23 +++++ .../environment/environment_receipt.py | 47 ++++++++++ .../environment/task-contract.json | 24 +++++ .../environment/verify_plan_contract.py | 28 ++++++ .../environment-receipt-v1/instruction.md | 1 + .../environment-receipt-v1/task.toml | 29 ++++++ .../environment-receipt-v1/tests/test.sh | 4 + .../eval/docs/08-harbor-artifact-timing.md | 29 ++++++ .../docs/09-gemini-agent-setup-boundary.md | 43 +++++++++ .../manifests/preflight-v4-evidence-lock.json | 90 +++++++++++++++++++ .../manifests/preflight-v5-evidence-lock.json | 33 +++++++ .../results/preflight-v5/classification.json | 15 ++++ .../conformance/environment-receipt-v1.json | 79 ++++++++++++++++ .../eval/results/preflight-v5/readiness.json | 9 ++ .../eval/results/preflight-v5/report.md | 18 ++++ .../D1-artifact-manifest-receipt.json | 28 ++++++ .../D1-end-to-end.json | 22 +++++ .../D1-environment-receipt.json | 55 ++++++++++++ .../D1-plan-contract-receipt.json | 6 ++ .../D1-semantic-result.json | 6 ++ .../D1-treatment-use.json | 11 +++ .../eval/scripts/v5/artifact_manifest.py | 31 +++++++ .../eval/scripts/v5/environment_receipt.py | 47 ++++++++++ .../scripts/v5/generate_v5_evidence_lock.py | 37 ++++++++ .../eval/scripts/v5/negative_controls.py | 38 ++++++++ .../eval/scripts/v5/run_conformance.py | 48 ++++++++++ .../eval/scripts/v5/test_artifact_manifest.py | 27 ++++++ .../scripts/v5/test_environment_receipt.py | 35 ++++++++ .../eval/scripts/v5/validate_integration.py | 46 ++++++++++ .../eval/scripts/v5/verify_evidence_locks.py | 32 +++++++ .../eval/scripts/validate_dataset.sh | 4 + 35 files changed, 985 insertions(+) create mode 100644 labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/config.json create mode 100644 labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/schemas/range-large.schema.json create mode 100644 labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.generated/.gitignore create mode 100644 labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/Dockerfile create mode 100644 labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/deltawire-environment-expectations.json create mode 100644 labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/environment_receipt.py create mode 100644 labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/task-contract.json create mode 100644 labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/verify_plan_contract.py create mode 100644 labs/20-deltawire/eval/conformance/environment-receipt-v1/instruction.md create mode 100644 labs/20-deltawire/eval/conformance/environment-receipt-v1/task.toml create mode 100755 labs/20-deltawire/eval/conformance/environment-receipt-v1/tests/test.sh create mode 100644 labs/20-deltawire/eval/docs/08-harbor-artifact-timing.md create mode 100644 labs/20-deltawire/eval/docs/09-gemini-agent-setup-boundary.md create mode 100644 labs/20-deltawire/eval/manifests/preflight-v4-evidence-lock.json create mode 100644 labs/20-deltawire/eval/manifests/preflight-v5-evidence-lock.json create mode 100644 labs/20-deltawire/eval/results/preflight-v5/classification.json create mode 100644 labs/20-deltawire/eval/results/preflight-v5/conformance/environment-receipt-v1.json create mode 100644 labs/20-deltawire/eval/results/preflight-v5/readiness.json create mode 100644 labs/20-deltawire/eval/results/preflight-v5/report.md create mode 100644 labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-artifact-manifest-receipt.json create mode 100644 labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-end-to-end.json create mode 100644 labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-environment-receipt.json create mode 100644 labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-plan-contract-receipt.json create mode 100644 labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-semantic-result.json create mode 100644 labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-treatment-use.json create mode 100644 labs/20-deltawire/eval/scripts/v5/artifact_manifest.py create mode 100644 labs/20-deltawire/eval/scripts/v5/environment_receipt.py create mode 100644 labs/20-deltawire/eval/scripts/v5/generate_v5_evidence_lock.py create mode 100644 labs/20-deltawire/eval/scripts/v5/negative_controls.py create mode 100644 labs/20-deltawire/eval/scripts/v5/run_conformance.py create mode 100644 labs/20-deltawire/eval/scripts/v5/test_artifact_manifest.py create mode 100644 labs/20-deltawire/eval/scripts/v5/test_environment_receipt.py create mode 100644 labs/20-deltawire/eval/scripts/v5/validate_integration.py create mode 100644 labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/config.json b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/config.json new file mode 100644 index 000000000..61a05415a --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/config.json @@ -0,0 +1,12 @@ +{ + "limits": { + "max_output_bytes": 104857600, + "max_plan_bytes": 1048576, + "max_records": 100000, + "max_schema_bytes": 1048576 + }, + "plans_dir": ".deltawire/plans", + "schemas_dir": ".deltawire/schemas", + "state_file": ".deltawire/state.json", + "version": "deltawire.config.v1" +} diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/schemas/range-large.schema.json b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/schemas/range-large.schema.json new file mode 100644 index 000000000..69b476237 --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/schemas/range-large.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "index": { + "maximum": 500, + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "index" + ], + "title": "range-large", + "type": "object" +} diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.generated/.gitignore b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.generated/.gitignore new file mode 100644 index 000000000..d6b7ef32c --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.generated/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/Dockerfile b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/Dockerfile new file mode 100644 index 000000000..023396b14 --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:22.04 +COPY .generated/deltawire /usr/local/bin/deltawire +COPY task-contract.json /task-contract.json +COPY .deltawire /.deltawire +COPY verify_plan_contract.py /usr/local/bin/verify_plan_contract.py +COPY environment_receipt.py /usr/local/bin/environment_receipt.py +COPY deltawire-environment-expectations.json /deltawire-environment-expectations.json +RUN chmod 0755 /usr/local/bin/deltawire /usr/local/bin/verify_plan_contract.py /usr/local/bin/environment_receipt.py && chmod a-w /task-contract.json /.deltawire/config.json /.deltawire/schemas/*.json /deltawire-environment-expectations.json +RUN command -v deltawire && deltawire version +RUN apt-get update && apt-get install -y python3 diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/deltawire-environment-expectations.json b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/deltawire-environment-expectations.json new file mode 100644 index 000000000..a47f4e360 --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/deltawire-environment-expectations.json @@ -0,0 +1,23 @@ +{ + "binary": { + "path": "/usr/local/bin/deltawire", + "realpath": "/usr/local/bin/deltawire", + "sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "version": "deltawire version dev" + }, + "files": { + "config": { + "path": "/.deltawire/config.json", + "sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f" + }, + "public_contract": { + "path": "/task-contract.json", + "sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5" + }, + "schema": { + "path": "/.deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + } + }, + "schema_version": "deltawire-environment-expectations.v1" +} diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/environment_receipt.py b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/environment_receipt.py new file mode 100644 index 000000000..84446c2d5 --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/environment_receipt.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Atomically emit deterministic DeltaWire environment evidence.""" +import argparse,hashlib,json,os,stat,subprocess,tempfile +from pathlib import Path + +def sha(path): + h=hashlib.sha256() + with Path(path).open("rb") as f: + for chunk in iter(lambda:f.read(1048576),b""):h.update(chunk) + return h.hexdigest() +def atomic_write(path,data): + target=Path(path);target.parent.mkdir(parents=True,exist_ok=True) + fd,tmp=tempfile.mkstemp(prefix=f".{target.name}.",suffix=".tmp",dir=target.parent) + try: + with os.fdopen(fd,"w",encoding="utf-8") as f:f.write(data);f.flush();os.fsync(f.fileno()) + os.replace(tmp,target) + except BaseException: + try:os.unlink(tmp) + except FileNotFoundError:pass + raise +def build(expectations): + expected=json.loads(Path(expectations).read_text());binary=Path(expected["binary"]["path"]) + exists=binary.exists();lst=binary.lstat() if exists else None + regular=bool(lst and stat.S_ISREG(lst.st_mode));symlink=binary.is_symlink() if exists else False + executable=exists and os.access(binary,os.X_OK);realpath=str(binary.resolve()) if exists else None + try:version=subprocess.run([str(binary),"version"],capture_output=True,text=True,check=False) if exists else None + except OSError:version=None + files={} + for name,item in sorted(expected["files"].items()): + path=Path(item["path"]);actual=sha(path) if path.is_file() else None + files[name]={"path":item["path"],"exists":path.is_file(),"actual_sha256":actual,"expected_sha256":item["sha256"],"hash_match":actual==item["sha256"]} + binary_hash=sha(binary) if regular else None + checks={"binary_exists":exists,"binary_regular":regular,"binary_not_symlink":not symlink,"binary_executable":executable, + "binary_path":str(binary)==expected["binary"]["path"],"binary_realpath":realpath==expected["binary"]["realpath"], + "binary_hash":binary_hash==expected["binary"]["sha256"],"version_exit_0":bool(version and version.returncode==0), + "version_exact":bool(version and version.stdout.splitlines() and version.stdout.splitlines()[0].strip()==expected["binary"]["version"]), + **{f"{name}_hash":item["hash_match"] for name,item in files.items()}} + return {"schema_version":"deltawire-environment-receipt.v1","binary":{"path":str(binary),"realpath":realpath,"exists":exists, + "is_regular":regular,"is_symlink":symlink,"executable":executable,"actual_sha256":binary_hash, + "expected_sha256":expected["binary"]["sha256"],"version_stdout":version.stdout.strip() if version else None, + "version_stderr":version.stderr.strip() if version else None,"version_exit_code":version.returncode if version else None, + "expected_version":expected["binary"]["version"]},"files":files,"checks":checks,"status":"pass" if all(checks.values()) else "fail"} +def main(): + p=argparse.ArgumentParser();p.add_argument("--expectations",required=True);p.add_argument("--receipt",required=True);a=p.parse_args() + receipt=build(a.expectations);atomic_write(a.receipt,json.dumps(receipt,indent=2,sort_keys=True)+"\n") + print(json.dumps(receipt,sort_keys=True));raise SystemExit(0 if receipt["status"]=="pass" else 1) +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/task-contract.json b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/task-contract.json new file mode 100644 index 000000000..281ba55ac --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/task-contract.json @@ -0,0 +1,24 @@ +{ + "authoritative_schema": { + "path": ".deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + }, + "deltawire_applicability": "expected", + "family": "range", + "generation": { + "field": "index", + "field_order": [ + "index" + ], + "start": 1 + }, + "ordering": "canonical_generation_order", + "output": { + "format": "ndjson", + "path": "testdata/generated/output.ndjson" + }, + "record_count": 500, + "schema_version": "task-spec.v1", + "size": "large", + "task": "range-large" +} diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/verify_plan_contract.py b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/verify_plan_contract.py new file mode 100644 index 000000000..205d4fc8a --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/verify_plan_contract.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import argparse, hashlib, json, subprocess +from pathlib import Path + +def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() +def write(path, value): + if path: + target=Path(path); target.parent.mkdir(parents=True,exist_ok=True); target.write_text(json.dumps(value,indent=2,sort_keys=True)+"\n") + +def main(): + p=argparse.ArgumentParser(); p.add_argument("--contract",required=True); p.add_argument("--plan",required=True); p.add_argument("--repo",default="."); p.add_argument("--receipt"); p.add_argument("--deltawire",default="deltawire"); a=p.parse_args() + repo=Path(a.repo).resolve(); contract_path=Path(a.contract).resolve(); plan=Path(a.plan).resolve(); contract=json.loads(contract_path.read_text()) + try: plan_arg=str(plan.relative_to(repo)) + except ValueError: plan_arg=str(plan) + result={"schema_version":"plan-contract-receipt.v1","contract_sha256":sha(contract_path),"plan_sha256":sha(plan),"checks":{},"status":"fail"} + try: + run=subprocess.run([a.deltawire,"inspect","--repo",str(repo),"--format","json",plan_arg],capture_output=True,text=True) + result["inspect_exit_code"]=run.returncode + if run.returncode: result["error"]=run.stderr.strip() or run.stdout.strip() + else: + inspected=json.loads(run.stdout); schema=contract["authoritative_schema"]; schema_path=repo/schema["path"] + checks={"output_path":inspected.get("output_path")==contract["output"]["path"],"output_format":inspected.get("output_format")==contract["output"]["format"],"projected_records":inspected.get("projected_records")==contract["record_count"],"schema_path":inspected.get("schema_path")==schema["path"],"schema_exists":schema_path.is_file(),"schema_sha256":schema_path.is_file() and sha(schema_path)==schema["sha256"],"exact_count_assertion":str(contract["record_count"]) in json.dumps(inspected.get("assertion_summary",{}))} + result.update({"checks":checks,"inspect":inspected,"status":"pass" if all(checks.values()) else "fail"}) + except (OSError,ValueError,KeyError) as error: result["error"]=str(error) + write(a.receipt,result) + if result["status"]!="pass": raise SystemExit(1) + print(json.dumps(result,sort_keys=True)) +if __name__=="__main__": main() diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/instruction.md b/labs/20-deltawire/eval/conformance/environment-receipt-v1/instruction.md new file mode 100644 index 000000000..d8dbe2d66 --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/instruction.md @@ -0,0 +1 @@ +No agent action is required. Exit without changing the task environment. diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/task.toml b/labs/20-deltawire/eval/conformance/environment-receipt-v1/task.toml new file mode 100644 index 000000000..de75749cc --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/task.toml @@ -0,0 +1,29 @@ +schema_version = "1.3" +artifacts = [] + +[task] +name = "operatorstack/deltawire-environment-receipt-conformance-v1" +description = "No-model Harbor main collect-hook conformance" +authors = [] +keywords = [] + +[metadata] +benchmark_result = false +conformance_version = "v1" + +[verifier] +timeout_sec = 60.0 + +[[verifier.collect]] +service = "main" +command = "python3 /usr/local/bin/environment_receipt.py --expectations /deltawire-environment-expectations.json --receipt /logs/artifacts/deltawire/environment-receipt.json" +timeout_sec = 30.0 + +[agent] +timeout_sec = 60.0 + +[environment] +network_mode = "public" +build_timeout_sec = 600.0 +os = "linux" +mcp_servers = [] diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/tests/test.sh b/labs/20-deltawire/eval/conformance/environment-receipt-v1/tests/test.sh new file mode 100755 index 000000000..39751523e --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/tests/test.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +mkdir -p /logs/verifier +printf '{"conformance": 1}\n' > /logs/verifier/reward.json diff --git a/labs/20-deltawire/eval/docs/08-harbor-artifact-timing.md b/labs/20-deltawire/eval/docs/08-harbor-artifact-timing.md new file mode 100644 index 000000000..6b52c1b22 --- /dev/null +++ b/labs/20-deltawire/eval/docs/08-harbor-artifact-timing.md @@ -0,0 +1,29 @@ +# Harbor 0.20.0 artifact timing + +This preflight is pinned to `harbor==0.20.0`. Inspection of the installed package on +`tbench-c4d` established the following order in +`harbor/trial/single_step.py` and `harbor/trial/trial.py`: + +1. the agent phase finishes; +2. `[[verifier.collect]]` hooks with `service = "main"` run while the main + container is still available; +3. main-container artifacts are downloaded; +4. the verifier runs; +5. verifier logs and reward are retained. + +`VerifierCollectConfig` in `harbor/models/task/config.py` accepts `command`, +`service`, `timeout_sec`, and `user`. The implicit convention entry for +`/logs/artifacts/` is created by `harbor/trial/artifact_handler.py` and maps to +`/artifacts/logs/artifacts/`. Its `artifacts/manifest.json` entry has +`source`, `destination`, `type`, `status`, and `service` fields. + +Collect hooks are best effort. A nonzero exit is written as a warning, including +the command, exit code, stdout, and stderr, in the job/trial logs, but collection +continues and no structured hook-exit field is added to the artifact manifest. +Consequently the v5 gate never treats Harbor success or hook exit alone as proof: +it requires the canonical retained receipt plus an `ok` convention-directory +manifest entry. + +The paid task retains the default container user. When that user is root, the +receipt detects accidental environment drift; it is not a cryptographic +anti-cheat boundary against the agent. diff --git a/labs/20-deltawire/eval/docs/09-gemini-agent-setup-boundary.md b/labs/20-deltawire/eval/docs/09-gemini-agent-setup-boundary.md new file mode 100644 index 000000000..f9008f676 --- /dev/null +++ b/labs/20-deltawire/eval/docs/09-gemini-agent-setup-boundary.md @@ -0,0 +1,43 @@ +# Gemini agent setup boundary + +## V5 observation + +The single v5 D1 probe used Harbor 0.20.0's built-in `gemini-cli` agent. The +DeltaWire task environment finished setup and the main collect hook retained a +green environment receipt. Agent execution never began. + +Harbor's built-in `GeminiCli.install` starts with the following sequence: + +1. `apt-get update && apt-get install -y curl` +2. install NVM and Node 22 +3. `npm install -g @google/gemini-cli@` +4. create Gemini settings +5. run `gemini --version` + +The v5 job log contains only the first command. Harbor then raised +`AgentSetupTimeoutError: Agent setup timed out after 360.0 seconds`. The exact +failure is recorded in +`labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/exception.txt` +at evidence commit `8c7f40eb96db71b3a5ed641277026adc3e4e2ecd`. + +The Harbor 0.20.0 source boundary is +`harbor/agents/installed/gemini_cli.py:109-135`; the timed-out root command is +issued at line 111. Harbor wraps setup with the 360-second timeout in +`harbor/trial/trial.py:1180-1188`. + +## Classification + +V5 is `agent_bootstrap_infrastructure_failure`, normalized more specifically as +`agent_setup_timeout` with trial status `failed_before_agent_execution`. + +This is not a Gemini model failure: no agent execution, provider request, +trajectory, or token accounting existed. It is not a DeltaWire semantic +failure: no DeltaWire plan or output was attempted. Harbor process exit zero is +diagnostic only and cannot override the non-null trial exception. + +## V6 boundary + +V6 preinstalls exact Node and Gemini CLI versions in the task image. Its custom +agent inherits Harbor's official Gemini run behavior and replaces only dynamic +installation with local identity verification. Increasing the paid setup +timeout is not an accepted fix. diff --git a/labs/20-deltawire/eval/manifests/preflight-v4-evidence-lock.json b/labs/20-deltawire/eval/manifests/preflight-v4-evidence-lock.json new file mode 100644 index 000000000..e47bac3fc --- /dev/null +++ b/labs/20-deltawire/eval/manifests/preflight-v4-evidence-lock.json @@ -0,0 +1,90 @@ +{ + "immutable_files": { + "labs/20-deltawire/eval/manifests/preflight-v2-evidence-lock.json": "dfaa4350681a36bcadef1e00d52bd90e16b2cf72638928ba2d0bc42b7af8f1b8", + "labs/20-deltawire/eval/manifests/preflight-v3-evidence-lock.json": "3318137e8f0da487297c3e95d07bac61fb7229d4ff751c334fe444e4444d6f61", + "labs/20-deltawire/eval/manifests/preflight-v4-range-large.json": "badcdcc4d7db11fb5aa715d26fbf6aefcd70f31ccc708afa7030486c2e26f116", + "labs/20-deltawire/eval/matrices/72-run-matrix.json": "f9dcddc4b7336418c7e2b8e7a7e51ca1e42b5a137255ff0823a732618f3568d5", + "labs/20-deltawire/eval/probes/range-large-v4/authoritative-schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/probes/range-large-v4/environment/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "labs/20-deltawire/eval/probes/range-large-v4/environment/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/probes/range-large-v4/environment/Dockerfile": "10f480e7735bab6ca1f9389af058951550284516a98c363b6eb113d4f9ff655f", + "labs/20-deltawire/eval/probes/range-large-v4/environment/deltawire-environment-expectations.json": "5b2c2f4876ef27633b8889ce4bc9209a8233b0ca2c7dfefca24b61de5cabb252", + "labs/20-deltawire/eval/probes/range-large-v4/environment/environment_receipt.py": "bf1a730b409faa8ee8a85a9a83a2f076dc2c1daad72f25c9613e6f18d630a90f", + "labs/20-deltawire/eval/probes/range-large-v4/environment/task-contract.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v4/environment/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af", + "labs/20-deltawire/eval/probes/range-large-v4/instruction.md": "e71b24acdf77f476573e4373a7f6a814f94e35adf5dcd0e6f048350574359725", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/authoritative-schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/environment/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/environment/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/environment/.generated/.gitignore": "240a3e0d37d2e86b614063f5347eb02d4f99ca6c254de6b82871ff8d95532a7d", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/environment/Dockerfile": "10f480e7735bab6ca1f9389af058951550284516a98c363b6eb113d4f9ff655f", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/environment/deltawire-environment-expectations.json": "5b2c2f4876ef27633b8889ce4bc9209a8233b0ca2c7dfefca24b61de5cabb252", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/environment/environment_receipt.py": "bf1a730b409faa8ee8a85a9a83a2f076dc2c1daad72f25c9613e6f18d630a90f", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/environment/task-contract.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/environment/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/instruction.md": "549f4047b1a4ba4f7a3f3dce9e60d1d70b141caa189aafdba942b15e6d022aee", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/task.toml": "01f68e68f24bd143fc2cf9cce1c89871694160a56992d1c19ef03f6f02dc1fc9", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/tests/semantic_oracle.py": "94b30fac117f40e977851966b5228e6c372bb48cbf2e5c29c24f428e1f98e91d", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/tests/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/tests/test.sh": "0c5eb4de20a4c76fb073742a684f15ff3431b38bfa0a9ca27186e4b1f9e65884", + "labs/20-deltawire/eval/probes/range-large-v4/skill/deltawire/SKILL.md": "69f266e086759f3b8371a5708464067d10daf30c7d01a8c8bc06dd39261ec19e", + "labs/20-deltawire/eval/probes/range-large-v4/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v4/task.toml": "81f4033d57800432e053aa9166b887f98fff6a4ca339753a5575917ecbe42bd9", + "labs/20-deltawire/eval/probes/range-large-v4/tests/semantic_oracle.py": "94b30fac117f40e977851966b5228e6c372bb48cbf2e5c29c24f428e1f98e91d", + "labs/20-deltawire/eval/probes/range-large-v4/tests/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v4/tests/test.sh": "0c5eb4de20a4c76fb073742a684f15ff3431b38bfa0a9ca27186e4b1f9e65884", + "labs/20-deltawire/eval/results/preflight-v1/treatment-probe-range-large/v1-evidence-lock.json": "26f4013270eb366c3c190f8235ef2564985b593005d499338273b1edc2c051d1", + "labs/20-deltawire/eval/results/preflight-v4/index.json": "0b6a2954c9597aab8d99e47958deb9159d2b24f98625be9d9ff5d54a723b152f", + "labs/20-deltawire/eval/results/preflight-v4/readiness.json": "5974d5a844dc7fb8659bb07b59a0625deadb275161dfd774f82df42db5962f4c", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/D1-end-to-end.json": "c99fd2df59f2162da1f245a1b5e8fad778c0ba8d18ad583e2a1751d8cc38f658", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/D1-environment-receipt.json": "c088201283f920cc39bc6d3b5621e9a86ba36a106e3d541aabd409ee7d70c231", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/D1-plan-contract-receipt.json": "9eb93bd1efb49fe95486d81529ff01ab6e2f20500eebd9146063380dea235f5a", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/D1-semantic-result.json": "a8f74965bc7c20d7b8de488e05b6c90e85caaab94b480adb71038165541a7c74", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/D1-treatment-use.json": "be48c00d06af98783f37814cd5ac68233447d19cb1de8ad0f36fabfea13162c5", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/config.json": "b0ba0743f1e6fce0151c0900520a4c4100126d77d114164fa4acb28581d71b52", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/job.log": "d8a2e5471b553e4fd839bb7c6ee4d18b9c7361647e7da8354d77868d850596a0", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/lock.json": "e27de1f7a41bc6f12afcaa3d8afc5e04d5f38936edf5a0f4da7158e266f872ef", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/agent/gemini-cli.trajectory.jsonl": "609a6b11bc5efdee6761986cbefa9b0a92db03a84e6d4fef6d842356ee5712c5", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/agent/gemini-cli.txt": "02dd467e913ee48ccbe01fb8652d888c02a22a1526d5d732b43a7dc6d42d6d17", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/agent/trajectory.json": "a2bf99ef6e09fc90008cc2ada0d9d1f0705887d1f5a2ed3603c8f0e3bc9cd5ad", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/artifacts/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/artifacts/.deltawire/plan-contract-receipt.json": "9eb93bd1efb49fe95486d81529ff01ab6e2f20500eebd9146063380dea235f5a", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/artifacts/.deltawire/plans/range-large.dw.json": "b1dcd5d61787bbc1a1873ae213b1105e7ac19184191eec599f5edd2a443404cb", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/artifacts/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/artifacts/.deltawire/state.json": "4c6bcc092e261346d8dd7ddcbc6a3cfc54e9908f08558840af43ba62dee1319d", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/artifacts/manifest.json": "d26ee4b66af08182faf6590f9cbd5c0ea57ad78825fcc3fc83beccd51ec61e9c", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/artifacts/testdata/generated/output.ndjson": "9d205b57ab166d23c81927899f99b6ccece261c679c96979c58fbde975637776", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/config.json": "66583cab10f7463245dceca31a943d19bcecdfd62c33e9a64cf1442808835692", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/lock.json": "474fe717247a1eb01267b9488acfdc0acd8ef6a5cd1ea939f54a7af34a01e996", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/result.json": "5297c6c91a002e4ea712bb3ccb616944efc55f03a3cef381eafe85a12b439e3f", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/trial.log": "d8a2e5471b553e4fd839bb7c6ee4d18b9c7361647e7da8354d77868d850596a0", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/verifier/reward.json": "36a492513204040f04d7bdf4146d84ede71353d8cc7df112fc3ebff66327683a", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/verifier/test-stdout.txt": "0813813de07909a65008adefbb04228b030d69a9f0acd33629b48a1f8e3fff96", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/result.json": "91eac94173146576acd04c2e38d5690fd36e9cd1b6306f869b68608174fd9145", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/redaction.json": "fd7ac8bda46226122e7ebd23ab4a0f05d3f508b0c2cae757276a73903d9c3760", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/report.md": "8ba26343e330ee864b9f51ab26854cbb15d1d6be34236b6e80d596bff6cb5423", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/run-ledger.json": "24d91aa5a9544c5c127e9eacea46727452aa9f0eeb0e404f92430e4eb900f8c0", + "labs/20-deltawire/eval/scripts/v4/build_and_stage.sh": "ad93bff7e05e1c6f40ffc8663ba688c77bced2a26c5bf11ccfda35ffc312af16", + "labs/20-deltawire/eval/scripts/v4/environment_receipt.py": "bf1a730b409faa8ee8a85a9a83a2f076dc2c1daad72f25c9613e6f18d630a90f", + "labs/20-deltawire/eval/scripts/v4/fidelity.py": "ae1963ea7021dc61eb38da757706ffe0dae204a96e757f452663080529f74077", + "labs/20-deltawire/eval/scripts/v4/generate_manifest.py": "f2223056e66ceb660b8954f9bc6cb7b32291819dd2ca0573608041bc1d5d8e2b", + "labs/20-deltawire/eval/scripts/v4/generate_v3_evidence_lock.py": "dfccde6c65d9357adf320bc8c28a7a5ad0341b3579b5505126d8e3b5ce3f7b43", + "labs/20-deltawire/eval/scripts/v4/runner.py": "789cd527a7e3659d77f5879347d5493784d30bb5d8ca02c9ed5cdcdb62312547", + "labs/20-deltawire/eval/scripts/v4/semantic_oracle.py": "94b30fac117f40e977851966b5228e6c372bb48cbf2e5c29c24f428e1f98e91d", + "labs/20-deltawire/eval/scripts/v4/shell_observation.py": "e5269e06350147715505a76b8e83235abecd5af34eba1c8837e19b22b0e4c9fe", + "labs/20-deltawire/eval/scripts/v4/test_environment_receipt.py": "59421625f731a5c5c9e555d970b87c1977eda86253cd18382b5aef4679c59d9e", + "labs/20-deltawire/eval/scripts/v4/test_fidelity.py": "531ca3f45bad8c3896829c430e2a01743bd24ceeb102170b7efa57525d545c19", + "labs/20-deltawire/eval/scripts/v4/test_runner.py": "ca72def4dc39be2bfb9b0cbe8e5eb5031b3c8711008abe7473e361eaf2901696", + "labs/20-deltawire/eval/scripts/v4/test_shell_observation.py": "5348c52e60b362408b433813466eb8cd8f03bde8f80033359eadb502bb71f3a0", + "labs/20-deltawire/eval/scripts/v4/validate.py": "571b7d95ac23595fd66268189691644872f0d78eaaf1f6814f27dde50cde9e9f", + "labs/20-deltawire/eval/scripts/v4/validate.sh": "c2ab7b0270a773f36fe877adb69829f0584902105fc2f12512ad6375533f5558", + "labs/20-deltawire/eval/scripts/v4/verify_evidence_locks.py": "8422b5f1bb156bffe6e399586262b474a6ac4bae48cfafb58b348bb2be4eafa0", + "labs/20-deltawire/eval/scripts/v4/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af", + "labs/20-deltawire/eval/tasks/range-large/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5" + }, + "probe_version": "v4", + "schema_version": "evidence-lock.v1", + "source_commit": "c200dc0eb3c4dff0e4732cc4b2daa1cd535703e3", + "source_tree": "ebafaafe33d871cce5ef26f39c99e37d0e78c9fa" +} diff --git a/labs/20-deltawire/eval/manifests/preflight-v5-evidence-lock.json b/labs/20-deltawire/eval/manifests/preflight-v5-evidence-lock.json new file mode 100644 index 000000000..77f25d955 --- /dev/null +++ b/labs/20-deltawire/eval/manifests/preflight-v5-evidence-lock.json @@ -0,0 +1,33 @@ +{ + "immutable_files": { + "labs/20-deltawire/eval/manifests/preflight-v5-range-large.json": "6f99b8b29181b4bf3021397e4c3d1b7827f1850f3d1cc25ddb638f021ccdbf57", + "labs/20-deltawire/eval/results/preflight-v5/approvals/paid-v5-probe.json": "86b5a778af39d4b41046ac85d55967ee3890580ac4dcab2c1555247c836d1b1b", + "labs/20-deltawire/eval/results/preflight-v5/conformance/environment-receipt-v1.json": "37c35894da0d23867743e710b096227e83744705dce24d1e6ecd94084a644fac", + "labs/20-deltawire/eval/results/preflight-v5/pre-paid-review.md": "926bde0a37cfc5c4be675d87f31e3087e6f02fc43b276035c3dd0fadba0de188", + "labs/20-deltawire/eval/results/preflight-v5/readiness.json": "5109463454d3de8d67a4d70647bccfb12a59797aae2a2e5cce6cfcbc7d2cf16c", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-artifact-manifest-receipt.json": "90aa12bdcd80ae70d8e053dae958e5848c45dceecd16759a9d6c410274a32c88", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-end-to-end.json": "ad7d53dc4ec75df7e06e0502c44e715d86df1b0444d8f895ce131b1eb5850ea8", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-environment-receipt.json": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-plan-contract-receipt.json": "e7ea2ae2fa9525b600e97e2a3c48c682e4a1909c6f15d75f58c3a9cee6f21513", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-semantic-result.json": "b6627f4ded6aa3685d6c966f03d78efeb8c693e5687e82d4aa4dc8a477d4113f", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-treatment-use.json": "44def672d5152acf149ed668043f673b92e9b8563ac857513c79d7053a6a5941", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/config.json": "0f5e22e8bed17acf2d7dfd5e7cfbd94929985cab310fbd091e2bc1109fb21708", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/job.log": "9c9e9ef588bfb14ddbf31b8872d06128d9c3db48962a9527bb1e5a625cb7428c", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/lock.json": "a0050425e26596063011fca44f0ee375f4bfd771850096cb22e4c47a455cfeeb", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/logs/artifacts/deltawire/environment-receipt.json": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/manifest.json": "6ed4dabb1ccea6827c700b6da5f5b45cd02b63b72b85d884a40950c54ea37121", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/config.json": "54f34aeb9acde46f4d802fc6de0b9dd9062036f3715b6c4af6fbaef2df5e26f4", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/exception.txt": "d29df1462cfce06fe934d2cd68ffae9fac852a920613f7f5b0f804e6c22df829", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/lock.json": "849d153b7ef9852b138c78addfcf588c078e9744fe338c605d7e2db3ade63157", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/result.json": "b1c29f30ba5677e51f57db870e74e91f966a0e62736ab81f1b0151d57b952da6", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/trial.log": "bb444b52b79168a19e1617bd24b88add55f959ce10052e1555d743adf1a98afb", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/result.json": "44d30e0daa3eda151ef152d6f63b9672cd26348329d4285acbb7a197abc380a6", + "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/run-ledger.json": "b3cc2619762178b5476c913c978681edc00c9182f44868454c1f24e921c10a16" + }, + "probe_version": "v5", + "schema_version": "evidence-lock.v1", + "source_commit": "8c7f40eb96db71b3a5ed641277026adc3e4e2ecd", + "source_tree": "fc69bc8ffe9f9a50737bfcce8995df7065fa5e0f" +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/classification.json b/labs/20-deltawire/eval/results/preflight-v5/classification.json new file mode 100644 index 000000000..d68770d99 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/classification.json @@ -0,0 +1,15 @@ +{ + "READY_FOR_72": false, + "agent_setup_status": "fail", + "artifact_status": "pass", + "end_to_end_status": "fail", + "environment_status": "pass", + "failure_class": "agent_bootstrap_infrastructure_failure", + "infrastructure_failure_class": "agent_setup_timeout", + "pair_launched": false, + "release_created": false, + "schema_version": "deltawire-v5-classification.v1", + "semantic_status": "fail", + "treatment_status": "fail", + "trial_status": "failed_before_agent_execution" +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/conformance/environment-receipt-v1.json b/labs/20-deltawire/eval/results/preflight-v5/conformance/environment-receipt-v1.json new file mode 100644 index 000000000..b226ed67f --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/conformance/environment-receipt-v1.json @@ -0,0 +1,79 @@ +{ + "agent": "nop", + "binary_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "expectation_hashes": { + "config": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "public_contract": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "schema": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + }, + "harbor_version": "0.20.0", + "model": null, + "negative_controls": { + "atomic_write_failure": true, + "binary_non_executable": true, + "binary_symlink": true, + "collect_hook_nonzero": true, + "copied_v4_receipt": true, + "malformed_receipt": true, + "manifest_missing_or_skipped": true, + "missing_binary": true, + "missing_collect_hook": true, + "missing_config": true, + "verifier_only_receipt": true, + "version_nonzero": true, + "wrong_binary_hash": true, + "wrong_contract_hash": true, + "wrong_receipt_path": true, + "wrong_schema_hash": true, + "wrong_version": true + }, + "positive_trials": [ + { + "agent": "nop", + "artifact_gate_status": "pass", + "artifact_manifest_entry": { + "destination": "artifacts/logs/artifacts", + "service": null, + "source": "/logs/artifacts", + "status": "ok", + "type": "directory" + }, + "artifact_manifest_sha256": "8d0a02e9c17d62af6176024f463c041ed30ff29d3a7b66dadbeab1d762b0cbd9", + "command": "uvx --from harbor==0.20.0 harbor run -a nop -p labs/20-deltawire/eval/conformance/environment-receipt-v1 -k 1 --jobs-dir /positive-1 --n-concurrent 1", + "environment_receipt_path": "positive-1/2026-07-22__14-44-04/environment-receipt-v1__sjjT4wM/artifacts/logs/artifacts/deltawire/environment-receipt.json", + "environment_receipt_sha256": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "harbor_exit_code": 0, + "model": null, + "reported_agent": "nop", + "trial": 1, + "trial_path": "positive-1/2026-07-22__14-44-04/environment-receipt-v1__sjjT4wM" + }, + { + "agent": "nop", + "artifact_gate_status": "pass", + "artifact_manifest_entry": { + "destination": "artifacts/logs/artifacts", + "service": null, + "source": "/logs/artifacts", + "status": "ok", + "type": "directory" + }, + "artifact_manifest_sha256": "8d0a02e9c17d62af6176024f463c041ed30ff29d3a7b66dadbeab1d762b0cbd9", + "command": "uvx --from harbor==0.20.0 harbor run -a nop -p labs/20-deltawire/eval/conformance/environment-receipt-v1 -k 1 --jobs-dir /positive-2 --n-concurrent 1", + "environment_receipt_path": "positive-2/2026-07-22__14-45-20/environment-receipt-v1__4jdxKnL/artifacts/logs/artifacts/deltawire/environment-receipt.json", + "environment_receipt_sha256": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "harbor_exit_code": 0, + "model": null, + "reported_agent": "nop", + "trial": 2, + "trial_path": "positive-2/2026-07-22__14-45-20/environment-receipt-v1__4jdxKnL" + } + ], + "ready_for_72": false, + "schema_version": "deltawire-no-model-conformance.v1", + "source_infrastructure_commit": "40bb6619819be468d66201d9e4aa947c6042415a", + "stable_receipt_sha256": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "status": "pass", + "workflow_url": "https://github.com/operatorstack/intelligence-flow/actions/runs/29929855472", + "zero_model_calls": true +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/readiness.json b/labs/20-deltawire/eval/results/preflight-v5/readiness.json new file mode 100644 index 000000000..d54c68365 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/readiness.json @@ -0,0 +1,9 @@ +{ + "READY_FOR_72": false, + "full_run_started": false, + "pair_complete": false, + "pair_launched": false, + "probe_complete": false, + "schema_version": "deltawire-readiness.v5", + "six_run_canary_started": false +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/report.md b/labs/20-deltawire/eval/results/preflight-v5/report.md new file mode 100644 index 000000000..4b1d4a9c0 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/report.md @@ -0,0 +1,18 @@ +# DeltaWire v5 terminal report + +Exactly one D1-only `range-large-v5` probe ran. Harbor retained a green +DeltaWire environment receipt and a green artifact-manifest receipt, then the +built-in Gemini CLI agent timed out during its first dynamic installation +command. Agent execution never began. + +- Trial: `574138ea-c743-4a05-b34c-cd578d24f49f` +- Evidence commit: `8c7f40eb96db71b3a5ed641277026adc3e4e2ecd` +- Failure: `AgentSetupTimeoutError` after the 360-second setup boundary +- Normalized class: `agent_bootstrap_infrastructure_failure` +- Trajectory, plan, output, and token usage: absent +- Release and pair: not created +- `READY_FOR_72`: false + +Raw evidence remains exclusively in PR #111's experiment branch. This clean +projection contains only compact receipts and the Git-object-backed evidence +lock. diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-artifact-manifest-receipt.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-artifact-manifest-receipt.json new file mode 100644 index 000000000..56ee9a012 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-artifact-manifest-receipt.json @@ -0,0 +1,28 @@ +{ + "artifact_manifest_sha256": "6ed4dabb1ccea6827c700b6da5f5b45cd02b63b72b85d884a40950c54ea37121", + "checks": { + "binary_hash_frozen": true, + "config_hash_frozen": true, + "manifest_exists": true, + "manifest_status_ok": true, + "public_contract_hash_frozen": true, + "receipt_canonical_path": true, + "receipt_checks_complete": true, + "receipt_checks_true": true, + "receipt_schema_exact": true, + "receipt_status_pass": true, + "schema_hash_frozen": true, + "single_convention_entry": true + }, + "convention_entry": { + "destination": "artifacts/logs/artifacts", + "service": null, + "source": "/logs/artifacts", + "status": "ok", + "type": "directory" + }, + "environment_receipt_path": "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/logs/artifacts/deltawire/environment-receipt.json", + "environment_receipt_sha256": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "schema_version": "artifact-manifest-receipt.v1", + "status": "pass" +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-end-to-end.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-end-to-end.json new file mode 100644 index 000000000..a423bd006 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-end-to-end.json @@ -0,0 +1,22 @@ +{ + "arm": "D1", + "artifact_manifest_status": "pass", + "environment_status": "pass", + "harbor_exit_0": true, + "model_match": true, + "plan_contract_status": "fail", + "ready_for_72": false, + "schema_version": "deltawire-end-to-end-receipt.v5", + "semantic_status": "fail", + "source_hashes": { + "artifact_manifest_receipt": "90aa12bdcd80ae70d8e053dae958e5848c45dceecd16759a9d6c410274a32c88", + "environment": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "output": null, + "result": "b1c29f30ba5677e51f57db870e74e91f966a0e62736ab81f1b0151d57b952da6", + "semantic": "b6627f4ded6aa3685d6c966f03d78efeb8c693e5687e82d4aa4dc8a477d4113f", + "trajectory": null, + "treatment_use": "44def672d5152acf149ed668043f673b92e9b8563ac857513c79d7053a6a5941" + }, + "status": "fail", + "treatment_status": "fail" +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-environment-receipt.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-environment-receipt.json new file mode 100644 index 000000000..55108c6ac --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-environment-receipt.json @@ -0,0 +1,55 @@ +{ + "binary": { + "actual_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "executable": true, + "exists": true, + "expected_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "expected_version": "deltawire version dev", + "is_regular": true, + "is_symlink": false, + "path": "/usr/local/bin/deltawire", + "realpath": "/usr/local/bin/deltawire", + "version_exit_code": 0, + "version_stderr": "", + "version_stdout": "deltawire version dev\nGo runtime version go1.26.5\nsupported config version deltawire.config.v1\nsupported plan version deltawire.plan.v1\nsupported state version deltawire.state.v1" + }, + "checks": { + "binary_executable": true, + "binary_exists": true, + "binary_hash": true, + "binary_not_symlink": true, + "binary_path": true, + "binary_realpath": true, + "binary_regular": true, + "config_hash": true, + "public_contract_hash": true, + "schema_hash": true, + "version_exact": true, + "version_exit_0": true + }, + "files": { + "config": { + "actual_sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "exists": true, + "expected_sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "hash_match": true, + "path": "/.deltawire/config.json" + }, + "public_contract": { + "actual_sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "exists": true, + "expected_sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "hash_match": true, + "path": "/task-contract.json" + }, + "schema": { + "actual_sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "exists": true, + "expected_sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "hash_match": true, + "path": "/.deltawire/schemas/range-large.schema.json" + } + }, + "schema_version": "deltawire-environment-receipt.v1", + "status": "pass" +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-plan-contract-receipt.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-plan-contract-receipt.json new file mode 100644 index 000000000..f3ba750aa --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-plan-contract-receipt.json @@ -0,0 +1,6 @@ +{ + "checks": {}, + "error": "expected one plan, found 0", + "schema_version": "plan-contract-receipt.v1", + "status": "fail" +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-semantic-result.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-semantic-result.json new file mode 100644 index 000000000..e94af9f0e --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-semantic-result.json @@ -0,0 +1,6 @@ +{ + "error": "output missing", + "exact_match": 0, + "schema_version": "semantic-result.v5", + "status": "fail" +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-treatment-use.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-treatment-use.json new file mode 100644 index 000000000..18aacdbeb --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-treatment-use.json @@ -0,0 +1,11 @@ +{ + "arm": "D1", + "checks": {}, + "classification": "attempted_failed", + "contamination": false, + "failure_reasons": [ + "missing artifacts: trajectory_path" + ], + "schema_version": "treatment-use-receipt.v5", + "status": "fail" +} diff --git a/labs/20-deltawire/eval/scripts/v5/artifact_manifest.py b/labs/20-deltawire/eval/scripts/v5/artifact_manifest.py new file mode 100644 index 000000000..1b8344ed7 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/artifact_manifest.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import hashlib,json +from pathlib import Path + +SOURCE="/logs/artifacts" +DESTINATION="artifacts/logs/artifacts" +RECEIPT="logs/artifacts/deltawire/environment-receipt.json" +def sha(path):return hashlib.sha256(Path(path).read_bytes()).hexdigest() +def validate(trial,expected_binary_sha=None,expected_files=None): + trial=Path(trial);manifest_path=trial/"artifacts/manifest.json";receipt_path=trial/"artifacts"/RECEIPT + checks={"manifest_exists":manifest_path.is_file(),"receipt_canonical_path":receipt_path.is_file()} + manifest=[];receipt={} + try:manifest=json.loads(manifest_path.read_text()) if checks["manifest_exists"] else [] + except (OSError,json.JSONDecodeError):manifest=[] + entries=[e for e in manifest if isinstance(e,dict) and e.get("source")==SOURCE and e.get("destination")==DESTINATION] + checks["single_convention_entry"]=len(entries)==1 + checks["manifest_status_ok"]=len(entries)==1 and entries[0].get("status")=="ok" and entries[0].get("type")=="directory" + try:receipt=json.loads(receipt_path.read_text()) if checks["receipt_canonical_path"] else {} + except (OSError,json.JSONDecodeError):receipt={} + checks["receipt_schema_exact"]=receipt.get("schema_version")=="deltawire-environment-receipt.v1" + checks["receipt_status_pass"]=receipt.get("status")=="pass" + required={"binary_exists","binary_regular","binary_not_symlink","binary_executable","binary_path","binary_realpath","binary_hash","version_exit_0","version_exact","config_hash","schema_hash","public_contract_hash"} + receipt_checks=receipt.get("checks") if isinstance(receipt.get("checks"),dict) else {} + checks["receipt_checks_complete"]=set(receipt_checks)==required + checks["receipt_checks_true"]=set(receipt_checks)==required and all(receipt_checks.values()) + if expected_binary_sha is not None:checks["binary_hash_frozen"]=receipt.get("binary",{}).get("actual_sha256")==expected_binary_sha + for name,want in sorted((expected_files or {}).items()):checks[f"{name}_hash_frozen"]=receipt.get("files",{}).get(name,{}).get("actual_sha256")==want + return {"schema_version":"artifact-manifest-receipt.v1","status":"pass" if all(checks.values()) else "fail","checks":checks, + "artifact_manifest_sha256":sha(manifest_path) if manifest_path.is_file() else None, + "environment_receipt_sha256":sha(receipt_path) if receipt_path.is_file() else None, + "environment_receipt_path":str(receipt_path),"convention_entry":entries[0] if len(entries)==1 else None} diff --git a/labs/20-deltawire/eval/scripts/v5/environment_receipt.py b/labs/20-deltawire/eval/scripts/v5/environment_receipt.py new file mode 100644 index 000000000..84446c2d5 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/environment_receipt.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Atomically emit deterministic DeltaWire environment evidence.""" +import argparse,hashlib,json,os,stat,subprocess,tempfile +from pathlib import Path + +def sha(path): + h=hashlib.sha256() + with Path(path).open("rb") as f: + for chunk in iter(lambda:f.read(1048576),b""):h.update(chunk) + return h.hexdigest() +def atomic_write(path,data): + target=Path(path);target.parent.mkdir(parents=True,exist_ok=True) + fd,tmp=tempfile.mkstemp(prefix=f".{target.name}.",suffix=".tmp",dir=target.parent) + try: + with os.fdopen(fd,"w",encoding="utf-8") as f:f.write(data);f.flush();os.fsync(f.fileno()) + os.replace(tmp,target) + except BaseException: + try:os.unlink(tmp) + except FileNotFoundError:pass + raise +def build(expectations): + expected=json.loads(Path(expectations).read_text());binary=Path(expected["binary"]["path"]) + exists=binary.exists();lst=binary.lstat() if exists else None + regular=bool(lst and stat.S_ISREG(lst.st_mode));symlink=binary.is_symlink() if exists else False + executable=exists and os.access(binary,os.X_OK);realpath=str(binary.resolve()) if exists else None + try:version=subprocess.run([str(binary),"version"],capture_output=True,text=True,check=False) if exists else None + except OSError:version=None + files={} + for name,item in sorted(expected["files"].items()): + path=Path(item["path"]);actual=sha(path) if path.is_file() else None + files[name]={"path":item["path"],"exists":path.is_file(),"actual_sha256":actual,"expected_sha256":item["sha256"],"hash_match":actual==item["sha256"]} + binary_hash=sha(binary) if regular else None + checks={"binary_exists":exists,"binary_regular":regular,"binary_not_symlink":not symlink,"binary_executable":executable, + "binary_path":str(binary)==expected["binary"]["path"],"binary_realpath":realpath==expected["binary"]["realpath"], + "binary_hash":binary_hash==expected["binary"]["sha256"],"version_exit_0":bool(version and version.returncode==0), + "version_exact":bool(version and version.stdout.splitlines() and version.stdout.splitlines()[0].strip()==expected["binary"]["version"]), + **{f"{name}_hash":item["hash_match"] for name,item in files.items()}} + return {"schema_version":"deltawire-environment-receipt.v1","binary":{"path":str(binary),"realpath":realpath,"exists":exists, + "is_regular":regular,"is_symlink":symlink,"executable":executable,"actual_sha256":binary_hash, + "expected_sha256":expected["binary"]["sha256"],"version_stdout":version.stdout.strip() if version else None, + "version_stderr":version.stderr.strip() if version else None,"version_exit_code":version.returncode if version else None, + "expected_version":expected["binary"]["version"]},"files":files,"checks":checks,"status":"pass" if all(checks.values()) else "fail"} +def main(): + p=argparse.ArgumentParser();p.add_argument("--expectations",required=True);p.add_argument("--receipt",required=True);a=p.parse_args() + receipt=build(a.expectations);atomic_write(a.receipt,json.dumps(receipt,indent=2,sort_keys=True)+"\n") + print(json.dumps(receipt,sort_keys=True));raise SystemExit(0 if receipt["status"]=="pass" else 1) +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v5/generate_v5_evidence_lock.py b/labs/20-deltawire/eval/scripts/v5/generate_v5_evidence_lock.py new file mode 100644 index 000000000..f473e06fe --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/generate_v5_evidence_lock.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +import hashlib +import json +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[5] +EVAL = ROOT / "labs/20-deltawire/eval" +SOURCE_COMMIT = "8c7f40eb96db71b3a5ed641277026adc3e4e2ecd" +OUTPUT = EVAL / "manifests/preflight-v5-evidence-lock.json" + + +def git(*args: str, binary: bool = False): + return subprocess.check_output(["git", *args], cwd=ROOT, text=not binary) + + +def main() -> None: + prefix = "labs/20-deltawire/eval/results/preflight-v5" + paths = git("ls-tree", "-r", "--name-only", SOURCE_COMMIT, prefix).splitlines() + paths.append("labs/20-deltawire/eval/manifests/preflight-v5-range-large.json") + immutable = {} + for name in sorted(set(paths)): + data = git("show", f"{SOURCE_COMMIT}:{name}", binary=True) + immutable[name] = hashlib.sha256(data).hexdigest() + value = { + "schema_version": "evidence-lock.v1", + "probe_version": "v5", + "source_commit": SOURCE_COMMIT, + "source_tree": git("rev-parse", f"{SOURCE_COMMIT}^{{tree}}").strip(), + "immutable_files": immutable, + } + OUTPUT.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + print(OUTPUT) + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/scripts/v5/negative_controls.py b/labs/20-deltawire/eval/scripts/v5/negative_controls.py new file mode 100644 index 000000000..cf02696a5 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/negative_controls.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +import json,os,tempfile +from pathlib import Path +from artifact_manifest import validate +from environment_receipt import atomic_write + +REQUIRED=("binary_exists","binary_regular","binary_not_symlink","binary_executable","binary_path","binary_realpath","binary_hash","version_exit_0","version_exact","config_hash","schema_hash","public_contract_hash") +def write(path,value):path.parent.mkdir(parents=True,exist_ok=True);path.write_text(json.dumps(value)+"\n") +def fixture(root): + trial=root/"trial";manifest=trial/"artifacts/manifest.json";receipt=trial/"artifacts/logs/artifacts/deltawire/environment-receipt.json" + value={"schema_version":"deltawire-environment-receipt.v1","status":"pass","checks":{x:True for x in REQUIRED},"binary":{"actual_sha256":"binary"},"files":{x:{"actual_sha256":x} for x in ("config","schema","public_contract")}} + write(receipt,value);write(manifest,[{"source":"/logs/artifacts","destination":"artifacts/logs/artifacts","type":"directory","status":"ok","service":None}]);return trial,manifest,receipt +def rejected(trial):return validate(trial,"binary",{x:x for x in ("config","schema","public_contract")})["status"]=="fail" +def run_negative_controls(): + results={} + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp) + trial,manifest,receipt=fixture(root);receipt.unlink();results["missing_collect_hook"]=rejected(trial) + trial,manifest,receipt=fixture(root);value=json.loads(receipt.read_text());value["status"]="fail";value["checks"]["binary_hash"]=False;write(receipt,value);results["collect_hook_nonzero"]=rejected(trial) + trial,manifest,receipt=fixture(root);late=trial/"artifacts/.deltawire/environment-receipt.json";late.parent.mkdir(parents=True,exist_ok=True);late.write_bytes(receipt.read_bytes());receipt.unlink();results["verifier_only_receipt"]=rejected(trial) + for key,field in (("wrong_binary_hash",("binary","actual_sha256")),("wrong_schema_hash",("files","schema","actual_sha256")),("wrong_contract_hash",("files","public_contract","actual_sha256"))): + trial,manifest,receipt=fixture(root);value=json.loads(receipt.read_text());target=value + for part in field[:-1]:target=target[part] + target[field[-1]]="wrong";write(receipt,value);results[key]=rejected(trial) + trial,manifest,receipt=fixture(root);value=json.loads(receipt.read_text());value["checks"]["version_exact"]=False;value["status"]="fail";write(receipt,value);results["wrong_version"]=rejected(trial) + for name,check in (("missing_binary","binary_exists"),("binary_symlink","binary_not_symlink"),("binary_non_executable","binary_executable"),("version_nonzero","version_exit_0")): + trial,manifest,receipt=fixture(root);value=json.loads(receipt.read_text());value["checks"][check]=False;value["status"]="fail";write(receipt,value);results[name]=rejected(trial) + trial,manifest,receipt=fixture(root);value=json.loads(receipt.read_text());value["checks"]["config_hash"]=False;value["status"]="fail";write(receipt,value);results["missing_config"]=rejected(trial) + trial,manifest,receipt=fixture(root);receipt.write_text("{");results["malformed_receipt"]=rejected(trial) + trial,manifest,receipt=fixture(root);write(manifest,[{"source":"/logs/artifacts","destination":"artifacts/logs/artifacts","type":"directory","status":"skipped"}]);results["manifest_missing_or_skipped"]=rejected(trial) + trial,manifest,receipt=fixture(root);wrong=trial/"artifacts/environment-receipt.json";wrong.write_bytes(receipt.read_bytes());receipt.unlink();results["wrong_receipt_path"]=rejected(trial) + trial,manifest,receipt=fixture(root);value=json.loads(receipt.read_text());value["binary"]["actual_sha256"]="v4";write(receipt,value);results["copied_v4_receipt"]=rejected(trial) + target=root/"atomic";target.mkdir() + try:atomic_write(target,"{}\n");results["atomic_write_failure"]=False + except OSError:results["atomic_write_failure"]=target.is_dir() and not list(root.glob(".atomic.*.tmp")) + return results +if __name__=="__main__": + value=run_negative_controls();print(json.dumps(value,sort_keys=True));raise SystemExit(0 if all(value.values()) else 1) diff --git a/labs/20-deltawire/eval/scripts/v5/run_conformance.py b/labs/20-deltawire/eval/scripts/v5/run_conformance.py new file mode 100644 index 000000000..48355b798 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/run_conformance.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Run exactly two no-model Harbor collect-hook conformance trials.""" +import argparse,hashlib,json,shlex,subprocess +from pathlib import Path +from artifact_manifest import validate +from negative_controls import run_negative_controls + +ROOT=Path(__file__).resolve().parents[5];EVAL=ROOT/"labs/20-deltawire/eval";TASK=EVAL/"conformance/environment-receipt-v1" +HARBOR=["uvx","--from","harbor==0.20.0","harbor"] +def sha(path):return hashlib.sha256(Path(path).read_bytes()).hexdigest() +def trials(root):return sorted({p.parent.parent for p in Path(root).rglob("artifacts/manifest.json")}) +def main(): + p=argparse.ArgumentParser();p.add_argument("--jobs-dir",required=True);p.add_argument("--report",required=True);a=p.parse_args() + jobs=Path(a.jobs_dir).resolve();report=Path(a.report).resolve() + if jobs.exists() and any(jobs.iterdir()):raise SystemExit("conformance jobs directory must be empty") + jobs.mkdir(parents=True,exist_ok=True) + version=subprocess.check_output(HARBOR+["--version"],text=True).strip() + if version!="0.20.0":raise SystemExit(f"Harbor version mismatch: {version}") + expected=json.loads((TASK/"environment/deltawire-environment-expectations.json").read_text()) + expected_files={name:item["sha256"] for name,item in expected["files"].items()} + records=[] + for number in (1,2): + target=jobs/f"positive-{number}";before=set(trials(target)) + cmd=HARBOR+["run","-a","nop","-p",str(TASK),"-k","1","--jobs-dir",str(target),"--n-concurrent","1"] + run=subprocess.run(cmd,cwd=ROOT,check=False);created=set(trials(target))-before + if run.returncode or len(created)!=1:raise SystemExit(f"positive conformance {number} failed: rc={run.returncode}, trials={len(created)}") + trial=created.pop();receipt=validate(trial,expected["binary"]["sha256"],expected_files) + if receipt["status"]!="pass":raise SystemExit(f"positive conformance {number} artifact gate failed") + result_files=sorted(trial.glob("result.json"));result=json.loads(result_files[0].read_text()) if result_files else {} + logical=f"positive-{number}/{trial.parent.name}/{trial.name}" + display_cmd=HARBOR+["run","-a","nop","-p",str(TASK.relative_to(ROOT)),"-k","1","--jobs-dir",f"/positive-{number}","--n-concurrent","1"] + records.append({"trial":number,"trial_path":logical,"command":shlex.join(display_cmd),"harbor_exit_code":run.returncode, + "artifact_manifest_entry":receipt["convention_entry"],"environment_receipt_path":receipt["environment_receipt_path"], + "environment_receipt_sha256":receipt["environment_receipt_sha256"],"artifact_manifest_sha256":receipt["artifact_manifest_sha256"], + "artifact_gate_status":receipt["status"],"agent":"nop","model":None,"reported_agent":(result.get("agent_info") or {}).get("name")}) + records[-1]["environment_receipt_path"]=logical+"/artifacts/logs/artifacts/deltawire/environment-receipt.json" + hashes={x["environment_receipt_sha256"] for x in records} + if len(hashes)!=1:raise SystemExit("positive conformance receipt hashes differ") + negatives=run_negative_controls() + value={"schema_version":"deltawire-no-model-conformance.v1","status":"pass","harbor_version":version,"agent":"nop","model":None, + "zero_model_calls":all(" -m " not in f" {r['command']} " and "--env-file" not in r["command"] for r in records), + "binary_sha256":expected["binary"]["sha256"],"expectation_hashes":{name:item["sha256"] for name,item in expected["files"].items()}, + "positive_trials":records,"stable_receipt_sha256":records[0]["environment_receipt_sha256"],"negative_controls":negatives, + "ready_for_72":False} + if not value["zero_model_calls"] or not all(negatives.values()):raise SystemExit("conformance proof failed") + report.parent.mkdir(parents=True,exist_ok=True);report.write_text(json.dumps(value,indent=2,sort_keys=True)+"\n") + print(report) +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v5/test_artifact_manifest.py b/labs/20-deltawire/eval/scripts/v5/test_artifact_manifest.py new file mode 100644 index 000000000..461fdfabe --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/test_artifact_manifest.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +import json,tempfile +from pathlib import Path +from artifact_manifest import validate + +def write(path,value):path.parent.mkdir(parents=True,exist_ok=True);path.write_text(json.dumps(value)+"\n") +def fixture(root): + trial=root/"trial";manifest=trial/"artifacts/manifest.json";receipt=trial/"artifacts/logs/artifacts/deltawire/environment-receipt.json" + checks={name:True for name in ("binary_exists","binary_regular","binary_not_symlink","binary_executable","binary_path","binary_realpath","binary_hash","version_exit_0","version_exact","config_hash","schema_hash","public_contract_hash")} + value={"schema_version":"deltawire-environment-receipt.v1","status":"pass","checks":checks,"binary":{"actual_sha256":"binary"},"files":{name:{"actual_sha256":name} for name in ("config","schema","public_contract")}} + write(receipt,value);write(manifest,[{"source":"/logs/artifacts","destination":"artifacts/logs/artifacts","type":"directory","status":"ok","service":None}]);return trial,manifest,receipt +def main(): + expected={name:name for name in ("config","schema","public_contract")} + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);trial,manifest,receipt=fixture(root);assert validate(trial,"binary",expected)["status"]=="pass" + cases=[] + cases.append(lambda:receipt.unlink()) + cases.append(lambda:write(receipt,{"bad":"json shape"})) + cases.append(lambda:write(manifest,[{"source":"/logs/artifacts","destination":"artifacts/logs/artifacts","type":"directory","status":"skipped"}])) + cases.append(lambda:write(manifest,[{"source":"/wrong","destination":"artifacts/logs/artifacts","type":"directory","status":"ok"}])) + cases.append(lambda:write(manifest,[])) + for mutate in cases: + trial,manifest,receipt=fixture(root);mutate();assert validate(trial,"binary",expected)["status"]=="fail" + trial,manifest,receipt=fixture(root);outside=trial/"artifacts/environment-receipt.json";outside.write_bytes(receipt.read_bytes());receipt.unlink();assert validate(trial,"binary",expected)["status"]=="fail" + trial,manifest,receipt=fixture(root);value=json.loads(receipt.read_text());value["binary"]["actual_sha256"]="v4";write(receipt,value);assert validate(trial,"binary",expected)["status"]=="fail" + print("Artifact-manifest and canonical receipt-path tests passed.") +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v5/test_environment_receipt.py b/labs/20-deltawire/eval/scripts/v5/test_environment_receipt.py new file mode 100644 index 000000000..4f586567f --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/test_environment_receipt.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +import hashlib,json,os,subprocess,tempfile +from pathlib import Path +HERE=Path(__file__).resolve().parent +def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() +def run(root,binary,mutate=None): + files={} + for name in ("config","schema","public_contract"): + p=root/f"{name}.json";p.write_text(name);files[name]={"path":str(p),"sha256":sha(p)} + exp={"binary":{"path":str(binary),"realpath":str(binary.resolve()),"sha256":sha(binary) if binary.is_file() else "0"*64,"version":"deltawire version dev"},"files":files} + if mutate:mutate(exp) + ep=root/"expect.json";rp=root/"receipt.json";ep.write_text(json.dumps(exp));result=subprocess.run(["python3",str(HERE/"environment_receipt.py"),"--expectations",str(ep),"--receipt",str(rp)],capture_output=True,text=True);return result.returncode,json.loads(rp.read_text()) +def main(): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);binary=root/"deltawire";binary.write_text("#!/bin/sh\nprintf 'deltawire version dev\\nsupported plan version deltawire.plan.v1\\n'\n");binary.chmod(0o755) + first=run(root,binary)[1];second=run(root,binary)[1];assert first["status"]=="pass" and first==second + assert run(root,binary,lambda e:e["binary"].update(sha256="0"*64))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["binary"].update(path=str(root/"wrong")))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["binary"].update(version="wrong"))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["files"]["config"].update(sha256="0"*64))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["files"]["schema"].update(sha256="0"*64))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["files"]["public_contract"].update(sha256="0"*64))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["binary"].update(realpath="/wrong"))[1]["status"]=="fail" + binary.chmod(0o644);assert run(root,binary)[1]["status"]=="fail";binary.chmod(0o755) + link=root/"link";link.symlink_to(binary);assert run(root,link)[1]["status"]=="fail" + missing=root/"missing";code,receipt=run(root,missing);assert code and receipt["status"]=="fail" + failing=root/"failing";failing.write_text("#!/bin/sh\nexit 7\n");failing.chmod(0o755);assert run(root,failing)[1]["binary"]["version_exit_code"]==7 + atomic=root/"atomic-target";atomic.mkdir();files={} + for name in ("config","schema","public_contract"): + p=root/f"atomic-{name}.json";p.write_text(name);files[name]={"path":str(p),"sha256":sha(p)} + exp={"binary":{"path":str(binary),"realpath":str(binary.resolve()),"sha256":sha(binary),"version":"deltawire version dev"},"files":files} + ep=root/"atomic-expect.json";ep.write_text(json.dumps(exp));result=subprocess.run(["python3",str(HERE/"environment_receipt.py"),"--expectations",str(ep),"--receipt",str(atomic)],capture_output=True,text=True) + assert result.returncode and atomic.is_dir() and not list(root.glob(".atomic-target.*.tmp")) + print("Deterministic environment receipt tests passed.") +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v5/validate_integration.py b/labs/20-deltawire/eval/scripts/v5/validate_integration.py new file mode 100644 index 000000000..531d31a22 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/validate_integration.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +import json +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[5] +EVAL = ROOT / "labs/20-deltawire/eval" + + +def load(path: Path): + return json.loads(path.read_text()) + + +def main() -> None: + base = EVAL / "results/preflight-v5" + classification = load(base / "classification.json") + expected = { + "environment_status": "pass", + "artifact_status": "pass", + "agent_setup_status": "fail", + "treatment_status": "fail", + "semantic_status": "fail", + "end_to_end_status": "fail", + "release_created": False, + "pair_launched": False, + "READY_FOR_72": False, + } + assert all(classification[key] == value for key, value in expected.items()) + assert classification["failure_class"] == "agent_bootstrap_infrastructure_failure" + assert classification["trial_status"] == "failed_before_agent_execution" + assert load(base / "treatment-probe-range-large-v5/D1-environment-receipt.json")["status"] == "pass" + assert load(base / "treatment-probe-range-large-v5/D1-artifact-manifest-receipt.json")["status"] == "pass" + for name in ("D1-treatment-use.json", "D1-plan-contract-receipt.json", "D1-semantic-result.json", "D1-end-to-end.json"): + assert load(base / f"treatment-probe-range-large-v5/{name}")["status"] == "fail" + tracked = subprocess.check_output(["git", "ls-files"], cwd=ROOT, text=True).splitlines() + assert not any("results/preflight-v5/" in name and "/raw/" in name for name in tracked) + assert not (base / "approvals").exists() + assert not (base / "treatment-probe-range-large-v5/run-ledger.json").exists() + doc = (EVAL / "docs/09-gemini-agent-setup-boundary.md").read_text() + assert "agent_bootstrap_infrastructure_failure" in doc + assert "not a Gemini model failure" in doc + print("Clean v5 projection and terminal classification passed.") + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py b/labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py new file mode 100644 index 000000000..d37641929 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +import hashlib,json,subprocess +from pathlib import Path + +ROOT=Path(__file__).resolve().parents[5] +EVAL=ROOT/"labs/20-deltawire/eval" +def sha(data):return hashlib.sha256(data).hexdigest() +def object_bytes(commit,name):return subprocess.check_output(["git","show",f"{commit}:{name}"],cwd=ROOT) +def main(): + evidence_commit="c200dc0eb3c4dff0e4732cc4b2daa1cd535703e3" + for path in (EVAL/"results/preflight-v1/treatment-probe-range-large/v1-evidence-lock.json",EVAL/"manifests/preflight-v2-evidence-lock.json"): + lock=json.loads(path.read_text()) + for name,want in lock["immutable_files"].items(): + target=ROOT/name + data=target.read_bytes() if target.is_file() else object_bytes(evidence_commit,name) + if sha(data)!=want:raise SystemExit(f"historical evidence mismatch: {name}") + v3=json.loads((EVAL/"manifests/preflight-v3-evidence-lock.json").read_text());v3_commit=v3["source_commit"] + if subprocess.check_output(["git","rev-parse",f"{v3_commit}^{{tree}}"],cwd=ROOT,text=True).strip()!=v3["source_tree"]:raise SystemExit("v3 source tree mismatch") + for name,want in v3["immutable_files"].items(): + if sha(object_bytes(v3_commit,name))!=want:raise SystemExit(f"v3 Git-object mismatch: {name}") + counts={} + for version in ("v4","v5"): + lock=json.loads((EVAL/f"manifests/preflight-{version}-evidence-lock.json").read_text()) + commit=lock["source_commit"] + tree=subprocess.check_output(["git","rev-parse",f"{commit}^{{tree}}"],cwd=ROOT,text=True).strip() + if tree!=lock["source_tree"]:raise SystemExit(f"{version} source tree mismatch") + for name,want in lock["immutable_files"].items(): + data=object_bytes(commit,name) + if sha(data)!=want:raise SystemExit(f"{version} Git-object mismatch: {name}") + counts[version]=len(lock["immutable_files"]) + print(f"v1-v5 evidence locks passed ({counts['v4']} v4 files, {counts['v5']} v5 files).") +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/validate_dataset.sh b/labs/20-deltawire/eval/scripts/validate_dataset.sh index 8db1060b4..7a0e62dda 100755 --- a/labs/20-deltawire/eval/scripts/validate_dataset.sh +++ b/labs/20-deltawire/eval/scripts/validate_dataset.sh @@ -17,4 +17,8 @@ assert paths and all(hashlib.sha256(path.read_bytes()).hexdigest()==expected for print(f"Identical Linux binary staged into {len(paths)-1} task contexts.") PY python3 labs/20-deltawire/eval/scripts/validate_preflight.py +python3 labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py +python3 labs/20-deltawire/eval/scripts/v5/test_environment_receipt.py +python3 labs/20-deltawire/eval/scripts/v5/test_artifact_manifest.py +python3 labs/20-deltawire/eval/scripts/v5/validate_integration.py bash labs/20-deltawire/scripts/check-production-cleanliness.sh From 656b05b919983f0a7f4e05dc724ce508083c8cbd Mon Sep 17 00:00:00 2001 From: bigboateng Date: Wed, 22 Jul 2026 17:13:26 +0100 Subject: [PATCH 2/2] fix(deltawire): verify historical evidence in clean checkouts --- .github/workflows/deltawire-lab.yml | 4 ++++ .../eval/scripts/v5/verify_evidence_locks.py | 10 +++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deltawire-lab.yml b/.github/workflows/deltawire-lab.yml index 3d1982bcf..290526980 100644 --- a/.github/workflows/deltawire-lab.yml +++ b/.github/workflows/deltawire-lab.yml @@ -15,6 +15,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 + with: + # Evidence locks intentionally verify historical experiment commits + # that are preserved on non-main branches. + fetch-depth: 0 - uses: actions/setup-go@v7 with: go-version-file: labs/20-deltawire/go.mod diff --git a/labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py b/labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py index d37641929..26fc37a2e 100644 --- a/labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py +++ b/labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py @@ -6,13 +6,17 @@ EVAL=ROOT/"labs/20-deltawire/eval" def sha(data):return hashlib.sha256(data).hexdigest() def object_bytes(commit,name):return subprocess.check_output(["git","show",f"{commit}:{name}"],cwd=ROOT) +def tracked(name):return subprocess.run(["git","ls-files","--error-unmatch",name],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL).returncode==0 def main(): - evidence_commit="c200dc0eb3c4dff0e4732cc4b2daa1cd535703e3" - for path in (EVAL/"results/preflight-v1/treatment-probe-range-large/v1-evidence-lock.json",EVAL/"manifests/preflight-v2-evidence-lock.json"): + historical=( + (EVAL/"results/preflight-v1/treatment-probe-range-large/v1-evidence-lock.json","e6abfc48e4874fc138bba321e9d60f95f39721f8"), + (EVAL/"manifests/preflight-v2-evidence-lock.json","103261d89b424eb572361797681f1c8177e72da3"), + ) + for path,evidence_commit in historical: lock=json.loads(path.read_text()) for name,want in lock["immutable_files"].items(): target=ROOT/name - data=target.read_bytes() if target.is_file() else object_bytes(evidence_commit,name) + data=target.read_bytes() if tracked(name) and target.is_file() else object_bytes(evidence_commit,name) if sha(data)!=want:raise SystemExit(f"historical evidence mismatch: {name}") v3=json.loads((EVAL/"manifests/preflight-v3-evidence-lock.json").read_text());v3_commit=v3["source_commit"] if subprocess.check_output(["git","rev-parse",f"{v3_commit}^{{tree}}"],cwd=ROOT,text=True).strip()!=v3["source_tree"]:raise SystemExit("v3 source tree mismatch")