From 49c2320d66e04fe43958d66e0daa9cfc595d3c69 Mon Sep 17 00:00:00 2001 From: bigboateng Date: Wed, 22 Jul 2026 20:14:07 +0100 Subject: [PATCH] eval: integrate terminal v6 runtime boundary --- .github/workflows/deltawire-runtime.yml | 37 ++++ labs/20-deltawire/eval/agents/__init__.py | 1 + .../eval/agents/preinstalled_gemini_cli.py | 73 ++++++ .../agents/test_preinstalled_gemini_cli.py | 60 +++++ .../environment/.gitkeep | 2 + .../instruction.md | 1 + .../preinstalled-gemini-setup-v1/task.toml | 30 +++ .../tests/test.sh | 3 + .../docs/10-preinstalled-gemini-boundary.md | 29 +++ .../preflight-v6-terminal-evidence-lock.json | 207 ++++++++++++++++++ .../preflight-v6-terminal/summary.json | 15 ++ .../eval/runtime/preinstalled-gemini-v1.json | 13 ++ .../eval/scripts/v6/artifact_manifest.py | 31 +++ .../eval/scripts/v6/environment_receipt.py | 47 ++++ .../eval/scripts/v6/negative_controls.py | 97 ++++++++ labs/20-deltawire/eval/scripts/v6/release.py | 29 +++ .../eval/scripts/v6/run_setup_conformance.py | 191 ++++++++++++++++ .../eval/scripts/v6/setup_receipt.py | 60 +++++ .../eval/scripts/v6/shell_observation.py | 70 ++++++ .../eval/scripts/v6/test_artifact_manifest.py | 27 +++ .../scripts/v6/test_environment_receipt.py | 35 +++ .../eval/scripts/v6/test_release.py | 57 +++++ .../eval/scripts/v6/test_setup_receipt.py | 67 ++++++ .../eval/scripts/v6/test_shell_observation.py | 19 ++ .../scripts/v6/test_terminal_evidence_lock.py | 37 ++++ .../eval/scripts/v6/test_trial_status.py | 43 ++++ .../eval/scripts/v6/trial_status.py | 45 ++++ .../v6/verify_terminal_evidence_lock.py | 67 ++++++ .../eval/scripts/validate_dataset.sh | 3 + 29 files changed, 1396 insertions(+) create mode 100644 .github/workflows/deltawire-runtime.yml create mode 100644 labs/20-deltawire/eval/agents/__init__.py create mode 100644 labs/20-deltawire/eval/agents/preinstalled_gemini_cli.py create mode 100644 labs/20-deltawire/eval/agents/test_preinstalled_gemini_cli.py create mode 100644 labs/20-deltawire/eval/conformance/preinstalled-gemini-setup-v1/environment/.gitkeep create mode 100644 labs/20-deltawire/eval/conformance/preinstalled-gemini-setup-v1/instruction.md create mode 100644 labs/20-deltawire/eval/conformance/preinstalled-gemini-setup-v1/task.toml create mode 100644 labs/20-deltawire/eval/conformance/preinstalled-gemini-setup-v1/tests/test.sh create mode 100644 labs/20-deltawire/eval/docs/10-preinstalled-gemini-boundary.md create mode 100644 labs/20-deltawire/eval/manifests/preflight-v6-terminal-evidence-lock.json create mode 100644 labs/20-deltawire/eval/results/preflight-v6-terminal/summary.json create mode 100644 labs/20-deltawire/eval/runtime/preinstalled-gemini-v1.json create mode 100644 labs/20-deltawire/eval/scripts/v6/artifact_manifest.py create mode 100644 labs/20-deltawire/eval/scripts/v6/environment_receipt.py create mode 100644 labs/20-deltawire/eval/scripts/v6/negative_controls.py create mode 100644 labs/20-deltawire/eval/scripts/v6/release.py create mode 100644 labs/20-deltawire/eval/scripts/v6/run_setup_conformance.py create mode 100644 labs/20-deltawire/eval/scripts/v6/setup_receipt.py create mode 100644 labs/20-deltawire/eval/scripts/v6/shell_observation.py create mode 100644 labs/20-deltawire/eval/scripts/v6/test_artifact_manifest.py create mode 100644 labs/20-deltawire/eval/scripts/v6/test_environment_receipt.py create mode 100644 labs/20-deltawire/eval/scripts/v6/test_release.py create mode 100644 labs/20-deltawire/eval/scripts/v6/test_setup_receipt.py create mode 100644 labs/20-deltawire/eval/scripts/v6/test_shell_observation.py create mode 100644 labs/20-deltawire/eval/scripts/v6/test_terminal_evidence_lock.py create mode 100644 labs/20-deltawire/eval/scripts/v6/test_trial_status.py create mode 100644 labs/20-deltawire/eval/scripts/v6/trial_status.py create mode 100644 labs/20-deltawire/eval/scripts/v6/verify_terminal_evidence_lock.py diff --git a/.github/workflows/deltawire-runtime.yml b/.github/workflows/deltawire-runtime.yml new file mode 100644 index 000000000..be4c3525c --- /dev/null +++ b/.github/workflows/deltawire-runtime.yml @@ -0,0 +1,37 @@ +name: Verify DeltaWire runtime integration + +on: + pull_request: + paths: + - "labs/20-deltawire/**" + - ".github/workflows/deltawire-runtime.yml" + push: + branches: [main] + paths: + - "labs/20-deltawire/**" + - ".github/workflows/deltawire-runtime.yml" + +permissions: + contents: read + +jobs: + validate-runtime: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@v7 + - run: python3 labs/20-deltawire/eval/scripts/v6/verify_terminal_evidence_lock.py + - run: python3 labs/20-deltawire/eval/scripts/v6/test_terminal_evidence_lock.py + - run: python3 labs/20-deltawire/eval/scripts/v6/test_release.py + - run: python3 labs/20-deltawire/eval/scripts/v6/test_environment_receipt.py + - run: python3 labs/20-deltawire/eval/scripts/v6/test_artifact_manifest.py + - run: python3 labs/20-deltawire/eval/scripts/v6/test_setup_receipt.py + - run: python3 labs/20-deltawire/eval/scripts/v6/test_trial_status.py + - run: python3 labs/20-deltawire/eval/scripts/v6/test_shell_observation.py + - run: python3 labs/20-deltawire/eval/scripts/v6/negative_controls.py --self-test + - run: PYTHONPATH=labs/20-deltawire/eval uv run --with harbor==0.20.0 python labs/20-deltawire/eval/agents/test_preinstalled_gemini_cli.py diff --git a/labs/20-deltawire/eval/agents/__init__.py b/labs/20-deltawire/eval/agents/__init__.py new file mode 100644 index 000000000..49302ad77 --- /dev/null +++ b/labs/20-deltawire/eval/agents/__init__.py @@ -0,0 +1 @@ +"""Versioned Harbor custom agents for DeltaWire evaluation.""" diff --git a/labs/20-deltawire/eval/agents/preinstalled_gemini_cli.py b/labs/20-deltawire/eval/agents/preinstalled_gemini_cli.py new file mode 100644 index 000000000..3fdc213a9 --- /dev/null +++ b/labs/20-deltawire/eval/agents/preinstalled_gemini_cli.py @@ -0,0 +1,73 @@ +"""Harbor Gemini CLI agent with a verification-only install boundary.""" +import json +import shlex + +from harbor.agents.installed.gemini_cli import GeminiCli + +AGENT_NAME = "preinstalled-gemini-cli-v1" +EXPECTED_NODE_PATH = "/usr/local/bin/node" +EXPECTED_NODE_VERSION = "v22.23.1" +EXPECTED_NPM_VERSION = "10.9.8" +EXPECTED_GEMINI_PATH = "/usr/local/bin/gemini" +EXPECTED_GEMINI_VERSION = "0.51.0" +EXPECTED_PACKAGE_VERSION = "0.51.0" +EXPECTED_IMAGE_ID = "sha256:589fce58d7ddf09910876afe0198bcee0086a840af6de7c08515bbce3e8367a7" +RECEIPT_PATH = "/logs/artifacts/preinstalled-gemini/setup-receipt.json" + + +class PreinstalledGeminiCli(GeminiCli): + @staticmethod + def name() -> str: + return AGENT_NAME + + async def install(self, environment) -> None: + async def checked(command: str) -> str: + result = await self.exec_as_agent(environment, command=command) + if result.return_code != 0: + raise RuntimeError(f"preinstalled runtime verification failed: {command}") + return (result.stdout or "").strip() + + node_path = await checked("command -v node") + node_version = await checked("node --version") + npm_version = await checked("npm --version") + gemini_path = await checked("command -v gemini") + gemini_version = await checked("gemini --version") + package_version = await checked( + "node -p \"require('/usr/local/lib/node_modules/@google/gemini-cli/package.json').version\"" + ) + checks = { + "agent_identity": self.name() == AGENT_NAME, + "gemini_executable": gemini_path == EXPECTED_GEMINI_PATH, + "gemini_version": gemini_version == EXPECTED_GEMINI_VERSION, + "image_id": EXPECTED_IMAGE_ID.startswith("sha256:") and len(EXPECTED_IMAGE_ID) == 71, + "node_executable": node_path == EXPECTED_NODE_PATH, + "node_version": node_version == EXPECTED_NODE_VERSION, + "npm_version": npm_version == EXPECTED_NPM_VERSION, + "package_identity": package_version == EXPECTED_PACKAGE_VERSION, + } + receipt = { + "agent": AGENT_NAME, + "checks": checks, + "gemini_path": gemini_path, + "gemini_version": gemini_version, + "image_id": EXPECTED_IMAGE_ID, + "node_path": node_path, + "node_version": node_version, + "npm_version": npm_version, + "package_version": package_version, + "schema_version": "preinstalled-gemini-setup-receipt.v1", + "status": "pass" if all(checks.values()) else "fail", + } + payload = json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n" + parent = RECEIPT_PATH.rsplit("/", 1)[0] + command = ( + f"mkdir -p {shlex.quote(parent)} && " + f"temporary={shlex.quote(RECEIPT_PATH + '.tmp')} && " + f"printf %s {shlex.quote(payload)} > \"$temporary\" && " + f"mv \"$temporary\" {shlex.quote(RECEIPT_PATH)}" + ) + result = await self.exec_as_agent(environment, command=command) + if result.return_code != 0: + raise RuntimeError("preinstalled setup receipt write failed") + if receipt["status"] != "pass": + raise RuntimeError("preinstalled runtime identity mismatch") diff --git a/labs/20-deltawire/eval/agents/test_preinstalled_gemini_cli.py b/labs/20-deltawire/eval/agents/test_preinstalled_gemini_cli.py new file mode 100644 index 000000000..d569bb1b7 --- /dev/null +++ b/labs/20-deltawire/eval/agents/test_preinstalled_gemini_cli.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +import ast +import asyncio +import importlib +import inspect +from pathlib import Path +from types import SimpleNamespace + +MODULE = importlib.import_module("agents.preinstalled_gemini_cli") +CLASS = MODULE.PreinstalledGeminiCli + + +def structural_test() -> None: + source = Path(inspect.getsourcefile(CLASS)).read_text() + tree = ast.parse(source) + node = next(item for item in tree.body if isinstance(item, ast.ClassDef) and item.name == CLASS.__name__) + methods = {item.name for item in node.body if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef))} + inherited = {name for name in methods if hasattr(CLASS.__mro__[1], name)} + assert inherited == {"name", "install"} + lowered = inspect.getsource(CLASS.install).lower() + for forbidden in ("apt-get", "apt ", "curl", "wget", "nvm", "npm install", "npm update"): + assert forbidden not in lowered + assert CLASS.run is CLASS.__mro__[1].run + + +def functional_test() -> None: + outputs = iter( + [ + "/usr/local/bin/node\n", + "v22.23.1\n", + "10.9.8\n", + "/usr/local/bin/gemini\n", + "0.51.0\n", + "0.51.0\n", + "", + ] + ) + commands = [] + + async def fake_exec(_environment, command): + commands.append(command) + return SimpleNamespace(return_code=0, stdout=next(outputs), stderr="") + + instance = object.__new__(CLASS) + instance.exec_as_agent = fake_exec + original = MODULE.EXPECTED_IMAGE_ID + MODULE.EXPECTED_IMAGE_ID = "sha256:" + "1" * 64 + try: + asyncio.run(instance.install(object())) + finally: + MODULE.EXPECTED_IMAGE_ID = original + assert len(commands) == 7 + assert "setup-receipt.json.tmp" in commands[-1] + assert '"status":"pass"' in commands[-1] + + +if __name__ == "__main__": + structural_test() + functional_test() + print("Preinstalled Gemini agent inheritance and install tests passed.") diff --git a/labs/20-deltawire/eval/conformance/preinstalled-gemini-setup-v1/environment/.gitkeep b/labs/20-deltawire/eval/conformance/preinstalled-gemini-setup-v1/environment/.gitkeep new file mode 100644 index 000000000..5b2efa12f --- /dev/null +++ b/labs/20-deltawire/eval/conformance/preinstalled-gemini-setup-v1/environment/.gitkeep @@ -0,0 +1,2 @@ +# Harbor task discovery requires the environment directory even when task.toml +# selects a prebuilt docker_image and no Dockerfile is used. diff --git a/labs/20-deltawire/eval/conformance/preinstalled-gemini-setup-v1/instruction.md b/labs/20-deltawire/eval/conformance/preinstalled-gemini-setup-v1/instruction.md new file mode 100644 index 000000000..2fcb2ecec --- /dev/null +++ b/labs/20-deltawire/eval/conformance/preinstalled-gemini-setup-v1/instruction.md @@ -0,0 +1 @@ +Verify the preinstalled Gemini CLI agent runtime. Do not execute an agent or model. diff --git a/labs/20-deltawire/eval/conformance/preinstalled-gemini-setup-v1/task.toml b/labs/20-deltawire/eval/conformance/preinstalled-gemini-setup-v1/task.toml new file mode 100644 index 000000000..4487f9484 --- /dev/null +++ b/labs/20-deltawire/eval/conformance/preinstalled-gemini-setup-v1/task.toml @@ -0,0 +1,30 @@ +schema_version = "1.3" +artifacts = [] + +[task] +name = "operatorstack/preinstalled-gemini-setup-v1" +description = "Credential-free install-only conformance for the v6 agent runtime" +authors = [] +keywords = [] + +[metadata] +benchmark_result = false +conformance_version = "v1" + +[verifier] +timeout_sec = 60.0 + +[[verifier.collect]] +service = "main" +command = "python3 /usr/local/bin/environment_receipt.py --expectations /deltawire-environment-expectations.json --receipt /logs/artifacts/deltawire/environment-receipt.json" +timeout_sec = 30.0 + +[agent] +timeout_sec = 60.0 + +[environment] +docker_image = "deltawire-preflight-v6:runtime-v1" +network_mode = "no-network" +build_timeout_sec = 60.0 +os = "linux" +mcp_servers = [] diff --git a/labs/20-deltawire/eval/conformance/preinstalled-gemini-setup-v1/tests/test.sh b/labs/20-deltawire/eval/conformance/preinstalled-gemini-setup-v1/tests/test.sh new file mode 100644 index 000000000..9879df4c3 --- /dev/null +++ b/labs/20-deltawire/eval/conformance/preinstalled-gemini-setup-v1/tests/test.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -euo pipefail +exit 0 diff --git a/labs/20-deltawire/eval/docs/10-preinstalled-gemini-boundary.md b/labs/20-deltawire/eval/docs/10-preinstalled-gemini-boundary.md new file mode 100644 index 000000000..76b2ab43c --- /dev/null +++ b/labs/20-deltawire/eval/docs/10-preinstalled-gemini-boundary.md @@ -0,0 +1,29 @@ +# Preinstalled Gemini CLI boundary + +DeltaWire preflight v6 separates agent bootstrap from paid treatment behavior. The task image contains the runtime; Harbor's custom agent performs local identity checks only. + +## Harbor 0.20.0 grounding + +- Custom agents use the `module.path:ClassName` import form. V6 uses `agents.preinstalled_gemini_cli:PreinstalledGeminiCli` with the eval directory on `PYTHONPATH`. +- `harbor run --install-only` performs environment and agent setup without agent execution or model verification. `--model` is optional on that path. +- Harbor's official `GeminiCli.install` begins at line 109 of `harbor/agents/installed/gemini_cli.py` and performs dynamic `apt-get`, NVM, and npm installation. Its SHA-256 in Harbor 0.20.0 is `dc8ce81e8f41b78f00abb54cd91fd7840d147f442960ca32e68c7a760e060a2a`. +- The inherited official `GeminiCli.run` begins at line 775 in that same frozen source. The custom class overrides only `name` and `install`; execution, authentication, settings, skills, prompt, trajectory, ATIF, token, cost, and resume behavior remain Harbor's implementation. + +Grounding commands: + +```text +uvx --from harbor==0.20.0 harbor run --help +uv run --with harbor==0.20.0 python +``` + +## Deterministic setup proof + +The image pins the Linux/amd64 base manifest, Node, npm, Gemini CLI package and DeltaWire binary. The custom install boundary runs only local path/version/package checks and writes a canonical setup receipt under `/logs/artifacts`. In Harbor 0.20.0, `--install-only` implies disabled verification, so verifier collect hooks do not run and `artifacts/manifest.json` is not emitted; Harbor still performs its automatic `/logs/artifacts` collection after agent setup. The paid path retains the main-service collect hook and artifact manifest as separate environment gates. Three credential-free install-only Harbor trials must agree on setup-receipt hash and image identity, finish setup within 60 seconds, and show no model, credential, skill, agent execution, verifier execution, provider use, or dynamic installation. + +Harbor process exit and trial exception are normalized independently. A zero Harbor process exit cannot turn a trial exception into a pass. + +## Authorization boundary + +The v6 runner exposes only `probe-v6` and `pair`. The paid probe is unavailable until the exact phrase `approve the paid v6 probe` is separately recorded after manifest-head review. The pair requires the later exact phrase `approve the frozen range-large/r1 pair`. Neither authorization is implied by building or validating v6. + +All terminal states keep `READY_FOR_72=false`, `pair_launched=false`, `six_run_canary_started=false`, and `full_run_started=false` until evidence says otherwise; v6 never exposes a full-run mode. diff --git a/labs/20-deltawire/eval/manifests/preflight-v6-terminal-evidence-lock.json b/labs/20-deltawire/eval/manifests/preflight-v6-terminal-evidence-lock.json new file mode 100644 index 000000000..2158c762c --- /dev/null +++ b/labs/20-deltawire/eval/manifests/preflight-v6-terminal-evidence-lock.json @@ -0,0 +1,207 @@ +{ + "schema_version": "deltawire-v6-terminal-evidence-lock.v1", + "sources": [ + { + "commit": "0dad5c01684b337f8a1bf42f17141f4789c95740", + "files": { + "labs/20-deltawire/eval/manifests/preflight-v6-range-large.json": "ea5a4b4073604cd940f3f016cae390ae214b48f281fd30ed4581eedd4f99bc7a", + "labs/20-deltawire/eval/results/preflight-v6/approvals/paid-v6-probe.json": "a4e4927c438d2600ce9cd8e0dee8bf5a7e99f49f421de9608394e8c971c6e9fa", + "labs/20-deltawire/eval/results/preflight-v6/approvals/pre-probe-validation.json": "5e51b48d677d8ef945becd077359a5e4bfdac1036bff26ad59c42fc8f74ab9d9", + "labs/20-deltawire/eval/results/preflight-v6/conformance/preinstalled-gemini-setup-v1.json": "6488387dbd1c4598be003aabc302ab9d37fe3d2c5eaab0d6a9263db6fad44f45", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/config.json": "85be64b52f53a67c79c7b654ab4a91f5e779b51bddd622ae9bcaf010a3d43712", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/job.log": "1ecb4067a02fe1142ea7d535825003b6d413c31b98fb1fc3c4c1f55f73916a3e", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/lock.json": "65b14219619b00c26f78c719cfcdaf8d042bd1a6f00c5e24c931636e1840047d", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/preinstalled-gemini-setup-v1__XCUXNb6/artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json": "a88d221e573265997797dcad31136aec0ed74a5bd37e50887595e5ea85cfa0d1", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/preinstalled-gemini-setup-v1__XCUXNb6/config.json": "29f7524ca459c21fb609e8edb8e2c74634a5c010a6f5db6780bdc2a42d113b8b", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/preinstalled-gemini-setup-v1__XCUXNb6/lock.json": "850d902431e78533c81138290891bc8b540adb7ccd88df39fd1c3ed8a946e2cf", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/preinstalled-gemini-setup-v1__XCUXNb6/result.json": "b48855441cc6d690a635baa4e01df2fa527958ee9544512da30a4f486ed95c54", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/preinstalled-gemini-setup-v1__XCUXNb6/trial.log": "1ecb4067a02fe1142ea7d535825003b6d413c31b98fb1fc3c4c1f55f73916a3e", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/result.json": "a2a192ea125102d2b27feb2ce1199cbcb2ea67a49215ea693d161d9239b57b6e", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/config.json": "9ea4ab2eaeb78ae6e5a3acc35bd9d54ee659a08ee2bc582b7dfae119e619d0b3", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/job.log": "1ecb4067a02fe1142ea7d535825003b6d413c31b98fb1fc3c4c1f55f73916a3e", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/lock.json": "118732ed7923df9af95ff56021010efd380760674a231a7fffd34654a1380a1e", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/preinstalled-gemini-setup-v1__8qbm7gq/artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json": "a88d221e573265997797dcad31136aec0ed74a5bd37e50887595e5ea85cfa0d1", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/preinstalled-gemini-setup-v1__8qbm7gq/config.json": "1c6e124060d841d946bac9afa993c119632e935f9bc9b5da84c3d32a905feef1", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/preinstalled-gemini-setup-v1__8qbm7gq/lock.json": "850d902431e78533c81138290891bc8b540adb7ccd88df39fd1c3ed8a946e2cf", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/preinstalled-gemini-setup-v1__8qbm7gq/result.json": "f011e4f6816b09476377abd949b497711dd853318a76757f0b4da3efd0d29f10", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/preinstalled-gemini-setup-v1__8qbm7gq/trial.log": "1ecb4067a02fe1142ea7d535825003b6d413c31b98fb1fc3c4c1f55f73916a3e", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/result.json": "a7ba1389c58415c243835566312b16736562b2ed8df84c47a6e5621124efda08", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/config.json": "c3c3e6a59f2c24a3668c7801a7db842d153000ac06b0336800d59bb59a7ec442", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/job.log": "1ecb4067a02fe1142ea7d535825003b6d413c31b98fb1fc3c4c1f55f73916a3e", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/lock.json": "80aef02cfff9fc59eb7467bda23fc1d3657610c07629cdf6df00e8acbe33dc8a", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/preinstalled-gemini-setup-v1__PuacLWm/artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json": "a88d221e573265997797dcad31136aec0ed74a5bd37e50887595e5ea85cfa0d1", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/preinstalled-gemini-setup-v1__PuacLWm/config.json": "d04bdda45279a562512c94d9709c82d270a03f105d071651cc8633ab65b0071a", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/preinstalled-gemini-setup-v1__PuacLWm/lock.json": "850d902431e78533c81138290891bc8b540adb7ccd88df39fd1c3ed8a946e2cf", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/preinstalled-gemini-setup-v1__PuacLWm/result.json": "0975b8c66193849d69caf5bb9dfabbb189e5415cd8ae97d6c8734a3505748d90", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/preinstalled-gemini-setup-v1__PuacLWm/trial.log": "1ecb4067a02fe1142ea7d535825003b6d413c31b98fb1fc3c4c1f55f73916a3e", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/result.json": "bfe98afda96acd6ed96fa39fb14fa720157010d7e4b23e07794f6e208224b24a", + "labs/20-deltawire/eval/results/preflight-v6/evidence-validation.json": "8b7742335a593b62da7a8151cf32cba72b4bbef9ca2166b67c5a4b30ad16ac89", + "labs/20-deltawire/eval/results/preflight-v6/post-probe-readiness.json": "db32b1c9411d5ebaa29fc9e2abde1cc1bb627ac8274ee36bead9345cf6927966", + "labs/20-deltawire/eval/results/preflight-v6/readiness.json": "e4b521320a259b226dbdbd8ecc2118b96e02973f8988f9bb563c9dec17e982dc", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/D1-artifact-manifest-receipt.json": "2b8c9f2817845dbbb8d3d0eae89f8005d7eb71f75b9e8df33f2e367b6850b1b0", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/D1-end-to-end.json": "39e4df8cd187534aaf1d26683d16b8ebbd279e62e945971ecddd3c4f0c44f578", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/D1-environment-receipt.json": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/D1-plan-contract-receipt.json": "41438f9e48d6bd15170f896be64a08894ae16afe2b517ec461e5e229c9f05c69", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/D1-semantic-result.json": "6378e7763dc0ad07d21d18076120f7a27856e19055057535d236982ef3679d0b", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/D1-setup-receipt-validation.json": "adca6d256d43f320bb542745bc37f58fcfa97db736eebcc2aa1ce885d84c5240", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/D1-treatment-use.json": "95e92414f70420ec09d94dd60594186fa39b994dca7c112a5285a672b56d0bdc", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/independent-recheck.json": "be6c7b154ce13053e8d06db89fd17a4a724c6a6f518b58b299c47d6f56d1d3a9", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/pre-pair-report.json": "33aba937891ee04a35f639b01ab20f2ee94d3ea72f0be35f017f471df583cbf7", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/probe-release.json": "b4e7880d3024488516bfe559b09fc699204cef3b31c9e00f8c78583f21e6ccce", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/config.json": "a7b4cd522dce89be44288a4a249bd8e87219866807aff5db75fa3df1a84820d2", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/job.log": "523c6c76b457b6c86a180ae1518f44a44e292a361341bdf05fbd61abaf219e39", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/lock.json": "5fd68e254be6655cdbac655ee8b3d900972496f882a9554fa8cd496871a2fc23", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/agent/gemini-cli.trajectory.jsonl": "a6060a94c1179634c5319163f6770aa35015a5f4be5264161ae3e97e5874c768", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/agent/gemini-cli.txt": "90df8e0ba690e6c3b74888d61dea279ddbfa4e96254da1e1baf3cf47a1783ce1", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/agent/trajectory.json": "af1c3810508fae524c9d127ab3b8f112cc5b44a31a502e9440c0c2f8f4d3ee7e", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/.deltawire/plan-contract-receipt.json": "41438f9e48d6bd15170f896be64a08894ae16afe2b517ec461e5e229c9f05c69", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/.deltawire/plans/range-large.dw.json": "fe81f8e427522a59f0379b02ed9535ce4a0f01a3cfc68411d6c045b7ea4b8e14", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/.deltawire/state.json": "7c69437d6459ca467cc19e282402992d22bc34b063b679c6933a485652eb969a", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/logs/artifacts/deltawire/environment-receipt.json": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json": "a88d221e573265997797dcad31136aec0ed74a5bd37e50887595e5ea85cfa0d1", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/manifest.json": "d3775d0ad85a9d1f6630c909bd8f9ad47a4b2a58b233fef0376a73f58dd91bb4", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/testdata/generated/output.ndjson": "9d205b57ab166d23c81927899f99b6ccece261c679c96979c58fbde975637776", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/config.json": "52e62989e2ced0442ed97b95cb0afecfcb92d3519f749d500776e465f5ef0325", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/lock.json": "907a2a1e3b05c17c68235882322e1c846810fdf030a0d1990ab2c6c62ecb5ead", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/result.json": "1beecae51c4e2a30cce01e6a6af5bba03346fbdd659906cf2ac6de615f2b6572", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/trial.log": "523c6c76b457b6c86a180ae1518f44a44e292a361341bdf05fbd61abaf219e39", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/verifier/reward.json": "36a492513204040f04d7bdf4146d84ede71353d8cc7df112fc3ebff66327683a", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/verifier/test-stdout.txt": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/result.json": "034fda6aec05ffda9c0410df548592cb603cae00adf0eba7d8e21ef007bb94c2", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/run-ledger.json": "c8d0a9ef209d00c358294becb52221a388d3240c8e157ea8e18c1247788329b1", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/security-scan.json": "ece9caa5361535ba84b23fe17ec15e24d5c880a6570c7c45173db6fdc2f6768c" + }, + "scopes": [ + "labs/20-deltawire/eval/manifests/preflight-v6-range-large.json", + "labs/20-deltawire/eval/results/preflight-v6" + ] + }, + { + "commit": "273e977c2f61e3b3f31dd675ee3512ef9182a695", + "files": { + "labs/20-deltawire/eval/manifests/preflight-v6-range-large.json": "ea5a4b4073604cd940f3f016cae390ae214b48f281fd30ed4581eedd4f99bc7a", + "labs/20-deltawire/eval/results/preflight-v6/approvals/paid-v6-probe.json": "a4e4927c438d2600ce9cd8e0dee8bf5a7e99f49f421de9608394e8c971c6e9fa", + "labs/20-deltawire/eval/results/preflight-v6/approvals/pre-pair-validation.json": "50a6aedd742b2a5ac652da048029d71ccef075f0db4b0550e7316e83d7b5f877", + "labs/20-deltawire/eval/results/preflight-v6/approvals/pre-probe-validation.json": "5e51b48d677d8ef945becd077359a5e4bfdac1036bff26ad59c42fc8f74ab9d9", + "labs/20-deltawire/eval/results/preflight-v6/approvals/range-large-r1-pair.json": "6cc42f58fd05188d09d8314db64d5d70d7894aac67ed25c23faa68966040e6bf", + "labs/20-deltawire/eval/results/preflight-v6/conformance/preinstalled-gemini-setup-v1.json": "6488387dbd1c4598be003aabc302ab9d37fe3d2c5eaab0d6a9263db6fad44f45", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/config.json": "85be64b52f53a67c79c7b654ab4a91f5e779b51bddd622ae9bcaf010a3d43712", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/job.log": "1ecb4067a02fe1142ea7d535825003b6d413c31b98fb1fc3c4c1f55f73916a3e", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/lock.json": "65b14219619b00c26f78c719cfcdaf8d042bd1a6f00c5e24c931636e1840047d", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/preinstalled-gemini-setup-v1__XCUXNb6/artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json": "a88d221e573265997797dcad31136aec0ed74a5bd37e50887595e5ea85cfa0d1", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/preinstalled-gemini-setup-v1__XCUXNb6/config.json": "29f7524ca459c21fb609e8edb8e2c74634a5c010a6f5db6780bdc2a42d113b8b", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/preinstalled-gemini-setup-v1__XCUXNb6/lock.json": "850d902431e78533c81138290891bc8b540adb7ccd88df39fd1c3ed8a946e2cf", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/preinstalled-gemini-setup-v1__XCUXNb6/result.json": "b48855441cc6d690a635baa4e01df2fa527958ee9544512da30a4f486ed95c54", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/preinstalled-gemini-setup-v1__XCUXNb6/trial.log": "1ecb4067a02fe1142ea7d535825003b6d413c31b98fb1fc3c4c1f55f73916a3e", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-1/2026-07-22__16-40-51/result.json": "a2a192ea125102d2b27feb2ce1199cbcb2ea67a49215ea693d161d9239b57b6e", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/config.json": "9ea4ab2eaeb78ae6e5a3acc35bd9d54ee659a08ee2bc582b7dfae119e619d0b3", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/job.log": "1ecb4067a02fe1142ea7d535825003b6d413c31b98fb1fc3c4c1f55f73916a3e", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/lock.json": "118732ed7923df9af95ff56021010efd380760674a231a7fffd34654a1380a1e", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/preinstalled-gemini-setup-v1__8qbm7gq/artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json": "a88d221e573265997797dcad31136aec0ed74a5bd37e50887595e5ea85cfa0d1", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/preinstalled-gemini-setup-v1__8qbm7gq/config.json": "1c6e124060d841d946bac9afa993c119632e935f9bc9b5da84c3d32a905feef1", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/preinstalled-gemini-setup-v1__8qbm7gq/lock.json": "850d902431e78533c81138290891bc8b540adb7ccd88df39fd1c3ed8a946e2cf", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/preinstalled-gemini-setup-v1__8qbm7gq/result.json": "f011e4f6816b09476377abd949b497711dd853318a76757f0b4da3efd0d29f10", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/preinstalled-gemini-setup-v1__8qbm7gq/trial.log": "1ecb4067a02fe1142ea7d535825003b6d413c31b98fb1fc3c4c1f55f73916a3e", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-2/2026-07-22__16-45-03/result.json": "a7ba1389c58415c243835566312b16736562b2ed8df84c47a6e5621124efda08", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/config.json": "c3c3e6a59f2c24a3668c7801a7db842d153000ac06b0336800d59bb59a7ec442", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/job.log": "1ecb4067a02fe1142ea7d535825003b6d413c31b98fb1fc3c4c1f55f73916a3e", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/lock.json": "80aef02cfff9fc59eb7467bda23fc1d3657610c07629cdf6df00e8acbe33dc8a", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/preinstalled-gemini-setup-v1__PuacLWm/artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json": "a88d221e573265997797dcad31136aec0ed74a5bd37e50887595e5ea85cfa0d1", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/preinstalled-gemini-setup-v1__PuacLWm/config.json": "d04bdda45279a562512c94d9709c82d270a03f105d071651cc8633ab65b0071a", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/preinstalled-gemini-setup-v1__PuacLWm/lock.json": "850d902431e78533c81138290891bc8b540adb7ccd88df39fd1c3ed8a946e2cf", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/preinstalled-gemini-setup-v1__PuacLWm/result.json": "0975b8c66193849d69caf5bb9dfabbb189e5415cd8ae97d6c8734a3505748d90", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/preinstalled-gemini-setup-v1__PuacLWm/trial.log": "1ecb4067a02fe1142ea7d535825003b6d413c31b98fb1fc3c4c1f55f73916a3e", + "labs/20-deltawire/eval/results/preflight-v6/conformance/raw/positive-3/2026-07-22__16-45-26/result.json": "bfe98afda96acd6ed96fa39fb14fa720157010d7e4b23e07794f6e208224b24a", + "labs/20-deltawire/eval/results/preflight-v6/evidence-validation.json": "8b7742335a593b62da7a8151cf32cba72b4bbef9ca2166b67c5a4b30ad16ac89", + "labs/20-deltawire/eval/results/preflight-v6/pair-evidence-closure.sha256": "20c7262a24e27fda998b04a4bd1a367fd801e7b60344b3b6870b9dc8fe7f94d7", + "labs/20-deltawire/eval/results/preflight-v6/pair-evidence-validation.json": "4e1ffacf58f4e8075d181009459d6ebb98c12c3edbdc5ae9af10b96579318264", + "labs/20-deltawire/eval/results/preflight-v6/post-pair-readiness.json": "3a318fecef1d358b3274d01a38d9d6db7c14397a6f845ea5e09d16f4f55a7412", + "labs/20-deltawire/eval/results/preflight-v6/post-probe-readiness.json": "db32b1c9411d5ebaa29fc9e2abde1cc1bb627ac8274ee36bead9345cf6927966", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/D1-artifact-manifest-receipt.json": "a917e8ac44395210d18a58159179b2ced2e22069fa25e2aa81c1a4e46c5c37b0", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/D1-end-to-end.json": "9e28f7c8f23b80036909cc820c7054df0e46d67e85b066483452156e2477c8f5", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/D1-environment-receipt.json": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/D1-plan-contract-receipt.json": "0559c00507e01fe17950cf5bb4f618c727da57ea11ba1dac29f35a3f62cb7899", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/D1-semantic-result.json": "6378e7763dc0ad07d21d18076120f7a27856e19055057535d236982ef3679d0b", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/D1-setup-receipt-validation.json": "adca6d256d43f320bb542745bc37f58fcfa97db736eebcc2aa1ce885d84c5240", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/D1-treatment-use.json": "a093dde0e121c9193e6ece124c394de847cf5f8229e204652dda92b7f929ac7c", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/evidence-hashes.sha256": "e581d5b72ca931198f54fd8cb811f677bbd906922d2ed4592d7210733acd90dc", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/pair-report.md": "52646cb6748469768fb6ab0b75932f8ba9f127cd7c5c3de6c40b313065509515", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/pair-summary.v2.json": "76eaafedf92267d01c93f2980e18c27348326a6109036e2c4a09211ac4383f00", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/config.json": "5aeb9f45203ea87764269636a2f90785c1d29cbf9d8b88e065cc632917647949", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/job.log": "8c773c7309b6814ab22a14058c171382c7183cdfb14394a5686cf446e371fbbf", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/lock.json": "07e9e7309bc410cecb15a2222c684e61f7d246ec690b509818d392f6384698b9", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/agent/gemini-cli.trajectory.jsonl": "994da1d3e8d52ca0341a31b90160e5f6be95379f7e411ec20aa252fb466939d1", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/agent/gemini-cli.txt": "eb5e8fff0cff39dd1c80eff3c147b4b1e9333b4bddd34f01517745a6a9efaa58", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/agent/trajectory.json": "bc1818778ae1391773bc6d0fc123abbf924936bbed0662646ce6026837831f9e", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/artifacts/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/artifacts/.deltawire/plan-contract-receipt.json": "0559c00507e01fe17950cf5bb4f618c727da57ea11ba1dac29f35a3f62cb7899", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/artifacts/.deltawire/plans/range-large.dw.json": "9e065135cde5c7bb5671a8ee12cc220c53fdc7ab664292e2c455028633ef3f13", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/artifacts/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/artifacts/.deltawire/state.json": "4c6bcc092e261346d8dd7ddcbc6a3cfc54e9908f08558840af43ba62dee1319d", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/artifacts/logs/artifacts/deltawire/environment-receipt.json": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json": "a88d221e573265997797dcad31136aec0ed74a5bd37e50887595e5ea85cfa0d1", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/artifacts/manifest.json": "d3775d0ad85a9d1f6630c909bd8f9ad47a4b2a58b233fef0376a73f58dd91bb4", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/artifacts/testdata/generated/output.ndjson": "9d205b57ab166d23c81927899f99b6ccece261c679c96979c58fbde975637776", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/config.json": "cb7a1e7d53d6585593b5dec9e70081af3cdbb4565f6ae2f2817a4f0d2f6f3ab7", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/lock.json": "210d9a16d3261951022c212162aae1cfc50cb085ad2e6bc5bc8060b9c56f4519", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/result.json": "572567cb7e83e61ea0e10956e465fe67eb09f1d3ad5c949e21b1378c744c7d1b", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/trial.log": "8c773c7309b6814ab22a14058c171382c7183cdfb14394a5686cf446e371fbbf", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/verifier/reward.json": "36a492513204040f04d7bdf4146d84ede71353d8cc7df112fc3ebff66327683a", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/pair-task__TwPpqPK/verifier/test-stdout.txt": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/raw/D1/2026-07-22__18-36-05/result.json": "06de12a6a91c2cba6eaadb856d42f949c8f3f2091c5e59dc928a0c298dca3c00", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/run-ledger.json": "b890510a100d18fe66b858d97100bbdffcbc06817de002b747015f43ee560213", + "labs/20-deltawire/eval/results/preflight-v6/range-large-r1/security-scan.json": "4f66171bbf8f93ecb981012b0d7e54f678ffedec141fc0c1fb47049ccd4c0b60", + "labs/20-deltawire/eval/results/preflight-v6/readiness.json": "e4b521320a259b226dbdbd8ecc2118b96e02973f8988f9bb563c9dec17e982dc", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/D1-artifact-manifest-receipt.json": "2b8c9f2817845dbbb8d3d0eae89f8005d7eb71f75b9e8df33f2e367b6850b1b0", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/D1-end-to-end.json": "39e4df8cd187534aaf1d26683d16b8ebbd279e62e945971ecddd3c4f0c44f578", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/D1-environment-receipt.json": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/D1-plan-contract-receipt.json": "41438f9e48d6bd15170f896be64a08894ae16afe2b517ec461e5e229c9f05c69", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/D1-semantic-result.json": "6378e7763dc0ad07d21d18076120f7a27856e19055057535d236982ef3679d0b", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/D1-setup-receipt-validation.json": "adca6d256d43f320bb542745bc37f58fcfa97db736eebcc2aa1ce885d84c5240", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/D1-treatment-use.json": "95e92414f70420ec09d94dd60594186fa39b994dca7c112a5285a672b56d0bdc", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/independent-recheck.json": "be6c7b154ce13053e8d06db89fd17a4a724c6a6f518b58b299c47d6f56d1d3a9", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/pre-pair-report.json": "33aba937891ee04a35f639b01ab20f2ee94d3ea72f0be35f017f471df583cbf7", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/probe-release.json": "b4e7880d3024488516bfe559b09fc699204cef3b31c9e00f8c78583f21e6ccce", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/config.json": "a7b4cd522dce89be44288a4a249bd8e87219866807aff5db75fa3df1a84820d2", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/job.log": "523c6c76b457b6c86a180ae1518f44a44e292a361341bdf05fbd61abaf219e39", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/lock.json": "5fd68e254be6655cdbac655ee8b3d900972496f882a9554fa8cd496871a2fc23", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/agent/gemini-cli.trajectory.jsonl": "a6060a94c1179634c5319163f6770aa35015a5f4be5264161ae3e97e5874c768", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/agent/gemini-cli.txt": "90df8e0ba690e6c3b74888d61dea279ddbfa4e96254da1e1baf3cf47a1783ce1", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/agent/trajectory.json": "af1c3810508fae524c9d127ab3b8f112cc5b44a31a502e9440c0c2f8f4d3ee7e", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/.deltawire/plan-contract-receipt.json": "41438f9e48d6bd15170f896be64a08894ae16afe2b517ec461e5e229c9f05c69", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/.deltawire/plans/range-large.dw.json": "fe81f8e427522a59f0379b02ed9535ce4a0f01a3cfc68411d6c045b7ea4b8e14", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/.deltawire/state.json": "7c69437d6459ca467cc19e282402992d22bc34b063b679c6933a485652eb969a", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/logs/artifacts/deltawire/environment-receipt.json": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json": "a88d221e573265997797dcad31136aec0ed74a5bd37e50887595e5ea85cfa0d1", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/manifest.json": "d3775d0ad85a9d1f6630c909bd8f9ad47a4b2a58b233fef0376a73f58dd91bb4", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/artifacts/testdata/generated/output.ndjson": "9d205b57ab166d23c81927899f99b6ccece261c679c96979c58fbde975637776", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/config.json": "52e62989e2ced0442ed97b95cb0afecfcb92d3519f749d500776e465f5ef0325", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/lock.json": "907a2a1e3b05c17c68235882322e1c846810fdf030a0d1990ab2c6c62ecb5ead", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/result.json": "1beecae51c4e2a30cce01e6a6af5bba03346fbdd659906cf2ac6de615f2b6572", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/trial.log": "523c6c76b457b6c86a180ae1518f44a44e292a361341bdf05fbd61abaf219e39", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/verifier/reward.json": "36a492513204040f04d7bdf4146d84ede71353d8cc7df112fc3ebff66327683a", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/range-large-v6__yAyBtNN/verifier/test-stdout.txt": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/raw/D1/2026-07-22__17-55-12/result.json": "034fda6aec05ffda9c0410df548592cb603cae00adf0eba7d8e21ef007bb94c2", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/run-ledger.json": "c8d0a9ef209d00c358294becb52221a388d3240c8e157ea8e18c1247788329b1", + "labs/20-deltawire/eval/results/preflight-v6/treatment-probe-range-large-v6/security-scan.json": "ece9caa5361535ba84b23fe17ec15e24d5c880a6570c7c45173db6fdc2f6768c" + }, + "scopes": [ + "labs/20-deltawire/eval/manifests/preflight-v6-range-large.json", + "labs/20-deltawire/eval/results/preflight-v6" + ] + } + ], + "terminal_classification": { + "B0": "not_run", + "D1_semantic": "pass", + "D1_treatment_fidelity": "fail", + "READY_FOR_72": false, + "pair_attempt_consumed": true, + "pair_valid": false, + "probe": "pass" + } +} diff --git a/labs/20-deltawire/eval/results/preflight-v6-terminal/summary.json b/labs/20-deltawire/eval/results/preflight-v6-terminal/summary.json new file mode 100644 index 000000000..36f59a72a --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v6-terminal/summary.json @@ -0,0 +1,15 @@ +{ + "B0": "not_run", + "D1_semantic": "pass", + "D1_treatment_fidelity": "fail", + "READY_FOR_72": false, + "evidence_commits": [ + "0dad5c01684b337f8a1bf42f17141f4789c95740", + "273e977c2f61e3b3f31dd675ee3512ef9182a695" + ], + "failure_class": "missing_structured_pre_activation_discovery", + "pair_attempt_consumed": true, + "pair_valid": false, + "probe": "pass", + "schema_version": "deltawire-v6-public-claim.v1" +} diff --git a/labs/20-deltawire/eval/runtime/preinstalled-gemini-v1.json b/labs/20-deltawire/eval/runtime/preinstalled-gemini-v1.json new file mode 100644 index 000000000..75c5551c4 --- /dev/null +++ b/labs/20-deltawire/eval/runtime/preinstalled-gemini-v1.json @@ -0,0 +1,13 @@ +{ + "base_image": "node:22.23.1-bookworm-slim@sha256:8607a9064d4a571140998ae9e52a3b3fcf9cff361d04642d5971e6cd76d39e27", + "base_image_linux_amd64_manifest": "sha256:8607a9064d4a571140998ae9e52a3b3fcf9cff361d04642d5971e6cd76d39e27", + "gemini_cli_npm_integrity": "sha512-Jx4fpzcp3Ya99cmFa1LBcfSdAnCNOSs7ZJbJd3uvprqUoG+kkyNnZVjjBOl84Y6dxlIgIF+5m8MGFRuSo93ijg==", + "gemini_cli_npm_shasum": "3144f4176635a340af254720be4d336af7a56d3a", + "gemini_cli_package": "@google/gemini-cli", + "gemini_cli_version": "0.51.0", + "image_id": "sha256:589fce58d7ddf09910876afe0198bcee0086a840af6de7c08515bbce3e8367a7", + "image_ref": "deltawire-preflight-v6:runtime-v1", + "node_version": "v22.23.1", + "npm_version": "10.9.8", + "schema_version": "preinstalled-gemini-runtime.v1" +} diff --git a/labs/20-deltawire/eval/scripts/v6/artifact_manifest.py b/labs/20-deltawire/eval/scripts/v6/artifact_manifest.py new file mode 100644 index 000000000..1b8344ed7 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v6/artifact_manifest.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import hashlib,json +from pathlib import Path + +SOURCE="/logs/artifacts" +DESTINATION="artifacts/logs/artifacts" +RECEIPT="logs/artifacts/deltawire/environment-receipt.json" +def sha(path):return hashlib.sha256(Path(path).read_bytes()).hexdigest() +def validate(trial,expected_binary_sha=None,expected_files=None): + trial=Path(trial);manifest_path=trial/"artifacts/manifest.json";receipt_path=trial/"artifacts"/RECEIPT + checks={"manifest_exists":manifest_path.is_file(),"receipt_canonical_path":receipt_path.is_file()} + manifest=[];receipt={} + try:manifest=json.loads(manifest_path.read_text()) if checks["manifest_exists"] else [] + except (OSError,json.JSONDecodeError):manifest=[] + entries=[e for e in manifest if isinstance(e,dict) and e.get("source")==SOURCE and e.get("destination")==DESTINATION] + checks["single_convention_entry"]=len(entries)==1 + checks["manifest_status_ok"]=len(entries)==1 and entries[0].get("status")=="ok" and entries[0].get("type")=="directory" + try:receipt=json.loads(receipt_path.read_text()) if checks["receipt_canonical_path"] else {} + except (OSError,json.JSONDecodeError):receipt={} + checks["receipt_schema_exact"]=receipt.get("schema_version")=="deltawire-environment-receipt.v1" + checks["receipt_status_pass"]=receipt.get("status")=="pass" + required={"binary_exists","binary_regular","binary_not_symlink","binary_executable","binary_path","binary_realpath","binary_hash","version_exit_0","version_exact","config_hash","schema_hash","public_contract_hash"} + receipt_checks=receipt.get("checks") if isinstance(receipt.get("checks"),dict) else {} + checks["receipt_checks_complete"]=set(receipt_checks)==required + checks["receipt_checks_true"]=set(receipt_checks)==required and all(receipt_checks.values()) + if expected_binary_sha is not None:checks["binary_hash_frozen"]=receipt.get("binary",{}).get("actual_sha256")==expected_binary_sha + for name,want in sorted((expected_files or {}).items()):checks[f"{name}_hash_frozen"]=receipt.get("files",{}).get(name,{}).get("actual_sha256")==want + return {"schema_version":"artifact-manifest-receipt.v1","status":"pass" if all(checks.values()) else "fail","checks":checks, + "artifact_manifest_sha256":sha(manifest_path) if manifest_path.is_file() else None, + "environment_receipt_sha256":sha(receipt_path) if receipt_path.is_file() else None, + "environment_receipt_path":str(receipt_path),"convention_entry":entries[0] if len(entries)==1 else None} diff --git a/labs/20-deltawire/eval/scripts/v6/environment_receipt.py b/labs/20-deltawire/eval/scripts/v6/environment_receipt.py new file mode 100644 index 000000000..84446c2d5 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v6/environment_receipt.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Atomically emit deterministic DeltaWire environment evidence.""" +import argparse,hashlib,json,os,stat,subprocess,tempfile +from pathlib import Path + +def sha(path): + h=hashlib.sha256() + with Path(path).open("rb") as f: + for chunk in iter(lambda:f.read(1048576),b""):h.update(chunk) + return h.hexdigest() +def atomic_write(path,data): + target=Path(path);target.parent.mkdir(parents=True,exist_ok=True) + fd,tmp=tempfile.mkstemp(prefix=f".{target.name}.",suffix=".tmp",dir=target.parent) + try: + with os.fdopen(fd,"w",encoding="utf-8") as f:f.write(data);f.flush();os.fsync(f.fileno()) + os.replace(tmp,target) + except BaseException: + try:os.unlink(tmp) + except FileNotFoundError:pass + raise +def build(expectations): + expected=json.loads(Path(expectations).read_text());binary=Path(expected["binary"]["path"]) + exists=binary.exists();lst=binary.lstat() if exists else None + regular=bool(lst and stat.S_ISREG(lst.st_mode));symlink=binary.is_symlink() if exists else False + executable=exists and os.access(binary,os.X_OK);realpath=str(binary.resolve()) if exists else None + try:version=subprocess.run([str(binary),"version"],capture_output=True,text=True,check=False) if exists else None + except OSError:version=None + files={} + for name,item in sorted(expected["files"].items()): + path=Path(item["path"]);actual=sha(path) if path.is_file() else None + files[name]={"path":item["path"],"exists":path.is_file(),"actual_sha256":actual,"expected_sha256":item["sha256"],"hash_match":actual==item["sha256"]} + binary_hash=sha(binary) if regular else None + checks={"binary_exists":exists,"binary_regular":regular,"binary_not_symlink":not symlink,"binary_executable":executable, + "binary_path":str(binary)==expected["binary"]["path"],"binary_realpath":realpath==expected["binary"]["realpath"], + "binary_hash":binary_hash==expected["binary"]["sha256"],"version_exit_0":bool(version and version.returncode==0), + "version_exact":bool(version and version.stdout.splitlines() and version.stdout.splitlines()[0].strip()==expected["binary"]["version"]), + **{f"{name}_hash":item["hash_match"] for name,item in files.items()}} + return {"schema_version":"deltawire-environment-receipt.v1","binary":{"path":str(binary),"realpath":realpath,"exists":exists, + "is_regular":regular,"is_symlink":symlink,"executable":executable,"actual_sha256":binary_hash, + "expected_sha256":expected["binary"]["sha256"],"version_stdout":version.stdout.strip() if version else None, + "version_stderr":version.stderr.strip() if version else None,"version_exit_code":version.returncode if version else None, + "expected_version":expected["binary"]["version"]},"files":files,"checks":checks,"status":"pass" if all(checks.values()) else "fail"} +def main(): + p=argparse.ArgumentParser();p.add_argument("--expectations",required=True);p.add_argument("--receipt",required=True);a=p.parse_args() + receipt=build(a.expectations);atomic_write(a.receipt,json.dumps(receipt,indent=2,sort_keys=True)+"\n") + print(json.dumps(receipt,sort_keys=True));raise SystemExit(0 if receipt["status"]=="pass" else 1) +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v6/negative_controls.py b/labs/20-deltawire/eval/scripts/v6/negative_controls.py new file mode 100644 index 000000000..c4d422d23 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v6/negative_controls.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +import ast +import argparse +import copy +import json +import tempfile +from pathlib import Path + +from setup_receipt import REQUIRED_CHECKS, validate + + +def metadata_gate(agent, model, credential_present, dynamic_install, agent_execution): + return ( + agent == "preinstalled-gemini-cli-v1" + and model is None + and not credential_present + and not dynamic_install + and agent_execution is None + ) + + +def run_negative_controls(runtime, agent_source): + base = { + "schema_version": "preinstalled-gemini-setup-receipt.v1", + "agent": "preinstalled-gemini-cli-v1", + "node_path": "/usr/local/bin/node", + "node_version": runtime["node_version"], + "npm_version": runtime["npm_version"], + "gemini_path": "/usr/local/bin/gemini", + "gemini_version": runtime["gemini_cli_version"], + "package_version": runtime["gemini_cli_version"], + "image_id": runtime["image_id"], + "checks": {name: True for name in sorted(REQUIRED_CHECKS)}, + "status": "pass", + } + mutations = { + "missing_node": ("node_path", None), + "wrong_node_version": ("node_version", "v22.22.3"), + "missing_gemini": ("gemini_path", None), + "wrong_gemini_version": ("gemini_version", "0.50.0"), + "wrong_package_identity": ("package_version", "0.50.0"), + "wrong_image_digest": ("image_id", "sha256:" + "0" * 64), + } + results = {} + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json" + path.parent.mkdir(parents=True) + for name, (field, value) in mutations.items(): + changed = copy.deepcopy(base) + changed[field] = value + path.write_text(json.dumps(changed) + "\n") + results[name] = validate(path, runtime)["status"] == "fail" + changed = copy.deepcopy(base) + changed["checks"]["gemini_executable"] = False + path.write_text(json.dumps(changed) + "\n") + results["gemini_non_executable"] = validate(path, runtime)["status"] == "fail" + path.write_text("malformed\n") + results["setup_receipt_malformed"] = validate(path, runtime)["status"] == "fail" + wrong = Path(tmp) / "setup-receipt.json" + wrong.write_text(json.dumps(base) + "\n") + results["setup_receipt_wrong_path"] = validate(wrong, runtime)["status"] == "fail" + source = Path(agent_source).read_text() + lowered = source.lower() + results["dynamic_install_rejected"] = not any( + token in lowered for token in ("apt-get", "curl ", "wget ", "nvm ", "npm install", "npm update") + ) + tree = ast.parse(source) + cls = next(node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == "PreinstalledGeminiCli") + methods = {node.name for node in cls.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))} + results["run_override_rejected"] = "run" not in methods + results["built_in_agent_rejected"] = not metadata_gate("gemini-cli", None, False, False, None) + results["model_rejected"] = not metadata_gate("preinstalled-gemini-cli-v1", "google/gemini", False, False, None) + results["credential_rejected"] = not metadata_gate("preinstalled-gemini-cli-v1", None, True, False, None) + results["agent_execution_rejected"] = not metadata_gate("preinstalled-gemini-cli-v1", None, False, False, {}) + if not all(results.values()): + raise RuntimeError(f"unexpectedly passing negative control: {results}") + return results + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--self-test", action="store_true", required=True) + args = parser.parse_args() + root = Path(__file__).resolve().parents[5] + eval_root = root / "labs/20-deltawire/eval" + runtime = json.loads((eval_root / "runtime/preinstalled-gemini-v1.json").read_text()) + # The structural controls do not require a built image. Use a valid-shaped + # deterministic identity while infrastructure is still being prepared. + if runtime["image_id"] == "UNFROZEN_IMAGE_ID": + runtime["image_id"] = "sha256:" + "1" * 64 + results = run_negative_controls(runtime, eval_root / "agents/preinstalled_gemini_cli.py") + assert args.self_test and len(results) >= 12 and all(results.values()) + print(f"Preinstalled Gemini negative controls passed ({len(results)} cases).") + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/scripts/v6/release.py b/labs/20-deltawire/eval/scripts/v6/release.py new file mode 100644 index 000000000..864d18a92 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v6/release.py @@ -0,0 +1,29 @@ +"""Content-addressed release verification independent of repository state.""" + +import hashlib +import json +from pathlib import Path + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def verify_release(release_path: Path, manifest_path: Path, root: Path, schema: str) -> dict: + if not release_path.is_file(): + raise RuntimeError(f"{schema} missing") + release = json.loads(release_path.read_text()) + release_id = release.pop("release_id", None) + wanted_id = hashlib.sha256( + json.dumps(release, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + release["release_id"] = release_id + if release.get("schema_version") != schema or release.get("status") != "pass": + raise RuntimeError("invalid release identity") + if release_id != wanted_id or release.get("manifest_sha256") != sha(manifest_path): + raise RuntimeError("invalid release hash") + for name, wanted in release.get("evidence_hashes", {}).items(): + target = root / name + if not target.is_file() or sha(target) != wanted: + raise RuntimeError(f"release evidence mismatch: {name}") + return release diff --git a/labs/20-deltawire/eval/scripts/v6/run_setup_conformance.py b/labs/20-deltawire/eval/scripts/v6/run_setup_conformance.py new file mode 100644 index 000000000..9774d56ee --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v6/run_setup_conformance.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""Run exactly three credential-free Harbor install-only trials.""" +import argparse +import json +import os +import shlex +import subprocess +from datetime import datetime +from pathlib import Path + +from negative_controls import metadata_gate, run_negative_controls +from setup_receipt import validate as validate_setup +from trial_status import normalize + +ROOT = Path(__file__).resolve().parents[5] +EVAL = ROOT / "labs/20-deltawire/eval" +TASK = EVAL / "conformance/preinstalled-gemini-setup-v1" +RUNTIME_PATH = EVAL / "runtime/preinstalled-gemini-v1.json" +AGENT_SOURCE = EVAL / "agents/preinstalled_gemini_cli.py" +AGENT_IMPORT = "agents.preinstalled_gemini_cli:PreinstalledGeminiCli" +HARBOR = ["uvx", "--from", "harbor==0.20.0", "harbor"] +FORBIDDEN = ("apt-get", "apt ", "curl ", "wget ", "nvm ", "npm install", "npm update") +CREDENTIALS = ( + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "GOOGLE_APPLICATION_CREDENTIALS", + "GEMINI_OAUTH_CREDS_PATH", + "GEMINI_FORCE_OAUTH", +) + + +def load(path): + return json.loads(Path(path).read_text()) + + +def trials(root): + found = [] + for path in Path(root).rglob("result.json"): + try: + value = load(path) + except (OSError, json.JSONDecodeError): + continue + if value.get("task_name") == "operatorstack/preinstalled-gemini-setup-v1": + found.append(path.parent) + return sorted(found) + + +def seconds(phase): + start = datetime.fromisoformat(phase["started_at"].replace("Z", "+00:00")) + finish = datetime.fromisoformat(phase["finished_at"].replace("Z", "+00:00")) + return (finish - start).total_seconds() + + +def inspect_trial(trial, runtime, harbor_exit_code, number): + result = load(trial / "result.json") + status = normalize(result, harbor_exit_code, install_only=True) + setup_path = trial / "artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json" + setup = validate_setup(setup_path, runtime) + logs = "\n".join(path.read_text(errors="replace") for path in trial.rglob("*.log")) + dynamic = [token for token in FORBIDDEN if token in logs.lower()] + agent = (result.get("agent_info") or {}).get("name") + model = (result.get("agent_info") or {}).get("model_info") + setup_seconds = seconds(result["agent_setup"]) + if status["trial_status"] != "install_only_pass" or setup["status"] != "pass": + raise SystemExit(f"setup conformance {number} receipt/status gate failed") + if setup_seconds >= 60 or dynamic or not metadata_gate(agent, model, False, False, result.get("agent_execution")): + raise SystemExit(f"setup conformance {number} identity/latency gate failed") + if result.get("agent_result") is not None or result.get("verifier_result") is not None: + raise SystemExit(f"setup conformance {number} unexpectedly executed agent or verifier") + if any((result.get("config") or {}).get("agent", {}).get("skills", [])): + raise SystemExit(f"setup conformance {number} unexpectedly received a skill") + logical = str(trial.relative_to(EVAL / "results/preflight-v6/conformance/raw")) + return { + "trial": number, + "trial_path": logical, + "harbor_exit_code": harbor_exit_code, + "agent": agent, + "model": None, + "credential": False, + "agent_execution": None, + "verifier_execution": None, + "setup_seconds": setup_seconds, + "setup_receipt_sha256": setup["receipt_sha256"], + "image_id": runtime["image_id"], + "gemini_version": runtime["gemini_cli_version"], + "dynamic_install_observed": False, + "provider_usage_observed": False, + "artifact_transport": "automatic_logs_artifact_collection", + "artifact_manifest_present": (trial / "artifacts/manifest.json").is_file(), + "environment_collect_hook_expected": False, + "status": "pass", + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--jobs-dir", required=True) + parser.add_argument("--report", required=True) + parser.add_argument( + "--continue-after-trial-one", + action="store_true", + help="Continue only after one retained, independently passing positive trial.", + ) + args = parser.parse_args() + jobs = Path(args.jobs_dir).resolve() + report = Path(args.report).resolve() + if report.exists(): + raise SystemExit("v6 setup conformance report already exists; rerun forbidden") + runtime = load(RUNTIME_PATH) + if not str(runtime.get("image_id", "")).startswith("sha256:"): + raise SystemExit("v6 image is not frozen") + actual_image = subprocess.check_output( + ["docker", "image", "inspect", runtime["image_ref"], "--format", "{{.Id}}"], text=True + ).strip() + if actual_image != runtime["image_id"]: + raise SystemExit("preinstalled image ID mismatch") + version = subprocess.check_output(HARBOR + ["--version"], text=True).strip() + if version != "0.20.0": + raise SystemExit(f"Harbor version mismatch: {version}") + existing = trials(jobs) if jobs.exists() else [] + if args.continue_after_trial_one: + if len(existing) != 1 or "positive-1" not in existing[0].parts: + raise SystemExit("continuation requires exactly one retained positive-1 trial") + elif jobs.exists() and any(jobs.iterdir()): + raise SystemExit("v6 setup conformance output is nonempty; rerun forbidden") + jobs.mkdir(parents=True, exist_ok=True) + env = os.environ.copy() + for name in CREDENTIALS: + env.pop(name, None) + env["PYTHONPATH"] = str(EVAL) + records = [] + if existing: + first = inspect_trial(existing[0], runtime, 0, 1) + first["command"] = "retained from interrupted three-trial conformance sequence" + records.append(first) + for number in ((2, 3) if existing else (1, 2, 3)): + target = jobs / f"positive-{number}" + before = set(trials(target)) + command = HARBOR + [ + "run", + "-a", + AGENT_IMPORT, + "--install-only", + "--agent-setup-timeout-multiplier", + "0.1666666667", + "-p", + str(TASK), + "-k", + "1", + "--jobs-dir", + str(target), + "--n-concurrent", + "1", + ] + run = subprocess.run(command, cwd=ROOT, env=env, check=False) + created = set(trials(target)) - before + if run.returncode != 0 or len(created) != 1: + raise SystemExit(f"setup conformance {number} failed: rc={run.returncode}, trials={len(created)}") + trial = created.pop() + record = inspect_trial(trial, runtime, run.returncode, number) + record["command"] = shlex.join( + HARBOR + + ["run", "-a", AGENT_IMPORT, "--install-only", "-p", str(TASK.relative_to(ROOT)), "-k", "1", "--jobs-dir", f"/positive-{number}", "--n-concurrent", "1"] + ) + records.append(record) + if len({item["setup_receipt_sha256"] for item in records}) != 1: + raise SystemExit("setup receipt hashes differ") + negatives = run_negative_controls(runtime, AGENT_SOURCE) + value = { + "schema_version": "preinstalled-gemini-conformance.v1", + "status": "pass", + "harbor_version": version, + "agent_import": AGENT_IMPORT, + "image_id": runtime["image_id"], + "model": None, + "credential": False, + "zero_model_calls": True, + "install_only_collect_hooks_disabled": True, + "setup_receipt_transport": "automatic /logs/artifacts collection after agent setup", + "positive_trials": records, + "stable_setup_receipt_sha256": records[0]["setup_receipt_sha256"], + "negative_controls": negatives, + "READY_FOR_72": False, + } + report.parent.mkdir(parents=True, exist_ok=True) + report.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + print(report) + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/scripts/v6/setup_receipt.py b/labs/20-deltawire/eval/scripts/v6/setup_receipt.py new file mode 100644 index 000000000..9239f84b9 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v6/setup_receipt.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +import hashlib +import json +from pathlib import Path + +SCHEMA = "preinstalled-gemini-setup-receipt.v1" +REQUIRED_CHECKS = { + "agent_identity", + "gemini_executable", + "gemini_version", + "image_id", + "node_executable", + "node_version", + "npm_version", + "package_identity", +} + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def validate(path, runtime): + path = Path(path) + checks = { + "receipt_exists": path.is_file(), + "receipt_canonical_path": path.as_posix().endswith( + "/artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json" + ), + } + try: + value = json.loads(path.read_text()) if path.is_file() else {} + except (OSError, json.JSONDecodeError): + value = {} + receipt_checks = value.get("checks") if isinstance(value.get("checks"), dict) else {} + checks.update( + { + "schema_exact": value.get("schema_version") == SCHEMA, + "agent_exact": value.get("agent") == "preinstalled-gemini-cli-v1", + "node_path_exact": value.get("node_path") == "/usr/local/bin/node", + "node_version_exact": value.get("node_version") == runtime["node_version"], + "npm_version_exact": value.get("npm_version") == runtime["npm_version"], + "gemini_path_exact": value.get("gemini_path") == "/usr/local/bin/gemini", + "gemini_version_exact": value.get("gemini_version") == runtime["gemini_cli_version"], + "package_version_exact": value.get("package_version") == runtime["gemini_cli_version"], + "image_id_exact": value.get("image_id") == runtime["image_id"], + "checks_exact": set(receipt_checks) == REQUIRED_CHECKS, + "checks_true": set(receipt_checks) == REQUIRED_CHECKS and all(receipt_checks.values()), + "status_pass": value.get("status") == "pass", + "no_timestamps": not any("time" in str(key).lower() for key in value), + } + ) + status = "pass" if all(checks.values()) else "fail" + return { + "schema_version": "preinstalled-gemini-setup-validation.v1", + "status": status, + "checks": checks, + "receipt_sha256": sha(path) if path.is_file() else None, + "receipt": value, + } diff --git a/labs/20-deltawire/eval/scripts/v6/shell_observation.py b/labs/20-deltawire/eval/scripts/v6/shell_observation.py new file mode 100644 index 000000000..1d816d864 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v6/shell_observation.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Canonicalize structured Gemini shell calls without trusting tool status.""" +import json,re,shlex +from pathlib import Path + +SHELL_NAMES={"run_shell_command","shell","execute"} +EXIT_KEYS=("exit_code","exitCode","returncode","return_code") +TRAILER=re.compile(r"(?:^|\n)Exit Code:\s*(-?\d+)\nProcess Group PGID:\s*\d+\s*(?:)?\s*$") + +def calls(path): + out=[] + for line in Path(path).read_text(encoding="utf-8",errors="replace").splitlines(): + try: event=json.loads(line) + except json.JSONDecodeError: continue + for call in event.get("toolCalls",[]): + outputs=[] + for item in call.get("result",[]): + response=(item.get("functionResponse") or {}).get("response") or {} + outputs.append(str(response.get("output",""))) + out.append({"name":call.get("name"),"args":call.get("args") or {},"tool_status":call.get("status"),"result":call.get("result") or [],"output":"\n".join(outputs)}) + return out + +def direct_code(value): + if isinstance(value,dict): + for key in EXIT_KEYS: + if key in value and isinstance(value[key],int): return value[key] + for child in value.values(): + found=direct_code(child) + if found is not None:return found + elif isinstance(value,list): + for child in value: + found=direct_code(child) + if found is not None:return found + return None + +def observation(call): + command=str(call["args"].get("command") or call["args"].get("cmd") or "") + code=direct_code(call["result"]); provenance="structured" if code is not None else None + if code is None: + matches=TRAILER.findall(call["output"]) + if len(matches)==1: code=int(matches[0]); provenance="explicit_output" + return {"command":command,"tool_status":call["tool_status"],"shell_exit_code":code,"shell_exit_provenance":provenance,"observable":call["name"] in SHELL_NAMES and bool(command)} + +def lifecycle(observation): + command=observation["command"]; parts=re.split(r"(&&|\|\||;|\n)",command); commands=parts[::2]; operators=parts[1::2] + found=[] + for segment in commands: + try: tokens=shlex.split(segment) + except ValueError: continue + while tokens and ("=" in tokens[0] and not tokens[0].startswith("/")): tokens=tokens[1:] + if tokens and tokens[0] in {"sudo","env"}: tokens=tokens[1:] + if len(tokens)>=2 and Path(tokens[0]).name=="deltawire" and tokens[1] in {"validate","render","check"}: + found.append({"command":tokens[1],"invocation":segment.strip(),"shell_exit_code":None}) + if len(found)==1 and len(commands)==1: found[0]["shell_exit_code"]=observation["shell_exit_code"] + elif found and observation["shell_exit_code"]==0 and operators and all(op=="&&" for op in operators): + for item in found:item["shell_exit_code"]=0 + return found + +def invokes_deltawire(observation): + for segment in re.split(r"&&|\|\||;|\n",observation["command"]): + try: tokens=shlex.split(segment) + except ValueError: continue + while tokens and ("=" in tokens[0] and not tokens[0].startswith("/")): tokens=tokens[1:] + if tokens and tokens[0] in {"sudo","env"}: tokens=tokens[1:] + if tokens and Path(tokens[0]).name=="deltawire": return True + return False + +def parse(path): + observations=[observation(c) for c in calls(path) if c["name"] in SHELL_NAMES] + return observations,[item for obs in observations for item in lifecycle(obs)] diff --git a/labs/20-deltawire/eval/scripts/v6/test_artifact_manifest.py b/labs/20-deltawire/eval/scripts/v6/test_artifact_manifest.py new file mode 100644 index 000000000..461fdfabe --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v6/test_artifact_manifest.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +import json,tempfile +from pathlib import Path +from artifact_manifest import validate + +def write(path,value):path.parent.mkdir(parents=True,exist_ok=True);path.write_text(json.dumps(value)+"\n") +def fixture(root): + trial=root/"trial";manifest=trial/"artifacts/manifest.json";receipt=trial/"artifacts/logs/artifacts/deltawire/environment-receipt.json" + checks={name:True for name in ("binary_exists","binary_regular","binary_not_symlink","binary_executable","binary_path","binary_realpath","binary_hash","version_exit_0","version_exact","config_hash","schema_hash","public_contract_hash")} + value={"schema_version":"deltawire-environment-receipt.v1","status":"pass","checks":checks,"binary":{"actual_sha256":"binary"},"files":{name:{"actual_sha256":name} for name in ("config","schema","public_contract")}} + write(receipt,value);write(manifest,[{"source":"/logs/artifacts","destination":"artifacts/logs/artifacts","type":"directory","status":"ok","service":None}]);return trial,manifest,receipt +def main(): + expected={name:name for name in ("config","schema","public_contract")} + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);trial,manifest,receipt=fixture(root);assert validate(trial,"binary",expected)["status"]=="pass" + cases=[] + cases.append(lambda:receipt.unlink()) + cases.append(lambda:write(receipt,{"bad":"json shape"})) + cases.append(lambda:write(manifest,[{"source":"/logs/artifacts","destination":"artifacts/logs/artifacts","type":"directory","status":"skipped"}])) + cases.append(lambda:write(manifest,[{"source":"/wrong","destination":"artifacts/logs/artifacts","type":"directory","status":"ok"}])) + cases.append(lambda:write(manifest,[])) + for mutate in cases: + trial,manifest,receipt=fixture(root);mutate();assert validate(trial,"binary",expected)["status"]=="fail" + trial,manifest,receipt=fixture(root);outside=trial/"artifacts/environment-receipt.json";outside.write_bytes(receipt.read_bytes());receipt.unlink();assert validate(trial,"binary",expected)["status"]=="fail" + trial,manifest,receipt=fixture(root);value=json.loads(receipt.read_text());value["binary"]["actual_sha256"]="v4";write(receipt,value);assert validate(trial,"binary",expected)["status"]=="fail" + print("Artifact-manifest and canonical receipt-path tests passed.") +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v6/test_environment_receipt.py b/labs/20-deltawire/eval/scripts/v6/test_environment_receipt.py new file mode 100644 index 000000000..4f586567f --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v6/test_environment_receipt.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +import hashlib,json,os,subprocess,tempfile +from pathlib import Path +HERE=Path(__file__).resolve().parent +def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() +def run(root,binary,mutate=None): + files={} + for name in ("config","schema","public_contract"): + p=root/f"{name}.json";p.write_text(name);files[name]={"path":str(p),"sha256":sha(p)} + exp={"binary":{"path":str(binary),"realpath":str(binary.resolve()),"sha256":sha(binary) if binary.is_file() else "0"*64,"version":"deltawire version dev"},"files":files} + if mutate:mutate(exp) + ep=root/"expect.json";rp=root/"receipt.json";ep.write_text(json.dumps(exp));result=subprocess.run(["python3",str(HERE/"environment_receipt.py"),"--expectations",str(ep),"--receipt",str(rp)],capture_output=True,text=True);return result.returncode,json.loads(rp.read_text()) +def main(): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);binary=root/"deltawire";binary.write_text("#!/bin/sh\nprintf 'deltawire version dev\\nsupported plan version deltawire.plan.v1\\n'\n");binary.chmod(0o755) + first=run(root,binary)[1];second=run(root,binary)[1];assert first["status"]=="pass" and first==second + assert run(root,binary,lambda e:e["binary"].update(sha256="0"*64))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["binary"].update(path=str(root/"wrong")))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["binary"].update(version="wrong"))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["files"]["config"].update(sha256="0"*64))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["files"]["schema"].update(sha256="0"*64))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["files"]["public_contract"].update(sha256="0"*64))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["binary"].update(realpath="/wrong"))[1]["status"]=="fail" + binary.chmod(0o644);assert run(root,binary)[1]["status"]=="fail";binary.chmod(0o755) + link=root/"link";link.symlink_to(binary);assert run(root,link)[1]["status"]=="fail" + missing=root/"missing";code,receipt=run(root,missing);assert code and receipt["status"]=="fail" + failing=root/"failing";failing.write_text("#!/bin/sh\nexit 7\n");failing.chmod(0o755);assert run(root,failing)[1]["binary"]["version_exit_code"]==7 + atomic=root/"atomic-target";atomic.mkdir();files={} + for name in ("config","schema","public_contract"): + p=root/f"atomic-{name}.json";p.write_text(name);files[name]={"path":str(p),"sha256":sha(p)} + exp={"binary":{"path":str(binary),"realpath":str(binary.resolve()),"sha256":sha(binary),"version":"deltawire version dev"},"files":files} + ep=root/"atomic-expect.json";ep.write_text(json.dumps(exp));result=subprocess.run(["python3",str(HERE/"environment_receipt.py"),"--expectations",str(ep),"--receipt",str(atomic)],capture_output=True,text=True) + assert result.returncode and atomic.is_dir() and not list(root.glob(".atomic-target.*.tmp")) + print("Deterministic environment receipt tests passed.") +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v6/test_release.py b/labs/20-deltawire/eval/scripts/v6/test_release.py new file mode 100644 index 000000000..fff889dbb --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v6/test_release.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +import copy +import hashlib +import json +import tempfile +from pathlib import Path + +from release import verify_release + + +def write(path: Path, value) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, sort_keys=True) + "\n" if not isinstance(value, str) else value) + + +def expect_failure(*args) -> None: + try: + verify_release(*args) + except RuntimeError: + return + raise AssertionError("release verifier accepted an invalid fixture") + + +def main() -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manifest = root / "manifest.json" + evidence = root / "evidence.json" + release_path = root / "release.json" + write(manifest, {"frozen": True}) + write(evidence, {"status": "pass"}) + body = { + "schema_version": "fixture-release.v1", + "status": "pass", + "manifest_sha256": hashlib.sha256(manifest.read_bytes()).hexdigest(), + "evidence_hashes": {"evidence.json": hashlib.sha256(evidence.read_bytes()).hexdigest()}, + } + body["release_id"] = hashlib.sha256( + json.dumps(body, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + write(release_path, body) + verify_release(release_path, manifest, root, "fixture-release.v1") + + missing = root / "missing-release.json" + expect_failure(missing, manifest, root, "fixture-release.v1") + changed = copy.deepcopy(body) + changed["release_id"] = "0" * 64 + write(release_path, changed) + expect_failure(release_path, manifest, root, "fixture-release.v1") + write(release_path, body) + evidence.write_text("changed\n") + expect_failure(release_path, manifest, root, "fixture-release.v1") + print("Temporary-fixture release tests passed.") + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/scripts/v6/test_setup_receipt.py b/labs/20-deltawire/eval/scripts/v6/test_setup_receipt.py new file mode 100644 index 000000000..aba2bcd44 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v6/test_setup_receipt.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +import copy +import json +import tempfile +from pathlib import Path + +from setup_receipt import REQUIRED_CHECKS, validate + +RUNTIME = { + "node_version": "v22.23.1", + "npm_version": "10.9.8", + "gemini_cli_version": "0.51.0", + "image_id": "sha256:" + "1" * 64, +} + + +def receipt(): + return { + "schema_version": "preinstalled-gemini-setup-receipt.v1", + "agent": "preinstalled-gemini-cli-v1", + "node_path": "/usr/local/bin/node", + "node_version": RUNTIME["node_version"], + "npm_version": RUNTIME["npm_version"], + "gemini_path": "/usr/local/bin/gemini", + "gemini_version": RUNTIME["gemini_cli_version"], + "package_version": RUNTIME["gemini_cli_version"], + "image_id": RUNTIME["image_id"], + "checks": {name: True for name in sorted(REQUIRED_CHECKS)}, + "status": "pass", + } + + +def main(): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json" + path.parent.mkdir(parents=True) + good = receipt() + path.write_text(json.dumps(good, sort_keys=True, separators=(",", ":")) + "\n") + assert validate(path, RUNTIME)["status"] == "pass" + mutations = { + "wrong_node": ("node_version", "v22.22.3"), + "wrong_npm": ("npm_version", "10.0.0"), + "wrong_gemini": ("gemini_version", "0.50.0"), + "wrong_package": ("package_version", "0.50.0"), + "wrong_image": ("image_id", "sha256:" + "2" * 64), + "wrong_agent": ("agent", "gemini-cli"), + "bad_status": ("status", "fail"), + } + for field, value in mutations.values(): + changed = copy.deepcopy(good) + changed[field] = value + path.write_text(json.dumps(changed) + "\n") + assert validate(path, RUNTIME)["status"] == "fail" + changed = copy.deepcopy(good) + changed["checks"]["node_executable"] = False + path.write_text(json.dumps(changed) + "\n") + assert validate(path, RUNTIME)["status"] == "fail" + path.write_text("not json\n") + assert validate(path, RUNTIME)["status"] == "fail" + misplaced = Path(tmp) / "setup-receipt.json" + misplaced.write_text(json.dumps(good) + "\n") + assert validate(misplaced, RUNTIME)["status"] == "fail" + print("Preinstalled Gemini setup receipt tests passed.") + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/scripts/v6/test_shell_observation.py b/labs/20-deltawire/eval/scripts/v6/test_shell_observation.py new file mode 100644 index 000000000..de3ba9bfe --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v6/test_shell_observation.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +import json,tempfile +from pathlib import Path +from shell_observation import parse +def event(command,output="",status="success",extra=None): + response={"output":output};response.update(extra or {}) + return {"toolCalls":[{"name":"run_shell_command","args":{"command":command},"status":status,"result":[{"functionResponse":{"response":response}}]}]} +def run(events): + with tempfile.TemporaryDirectory() as tmp: + p=Path(tmp)/"t.jsonl";p.write_text("\n".join(json.dumps(e) for e in events));return parse(p) +def main(): + obs,life=run([event("deltawire validate p","ok\nExit Code: 2\nProcess Group PGID: 7\n")]);assert obs[0]["tool_status"]=="success" and obs[0]["shell_exit_code"]==2 and life[0]["shell_exit_code"]==2 + obs,_=run([event("which deltawire","/usr/local/bin/deltawire")]);assert obs[0]["shell_exit_code"] is None + obs,_=run([event("type -P deltawire","/fake\nExit Code: 0\nProcess Group PGID: 1",extra={"exit_code":3})]);assert obs[0]["shell_exit_code"]==3 and obs[0]["shell_exit_provenance"]=="structured" + _,life=run([event("/usr/local/bin/deltawire validate p && deltawire render p && deltawire check p","Exit Code: 0\nProcess Group PGID: 9")]);assert [x["command"] for x in life]==["validate","render","check"] and all(x["shell_exit_code"]==0 for x in life) + _,life=run([event("deltawire validate p; deltawire check p","Exit Code: 0\nProcess Group PGID: 9")]);assert all(x["shell_exit_code"] is None for x in life) + _,life=run([event("echo 'deltawire validate'",status="success")]);assert not life + print("Canonical shell observation tests passed.") +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v6/test_terminal_evidence_lock.py b/labs/20-deltawire/eval/scripts/v6/test_terminal_evidence_lock.py new file mode 100644 index 000000000..9de236796 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v6/test_terminal_evidence_lock.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +import copy +import json +from pathlib import Path + +import verify_terminal_evidence_lock as verifier + + +def rejected(value: dict) -> bool: + try: + verifier.verify(value) + except RuntimeError: + return True + return False + + +def main() -> None: + value = json.loads(verifier.LOCK.read_text()) + verifier.verify(value) + + changed = copy.deepcopy(value) + changed["terminal_classification"]["pair_valid"] = True + assert rejected(changed) + + changed = copy.deepcopy(value) + first = next(iter(changed["sources"][1]["files"])) + changed["sources"][1]["files"][first] = "0" * 64 + assert rejected(changed) + + changed = copy.deepcopy(value) + changed["sources"][1]["files"].pop(first) + assert rejected(changed) + print("v6 terminal evidence-lock mutation tests passed.") + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/scripts/v6/test_trial_status.py b/labs/20-deltawire/eval/scripts/v6/test_trial_status.py new file mode 100644 index 000000000..821f1328d --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v6/test_trial_status.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +from trial_status import normalize + +PHASE = {"started_at": "2026-01-01T00:00:00Z", "finished_at": "2026-01-01T00:00:01Z"} + + +def main(): + failed = { + "environment_setup": PHASE, + "agent_setup": PHASE, + "agent_execution": None, + "verifier": None, + "exception_info": { + "exception_type": "AgentSetupTimeoutError", + "exception_message": "Agent setup timed out after 360.0 seconds", + }, + } + value = normalize(failed, 0) + assert value["harbor_process_exit_code"] == 0 + assert value["trial_status"] == "failed_before_agent_execution" + assert value["infrastructure_failure_class"] == "agent_setup_timeout" + assert value["agent_setup_status"] == "fail" + assert not value["agent_execution_started"] + install = normalize( + {"environment_setup": PHASE, "agent_setup": PHASE, "agent_execution": None, "verifier": None, "exception_info": None}, + 0, + install_only=True, + ) + assert install["trial_status"] == "install_only_pass" + assert install["agent_setup_status"] == "pass" + executed = normalize( + {"environment_setup": PHASE, "agent_setup": PHASE, "agent_execution": PHASE, "verifier": PHASE, "exception_info": None}, + 0, + ) + assert executed["trial_status"] == "completed" + assert executed["agent_execution_started"] and executed["verifier_started"] + missing = normalize({"environment_setup": PHASE, "agent_setup": PHASE}, 0) + assert missing["trial_status"] == "incomplete" + print("Harbor process/trial status normalization tests passed.") + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/scripts/v6/trial_status.py b/labs/20-deltawire/eval/scripts/v6/trial_status.py new file mode 100644 index 000000000..5a8b8da27 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v6/trial_status.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + + +def _phase_status(value): + if not isinstance(value, dict): + return "not_started" + return "pass" if value.get("started_at") and value.get("finished_at") else "incomplete" + + +def normalize(result, harbor_process_exit_code, install_only=False): + exception = result.get("exception_info") if isinstance(result, dict) else None + exception = exception if isinstance(exception, dict) else None + execution = result.get("agent_execution") if isinstance(result, dict) else None + verifier = result.get("verifier") if isinstance(result, dict) else None + environment_status = _phase_status(result.get("environment_setup")) + agent_status = _phase_status(result.get("agent_setup")) + if exception: + agent_status = "fail" + trial_status = "failed_before_agent_execution" if execution is None else "failed_after_agent_execution" + failure_class = ( + "agent_setup_timeout" + if exception.get("exception_type") == "AgentSetupTimeoutError" and execution is None + else "trial_exception" + ) + elif install_only and agent_status == "pass" and execution is None: + trial_status = "install_only_pass" + failure_class = None + elif execution is not None: + trial_status = "completed" + failure_class = None + else: + trial_status = "incomplete" + failure_class = "missing_agent_execution" + return { + "schema_version": "harbor-trial-status.v1", + "harbor_process_exit_code": harbor_process_exit_code, + "trial_exception_type": exception.get("exception_type") if exception else None, + "trial_exception_message": exception.get("exception_message") if exception else None, + "environment_setup_status": environment_status, + "agent_setup_status": agent_status, + "agent_execution_started": execution is not None, + "verifier_started": verifier is not None, + "trial_status": trial_status, + "infrastructure_failure_class": failure_class, + } diff --git a/labs/20-deltawire/eval/scripts/v6/verify_terminal_evidence_lock.py b/labs/20-deltawire/eval/scripts/v6/verify_terminal_evidence_lock.py new file mode 100644 index 000000000..a88266865 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v6/verify_terminal_evidence_lock.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Verify the terminal v6 experiment exclusively through immutable Git objects.""" + +import hashlib +import json +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[5] +LOCK = ROOT / "labs/20-deltawire/eval/manifests/preflight-v6-terminal-evidence-lock.json" +EXPECTED_TERMINAL = { + "B0": "not_run", + "D1_semantic": "pass", + "D1_treatment_fidelity": "fail", + "READY_FOR_72": False, + "pair_attempt_consumed": True, + "pair_valid": False, + "probe": "pass", +} + + +def git(*args: str) -> bytes: + return subprocess.check_output(["git", *args], cwd=ROOT) + + +def files_for(commit: str, scopes: list[str]) -> set[str]: + found: set[str] = set() + for scope in scopes: + output = git("ls-tree", "-r", "--name-only", commit, "--", scope).decode() + found.update(line for line in output.splitlines() if line) + return found + + +def verify(lock: dict) -> tuple[int, int]: + if lock.get("schema_version") != "deltawire-v6-terminal-evidence-lock.v1": + raise RuntimeError("v6 terminal lock schema mismatch") + if lock.get("terminal_classification") != EXPECTED_TERMINAL: + raise RuntimeError("v6 terminal classification mismatch") + sources = lock.get("sources") + if not isinstance(sources, list) or [item.get("commit") for item in sources] != [ + "0dad5c01684b337f8a1bf42f17141f4789c95740", + "273e977c2f61e3b3f31dd675ee3512ef9182a695", + ]: + raise RuntimeError("v6 evidence commits mismatch") + counts = [] + for source in sources: + commit = source["commit"] + git("rev-parse", f"{commit}^{{tree}}") + expected_names = files_for(commit, source["scopes"]) + recorded = source.get("files", {}) + if set(recorded) != expected_names: + raise RuntimeError(f"v6 evidence closure mismatch at {commit}") + for name, wanted in recorded.items(): + actual = hashlib.sha256(git("show", f"{commit}:{name}")).hexdigest() + if actual != wanted: + raise RuntimeError(f"v6 Git-object mismatch: {commit}:{name}") + counts.append(len(recorded)) + return counts[0], counts[1] + + +def main() -> None: + before, terminal = verify(json.loads(LOCK.read_text())) + print(f"v6 terminal evidence lock passed ({before} pre-pair files, {terminal} terminal files).") + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/scripts/validate_dataset.sh b/labs/20-deltawire/eval/scripts/validate_dataset.sh index 7a0e62dda..d7628158a 100755 --- a/labs/20-deltawire/eval/scripts/validate_dataset.sh +++ b/labs/20-deltawire/eval/scripts/validate_dataset.sh @@ -18,6 +18,9 @@ print(f"Identical Linux binary staged into {len(paths)-1} task contexts.") PY python3 labs/20-deltawire/eval/scripts/validate_preflight.py python3 labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py +python3 labs/20-deltawire/eval/scripts/v6/verify_terminal_evidence_lock.py +python3 labs/20-deltawire/eval/scripts/v6/test_terminal_evidence_lock.py +python3 labs/20-deltawire/eval/scripts/v6/test_release.py python3 labs/20-deltawire/eval/scripts/v5/test_environment_receipt.py python3 labs/20-deltawire/eval/scripts/v5/test_artifact_manifest.py python3 labs/20-deltawire/eval/scripts/v5/validate_integration.py