diff --git a/.github/workflows/deltawire-lab.yml b/.github/workflows/deltawire-lab.yml index 290526980..1c8d63a1c 100644 --- a/.github/workflows/deltawire-lab.yml +++ b/.github/workflows/deltawire-lab.yml @@ -24,5 +24,6 @@ jobs: go-version-file: labs/20-deltawire/go.mod - uses: actions/setup-python@v6 with: - python-version: "3.11" + python-version: "3.12" + - uses: astral-sh/setup-uv@v7 - run: bash labs/20-deltawire/scripts/validate.sh diff --git a/.github/workflows/deltawire-v7.yml b/.github/workflows/deltawire-v7.yml new file mode 100644 index 000000000..0b0f02827 --- /dev/null +++ b/.github/workflows/deltawire-v7.yml @@ -0,0 +1,55 @@ +name: DeltaWire v7 no-live validation + +on: + push: + branches: [eval/deltawire-v7] + paths: + - "labs/20-deltawire/**" + - ".github/workflows/deltawire-v7.yml" + pull_request: + paths: + - "labs/20-deltawire/**" + - ".github/workflows/deltawire-v7.yml" + +permissions: + contents: read + +jobs: + validate-v7: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: actions/setup-go@v6 + with: + go-version-file: labs/20-deltawire/go.mod + cache-dependency-path: labs/20-deltawire/go.sum + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@v7 + - name: Build and stage the frozen Linux binary + run: | + cd labs/20-deltawire + mkdir -p eval/.generated eval/pairs/range-large-v7-r1/environment/.generated + GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -o eval/.generated/deltawire-linux-amd64 ./cmd/deltawire + cp eval/.generated/deltawire-linux-amd64 eval/pairs/range-large-v7-r1/environment/.generated/deltawire + test "$(sha256sum eval/.generated/deltawire-linux-amd64 | cut -d' ' -f1)" = e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4 + - name: Validate skill examples using the actual Linux binary + run: | + python3 labs/20-deltawire/eval/scripts/v7/skill_conformance.py \ + --skill labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire \ + --binary labs/20-deltawire/eval/.generated/deltawire-linux-amd64 \ + --report /tmp/skill-conformance.json + if test -f labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json; then + cmp /tmp/skill-conformance.json labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json + fi + - name: No-live v7 validation + run: bash labs/20-deltawire/eval/scripts/v7/validate.sh + - name: Prove workflow has no model invocation + run: | + live_runner='runner.py .*pair'"-v7" + harbor_live='harbor '"run" + credential='GEMINI_API'"_KEY" + ! grep -E "$live_runner|$harbor_live|$credential" .github/workflows/deltawire-v7.yml diff --git a/labs/20-deltawire/eval/docs/v7-treatment-boundary.md b/labs/20-deltawire/eval/docs/v7-treatment-boundary.md new file mode 100644 index 000000000..5a66f3dd5 --- /dev/null +++ b/labs/20-deltawire/eval/docs/v7-treatment-boundary.md @@ -0,0 +1,28 @@ +# DeltaWire v7 treatment boundary + +V7 keeps the v6 binary, preinstalled runtime image, Harbor 0.20.0, Gemini CLI +0.51.0, model, task semantics, verifier, and credential boundary unchanged. +Only the treatment bundle and its measurement contract are versioned. + +Harbor recursively copies the complete assigned skill directory into the Gemini +skill directory. The v7 assignment receipt therefore verifies the trial lock's +skill name and Harbor digest, the trial configuration's assigned skill path, +and the frozen local bundle tree. Arm labels and model prose are not evidence +of assignment. + +The skill reference is grounded in the Go plan types, strict decoder, matrix, +rows and variants engines, existing fixtures, and credential-free execution of +all four examples. The incomplete standalone JSON plan schema is not treated as +the protocol authority. + +`gemini skills list`, direct skill reads, reference/example reads, validation +counts, repairs, subprocess loops, and binary introspection are diagnostics. +D1 fidelity instead requires assignment, structured activation, public-contract +access, lifecycle invocations, retained artifacts, two contract checks, schema +integrity, and semantic exactness. B0 is contaminated by activation, a +structured DeltaWire invocation, a retained plan/state, or DeltaWire-managed +output; immutable config and schema files are shared infrastructure. + +The runner exposes only `pair-v7`, freezes D1 before conditional B0, refuses a +nonempty result root, and requires the exact external approval receipt before a +live call. CI has no credential or live-run path. diff --git a/labs/20-deltawire/eval/manifests/deltawire-range-large-v7-pair.json b/labs/20-deltawire/eval/manifests/deltawire-range-large-v7-pair.json new file mode 100644 index 000000000..2fe4b3329 --- /dev/null +++ b/labs/20-deltawire/eval/manifests/deltawire-range-large-v7-pair.json @@ -0,0 +1,95 @@ +{ + "READY_FOR_72": false, + "agent": "preinstalled-gemini-cli-v1", + "agent_import": "agents.preinstalled_gemini_cli:PreinstalledGeminiCli", + "binary_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "expected_reported_model": "gemini-3.1-pro-preview", + "failure_policy": "withhold_B0_unless_D1_passes", + "full_run_started": false, + "gemini_cli_version": "0.51.0", + "harbor_command": [ + "uvx", + "--from", + "harbor==0.20.0", + "harbor" + ], + "harbor_source": [ + { + "module": "harbor.agents.installed.gemini_cli", + "sha256": "dc8ce81e8f41b78f00abb54cd91fd7840d147f442960ca32e68c7a760e060a2a" + }, + { + "module": "harbor.skills", + "sha256": "45fa5d0f45dce8029c47eac7468dc8bdb01157cc2b7c9271503719fc157cc7f3" + } + ], + "harbor_version": "0.20.0", + "image_id": "sha256:589fce58d7ddf09910876afe0198bcee0086a840af6de7c08515bbce3e8367a7", + "image_ref": "deltawire-preflight-v6:runtime-v1", + "infrastructure_commit": "a8d2d8260ceed56baf0d1cf8fcc2f40776e9f7b6", + "input_hashes": { + "labs/20-deltawire/eval/agents/preinstalled_gemini_cli.py": "dfd0d27258ac20b8778b9aacce5f302db93576d2f735c6645e489be6c6b96058", + "labs/20-deltawire/eval/manifests/preflight-v6-terminal-evidence-lock.json": "76a6ac42fe2dfc41d1af2d73502c11724ed632a21e1d8d10b065e9bc619b7fcc", + "labs/20-deltawire/eval/matrices/72-run-matrix.json": "f9dcddc4b7336418c7e2b8e7a7e51ca1e42b5a137255ff0823a732618f3568d5", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/authoritative-schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/Dockerfile": "cd768e64efba435604b265088bce6269e8dbd09e54bf51a4c5bf0dd94d10c05f", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/deltawire-environment-expectations.json": "03fe7a40964e908f2fb3736add8b8bea890b407d837f2b185d92bd8d85d4d114", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/environment_receipt.py": "3ab3bbcbfae2093ac658063d1e26f26b43c35dfdb81d8ddae5c9989f967ba623", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/task-contract.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/instruction.md": "549f4047b1a4ba4f7a3f3dce9e60d1d70b141caa189aafdba942b15e6d022aee", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/SKILL.md": "e7d3a1e8ccef1b72338d5dac7a7b7ac412164846018ab55b7f2ecadf5a07a89a", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/matrix.dw.json": "56d16a5b20086c8202cbc1bbdccf41d467a5d4c3806d701227ebcdbf72e54b1a", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/range.dw.json": "b31e078409547f4fbb5a855f3bf0b1bd6f52386696c07401622f34d224fe148b", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/rows.dw.json": "e172cc7d6224e4566a0cd46a73465392976338fea0e225512a089322ba81aca3", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/variants.dw.json": "3183405b76e11874309728f3a4025687af6c54add8d2b97b94daff2d6ef8cdca", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/references/plan-v1.md": "f78d41266db73592526a3c5e7d6e4c883f549f99b01c6387c7f5337fc269dd5e", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/task.toml": "4adc78ba9c59f8c743770ee9b1b31a55364e7303010e3c3f109da3b3d61d96e3", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/semantic_oracle.py": "94b30fac117f40e977851966b5228e6c372bb48cbf2e5c29c24f428e1f98e91d", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/test.sh": "6f7cfbefdf7eaf3a082fd0c88f264ba3d41bfe84fc53498df4464bef66dc116e", + "labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json": "4d0e142a1bc801642b5de8c77b9cbebc9680a42f6f64da95b3d48b8624fe7c46", + "labs/20-deltawire/eval/runtime/preinstalled-gemini-v1.json": "19d9bf6ed82d3f6fa8e9bdad65bb74538ef9d7d161afb3f1f76771ac2e6b0c2d", + "labs/20-deltawire/eval/scripts/semantic_oracle.py": "94b30fac117f40e977851966b5228e6c372bb48cbf2e5c29c24f428e1f98e91d", + "labs/20-deltawire/eval/scripts/v6/artifact_manifest.py": "722f71bfbec56cbdbaea85a2f975aaa61f58d2dbbbd41e4a6b02741f3ac90ca0", + "labs/20-deltawire/eval/scripts/v6/setup_receipt.py": "c57a5a614663e4b3b81f4e1b5929c3aadfbb3a319198c9700213da0308a3d634", + "labs/20-deltawire/eval/scripts/v6/shell_observation.py": "e5269e06350147715505a76b8e83235abecd5af34eba1c8837e19b22b0e4c9fe", + "labs/20-deltawire/eval/scripts/v6/trial_status.py": "47fea192f0853a3ea880d992c27934e87cd19bc84a7a2c01446d3e9a0be8379a", + "labs/20-deltawire/eval/scripts/v6/verify_terminal_evidence_lock.py": "5adf6d90f641fb80fea894709f1a6e219e1b430454a11d19fc4c1c402bb0affe", + "labs/20-deltawire/eval/scripts/v7/generate_manifest.py": "60fbcd9a31514b94f606e87b7df07c4442259a9ed15c00f169b344675596b1c6", + "labs/20-deltawire/eval/scripts/v7/runner.py": "cc551f8f943f39c767c0cc88bc07f584972777b4d775ca586d3502eb653399b8", + "labs/20-deltawire/eval/scripts/v7/skill_conformance.py": "31617b88fef9185635ea3bc78f60368df8a894212a804c4f5e1d60b2365d9491", + "labs/20-deltawire/eval/scripts/v7/test_runner.py": "6e0ca10b09f1fd2624bbbe70892c9faf31ef630644f710ba9cd05fbaa92556ab", + "labs/20-deltawire/eval/scripts/v7/test_skill.py": "6fd6b49e06de82e4af7a97a449a9981fd860dca96c806c069834cf96287d288e", + "labs/20-deltawire/eval/scripts/v7/test_treatment.py": "4f2cb7744ce65d5cfe8ec1da8b1aca804609977a44c20fdbafa12397110409ce", + "labs/20-deltawire/eval/scripts/v7/treatment.py": "5587514cd678223105a2aff6b1a01f58fff6b6e138d06b9461db0fc7f77e7ee8", + "labs/20-deltawire/eval/scripts/v7/validate.py": "c96eadbac126fb6546816f83513e3a38dea0b29091b43fec620b929768e1e06a", + "labs/20-deltawire/eval/scripts/v7/validate.sh": "a24c183a3824df10cf5dffcdd07c0bd336b7d51705a9e311127587d56aa87200", + "labs/20-deltawire/eval/scripts/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af" + }, + "integration_commit": "82a7963bdd98ca41435e756fb4212b3b37b19420", + "model": "google/gemini-3.1-pro-preview", + "order": [ + "D1", + "B0" + ], + "pair_id": "range-large/v7-r1", + "result_root": "labs/20-deltawire/eval/results/range-large-v7-r1", + "schema_version": "deltawire-range-large-v7-pair-manifest.v1", + "six_run_canary_started": false, + "skill": { + "conformance_sha256": "4d0e142a1bc801642b5de8c77b9cbebc9680a42f6f64da95b3d48b8624fe7c46", + "harbor_digest": "sha256:a841ca12dc687b1e467f90d5d9f5ccef120bc810742fab693e8e976135a75984", + "reference_sha256": "f78d41266db73592526a3c5e7d6e4c883f549f99b01c6387c7f5337fc269dd5e", + "skill_sha256": "e7d3a1e8ccef1b72338d5dac7a7b7ac412164846018ab55b7f2ecadf5a07a89a", + "tree_sha256": "a841ca12dc687b1e467f90d5d9f5ccef120bc810742fab693e8e976135a75984" + }, + "source_matrix_pair_id": "range-large/r1", + "source_matrix_positions_zero_based": [ + 36, + 37 + ] +} diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/authoritative-schema.json b/labs/20-deltawire/eval/pairs/range-large-v7-r1/authoritative-schema.json new file mode 100644 index 000000000..69b476237 --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/authoritative-schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "index": { + "maximum": 500, + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "index" + ], + "title": "range-large", + "type": "object" +} diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.deltawire/config.json b/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.deltawire/config.json new file mode 100644 index 000000000..61a05415a --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.deltawire/config.json @@ -0,0 +1,12 @@ +{ + "limits": { + "max_output_bytes": 104857600, + "max_plan_bytes": 1048576, + "max_records": 100000, + "max_schema_bytes": 1048576 + }, + "plans_dir": ".deltawire/plans", + "schemas_dir": ".deltawire/schemas", + "state_file": ".deltawire/state.json", + "version": "deltawire.config.v1" +} diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.deltawire/schemas/range-large.schema.json b/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.deltawire/schemas/range-large.schema.json new file mode 100644 index 000000000..69b476237 --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.deltawire/schemas/range-large.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "index": { + "maximum": 500, + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "index" + ], + "title": "range-large", + "type": "object" +} diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.generated/.gitignore b/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.generated/.gitignore new file mode 100644 index 000000000..d6b7ef32c --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.generated/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/Dockerfile b/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/Dockerfile new file mode 100644 index 000000000..0e0f65c19 --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/Dockerfile @@ -0,0 +1,26 @@ +FROM node:22.23.1-bookworm-slim@sha256:8607a9064d4a571140998ae9e52a3b3fcf9cff361d04642d5971e6cd76d39e27 +ARG GEMINI_CLI_VERSION=0.51.0 +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates python3 \ + && rm -rf /var/lib/apt/lists/* +RUN npm install --global --omit=dev "@google/gemini-cli@${GEMINI_CLI_VERSION}" \ + && test "$(node --version)" = "v22.23.1" \ + && test "$(npm --version)" = "10.9.8" \ + && test "$(gemini --version)" = "${GEMINI_CLI_VERSION}" \ + && test "$(node -p 'require("/usr/local/lib/node_modules/@google/gemini-cli/package.json").version')" = "${GEMINI_CLI_VERSION}" +COPY .generated/deltawire /usr/local/bin/deltawire +COPY task-contract.json /task-contract.json +COPY .deltawire /.deltawire +COPY verify_plan_contract.py /usr/local/bin/verify_plan_contract.py +COPY environment_receipt.py /usr/local/bin/environment_receipt.py +COPY deltawire-environment-expectations.json /deltawire-environment-expectations.json +RUN mkdir -p /root/.gemini \ + && printf '%s\n' '{"experimental":{"skills":true}}' > /root/.gemini/settings.json \ + && chmod 0755 /usr/local/bin/deltawire /usr/local/bin/verify_plan_contract.py /usr/local/bin/environment_receipt.py \ + && chmod a-w /task-contract.json /.deltawire/config.json /.deltawire/schemas/*.json /deltawire-environment-expectations.json \ + && command -v node \ + && command -v npm \ + && command -v gemini \ + && command -v deltawire \ + && deltawire version +WORKDIR / diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/deltawire-environment-expectations.json b/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/deltawire-environment-expectations.json new file mode 100644 index 000000000..a47f4e360 --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/deltawire-environment-expectations.json @@ -0,0 +1,23 @@ +{ + "binary": { + "path": "/usr/local/bin/deltawire", + "realpath": "/usr/local/bin/deltawire", + "sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "version": "deltawire version dev" + }, + "files": { + "config": { + "path": "/.deltawire/config.json", + "sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f" + }, + "public_contract": { + "path": "/task-contract.json", + "sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5" + }, + "schema": { + "path": "/.deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + } + }, + "schema_version": "deltawire-environment-expectations.v1" +} diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/environment_receipt.py b/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/environment_receipt.py new file mode 100644 index 000000000..84446c2d5 --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/environment_receipt.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Atomically emit deterministic DeltaWire environment evidence.""" +import argparse,hashlib,json,os,stat,subprocess,tempfile +from pathlib import Path + +def sha(path): + h=hashlib.sha256() + with Path(path).open("rb") as f: + for chunk in iter(lambda:f.read(1048576),b""):h.update(chunk) + return h.hexdigest() +def atomic_write(path,data): + target=Path(path);target.parent.mkdir(parents=True,exist_ok=True) + fd,tmp=tempfile.mkstemp(prefix=f".{target.name}.",suffix=".tmp",dir=target.parent) + try: + with os.fdopen(fd,"w",encoding="utf-8") as f:f.write(data);f.flush();os.fsync(f.fileno()) + os.replace(tmp,target) + except BaseException: + try:os.unlink(tmp) + except FileNotFoundError:pass + raise +def build(expectations): + expected=json.loads(Path(expectations).read_text());binary=Path(expected["binary"]["path"]) + exists=binary.exists();lst=binary.lstat() if exists else None + regular=bool(lst and stat.S_ISREG(lst.st_mode));symlink=binary.is_symlink() if exists else False + executable=exists and os.access(binary,os.X_OK);realpath=str(binary.resolve()) if exists else None + try:version=subprocess.run([str(binary),"version"],capture_output=True,text=True,check=False) if exists else None + except OSError:version=None + files={} + for name,item in sorted(expected["files"].items()): + path=Path(item["path"]);actual=sha(path) if path.is_file() else None + files[name]={"path":item["path"],"exists":path.is_file(),"actual_sha256":actual,"expected_sha256":item["sha256"],"hash_match":actual==item["sha256"]} + binary_hash=sha(binary) if regular else None + checks={"binary_exists":exists,"binary_regular":regular,"binary_not_symlink":not symlink,"binary_executable":executable, + "binary_path":str(binary)==expected["binary"]["path"],"binary_realpath":realpath==expected["binary"]["realpath"], + "binary_hash":binary_hash==expected["binary"]["sha256"],"version_exit_0":bool(version and version.returncode==0), + "version_exact":bool(version and version.stdout.splitlines() and version.stdout.splitlines()[0].strip()==expected["binary"]["version"]), + **{f"{name}_hash":item["hash_match"] for name,item in files.items()}} + return {"schema_version":"deltawire-environment-receipt.v1","binary":{"path":str(binary),"realpath":realpath,"exists":exists, + "is_regular":regular,"is_symlink":symlink,"executable":executable,"actual_sha256":binary_hash, + "expected_sha256":expected["binary"]["sha256"],"version_stdout":version.stdout.strip() if version else None, + "version_stderr":version.stderr.strip() if version else None,"version_exit_code":version.returncode if version else None, + "expected_version":expected["binary"]["version"]},"files":files,"checks":checks,"status":"pass" if all(checks.values()) else "fail"} +def main(): + p=argparse.ArgumentParser();p.add_argument("--expectations",required=True);p.add_argument("--receipt",required=True);a=p.parse_args() + receipt=build(a.expectations);atomic_write(a.receipt,json.dumps(receipt,indent=2,sort_keys=True)+"\n") + print(json.dumps(receipt,sort_keys=True));raise SystemExit(0 if receipt["status"]=="pass" else 1) +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/task-contract.json b/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/task-contract.json new file mode 100644 index 000000000..281ba55ac --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/task-contract.json @@ -0,0 +1,24 @@ +{ + "authoritative_schema": { + "path": ".deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + }, + "deltawire_applicability": "expected", + "family": "range", + "generation": { + "field": "index", + "field_order": [ + "index" + ], + "start": 1 + }, + "ordering": "canonical_generation_order", + "output": { + "format": "ndjson", + "path": "testdata/generated/output.ndjson" + }, + "record_count": 500, + "schema_version": "task-spec.v1", + "size": "large", + "task": "range-large" +} diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/verify_plan_contract.py b/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/verify_plan_contract.py new file mode 100644 index 000000000..205d4fc8a --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/verify_plan_contract.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import argparse, hashlib, json, subprocess +from pathlib import Path + +def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() +def write(path, value): + if path: + target=Path(path); target.parent.mkdir(parents=True,exist_ok=True); target.write_text(json.dumps(value,indent=2,sort_keys=True)+"\n") + +def main(): + p=argparse.ArgumentParser(); p.add_argument("--contract",required=True); p.add_argument("--plan",required=True); p.add_argument("--repo",default="."); p.add_argument("--receipt"); p.add_argument("--deltawire",default="deltawire"); a=p.parse_args() + repo=Path(a.repo).resolve(); contract_path=Path(a.contract).resolve(); plan=Path(a.plan).resolve(); contract=json.loads(contract_path.read_text()) + try: plan_arg=str(plan.relative_to(repo)) + except ValueError: plan_arg=str(plan) + result={"schema_version":"plan-contract-receipt.v1","contract_sha256":sha(contract_path),"plan_sha256":sha(plan),"checks":{},"status":"fail"} + try: + run=subprocess.run([a.deltawire,"inspect","--repo",str(repo),"--format","json",plan_arg],capture_output=True,text=True) + result["inspect_exit_code"]=run.returncode + if run.returncode: result["error"]=run.stderr.strip() or run.stdout.strip() + else: + inspected=json.loads(run.stdout); schema=contract["authoritative_schema"]; schema_path=repo/schema["path"] + checks={"output_path":inspected.get("output_path")==contract["output"]["path"],"output_format":inspected.get("output_format")==contract["output"]["format"],"projected_records":inspected.get("projected_records")==contract["record_count"],"schema_path":inspected.get("schema_path")==schema["path"],"schema_exists":schema_path.is_file(),"schema_sha256":schema_path.is_file() and sha(schema_path)==schema["sha256"],"exact_count_assertion":str(contract["record_count"]) in json.dumps(inspected.get("assertion_summary",{}))} + result.update({"checks":checks,"inspect":inspected,"status":"pass" if all(checks.values()) else "fail"}) + except (OSError,ValueError,KeyError) as error: result["error"]=str(error) + write(a.receipt,result) + if result["status"]!="pass": raise SystemExit(1) + print(json.dumps(result,sort_keys=True)) +if __name__=="__main__": main() diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/instruction.md b/labs/20-deltawire/eval/pairs/range-large-v7-r1/instruction.md new file mode 100644 index 000000000..9181f4398 --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/instruction.md @@ -0,0 +1,28 @@ +Generate the exact `range-large` dataset defined by public `/task-contract.json`. Write exactly 500 JSON records in canonical order as NDJSON to `testdata/generated/output.ndjson`. Every record, field, value, and ordering rule must match this complete embedded contract: + +```json +{ + "authoritative_schema": { + "path": ".deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + }, + "deltawire_applicability": "expected", + "family": "range", + "generation": { + "field": "index", + "field_order": [ + "index" + ], + "start": 1 + }, + "ordering": "canonical_generation_order", + "output": { + "format": "ndjson", + "path": "testdata/generated/output.ndjson" + }, + "record_count": 500, + "schema_version": "task-spec.v1", + "size": "large", + "task": "range-large" +} +``` diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/SKILL.md b/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/SKILL.md new file mode 100644 index 000000000..f02d051a8 --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/SKILL.md @@ -0,0 +1,29 @@ +--- +name: deltawire +description: Generate large deterministic JSON or NDJSON datasets from compact ranges, Cartesian products, repeated rows, variants, defaults, and regular exceptions. Use when the requested output has a compact exact plan; abstain when records are small or too irregular for a deterministic description. +--- + +# DeltaWire + +Read the public task contract first. Preserve its output path, format, schema +path and hash, record count, field set, values, and ordering. + +Read `references/plan-v1.md` and the matching bundled example before creating a +plan. Use `examples/range.dw.json` for integer sequences, +`examples/matrix.dw.json` for Cartesian products, `examples/rows.dw.json` for +explicit repeated rows, or `examples/variants.dw.json` for base records with +small changes. + +Create only the requested plan. Do not edit the installed config or schema, do +not generate the expanded output manually, and do not inspect or reverse +engineer the DeltaWire binary. Resolve validation errors by comparing the plan +with the reference and example instead of guessing field names. + +Run this lifecycle in order: + +1. `deltawire validate --repo . ` +2. Run the installed independent contract checker for the public contract. +3. `deltawire render --repo . ` +4. `deltawire check --repo . ` + +Stop after all four steps pass and the required artifacts are retained. diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/matrix.dw.json b/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/matrix.dw.json new file mode 100644 index 000000000..ca1c24521 --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/matrix.dw.json @@ -0,0 +1,17 @@ +{ + "version": "deltawire.plan.v1", + "id": "matrix-example", + "description": "A small Cartesian product", + "record_schema": ".deltawire/schemas/example.schema.json", + "output": {"path": "testdata/generated/matrix.ndjson", "format": "ndjson", "pretty": false}, + "generators": [{ + "kind": "matrix", + "name": "requests", + "dimensions": [ + {"name": "role", "values": ["guest", "member"]}, + {"name": "method", "values": ["GET", "POST"]} + ], + "record": {"role": "${role}", "method": "${method}"} + }], + "assertions": {"count": 4} +} diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/range.dw.json b/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/range.dw.json new file mode 100644 index 000000000..9c467bf42 --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/range.dw.json @@ -0,0 +1,14 @@ +{ + "version": "deltawire.plan.v1", + "id": "range-example", + "description": "Three ordered integer records", + "record_schema": ".deltawire/schemas/example.schema.json", + "output": {"path": "testdata/generated/range.ndjson", "format": "ndjson", "pretty": false}, + "generators": [{ + "kind": "matrix", + "name": "records", + "dimensions": [{"name": "index", "range": {"start": 1, "end_exclusive": 4, "step": 1}}], + "record": {"index": "${index}"} + }], + "assertions": {"count": 3} +} diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/rows.dw.json b/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/rows.dw.json new file mode 100644 index 000000000..543ddac69 --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/rows.dw.json @@ -0,0 +1,15 @@ +{ + "version": "deltawire.plan.v1", + "id": "rows-example", + "description": "Two explicit records with defaults", + "record_schema": ".deltawire/schemas/example.schema.json", + "output": {"path": "testdata/generated/rows.ndjson", "format": "ndjson", "pretty": false}, + "defaults": {"enabled": true}, + "generators": [{ + "kind": "rows", + "name": "items", + "columns": ["/name", "/rank"], + "rows": [["alpha", 1], ["beta", 2]] + }], + "assertions": {"count": 2} +} diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/variants.dw.json b/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/variants.dw.json new file mode 100644 index 000000000..c9adb90bb --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/variants.dw.json @@ -0,0 +1,18 @@ +{ + "version": "deltawire.plan.v1", + "id": "variants-example", + "description": "Base records with small deterministic changes", + "record_schema": ".deltawire/schemas/example.schema.json", + "output": {"path": "testdata/generated/variants.ndjson", "format": "ndjson", "pretty": false}, + "defaults": {"active": true}, + "generators": [{ + "kind": "variants", + "name": "cases", + "base": {"role": "member", "status": "allow"}, + "variants": [ + {"name": "deny", "set": {"/status": "deny"}}, + {"name": "inactive", "set": {"/active": false}} + ] + }], + "assertions": {"count": 2} +} diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/references/plan-v1.md b/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/references/plan-v1.md new file mode 100644 index 000000000..8a10230ed --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/references/plan-v1.md @@ -0,0 +1,45 @@ +# DeltaWire plan v1 + +A plan is strict JSON. Unknown fields fail decoding. + +## Plan fields + +- `version`: exactly `deltawire.plan.v1`. +- `id`, `description`: descriptive strings. +- `record_schema`: repository-relative JSON Schema path used by `check`. +- `output.path`: repository-relative output path. +- `output.format`: `json` or `ndjson`. +- `output.pretty`: JSON pretty-printing flag. +- `defaults`: optional object deep-merged into each record. +- `sets`: optional named arrays of scalar dimension values. +- `generators`: ordered array of `matrix`, `rows`, or `variants` generators. +- `assertions.count`: exact projected record count. +- `assertions.unique`: optional JSON Pointer paths that must be unique. +- `assertions.coverage`: optional JSON Pointer/value coverage checks. + +## Matrix + +A matrix generator has `kind`, `name`, `dimensions`, and `record`. Each +dimension has a `name` plus exactly one of `values`, `set`, or `range`. An +inline integer range uses `start`, `end_exclusive`, and nonzero `step`. + +Reference dimensions from the record with `${name}`. When a string is exactly +one placeholder, the generated scalar retains its original type; for example, +an integer range produces JSON integers. Embedded placeholders stringify their +scalar values. Every dimension must be referenced by the record. + +## Rows + +A rows generator has `kind`, `name`, `columns`, and `rows`. Each column is a +JSON Pointer such as `/role`; every row must have exactly one value per column. +Rows begin with plan defaults, then column values are assigned in order. + +## Variants + +A variants generator has `kind`, `name`, `base`, and `variants`. Each variant +has a unique `name`, optional `set` JSON Pointer/value assignments, and optional +`omit` JSON Pointer paths. Records are built by deep-merging plan defaults and +the generator base, then applying the variant changes. + +Generators emit records in plan order. Matrix dimensions vary from the last +dimension inward, preserving each values/range order. diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/task-spec.json b/labs/20-deltawire/eval/pairs/range-large-v7-r1/task-spec.json new file mode 100644 index 000000000..281ba55ac --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/task-spec.json @@ -0,0 +1,24 @@ +{ + "authoritative_schema": { + "path": ".deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + }, + "deltawire_applicability": "expected", + "family": "range", + "generation": { + "field": "index", + "field_order": [ + "index" + ], + "start": 1 + }, + "ordering": "canonical_generation_order", + "output": { + "format": "ndjson", + "path": "testdata/generated/output.ndjson" + }, + "record_count": 500, + "schema_version": "task-spec.v1", + "size": "large", + "task": "range-large" +} diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/task.toml b/labs/20-deltawire/eval/pairs/range-large-v7-r1/task.toml new file mode 100644 index 000000000..4229f8b02 --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/task.toml @@ -0,0 +1,30 @@ +schema_version = "1.3" +artifacts = ["testdata/generated/output.ndjson", ".deltawire/config.json", ".deltawire/plans/range-large.dw.json", ".deltawire/schemas/range-large.schema.json", ".deltawire/state.json", ".deltawire/plan-contract-receipt.json"] + +[task] +name = "operatorstack/range-large-v7-r1" +description = "Frozen range-large/v7-r1 causal pair task" +authors = [] +keywords = [] + +[metadata] +benchmark_result = true +pair_version = "v7-r1" + +[verifier] +timeout_sec = 600.0 + +[[verifier.collect]] +service = "main" +command = "python3 /usr/local/bin/environment_receipt.py --expectations /deltawire-environment-expectations.json --receipt /logs/artifacts/deltawire/environment-receipt.json" +timeout_sec = 30.0 + +[agent] +timeout_sec = 600.0 + +[environment] +docker_image = "deltawire-preflight-v6:runtime-v1" +network_mode = "public" +build_timeout_sec = 600.0 +os = "linux" +mcp_servers = [] diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/semantic_oracle.py b/labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/semantic_oracle.py new file mode 100644 index 000000000..00cfbfbc6 --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/semantic_oracle.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + + +def canonical(spec): + g, count, family = spec["generation"], spec["record_count"], spec["family"] + if family == "matrix": + rows = [{}] + for key, allowed in g["dimensions"].items(): + rows = [{**row, key: value} for row in rows for value in allowed] + return [{**row, **g["constants"]} for row in rows] + if family == "range": + return [{g["field"]: i} for i in range(g["start"], g["start"] + count)] + if family == "rows": + return [{"id": i, "name": g["name_template"].format(id=i), **g["constants"]} for i in range(g["start"], g["start"] + count)] + if family == "variants": + return [{**g["base"], "value": g["base"]["value"] + i * g["value_step"]} for i in range(count)] + if family == "mixed": + return ([{"id": i, "type": "std"} for i in range(1, g["standard_count"] + 1)] + + [{"id": i, "type": "exc", "error_code": g["error_code"]} for i in range(g["standard_count"] + 1, count + 1)]) + return [{"id": i, f'{g["dynamic_field_prefix"]}{i}': i} for i in range(g["id_start"], g["id_start"] + count)] + + +def main(): + if len(sys.argv) != 3: + print('{"exact_match": 0}'); return + spec_arg, output_arg = Path(sys.argv[1]), Path(sys.argv[2]) + if not spec_arg.is_file(): + candidate = Path(__file__).resolve().parents[1] / "tasks" / sys.argv[1] / "task-spec.json" + spec_arg = candidate + try: + spec = json.loads(spec_arg.read_text(encoding="utf-8")) + lines = output_arg.read_text(encoding="utf-8").splitlines() + if not lines or any(not line.strip() for line in lines): + raise ValueError("empty line") + actual = [json.loads(line) for line in lines] + passed = actual == canonical(spec) + except Exception: + passed = False + print(json.dumps({"exact_match": 1 if passed else 0})) + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/task-spec.json b/labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/task-spec.json new file mode 100644 index 000000000..281ba55ac --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/task-spec.json @@ -0,0 +1,24 @@ +{ + "authoritative_schema": { + "path": ".deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + }, + "deltawire_applicability": "expected", + "family": "range", + "generation": { + "field": "index", + "field_order": [ + "index" + ], + "start": 1 + }, + "ordering": "canonical_generation_order", + "output": { + "format": "ndjson", + "path": "testdata/generated/output.ndjson" + }, + "record_count": 500, + "schema_version": "task-spec.v1", + "size": "large", + "task": "range-large" +} diff --git a/labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/test.sh b/labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/test.sh new file mode 100755 index 000000000..9c1039618 --- /dev/null +++ b/labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/test.sh @@ -0,0 +1,8 @@ +#!/bin/bash +mkdir -p /logs/verifier +OUTPUT_FILE="testdata/generated/output.ndjson" +if [ ! -f "$OUTPUT_FILE" ]; then + mkdir -p /logs/verifier && echo '{"exact_match": 0}' > /logs/verifier/reward.json + exit 0 +fi +mkdir -p /logs/verifier && python3 /tests/semantic_oracle.py /tests/task-spec.json "$OUTPUT_FILE" > /logs/verifier/reward.json diff --git a/labs/20-deltawire/eval/results/preflight-v7/pair-approval.json b/labs/20-deltawire/eval/results/preflight-v7/pair-approval.json new file mode 100644 index 000000000..9053beb00 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v7/pair-approval.json @@ -0,0 +1,19 @@ +{ + "READY_FOR_72": false, + "authorization": "approve the frozen range-large/v7-r1 pair", + "binary_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "full_run_authorized": false, + "image_id": "sha256:589fce58d7ddf09910876afe0198bcee0086a840af6de7c08515bbce3e8367a7", + "infrastructure_commit": "a8d2d8260ceed56baf0d1cf8fcc2f40776e9f7b6", + "manifest_commit": "369f3c7c0d751da72fa658a2ec810557d6669e39", + "manifest_sha256": "67789b85c08802a87f5d14bb72aa2d36068c43a45aa27187ddab5d66dba3eca4", + "order": [ + "D1", + "B0" + ], + "pair_id": "range-large/v7-r1", + "probe_authorized": false, + "rerun_authorized": false, + "schema_version": "deltawire-v7-pair-approval.v1", + "six_run_canary_authorized": false +} diff --git a/labs/20-deltawire/eval/results/preflight-v7/pair-security-scan.json b/labs/20-deltawire/eval/results/preflight-v7/pair-security-scan.json new file mode 100644 index 000000000..470e2b88a --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v7/pair-security-scan.json @@ -0,0 +1,11 @@ +{ + "active_pair_processes": 0, + "approval_sha256": "c6b493851951e72ead0d15e38231f47d036331f94ded94d9d3dceb38db7c3154", + "bytes_scanned": 143725, + "exact_credential_matches": 0, + "files_scanned": 29, + "generic_secret_matches": 0, + "schema_version": "deltawire-v7-security-scan.v1", + "status": "pass", + "temporary_credential_files_remaining": 0 +} diff --git a/labs/20-deltawire/eval/results/preflight-v7/pre-live-report.json b/labs/20-deltawire/eval/results/preflight-v7/pre-live-report.json new file mode 100644 index 000000000..db09290b3 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v7/pre-live-report.json @@ -0,0 +1,101 @@ +{ + "READY_FOR_72": false, + "authorization": { + "accepted_phrase": "approve the frozen range-large/v7-r1 pair", + "pair_authorized": false, + "rerun_authorized": false, + "six_run_canary_authorized": false, + "full_run_authorized": false + }, + "ci": [ + { + "name": "general DeltaWire validation", + "run_id": 29952454841, + "status": "pass", + "url": "https://github.com/operatorstack/intelligence-flow/actions/runs/29952454841" + }, + { + "name": "v6 runtime integration validation", + "run_id": 29952454820, + "status": "pass", + "url": "https://github.com/operatorstack/intelligence-flow/actions/runs/29952454820" + }, + { + "event": "push", + "name": "v7 no-live validation", + "run_id": 29952451076, + "status": "pass", + "url": "https://github.com/operatorstack/intelligence-flow/actions/runs/29952451076" + }, + { + "event": "pull_request", + "name": "v7 no-live validation", + "run_id": 29952454863, + "status": "pass", + "url": "https://github.com/operatorstack/intelligence-flow/actions/runs/29952454863" + } + ], + "dry_run": { + "B0_conditional_on_D1": true, + "commands": [ + { + "arm": "D1", + "skill_assigned": true + }, + { + "arm": "B0", + "skill_assigned": false + } + ], + "model_calls": 0, + "order": [ + "D1", + "B0" + ], + "pair_root_empty_after": true, + "status": "pass" + }, + "frozen_identity": { + "agent": "preinstalled-gemini-cli-v1", + "agent_import": "agents.preinstalled_gemini_cli:PreinstalledGeminiCli", + "binary_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "gemini_cli_version": "0.51.0", + "harbor_version": "0.20.0", + "image_id": "sha256:589fce58d7ddf09910876afe0198bcee0086a840af6de7c08515bbce3e8367a7", + "infrastructure_commit": "a8d2d8260ceed56baf0d1cf8fcc2f40776e9f7b6", + "manifest_commit": "369f3c7c0d751da72fa658a2ec810557d6669e39", + "manifest_sha256": "67789b85c08802a87f5d14bb72aa2d36068c43a45aa27187ddab5d66dba3eca4", + "model": "google/gemini-3.1-pro-preview", + "pair_id": "range-large/v7-r1", + "skill_conformance_sha256": "4d0e142a1bc801642b5de8c77b9cbebc9680a42f6f64da95b3d48b8624fe7c46", + "skill_tree_sha256": "a841ca12dc687b1e467f90d5d9f5ccef120bc810742fab693e8e976135a75984" + }, + "locks": { + "v1_through_v5": "pass", + "v6_terminal": "pass" + }, + "pair_root": "labs/20-deltawire/eval/results/range-large-v7-r1", + "pair_root_empty": true, + "pull_request": { + "draft": true, + "number": 118, + "url": "https://github.com/operatorstack/intelligence-flow/pull/118" + }, + "schema_version": "deltawire-v7-pre-live-report.v1", + "skill_conformance": { + "examples": [ + "range", + "matrix", + "rows", + "variants" + ], + "malformed_json_rejected": true, + "model_calls": 0, + "repeated_byte_determinism": true, + "status": "pass", + "unsupported_field_rejected": true + }, + "six_run_canary_started": false, + "full_run_started": false, + "value_statement": "The frozen pair can now distinguish assigned and activated DeltaWire use from Gemini-specific discovery ceremony while retaining fail-closed artifact and semantic verification." +} diff --git a/labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json b/labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json new file mode 100644 index 000000000..b2c019c32 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json @@ -0,0 +1,141 @@ +{ + "binary_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "checks": { + "binary_exists": true, + "credentials_absent": true, + "examples_exact": true, + "examples_pass": true, + "negative_controls_pass": true, + "reference_exists": true, + "skill_file_exists": true, + "zero_model_calls": true + }, + "examples": [ + { + "example": "matrix.dw.json", + "example_sha256": "56d16a5b20086c8202cbc1bbdccf41d467a5d4c3806d701227ebcdbf72e54b1a", + "expected_records": [ + { + "method": "GET", + "role": "guest" + }, + { + "method": "POST", + "role": "guest" + }, + { + "method": "GET", + "role": "member" + }, + { + "method": "POST", + "role": "member" + } + ], + "record_count": 4, + "rendered_sha256": [ + "01ee7ae45519e704708945436792560f537ae7e67b435623190e7eb3c1ad8660", + "01ee7ae45519e704708945436792560f537ae7e67b435623190e7eb3c1ad8660" + ], + "repeated_byte_determinism": true, + "runs_pass": [ + true, + true + ], + "status": "pass" + }, + { + "example": "range.dw.json", + "example_sha256": "b31e078409547f4fbb5a855f3bf0b1bd6f52386696c07401622f34d224fe148b", + "expected_records": [ + { + "index": 1 + }, + { + "index": 2 + }, + { + "index": 3 + } + ], + "record_count": 3, + "rendered_sha256": [ + "308b0a63452f680897077bfec12487712eaf512a7d49d454c011d209b11557d7", + "308b0a63452f680897077bfec12487712eaf512a7d49d454c011d209b11557d7" + ], + "repeated_byte_determinism": true, + "runs_pass": [ + true, + true + ], + "status": "pass" + }, + { + "example": "rows.dw.json", + "example_sha256": "e172cc7d6224e4566a0cd46a73465392976338fea0e225512a089322ba81aca3", + "expected_records": [ + { + "enabled": true, + "name": "alpha", + "rank": 1 + }, + { + "enabled": true, + "name": "beta", + "rank": 2 + } + ], + "record_count": 2, + "rendered_sha256": [ + "77a7cabbfdbd71945310803ce919530ebf1b888d2e5c417c1344f0825fb63bad", + "77a7cabbfdbd71945310803ce919530ebf1b888d2e5c417c1344f0825fb63bad" + ], + "repeated_byte_determinism": true, + "runs_pass": [ + true, + true + ], + "status": "pass" + }, + { + "example": "variants.dw.json", + "example_sha256": "3183405b76e11874309728f3a4025687af6c54add8d2b97b94daff2d6ef8cdca", + "expected_records": [ + { + "active": true, + "role": "member", + "status": "deny" + }, + { + "active": false, + "role": "member", + "status": "allow" + } + ], + "record_count": 2, + "rendered_sha256": [ + "e6ac2a1b8f43a7557f3d8b9b93e290618bbf0e44f6137515bdf7c61403d428c8", + "e6ac2a1b8f43a7557f3d8b9b93e290618bbf0e44f6137515bdf7c61403d428c8" + ], + "repeated_byte_determinism": true, + "runs_pass": [ + true, + true + ], + "status": "pass" + } + ], + "harbor_skill_digest": "sha256:a841ca12dc687b1e467f90d5d9f5ccef120bc810742fab693e8e976135a75984", + "negative_controls": { + "checks": { + "malformed_json_rejected": true, + "unsupported_field_rejected": true + }, + "status": "pass" + }, + "reference_sha256": "f78d41266db73592526a3c5e7d6e4c883f549f99b01c6387c7f5337fc269dd5e", + "schema_version": "deltawire-skill-conformance.v1", + "skill_sha256": "e7d3a1e8ccef1b72338d5dac7a7b7ac412164846018ab55b7f2ecadf5a07a89a", + "skill_tree_sha256": "a841ca12dc687b1e467f90d5d9f5ccef120bc810742fab693e8e976135a75984", + "status": "pass" +} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/D1-artifact-manifest.json b/labs/20-deltawire/eval/results/range-large-v7-r1/D1-artifact-manifest.json new file mode 100644 index 000000000..c7573a7dc --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/D1-artifact-manifest.json @@ -0,0 +1,28 @@ +{ + "artifact_manifest_sha256": "0c8e85115c5bc9a68deb3aa39b84e4f734bdeb8b7742a97b72bc139b1192b386", + "checks": { + "binary_hash_frozen": true, + "config_hash_frozen": true, + "manifest_exists": true, + "manifest_status_ok": true, + "public_contract_hash_frozen": true, + "receipt_canonical_path": true, + "receipt_checks_complete": true, + "receipt_checks_true": true, + "receipt_schema_exact": true, + "receipt_status_pass": true, + "schema_hash_frozen": true, + "single_convention_entry": true + }, + "convention_entry": { + "destination": "artifacts/logs/artifacts", + "service": null, + "source": "/logs/artifacts", + "status": "ok", + "type": "directory" + }, + "environment_receipt_path": "/home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/logs/artifacts/deltawire/environment-receipt.json", + "environment_receipt_sha256": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "schema_version": "artifact-manifest-receipt.v1", + "status": "pass" +} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/D1-end-to-end.json b/labs/20-deltawire/eval/results/range-large-v7-r1/D1-end-to-end.json new file mode 100644 index 000000000..d7d22f367 --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/D1-end-to-end.json @@ -0,0 +1,30 @@ +{ + "READY_FOR_72": false, + "arm": "D1", + "component_checks": { + "artifact_manifest": true, + "environment": true, + "plan_contract": false, + "semantic": true, + "setup": true, + "treatment": false + }, + "identity_checks": { + "agent_match": true, + "harbor_exit_zero": true, + "model_match": true, + "trial_completed": true + }, + "schema_version": "deltawire-end-to-end-receipt.v7", + "source_hashes": { + "artifact_manifest": "40cf6ddc54e657621f177afcb92e88a94eec553ef2f29d829174d920b86ab469", + "environment": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "output": "9d205b57ab166d23c81927899f99b6ccece261c679c96979c58fbde975637776", + "result": "22f50c59c4eac9f5f1a680572304f25a11270c3fb2caa66be584bbd9d7a3525b", + "semantic": "09654564d33620bb4b39ae7cb64af13243078013118e6595adaf3a40af59cba0", + "setup": "adca6d256d43f320bb542745bc37f58fcfa97db736eebcc2aa1ce885d84c5240", + "trajectory": "d6659287d950b37b9ea1364e6256d715ce0220b652c45689355b79e623be7af5", + "treatment": "298daa00fb33afe6f7a93b67028bcc243fe1ef809f58199f89b36d35438148aa" + }, + "status": "fail" +} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/D1-environment.json b/labs/20-deltawire/eval/results/range-large-v7-r1/D1-environment.json new file mode 100644 index 000000000..55108c6ac --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/D1-environment.json @@ -0,0 +1,55 @@ +{ + "binary": { + "actual_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "executable": true, + "exists": true, + "expected_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "expected_version": "deltawire version dev", + "is_regular": true, + "is_symlink": false, + "path": "/usr/local/bin/deltawire", + "realpath": "/usr/local/bin/deltawire", + "version_exit_code": 0, + "version_stderr": "", + "version_stdout": "deltawire version dev\nGo runtime version go1.26.5\nsupported config version deltawire.config.v1\nsupported plan version deltawire.plan.v1\nsupported state version deltawire.state.v1" + }, + "checks": { + "binary_executable": true, + "binary_exists": true, + "binary_hash": true, + "binary_not_symlink": true, + "binary_path": true, + "binary_realpath": true, + "binary_regular": true, + "config_hash": true, + "public_contract_hash": true, + "schema_hash": true, + "version_exact": true, + "version_exit_0": true + }, + "files": { + "config": { + "actual_sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "exists": true, + "expected_sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "hash_match": true, + "path": "/.deltawire/config.json" + }, + "public_contract": { + "actual_sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "exists": true, + "expected_sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "hash_match": true, + "path": "/task-contract.json" + }, + "schema": { + "actual_sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "exists": true, + "expected_sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "hash_match": true, + "path": "/.deltawire/schemas/range-large.schema.json" + } + }, + "schema_version": "deltawire-environment-receipt.v1", + "status": "pass" +} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/D1-plan-contract-receipt.json b/labs/20-deltawire/eval/results/range-large-v7-r1/D1-plan-contract-receipt.json new file mode 100644 index 000000000..f1e010ed5 --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/D1-plan-contract-receipt.json @@ -0,0 +1,6 @@ +{ + "checks": {}, + "error": "expected one plan; found 0", + "schema_version": "plan-contract-receipt.v1", + "status": "fail" +} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/D1-semantic-result.json b/labs/20-deltawire/eval/results/range-large-v7-r1/D1-semantic-result.json new file mode 100644 index 000000000..14b571937 --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/D1-semantic-result.json @@ -0,0 +1,8 @@ +{ + "exact_match": 1, + "oracle": { + "exact_match": 1 + }, + "schema_version": "semantic-result.v7", + "status": "pass" +} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/D1-setup.json b/labs/20-deltawire/eval/results/range-large-v7-r1/D1-setup.json new file mode 100644 index 000000000..2e3d844f2 --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/D1-setup.json @@ -0,0 +1,44 @@ +{ + "checks": { + "agent_exact": true, + "checks_exact": true, + "checks_true": true, + "gemini_path_exact": true, + "gemini_version_exact": true, + "image_id_exact": true, + "no_timestamps": true, + "node_path_exact": true, + "node_version_exact": true, + "npm_version_exact": true, + "package_version_exact": true, + "receipt_canonical_path": true, + "receipt_exists": true, + "schema_exact": true, + "status_pass": true + }, + "receipt": { + "agent": "preinstalled-gemini-cli-v1", + "checks": { + "agent_identity": true, + "gemini_executable": true, + "gemini_version": true, + "image_id": true, + "node_executable": true, + "node_version": true, + "npm_version": true, + "package_identity": true + }, + "gemini_path": "/usr/local/bin/gemini", + "gemini_version": "0.51.0", + "image_id": "sha256:589fce58d7ddf09910876afe0198bcee0086a840af6de7c08515bbce3e8367a7", + "node_path": "/usr/local/bin/node", + "node_version": "v22.23.1", + "npm_version": "10.9.8", + "package_version": "0.51.0", + "schema_version": "preinstalled-gemini-setup-receipt.v1", + "status": "pass" + }, + "receipt_sha256": "a88d221e573265997797dcad31136aec0ed74a5bd37e50887595e5ea85cfa0d1", + "schema_version": "preinstalled-gemini-setup-validation.v1", + "status": "pass" +} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/D1-treatment.json b/labs/20-deltawire/eval/results/range-large-v7-r1/D1-treatment.json new file mode 100644 index 000000000..bbe547126 --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/D1-treatment.json @@ -0,0 +1,131 @@ +{ + "arm": "D1", + "artifact_hashes": { + "config": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "output": "9d205b57ab166d23c81927899f99b6ccece261c679c96979c58fbde975637776", + "schema": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "state": "5e1cf20f1d0945a30accb87152086fc61c848cdf18ac46ad2f0fa65fbd5b8c76" + }, + "assignment": { + "checks": { + "config_exactly_one_skill": true, + "config_skill_name": true, + "local_tree_hash": true, + "lock_exactly_one_skill": true, + "lock_skill_digest": true, + "lock_skill_name": true + }, + "status": "pass" + }, + "checks": { + "assignment_pass": true, + "check_invoked": true, + "check_not_proven_failed": true, + "in_container_contract_pass": false, + "independent_contract_pass": false, + "public_contract_access": false, + "render_invoked": true, + "render_not_proven_failed": true, + "retained_config": true, + "retained_in_container_contract": false, + "retained_output": true, + "retained_plan": false, + "retained_schema": true, + "retained_state": true, + "schema_integrity": true, + "semantic_exact_pass": true, + "structured_activation": true, + "validate_invoked": true, + "validate_not_proven_failed": true + }, + "classification": "attempted_failed", + "contamination": false, + "diagnostics": { + "binary_introspection": false, + "direct_skill_read": true, + "direct_validate_count": 1, + "failed_direct_validate_count": 0, + "gemini_skills_list_observed": false, + "matching_example_read": true, + "reference_read": true, + "subprocess_bruteforce_loops": 0 + }, + "lifecycle_invocations": [ + { + "command": "validate", + "invocation": "deltawire validate --repo . plan.json", + "shell_exit_code": null + }, + { + "command": "render", + "invocation": "deltawire render --repo . plan.json", + "shell_exit_code": null + }, + { + "command": "check", + "invocation": "deltawire check --repo . plan.json", + "shell_exit_code": null + } + ], + "schema_version": "deltawire-treatment-receipt.v7", + "shell_observations": [ + { + "command": "deltawire validate --repo . plan.json", + "observable": true, + "shell_exit_code": null, + "shell_exit_provenance": null, + "tool_status": "success" + }, + { + "command": "find / -maxdepth 3 -name \"*contract*check*\" -o -name \"*check*contract*\" 2>/dev/null", + "observable": true, + "shell_exit_code": null, + "shell_exit_provenance": null, + "tool_status": "success" + }, + { + "command": "ls -l /usr/local/bin | grep check", + "observable": true, + "shell_exit_code": 1, + "shell_exit_provenance": "explicit_output", + "tool_status": "success" + }, + { + "command": "ls -l /usr/local/bin", + "observable": true, + "shell_exit_code": null, + "shell_exit_provenance": null, + "tool_status": "success" + }, + { + "command": "verify_plan_contract.py plan.json", + "observable": true, + "shell_exit_code": 2, + "shell_exit_provenance": "explicit_output", + "tool_status": "success" + }, + { + "command": "verify_plan_contract.py --contract task-contract.json --plan plan.json", + "observable": true, + "shell_exit_code": null, + "shell_exit_provenance": null, + "tool_status": "success" + }, + { + "command": "deltawire render --repo . plan.json", + "observable": true, + "shell_exit_code": null, + "shell_exit_provenance": null, + "tool_status": "success" + }, + { + "command": "deltawire check --repo . plan.json", + "observable": true, + "shell_exit_code": null, + "shell_exit_provenance": null, + "tool_status": "success" + } + ], + "status": "fail", + "trajectory_sha256": "d6659287d950b37b9ea1364e6256d715ce0220b652c45689355b79e623be7af5" +} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/evidence-closure.json b/labs/20-deltawire/eval/results/range-large-v7-r1/evidence-closure.json new file mode 100644 index 000000000..514e26626 --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/evidence-closure.json @@ -0,0 +1,44 @@ +{ + "READY_FOR_72": false, + "file_count": 33, + "files": { + "labs/20-deltawire/eval/results/preflight-v7/pair-approval.json": "c6b493851951e72ead0d15e38231f47d036331f94ded94d9d3dceb38db7c3154", + "labs/20-deltawire/eval/results/preflight-v7/pair-security-scan.json": "4ab32a7795109d3973bd8665dbe60df485c6ac0af25b0b2e752e15a448650411", + "labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json": "4d0e142a1bc801642b5de8c77b9cbebc9680a42f6f64da95b3d48b8624fe7c46", + "labs/20-deltawire/eval/results/range-large-v7-r1/D1-artifact-manifest.json": "40cf6ddc54e657621f177afcb92e88a94eec553ef2f29d829174d920b86ab469", + "labs/20-deltawire/eval/results/range-large-v7-r1/D1-end-to-end.json": "207e6e300158c9d87b92589c412a7ae15aa732549dbb7bdaa8654a366efc62e1", + "labs/20-deltawire/eval/results/range-large-v7-r1/D1-environment.json": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "labs/20-deltawire/eval/results/range-large-v7-r1/D1-plan-contract-receipt.json": "d3c6d569d9353120583551f280753e457683c2b886594cda291b93b4e30515a3", + "labs/20-deltawire/eval/results/range-large-v7-r1/D1-semantic-result.json": "09654564d33620bb4b39ae7cb64af13243078013118e6595adaf3a40af59cba0", + "labs/20-deltawire/eval/results/range-large-v7-r1/D1-setup.json": "adca6d256d43f320bb542745bc37f58fcfa97db736eebcc2aa1ce885d84c5240", + "labs/20-deltawire/eval/results/range-large-v7-r1/D1-treatment.json": "298daa00fb33afe6f7a93b67028bcc243fe1ef809f58199f89b36d35438148aa", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/config.json": "27de86441290364f9bc0c6c66854b4bc0194f591b909dbaf275b628f40471728", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/job.log": "ad20c3e97e7c89bf6f07934e3cc709c4880709c400461527d363ea53ba121dfe", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/lock.json": "0a58888ac484406279a52b2109f9aa8215cc7890204b7ff2f17022bd5d703286", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/gemini-cli.trajectory.jsonl": "d6659287d950b37b9ea1364e6256d715ce0220b652c45689355b79e623be7af5", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/gemini-cli.txt": "54e1b5e6d019e43be0e7141827f3dccb32b4a613792551ca6b80e08452e9ee6c", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/trajectory.json": "d6c4bae360f51d98c467028fa0354f351e8b98d6b157b19baae328ea5f514db5", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/state.json": "5e1cf20f1d0945a30accb87152086fc61c848cdf18ac46ad2f0fa65fbd5b8c76", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/logs/artifacts/deltawire/environment-receipt.json": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json": "a88d221e573265997797dcad31136aec0ed74a5bd37e50887595e5ea85cfa0d1", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/manifest.json": "0c8e85115c5bc9a68deb3aa39b84e4f734bdeb8b7742a97b72bc139b1192b386", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/testdata/generated/output.ndjson": "9d205b57ab166d23c81927899f99b6ccece261c679c96979c58fbde975637776", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/config.json": "0768e4213b3a05725c0ef62c851d2c0bd902b102dfa134de749cfe2e992f5bad", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/lock.json": "5a931c5b1058e58b3792de2f7a9303139778180b3efedc2d3b91081ef14679a3", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/result.json": "22f50c59c4eac9f5f1a680572304f25a11270c3fb2caa66be584bbd9d7a3525b", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/trial.log": "ad20c3e97e7c89bf6f07934e3cc709c4880709c400461527d363ea53ba121dfe", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/verifier/reward.json": "36a492513204040f04d7bdf4146d84ede71353d8cc7df112fc3ebff66327683a", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/verifier/test-stdout.txt": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/result.json": "5596535bf83df1d66ae5fa4f6388734fd456a6825505e7f6ad11295552c48cb3", + "labs/20-deltawire/eval/results/range-large-v7-r1/readiness.json": "93124d41c2c3c66043bf5fa46673893faf31737bdfcada75656320a98ea543e9", + "labs/20-deltawire/eval/results/range-large-v7-r1/run-ledger.json": "8b4438d59fffcc5b193c68cae268d400f30200bc28b2156249d978f544b5f541", + "labs/20-deltawire/eval/results/range-large-v7-r1/terminal-report.json": "86dcc3abf7f998089ebdfed173322a7db96e28a36eac7eb9a5087630b89a85aa" + }, + "pair_id": "range-large/v7-r1", + "pair_valid": false, + "rerun_authorized": false, + "schema_version": "deltawire-v7-evidence-closure.v1", + "status": "closed" +} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/config.json b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/config.json new file mode 100644 index 000000000..c06fb67fd --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/config.json @@ -0,0 +1,18 @@ +{ + "jobs_dir": "/home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1", + "n_concurrent_trials": 1, + "agents": [ + { + "name": "agents.preinstalled_gemini_cli:PreinstalledGeminiCli", + "model_name": "google/gemini-3.1-pro-preview", + "skills": [ + "/home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire" + ] + } + ], + "tasks": [ + { + "path": "/home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/pairs/range-large-v7-r1" + } + ] +} \ No newline at end of file diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/lock.json b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/lock.json new file mode 100644 index 000000000..b6dc2a1f9 --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/lock.json @@ -0,0 +1,70 @@ +{ + "schema_version": 2, + "created_at": "2026-07-22T19:57:08.288018Z", + "harbor": { + "version": "0.20.0", + "is_editable": false + }, + "n_concurrent_trials": 1, + "retry": { + "max_retries": 0, + "exclude_exceptions": [ + "VerifierOutputParseError", + "VerifierTimeoutError", + "ApiUsageLimitError", + "AgentSafetyRefusalError", + "AgentAuthenticationError", + "AgentTimeoutError", + "RewardFileNotFoundError", + "RewardFileEmptyError", + "ModelNotFoundError" + ], + "wait_multiplier": 1.0, + "min_wait_sec": 1.0, + "max_wait_sec": 60.0 + }, + "trials": [ + { + "schema_version": 1, + "task": { + "name": "range-large-v7-r1", + "type": "local", + "digest": "sha256:f4b01bec8b34ab576ed35e9b9512b88945f0dd566a5f5af1e066476568d54cc1", + "path": "/home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/pairs/range-large-v7-r1" + }, + "install_only": false, + "timeout_multiplier": 1.0, + "agent": { + "name": "agents.preinstalled_gemini_cli:PreinstalledGeminiCli", + "model_name": "google/gemini-3.1-pro-preview", + "skills": [ + "/home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire" + ], + "resume_trajectory": false, + "extra_allowed_hosts": [], + "kwargs": {}, + "mcp_servers": [] + }, + "skills": [ + { + "name": "deltawire", + "source": "/home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire", + "digest": "sha256:a841ca12dc687b1e467f90d5d9f5ccef120bc810742fab693e8e976135a75984" + } + ], + "environment": { + "type": "docker", + "force_build": false, + "delete": true, + "cpu_enforcement_policy": "auto", + "memory_enforcement_policy": "auto", + "extra_docker_compose": [], + "kwargs": {}, + "extra_allowed_hosts": [] + }, + "verifier": { + "disable": false + } + } + ] +} \ No newline at end of file diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/gemini-cli.trajectory.jsonl b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/gemini-cli.trajectory.jsonl new file mode 100644 index 000000000..5bb65369f --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/gemini-cli.trajectory.jsonl @@ -0,0 +1,56 @@ +{"sessionId":"e8bc4290-e6c1-479c-88ef-e3a1d02d4477","projectHash":"8a5edab282632443219e051e4ade2d1d5bbc671c781051bf1437897cbdfea0f1","startTime":"2026-07-22T19:57:16.281Z","lastUpdated":"2026-07-22T19:57:16.281Z","kind":"main"} +{"$set":{"messages":[{"id":"d04923d38bb0f6017037e74183378ef4","timestamp":"2026-07-22T19:57:16.281Z","type":"user","content":[{"text":"\nThis is the Gemini CLI. We are setting up the context for our chat.\nToday's date is Wednesday, July 22, 2026 (formatted according to the user's locale).\nMy operating system is: linux\nThe project's temporary directory is: /root/.gemini/tmp/project\n- **Workspace Directories:**\n - /\n- **Directory Structure:**\n\nShowing up to 200 items (files + folders). Folders or files indicated with ... contain more items not shown, were ignored, or the display limit (200 items) was reached.\n\n//\n├───.dockerenv\n├───deltawire-environment-expectations.json\n├───task-contract.json\n├───.deltawire/\n│ ├───config.json\n│ └───schemas/\n├───boot/\n├───dev/\n│ ├───mqueue/\n│ ├───pts/\n│ └───shm/\n├───etc/\n│ ├───.pwd.lock\n│ ├───adduser.conf\n│ ├───bash.bashrc\n│ ├───bindresvport.blacklist\n│ ├───ca-certificates.conf\n│ ├───debconf.conf\n│ ├───debian_version\n│ ├───deluser.conf\n│ ├───e2scrub.conf\n│ ├───environment\n│ ├───fstab\n│ ├───gai.conf\n│ ├───group\n│ ├───group-\n│ ├───gshadow\n│ ├───gshadow-\n│ ├───host.conf\n│ ├───hostname\n│ ├───hosts\n│ ├───inputrc\n│ ├───issue\n│ ├───issue.net\n│ ├───ld.so.cache\n│ ├───ld.so.conf\n│ ├───libaudit.conf\n│ ├───login.defs\n│ ├───mime.types\n│ ├───mke2fs.conf\n│ ├───motd\n│ ├───netconfig\n│ ├───nsswitch.conf\n│ ├───pam.conf\n│ ├───passwd\n│ ├───passwd-\n│ ├───profile\n│ ├───resolv.conf\n│ ├───shadow\n│ ├───shadow-\n│ ├───shells\n│ ├───subgid\n│ ├───subgid-\n│ ├───subuid\n│ ├───subuid-\n│ ├───timezone\n│ ├───xattr.conf\n│ ├───alternatives/\n│ ├───apt/\n│ ├───ca-certificates/\n│ ├───cron.d/\n│ ├───cron.daily/\n│ ├───default/\n│ ├───dpkg/\n│ ├───gss/\n│ ├───init.d/\n│ ├───kernel/\n│ ├───ld.so.conf.d/\n│ ├───logrotate.d/\n│ ├───opt/\n│ ├───pam.d/\n│ ├───profile.d/\n│ ├───python3/\n│ ├───python3.11/\n│ ├───rc0.d/\n│ ├───rc1.d/\n│ ├───rc2.d/\n│ ├───rc3.d/\n│ ├───rc4.d/\n│ ├───rc5.d/\n│ ├───rc6.d/\n│ ├───rcS.d/\n│ ├───security/\n│ ├───selinux/\n│ ├───skel/\n│ ├───ssl/\n│ ├───systemd/\n│ ├───terminfo/\n│ └───update-motd.d/\n├───harbor/\n│ └───skills/\n├───home/\n│ └───node/\n├───installed-agent/\n├───logs/\n│ ├───agent/\n│ ├───artifacts/\n│ └───verifier/\n├───media/\n├───mnt/\n├───opt/\n│ └───yarn-v1.22.22/\n├───proc/\n│ ├───bootconfig\n│ ├───buddyinfo\n│ ├───cgroups\n│ ├───cmdline\n│ ├───consoles\n│ ├───cpuinfo\n│ ├───crypto\n│ ├───devices\n│ ├───diskstats\n│ ├───dma\n│ ├───execdomains\n│ ├───fb\n│ ├───filesystems\n│ ├───interrupts\n│ ├───iomem\n│ ├───ioports\n│ ├───kallsyms\n│ ├───kcore\n│ ├───key-users\n│ ├───keys\n│ ├───kmsg\n│ ├───kpagecgroup\n│ ├───kpagecount\n│ ├───kpageflags\n│ ├───latency_stats\n│ ├───loadavg\n│ ├───locks\n│ ├───mdstat\n│ ├───meminfo\n│ ├───misc\n│ ├───modules\n│ ├───mtrr\n│ ├───pagetypeinfo\n│ ├───partitions\n│ ├───schedstat\n│ ├───slabinfo\n│ ├───softirqs\n│ ├───stat\n│ ├───swaps\n│ ├───sysrq-trigger\n│ ├───timer_list\n│ ├───uptime\n│ ├───version\n│ ├───version_signature\n│ ├───vmallocinfo\n│ ├───vmstat\n│ ├───zoneinfo\n│ ├───1/\n│ ├───154/\n│ ├───160/\n│ ├───161/\n│ ├───169/\n│ ├───8/\n│ ├───acpi/\n│ ├───bus/\n│ ├───driver/\n│ ├───dynamic_debug/\n│ ├───fs/\n│ ├───irq/\n│ ├───pressure/\n│ ├───scsi/\n│ ├───sys/\n│ ├───sysvipc/\n│ └───tty/\n├───root/\n│ ├───.bashrc\n│ ├───.profile\n│ ├───.cache/\n│ ├───.gemini/\n│ └───.npm/\n├───run/\n│ └───lock/\n├───srv/\n├───sys/\n│ ├───block/\n│ ├───bus/\n│ ├───class/\n│ ├───dev/\n│ ├───devices/\n│ ├───firmware/\n│ ├───fs/\n│ ├───hypervisor/\n│ ├───kernel/\n│ ├───module/\n│ └───power/\n├───tmp/\n│ └───node-compile-cache/\n├───usr/\n│ ├───bin/\n│ ├───games/\n│ ├───include/\n│ ├───lib/\n│ ├───lib64/\n│ ├───libexec/\n│ ├───local/\n│ ├───sbin/\n│ └───...\n└───var/\n\n\n"}]}],"lastUpdated":"2026-07-22T19:57:16.281Z"}} +{"id":"0f3b73f6-73c5-472a-879d-93a31eb28c16","timestamp":"2026-07-22T19:57:16.294Z","type":"user","content":[{"text":"Generate the exact `range-large` dataset defined by public `/task-contract.json`. Write exactly 500 JSON records in canonical order as NDJSON to `testdata/generated/output.ndjson`. Every record, field, value, and ordering rule must match this complete embedded contract:\n\n```json\n{\n \"authoritative_schema\": {\n \"path\": \".deltawire/schemas/range-large.schema.json\",\n \"sha256\": \"2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5\"\n },\n \"deltawire_applicability\": \"expected\",\n \"family\": \"range\",\n \"generation\": {\n \"field\": \"index\",\n \"field_order\": [\n \"index\"\n ],\n \"start\": 1\n },\n \"ordering\": \"canonical_generation_order\",\n \"output\": {\n \"format\": \"ndjson\",\n \"path\": \"testdata/generated/output.ndjson\"\n },\n \"record_count\": 500,\n \"schema_version\": \"task-spec.v1\",\n \"size\": \"large\",\n \"task\": \"range-large\"\n}\n```\n"}]} +{"$set":{"lastUpdated":"2026-07-22T19:57:16.294Z"}} +{"id":"682ab368-e99f-4c66-884a-7238a662f467","timestamp":"2026-07-22T19:57:20.180Z","type":"gemini","content":"","thoughts":[{"subject":"Generating Test Data","description":"I'm currently focused on creating a dataset per the specified contract for the 'range-large' task. My goal is to produce precisely 500 NDJSON records to `testdata/generated/output.ndjson`. The contract details are guiding this generation process.","timestamp":"2026-07-22T19:57:19.301Z"}],"tokens":{"input":11677,"output":147,"cached":0,"thoughts":164,"tool":0,"total":11988},"model":"gemini-3.1-pro-preview"} +{"$set":{"lastUpdated":"2026-07-22T19:57:20.180Z"}} +{"id":"682ab368-e99f-4c66-884a-7238a662f467","timestamp":"2026-07-22T19:57:20.180Z","type":"gemini","content":"","thoughts":[{"subject":"Generating Test Data","description":"I'm currently focused on creating a dataset per the specified contract for the 'range-large' task. My goal is to produce precisely 500 NDJSON records to `testdata/generated/output.ndjson`. The contract details are guiding this generation process.","timestamp":"2026-07-22T19:57:19.301Z"}],"tokens":{"input":11677,"output":147,"cached":0,"thoughts":164,"tool":0,"total":11988},"model":"gemini-3.1-pro-preview","toolCalls":[{"id":"update_topic__1k3g0da4","name":"update_topic","args":{"strategic_intent":"Activate the deltawire skill and read the authoritative schema to understand the dataset generation rules.","summary":"I am activating the deltawire skill to generate a deterministic JSON dataset. I will read the schema provided in the contract to understand the fields and rules required to generate exactly 500 records in NDJSON format.","title":"Generating Range-Large Dataset","wait_for_previous":false},"result":[{"functionResponse":{"id":"update_topic__1k3g0da4","name":"update_topic","response":{"output":"Current topic: \"Generating Range-Large Dataset\"\nTopic summary: I am activating the deltawire skill to generate a deterministic JSON dataset. I will read the schema provided in the contract to understand the fields and rules required to generate exactly 500 records in NDJSON format.\n\nStrategic Intent: Activate the deltawire skill and read the authoritative schema to understand the dataset generation rules."}}}],"status":"success","timestamp":"2026-07-22T19:57:20.196Z","resultDisplay":"## 📂 Topic: **Generating Range-Large Dataset**\n\n**Summary:**\nI am activating the deltawire skill to generate a deterministic JSON dataset. I will read the schema provided in the contract to understand the fields and rules required to generate exactly 500 records in NDJSON format.\n\n> [!STRATEGY]\n> **Intent:** Activate the deltawire skill and read the authoritative schema to understand the dataset generation rules.","description":"Update topic to: \"Generating Range-Large Dataset\"","displayName":"Update Topic Context","renderOutputAsMarkdown":true},{"id":"activate_skill__42osg4cn","name":"activate_skill","args":{"name":"deltawire"},"result":[{"functionResponse":{"id":"activate_skill__42osg4cn","name":"activate_skill","response":{"output":"\n \n # DeltaWire\n\nRead the public task contract first. Preserve its output path, format, schema\npath and hash, record count, field set, values, and ordering.\n\nRead `references/plan-v1.md` and the matching bundled example before creating a\nplan. Use `examples/range.dw.json` for integer sequences,\n`examples/matrix.dw.json` for Cartesian products, `examples/rows.dw.json` for\nexplicit repeated rows, or `examples/variants.dw.json` for base records with\nsmall changes.\n\nCreate only the requested plan. Do not edit the installed config or schema, do\nnot generate the expanded output manually, and do not inspect or reverse\nengineer the DeltaWire binary. Resolve validation errors by comparing the plan\nwith the reference and example instead of guessing field names.\n\nRun this lifecycle in order:\n\n1. `deltawire validate --repo . `\n2. Run the installed independent contract checker for the public contract.\n3. `deltawire render --repo . `\n4. `deltawire check --repo . `\n\nStop after all four steps pass and the required artifacts are retained.\n \n\n \n Showing up to 200 items (files + folders).\n\n/root/.gemini/skills/deltawire/\n├───SKILL.md\n├───examples/\n│ ├───matrix.dw.json\n│ ├───range.dw.json\n│ ├───rows.dw.json\n│ └───variants.dw.json\n└───references/\n └───plan-v1.md\n \n"}}}],"status":"success","timestamp":"2026-07-22T19:57:20.196Z","resultDisplay":"Skill **deltawire** activated. Resources loaded from `/root/.gemini/skills/deltawire`:\n\nShowing up to 200 items (files + folders).\n\n/root/.gemini/skills/deltawire/\n├───SKILL.md\n├───examples/\n│ ├───matrix.dw.json\n│ ├───range.dw.json\n│ ├───rows.dw.json\n│ └───variants.dw.json\n└───references/\n └───plan-v1.md","description":"\"deltawire\": Generate large deterministic JSON or NDJSON datasets from compact ranges, Cartesian products, repeated rows, variants, defaults, and regular exceptions. Use when the requested output has a compact exact plan; abstain when records are small or too irregular for a deterministic description.","displayName":"Activate Skill","renderOutputAsMarkdown":true},{"id":"read_file__hwsp96t0","name":"read_file","args":{"file_path":"/.deltawire/schemas/range-large.schema.json"},"result":[{"functionResponse":{"id":"read_file__hwsp96t0","name":"read_file","response":{"output":"{\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"additionalProperties\": false,\n \"properties\": {\n \"index\": {\n \"maximum\": 500,\n \"minimum\": 1,\n \"type\": \"integer\"\n }\n },\n \"required\": [\n \"index\"\n ],\n \"title\": \"range-large\",\n \"type\": \"object\"\n}\n"}}}],"status":"success","timestamp":"2026-07-22T19:57:20.196Z","resultDisplay":"","description":".del.../.../range-large.schema.json","displayName":"ReadFile","renderOutputAsMarkdown":true}]} +{"id":"f0a60467-d7cd-4c8f-b62a-e5674a06fe86","timestamp":"2026-07-22T19:57:20.197Z","type":"user","content":[{"functionResponse":{"id":"update_topic__1k3g0da4","name":"update_topic","response":{"output":"Current topic: \"Generating Range-Large Dataset\"\nTopic summary: I am activating the deltawire skill to generate a deterministic JSON dataset. I will read the schema provided in the contract to understand the fields and rules required to generate exactly 500 records in NDJSON format.\n\nStrategic Intent: Activate the deltawire skill and read the authoritative schema to understand the dataset generation rules."}}},{"functionResponse":{"id":"activate_skill__42osg4cn","name":"activate_skill","response":{"output":"\n \n # DeltaWire\n\nRead the public task contract first. Preserve its output path, format, schema\npath and hash, record count, field set, values, and ordering.\n\nRead `references/plan-v1.md` and the matching bundled example before creating a\nplan. Use `examples/range.dw.json` for integer sequences,\n`examples/matrix.dw.json` for Cartesian products, `examples/rows.dw.json` for\nexplicit repeated rows, or `examples/variants.dw.json` for base records with\nsmall changes.\n\nCreate only the requested plan. Do not edit the installed config or schema, do\nnot generate the expanded output manually, and do not inspect or reverse\nengineer the DeltaWire binary. Resolve validation errors by comparing the plan\nwith the reference and example instead of guessing field names.\n\nRun this lifecycle in order:\n\n1. `deltawire validate --repo . `\n2. Run the installed independent contract checker for the public contract.\n3. `deltawire render --repo . `\n4. `deltawire check --repo . `\n\nStop after all four steps pass and the required artifacts are retained.\n \n\n \n Showing up to 200 items (files + folders).\n\n/root/.gemini/skills/deltawire/\n├───SKILL.md\n├───examples/\n│ ├───matrix.dw.json\n│ ├───range.dw.json\n│ ├───rows.dw.json\n│ └───variants.dw.json\n└───references/\n └───plan-v1.md\n \n"}}},{"functionResponse":{"id":"read_file__hwsp96t0","name":"read_file","response":{"output":"{\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"additionalProperties\": false,\n \"properties\": {\n \"index\": {\n \"maximum\": 500,\n \"minimum\": 1,\n \"type\": \"integer\"\n }\n },\n \"required\": [\n \"index\"\n ],\n \"title\": \"range-large\",\n \"type\": \"object\"\n}\n"}}}]} +{"$set":{"lastUpdated":"2026-07-22T19:57:20.197Z"}} +{"id":"00d15ab8-29f4-4197-9356-8689fc72a51f","timestamp":"2026-07-22T19:57:25.045Z","type":"gemini","content":"","thoughts":[{"subject":"Reviewing Plan and Examples","description":"I'm currently reviewing the initial plan document and the provided examples. I'm focusing on how they structure integer sequences, Cartesian products, and explicit row repetition. This is to ensure my plan adheres to the established patterns. I'm leveraging the `examples` directory for patterns.","timestamp":"2026-07-22T19:57:24.071Z"}],"tokens":{"input":12600,"output":192,"cached":8071,"thoughts":137,"tool":0,"total":12929},"model":"gemini-3.1-pro-preview"} +{"$set":{"lastUpdated":"2026-07-22T19:57:25.045Z"}} +{"id":"00d15ab8-29f4-4197-9356-8689fc72a51f","timestamp":"2026-07-22T19:57:25.045Z","type":"gemini","content":"","thoughts":[{"subject":"Reviewing Plan and Examples","description":"I'm currently reviewing the initial plan document and the provided examples. I'm focusing on how they structure integer sequences, Cartesian products, and explicit row repetition. This is to ensure my plan adheres to the established patterns. I'm leveraging the `examples` directory for patterns.","timestamp":"2026-07-22T19:57:24.071Z"}],"tokens":{"input":12600,"output":192,"cached":8071,"thoughts":137,"tool":0,"total":12929},"model":"gemini-3.1-pro-preview","toolCalls":[{"id":"update_topic__klcvc3mx","name":"update_topic","args":{"title":"Generating Range-Large Dataset","summary":"I have read the schema for the `range-large` task, which consists of a single `index` integer property. I will now read the DeltaWire reference documentation and the `range.dw.json` example to learn how to construct a valid DeltaWire plan for this dataset.","wait_for_previous":false,"strategic_intent":"Read the DeltaWire plan reference and range example to understand how to write a DeltaWire plan for generating this dataset."},"result":[{"functionResponse":{"id":"update_topic__klcvc3mx","name":"update_topic","response":{"output":"Strategic Intent: Read the DeltaWire plan reference and range example to understand how to write a DeltaWire plan for generating this dataset."}}}],"status":"success","timestamp":"2026-07-22T19:57:25.055Z","resultDisplay":"> [!STRATEGY]\n> **Intent:** Read the DeltaWire plan reference and range example to understand how to write a DeltaWire plan for generating this dataset.","description":"Update topic to: \"Generating Range-Large Dataset\"","displayName":"Update Topic Context","renderOutputAsMarkdown":true},{"id":"read_file__untyuu15","name":"read_file","args":{"file_path":"/root/.gemini/skills/deltawire/references/plan-v1.md"},"result":[{"functionResponse":{"id":"read_file__untyuu15","name":"read_file","response":{"output":"# DeltaWire plan v1\n\nA plan is strict JSON. Unknown fields fail decoding.\n\n## Plan fields\n\n- `version`: exactly `deltawire.plan.v1`.\n- `id`, `description`: descriptive strings.\n- `record_schema`: repository-relative JSON Schema path used by `check`.\n- `output.path`: repository-relative output path.\n- `output.format`: `json` or `ndjson`.\n- `output.pretty`: JSON pretty-printing flag.\n- `defaults`: optional object deep-merged into each record.\n- `sets`: optional named arrays of scalar dimension values.\n- `generators`: ordered array of `matrix`, `rows`, or `variants` generators.\n- `assertions.count`: exact projected record count.\n- `assertions.unique`: optional JSON Pointer paths that must be unique.\n- `assertions.coverage`: optional JSON Pointer/value coverage checks.\n\n## Matrix\n\nA matrix generator has `kind`, `name`, `dimensions`, and `record`. Each\ndimension has a `name` plus exactly one of `values`, `set`, or `range`. An\ninline integer range uses `start`, `end_exclusive`, and nonzero `step`.\n\nReference dimensions from the record with `${name}`. When a string is exactly\none placeholder, the generated scalar retains its original type; for example,\nan integer range produces JSON integers. Embedded placeholders stringify their\nscalar values. Every dimension must be referenced by the record.\n\n## Rows\n\nA rows generator has `kind`, `name`, `columns`, and `rows`. Each column is a\nJSON Pointer such as `/role`; every row must have exactly one value per column.\nRows begin with plan defaults, then column values are assigned in order.\n\n## Variants\n\nA variants generator has `kind`, `name`, `base`, and `variants`. Each variant\nhas a unique `name`, optional `set` JSON Pointer/value assignments, and optional\n`omit` JSON Pointer paths. Records are built by deep-merging plan defaults and\nthe generator base, then applying the variant changes.\n\nGenerators emit records in plan order. Matrix dimensions vary from the last\ndimension inward, preserving each values/range order.\n"}}}],"status":"success","timestamp":"2026-07-22T19:57:25.055Z","resultDisplay":"","description":"root/.../references/plan-v1.md","displayName":"ReadFile","renderOutputAsMarkdown":true},{"id":"read_file__3od2weo9","name":"read_file","args":{"file_path":"/root/.gemini/skills/deltawire/examples/range.dw.json"},"result":[{"functionResponse":{"id":"read_file__3od2weo9","name":"read_file","response":{"output":"{\n \"version\": \"deltawire.plan.v1\",\n \"id\": \"range-example\",\n \"description\": \"Three ordered integer records\",\n \"record_schema\": \".deltawire/schemas/example.schema.json\",\n \"output\": {\"path\": \"testdata/generated/range.ndjson\", \"format\": \"ndjson\", \"pretty\": false},\n \"generators\": [{\n \"kind\": \"matrix\",\n \"name\": \"records\",\n \"dimensions\": [{\"name\": \"index\", \"range\": {\"start\": 1, \"end_exclusive\": 4, \"step\": 1}}],\n \"record\": {\"index\": \"${index}\"}\n }],\n \"assertions\": {\"count\": 3}\n}\n"}}}],"status":"success","timestamp":"2026-07-22T19:57:25.055Z","resultDisplay":"","description":"root/.../examples/range.dw.json","displayName":"ReadFile","renderOutputAsMarkdown":true}]} +{"id":"3798be68-9048-4a28-8e15-43cb9a55afd7","timestamp":"2026-07-22T19:57:25.056Z","type":"user","content":[{"functionResponse":{"id":"update_topic__klcvc3mx","name":"update_topic","response":{"output":"Strategic Intent: Read the DeltaWire plan reference and range example to understand how to write a DeltaWire plan for generating this dataset."}}},{"functionResponse":{"id":"read_file__untyuu15","name":"read_file","response":{"output":"# DeltaWire plan v1\n\nA plan is strict JSON. Unknown fields fail decoding.\n\n## Plan fields\n\n- `version`: exactly `deltawire.plan.v1`.\n- `id`, `description`: descriptive strings.\n- `record_schema`: repository-relative JSON Schema path used by `check`.\n- `output.path`: repository-relative output path.\n- `output.format`: `json` or `ndjson`.\n- `output.pretty`: JSON pretty-printing flag.\n- `defaults`: optional object deep-merged into each record.\n- `sets`: optional named arrays of scalar dimension values.\n- `generators`: ordered array of `matrix`, `rows`, or `variants` generators.\n- `assertions.count`: exact projected record count.\n- `assertions.unique`: optional JSON Pointer paths that must be unique.\n- `assertions.coverage`: optional JSON Pointer/value coverage checks.\n\n## Matrix\n\nA matrix generator has `kind`, `name`, `dimensions`, and `record`. Each\ndimension has a `name` plus exactly one of `values`, `set`, or `range`. An\ninline integer range uses `start`, `end_exclusive`, and nonzero `step`.\n\nReference dimensions from the record with `${name}`. When a string is exactly\none placeholder, the generated scalar retains its original type; for example,\nan integer range produces JSON integers. Embedded placeholders stringify their\nscalar values. Every dimension must be referenced by the record.\n\n## Rows\n\nA rows generator has `kind`, `name`, `columns`, and `rows`. Each column is a\nJSON Pointer such as `/role`; every row must have exactly one value per column.\nRows begin with plan defaults, then column values are assigned in order.\n\n## Variants\n\nA variants generator has `kind`, `name`, `base`, and `variants`. Each variant\nhas a unique `name`, optional `set` JSON Pointer/value assignments, and optional\n`omit` JSON Pointer paths. Records are built by deep-merging plan defaults and\nthe generator base, then applying the variant changes.\n\nGenerators emit records in plan order. Matrix dimensions vary from the last\ndimension inward, preserving each values/range order.\n"}}},{"functionResponse":{"id":"read_file__3od2weo9","name":"read_file","response":{"output":"{\n \"version\": \"deltawire.plan.v1\",\n \"id\": \"range-example\",\n \"description\": \"Three ordered integer records\",\n \"record_schema\": \".deltawire/schemas/example.schema.json\",\n \"output\": {\"path\": \"testdata/generated/range.ndjson\", \"format\": \"ndjson\", \"pretty\": false},\n \"generators\": [{\n \"kind\": \"matrix\",\n \"name\": \"records\",\n \"dimensions\": [{\"name\": \"index\", \"range\": {\"start\": 1, \"end_exclusive\": 4, \"step\": 1}}],\n \"record\": {\"index\": \"${index}\"}\n }],\n \"assertions\": {\"count\": 3}\n}\n"}}}]} +{"$set":{"lastUpdated":"2026-07-22T19:57:25.057Z"}} +{"id":"88122d10-64e1-4e82-ae18-6062c223c1d8","timestamp":"2026-07-22T19:57:34.705Z","type":"gemini","content":"","thoughts":[{"subject":"Defining Output Location","description":"I've nailed down the output file path as `testdata/generated/output.ndjson`. This aligns directly with the specifications. Also, I have confirmed that the schema for the records will be `.deltawire/schemas/range-large.schema.json`, and the output format should be ndjson as required.","timestamp":"2026-07-22T19:57:30.846Z"},{"subject":"Formulating Execution Plan","description":"I'm now sketching out the execution plan, and have decided to name the file `plan.json`. Based on the steps, I'll place it as `.deltawire/plan.json`. This seems correct, as the step specifies validating from the repository's root directory. The plan will include `generators` for the data generation and `assertions.count` to validate the record count as 500, with `start: 1, end_exclusive: 501, step: 1`. I need to review and check the contract and schema.","timestamp":"2026-07-22T19:57:30.846Z"}],"tokens":{"input":13635,"output":395,"cached":8065,"thoughts":756,"tool":0,"total":14786},"model":"gemini-3.1-pro-preview"} +{"$set":{"lastUpdated":"2026-07-22T19:57:34.705Z"}} +{"id":"88122d10-64e1-4e82-ae18-6062c223c1d8","timestamp":"2026-07-22T19:57:34.705Z","type":"gemini","content":"","thoughts":[{"subject":"Defining Output Location","description":"I've nailed down the output file path as `testdata/generated/output.ndjson`. This aligns directly with the specifications. Also, I have confirmed that the schema for the records will be `.deltawire/schemas/range-large.schema.json`, and the output format should be ndjson as required.","timestamp":"2026-07-22T19:57:30.846Z"},{"subject":"Formulating Execution Plan","description":"I'm now sketching out the execution plan, and have decided to name the file `plan.json`. Based on the steps, I'll place it as `.deltawire/plan.json`. This seems correct, as the step specifies validating from the repository's root directory. The plan will include `generators` for the data generation and `assertions.count` to validate the record count as 500, with `start: 1, end_exclusive: 501, step: 1`. I need to review and check the contract and schema.","timestamp":"2026-07-22T19:57:30.846Z"}],"tokens":{"input":13635,"output":395,"cached":8065,"thoughts":756,"tool":0,"total":14786},"model":"gemini-3.1-pro-preview","toolCalls":[{"id":"update_topic__bqa8f8l2","name":"update_topic","args":{"wait_for_previous":false,"title":"Generating Range-Large Dataset","strategic_intent":"Create the DeltaWire plan and run the required lifecycle commands to generate and validate the dataset.","summary":"I will write the `plan.json` containing the rules to generate the 500 records. After saving the plan, I will validate it using `deltawire validate`, run any available contract checkers in the project root, render the output with `deltawire render`, and finally check the generated data with `deltawire check`."},"result":[{"functionResponse":{"id":"update_topic__bqa8f8l2","name":"update_topic","response":{"output":"Strategic Intent: Create the DeltaWire plan and run the required lifecycle commands to generate and validate the dataset."}}}],"status":"success","timestamp":"2026-07-22T19:57:34.719Z","resultDisplay":"> [!STRATEGY]\n> **Intent:** Create the DeltaWire plan and run the required lifecycle commands to generate and validate the dataset.","description":"Update topic to: \"Generating Range-Large Dataset\"","displayName":"Update Topic Context","renderOutputAsMarkdown":true},{"id":"write_file__mr1cteab","name":"write_file","args":{"file_path":"plan.json","content":"{\n \"version\": \"deltawire.plan.v1\",\n \"id\": \"range-large\",\n \"description\": \"500 ordered integer records\",\n \"record_schema\": \".deltawire/schemas/range-large.schema.json\",\n \"output\": {\n \"path\": \"testdata/generated/output.ndjson\",\n \"format\": \"ndjson\",\n \"pretty\": false\n },\n \"generators\": [\n {\n \"kind\": \"matrix\",\n \"name\": \"records\",\n \"dimensions\": [\n {\n \"name\": \"index\",\n \"range\": {\n \"start\": 1,\n \"end_exclusive\": 501,\n \"step\": 1\n }\n }\n ],\n \"record\": {\n \"index\": \"${index}\"\n }\n }\n ],\n \"assertions\": {\n \"count\": 500\n }\n}\n"},"result":[{"functionResponse":{"id":"write_file__mr1cteab","name":"write_file","response":{"output":"Successfully created and wrote to new file: /plan.json. Here is the updated code:\n{\n \"version\": \"deltawire.plan.v1\",\n \"id\": \"range-large\",\n \"description\": \"500 ordered integer records\",\n \"record_schema\": \".deltawire/schemas/range-large.schema.json\",\n \"output\": {\n \"path\": \"testdata/generated/output.ndjson\",\n \"format\": \"ndjson\",\n \"pretty\": false\n },\n \"generators\": [\n {\n \"kind\": \"matrix\",\n \"name\": \"records\",\n \"dimensions\": [\n {\n \"name\": \"index\",\n \"range\": {\n \"start\": 1,\n \"end_exclusive\": 501,\n \"step\": 1\n }\n }\n ],\n \"record\": {\n \"index\": \"${index}\"\n }\n }\n ],\n \"assertions\": {\n \"count\": 500\n }\n}\n"}}}],"status":"success","timestamp":"2026-07-22T19:57:34.719Z","resultDisplay":{"fileDiff":"Index: plan.json\n===================================================================\n--- plan.json\tOriginal\n+++ plan.json\tWritten\n@@ -0,0 +1,33 @@\n+{\n+ \"version\": \"deltawire.plan.v1\",\n+ \"id\": \"range-large\",\n+ \"description\": \"500 ordered integer records\",\n+ \"record_schema\": \".deltawire/schemas/range-large.schema.json\",\n+ \"output\": {\n+ \"path\": \"testdata/generated/output.ndjson\",\n+ \"format\": \"ndjson\",\n+ \"pretty\": false\n+ },\n+ \"generators\": [\n+ {\n+ \"kind\": \"matrix\",\n+ \"name\": \"records\",\n+ \"dimensions\": [\n+ {\n+ \"name\": \"index\",\n+ \"range\": {\n+ \"start\": 1,\n+ \"end_exclusive\": 501,\n+ \"step\": 1\n+ }\n+ }\n+ ],\n+ \"record\": {\n+ \"index\": \"${index}\"\n+ }\n+ }\n+ ],\n+ \"assertions\": {\n+ \"count\": 500\n+ }\n+}\n","fileName":"plan.json","filePath":"/plan.json","originalContent":"","newContent":"{\n \"version\": \"deltawire.plan.v1\",\n \"id\": \"range-large\",\n \"description\": \"500 ordered integer records\",\n \"record_schema\": \".deltawire/schemas/range-large.schema.json\",\n \"output\": {\n \"path\": \"testdata/generated/output.ndjson\",\n \"format\": \"ndjson\",\n \"pretty\": false\n },\n \"generators\": [\n {\n \"kind\": \"matrix\",\n \"name\": \"records\",\n \"dimensions\": [\n {\n \"name\": \"index\",\n \"range\": {\n \"start\": 1,\n \"end_exclusive\": 501,\n \"step\": 1\n }\n }\n ],\n \"record\": {\n \"index\": \"${index}\"\n }\n }\n ],\n \"assertions\": {\n \"count\": 500\n }\n}\n","diffStat":{"model_added_lines":33,"model_removed_lines":0,"model_added_chars":620,"model_removed_chars":0,"user_added_lines":0,"user_removed_lines":0,"user_added_chars":0,"user_removed_chars":0},"isNewFile":true},"description":"Writing to plan.json","displayName":"WriteFile","renderOutputAsMarkdown":true},{"id":"list_directory__fs6c47lj","name":"list_directory","args":{"dir_path":"."},"result":[{"functionResponse":{"id":"list_directory__fs6c47lj","name":"list_directory","response":{"output":"Directory listing for /:\n[DIR] .deltawire\n[DIR] bin\n[DIR] boot\n[DIR] dev\n[DIR] etc\n[DIR] harbor\n[DIR] home\n[DIR] installed-agent\n[DIR] lib\n[DIR] lib64\n[DIR] logs\n[DIR] media\n[DIR] mnt\n[DIR] opt\n[DIR] proc\n[DIR] root\n[DIR] run\n[DIR] sbin\n[DIR] srv\n[DIR] sys\n[DIR] tmp\n[DIR] usr\n[DIR] var\n.dockerenv (0 bytes)\ndeltawire-environment-expectations.json (771 bytes)\nplan.json (653 bytes)\ntask-contract.json (576 bytes)"}}}],"status":"success","timestamp":"2026-07-22T19:57:34.719Z","resultDisplay":{"summary":"Found 27 item(s).","files":["[DIR] .deltawire","[DIR] bin","[DIR] boot","[DIR] dev","[DIR] etc","[DIR] harbor","[DIR] home","[DIR] installed-agent","[DIR] lib","[DIR] lib64","[DIR] logs","[DIR] media","[DIR] mnt","[DIR] opt","[DIR] proc","[DIR] root","[DIR] run","[DIR] sbin","[DIR] srv","[DIR] sys","[DIR] tmp","[DIR] usr","[DIR] var",".dockerenv","deltawire-environment-expectations.json","plan.json","task-contract.json"]},"description":".","displayName":"ReadFolder","renderOutputAsMarkdown":true}]} +{"id":"d3f6aa0c-5bc3-4507-a06d-a49ca7d94a1e","timestamp":"2026-07-22T19:57:34.720Z","type":"user","content":[{"functionResponse":{"id":"update_topic__bqa8f8l2","name":"update_topic","response":{"output":"Strategic Intent: Create the DeltaWire plan and run the required lifecycle commands to generate and validate the dataset."}}},{"functionResponse":{"id":"write_file__mr1cteab","name":"write_file","response":{"output":"Successfully created and wrote to new file: /plan.json. Here is the updated code:\n{\n \"version\": \"deltawire.plan.v1\",\n \"id\": \"range-large\",\n \"description\": \"500 ordered integer records\",\n \"record_schema\": \".deltawire/schemas/range-large.schema.json\",\n \"output\": {\n \"path\": \"testdata/generated/output.ndjson\",\n \"format\": \"ndjson\",\n \"pretty\": false\n },\n \"generators\": [\n {\n \"kind\": \"matrix\",\n \"name\": \"records\",\n \"dimensions\": [\n {\n \"name\": \"index\",\n \"range\": {\n \"start\": 1,\n \"end_exclusive\": 501,\n \"step\": 1\n }\n }\n ],\n \"record\": {\n \"index\": \"${index}\"\n }\n }\n ],\n \"assertions\": {\n \"count\": 500\n }\n}\n"}}},{"functionResponse":{"id":"list_directory__fs6c47lj","name":"list_directory","response":{"output":"Directory listing for /:\n[DIR] .deltawire\n[DIR] bin\n[DIR] boot\n[DIR] dev\n[DIR] etc\n[DIR] harbor\n[DIR] home\n[DIR] installed-agent\n[DIR] lib\n[DIR] lib64\n[DIR] logs\n[DIR] media\n[DIR] mnt\n[DIR] opt\n[DIR] proc\n[DIR] root\n[DIR] run\n[DIR] sbin\n[DIR] srv\n[DIR] sys\n[DIR] tmp\n[DIR] usr\n[DIR] var\n.dockerenv (0 bytes)\ndeltawire-environment-expectations.json (771 bytes)\nplan.json (653 bytes)\ntask-contract.json (576 bytes)"}}}]} +{"$set":{"lastUpdated":"2026-07-22T19:57:34.720Z"}} +{"id":"c5131ec2-b4fe-46d1-8a21-6ce95786a0c7","timestamp":"2026-07-22T19:57:38.793Z","type":"gemini","content":"","thoughts":[{"subject":"Locating Project Directory","description":"I've made a discovery: the task resides within the project's temporary directory, confirmed by the `session_context`. However, I found a discrepancy in the path information, so I need to verify by checking the working directory. I will list the actual project directory to confirm my current working directory.","timestamp":"2026-07-22T19:57:37.453Z"}],"tokens":{"input":15264,"output":92,"cached":12097,"thoughts":256,"tool":0,"total":15612},"model":"gemini-3.1-pro-preview"} +{"$set":{"lastUpdated":"2026-07-22T19:57:38.793Z"}} +{"id":"c5131ec2-b4fe-46d1-8a21-6ce95786a0c7","timestamp":"2026-07-22T19:57:38.793Z","type":"gemini","content":"","thoughts":[{"subject":"Locating Project Directory","description":"I've made a discovery: the task resides within the project's temporary directory, confirmed by the `session_context`. However, I found a discrepancy in the path information, so I need to verify by checking the working directory. I will list the actual project directory to confirm my current working directory.","timestamp":"2026-07-22T19:57:37.453Z"}],"tokens":{"input":15264,"output":92,"cached":12097,"thoughts":256,"tool":0,"total":15612},"model":"gemini-3.1-pro-preview","toolCalls":[{"id":"run_shell_command__vve0s0po","name":"run_shell_command","args":{"description":"Validate the DeltaWire plan","command":"deltawire validate --repo . plan.json"},"result":[{"functionResponse":{"id":"run_shell_command__vve0s0po","name":"run_shell_command","response":{"output":"\nOutput: Validated plan.json: 500 records, 6892 bytes\nProcess Group PGID: 190\n"}}}],"status":"success","timestamp":"2026-07-22T19:57:38.834Z","resultDisplay":"Validated plan.json: 500 records, 6892 bytes","description":"deltawire validate --repo . plan.json","displayName":"Shell","renderOutputAsMarkdown":false},{"id":"run_shell_command__l9vbn1ct","name":"run_shell_command","args":{"command":"find / -maxdepth 3 -name \"*contract*check*\" -o -name \"*check*contract*\" 2>/dev/null","description":"Search for contract checker script"},"result":[{"functionResponse":{"id":"run_shell_command__l9vbn1ct","name":"run_shell_command","response":{"output":"\nOutput: (empty)\nProcess Group PGID: 193\n"}}}],"status":"success","timestamp":"2026-07-22T19:57:38.834Z","resultDisplay":"","description":"find / -maxdepth 3 -name \"*contract*check*\" -o -name \"*check*contract*\" 2>/dev/null","displayName":"Shell","renderOutputAsMarkdown":false}]} +{"id":"c197be61-9c3b-4542-ab9e-30c462be617a","timestamp":"2026-07-22T19:57:38.835Z","type":"user","content":[{"functionResponse":{"id":"run_shell_command__vve0s0po","name":"run_shell_command","response":{"output":"\nOutput: Validated plan.json: 500 records, 6892 bytes\nProcess Group PGID: 190\n"}}},{"functionResponse":{"id":"run_shell_command__l9vbn1ct","name":"run_shell_command","response":{"output":"\nOutput: (empty)\nProcess Group PGID: 193\n"}}}]} +{"$set":{"lastUpdated":"2026-07-22T19:57:38.835Z"}} +{"id":"d1e9370f-bebe-4401-96c5-6cbbca833c3c","timestamp":"2026-07-22T19:57:41.554Z","type":"gemini","content":"","thoughts":[{"subject":"Refining Command Sequences","description":"I've got the validation to be working! Now, I am refining the command sequences to be run. I am thinking it is best to use a more specific sequence of commands, `deltawire validate`, `deltawire render`, and then `deltawire check`. I'm starting to think the independent checker will handle general checks.","timestamp":"2026-07-22T19:57:41.177Z"}],"tokens":{"input":15713,"output":46,"cached":12089,"thoughts":112,"tool":0,"total":15871},"model":"gemini-3.1-pro-preview"} +{"$set":{"lastUpdated":"2026-07-22T19:57:41.554Z"}} +{"id":"d1e9370f-bebe-4401-96c5-6cbbca833c3c","timestamp":"2026-07-22T19:57:41.554Z","type":"gemini","content":"","thoughts":[{"subject":"Refining Command Sequences","description":"I've got the validation to be working! Now, I am refining the command sequences to be run. I am thinking it is best to use a more specific sequence of commands, `deltawire validate`, `deltawire render`, and then `deltawire check`. I'm starting to think the independent checker will handle general checks.","timestamp":"2026-07-22T19:57:41.177Z"}],"tokens":{"input":15713,"output":46,"cached":12089,"thoughts":112,"tool":0,"total":15871},"model":"gemini-3.1-pro-preview","toolCalls":[{"id":"run_shell_command__auimhbi6","name":"run_shell_command","args":{"command":"ls -l /usr/local/bin | grep check","description":"Look for a contract checker in /usr/local/bin"},"result":[{"functionResponse":{"id":"run_shell_command__auimhbi6","name":"run_shell_command","response":{"output":"\nOutput: (empty)\nExit Code: 1\nProcess Group PGID: 216\n"}}}],"status":"success","timestamp":"2026-07-22T19:57:41.568Z","resultDisplay":"Command exited with code: 1","description":"ls -l /usr/local/bin | grep check","displayName":"Shell","renderOutputAsMarkdown":false}]} +{"id":"a90ec30a-969f-47ae-b317-dfa152ad1149","timestamp":"2026-07-22T19:57:41.568Z","type":"user","content":[{"functionResponse":{"id":"run_shell_command__auimhbi6","name":"run_shell_command","response":{"output":"\nOutput: (empty)\nExit Code: 1\nProcess Group PGID: 216\n"}}}]} +{"$set":{"lastUpdated":"2026-07-22T19:57:41.568Z"}} +{"id":"0eafbac6-e40b-4cf0-9f4c-d8e89dbc43fc","timestamp":"2026-07-22T19:57:43.510Z","type":"gemini","content":"","thoughts":[],"tokens":{"input":15921,"output":38,"cached":12084,"thoughts":15,"tool":0,"total":15974},"model":"gemini-3.1-pro-preview"} +{"$set":{"lastUpdated":"2026-07-22T19:57:43.510Z"}} +{"id":"0eafbac6-e40b-4cf0-9f4c-d8e89dbc43fc","timestamp":"2026-07-22T19:57:43.510Z","type":"gemini","content":"","thoughts":[],"tokens":{"input":15921,"output":38,"cached":12084,"thoughts":15,"tool":0,"total":15974},"model":"gemini-3.1-pro-preview","toolCalls":[{"id":"run_shell_command__hq6ookf4","name":"run_shell_command","args":{"description":"List /usr/local/bin","command":"ls -l /usr/local/bin"},"result":[{"functionResponse":{"id":"run_shell_command__hq6ookf4","name":"run_shell_command","response":{"output":"\nOutput: total 127252\nlrwxrwxrwx 1 root root 45 Jun 23 16:17 corepack -> ../lib/node_modules/corepack/dist/corepack.js\n-rwxr-xr-x 1 root root 5451215 Jul 22 16:36 deltawire\n-rwxr-xr-x 1 root root 388 Jul 14 01:48 docker-entrypoint.sh\n-rwxr-xr-x 1 root root 3167 Jul 22 16:36 environment_receipt.py\nlrwxrwxrwx 1 root root 55 Jul 22 16:37 gemini -> ../lib/node_modules/@google/gemini-cli/bundle/gemini.js\n-rwxr-xr-x 1 root root 124835376 Jun 23 16:17 node\nlrwxrwxrwx 1 root root 19 Jul 14 01:48 nodejs -> /usr/local/bin/node\nlrwxrwxrwx 1 root root 38 Jun 23 16:17 npm -> ../lib/node_modules/npm/bin/npm-cli.js\nlrwxrwxrwx 1 root root 38 Jun 23 16:17 npx -> ../lib/node_modules/npm/bin/npx-cli.js\n-rwxr-xr-x 1 root root 2284 Jul 22 16:36 verify_plan_contract.py\nlrwxrwxrwx 1 root root 27 Jul 14 01:48 yarn -> /opt/yarn-v1.22.22/bin/yarn\nlrwxrwxrwx 1 root root 30 Jul 14 01:48 yarnpkg -> /opt/yarn-v1.22.22/bin/yarnpkg\nProcess Group PGID: 220\n"}}}],"status":"success","timestamp":"2026-07-22T19:57:43.523Z","resultDisplay":"total 127252\nlrwxrwxrwx 1 root root 45 Jun 23 16:17 corepack -> ../lib/node_modules/corepack/dist/corepack.js\n-rwxr-xr-x 1 root root 5451215 Jul 22 16:36 deltawire\n-rwxr-xr-x 1 root root 388 Jul 14 01:48 docker-entrypoint.sh\n-rwxr-xr-x 1 root root 3167 Jul 22 16:36 environment_receipt.py\nlrwxrwxrwx 1 root root 55 Jul 22 16:37 gemini -> ../lib/node_modules/@google/gemini-cli/bundle/gemini.js\n-rwxr-xr-x 1 root root 124835376 Jun 23 16:17 node\nlrwxrwxrwx 1 root root 19 Jul 14 01:48 nodejs -> /usr/local/bin/node\nlrwxrwxrwx 1 root root 38 Jun 23 16:17 npm -> ../lib/node_modules/npm/bin/npm-cli.js\nlrwxrwxrwx 1 root root 38 Jun 23 16:17 npx -> ../lib/node_modules/npm/bin/npx-cli.js\n-rwxr-xr-x 1 root root 2284 Jul 22 16:36 verify_plan_contract.py\nlrwxrwxrwx 1 root root 27 Jul 14 01:48 yarn -> /opt/yarn-v1.22.22/bin/yarn\nlrwxrwxrwx 1 root root 30 Jul 14 01:48 yarnpkg -> /opt/yarn-v1.22.22/bin/yarnpkg","description":"ls -l /usr/local/bin","displayName":"Shell","renderOutputAsMarkdown":false}]} +{"id":"1c95c096-7e52-48d5-833c-d821e0f78727","timestamp":"2026-07-22T19:57:43.524Z","type":"user","content":[{"functionResponse":{"id":"run_shell_command__hq6ookf4","name":"run_shell_command","response":{"output":"\nOutput: total 127252\nlrwxrwxrwx 1 root root 45 Jun 23 16:17 corepack -> ../lib/node_modules/corepack/dist/corepack.js\n-rwxr-xr-x 1 root root 5451215 Jul 22 16:36 deltawire\n-rwxr-xr-x 1 root root 388 Jul 14 01:48 docker-entrypoint.sh\n-rwxr-xr-x 1 root root 3167 Jul 22 16:36 environment_receipt.py\nlrwxrwxrwx 1 root root 55 Jul 22 16:37 gemini -> ../lib/node_modules/@google/gemini-cli/bundle/gemini.js\n-rwxr-xr-x 1 root root 124835376 Jun 23 16:17 node\nlrwxrwxrwx 1 root root 19 Jul 14 01:48 nodejs -> /usr/local/bin/node\nlrwxrwxrwx 1 root root 38 Jun 23 16:17 npm -> ../lib/node_modules/npm/bin/npm-cli.js\nlrwxrwxrwx 1 root root 38 Jun 23 16:17 npx -> ../lib/node_modules/npm/bin/npx-cli.js\n-rwxr-xr-x 1 root root 2284 Jul 22 16:36 verify_plan_contract.py\nlrwxrwxrwx 1 root root 27 Jul 14 01:48 yarn -> /opt/yarn-v1.22.22/bin/yarn\nlrwxrwxrwx 1 root root 30 Jul 14 01:48 yarnpkg -> /opt/yarn-v1.22.22/bin/yarnpkg\nProcess Group PGID: 220\n"}}}]} +{"$set":{"lastUpdated":"2026-07-22T19:57:43.524Z"}} +{"id":"05ae1fb0-241e-4873-a073-41b389f1158a","timestamp":"2026-07-22T19:57:45.894Z","type":"gemini","content":"","thoughts":[],"tokens":{"input":16471,"output":37,"cached":12084,"thoughts":31,"tool":0,"total":16539},"model":"gemini-3.1-pro-preview"} +{"$set":{"lastUpdated":"2026-07-22T19:57:45.894Z"}} +{"id":"05ae1fb0-241e-4873-a073-41b389f1158a","timestamp":"2026-07-22T19:57:45.894Z","type":"gemini","content":"","thoughts":[],"tokens":{"input":16471,"output":37,"cached":12084,"thoughts":31,"tool":0,"total":16539},"model":"gemini-3.1-pro-preview","toolCalls":[{"id":"run_shell_command__mhxmzlct","name":"run_shell_command","args":{"command":"verify_plan_contract.py plan.json","description":"Run the independent contract checker"},"result":[{"functionResponse":{"id":"run_shell_command__mhxmzlct","name":"run_shell_command","response":{"output":"\nOutput: usage: verify_plan_contract.py [-h] --contract CONTRACT --plan PLAN\n [--repo REPO] [--receipt RECEIPT]\n [--deltawire DELTAWIRE]\nverify_plan_contract.py: error: the following arguments are required: --contract, --plan\nExit Code: 2\nProcess Group PGID: 223\n"}}}],"status":"success","timestamp":"2026-07-22T19:57:45.943Z","resultDisplay":"usage: verify_plan_contract.py [-h] --contract CONTRACT --plan PLAN\n [--repo REPO] [--receipt RECEIPT]\n [--deltawire DELTAWIRE]\nverify_plan_contract.py: error: the following arguments are required: --contract, --plan","description":"verify_plan_contract.py plan.json","displayName":"Shell","renderOutputAsMarkdown":false}]} +{"id":"ae3e9240-ab21-4dce-b379-754504487977","timestamp":"2026-07-22T19:57:45.943Z","type":"user","content":[{"functionResponse":{"id":"run_shell_command__mhxmzlct","name":"run_shell_command","response":{"output":"\nOutput: usage: verify_plan_contract.py [-h] --contract CONTRACT --plan PLAN\n [--repo REPO] [--receipt RECEIPT]\n [--deltawire DELTAWIRE]\nverify_plan_contract.py: error: the following arguments are required: --contract, --plan\nExit Code: 2\nProcess Group PGID: 223\n"}}}]} +{"$set":{"lastUpdated":"2026-07-22T19:57:45.943Z"}} +{"id":"c25f138e-f8b2-4fd5-8b3c-05a1c397cb0b","timestamp":"2026-07-22T19:57:47.992Z","type":"gemini","content":"","thoughts":[],"tokens":{"input":16649,"output":49,"cached":12079,"thoughts":19,"tool":0,"total":16717},"model":"gemini-3.1-pro-preview"} +{"$set":{"lastUpdated":"2026-07-22T19:57:47.992Z"}} +{"id":"c25f138e-f8b2-4fd5-8b3c-05a1c397cb0b","timestamp":"2026-07-22T19:57:47.992Z","type":"gemini","content":"","thoughts":[],"tokens":{"input":16649,"output":49,"cached":12079,"thoughts":19,"tool":0,"total":16717},"model":"gemini-3.1-pro-preview","toolCalls":[{"id":"run_shell_command__c153tt8e","name":"run_shell_command","args":{"command":"verify_plan_contract.py --contract task-contract.json --plan plan.json","description":"Run the independent contract checker with required arguments"},"result":[{"functionResponse":{"id":"run_shell_command__c153tt8e","name":"run_shell_command","response":{"output":"\nOutput: {\"checks\": {\"exact_count_assertion\": true, \"output_format\": true, \"output_path\": true, \"projected_records\": true, \"schema_exists\": true, \"schema_path\": true, \"schema_sha256\": true}, \"contract_sha256\": \"d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5\", \"inspect\": {\"assertion_summary\": [\"exact count 500\"], \"cmd_check\": \"deltawire check plan.json\", \"cmd_render\": \"deltawire render plan.json\", \"cmd_validate\": \"deltawire validate plan.json\", \"cold_amp_ratio\": \"6892/939\", \"output_format\": \"ndjson\", \"output_path\": \"testdata/generated/output.ndjson\", \"plan_id\": \"range-large\", \"plan_only_amp_ratio\": \"6892/653\", \"plan_path\": \"plan.json\", \"plan_source_bytes\": 653, \"primitives_used\": [\"matrix\", \"ranges\"], \"projected_output_bytes\": 6892, \"projected_records\": 500, \"schema_path\": \".deltawire/schemas/range-large.schema.json\", \"schema_source_bytes\": 286}, \"inspect_exit_code\": 0, \"plan_sha256\": \"3a32efdad6691d5a8e6bab88131e2f5bf038e6d4d48a78318b6f1e2840b1c919\", \"schema_version\": \"plan-contract-receipt.v1\", \"status\": \"pass\"}\nProcess Group PGID: 226\n"}}}],"status":"success","timestamp":"2026-07-22T19:57:48.054Z","resultDisplay":"{\"checks\": {\"exact_count_assertion\": true, \"output_format\": true, \"output_path\": true, \"projected_records\": true, \"schema_exists\": true, \"schema_path\": true, \"schema_sha256\": true}, \"contract_sha256\": \"d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5\", \"inspect\": {\"assertion_summary\": [\"exact count 500\"], \"cmd_check\": \"deltawire check plan.json\", \"cmd_render\": \"deltawire render plan.json\", \"cmd_validate\": \"deltawire validate plan.json\", \"cold_amp_ratio\": \"6892/939\", \"output_format\": \"ndjson\", \"output_path\": \"testdata/generated/output.ndjson\", \"plan_id\": \"range-large\", \"plan_only_amp_ratio\": \"6892/653\", \"plan_path\": \"plan.json\", \"plan_source_bytes\": 653, \"primitives_used\": [\"matrix\", \"ranges\"], \"projected_output_bytes\": 6892, \"projected_records\": 500, \"schema_path\": \".deltawire/schemas/range-large.schema.json\", \"schema_source_bytes\": 286}, \"inspect_exit_code\": 0, \"plan_sha256\": \"3a32efdad6691d5a8e6bab88131e2f5bf038e6d4d48a78318b6f1e2840b1c919\", \"schema_version\": \"plan-contract-receipt.v1\", \"status\": \"pass\"}","description":"verify_plan_contract.py --contract task-contract.json --plan plan.json","displayName":"Shell","renderOutputAsMarkdown":false}]} +{"id":"cd0ffd9d-41fe-4506-93db-db220cd4cbc3","timestamp":"2026-07-22T19:57:48.054Z","type":"user","content":[{"functionResponse":{"id":"run_shell_command__c153tt8e","name":"run_shell_command","response":{"output":"\nOutput: {\"checks\": {\"exact_count_assertion\": true, \"output_format\": true, \"output_path\": true, \"projected_records\": true, \"schema_exists\": true, \"schema_path\": true, \"schema_sha256\": true}, \"contract_sha256\": \"d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5\", \"inspect\": {\"assertion_summary\": [\"exact count 500\"], \"cmd_check\": \"deltawire check plan.json\", \"cmd_render\": \"deltawire render plan.json\", \"cmd_validate\": \"deltawire validate plan.json\", \"cold_amp_ratio\": \"6892/939\", \"output_format\": \"ndjson\", \"output_path\": \"testdata/generated/output.ndjson\", \"plan_id\": \"range-large\", \"plan_only_amp_ratio\": \"6892/653\", \"plan_path\": \"plan.json\", \"plan_source_bytes\": 653, \"primitives_used\": [\"matrix\", \"ranges\"], \"projected_output_bytes\": 6892, \"projected_records\": 500, \"schema_path\": \".deltawire/schemas/range-large.schema.json\", \"schema_source_bytes\": 286}, \"inspect_exit_code\": 0, \"plan_sha256\": \"3a32efdad6691d5a8e6bab88131e2f5bf038e6d4d48a78318b6f1e2840b1c919\", \"schema_version\": \"plan-contract-receipt.v1\", \"status\": \"pass\"}\nProcess Group PGID: 226\n"}}}]} +{"$set":{"lastUpdated":"2026-07-22T19:57:48.054Z"}} +{"id":"5cb0b5a4-f102-4602-8d63-2d9c61c45720","timestamp":"2026-07-22T19:57:50.482Z","type":"gemini","content":"","thoughts":[],"tokens":{"input":17205,"output":76,"cached":12078,"thoughts":49,"tool":0,"total":17330},"model":"gemini-3.1-pro-preview"} +{"$set":{"lastUpdated":"2026-07-22T19:57:50.482Z"}} +{"id":"5cb0b5a4-f102-4602-8d63-2d9c61c45720","timestamp":"2026-07-22T19:57:50.482Z","type":"gemini","content":"","thoughts":[],"tokens":{"input":17205,"output":76,"cached":12078,"thoughts":49,"tool":0,"total":17330},"model":"gemini-3.1-pro-preview","toolCalls":[{"id":"run_shell_command__tfqwy76e","name":"run_shell_command","args":{"command":"deltawire render --repo . plan.json","description":"Render the dataset"},"result":[{"functionResponse":{"id":"run_shell_command__tfqwy76e","name":"run_shell_command","response":{"output":"\nOutput: Rendered /testdata/generated/output.ndjson\nProcess Group PGID: 249\n"}}}],"status":"success","timestamp":"2026-07-22T19:57:50.536Z","resultDisplay":"Rendered /testdata/generated/output.ndjson","description":"deltawire render --repo . plan.json","displayName":"Shell","renderOutputAsMarkdown":false},{"id":"run_shell_command__wtnoafp3","name":"run_shell_command","args":{"command":"deltawire check --repo . plan.json","wait_for_previous":true,"description":"Check the dataset"},"result":[{"functionResponse":{"id":"run_shell_command__wtnoafp3","name":"run_shell_command","response":{"output":"\nOutput: Check ok plan.json\nProcess Group PGID: 271\n"}}}],"status":"success","timestamp":"2026-07-22T19:57:50.536Z","resultDisplay":"Check ok plan.json","description":"deltawire check --repo . plan.json","displayName":"Shell","renderOutputAsMarkdown":false}]} +{"id":"59109a1b-a962-4557-b629-ba828d5c8119","timestamp":"2026-07-22T19:57:50.536Z","type":"user","content":[{"functionResponse":{"id":"run_shell_command__tfqwy76e","name":"run_shell_command","response":{"output":"\nOutput: Rendered /testdata/generated/output.ndjson\nProcess Group PGID: 249\n"}}},{"functionResponse":{"id":"run_shell_command__wtnoafp3","name":"run_shell_command","response":{"output":"\nOutput: Check ok plan.json\nProcess Group PGID: 271\n"}}}]} +{"$set":{"lastUpdated":"2026-07-22T19:57:50.537Z"}} +{"id":"ab4edc74-ad59-4028-9005-5db73db20f36","timestamp":"2026-07-22T19:57:54.312Z","type":"gemini","content":"","thoughts":[{"subject":"Verifying Final Steps","description":"I've confirmed the render and initial checks were successful. Now, I'm focusing on the final steps. I need to run `deltawire validate` and the independent contract checker, followed by `deltawire render`. It appears those commands form the complete lifecycle needed to complete this task.","timestamp":"2026-07-22T19:57:53.497Z"}],"tokens":{"input":17427,"output":129,"cached":12069,"thoughts":134,"tool":0,"total":17690},"model":"gemini-3.1-pro-preview"} +{"$set":{"lastUpdated":"2026-07-22T19:57:54.312Z"}} +{"id":"ab4edc74-ad59-4028-9005-5db73db20f36","timestamp":"2026-07-22T19:57:54.312Z","type":"gemini","content":"","thoughts":[{"subject":"Verifying Final Steps","description":"I've confirmed the render and initial checks were successful. Now, I'm focusing on the final steps. I need to run `deltawire validate` and the independent contract checker, followed by `deltawire render`. It appears those commands form the complete lifecycle needed to complete this task.","timestamp":"2026-07-22T19:57:53.497Z"}],"tokens":{"input":17427,"output":129,"cached":12069,"thoughts":134,"tool":0,"total":17690},"model":"gemini-3.1-pro-preview","toolCalls":[{"id":"update_topic__x2x6k9dc","name":"update_topic","args":{"summary":"I have successfully written the plan and ran all the required commands: `deltawire validate`, the independent `verify_plan_contract.py` contract checker, `deltawire render` to generate the NDJSON, and `deltawire check` to assert output validity. All steps passed, and the `testdata/generated/output.ndjson` file was generated properly.","title":"Generating Range-Large Dataset","strategic_intent":"All lifecycle steps are successfully completed and artifacts are generated and retained.","wait_for_previous":false},"result":[{"functionResponse":{"id":"update_topic__x2x6k9dc","name":"update_topic","response":{"output":"Strategic Intent: All lifecycle steps are successfully completed and artifacts are generated and retained."}}}],"status":"success","timestamp":"2026-07-22T19:57:54.314Z","resultDisplay":"> [!STRATEGY]\n> **Intent:** All lifecycle steps are successfully completed and artifacts are generated and retained.","description":"Update topic to: \"Generating Range-Large Dataset\"","displayName":"Update Topic Context","renderOutputAsMarkdown":true}]} +{"id":"3d80c7b2-ca24-484a-a63a-d2738ba41a6f","timestamp":"2026-07-22T19:57:54.315Z","type":"user","content":[{"functionResponse":{"id":"update_topic__x2x6k9dc","name":"update_topic","response":{"output":"Strategic Intent: All lifecycle steps are successfully completed and artifacts are generated and retained."}}}]} +{"$set":{"lastUpdated":"2026-07-22T19:57:54.315Z"}} +{"id":"1b77ebfc-af61-4f75-8b05-1b6bac30fcdb","timestamp":"2026-07-22T19:57:57.044Z","type":"gemini","content":"I have successfully generated the `range-large` dataset. \n\nI created a deterministic DeltaWire plan (`plan.json`) adhering to the rules described in `/task-contract.json` and followed the mandated lifecycle steps:\n1. Validated the plan with `deltawire validate`\n2. Passed the contract checks via `verify_plan_contract.py`\n3. Rendered exactly 500 ordered records to `testdata/generated/output.ndjson`\n4. Asserted the dataset integrity with `deltawire check`\n\nThe artifact is now retained and ready in your workspace.","thoughts":[],"tokens":{"input":17719,"output":126,"cached":12066,"thoughts":10,"tool":0,"total":17855},"model":"gemini-3.1-pro-preview"} +{"$set":{"lastUpdated":"2026-07-22T19:57:57.044Z"}} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/gemini-cli.txt b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/gemini-cli.txt new file mode 100644 index 000000000..1133ff729 --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/gemini-cli.txt @@ -0,0 +1,14 @@ +Warning: You are running Gemini CLI in the root directory. Your entire folder structure will be used for context. It is strongly recommended to run in a project-specific directory. +Warning: 256-color support not detected. Using a terminal with at least 256-color support is recommended for a better visual experience. +YOLO mode is enabled. All tool calls will be automatically approved. +YOLO mode is enabled. All tool calls will be automatically approved. +Ripgrep is not available. Falling back to GrepTool. +I have successfully generated the `range-large` dataset. + +I created a deterministic DeltaWire plan (`plan.json`) adhering to the rules described in `/task-contract.json` and followed the mandated lifecycle steps: +1. Validated the plan with `deltawire validate` +2. Passed the contract checks via `verify_plan_contract.py` +3. Rendered exactly 500 ordered records to `testdata/generated/output.ndjson` +4. Asserted the dataset integrity with `deltawire check` + +The artifact is now retained and ready in your workspace. diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/trajectory.json b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/trajectory.json new file mode 100644 index 000000000..8d4d3abeb --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/trajectory.json @@ -0,0 +1,556 @@ +{ + "schema_version": "ATIF-v1.7", + "session_id": "e8bc4290-e6c1-479c-88ef-e3a1d02d4477", + "agent": { + "name": "gemini-cli", + "version": "0.51.0", + "model_name": "gemini-3.1-pro-preview" + }, + "steps": [ + { + "step_id": 1, + "timestamp": "2026-07-22T19:57:16.294Z", + "source": "user", + "message": "Generate the exact `range-large` dataset defined by public `/task-contract.json`. Write exactly 500 JSON records in canonical order as NDJSON to `testdata/generated/output.ndjson`. Every record, field, value, and ordering rule must match this complete embedded contract:\n\n```json\n{\n \"authoritative_schema\": {\n \"path\": \".deltawire/schemas/range-large.schema.json\",\n \"sha256\": \"2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5\"\n },\n \"deltawire_applicability\": \"expected\",\n \"family\": \"range\",\n \"generation\": {\n \"field\": \"index\",\n \"field_order\": [\n \"index\"\n ],\n \"start\": 1\n },\n \"ordering\": \"canonical_generation_order\",\n \"output\": {\n \"format\": \"ndjson\",\n \"path\": \"testdata/generated/output.ndjson\"\n },\n \"record_count\": 500,\n \"schema_version\": \"task-spec.v1\",\n \"size\": \"large\",\n \"task\": \"range-large\"\n}\n```\n" + }, + { + "step_id": 2, + "timestamp": "2026-07-22T19:57:20.180Z", + "source": "agent", + "model_name": "gemini-3.1-pro-preview", + "message": "", + "reasoning_content": "Generating Test Data: I'm currently focused on creating a dataset per the specified contract for the 'range-large' task. My goal is to produce precisely 500 NDJSON records to `testdata/generated/output.ndjson`. The contract details are guiding this generation process.", + "tool_calls": [ + { + "tool_call_id": "update_topic__1k3g0da4", + "function_name": "update_topic", + "arguments": { + "strategic_intent": "Activate the deltawire skill and read the authoritative schema to understand the dataset generation rules.", + "summary": "I am activating the deltawire skill to generate a deterministic JSON dataset. I will read the schema provided in the contract to understand the fields and rules required to generate exactly 500 records in NDJSON format.", + "title": "Generating Range-Large Dataset", + "wait_for_previous": false + } + }, + { + "tool_call_id": "activate_skill__42osg4cn", + "function_name": "activate_skill", + "arguments": { + "name": "deltawire" + } + }, + { + "tool_call_id": "read_file__hwsp96t0", + "function_name": "read_file", + "arguments": { + "file_path": "/.deltawire/schemas/range-large.schema.json" + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "update_topic__1k3g0da4", + "content": "Current topic: \"Generating Range-Large Dataset\"\nTopic summary: I am activating the deltawire skill to generate a deterministic JSON dataset. I will read the schema provided in the contract to understand the fields and rules required to generate exactly 500 records in NDJSON format.\n\nStrategic Intent: Activate the deltawire skill and read the authoritative schema to understand the dataset generation rules." + }, + { + "source_call_id": "activate_skill__42osg4cn", + "content": "\n \n # DeltaWire\n\nRead the public task contract first. Preserve its output path, format, schema\npath and hash, record count, field set, values, and ordering.\n\nRead `references/plan-v1.md` and the matching bundled example before creating a\nplan. Use `examples/range.dw.json` for integer sequences,\n`examples/matrix.dw.json` for Cartesian products, `examples/rows.dw.json` for\nexplicit repeated rows, or `examples/variants.dw.json` for base records with\nsmall changes.\n\nCreate only the requested plan. Do not edit the installed config or schema, do\nnot generate the expanded output manually, and do not inspect or reverse\nengineer the DeltaWire binary. Resolve validation errors by comparing the plan\nwith the reference and example instead of guessing field names.\n\nRun this lifecycle in order:\n\n1. `deltawire validate --repo . `\n2. Run the installed independent contract checker for the public contract.\n3. `deltawire render --repo . `\n4. `deltawire check --repo . `\n\nStop after all four steps pass and the required artifacts are retained.\n \n\n \n Showing up to 200 items (files + folders).\n\n/root/.gemini/skills/deltawire/\n\u251c\u2500\u2500\u2500SKILL.md\n\u251c\u2500\u2500\u2500examples/\n\u2502 \u251c\u2500\u2500\u2500matrix.dw.json\n\u2502 \u251c\u2500\u2500\u2500range.dw.json\n\u2502 \u251c\u2500\u2500\u2500rows.dw.json\n\u2502 \u2514\u2500\u2500\u2500variants.dw.json\n\u2514\u2500\u2500\u2500references/\n \u2514\u2500\u2500\u2500plan-v1.md\n \n" + }, + { + "source_call_id": "read_file__hwsp96t0", + "content": "{\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"additionalProperties\": false,\n \"properties\": {\n \"index\": {\n \"maximum\": 500,\n \"minimum\": 1,\n \"type\": \"integer\"\n }\n },\n \"required\": [\n \"index\"\n ],\n \"title\": \"range-large\",\n \"type\": \"object\"\n}\n" + } + ] + }, + "metrics": { + "prompt_tokens": 11677, + "completion_tokens": 311, + "cached_tokens": 0, + "extra": { + "thoughts_tokens": 164, + "tool_tokens": 0 + } + }, + "llm_call_count": 1 + }, + { + "step_id": 3, + "timestamp": "2026-07-22T19:57:20.197Z", + "source": "user", + "message": "\n\n" + }, + { + "step_id": 4, + "timestamp": "2026-07-22T19:57:25.045Z", + "source": "agent", + "model_name": "gemini-3.1-pro-preview", + "message": "", + "reasoning_content": "Reviewing Plan and Examples: I'm currently reviewing the initial plan document and the provided examples. I'm focusing on how they structure integer sequences, Cartesian products, and explicit row repetition. This is to ensure my plan adheres to the established patterns. I'm leveraging the `examples` directory for patterns.", + "tool_calls": [ + { + "tool_call_id": "update_topic__klcvc3mx", + "function_name": "update_topic", + "arguments": { + "title": "Generating Range-Large Dataset", + "summary": "I have read the schema for the `range-large` task, which consists of a single `index` integer property. I will now read the DeltaWire reference documentation and the `range.dw.json` example to learn how to construct a valid DeltaWire plan for this dataset.", + "wait_for_previous": false, + "strategic_intent": "Read the DeltaWire plan reference and range example to understand how to write a DeltaWire plan for generating this dataset." + } + }, + { + "tool_call_id": "read_file__untyuu15", + "function_name": "read_file", + "arguments": { + "file_path": "/root/.gemini/skills/deltawire/references/plan-v1.md" + } + }, + { + "tool_call_id": "read_file__3od2weo9", + "function_name": "read_file", + "arguments": { + "file_path": "/root/.gemini/skills/deltawire/examples/range.dw.json" + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "update_topic__klcvc3mx", + "content": "Strategic Intent: Read the DeltaWire plan reference and range example to understand how to write a DeltaWire plan for generating this dataset." + }, + { + "source_call_id": "read_file__untyuu15", + "content": "# DeltaWire plan v1\n\nA plan is strict JSON. Unknown fields fail decoding.\n\n## Plan fields\n\n- `version`: exactly `deltawire.plan.v1`.\n- `id`, `description`: descriptive strings.\n- `record_schema`: repository-relative JSON Schema path used by `check`.\n- `output.path`: repository-relative output path.\n- `output.format`: `json` or `ndjson`.\n- `output.pretty`: JSON pretty-printing flag.\n- `defaults`: optional object deep-merged into each record.\n- `sets`: optional named arrays of scalar dimension values.\n- `generators`: ordered array of `matrix`, `rows`, or `variants` generators.\n- `assertions.count`: exact projected record count.\n- `assertions.unique`: optional JSON Pointer paths that must be unique.\n- `assertions.coverage`: optional JSON Pointer/value coverage checks.\n\n## Matrix\n\nA matrix generator has `kind`, `name`, `dimensions`, and `record`. Each\ndimension has a `name` plus exactly one of `values`, `set`, or `range`. An\ninline integer range uses `start`, `end_exclusive`, and nonzero `step`.\n\nReference dimensions from the record with `${name}`. When a string is exactly\none placeholder, the generated scalar retains its original type; for example,\nan integer range produces JSON integers. Embedded placeholders stringify their\nscalar values. Every dimension must be referenced by the record.\n\n## Rows\n\nA rows generator has `kind`, `name`, `columns`, and `rows`. Each column is a\nJSON Pointer such as `/role`; every row must have exactly one value per column.\nRows begin with plan defaults, then column values are assigned in order.\n\n## Variants\n\nA variants generator has `kind`, `name`, `base`, and `variants`. Each variant\nhas a unique `name`, optional `set` JSON Pointer/value assignments, and optional\n`omit` JSON Pointer paths. Records are built by deep-merging plan defaults and\nthe generator base, then applying the variant changes.\n\nGenerators emit records in plan order. Matrix dimensions vary from the last\ndimension inward, preserving each values/range order.\n" + }, + { + "source_call_id": "read_file__3od2weo9", + "content": "{\n \"version\": \"deltawire.plan.v1\",\n \"id\": \"range-example\",\n \"description\": \"Three ordered integer records\",\n \"record_schema\": \".deltawire/schemas/example.schema.json\",\n \"output\": {\"path\": \"testdata/generated/range.ndjson\", \"format\": \"ndjson\", \"pretty\": false},\n \"generators\": [{\n \"kind\": \"matrix\",\n \"name\": \"records\",\n \"dimensions\": [{\"name\": \"index\", \"range\": {\"start\": 1, \"end_exclusive\": 4, \"step\": 1}}],\n \"record\": {\"index\": \"${index}\"}\n }],\n \"assertions\": {\"count\": 3}\n}\n" + } + ] + }, + "metrics": { + "prompt_tokens": 12600, + "completion_tokens": 329, + "cached_tokens": 8071, + "extra": { + "thoughts_tokens": 137, + "tool_tokens": 0 + } + }, + "llm_call_count": 1 + }, + { + "step_id": 5, + "timestamp": "2026-07-22T19:57:25.056Z", + "source": "user", + "message": "\n\n" + }, + { + "step_id": 6, + "timestamp": "2026-07-22T19:57:34.705Z", + "source": "agent", + "model_name": "gemini-3.1-pro-preview", + "message": "", + "reasoning_content": "Defining Output Location: I've nailed down the output file path as `testdata/generated/output.ndjson`. This aligns directly with the specifications. Also, I have confirmed that the schema for the records will be `.deltawire/schemas/range-large.schema.json`, and the output format should be ndjson as required.\nFormulating Execution Plan: I'm now sketching out the execution plan, and have decided to name the file `plan.json`. Based on the steps, I'll place it as `.deltawire/plan.json`. This seems correct, as the step specifies validating from the repository's root directory. The plan will include `generators` for the data generation and `assertions.count` to validate the record count as 500, with `start: 1, end_exclusive: 501, step: 1`. I need to review and check the contract and schema.", + "tool_calls": [ + { + "tool_call_id": "update_topic__bqa8f8l2", + "function_name": "update_topic", + "arguments": { + "wait_for_previous": false, + "title": "Generating Range-Large Dataset", + "strategic_intent": "Create the DeltaWire plan and run the required lifecycle commands to generate and validate the dataset.", + "summary": "I will write the `plan.json` containing the rules to generate the 500 records. After saving the plan, I will validate it using `deltawire validate`, run any available contract checkers in the project root, render the output with `deltawire render`, and finally check the generated data with `deltawire check`." + } + }, + { + "tool_call_id": "write_file__mr1cteab", + "function_name": "write_file", + "arguments": { + "file_path": "plan.json", + "content": "{\n \"version\": \"deltawire.plan.v1\",\n \"id\": \"range-large\",\n \"description\": \"500 ordered integer records\",\n \"record_schema\": \".deltawire/schemas/range-large.schema.json\",\n \"output\": {\n \"path\": \"testdata/generated/output.ndjson\",\n \"format\": \"ndjson\",\n \"pretty\": false\n },\n \"generators\": [\n {\n \"kind\": \"matrix\",\n \"name\": \"records\",\n \"dimensions\": [\n {\n \"name\": \"index\",\n \"range\": {\n \"start\": 1,\n \"end_exclusive\": 501,\n \"step\": 1\n }\n }\n ],\n \"record\": {\n \"index\": \"${index}\"\n }\n }\n ],\n \"assertions\": {\n \"count\": 500\n }\n}\n" + } + }, + { + "tool_call_id": "list_directory__fs6c47lj", + "function_name": "list_directory", + "arguments": { + "dir_path": "." + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "update_topic__bqa8f8l2", + "content": "Strategic Intent: Create the DeltaWire plan and run the required lifecycle commands to generate and validate the dataset." + }, + { + "source_call_id": "write_file__mr1cteab", + "content": "Successfully created and wrote to new file: /plan.json. Here is the updated code:\n{\n \"version\": \"deltawire.plan.v1\",\n \"id\": \"range-large\",\n \"description\": \"500 ordered integer records\",\n \"record_schema\": \".deltawire/schemas/range-large.schema.json\",\n \"output\": {\n \"path\": \"testdata/generated/output.ndjson\",\n \"format\": \"ndjson\",\n \"pretty\": false\n },\n \"generators\": [\n {\n \"kind\": \"matrix\",\n \"name\": \"records\",\n \"dimensions\": [\n {\n \"name\": \"index\",\n \"range\": {\n \"start\": 1,\n \"end_exclusive\": 501,\n \"step\": 1\n }\n }\n ],\n \"record\": {\n \"index\": \"${index}\"\n }\n }\n ],\n \"assertions\": {\n \"count\": 500\n }\n}\n" + }, + { + "source_call_id": "list_directory__fs6c47lj", + "content": "Directory listing for /:\n[DIR] .deltawire\n[DIR] bin\n[DIR] boot\n[DIR] dev\n[DIR] etc\n[DIR] harbor\n[DIR] home\n[DIR] installed-agent\n[DIR] lib\n[DIR] lib64\n[DIR] logs\n[DIR] media\n[DIR] mnt\n[DIR] opt\n[DIR] proc\n[DIR] root\n[DIR] run\n[DIR] sbin\n[DIR] srv\n[DIR] sys\n[DIR] tmp\n[DIR] usr\n[DIR] var\n.dockerenv (0 bytes)\ndeltawire-environment-expectations.json (771 bytes)\nplan.json (653 bytes)\ntask-contract.json (576 bytes)" + } + ] + }, + "metrics": { + "prompt_tokens": 13635, + "completion_tokens": 1151, + "cached_tokens": 8065, + "extra": { + "thoughts_tokens": 756, + "tool_tokens": 0 + } + }, + "llm_call_count": 1 + }, + { + "step_id": 7, + "timestamp": "2026-07-22T19:57:34.720Z", + "source": "user", + "message": "\n\n" + }, + { + "step_id": 8, + "timestamp": "2026-07-22T19:57:38.793Z", + "source": "agent", + "model_name": "gemini-3.1-pro-preview", + "message": "", + "reasoning_content": "Locating Project Directory: I've made a discovery: the task resides within the project's temporary directory, confirmed by the `session_context`. However, I found a discrepancy in the path information, so I need to verify by checking the working directory. I will list the actual project directory to confirm my current working directory.", + "tool_calls": [ + { + "tool_call_id": "run_shell_command__vve0s0po", + "function_name": "run_shell_command", + "arguments": { + "description": "Validate the DeltaWire plan", + "command": "deltawire validate --repo . plan.json" + } + }, + { + "tool_call_id": "run_shell_command__l9vbn1ct", + "function_name": "run_shell_command", + "arguments": { + "command": "find / -maxdepth 3 -name \"*contract*check*\" -o -name \"*check*contract*\" 2>/dev/null", + "description": "Search for contract checker script" + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "run_shell_command__vve0s0po", + "content": "\nOutput: Validated plan.json: 500 records, 6892 bytes\nProcess Group PGID: 190\n" + }, + { + "source_call_id": "run_shell_command__l9vbn1ct", + "content": "\nOutput: (empty)\nProcess Group PGID: 193\n" + } + ] + }, + "metrics": { + "prompt_tokens": 15264, + "completion_tokens": 348, + "cached_tokens": 12097, + "extra": { + "thoughts_tokens": 256, + "tool_tokens": 0 + } + }, + "llm_call_count": 1 + }, + { + "step_id": 9, + "timestamp": "2026-07-22T19:57:38.835Z", + "source": "user", + "message": "\n" + }, + { + "step_id": 10, + "timestamp": "2026-07-22T19:57:41.554Z", + "source": "agent", + "model_name": "gemini-3.1-pro-preview", + "message": "", + "reasoning_content": "Refining Command Sequences: I've got the validation to be working! Now, I am refining the command sequences to be run. I am thinking it is best to use a more specific sequence of commands, `deltawire validate`, `deltawire render`, and then `deltawire check`. I'm starting to think the independent checker will handle general checks.", + "tool_calls": [ + { + "tool_call_id": "run_shell_command__auimhbi6", + "function_name": "run_shell_command", + "arguments": { + "command": "ls -l /usr/local/bin | grep check", + "description": "Look for a contract checker in /usr/local/bin" + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "run_shell_command__auimhbi6", + "content": "\nOutput: (empty)\nExit Code: 1\nProcess Group PGID: 216\n" + } + ] + }, + "metrics": { + "prompt_tokens": 15713, + "completion_tokens": 158, + "cached_tokens": 12089, + "extra": { + "thoughts_tokens": 112, + "tool_tokens": 0 + } + }, + "llm_call_count": 1 + }, + { + "step_id": 11, + "timestamp": "2026-07-22T19:57:41.568Z", + "source": "user", + "message": "" + }, + { + "step_id": 12, + "timestamp": "2026-07-22T19:57:43.510Z", + "source": "agent", + "model_name": "gemini-3.1-pro-preview", + "message": "", + "tool_calls": [ + { + "tool_call_id": "run_shell_command__hq6ookf4", + "function_name": "run_shell_command", + "arguments": { + "description": "List /usr/local/bin", + "command": "ls -l /usr/local/bin" + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "run_shell_command__hq6ookf4", + "content": "\nOutput: total 127252\nlrwxrwxrwx 1 root root 45 Jun 23 16:17 corepack -> ../lib/node_modules/corepack/dist/corepack.js\n-rwxr-xr-x 1 root root 5451215 Jul 22 16:36 deltawire\n-rwxr-xr-x 1 root root 388 Jul 14 01:48 docker-entrypoint.sh\n-rwxr-xr-x 1 root root 3167 Jul 22 16:36 environment_receipt.py\nlrwxrwxrwx 1 root root 55 Jul 22 16:37 gemini -> ../lib/node_modules/@google/gemini-cli/bundle/gemini.js\n-rwxr-xr-x 1 root root 124835376 Jun 23 16:17 node\nlrwxrwxrwx 1 root root 19 Jul 14 01:48 nodejs -> /usr/local/bin/node\nlrwxrwxrwx 1 root root 38 Jun 23 16:17 npm -> ../lib/node_modules/npm/bin/npm-cli.js\nlrwxrwxrwx 1 root root 38 Jun 23 16:17 npx -> ../lib/node_modules/npm/bin/npx-cli.js\n-rwxr-xr-x 1 root root 2284 Jul 22 16:36 verify_plan_contract.py\nlrwxrwxrwx 1 root root 27 Jul 14 01:48 yarn -> /opt/yarn-v1.22.22/bin/yarn\nlrwxrwxrwx 1 root root 30 Jul 14 01:48 yarnpkg -> /opt/yarn-v1.22.22/bin/yarnpkg\nProcess Group PGID: 220\n" + } + ] + }, + "metrics": { + "prompt_tokens": 15921, + "completion_tokens": 53, + "cached_tokens": 12084, + "extra": { + "thoughts_tokens": 15, + "tool_tokens": 0 + } + }, + "llm_call_count": 1 + }, + { + "step_id": 13, + "timestamp": "2026-07-22T19:57:43.524Z", + "source": "user", + "message": "" + }, + { + "step_id": 14, + "timestamp": "2026-07-22T19:57:45.894Z", + "source": "agent", + "model_name": "gemini-3.1-pro-preview", + "message": "", + "tool_calls": [ + { + "tool_call_id": "run_shell_command__mhxmzlct", + "function_name": "run_shell_command", + "arguments": { + "command": "verify_plan_contract.py plan.json", + "description": "Run the independent contract checker" + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "run_shell_command__mhxmzlct", + "content": "\nOutput: usage: verify_plan_contract.py [-h] --contract CONTRACT --plan PLAN\n [--repo REPO] [--receipt RECEIPT]\n [--deltawire DELTAWIRE]\nverify_plan_contract.py: error: the following arguments are required: --contract, --plan\nExit Code: 2\nProcess Group PGID: 223\n" + } + ] + }, + "metrics": { + "prompt_tokens": 16471, + "completion_tokens": 68, + "cached_tokens": 12084, + "extra": { + "thoughts_tokens": 31, + "tool_tokens": 0 + } + }, + "llm_call_count": 1 + }, + { + "step_id": 15, + "timestamp": "2026-07-22T19:57:45.943Z", + "source": "user", + "message": "" + }, + { + "step_id": 16, + "timestamp": "2026-07-22T19:57:47.992Z", + "source": "agent", + "model_name": "gemini-3.1-pro-preview", + "message": "", + "tool_calls": [ + { + "tool_call_id": "run_shell_command__c153tt8e", + "function_name": "run_shell_command", + "arguments": { + "command": "verify_plan_contract.py --contract task-contract.json --plan plan.json", + "description": "Run the independent contract checker with required arguments" + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "run_shell_command__c153tt8e", + "content": "\nOutput: {\"checks\": {\"exact_count_assertion\": true, \"output_format\": true, \"output_path\": true, \"projected_records\": true, \"schema_exists\": true, \"schema_path\": true, \"schema_sha256\": true}, \"contract_sha256\": \"d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5\", \"inspect\": {\"assertion_summary\": [\"exact count 500\"], \"cmd_check\": \"deltawire check plan.json\", \"cmd_render\": \"deltawire render plan.json\", \"cmd_validate\": \"deltawire validate plan.json\", \"cold_amp_ratio\": \"6892/939\", \"output_format\": \"ndjson\", \"output_path\": \"testdata/generated/output.ndjson\", \"plan_id\": \"range-large\", \"plan_only_amp_ratio\": \"6892/653\", \"plan_path\": \"plan.json\", \"plan_source_bytes\": 653, \"primitives_used\": [\"matrix\", \"ranges\"], \"projected_output_bytes\": 6892, \"projected_records\": 500, \"schema_path\": \".deltawire/schemas/range-large.schema.json\", \"schema_source_bytes\": 286}, \"inspect_exit_code\": 0, \"plan_sha256\": \"3a32efdad6691d5a8e6bab88131e2f5bf038e6d4d48a78318b6f1e2840b1c919\", \"schema_version\": \"plan-contract-receipt.v1\", \"status\": \"pass\"}\nProcess Group PGID: 226\n" + } + ] + }, + "metrics": { + "prompt_tokens": 16649, + "completion_tokens": 68, + "cached_tokens": 12079, + "extra": { + "thoughts_tokens": 19, + "tool_tokens": 0 + } + }, + "llm_call_count": 1 + }, + { + "step_id": 17, + "timestamp": "2026-07-22T19:57:48.054Z", + "source": "user", + "message": "" + }, + { + "step_id": 18, + "timestamp": "2026-07-22T19:57:50.482Z", + "source": "agent", + "model_name": "gemini-3.1-pro-preview", + "message": "", + "tool_calls": [ + { + "tool_call_id": "run_shell_command__tfqwy76e", + "function_name": "run_shell_command", + "arguments": { + "command": "deltawire render --repo . plan.json", + "description": "Render the dataset" + } + }, + { + "tool_call_id": "run_shell_command__wtnoafp3", + "function_name": "run_shell_command", + "arguments": { + "command": "deltawire check --repo . plan.json", + "wait_for_previous": true, + "description": "Check the dataset" + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "run_shell_command__tfqwy76e", + "content": "\nOutput: Rendered /testdata/generated/output.ndjson\nProcess Group PGID: 249\n" + }, + { + "source_call_id": "run_shell_command__wtnoafp3", + "content": "\nOutput: Check ok plan.json\nProcess Group PGID: 271\n" + } + ] + }, + "metrics": { + "prompt_tokens": 17205, + "completion_tokens": 125, + "cached_tokens": 12078, + "extra": { + "thoughts_tokens": 49, + "tool_tokens": 0 + } + }, + "llm_call_count": 1 + }, + { + "step_id": 19, + "timestamp": "2026-07-22T19:57:50.536Z", + "source": "user", + "message": "\n" + }, + { + "step_id": 20, + "timestamp": "2026-07-22T19:57:54.312Z", + "source": "agent", + "model_name": "gemini-3.1-pro-preview", + "message": "", + "reasoning_content": "Verifying Final Steps: I've confirmed the render and initial checks were successful. Now, I'm focusing on the final steps. I need to run `deltawire validate` and the independent contract checker, followed by `deltawire render`. It appears those commands form the complete lifecycle needed to complete this task.", + "tool_calls": [ + { + "tool_call_id": "update_topic__x2x6k9dc", + "function_name": "update_topic", + "arguments": { + "summary": "I have successfully written the plan and ran all the required commands: `deltawire validate`, the independent `verify_plan_contract.py` contract checker, `deltawire render` to generate the NDJSON, and `deltawire check` to assert output validity. All steps passed, and the `testdata/generated/output.ndjson` file was generated properly.", + "title": "Generating Range-Large Dataset", + "strategic_intent": "All lifecycle steps are successfully completed and artifacts are generated and retained.", + "wait_for_previous": false + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "update_topic__x2x6k9dc", + "content": "Strategic Intent: All lifecycle steps are successfully completed and artifacts are generated and retained." + } + ] + }, + "metrics": { + "prompt_tokens": 17427, + "completion_tokens": 263, + "cached_tokens": 12069, + "extra": { + "thoughts_tokens": 134, + "tool_tokens": 0 + } + }, + "llm_call_count": 1 + }, + { + "step_id": 21, + "timestamp": "2026-07-22T19:57:54.315Z", + "source": "user", + "message": "" + }, + { + "step_id": 22, + "timestamp": "2026-07-22T19:57:57.044Z", + "source": "agent", + "model_name": "gemini-3.1-pro-preview", + "message": "I have successfully generated the `range-large` dataset. \n\nI created a deterministic DeltaWire plan (`plan.json`) adhering to the rules described in `/task-contract.json` and followed the mandated lifecycle steps:\n1. Validated the plan with `deltawire validate`\n2. Passed the contract checks via `verify_plan_contract.py`\n3. Rendered exactly 500 ordered records to `testdata/generated/output.ndjson`\n4. Asserted the dataset integrity with `deltawire check`\n\nThe artifact is now retained and ready in your workspace.", + "metrics": { + "prompt_tokens": 17719, + "completion_tokens": 136, + "cached_tokens": 12066, + "extra": { + "thoughts_tokens": 10, + "tool_tokens": 0 + } + }, + "llm_call_count": 1 + } + ], + "final_metrics": { + "total_prompt_tokens": 170281, + "total_completion_tokens": 3010, + "total_cached_tokens": 112782, + "total_steps": 22 + } +} \ No newline at end of file diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/config.json b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/config.json new file mode 100644 index 000000000..61a05415a --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/config.json @@ -0,0 +1,12 @@ +{ + "limits": { + "max_output_bytes": 104857600, + "max_plan_bytes": 1048576, + "max_records": 100000, + "max_schema_bytes": 1048576 + }, + "plans_dir": ".deltawire/plans", + "schemas_dir": ".deltawire/schemas", + "state_file": ".deltawire/state.json", + "version": "deltawire.config.v1" +} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/schemas/range-large.schema.json b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/schemas/range-large.schema.json new file mode 100644 index 000000000..69b476237 --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/schemas/range-large.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "index": { + "maximum": 500, + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "index" + ], + "title": "range-large", + "type": "object" +} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/state.json b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/state.json new file mode 100644 index 000000000..541959a06 --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/state.json @@ -0,0 +1,18 @@ +{ + "version": "deltawire.state.v1", + "entries": [ + { + "plan_id": "range-large", + "plan_path": "plan.json", + "schema_path": ".deltawire/schemas/range-large.schema.json", + "output_path": "testdata/generated/output.ndjson", + "plan_sha256": "1658a42bd1d4ab1b6385a59c62d713b9b8cd191ed4b2bf88ce85c537adfa82da", + "schema_sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "output_sha256": "9d205b57ab166d23c81927899f99b6ccece261c679c96979c58fbde975637776", + "record_count": 500, + "plan_source_bytes": 653, + "schema_source_bytes": 286, + "output_bytes": 6892 + } + ] +} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/logs/artifacts/deltawire/environment-receipt.json b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/logs/artifacts/deltawire/environment-receipt.json new file mode 100644 index 000000000..55108c6ac --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/logs/artifacts/deltawire/environment-receipt.json @@ -0,0 +1,55 @@ +{ + "binary": { + "actual_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "executable": true, + "exists": true, + "expected_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "expected_version": "deltawire version dev", + "is_regular": true, + "is_symlink": false, + "path": "/usr/local/bin/deltawire", + "realpath": "/usr/local/bin/deltawire", + "version_exit_code": 0, + "version_stderr": "", + "version_stdout": "deltawire version dev\nGo runtime version go1.26.5\nsupported config version deltawire.config.v1\nsupported plan version deltawire.plan.v1\nsupported state version deltawire.state.v1" + }, + "checks": { + "binary_executable": true, + "binary_exists": true, + "binary_hash": true, + "binary_not_symlink": true, + "binary_path": true, + "binary_realpath": true, + "binary_regular": true, + "config_hash": true, + "public_contract_hash": true, + "schema_hash": true, + "version_exact": true, + "version_exit_0": true + }, + "files": { + "config": { + "actual_sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "exists": true, + "expected_sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "hash_match": true, + "path": "/.deltawire/config.json" + }, + "public_contract": { + "actual_sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "exists": true, + "expected_sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "hash_match": true, + "path": "/task-contract.json" + }, + "schema": { + "actual_sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "exists": true, + "expected_sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "hash_match": true, + "path": "/.deltawire/schemas/range-large.schema.json" + } + }, + "schema_version": "deltawire-environment-receipt.v1", + "status": "pass" +} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json new file mode 100644 index 000000000..0e4289eb0 --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json @@ -0,0 +1 @@ +{"agent":"preinstalled-gemini-cli-v1","checks":{"agent_identity":true,"gemini_executable":true,"gemini_version":true,"image_id":true,"node_executable":true,"node_version":true,"npm_version":true,"package_identity":true},"gemini_path":"/usr/local/bin/gemini","gemini_version":"0.51.0","image_id":"sha256:589fce58d7ddf09910876afe0198bcee0086a840af6de7c08515bbce3e8367a7","node_path":"/usr/local/bin/node","node_version":"v22.23.1","npm_version":"10.9.8","package_version":"0.51.0","schema_version":"preinstalled-gemini-setup-receipt.v1","status":"pass"} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/manifest.json b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/manifest.json new file mode 100644 index 000000000..672df784c --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/manifest.json @@ -0,0 +1,51 @@ +[ + { + "source": "/logs/artifacts", + "destination": "artifacts/logs/artifacts", + "type": "directory", + "status": "ok", + "service": null + }, + { + "source": "testdata/generated/output.ndjson", + "destination": "artifacts/testdata/generated/output.ndjson", + "type": "file", + "status": "ok", + "service": null + }, + { + "source": ".deltawire/config.json", + "destination": "artifacts/.deltawire/config.json", + "type": "file", + "status": "ok", + "service": null + }, + { + "source": ".deltawire/plans/range-large.dw.json", + "destination": "artifacts/.deltawire/plans/range-large.dw.json", + "type": "file", + "status": "failed", + "service": null + }, + { + "source": ".deltawire/schemas/range-large.schema.json", + "destination": "artifacts/.deltawire/schemas/range-large.schema.json", + "type": "file", + "status": "ok", + "service": null + }, + { + "source": ".deltawire/state.json", + "destination": "artifacts/.deltawire/state.json", + "type": "file", + "status": "ok", + "service": null + }, + { + "source": ".deltawire/plan-contract-receipt.json", + "destination": "artifacts/.deltawire/plan-contract-receipt.json", + "type": "file", + "status": "failed", + "service": null + } +] \ No newline at end of file diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/testdata/generated/output.ndjson b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/testdata/generated/output.ndjson new file mode 100644 index 000000000..eaca8ffcd --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/testdata/generated/output.ndjson @@ -0,0 +1,500 @@ +{"index":1} +{"index":2} +{"index":3} +{"index":4} +{"index":5} +{"index":6} +{"index":7} +{"index":8} +{"index":9} +{"index":10} +{"index":11} +{"index":12} +{"index":13} +{"index":14} +{"index":15} +{"index":16} +{"index":17} +{"index":18} +{"index":19} +{"index":20} +{"index":21} +{"index":22} +{"index":23} +{"index":24} +{"index":25} +{"index":26} +{"index":27} +{"index":28} +{"index":29} +{"index":30} +{"index":31} +{"index":32} +{"index":33} +{"index":34} +{"index":35} +{"index":36} +{"index":37} +{"index":38} +{"index":39} +{"index":40} +{"index":41} +{"index":42} +{"index":43} +{"index":44} +{"index":45} +{"index":46} +{"index":47} +{"index":48} +{"index":49} +{"index":50} +{"index":51} +{"index":52} +{"index":53} +{"index":54} +{"index":55} +{"index":56} +{"index":57} +{"index":58} +{"index":59} +{"index":60} +{"index":61} +{"index":62} +{"index":63} +{"index":64} +{"index":65} +{"index":66} +{"index":67} +{"index":68} +{"index":69} +{"index":70} +{"index":71} +{"index":72} +{"index":73} +{"index":74} +{"index":75} +{"index":76} +{"index":77} +{"index":78} +{"index":79} +{"index":80} +{"index":81} +{"index":82} +{"index":83} +{"index":84} +{"index":85} +{"index":86} +{"index":87} +{"index":88} +{"index":89} +{"index":90} +{"index":91} +{"index":92} +{"index":93} +{"index":94} +{"index":95} +{"index":96} +{"index":97} +{"index":98} +{"index":99} +{"index":100} +{"index":101} +{"index":102} +{"index":103} +{"index":104} +{"index":105} +{"index":106} +{"index":107} +{"index":108} +{"index":109} +{"index":110} +{"index":111} +{"index":112} +{"index":113} +{"index":114} +{"index":115} +{"index":116} +{"index":117} +{"index":118} +{"index":119} +{"index":120} +{"index":121} +{"index":122} +{"index":123} +{"index":124} +{"index":125} +{"index":126} +{"index":127} +{"index":128} +{"index":129} +{"index":130} +{"index":131} +{"index":132} +{"index":133} +{"index":134} +{"index":135} +{"index":136} +{"index":137} +{"index":138} +{"index":139} +{"index":140} +{"index":141} +{"index":142} +{"index":143} +{"index":144} +{"index":145} +{"index":146} +{"index":147} +{"index":148} +{"index":149} +{"index":150} +{"index":151} +{"index":152} +{"index":153} +{"index":154} +{"index":155} +{"index":156} +{"index":157} +{"index":158} +{"index":159} +{"index":160} +{"index":161} +{"index":162} +{"index":163} +{"index":164} +{"index":165} +{"index":166} +{"index":167} +{"index":168} +{"index":169} +{"index":170} +{"index":171} +{"index":172} +{"index":173} +{"index":174} +{"index":175} +{"index":176} +{"index":177} +{"index":178} +{"index":179} +{"index":180} +{"index":181} +{"index":182} +{"index":183} +{"index":184} +{"index":185} +{"index":186} +{"index":187} +{"index":188} +{"index":189} +{"index":190} +{"index":191} +{"index":192} +{"index":193} +{"index":194} +{"index":195} +{"index":196} +{"index":197} +{"index":198} +{"index":199} +{"index":200} +{"index":201} +{"index":202} +{"index":203} +{"index":204} +{"index":205} +{"index":206} +{"index":207} +{"index":208} +{"index":209} +{"index":210} +{"index":211} +{"index":212} +{"index":213} +{"index":214} +{"index":215} +{"index":216} +{"index":217} +{"index":218} +{"index":219} +{"index":220} +{"index":221} +{"index":222} +{"index":223} +{"index":224} +{"index":225} +{"index":226} +{"index":227} +{"index":228} +{"index":229} +{"index":230} +{"index":231} +{"index":232} +{"index":233} +{"index":234} +{"index":235} +{"index":236} +{"index":237} +{"index":238} +{"index":239} +{"index":240} +{"index":241} +{"index":242} +{"index":243} +{"index":244} +{"index":245} +{"index":246} +{"index":247} +{"index":248} +{"index":249} +{"index":250} +{"index":251} +{"index":252} +{"index":253} +{"index":254} +{"index":255} +{"index":256} +{"index":257} +{"index":258} +{"index":259} +{"index":260} +{"index":261} +{"index":262} +{"index":263} +{"index":264} +{"index":265} +{"index":266} +{"index":267} +{"index":268} +{"index":269} +{"index":270} +{"index":271} +{"index":272} +{"index":273} +{"index":274} +{"index":275} +{"index":276} +{"index":277} +{"index":278} +{"index":279} +{"index":280} +{"index":281} +{"index":282} +{"index":283} +{"index":284} +{"index":285} +{"index":286} +{"index":287} +{"index":288} +{"index":289} +{"index":290} +{"index":291} +{"index":292} +{"index":293} +{"index":294} +{"index":295} +{"index":296} +{"index":297} +{"index":298} +{"index":299} +{"index":300} +{"index":301} +{"index":302} +{"index":303} +{"index":304} +{"index":305} +{"index":306} +{"index":307} +{"index":308} +{"index":309} +{"index":310} +{"index":311} +{"index":312} +{"index":313} +{"index":314} +{"index":315} +{"index":316} +{"index":317} +{"index":318} +{"index":319} +{"index":320} +{"index":321} +{"index":322} +{"index":323} +{"index":324} +{"index":325} +{"index":326} +{"index":327} +{"index":328} +{"index":329} +{"index":330} +{"index":331} +{"index":332} +{"index":333} +{"index":334} +{"index":335} +{"index":336} +{"index":337} +{"index":338} +{"index":339} +{"index":340} +{"index":341} +{"index":342} +{"index":343} +{"index":344} +{"index":345} +{"index":346} +{"index":347} +{"index":348} +{"index":349} +{"index":350} +{"index":351} +{"index":352} +{"index":353} +{"index":354} +{"index":355} +{"index":356} +{"index":357} +{"index":358} +{"index":359} +{"index":360} +{"index":361} +{"index":362} +{"index":363} +{"index":364} +{"index":365} +{"index":366} +{"index":367} +{"index":368} +{"index":369} +{"index":370} +{"index":371} +{"index":372} +{"index":373} +{"index":374} +{"index":375} +{"index":376} +{"index":377} +{"index":378} +{"index":379} +{"index":380} +{"index":381} +{"index":382} +{"index":383} +{"index":384} +{"index":385} +{"index":386} +{"index":387} +{"index":388} +{"index":389} +{"index":390} +{"index":391} +{"index":392} +{"index":393} +{"index":394} +{"index":395} +{"index":396} +{"index":397} +{"index":398} +{"index":399} +{"index":400} +{"index":401} +{"index":402} +{"index":403} +{"index":404} +{"index":405} +{"index":406} +{"index":407} +{"index":408} +{"index":409} +{"index":410} +{"index":411} +{"index":412} +{"index":413} +{"index":414} +{"index":415} +{"index":416} +{"index":417} +{"index":418} +{"index":419} +{"index":420} +{"index":421} +{"index":422} +{"index":423} +{"index":424} +{"index":425} +{"index":426} +{"index":427} +{"index":428} +{"index":429} +{"index":430} +{"index":431} +{"index":432} +{"index":433} +{"index":434} +{"index":435} +{"index":436} +{"index":437} +{"index":438} +{"index":439} +{"index":440} +{"index":441} +{"index":442} +{"index":443} +{"index":444} +{"index":445} +{"index":446} +{"index":447} +{"index":448} +{"index":449} +{"index":450} +{"index":451} +{"index":452} +{"index":453} +{"index":454} +{"index":455} +{"index":456} +{"index":457} +{"index":458} +{"index":459} +{"index":460} +{"index":461} +{"index":462} +{"index":463} +{"index":464} +{"index":465} +{"index":466} +{"index":467} +{"index":468} +{"index":469} +{"index":470} +{"index":471} +{"index":472} +{"index":473} +{"index":474} +{"index":475} +{"index":476} +{"index":477} +{"index":478} +{"index":479} +{"index":480} +{"index":481} +{"index":482} +{"index":483} +{"index":484} +{"index":485} +{"index":486} +{"index":487} +{"index":488} +{"index":489} +{"index":490} +{"index":491} +{"index":492} +{"index":493} +{"index":494} +{"index":495} +{"index":496} +{"index":497} +{"index":498} +{"index":499} +{"index":500} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/config.json b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/config.json new file mode 100644 index 000000000..0a0a8fd2f --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/config.json @@ -0,0 +1,15 @@ +{ + "task": { + "path": "/home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/pairs/range-large-v7-r1" + }, + "trial_name": "range-large-v7-r1__3wEHYmv", + "trials_dir": "/home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08", + "agent": { + "name": "agents.preinstalled_gemini_cli:PreinstalledGeminiCli", + "model_name": "google/gemini-3.1-pro-preview", + "skills": [ + "/home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire" + ] + }, + "job_id": "0bb5b3c3-144c-4aa0-85d4-5b4aeb9f578e" +} \ No newline at end of file diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/lock.json b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/lock.json new file mode 100644 index 000000000..42cea6683 --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/lock.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "task": { + "name": "range-large-v7-r1", + "type": "local", + "digest": "sha256:f4b01bec8b34ab576ed35e9b9512b88945f0dd566a5f5af1e066476568d54cc1", + "path": "/home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/pairs/range-large-v7-r1" + }, + "install_only": false, + "timeout_multiplier": 1.0, + "agent": { + "name": "agents.preinstalled_gemini_cli:PreinstalledGeminiCli", + "model_name": "google/gemini-3.1-pro-preview", + "skills": [ + "/home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire" + ], + "resume_trajectory": false, + "extra_allowed_hosts": [], + "kwargs": {}, + "mcp_servers": [] + }, + "skills": [ + { + "name": "deltawire", + "source": "/home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire", + "digest": "sha256:a841ca12dc687b1e467f90d5d9f5ccef120bc810742fab693e8e976135a75984" + } + ], + "environment": { + "type": "docker", + "force_build": false, + "delete": true, + "cpu_enforcement_policy": "auto", + "memory_enforcement_policy": "auto", + "extra_docker_compose": [], + "kwargs": {}, + "extra_allowed_hosts": [] + }, + "verifier": { + "disable": false + } +} \ No newline at end of file diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/result.json b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/result.json new file mode 100644 index 000000000..1eca14d1f --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/result.json @@ -0,0 +1,115 @@ +{ + "id": "db50c949-e6b2-4fcf-8ba0-3d0f56b0109e", + "task_name": "operatorstack/range-large-v7-r1", + "trial_name": "range-large-v7-r1__3wEHYmv", + "trial_uri": "file:///home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv", + "task_id": { + "path": "/home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/pairs/range-large-v7-r1" + }, + "source": null, + "task_checksum": "940f55834992b2d3560bc45559a4d01151e0f50f6d0892ec65ec135d0312667a", + "config": { + "task": { + "path": "/home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/pairs/range-large-v7-r1", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "range-large-v7-r1__3wEHYmv", + "trials_dir": "/home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08", + "install_only": false, + "timeout_multiplier": 1.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": "agents.preinstalled_gemini_cli:PreinstalledGeminiCli", + "import_path": null, + "model_name": "google/gemini-3.1-pro-preview", + "n_concurrent": null, + "concurrency_group": null, + "skills": [ + "/home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire" + ], + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "resume_trajectory": false, + "load_trajectory": null, + "extra_allowed_hosts": [], + "kwargs": {}, + "mcp_servers": [] + }, + "environment": { + "type": "docker", + "import_path": null, + "force_build": false, + "delete": true, + "cpu_enforcement_policy": "auto", + "memory_enforcement_policy": "auto", + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "override_tpu": null, + "mounts": null, + "extra_docker_compose": [], + "kwargs": {}, + "extra_allowed_hosts": [] + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "disable": false + }, + "artifacts": [], + "extra_instruction_paths": [], + "job_id": "0bb5b3c3-144c-4aa0-85d4-5b4aeb9f578e" + }, + "agent_info": { + "name": "preinstalled-gemini-cli-v1", + "version": "0.51.0", + "model_info": { + "name": "gemini-3.1-pro-preview", + "provider": "google" + } + }, + "agent_result": { + "n_input_tokens": 170281, + "n_cache_tokens": 112782, + "n_output_tokens": 3010, + "cost_usd": 0.1736744, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "exact_match": 1 + } + }, + "exception_info": null, + "started_at": "2026-07-22T19:57:08.348170Z", + "finished_at": "2026-07-22T19:58:12.511894Z", + "environment_setup": { + "started_at": "2026-07-22T19:57:08.374913Z", + "finished_at": "2026-07-22T19:57:09.590605Z" + }, + "agent_setup": { + "started_at": "2026-07-22T19:57:09.883789Z", + "finished_at": "2026-07-22T19:57:14.203542Z" + }, + "agent_execution": { + "started_at": "2026-07-22T19:57:14.203652Z", + "finished_at": "2026-07-22T19:57:57.314481Z" + }, + "verifier": { + "started_at": "2026-07-22T19:58:01.300069Z", + "finished_at": "2026-07-22T19:58:01.619934Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/verifier/reward.json b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/verifier/reward.json new file mode 100644 index 000000000..d0f7c4741 --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/verifier/reward.json @@ -0,0 +1 @@ +{"exact_match": 1} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/verifier/test-stdout.txt b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/verifier/test-stdout.txt new file mode 100644 index 000000000..e69de29bb diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/result.json b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/result.json new file mode 100644 index 000000000..52ce8248a --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/result.json @@ -0,0 +1,39 @@ +{ + "id": "0bb5b3c3-144c-4aa0-85d4-5b4aeb9f578e", + "started_at": "2026-07-22T19:57:08.275254", + "updated_at": "2026-07-22T19:58:12.514642", + "finished_at": "2026-07-22T19:58:12.514642", + "n_total_trials": 1, + "stats": { + "n_completed_trials": 1, + "n_errored_trials": 0, + "n_running_trials": 0, + "n_pending_trials": 0, + "n_cancelled_trials": 0, + "n_retries": 0, + "evals": { + "preinstalled-gemini-cli-v1__gemini-3.1-pro-preview__adhoc": { + "n_trials": 1, + "n_errors": 0, + "metrics": [ + { + "mean": 1.0 + } + ], + "pass_at_k": {}, + "reward_stats": { + "exact_match": { + "1": [ + "range-large-v7-r1__3wEHYmv" + ] + } + }, + "exception_stats": {} + } + }, + "n_input_tokens": 170281, + "n_cache_tokens": 112782, + "n_output_tokens": 3010, + "cost_usd": 0.1736744 + } +} \ No newline at end of file diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/readiness.json b/labs/20-deltawire/eval/results/range-large-v7-r1/readiness.json new file mode 100644 index 000000000..6531e477f --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/readiness.json @@ -0,0 +1,20 @@ +{ + "B0_launched": false, + "D1_launched": true, + "READY_FOR_72": false, + "blockers": [ + "v7 D1 treatment fidelity failed", + "v7 B0 was withheld", + "no valid causal pair", + "no six-run canary", + "no final confirmatory manifest", + "no explicit 72-run authorization" + ], + "full_run_started": false, + "pair_complete": false, + "pair_launched": true, + "pair_valid": false, + "rerun_authorized": false, + "schema_version": "deltawire-readiness.v7", + "six_run_canary_started": false +} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/run-ledger.json b/labs/20-deltawire/eval/results/range-large-v7-r1/run-ledger.json new file mode 100644 index 000000000..88f8624b2 --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/run-ledger.json @@ -0,0 +1,46 @@ +{ + "READY_FOR_72": false, + "order": [ + "D1", + "B0" + ], + "pair_id": "range-large/v7-r1", + "runs": [ + { + "agent_execution_started": true, + "agent_setup_status": "pass", + "arm": "D1", + "cache_tokens": 112782, + "config_path": "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/config.json", + "config_sha256": "0768e4213b3a05725c0ef62c851d2c0bd902b102dfa134de749cfe2e992f5bad", + "environment_setup_status": "pass", + "harbor_exit_code": 0, + "harbor_process_exit_code": 0, + "infrastructure_failure_class": null, + "input_tokens": 170281, + "lock_path": "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/lock.json", + "lock_sha256": "5a931c5b1058e58b3792de2f7a9303139778180b3efedc2d3b91081ef14679a3", + "output_path": "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/testdata/generated/output.ndjson", + "output_sha256": "9d205b57ab166d23c81927899f99b6ccece261c679c96979c58fbde975637776", + "output_tokens": 3010, + "redacted_command": "uvx --from harbor==0.20.0 harbor run -a agents.preinstalled_gemini_cli:PreinstalledGeminiCli -m google/gemini-3.1-pro-preview -p /home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/pairs/range-large-v7-r1 -k 1 --jobs-dir /home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1 --n-concurrent 1 --skill /home/apple/deltawire-preflight-v7-frozen/labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire --env-file ''", + "reported_agent": "preinstalled-gemini-cli-v1", + "reported_cost": 0.1736744, + "reported_model": "gemini-3.1-pro-preview", + "result_path": "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/result.json", + "result_sha256": "22f50c59c4eac9f5f1a680572304f25a11270c3fb2caa66be584bbd9d7a3525b", + "schema_version": "harbor-trial-status.v1", + "status": "completed", + "trajectory_path": "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/gemini-cli.trajectory.jsonl", + "trajectory_sha256": "d6659287d950b37b9ea1364e6256d715ce0220b652c45689355b79e623be7af5", + "trial_dir": "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv", + "trial_exception_message": null, + "trial_exception_type": null, + "trial_id": "db50c949-e6b2-4fcf-8ba0-3d0f56b0109e", + "trial_status": "completed", + "verifier_started": true, + "wall_time_seconds": 64.163724 + } + ], + "schema_version": "deltawire-v7-pair-ledger.v1" +} diff --git a/labs/20-deltawire/eval/results/range-large-v7-r1/terminal-report.json b/labs/20-deltawire/eval/results/range-large-v7-r1/terminal-report.json new file mode 100644 index 000000000..137fd93f5 --- /dev/null +++ b/labs/20-deltawire/eval/results/range-large-v7-r1/terminal-report.json @@ -0,0 +1,58 @@ +{ + "B0": { + "launched": false, + "reason": "withheld after D1 end-to-end failure", + "trial_count": 0 + }, + "D1": { + "activation": true, + "agent": "preinstalled-gemini-cli-v1", + "artifact_manifest_status": "pass", + "assignment_status": "pass", + "end_to_end_status": "fail", + "environment_status": "pass", + "failure_reasons": [ + "plan was created as plan.json instead of retained .deltawire/plans/range-large.dw.json", + "in-container plan-contract receipt was not retained", + "independent retained-plan contract check found zero canonical plans", + "structured public-contract access gate was not proven" + ], + "harbor_exit_code": 0, + "mechanism": { + "binary_introspection": false, + "contract_checker_failed_attempts": 1, + "direct_validates": 1, + "failed_direct_validates": 0, + "lifecycle_calls": 3, + "matching_example_read": true, + "reference_read": true, + "subprocess_bruteforce_loops": 0, + "tool_calls": 18 + }, + "model": "gemini-3.1-pro-preview", + "semantic_status": "pass", + "setup_status": "pass", + "treatment_status": "fail", + "trial_exception_type": null, + "trial_id": "db50c949-e6b2-4fcf-8ba0-3d0f56b0109e", + "usage": { + "cache_tokens": 112782, + "harbor_reported_cost": 0.1736744, + "input_plus_output_tokens": 173291, + "input_tokens": 170281, + "output_tokens": 3010, + "wall_time_seconds": 64.163724 + } + }, + "READY_FOR_72": false, + "claim": "no causal comparison; D1-only failed-fidelity observation", + "comparison_available": false, + "full_run_started": false, + "outcome": "D1_semantic_pass_treatment_fidelity_fail_B0_withheld", + "pair_attempt_consumed": true, + "pair_id": "range-large/v7-r1", + "pair_valid": false, + "rerun_authorized": false, + "schema_version": "deltawire-v7-terminal-report.v1", + "six_run_canary_started": false +} diff --git a/labs/20-deltawire/eval/scripts/v7/generate_manifest.py b/labs/20-deltawire/eval/scripts/v7/generate_manifest.py new file mode 100644 index 000000000..61333b25b --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v7/generate_manifest.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Generate or verify the frozen DeltaWire range-large/v7-r1 pair manifest.""" +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import subprocess +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[4] +EVAL = ROOT / "labs/20-deltawire/eval" +TASK = EVAL / "pairs/range-large-v7-r1" +SKILL = TASK / "skill/deltawire" +REPORT = EVAL / "results/preflight-v7/skill-conformance.json" +TARGET = EVAL / "manifests/deltawire-range-large-v7-pair.json" + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def tree_hash(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(item for item in root.rglob("*") if item.is_file() and "__pycache__" not in item.parts): + digest.update(path.relative_to(root).as_posix().encode()) + digest.update(b"\0") + digest.update(sha(path).encode()) + digest.update(b"\0") + return digest.hexdigest() + + +def repo_files() -> list[Path]: + roots = [TASK, HERE] + files = [path for root in roots for path in root.rglob("*") if path.is_file() and "__pycache__" not in path.parts and ".generated" not in path.parts] + files += [ + EVAL / "manifests/preflight-v6-terminal-evidence-lock.json", + EVAL / "scripts/v6/verify_terminal_evidence_lock.py", + EVAL / "scripts/v6/shell_observation.py", + EVAL / "scripts/v6/artifact_manifest.py", + EVAL / "scripts/v6/setup_receipt.py", + EVAL / "scripts/v6/trial_status.py", + EVAL / "scripts/semantic_oracle.py", + EVAL / "scripts/verify_plan_contract.py", + EVAL / "runtime/preinstalled-gemini-v1.json", + EVAL / "agents/preinstalled_gemini_cli.py", + EVAL / "matrices/72-run-matrix.json", + REPORT, + ] + return sorted(set(files)) + + +def package_source(module: str) -> dict: + spec = importlib.util.find_spec(module) + if not spec or not spec.origin: + raise RuntimeError(f"cannot locate {module}; run with harbor==0.20.0 installed") + path = Path(spec.origin) + return {"module": module, "sha256": sha(path)} + + +def build(source_commit: str) -> dict: + if subprocess.check_output(["git", "rev-parse", source_commit], cwd=ROOT, text=True).strip() != source_commit: + raise RuntimeError("source commit must be a full existing commit hash") + conformance = json.loads(REPORT.read_text()) + if conformance.get("status") != "pass": + raise RuntimeError("skill conformance must pass before freezing") + input_hashes = {path.relative_to(ROOT).as_posix(): sha(path) for path in repo_files()} + return { + "schema_version": "deltawire-range-large-v7-pair-manifest.v1", + "pair_id": "range-large/v7-r1", + "source_matrix_positions_zero_based": [36, 37], + "source_matrix_pair_id": "range-large/r1", + "order": ["D1", "B0"], + "failure_policy": "withhold_B0_unless_D1_passes", + "infrastructure_commit": source_commit, + "integration_commit": "82a7963bdd98ca41435e756fb4212b3b37b19420", + "result_root": "labs/20-deltawire/eval/results/range-large-v7-r1", + "harbor_version": "0.20.0", + "harbor_command": ["uvx", "--from", "harbor==0.20.0", "harbor"], + "harbor_source": [package_source("harbor.agents.installed.gemini_cli"), package_source("harbor.skills")], + "model": "google/gemini-3.1-pro-preview", + "expected_reported_model": "gemini-3.1-pro-preview", + "agent": "preinstalled-gemini-cli-v1", + "agent_import": "agents.preinstalled_gemini_cli:PreinstalledGeminiCli", + "gemini_cli_version": "0.51.0", + "image_ref": "deltawire-preflight-v6:runtime-v1", + "image_id": "sha256:589fce58d7ddf09910876afe0198bcee0086a840af6de7c08515bbce3e8367a7", + "binary_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "skill": { + "tree_sha256": tree_hash(SKILL), + "harbor_digest": conformance["harbor_skill_digest"], + "skill_sha256": conformance["skill_sha256"], + "reference_sha256": conformance["reference_sha256"], + "conformance_sha256": sha(REPORT), + }, + "input_hashes": input_hashes, + "READY_FOR_72": False, + "six_run_canary_started": False, + "full_run_started": False, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--source-commit") + parser.add_argument("--output", default=str(TARGET)) + parser.add_argument("--verify", action="store_true") + args = parser.parse_args() + output = Path(args.output) + if args.verify: + current = json.loads(output.read_text()) + expected = build(current["infrastructure_commit"]) + if current != expected: + raise SystemExit("manifest drift") + print(sha(output)) + return + if not args.source_commit: + parser.error("--source-commit is required when generating") + value = build(args.source_commit) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + print(sha(output)) + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/scripts/v7/runner.py b/labs/20-deltawire/eval/scripts/v7/runner.py new file mode 100644 index 000000000..23a7b101e --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v7/runner.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +"""One-shot DeltaWire v7 causal-pair runner; no probe or full modes exist.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shlex +import subprocess +import sys +import tempfile +from datetime import datetime +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[4] +EVAL = ROOT / "labs/20-deltawire/eval" +V6 = EVAL / "scripts/v6" +sys.path.insert(0, str(V6)) +from artifact_manifest import validate as validate_artifacts # noqa: E402 +from setup_receipt import validate as validate_setup # noqa: E402 +from trial_status import normalize # noqa: E402 +sys.path.insert(0, str(HERE)) +from treatment import build as treatment_build # noqa: E402 + +PAIR_ID = "range-large/v7-r1" +TASK = EVAL / "pairs/range-large-v7-r1" +SKILL = TASK / "skill/deltawire" +RESULT_ROOT = EVAL / "results/range-large-v7-r1" +MANIFEST = EVAL / "manifests/deltawire-range-large-v7-pair.json" +BINARY = EVAL / ".generated/deltawire-linux-amd64" +RUNTIME = EVAL / "runtime/preinstalled-gemini-v1.json" +APPROVAL_TEXT = "approve the frozen range-large/v7-r1 pair" + + +def sha(path: Path) -> str: + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def load(path: Path) -> dict: + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def write(path: Path, value: dict) -> None: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def relative(path: Path) -> str: + return Path(path).resolve().relative_to(ROOT.resolve()).as_posix() + + +def refuse_nonempty(path: Path) -> None: + if path.exists() and any(path.iterdir()): + raise RuntimeError("v7 pair result root is nonempty; the one-shot attempt is consumed") + + +def verify_manifest(path: Path, approved_sha: str) -> dict: + path = Path(path).resolve() + if path != MANIFEST.resolve() or sha(path) != approved_sha: + raise RuntimeError("v7 manifest path or approved SHA mismatch") + manifest = load(path) + if manifest.get("schema_version") != "deltawire-range-large-v7-pair-manifest.v1": + raise RuntimeError("v7 manifest schema mismatch") + identities = { + "pair_id": PAIR_ID, + "harbor_version": "0.20.0", + "model": "google/gemini-3.1-pro-preview", + "agent": "preinstalled-gemini-cli-v1", + "agent_import": "agents.preinstalled_gemini_cli:PreinstalledGeminiCli", + "image_id": "sha256:589fce58d7ddf09910876afe0198bcee0086a840af6de7c08515bbce3e8367a7", + "binary_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + } + for key, expected in identities.items(): + if manifest.get(key) != expected: + raise RuntimeError(f"frozen identity mismatch: {key}") + if manifest.get("order") != ["D1", "B0"] or manifest.get("failure_policy") != "withhold_B0_unless_D1_passes": + raise RuntimeError("frozen order or failure policy mismatch") + for name, expected in manifest.get("input_hashes", {}).items(): + target = ROOT / name + if not target.is_file() or sha(target) != expected: + raise RuntimeError(f"frozen input mismatch: {name}") + runtime = load(RUNTIME) + if runtime.get("image_id") != manifest["image_id"] or runtime.get("gemini_cli_version") != "0.51.0": + raise RuntimeError("runtime metadata mismatch") + actual_harbor = subprocess.check_output(manifest["harbor_command"] + ["--version"], cwd=ROOT, text=True).strip() + if actual_harbor != "0.20.0": + raise RuntimeError("Harbor version mismatch") + subprocess.run(["python3", str(V6 / "verify_terminal_evidence_lock.py")], cwd=ROOT, check=True) + matrix = load(EVAL / "matrices/72-run-matrix.json") + source = matrix[36:38] + if [item.get("pair_id") for item in source] != ["range-large/r1", "range-large/r1"] or [item.get("arm") for item in source] != ["D1", "B0"]: + raise RuntimeError("frozen source matrix positions 36-37 mismatch") + report = load(EVAL / "results/preflight-v7/skill-conformance.json") + if report.get("status") != "pass" or sha(EVAL / "results/preflight-v7/skill-conformance.json") != manifest["skill"]["conformance_sha256"]: + raise RuntimeError("skill conformance mismatch") + return manifest + + +def verify_binary(manifest: dict) -> None: + expected = manifest["binary_sha256"] + for path in (BINARY, TASK / "environment/.generated/deltawire"): + if not path.is_file() or sha(path) != expected: + raise RuntimeError(f"binary parity mismatch: {relative(path)}") + + +def verify_image(manifest: dict) -> None: + actual = subprocess.check_output( + ["docker", "image", "inspect", manifest["image_ref"], "--format", "{{.Id}}"], text=True + ).strip() + if actual != manifest["image_id"]: + raise RuntimeError("preinstalled image identity mismatch") + + +def verify_approval(path: Path, manifest_sha: str) -> dict: + value = load(path) + checks = { + "schema": value.get("schema_version") == "deltawire-v7-pair-approval.v1", + "authorization": value.get("authorization") == APPROVAL_TEXT, + "pair_id": value.get("pair_id") == PAIR_ID, + "order": value.get("order") == ["D1", "B0"], + "manifest_sha": value.get("manifest_sha256") == manifest_sha, + "no_rerun": value.get("rerun_authorized") is False, + "no_canary": value.get("six_run_canary_authorized") is False, + "no_full": value.get("full_run_authorized") is False, + "not_ready": value.get("READY_FOR_72") is False, + } + if not all(checks.values()): + raise RuntimeError("pair approval receipt is invalid") + return value + + +def private_env() -> Path: + key = os.environ.get("GEMINI_API_KEY", "").strip() + if not key: + raise RuntimeError("GEMINI_API_KEY is required for live execution") + handle = tempfile.NamedTemporaryFile("w", prefix="deltawire-v7-", suffix=".env", delete=False) + handle.write("GEMINI_API_KEY=" + key.replace("\n", "") + "\n") + handle.close() + os.chmod(handle.name, 0o600) + return Path(handle.name) + + +def trial_dirs(jobs: Path) -> set[Path]: + return {path.resolve() for path in jobs.glob("*/*__*") if path.is_dir()} + + +def command(manifest: dict, arm: str, jobs: Path) -> list[str]: + result = manifest["harbor_command"] + [ + "run", "-a", manifest["agent_import"], "-m", manifest["model"], + "-p", str(TASK), "-k", "1", "--jobs-dir", str(jobs), "--n-concurrent", "1", + ] + if arm == "D1": + result += ["--skill", str(SKILL)] + return result + + +def run_one(manifest: dict, arm: str, jobs: Path, dry_run: bool) -> dict: + cmd = command(manifest, arm, jobs) + redacted = shlex.join(cmd + ["--env-file", ""]) + if dry_run: + return {"arm": arm, "status": "dry_run", "redacted_command": redacted} + env_file = private_env() + before = trial_dirs(jobs) + jobs.mkdir(parents=True, exist_ok=True) + env = os.environ.copy() + env.pop("GEMINI_API_KEY", None) + env["PYTHONPATH"] = str(EVAL) + try: + process = subprocess.run(cmd + ["--env-file", str(env_file)], cwd=ROOT, env=env, check=False) + finally: + env_file.unlink(missing_ok=True) + created = trial_dirs(jobs) - before + entry = {"arm": arm, "harbor_exit_code": process.returncode, "redacted_command": redacted} + if len(created) != 1: + entry.update({"status": "trial_failed", "artifact_error": f"expected one trial directory; found {len(created)}"}) + return entry + trial = created.pop() + entry["trial_dir"] = relative(trial) + candidates = { + "result_path": trial / "result.json", + "trajectory_path": trial / "agent/gemini-cli.trajectory.jsonl", + "lock_path": trial / "lock.json", + "config_path": trial / "config.json", + } + outputs = sorted((trial / "artifacts").rglob("output.ndjson")) + if outputs: + candidates["output_path"] = outputs[0] + for name, path in candidates.items(): + if path.is_file(): + entry[name] = relative(path) + entry[name.replace("_path", "_sha256")] = sha(path) + if entry.get("result_path"): + result = load(ROOT / entry["result_path"]) + entry.update(normalize(result, process.returncode)) + agent = result.get("agent_result") or {} + info = result.get("agent_info") or {} + entry.update( + { + "trial_id": result.get("id") or result.get("trial_name"), + "reported_agent": info.get("name"), + "reported_model": (info.get("model_info") or {}).get("name"), + "input_tokens": agent.get("n_input_tokens"), + "output_tokens": agent.get("n_output_tokens"), + "cache_tokens": agent.get("n_cache_tokens"), + "reported_cost": agent.get("cost_usd"), + "wall_time_seconds": duration(result), + } + ) + required = ("result_path", "trajectory_path", "lock_path", "config_path", "output_path") + missing = [name for name in required if name not in entry] + if missing: + entry["artifact_error"] = "missing: " + ", ".join(missing) + entry["status"] = "completed" if entry.get("trial_status") == "completed" and process.returncode == 0 and not missing else "trial_failed" + return entry + + +def duration(result: dict): + try: + started = datetime.fromisoformat(result["started_at"].replace("Z", "+00:00")) + finished = datetime.fromisoformat(result["finished_at"].replace("Z", "+00:00")) + return (finished - started).total_seconds() + except (KeyError, TypeError, ValueError): + return None + + +def semantic(output: Path) -> dict: + process = subprocess.run( + ["python3", str(EVAL / "scripts/semantic_oracle.py"), str(TASK / "task-spec.json"), str(output)], + capture_output=True, text=True, check=False, + ) + try: + oracle = json.loads(process.stdout) + except json.JSONDecodeError: + oracle = {"exact_match": 0} + exact = oracle.get("exact_match") == 1 and process.returncode == 0 + return {"schema_version": "semantic-result.v7", "status": "pass" if exact else "fail", "exact_match": 1 if exact else 0, "oracle": oracle} + + +def independent_contract(trial: Path, receipt: Path) -> dict: + plans = sorted((trial / "artifacts/.deltawire/plans").glob("*.dw.json")) + if len(plans) != 1: + value = {"schema_version": "plan-contract-receipt.v1", "status": "fail", "error": f"expected one plan; found {len(plans)}", "checks": {}} + write(receipt, value) + return value + subprocess.run( + ["python3", str(EVAL / "scripts/verify_plan_contract.py"), "--contract", str(TASK / "task-spec.json"), + "--plan", str(plans[0]), "--repo", str(trial / "artifacts"), "--deltawire", str(BINARY), "--receipt", str(receipt)], + cwd=ROOT, capture_output=True, text=True, check=False, + ) + return load(receipt) if receipt.is_file() else {"status": "fail"} + + +def failure_receipts(arm: str, root: Path, reason: str) -> dict: + schemas = { + "semantic-result": "semantic-result.v7", + "treatment": "deltawire-treatment-receipt.v7", + "environment": "deltawire-environment-receipt.v1", + "artifact-manifest": "artifact-manifest-receipt.v1", + "setup": "preinstalled-gemini-setup-validation.v1", + "end-to-end": "deltawire-end-to-end-receipt.v7", + } + paths = {} + for name, schema in schemas.items(): + path = root / f"{arm}-{name}.json" + write(path, {"schema_version": schema, "arm": arm, "status": "fail", "error": reason, "READY_FOR_72": False}) + paths[name] = path + return {"status": "fail", "paths": paths} + + +def finalize(manifest: dict, entry: dict, root: Path) -> dict: + arm = entry["arm"] + if not entry.get("trial_dir"): + return failure_receipts(arm, root, entry.get("artifact_error", "trial directory missing")) + trial = ROOT / entry["trial_dir"] + trajectory = ROOT / entry["trajectory_path"] if entry.get("trajectory_path") else None + output = ROOT / entry["output_path"] if entry.get("output_path") else None + sem = semantic(output) if output and output.is_file() else {"schema_version": "semantic-result.v7", "status": "fail", "exact_match": 0, "error": "output missing"} + sem_path = root / f"{arm}-semantic-result.json" + write(sem_path, sem) + independent_path = root / f"{arm}-plan-contract-receipt.json" + independent = independent_contract(trial, independent_path) if arm == "D1" else {"status": "not_applicable"} + if not trajectory or not trajectory.is_file(): + return failure_receipts(arm, root, "structured trajectory missing") + use = treatment_build( + trajectory, trial, arm, load(TASK / "task-spec.json"), sem["exact_match"], independent_path, + ROOT / entry["lock_path"], ROOT / entry["config_path"], SKILL, manifest, + ) + use_path = root / f"{arm}-treatment.json" + write(use_path, use) + env_source = trial / "artifacts/logs/artifacts/deltawire/environment-receipt.json" + try: + environment = load(env_source) + except (OSError, json.JSONDecodeError): + environment = {"schema_version": "deltawire-environment-receipt.v1", "status": "fail", "error": "receipt missing"} + env_path = root / f"{arm}-environment.json" + write(env_path, environment) + expectations = load(TASK / "environment/deltawire-environment-expectations.json") + expected_files = {name: item["sha256"] for name, item in expectations["files"].items()} + artifact = validate_artifacts(trial, manifest["binary_sha256"], expected_files) + artifact_path = root / f"{arm}-artifact-manifest.json" + write(artifact_path, artifact) + setup_source = trial / "artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json" + setup = validate_setup(setup_source, load(RUNTIME)) + setup_path = root / f"{arm}-setup.json" + write(setup_path, setup) + identity = { + "harbor_exit_zero": entry.get("harbor_exit_code") == 0, + "trial_completed": entry.get("trial_status") == "completed", + "agent_match": entry.get("reported_agent") == manifest["agent"], + "model_match": entry.get("reported_model") == manifest["expected_reported_model"], + } + component = { + "setup": setup.get("status") == "pass", + "environment": environment.get("status") == "pass", + "artifact_manifest": artifact.get("status") == "pass", + "treatment": use.get("status") == "pass", + "semantic": sem.get("status") == "pass", + "plan_contract": independent.get("status") == "pass" if arm == "D1" else True, + } + end = { + "schema_version": "deltawire-end-to-end-receipt.v7", "arm": arm, + "status": "pass" if all(identity.values()) and all(component.values()) and not entry.get("artifact_error") else "fail", + "identity_checks": identity, "component_checks": component, "READY_FOR_72": False, + "source_hashes": {"semantic": sha(sem_path), "treatment": sha(use_path), "environment": sha(env_path), "artifact_manifest": sha(artifact_path), "setup": sha(setup_path), "trajectory": entry.get("trajectory_sha256"), "result": entry.get("result_sha256"), "output": entry.get("output_sha256")}, + } + end_path = root / f"{arm}-end-to-end.json" + write(end_path, end) + return {"status": end["status"], "end": end, "treatment": use} + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("mode", choices=("pair-v7",)) + parser.add_argument("--manifest", required=True) + parser.add_argument("--approve-manifest-sha", required=True) + parser.add_argument("--pair-id", required=True) + parser.add_argument("--approval") + parser.add_argument("--dry-run", action="store_true") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if args.pair_id != PAIR_ID: + raise RuntimeError(f"only {PAIR_ID} is permitted") + manifest = verify_manifest(Path(args.manifest), args.approve_manifest_sha) + refuse_nonempty(RESULT_ROOT) + verify_binary(manifest) + verify_image(manifest) + runs = ["D1", "B0"] + if args.dry_run: + resolved = [run_one(manifest, arm, RESULT_ROOT / "raw" / arm, True) for arm in runs] + print(json.dumps({"schema_version": "deltawire-v7-dry-run.v1", "pair_id": PAIR_ID, "order": runs, "B0_conditional_on_D1": True, "runs": resolved, "pair_authorized": False, "READY_FOR_72": False}, indent=2, sort_keys=True)) + return + if not args.approval: + raise RuntimeError("live pair requires an approval receipt") + verify_approval(Path(args.approval), args.approve_manifest_sha) + ledger = {"schema_version": "deltawire-v7-pair-ledger.v1", "pair_id": PAIR_ID, "order": runs, "runs": [], "READY_FOR_72": False} + ledger_path = RESULT_ROOT / "run-ledger.json" + for arm in runs: + entry = run_one(manifest, arm, RESULT_ROOT / "raw" / arm, False) + ledger["runs"].append(entry) + write(ledger_path, ledger) + try: + outcome = finalize(manifest, entry, RESULT_ROOT) + except Exception as error: + failure_receipts(arm, RESULT_ROOT, str(error)) + raise RuntimeError(f"{arm} finalization failed: {error}") from error + if outcome["status"] != "pass": + raise RuntimeError(f"{arm} end-to-end gate failed; later arm withheld") + d1, b0 = ledger["runs"] + write( + RESULT_ROOT / "pair-summary.json", + { + "schema_version": "deltawire-v7-causal-pair.v1", "pair_id": PAIR_ID, "status": "pass", "order": runs, + "D1": {key: d1.get(key) for key in ("trial_id", "input_tokens", "output_tokens", "cache_tokens", "reported_cost", "wall_time_seconds")}, + "B0": {key: b0.get(key) for key in ("trial_id", "input_tokens", "output_tokens", "cache_tokens", "reported_cost", "wall_time_seconds")}, + "claim_scope": "single_frozen_pair_observation", "READY_FOR_72": False, "six_run_canary_started": False, "full_run_started": False, + }, + ) + + +if __name__ == "__main__": + try: + main() + except (OSError, KeyError, ValueError, RuntimeError, subprocess.CalledProcessError) as error: + print(f"ERROR: {error}", file=sys.stderr) + raise SystemExit(1) diff --git a/labs/20-deltawire/eval/scripts/v7/skill_conformance.py b/labs/20-deltawire/eval/scripts/v7/skill_conformance.py new file mode 100644 index 000000000..87b258fd6 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v7/skill_conformance.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Credential-free conformance for the host-neutral DeltaWire skill bundle.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import subprocess +import tempfile +from pathlib import Path + + +EXPECTED = { + "range.dw.json": [{"index": 1}, {"index": 2}, {"index": 3}], + "matrix.dw.json": [ + {"method": "GET", "role": "guest"}, + {"method": "POST", "role": "guest"}, + {"method": "GET", "role": "member"}, + {"method": "POST", "role": "member"}, + ], + "rows.dw.json": [ + {"enabled": True, "name": "alpha", "rank": 1}, + {"enabled": True, "name": "beta", "rank": 2}, + ], + "variants.dw.json": [ + {"active": True, "role": "member", "status": "deny"}, + {"active": False, "role": "member", "status": "allow"}, + ], +} + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def tree_hash(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(item for item in root.rglob("*") if item.is_file()): + relative = path.relative_to(root).as_posix() + digest.update(relative.encode()) + digest.update(b"\0") + digest.update(sha(path).encode()) + digest.update(b"\0") + return digest.hexdigest() + + +def harbor_digest(root: Path) -> str: + return "sha256:" + tree_hash(root) + + +def run(binary: Path, repo: Path, command: str, plan: Path) -> dict: + completed = subprocess.run( + [str(binary), command, "--repo", str(repo), str(plan.relative_to(repo))], + capture_output=True, + text=True, + check=False, + ) + return { + "command": command, + "exit_code": completed.returncode, + "stdout": completed.stdout, + "stderr": completed.stderr, + } + + +def permissive_schema() -> dict: + return {"$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object"} + + +def verify_example(binary: Path, example: Path) -> dict: + expected = EXPECTED[example.name] + with tempfile.TemporaryDirectory(prefix="deltawire-v7-skill-") as raw: + repo = Path(raw) + (repo / ".deltawire/schemas").mkdir(parents=True) + (repo / ".deltawire/plans").mkdir(parents=True) + (repo / ".deltawire/config.json").write_text( + json.dumps( + { + "version": "deltawire.config.v1", + "plans_dir": ".deltawire/plans", + "schemas_dir": ".deltawire/schemas", + "state_file": ".deltawire/state.json", + "limits": { + "max_plan_bytes": 1048576, + "max_schema_bytes": 1048576, + "max_records": 100000, + "max_output_bytes": 104857600, + }, + }, + sort_keys=True, + ) + + "\n" + ) + (repo / ".deltawire/schemas/example.schema.json").write_text( + json.dumps(permissive_schema(), sort_keys=True) + "\n" + ) + plan = repo / ".deltawire/plans" / example.name + shutil.copy2(example, plan) + passes = [] + rendered = [] + for _ in range(2): + lifecycle = [run(binary, repo, name, plan) for name in ("validate", "render", "check")] + output_path = repo / json.loads(plan.read_text())["output"]["path"] + output_bytes = output_path.read_bytes() if output_path.is_file() else b"" + try: + records = [json.loads(line) for line in output_bytes.decode().splitlines()] + except (UnicodeDecodeError, json.JSONDecodeError): + records = [] + clean = all( + item["exit_code"] == 0 and not item["stderr"].strip() + for item in lifecycle + ) + passes.append(clean and records == expected) + rendered.append(hashlib.sha256(output_bytes).hexdigest()) + return { + "example": example.name, + "example_sha256": sha(example), + "expected_records": expected, + "record_count": len(expected), + "runs_pass": passes, + "rendered_sha256": rendered, + "repeated_byte_determinism": len(set(rendered)) == 1, + "status": "pass" if all(passes) and len(set(rendered)) == 1 else "fail", + } + + +def negative_controls(binary: Path, example: Path) -> dict: + with tempfile.TemporaryDirectory(prefix="deltawire-v7-negative-") as raw: + repo = Path(raw) + (repo / ".deltawire/plans").mkdir(parents=True) + (repo / ".deltawire/schemas").mkdir(parents=True) + (repo / ".deltawire/config.json").write_text(json.dumps({ + "version": "deltawire.config.v1", "plans_dir": ".deltawire/plans", + "schemas_dir": ".deltawire/schemas", "state_file": ".deltawire/state.json", + "limits": {"max_plan_bytes": 1048576, "max_schema_bytes": 1048576, "max_records": 100000, "max_output_bytes": 104857600}, + }, sort_keys=True) + "\n") + (repo / ".deltawire/schemas/example.schema.json").write_text(json.dumps(permissive_schema()) + "\n") + unsupported = json.loads(example.read_text()) + unsupported["undocumented_field"] = True + unsupported_path = repo / ".deltawire/plans/unsupported.dw.json" + unsupported_path.write_text(json.dumps(unsupported) + "\n") + malformed_path = repo / ".deltawire/plans/malformed.dw.json" + malformed_path.write_text('{"version":') + unsupported_run = run(binary, repo, "validate", unsupported_path) + malformed_run = run(binary, repo, "validate", malformed_path) + checks = { + "unsupported_field_rejected": unsupported_run["exit_code"] != 0, + "malformed_json_rejected": malformed_run["exit_code"] != 0, + } + return {"status": "pass" if all(checks.values()) else "fail", "checks": checks} + + +def build(skill: Path, binary: Path) -> dict: + examples = [verify_example(binary, path) for path in sorted((skill / "examples").glob("*.dw.json"))] + negatives = negative_controls(binary, skill / "examples/range.dw.json") + required = set(EXPECTED) + present = {item["example"] for item in examples} + checks = { + "binary_exists": binary.is_file(), + "skill_file_exists": (skill / "SKILL.md").is_file(), + "reference_exists": (skill / "references/plan-v1.md").is_file(), + "examples_exact": present == required, + "examples_pass": present == required and all(item["status"] == "pass" for item in examples), + "negative_controls_pass": negatives["status"] == "pass", + "zero_model_calls": True, + "credentials_absent": True, + } + return { + "schema_version": "deltawire-skill-conformance.v1", + "status": "pass" if all(checks.values()) else "fail", + "checks": checks, + "binary_sha256": sha(binary) if binary.is_file() else None, + "skill_tree_sha256": tree_hash(skill), + "harbor_skill_digest": harbor_digest(skill), + "skill_sha256": sha(skill / "SKILL.md"), + "reference_sha256": sha(skill / "references/plan-v1.md"), + "examples": examples, + "negative_controls": negatives, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--skill", required=True) + parser.add_argument("--binary", required=True) + parser.add_argument("--report", required=True) + args = parser.parse_args() + report = build(Path(args.skill).resolve(), Path(args.binary).resolve()) + target = Path(args.report) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + if report["status"] != "pass": + raise SystemExit(1) + print(json.dumps({"status": report["status"], "skill_tree_sha256": report["skill_tree_sha256"]}, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/scripts/v7/test_runner.py b/labs/20-deltawire/eval/scripts/v7/test_runner.py new file mode 100644 index 000000000..58743a132 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v7/test_runner.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +import json +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import runner + + +class RunnerTest(unittest.TestCase): + def test_only_pair_v7_mode_is_exposed(self): + process = subprocess.run( + ["python3", str(Path(runner.__file__)), "probe-v7", "--manifest", "x", "--approve-manifest-sha", "x", "--pair-id", runner.PAIR_ID], + capture_output=True, text=True, + ) + self.assertNotEqual(process.returncode, 0) + self.assertIn("invalid choice", process.stderr) + + def test_commands_freeze_d1_skill_then_b0_without_skill(self): + manifest = {"harbor_command": ["uvx", "--from", "harbor==0.20.0", "harbor"], "agent_import": "agent:Class", "model": "model"} + d1 = runner.command(manifest, "D1", Path("jobs/D1")) + b0 = runner.command(manifest, "B0", Path("jobs/B0")) + self.assertIn("--skill", d1) + self.assertNotIn("--skill", b0) + self.assertEqual(d1[:4], b0[:4]) + + def test_nonempty_root_consumes_attempt(self): + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + runner.refuse_nonempty(root) + (root / "partial").write_text("x") + with self.assertRaisesRegex(RuntimeError, "consumed"): + runner.refuse_nonempty(root) + + def test_approval_exact_and_excludes_other_execution(self): + with tempfile.TemporaryDirectory() as raw: + path = Path(raw) / "approval.json" + value = { + "schema_version": "deltawire-v7-pair-approval.v1", + "authorization": runner.APPROVAL_TEXT, + "pair_id": runner.PAIR_ID, + "order": ["D1", "B0"], + "manifest_sha256": "abc", + "rerun_authorized": False, + "six_run_canary_authorized": False, + "full_run_authorized": False, + "READY_FOR_72": False, + } + path.write_text(json.dumps(value)) + self.assertEqual(runner.verify_approval(path, "abc")["authorization"], runner.APPROVAL_TEXT) + value["authorization"] = "approve something else" + path.write_text(json.dumps(value)) + with self.assertRaises(RuntimeError): + runner.verify_approval(path, "abc") + + def test_d1_failure_withholds_b0_by_loop_contract(self): + arms_run = [] + def fake_run(_manifest, arm, _jobs, _dry): + arms_run.append(arm) + return {"arm": arm, "status": "completed"} + with mock.patch.object(runner, "parse_args", return_value=type("A", (), {"pair_id": runner.PAIR_ID, "manifest": "m", "approve_manifest_sha": "s", "dry_run": False, "approval": "a"})()), \ + mock.patch.object(runner, "verify_manifest", return_value={}), \ + mock.patch.object(runner, "refuse_nonempty"), \ + mock.patch.object(runner, "verify_binary"), \ + mock.patch.object(runner, "verify_image"), \ + mock.patch.object(runner, "verify_approval"), \ + mock.patch.object(runner, "run_one", side_effect=fake_run), \ + mock.patch.object(runner, "finalize", return_value={"status": "fail"}), \ + mock.patch.object(runner, "write"): + with self.assertRaisesRegex(RuntimeError, "later arm withheld"): + runner.main() + self.assertEqual(arms_run, ["D1"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/labs/20-deltawire/eval/scripts/v7/test_skill.py b/labs/20-deltawire/eval/scripts/v7/test_skill.py new file mode 100644 index 000000000..109d63ef2 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v7/test_skill.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +import json +import os +import tempfile +import unittest +from pathlib import Path + +from skill_conformance import EXPECTED, build, harbor_digest, tree_hash + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[4] +SKILL = ROOT / "labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire" + + +class SkillBundleTest(unittest.TestCase): + def test_frontmatter_and_host_neutral_guidance(self): + text = (SKILL / "SKILL.md").read_text() + self.assertTrue(text.startswith("---\nname: deltawire\ndescription:")) + self.assertEqual(text.count("\n---\n"), 1) + lowered = text.lower() + self.assertNotIn("gemini skills list", lowered) + self.assertNotIn("range-large", lowered) + self.assertNotIn("500", lowered) + self.assertIn("not inspect or reverse\nengineer", lowered) + self.assertIn("references/plan-v1.md", text) + for name in ("range", "matrix", "rows", "variants"): + self.assertIn(f"examples/{name}.dw.json", text) + + def test_reference_has_actual_parser_fields(self): + text = (SKILL / "references/plan-v1.md").read_text() + for token in ( + "version", "record_schema", "output.path", "output.format", "defaults", + "sets", "generators", "assertions.count", "matrix", "rows", "variants", + "start", "end_exclusive", "step", "${name}", + ): + self.assertIn(token, text) + self.assertNotIn("range-large", text) + self.assertNotIn("500", text) + parser = (ROOT / "labs/20-deltawire/internal/plan/plan.go").read_text() + matrix = (ROOT / "labs/20-deltawire/internal/engine/matrix.go").read_text() + for source_token in ("RecordSchema", "Defaults", "Sets", "Generators", "Assertions", "EndExclusive", "Variants", "Rows"): + self.assertIn(source_token, parser) + self.assertIn("${", matrix) + + def test_examples_are_generic_and_exact(self): + self.assertEqual({p.name for p in (SKILL / "examples").glob("*.dw.json")}, set(EXPECTED)) + for path in (SKILL / "examples").glob("*.dw.json"): + value = json.loads(path.read_text()) + raw = path.read_text().lower() + self.assertNotIn("range-large", raw) + self.assertNotIn("500", raw) + self.assertEqual(value["version"], "deltawire.plan.v1") + + def test_digest_matches_harbor_algorithm(self): + self.assertEqual(harbor_digest(SKILL), "sha256:" + tree_hash(SKILL)) + + @unittest.skipUnless(os.environ.get("DELTAWIRE_TEST_BINARY"), "set DELTAWIRE_TEST_BINARY for binary conformance") + def test_actual_binary_conformance(self): + report = build(SKILL, Path(os.environ["DELTAWIRE_TEST_BINARY"])) + self.assertEqual(report["status"], "pass", json.dumps(report, indent=2)) + range_result = next(item for item in report["examples"] if item["example"] == "range.dw.json") + self.assertIsInstance(range_result["expected_records"][0]["index"], int) + + +if __name__ == "__main__": + unittest.main() diff --git a/labs/20-deltawire/eval/scripts/v7/test_treatment.py b/labs/20-deltawire/eval/scripts/v7/test_treatment.py new file mode 100644 index 000000000..207e555f2 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v7/test_treatment.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from treatment import build, tree_hash + + +def write(path: Path, value) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(value, str): + path.write_text(value) + else: + path.write_text(json.dumps(value) + "\n") + + +def tool(name, args, status="success", output="", exit_code=0): + response = {"output": output, "exit_code": exit_code} + return {"toolCalls": [{"name": name, "args": args, "status": status, "result": [{"functionResponse": {"response": response}}]}]} + + +class TreatmentTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + self.trial = self.root / "trial" + self.artifacts = self.trial / "artifacts" + self.skill = self.root / "skill/deltawire" + write(self.skill / "SKILL.md", "---\nname: deltawire\ndescription: test\n---\n") + self.spec = {"authoritative_schema": {"sha256": hashlib.sha256(b"{}\n").hexdigest()}} + paths = { + ".deltawire/plans/range-large.dw.json": {}, + ".deltawire/config.json": {}, + ".deltawire/schemas/range-large.schema.json": "{}\n", + ".deltawire/state.json": {}, + "testdata/generated/output.ndjson": '{"index":1}\n', + ".deltawire/plan-contract-receipt.json": {"status": "pass"}, + } + for name, value in paths.items(): + write(self.artifacts / name, value) + self.independent = self.root / "independent.json" + write(self.independent, {"status": "pass"}) + digest = "sha256:" + tree_hash(self.skill) + self.manifest = {"skill": {"harbor_digest": digest, "tree_sha256": tree_hash(self.skill)}} + self.lock = self.trial / "lock.json" + self.config = self.trial / "config.json" + write(self.lock, {"skills": [{"name": "deltawire", "digest": digest}]}) + write(self.config, {"agent": {"skills": [str(self.skill)]}}) + + def tearDown(self): + self.temp.cleanup() + + def trajectory(self, include_list=False): + events = [] + if include_list: + events.append(tool("run_shell_command", {"command": "gemini skills list"}, output="deltawire")) + events += [ + tool("activate_skill", {"name": "deltawire"}), + tool("read_file", {"file_path": "/task-contract.json"}), + tool("read_file", {"file_path": "references/plan-v1.md"}), + tool("read_file", {"file_path": "examples/range.dw.json"}), + tool("run_shell_command", {"command": "deltawire validate --repo . .deltawire/plans/range-large.dw.json"}), + tool("run_shell_command", {"command": "python3 /usr/local/bin/verify_plan_contract.py"}), + tool("run_shell_command", {"command": "/usr/local/bin/deltawire render --repo . .deltawire/plans/range-large.dw.json"}), + tool("run_shell_command", {"command": "deltawire check --repo . .deltawire/plans/range-large.dw.json"}), + ] + path = self.root / "trajectory.jsonl" + path.write_text("".join(json.dumps(event) + "\n" for event in events)) + return path + + def receipt(self, arm="D1", include_list=False, semantic=1): + if arm == "B0": + write(self.lock, {"skills": []}) + write(self.config, {"agent": {"skills": []}}) + return build(self.trajectory(include_list), self.trial, arm, self.spec, semantic, self.independent, self.lock, self.config, self.skill, self.manifest) + + def test_d1_pass_does_not_require_skills_list(self): + receipt = self.receipt() + self.assertEqual(receipt["status"], "pass", json.dumps(receipt, indent=2)) + self.assertFalse(receipt["diagnostics"]["gemini_skills_list_observed"]) + + def test_skills_list_is_diagnostic_only(self): + receipt = self.receipt(include_list=True) + self.assertEqual(receipt["status"], "pass") + self.assertTrue(receipt["diagnostics"]["gemini_skills_list_observed"]) + + def test_assignment_hash_mismatch_fails(self): + self.manifest["skill"]["harbor_digest"] = "sha256:" + "0" * 64 + receipt = self.receipt() + self.assertEqual(receipt["status"], "fail") + self.assertFalse(receipt["checks"]["assignment_pass"]) + + def test_each_required_gate_fails_closed(self): + base = self.trajectory().read_text().splitlines() + for index in (0, 1, 4, 6, 7): + with self.subTest(index=index): + path = self.root / f"missing-{index}.jsonl" + path.write_text("\n".join(line for i, line in enumerate(base) if i != index) + "\n") + receipt = build(path, self.trial, "D1", self.spec, 1, self.independent, self.lock, self.config, self.skill, self.manifest) + self.assertEqual(receipt["status"], "fail") + + def test_semantic_failure_fails(self): + self.assertEqual(self.receipt(semantic=0)["status"], "fail") + + def test_clean_b0_passes_without_shared_config_schema_counting_as_contamination(self): + for path in ( + self.artifacts / ".deltawire/plans/range-large.dw.json", + self.artifacts / ".deltawire/state.json", + self.artifacts / "testdata/generated/output.ndjson", + self.artifacts / ".deltawire/plan-contract-receipt.json", + ): + path.unlink() + trajectory = self.root / "b0.jsonl" + trajectory.write_text(json.dumps(tool("run_shell_command", {"command": "python3 generate.py"})) + "\n") + write(self.lock, {"skills": []}) + write(self.config, {"agent": {"skills": []}}) + receipt = build(trajectory, self.trial, "B0", self.spec, 1, self.independent, self.lock, self.config, self.skill, self.manifest) + self.assertEqual(receipt["status"], "pass") + self.assertFalse(receipt["contamination"]) + + def test_b0_activation_invocation_plan_and_state_contaminate(self): + write(self.lock, {"skills": []}) + write(self.config, {"agent": {"skills": []}}) + for scenario in ("activation", "invocation", "plan", "state"): + with self.subTest(scenario=scenario): + for path in (self.artifacts / ".deltawire/plans/range-large.dw.json", self.artifacts / ".deltawire/state.json"): + path.unlink(missing_ok=True) + event = tool("run_shell_command", {"command": "python3 generate.py"}) + if scenario == "activation": event = tool("activate_skill", {"name": "deltawire"}) + if scenario == "invocation": event = tool("run_shell_command", {"command": "deltawire validate plan.json"}) + if scenario == "plan": write(self.artifacts / ".deltawire/plans/range-large.dw.json", {}) + if scenario == "state": write(self.artifacts / ".deltawire/state.json", {}) + trajectory = self.root / f"b0-{scenario}.jsonl" + trajectory.write_text(json.dumps(event) + "\n") + receipt = build(trajectory, self.trial, "B0", self.spec, 1, self.independent, self.lock, self.config, self.skill, self.manifest) + self.assertTrue(receipt["contamination"]) + self.assertEqual(receipt["status"], "fail") + + +if __name__ == "__main__": + unittest.main() diff --git a/labs/20-deltawire/eval/scripts/v7/treatment.py b/labs/20-deltawire/eval/scripts/v7/treatment.py new file mode 100644 index 000000000..9370cf839 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v7/treatment.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Build assignment- and artifact-grounded DeltaWire v7 treatment receipts.""" +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + +V6 = Path(__file__).resolve().parents[1] / "v6" +sys.path.insert(0, str(V6)) +from shell_observation import calls, invokes_deltawire, parse # noqa: E402 + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def tree_hash(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(item for item in root.rglob("*") if item.is_file()): + digest.update(path.relative_to(root).as_posix().encode()) + digest.update(b"\0") + digest.update(sha(path).encode()) + digest.update(b"\0") + return digest.hexdigest() + + +def tool_ok(call: dict) -> bool: + return call.get("tool_status") == "success" + + +def _skill_entries(value: dict) -> list[dict]: + trials = value.get("trials") or [] + if trials and isinstance(trials[0], dict): + value = trials[0] + entries = value.get("skills") or [] + return entries if isinstance(entries, list) else [] + + +def assignment(lock_path: Path, config_path: Path, skill: Path, manifest: dict, arm: str) -> dict: + try: + lock = json.loads(lock_path.read_text()) + config = json.loads(config_path.read_text()) + except (OSError, json.JSONDecodeError): + lock, config = {}, {} + lock_skills = _skill_entries(lock) + config_skills = (config.get("agent") or {}).get("skills") or [] + expected_digest = manifest.get("skill", {}).get("harbor_digest") + expected_tree = manifest.get("skill", {}).get("tree_sha256") + local_tree = tree_hash(skill) if skill.is_dir() else None + if arm == "D1": + checks = { + "lock_exactly_one_skill": len(lock_skills) == 1, + "lock_skill_name": len(lock_skills) == 1 and lock_skills[0].get("name") == "deltawire", + "lock_skill_digest": len(lock_skills) == 1 and lock_skills[0].get("digest") == expected_digest, + "config_exactly_one_skill": len(config_skills) == 1, + "config_skill_name": len(config_skills) == 1 and Path(str(config_skills[0])).name == "deltawire", + "local_tree_hash": local_tree == expected_tree, + } + else: + checks = { + "lock_has_no_skills": lock_skills == [], + "config_has_no_skills": config_skills == [], + "local_tree_hash": local_tree == expected_tree, + } + return {"status": "pass" if all(checks.values()) else "fail", "checks": checks} + + +def build( + trajectory: Path, + trial: Path, + arm: str, + spec: dict, + semantic_exact: int, + independent_receipt: Path, + lock_path: Path, + config_path: Path, + skill: Path, + manifest: dict, +) -> dict: + trial = Path(trial) + artifacts = trial / "artifacts" + structured = calls(trajectory) + observations, lifecycle = parse(trajectory) + activation = any( + call["name"] == "activate_skill" + and str(call["args"].get("name") or call["args"].get("skill_name") or "").lower() == "deltawire" + and tool_ok(call) + for call in structured + ) + contract_access = any( + tool_ok(call) + and "/task-contract.json" in json.dumps(call.get("args") or {}) + for call in structured + if call["name"] in {"read_file", "read_many_files", "run_shell_command", "shell", "execute"} + ) + by_name = {name: [item for item in lifecycle if item["command"] == name] for name in ("validate", "render", "check")} + lifecycle_invoked = {name: bool(items) for name, items in by_name.items()} + lifecycle_not_failed = { + name: bool(items) and (items[-1]["shell_exit_code"] is None or items[-1]["shell_exit_code"] == 0) + for name, items in by_name.items() + } + paths = { + "plan": artifacts / ".deltawire/plans/range-large.dw.json", + "config": artifacts / ".deltawire/config.json", + "schema": artifacts / ".deltawire/schemas/range-large.schema.json", + "state": artifacts / ".deltawire/state.json", + "output": artifacts / "testdata/generated/output.ndjson", + "in_container_contract": artifacts / ".deltawire/plan-contract-receipt.json", + } + retained = {name: path.is_file() for name, path in paths.items()} + try: + in_container = retained["in_container_contract"] and json.loads(paths["in_container_contract"].read_text()).get("status") == "pass" + except (OSError, json.JSONDecodeError): + in_container = False + try: + independent = Path(independent_receipt).is_file() and json.loads(Path(independent_receipt).read_text()).get("status") == "pass" + except (OSError, json.JSONDecodeError): + independent = False + schema_integrity = retained["schema"] and sha(paths["schema"]) == spec["authoritative_schema"]["sha256"] + assigned = assignment(lock_path, config_path, skill, manifest, arm) + invocations = [item for item in observations if invokes_deltawire(item)] + managed_output = retained["output"] and retained["state"] + contamination = arm == "B0" and (activation or bool(invocations) or retained["plan"] or retained["state"] or managed_output) + required = { + "assignment_pass": assigned["status"] == "pass", + "structured_activation": activation, + "public_contract_access": contract_access, + **{f"{name}_invoked": lifecycle_invoked[name] for name in lifecycle_invoked}, + **{f"{name}_not_proven_failed": lifecycle_not_failed[name] for name in lifecycle_not_failed}, + **{f"retained_{name}": value for name, value in retained.items()}, + "in_container_contract_pass": in_container, + "independent_contract_pass": independent, + "schema_integrity": schema_integrity, + "semantic_exact_pass": semantic_exact == 1, + } + text = json.dumps(structured).lower() + diagnostics = { + "gemini_skills_list_observed": "gemini skills list" in text, + "direct_skill_read": "skill.md" in text, + "matching_example_read": "examples/range.dw.json" in text, + "reference_read": "references/plan-v1.md" in text, + "direct_validate_count": len(by_name["validate"]), + "failed_direct_validate_count": sum(item["shell_exit_code"] not in (None, 0) for item in by_name["validate"]), + "subprocess_bruteforce_loops": sum("for " in item["command"] and "deltawire" in item["command"] for item in observations), + "binary_introspection": any(token in text for token in ("strings /usr/local/bin/deltawire", "objdump", "strace", "go tool")), + } + if arm == "D1": + status = "pass" if all(required.values()) else "fail" + classification = "used_successfully" if status == "pass" else "attempted_failed" + else: + status = "pass" if assigned["status"] == "pass" and not contamination and semantic_exact == 1 else "fail" + classification = "not_used" if not contamination else "contaminated" + return { + "schema_version": "deltawire-treatment-receipt.v7", + "arm": arm, + "status": status, + "classification": classification, + "assignment": assigned, + "checks": required, + "diagnostics": diagnostics, + "contamination": contamination, + "shell_observations": observations, + "lifecycle_invocations": lifecycle, + "artifact_hashes": {name: sha(path) for name, path in paths.items() if path.is_file()}, + "trajectory_sha256": sha(Path(trajectory)), + } diff --git a/labs/20-deltawire/eval/scripts/v7/validate.py b/labs/20-deltawire/eval/scripts/v7/validate.py new file mode 100644 index 000000000..bb8307f7c --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v7/validate.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Static v7 boundary checks that never invoke a model.""" +import hashlib +import json +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[4] +TASK = ROOT / "labs/20-deltawire/eval/pairs/range-large-v7-r1" + + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def main(): + task = json.loads((TASK / "task-spec.json").read_text()) + public = TASK / "environment/task-contract.json" + hidden = TASK / "tests/task-spec.json" + assert (TASK / "task-spec.json").read_bytes() == public.read_bytes() == hidden.read_bytes() + assert task["record_count"] == 500 and task["family"] == "range" + assert task["output"] == {"format": "ndjson", "path": "testdata/generated/output.ndjson"} + schema = TASK / "environment/.deltawire/schemas/range-large.schema.json" + assert schema.read_bytes() == (TASK / "authoritative-schema.json").read_bytes() + assert sha(schema) == task["authoritative_schema"]["sha256"] + instruction = (TASK / "instruction.md").read_text() + prose = instruction.split("```json", 1)[0].lower() + for forbidden in ("deltawire", "skill", "treatment", "control", "measurement", "/tests/"): + assert forbidden not in prose, forbidden + assert "gemini skills list" not in (TASK / "skill/deltawire/SKILL.md").read_text().lower() + assert (TASK / "task.toml").read_text().count("operatorstack/range-large-v7-r1") == 1 + print("v7 static treatment boundary: pass") + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/scripts/v7/validate.sh b/labs/20-deltawire/eval/scripts/v7/validate.sh new file mode 100644 index 000000000..c48f31e27 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v7/validate.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../../../../.." && pwd)" +cd "$ROOT" +python3 labs/20-deltawire/eval/scripts/v6/verify_terminal_evidence_lock.py +python3 labs/20-deltawire/eval/scripts/v7/validate.py +python3 -m unittest discover -s labs/20-deltawire/eval/scripts/v7 -p 'test_*.py' -v +python3 - <<'PY' +from pathlib import Path +p=Path('labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/SKILL.md') +t=p.read_text(); assert t.startswith('---\nname: deltawire\ndescription:') and '\n---\n' in t[4:] +print('Skill frontmatter validation: pass') +PY +if [ -f labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json ]; then + test "$(python3 -c 'import json;print(json.load(open("labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json"))["status"])')" = pass +fi +if [ -f labs/20-deltawire/eval/manifests/deltawire-range-large-v7-pair.json ]; then + uv run --with harbor==0.20.0 python labs/20-deltawire/eval/scripts/v7/generate_manifest.py --verify +fi diff --git a/labs/20-deltawire/eval/scripts/validate_dataset.sh b/labs/20-deltawire/eval/scripts/validate_dataset.sh index d7628158a..a96eb2f33 100755 --- a/labs/20-deltawire/eval/scripts/validate_dataset.sh +++ b/labs/20-deltawire/eval/scripts/validate_dataset.sh @@ -21,6 +21,9 @@ python3 labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py python3 labs/20-deltawire/eval/scripts/v6/verify_terminal_evidence_lock.py python3 labs/20-deltawire/eval/scripts/v6/test_terminal_evidence_lock.py python3 labs/20-deltawire/eval/scripts/v6/test_release.py +if [ -d labs/20-deltawire/eval/scripts/v7 ]; then + bash labs/20-deltawire/eval/scripts/v7/validate.sh +fi python3 labs/20-deltawire/eval/scripts/v5/test_environment_receipt.py python3 labs/20-deltawire/eval/scripts/v5/test_artifact_manifest.py python3 labs/20-deltawire/eval/scripts/v5/validate_integration.py