Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/deltawire-lab.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,6 @@ jobs:
go-version-file: labs/20-deltawire/go.mod
- uses: actions/setup-python@v6
with:
python-version: "3.11"
python-version: "3.12"
- uses: astral-sh/setup-uv@v7
- run: bash labs/20-deltawire/scripts/validate.sh
55 changes: 55 additions & 0 deletions .github/workflows/deltawire-v7.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
name: DeltaWire v7 no-live validation

on:
push:
branches: [eval/deltawire-v7]
paths:
- "labs/20-deltawire/**"
- ".github/workflows/deltawire-v7.yml"
pull_request:
paths:
- "labs/20-deltawire/**"
- ".github/workflows/deltawire-v7.yml"

permissions:
contents: read

jobs:
validate-v7:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
with:
fetch-depth: 0
- uses: actions/setup-go@v6
with:
go-version-file: labs/20-deltawire/go.mod
cache-dependency-path: labs/20-deltawire/go.sum
- uses: actions/setup-python@v6
with:
python-version: "3.12"
- uses: astral-sh/setup-uv@v7
- name: Build and stage the frozen Linux binary
run: |
cd labs/20-deltawire
mkdir -p eval/.generated eval/pairs/range-large-v7-r1/environment/.generated
GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -o eval/.generated/deltawire-linux-amd64 ./cmd/deltawire
cp eval/.generated/deltawire-linux-amd64 eval/pairs/range-large-v7-r1/environment/.generated/deltawire
test "$(sha256sum eval/.generated/deltawire-linux-amd64 | cut -d' ' -f1)" = e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4
- name: Validate skill examples using the actual Linux binary
run: |
python3 labs/20-deltawire/eval/scripts/v7/skill_conformance.py \
--skill labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire \
--binary labs/20-deltawire/eval/.generated/deltawire-linux-amd64 \
--report /tmp/skill-conformance.json
if test -f labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json; then
cmp /tmp/skill-conformance.json labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json
fi
- name: No-live v7 validation
run: bash labs/20-deltawire/eval/scripts/v7/validate.sh
- name: Prove workflow has no model invocation
run: |
live_runner='runner.py .*pair'"-v7"
harbor_live='harbor '"run"
credential='GEMINI_API'"_KEY"
! grep -E "$live_runner|$harbor_live|$credential" .github/workflows/deltawire-v7.yml
28 changes: 28 additions & 0 deletions labs/20-deltawire/eval/docs/v7-treatment-boundary.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# DeltaWire v7 treatment boundary

V7 keeps the v6 binary, preinstalled runtime image, Harbor 0.20.0, Gemini CLI
0.51.0, model, task semantics, verifier, and credential boundary unchanged.
Only the treatment bundle and its measurement contract are versioned.

Harbor recursively copies the complete assigned skill directory into the Gemini
skill directory. The v7 assignment receipt therefore verifies the trial lock's
skill name and Harbor digest, the trial configuration's assigned skill path,
and the frozen local bundle tree. Arm labels and model prose are not evidence
of assignment.

The skill reference is grounded in the Go plan types, strict decoder, matrix,
rows and variants engines, existing fixtures, and credential-free execution of
all four examples. The incomplete standalone JSON plan schema is not treated as
the protocol authority.

`gemini skills list`, direct skill reads, reference/example reads, validation
counts, repairs, subprocess loops, and binary introspection are diagnostics.
D1 fidelity instead requires assignment, structured activation, public-contract
access, lifecycle invocations, retained artifacts, two contract checks, schema
integrity, and semantic exactness. B0 is contaminated by activation, a
structured DeltaWire invocation, a retained plan/state, or DeltaWire-managed
output; immutable config and schema files are shared infrastructure.

The runner exposes only `pair-v7`, freezes D1 before conditional B0, refuses a
nonempty result root, and requires the exact external approval receipt before a
live call. CI has no credential or live-run path.
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
{
"READY_FOR_72": false,
"agent": "preinstalled-gemini-cli-v1",
"agent_import": "agents.preinstalled_gemini_cli:PreinstalledGeminiCli",
"binary_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4",
"expected_reported_model": "gemini-3.1-pro-preview",
"failure_policy": "withhold_B0_unless_D1_passes",
"full_run_started": false,
"gemini_cli_version": "0.51.0",
"harbor_command": [
"uvx",
"--from",
"harbor==0.20.0",
"harbor"
],
"harbor_source": [
{
"module": "harbor.agents.installed.gemini_cli",
"sha256": "dc8ce81e8f41b78f00abb54cd91fd7840d147f442960ca32e68c7a760e060a2a"
},
{
"module": "harbor.skills",
"sha256": "45fa5d0f45dce8029c47eac7468dc8bdb01157cc2b7c9271503719fc157cc7f3"
}
],
"harbor_version": "0.20.0",
"image_id": "sha256:589fce58d7ddf09910876afe0198bcee0086a840af6de7c08515bbce3e8367a7",
"image_ref": "deltawire-preflight-v6:runtime-v1",
"infrastructure_commit": "a8d2d8260ceed56baf0d1cf8fcc2f40776e9f7b6",
"input_hashes": {
"labs/20-deltawire/eval/agents/preinstalled_gemini_cli.py": "dfd0d27258ac20b8778b9aacce5f302db93576d2f735c6645e489be6c6b96058",
"labs/20-deltawire/eval/manifests/preflight-v6-terminal-evidence-lock.json": "76a6ac42fe2dfc41d1af2d73502c11724ed632a21e1d8d10b065e9bc619b7fcc",
"labs/20-deltawire/eval/matrices/72-run-matrix.json": "f9dcddc4b7336418c7e2b8e7a7e51ca1e42b5a137255ff0823a732618f3568d5",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/authoritative-schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/Dockerfile": "cd768e64efba435604b265088bce6269e8dbd09e54bf51a4c5bf0dd94d10c05f",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/deltawire-environment-expectations.json": "03fe7a40964e908f2fb3736add8b8bea890b407d837f2b185d92bd8d85d4d114",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/environment_receipt.py": "3ab3bbcbfae2093ac658063d1e26f26b43c35dfdb81d8ddae5c9989f967ba623",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/task-contract.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/instruction.md": "549f4047b1a4ba4f7a3f3dce9e60d1d70b141caa189aafdba942b15e6d022aee",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/SKILL.md": "e7d3a1e8ccef1b72338d5dac7a7b7ac412164846018ab55b7f2ecadf5a07a89a",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/matrix.dw.json": "56d16a5b20086c8202cbc1bbdccf41d467a5d4c3806d701227ebcdbf72e54b1a",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/range.dw.json": "b31e078409547f4fbb5a855f3bf0b1bd6f52386696c07401622f34d224fe148b",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/rows.dw.json": "e172cc7d6224e4566a0cd46a73465392976338fea0e225512a089322ba81aca3",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/variants.dw.json": "3183405b76e11874309728f3a4025687af6c54add8d2b97b94daff2d6ef8cdca",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/references/plan-v1.md": "f78d41266db73592526a3c5e7d6e4c883f549f99b01c6387c7f5337fc269dd5e",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/task.toml": "4adc78ba9c59f8c743770ee9b1b31a55364e7303010e3c3f109da3b3d61d96e3",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/semantic_oracle.py": "94b30fac117f40e977851966b5228e6c372bb48cbf2e5c29c24f428e1f98e91d",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/test.sh": "6f7cfbefdf7eaf3a082fd0c88f264ba3d41bfe84fc53498df4464bef66dc116e",
"labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json": "4d0e142a1bc801642b5de8c77b9cbebc9680a42f6f64da95b3d48b8624fe7c46",
"labs/20-deltawire/eval/runtime/preinstalled-gemini-v1.json": "19d9bf6ed82d3f6fa8e9bdad65bb74538ef9d7d161afb3f1f76771ac2e6b0c2d",
"labs/20-deltawire/eval/scripts/semantic_oracle.py": "94b30fac117f40e977851966b5228e6c372bb48cbf2e5c29c24f428e1f98e91d",
"labs/20-deltawire/eval/scripts/v6/artifact_manifest.py": "722f71bfbec56cbdbaea85a2f975aaa61f58d2dbbbd41e4a6b02741f3ac90ca0",
"labs/20-deltawire/eval/scripts/v6/setup_receipt.py": "c57a5a614663e4b3b81f4e1b5929c3aadfbb3a319198c9700213da0308a3d634",
"labs/20-deltawire/eval/scripts/v6/shell_observation.py": "e5269e06350147715505a76b8e83235abecd5af34eba1c8837e19b22b0e4c9fe",
"labs/20-deltawire/eval/scripts/v6/trial_status.py": "47fea192f0853a3ea880d992c27934e87cd19bc84a7a2c01446d3e9a0be8379a",
"labs/20-deltawire/eval/scripts/v6/verify_terminal_evidence_lock.py": "5adf6d90f641fb80fea894709f1a6e219e1b430454a11d19fc4c1c402bb0affe",
"labs/20-deltawire/eval/scripts/v7/generate_manifest.py": "60fbcd9a31514b94f606e87b7df07c4442259a9ed15c00f169b344675596b1c6",
"labs/20-deltawire/eval/scripts/v7/runner.py": "cc551f8f943f39c767c0cc88bc07f584972777b4d775ca586d3502eb653399b8",
"labs/20-deltawire/eval/scripts/v7/skill_conformance.py": "31617b88fef9185635ea3bc78f60368df8a894212a804c4f5e1d60b2365d9491",
"labs/20-deltawire/eval/scripts/v7/test_runner.py": "6e0ca10b09f1fd2624bbbe70892c9faf31ef630644f710ba9cd05fbaa92556ab",
"labs/20-deltawire/eval/scripts/v7/test_skill.py": "6fd6b49e06de82e4af7a97a449a9981fd860dca96c806c069834cf96287d288e",
"labs/20-deltawire/eval/scripts/v7/test_treatment.py": "4f2cb7744ce65d5cfe8ec1da8b1aca804609977a44c20fdbafa12397110409ce",
"labs/20-deltawire/eval/scripts/v7/treatment.py": "5587514cd678223105a2aff6b1a01f58fff6b6e138d06b9461db0fc7f77e7ee8",
"labs/20-deltawire/eval/scripts/v7/validate.py": "c96eadbac126fb6546816f83513e3a38dea0b29091b43fec620b929768e1e06a",
"labs/20-deltawire/eval/scripts/v7/validate.sh": "a24c183a3824df10cf5dffcdd07c0bd336b7d51705a9e311127587d56aa87200",
"labs/20-deltawire/eval/scripts/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af"
},
"integration_commit": "82a7963bdd98ca41435e756fb4212b3b37b19420",
"model": "google/gemini-3.1-pro-preview",
"order": [
"D1",
"B0"
],
"pair_id": "range-large/v7-r1",
"result_root": "labs/20-deltawire/eval/results/range-large-v7-r1",
"schema_version": "deltawire-range-large-v7-pair-manifest.v1",
"six_run_canary_started": false,
"skill": {
"conformance_sha256": "4d0e142a1bc801642b5de8c77b9cbebc9680a42f6f64da95b3d48b8624fe7c46",
"harbor_digest": "sha256:a841ca12dc687b1e467f90d5d9f5ccef120bc810742fab693e8e976135a75984",
"reference_sha256": "f78d41266db73592526a3c5e7d6e4c883f549f99b01c6387c7f5337fc269dd5e",
"skill_sha256": "e7d3a1e8ccef1b72338d5dac7a7b7ac412164846018ab55b7f2ecadf5a07a89a",
"tree_sha256": "a841ca12dc687b1e467f90d5d9f5ccef120bc810742fab693e8e976135a75984"
},
"source_matrix_pair_id": "range-large/r1",
"source_matrix_positions_zero_based": [
36,
37
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": false,
"properties": {
"index": {
"maximum": 500,
"minimum": 1,
"type": "integer"
}
},
"required": [
"index"
],
"title": "range-large",
"type": "object"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"limits": {
"max_output_bytes": 104857600,
"max_plan_bytes": 1048576,
"max_records": 100000,
"max_schema_bytes": 1048576
},
"plans_dir": ".deltawire/plans",
"schemas_dir": ".deltawire/schemas",
"state_file": ".deltawire/state.json",
"version": "deltawire.config.v1"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": false,
"properties": {
"index": {
"maximum": 500,
"minimum": 1,
"type": "integer"
}
},
"required": [
"index"
],
"title": "range-large",
"type": "object"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
*
!.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
FROM node:22.23.1-bookworm-slim@sha256:8607a9064d4a571140998ae9e52a3b3fcf9cff361d04642d5971e6cd76d39e27
ARG GEMINI_CLI_VERSION=0.51.0
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates python3 \
&& rm -rf /var/lib/apt/lists/*
RUN npm install --global --omit=dev "@google/gemini-cli@${GEMINI_CLI_VERSION}" \
&& test "$(node --version)" = "v22.23.1" \
&& test "$(npm --version)" = "10.9.8" \
&& test "$(gemini --version)" = "${GEMINI_CLI_VERSION}" \
&& test "$(node -p 'require("/usr/local/lib/node_modules/@google/gemini-cli/package.json").version')" = "${GEMINI_CLI_VERSION}"
COPY .generated/deltawire /usr/local/bin/deltawire
COPY task-contract.json /task-contract.json
COPY .deltawire /.deltawire
COPY verify_plan_contract.py /usr/local/bin/verify_plan_contract.py
COPY environment_receipt.py /usr/local/bin/environment_receipt.py
COPY deltawire-environment-expectations.json /deltawire-environment-expectations.json
RUN mkdir -p /root/.gemini \
&& printf '%s\n' '{"experimental":{"skills":true}}' > /root/.gemini/settings.json \
&& chmod 0755 /usr/local/bin/deltawire /usr/local/bin/verify_plan_contract.py /usr/local/bin/environment_receipt.py \
&& chmod a-w /task-contract.json /.deltawire/config.json /.deltawire/schemas/*.json /deltawire-environment-expectations.json \
&& command -v node \
&& command -v npm \
&& command -v gemini \
&& command -v deltawire \
&& deltawire version
WORKDIR /
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"binary": {
"path": "/usr/local/bin/deltawire",
"realpath": "/usr/local/bin/deltawire",
"sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4",
"version": "deltawire version dev"
},
"files": {
"config": {
"path": "/.deltawire/config.json",
"sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f"
},
"public_contract": {
"path": "/task-contract.json",
"sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5"
},
"schema": {
"path": "/.deltawire/schemas/range-large.schema.json",
"sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5"
}
},
"schema_version": "deltawire-environment-expectations.v1"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""Atomically emit deterministic DeltaWire environment evidence."""
import argparse,hashlib,json,os,stat,subprocess,tempfile
from pathlib import Path

def sha(path):
h=hashlib.sha256()
with Path(path).open("rb") as f:
for chunk in iter(lambda:f.read(1048576),b""):h.update(chunk)
return h.hexdigest()
def atomic_write(path,data):
target=Path(path);target.parent.mkdir(parents=True,exist_ok=True)
fd,tmp=tempfile.mkstemp(prefix=f".{target.name}.",suffix=".tmp",dir=target.parent)
try:
with os.fdopen(fd,"w",encoding="utf-8") as f:f.write(data);f.flush();os.fsync(f.fileno())
os.replace(tmp,target)
except BaseException:
try:os.unlink(tmp)
except FileNotFoundError:pass
raise
def build(expectations):
expected=json.loads(Path(expectations).read_text());binary=Path(expected["binary"]["path"])
exists=binary.exists();lst=binary.lstat() if exists else None
regular=bool(lst and stat.S_ISREG(lst.st_mode));symlink=binary.is_symlink() if exists else False
executable=exists and os.access(binary,os.X_OK);realpath=str(binary.resolve()) if exists else None
try:version=subprocess.run([str(binary),"version"],capture_output=True,text=True,check=False) if exists else None
except OSError:version=None
files={}
for name,item in sorted(expected["files"].items()):
path=Path(item["path"]);actual=sha(path) if path.is_file() else None
files[name]={"path":item["path"],"exists":path.is_file(),"actual_sha256":actual,"expected_sha256":item["sha256"],"hash_match":actual==item["sha256"]}
binary_hash=sha(binary) if regular else None
checks={"binary_exists":exists,"binary_regular":regular,"binary_not_symlink":not symlink,"binary_executable":executable,
"binary_path":str(binary)==expected["binary"]["path"],"binary_realpath":realpath==expected["binary"]["realpath"],
"binary_hash":binary_hash==expected["binary"]["sha256"],"version_exit_0":bool(version and version.returncode==0),
"version_exact":bool(version and version.stdout.splitlines() and version.stdout.splitlines()[0].strip()==expected["binary"]["version"]),
**{f"{name}_hash":item["hash_match"] for name,item in files.items()}}
return {"schema_version":"deltawire-environment-receipt.v1","binary":{"path":str(binary),"realpath":realpath,"exists":exists,
"is_regular":regular,"is_symlink":symlink,"executable":executable,"actual_sha256":binary_hash,
"expected_sha256":expected["binary"]["sha256"],"version_stdout":version.stdout.strip() if version else None,
"version_stderr":version.stderr.strip() if version else None,"version_exit_code":version.returncode if version else None,
"expected_version":expected["binary"]["version"]},"files":files,"checks":checks,"status":"pass" if all(checks.values()) else "fail"}
def main():
p=argparse.ArgumentParser();p.add_argument("--expectations",required=True);p.add_argument("--receipt",required=True);a=p.parse_args()
receipt=build(a.expectations);atomic_write(a.receipt,json.dumps(receipt,indent=2,sort_keys=True)+"\n")
print(json.dumps(receipt,sort_keys=True));raise SystemExit(0 if receipt["status"]=="pass" else 1)
if __name__=="__main__":main()
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{
"authoritative_schema": {
"path": ".deltawire/schemas/range-large.schema.json",
"sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5"
},
"deltawire_applicability": "expected",
"family": "range",
"generation": {
"field": "index",
"field_order": [
"index"
],
"start": 1
},
"ordering": "canonical_generation_order",
"output": {
"format": "ndjson",
"path": "testdata/generated/output.ndjson"
},
"record_count": 500,
"schema_version": "task-spec.v1",
"size": "large",
"task": "range-large"
}
Loading
Loading