DeltaWire evaluation evidence and direct-specification research plan

.github/workflows/deltawire-v8.yml

@@ -0,0 +1,62 @@
+name: DeltaWire v8 zero-call validation
+
+on:
+  push:
+    branches: [eval/deltawire-v8]
+    paths:
+      - "labs/20-deltawire/**"
+      - ".github/workflows/deltawire-v8.yml"
+  pull_request:
+    paths:
+      - "labs/20-deltawire/**"
+      - ".github/workflows/deltawire-v8.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  validate-v8:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: labs/20-deltawire/go.mod
+          cache-dependency-path: labs/20-deltawire/go.sum
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - name: Stage the frozen Linux binary
+        run: |
+          cd labs/20-deltawire
+          mkdir -p eval/.generated
+          GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -o eval/.generated/deltawire-linux-amd64 ./cmd/deltawire
+          test "$(sha256sum eval/.generated/deltawire-linux-amd64 | cut -d' ' -f1)" = e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4
+          cd ../..
+          python3 labs/20-deltawire/eval/scripts/v8/generate_tasks.py
+      - name: Validate v8 harness without a provider
+        run: bash labs/20-deltawire/eval/scripts/v8/validate.sh
+      - name: Validate historical evidence locks
+        run: |
+          python3 labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py
+          python3 labs/20-deltawire/eval/scripts/v6/verify_terminal_evidence_lock.py
+          python3 labs/20-deltawire/eval/scripts/v7/verify_terminal_evidence_lock.py
+      - name: Validate parser-grounded skill examples
+        run: |
+          python3 labs/20-deltawire/eval/scripts/v7/skill_conformance.py \
+            --skill labs/20-deltawire/eval/campaigns/v8/skill/deltawire \
+            --binary labs/20-deltawire/eval/.generated/deltawire-linux-amd64 \
+            --report /tmp/deltawire-v8-skill-conformance.json
+      - name: Production cleanliness and diff checks
+        run: |
+          bash labs/20-deltawire/scripts/check-production-cleanliness.sh
+          git diff --check
+      - name: Prove CI contains no live execution or credential reference
+        run: |
+          workflow=.github/workflows/deltawire-v8.yml
+          forbidden_harbor='harbor '"run"
+          forbidden_campaign='campaign.py '"campaign"
+          forbidden_secret='GEMINI_'"API_KEY"
+          ! grep -E "$forbidden_harbor|$forbidden_campaign|$forbidden_secret" "$workflow"

.gitignore

@@ -29,3 +29,5 @@ runner.env
 # large run corpora / trial artifacts — kept locally, not committed (for now)
 labs/11-harbor-submit/data/
 labs/11-harbor-submit/jobs/
+labs/20-deltawire/eval/results/deltawire-v8-live/
+labs/20-deltawire/eval/campaigns/v8/environment/.generated/

labs/20-deltawire/eval/agents/preinstalled_gemini_cli_v8.py

@@ -0,0 +1,72 @@
+"""Gemini CLI with frozen setup checks and an evaluation-only v8 adapter."""
+import json
+import os
+import shlex
+
+from harbor.agents.installed.gemini_cli import GeminiCli
+
+AGENT_NAME = "preinstalled-gemini-cli-v8"
+EXPECTED = {
+    "node_path": "/usr/local/bin/node",
+    "node_version": "v22.23.1",
+    "npm_version": "10.9.8",
+    "gemini_path": "/usr/local/bin/gemini",
+    "gemini_version": "0.51.0",
+    "package_version": "0.51.0",
+}
+ROOT = "/logs/artifacts/deltawire-v8"
+
+
+class PreinstalledGeminiCliV8(GeminiCli):
+    @staticmethod
+    def name() -> str:
+        return AGENT_NAME
+
+    async def install(self, environment) -> None:
+        async def checked(command: str) -> str:
+            result = await self.exec_as_agent(environment, command=command)
+            if result.return_code != 0:
+                raise RuntimeError(f"preinstalled runtime verification failed: {command}")
+            return (result.stdout or "").strip()
+
+        values = {
+            "node_path": await checked("command -v node"),
+            "node_version": await checked("node --version"),
+            "npm_version": await checked("npm --version"),
+            "gemini_path": await checked("command -v gemini"),
+            "gemini_version": await checked("gemini --version"),
+            "package_version": await checked("node -p \"require('/usr/local/lib/node_modules/@google/gemini-cli/package.json').version\""),
+        }
+        task = os.environ.get("DELTAWIRE_V8_TASK", "")
+        checks = {name: values[name] == wanted for name, wanted in EXPECTED.items()}
+        checks.update({"agent_identity": self.name() == AGENT_NAME, "task_selected": bool(task)})
+        install_command = (
+            "DELTAWIRE_V8_BINARY_SHA256=" + shlex.quote(os.environ.get("DELTAWIRE_V8_BINARY_SHA256", "")) + " "
+            "/usr/local/bin/deltawire_v8_adapter.py install-contract "
+            "--catalog /opt/deltawire-v8/contracts --task " + shlex.quote(task) + " --repo /"
+        )
+        installed = await self.exec_as_agent(environment, command=install_command)
+        checks["contract_installed"] = installed.return_code == 0
+        receipt = {
+            "agent": AGENT_NAME,
+            "checks": checks,
+            **values,
+            "schema_version": "preinstalled-gemini-v8-setup-receipt.v1",
+            "status": "pass" if all(checks.values()) else "fail",
+            "task": task,
+        }
+        payload = json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n"
+        command = f"mkdir -p {ROOT} && printf %s {shlex.quote(payload)} > {ROOT}/setup-receipt.json"
+        written = await self.exec_as_agent(environment, command=command)
+        if written.return_code != 0 or receipt["status"] != "pass":
+            raise RuntimeError("preinstalled v8 setup failed")
+        if self.skills_dir:
+            prepared = await self.exec_as_agent(
+                environment,
+                command=(
+                    "/usr/local/bin/deltawire_v8_adapter.py prepare --repo / "
+                    f"--receipt {ROOT}/pre-run-receipt.json --snapshot {ROOT}/pre-run-plan.json"
+                ),
+            )
+            if prepared.return_code != 0:
+                raise RuntimeError("v8 canonical-plan preparation failed")

labs/20-deltawire/eval/campaigns/v8/conformance/environment/.gitkeep

@@ -0,0 +1 @@
+

labs/20-deltawire/eval/campaigns/v8/conformance/instruction.md

@@ -0,0 +1 @@
+This install-only conformance task must never execute an agent or model.

labs/20-deltawire/eval/campaigns/v8/conformance/task.toml

@@ -0,0 +1,24 @@
+schema_version = "1.3"
+artifacts = []
+
+[task]
+name = "operatorstack/deltawire-v8-install-boundary"
+description = "Credential-free install-only v8 treatment boundary conformance"
+authors = []
+keywords = []
+
+[metadata]
+benchmark_result = false
+
+[verifier]
+timeout_sec = 60.0
+
+[agent]
+timeout_sec = 60.0
+
+[environment]
+docker_image = "deltawire-v8:campaign-v1"
+network_mode = "public"
+build_timeout_sec = 60.0
+os = "linux"
+mcp_servers = []

labs/20-deltawire/eval/campaigns/v8/conformance/tests/test.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+set -euo pipefail
+exit 0

labs/20-deltawire/eval/campaigns/v8/environment/Dockerfile

@@ -0,0 +1,22 @@
+FROM node:22.23.1-bookworm-slim@sha256:8607a9064d4a571140998ae9e52a3b3fcf9cff361d04642d5971e6cd76d39e27
+ARG GEMINI_CLI_VERSION=0.51.0
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates python3 \
+    && rm -rf /var/lib/apt/lists/*
+RUN npm install --global --omit=dev "@google/gemini-cli@${GEMINI_CLI_VERSION}" \
+    && test "$(node --version)" = "v22.23.1" \
+    && test "$(npm --version)" = "10.9.8" \
+    && test "$(gemini --version)" = "${GEMINI_CLI_VERSION}" \
+    && test "$(node -p 'require("/usr/local/lib/node_modules/@google/gemini-cli/package.json").version')" = "${GEMINI_CLI_VERSION}"
+COPY .generated/deltawire /usr/local/bin/deltawire
+COPY contracts /opt/deltawire-v8/contracts
+COPY adapter.py /usr/local/bin/deltawire_v8_adapter.py
+COPY finalizer.py /usr/local/bin/deltawire_v8_finalizer.py
+COPY collect.py /usr/local/bin/deltawire_v8_collect.py
+COPY verify_plan_contract.py /usr/local/bin/verify_plan_contract.py
+COPY environment_receipt.py /usr/local/bin/environment_receipt.py
+RUN mkdir -p /root/.gemini \
+    && printf '%s\n' '{"experimental":{"skills":true}}' > /root/.gemini/settings.json \
+    && chmod 0755 /usr/local/bin/deltawire /usr/local/bin/deltawire_v8_*.py /usr/local/bin/verify_plan_contract.py /usr/local/bin/environment_receipt.py \
+    && command -v deltawire && deltawire version
+WORKDIR /

labs/20-deltawire/eval/campaigns/v8/environment/adapter.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+"""Evaluation-owned contract installation and canonical-plan preparation."""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import tempfile
+from pathlib import Path
+
+SCHEMA = "deltawire-v8-adapter-receipt.v1"
+PLAN_VERSION = "deltawire.plan.v1"
+GENERATOR_KEYS = ("defaults", "sets", "generators")
+
+
+def canonical(value: object) -> bytes:
+    return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode()
+
+
+def sha_bytes(value: bytes) -> str:
+    return hashlib.sha256(value).hexdigest()
+
+
+def sha(path: Path) -> str:
+    return sha_bytes(path.read_bytes())
+
+
+def atomic_bytes(path: Path, payload: bytes, mode: int = 0o644) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    descriptor, name = tempfile.mkstemp(prefix=path.name + ".", dir=path.parent)
+    try:
+        with os.fdopen(descriptor, "wb") as handle:
+            handle.write(payload)
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.chmod(name, mode)
+        os.replace(name, path)
+    finally:
+        Path(name).unlink(missing_ok=True)
+
+
+def atomic_json(path: Path, value: object, mode: int = 0o644) -> None:
+    atomic_bytes(path, canonical(value), mode)
+
+
+def skeleton(contract: dict) -> dict:
+    task = contract["task"]
+    return {
+        "assertions": {"count": contract["record_count"]},
+        "description": f"Evaluation-owned canonical plan for {task}",
+        "generators": [],
+        "id": task,
+        "output": {
+            "format": contract["output"]["format"],
+            "path": contract["output"]["path"],
+            "pretty": False,
+        },
+        "record_schema": contract["authoritative_schema"]["path"],
+        "version": PLAN_VERSION,
+    }
+
+
+def bound_fields(contract: dict) -> dict:
+    value = skeleton(contract)
+    return {key: value[key] for key in value if key not in GENERATOR_KEYS}
+
+
+def install(catalog: Path, task: str, repo: Path) -> dict:
+    source = catalog / task
+    contract_source = source / "task-contract.json"
+    schema_source = source / "authoritative-schema.json"
+    config_source = catalog / "config.json"
+    if not all(path.is_file() for path in (contract_source, schema_source, config_source)):
+        raise RuntimeError(f"incomplete contract catalog for {task}")
+    contract = json.loads(contract_source.read_text())
+    if contract.get("task") != task:
+        raise RuntimeError("catalog task mismatch")
+    contract_target = repo / "task-contract.json"
+    config_target = repo / ".deltawire/config.json"
+    schema_target = repo / contract["authoritative_schema"]["path"]
+    for source_path, target in (
+        (contract_source, contract_target),
+        (config_source, config_target),
+        (schema_source, schema_target),
+    ):
+        atomic_bytes(target, source_path.read_bytes(), 0o444)
+    expectations = {
+        "binary": {
+            "path": "/usr/local/bin/deltawire",
+            "realpath": "/usr/local/bin/deltawire",
+            "sha256": os.environ.get("DELTAWIRE_V8_BINARY_SHA256", ""),
+            "version": "deltawire version dev",
+        },
+        "files": {
+            "config": {"path": "/.deltawire/config.json", "sha256": sha(config_target)},
+            "public_contract": {"path": "/task-contract.json", "sha256": sha(contract_target)},
+            "schema": {"path": "/" + contract["authoritative_schema"]["path"], "sha256": sha(schema_target)},
+        },
+        "schema_version": "deltawire-environment-expectations.v1",
+    }
+    atomic_json(repo / "deltawire-environment-expectations.json", expectations, 0o444)
+    return {
+        "contract_path": "/task-contract.json",
+        "contract_sha256": sha(contract_target),
+        "schema_path": "/" + contract["authoritative_schema"]["path"],
+        "schema_sha256": sha(schema_target),
+        "task": task,
+    }
+
+
+def prepare(repo: Path, receipt_path: Path, snapshot_path: Path) -> dict:
+    contract_path = repo / "task-contract.json"
+    contract = json.loads(contract_path.read_text())
+    plan = skeleton(contract)
+    plan_path = repo / ".deltawire/plans" / f"{contract['task']}.dw.json"
+    payload = canonical(plan)
+    atomic_bytes(plan_path, payload)
+    atomic_bytes(snapshot_path, payload)
+    receipt = {
+        "bound_fields": bound_fields(contract),
+        "contract_sha256": sha(contract_path),
+        "generator_owned_fields": list(GENERATOR_KEYS),
+        "plan_path": f".deltawire/plans/{contract['task']}.dw.json",
+        "pre_plan_sha256": sha_bytes(payload),
+        "schema_version": SCHEMA,
+        "status": "pass",
+        "task": contract["task"],
+    }
+    atomic_json(receipt_path, receipt)
+    return receipt
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    sub = parser.add_subparsers(dest="mode", required=True)
+    install_parser = sub.add_parser("install-contract")
+    install_parser.add_argument("--catalog", required=True)
+    install_parser.add_argument("--task", required=True)
+    install_parser.add_argument("--repo", default="/")
+    prepare_parser = sub.add_parser("prepare")
+    prepare_parser.add_argument("--repo", default="/")
+    prepare_parser.add_argument("--receipt", required=True)
+    prepare_parser.add_argument("--snapshot", required=True)
+    args = parser.parse_args()
+    if args.mode == "install-contract":
+        print(json.dumps(install(Path(args.catalog), args.task, Path(args.repo)), sort_keys=True))
+    else:
+        print(json.dumps(prepare(Path(args.repo), Path(args.receipt), Path(args.snapshot)), sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

labs/20-deltawire/eval/campaigns/v8/environment/collect.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+"""Run deterministic v8 collect-boundary receipts without suppressing evidence."""
+import subprocess
+from pathlib import Path
+
+
+def main() -> None:
+    root = Path("/logs/artifacts/deltawire-v8")
+    root.mkdir(parents=True, exist_ok=True)
+    commands = [[
+        "python3", "/usr/local/bin/environment_receipt.py",
+        "--expectations", "/deltawire-environment-expectations.json",
+        "--receipt", str(root / "environment-receipt.json"),
+    ]]
+    pre = root / "pre-run-receipt.json"
+    if pre.is_file():
+        commands.append([
+            "python3", "/usr/local/bin/deltawire_v8_finalizer.py", "finalize",
+            "--repo", "/", "--pre-receipt", str(pre),
+            "--receipt", str(root / "finalizer-receipt.json"),
+            "--retained", str(root / "retained"),
+        ])
+    else:
+        commands.append([
+            "python3", "/usr/local/bin/deltawire_v8_finalizer.py", "boundary",
+            "--repo", "/", "--receipt", str(root / "b0-boundary-receipt.json"),
+            "--retained", str(root / "retained"),
+        ])
+    codes = [subprocess.run(command, check=False).returncode for command in commands]
+    raise SystemExit(0 if all(code == 0 for code in codes) else 1)
+
+
+if __name__ == "__main__":
+    main()

labs/20-deltawire/eval/campaigns/v8/environment/contracts/config.json

@@ -0,0 +1,12 @@
+{
+  "limits": {
+    "max_output_bytes": 104857600,
+    "max_plan_bytes": 1048576,
+    "max_records": 100000,
+    "max_schema_bytes": 1048576
+  },
+  "plans_dir": ".deltawire/plans",
+  "schemas_dir": ".deltawire/schemas",
+  "state_file": ".deltawire/state.json",
+  "version": "deltawire.config.v1"
+}