Merge pull request #123 from operatorstack/codex/deltawire-supervisory-control

DeltaWire supervisory generation control

Author: bigboateng

.github/workflows/deltawire-supervision.yml

@@ -0,0 +1,40 @@
+name: DeltaWire supervisory control
+
+on:
+  push:
+    branches: [codex/deltawire-supervisory-control]
+    paths:
+      - "labs/20-deltawire/**"
+      - ".github/workflows/deltawire-supervision.yml"
+  pull_request:
+    paths:
+      - "labs/20-deltawire/**"
+      - ".github/workflows/deltawire-supervision.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  validate-supervisor:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v7
+        with:
+          go-version-file: labs/20-deltawire/go.mod
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - name: Validate synthesized supervisor and deterministic framework corpus
+        run: bash labs/20-deltawire/scripts/validate-supervision.sh
+      - name: Validate complete DeltaWire lab
+        run: bash labs/20-deltawire/scripts/validate.sh
+      - name: Confirm no live provider boundary
+        run: |
+          forbidden_harbor='harbor[[:space:]]+'"run"
+          forbidden_uvx='uvx.*har'"bor"
+          forbidden_campaign='campaign.*--'"live"
+          ! grep -E "$forbidden_harbor|$forbidden_uvx|$forbidden_campaign" .github/workflows/deltawire-supervision.yml
+          git diff --check

.github/workflows/deltawire-v7.yml

@@ -31,9 +31,13 @@ jobs:
       - uses: astral-sh/setup-uv@v7
       - name: Build and stage the frozen Linux binary
         run: |
-          cd labs/20-deltawire
+          frozen_tree="$RUNNER_TEMP/deltawire-v7-terminal-evidence"
+          git worktree add --detach "$frozen_tree" deltawire-v7-terminal-evidence
+          cd "$frozen_tree/labs/20-deltawire"
+          GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -o "$RUNNER_TEMP/deltawire-linux-amd64" ./cmd/deltawire
+          cd "$GITHUB_WORKSPACE/labs/20-deltawire"
           mkdir -p eval/.generated eval/pairs/range-large-v7-r1/environment/.generated
-          GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -o eval/.generated/deltawire-linux-amd64 ./cmd/deltawire
+          cp "$RUNNER_TEMP/deltawire-linux-amd64" eval/.generated/deltawire-linux-amd64
           cp eval/.generated/deltawire-linux-amd64 eval/pairs/range-large-v7-r1/environment/.generated/deltawire
           test "$(sha256sum eval/.generated/deltawire-linux-amd64 | cut -d' ' -f1)" = e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4
       - name: Validate skill examples using the actual Linux binary

.github/workflows/deltawire-v8.yml

@@ -30,9 +30,13 @@ jobs:
           python-version: "3.12"
       - name: Stage the frozen Linux binary
         run: |
-          cd labs/20-deltawire
+          frozen_tree="$RUNNER_TEMP/deltawire-v7-terminal-evidence"
+          git worktree add --detach "$frozen_tree" deltawire-v7-terminal-evidence
+          cd "$frozen_tree/labs/20-deltawire"
+          GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -o "$RUNNER_TEMP/deltawire-linux-amd64" ./cmd/deltawire
+          cd "$GITHUB_WORKSPACE/labs/20-deltawire"
           mkdir -p eval/.generated
-          GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -o eval/.generated/deltawire-linux-amd64 ./cmd/deltawire
+          cp "$RUNNER_TEMP/deltawire-linux-amd64" eval/.generated/deltawire-linux-amd64
           test "$(sha256sum eval/.generated/deltawire-linux-amd64 | cut -d' ' -f1)" = e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4
           cd ../..
           python3 labs/20-deltawire/eval/scripts/v8/generate_tasks.py

go.work.sum

@@ -1,4 +1,5 @@
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

labs/20-deltawire/GAPS.md

@@ -0,0 +1,50 @@
+# Supervisory generation-control gaps
+
+Status: non-blocking follow-ups to the supervisory-control implementation in PR #123.
+
+The main concept is working: a synthesized supervisor controls compile, stage, verify,
+promote, recovery, and abstention; strict YAML compiles to the existing JSON plan IR;
+an independent evaluator checks the generated bytes; and tested failures preserve the
+previous artifact. The deterministic evaluation and CI require no model calls.
+
+These items are hardening or measurement improvements, not merge blockers.
+
+## Higher-priority hardening
+
+1. **Crash-durable promotion recovery.** Promotion has in-process rollback, but there is
+   no durable journal spanning a process or machine crash between rename, state, and
+   final-receipt writes. A future change should add a small write-ahead promotion record
+   and recovery test matrix.
+2. **Stronger replay closure.** Stored receipts recheck the decision chain and released
+   output. They could additionally bind and revalidate every retained lifecycle artifact,
+   including the compiled plan, reference output, staged output, and engine state.
+3. **Narrow schema TOCTOU exposure.** Freeze schema bytes once and pass that immutable
+   value through staging and verification rather than reopening the source path at later
+   phases.
+4. **Pre-materialization resource bounds.** Reject obviously oversized finite products
+   from their cardinality before expanding rows or compiling a plan.
+
+## Lower-priority completeness
+
+5. Preserve exact integer condition comparisons above JavaScript's safe-integer range;
+   the current condition evaluator uses floating-point numeric comparison.
+6. Reject additional ambiguous YAML scalar spellings if interoperability demands a
+   narrower lexical format.
+7. Increase common-mode independence between the compiler and reference evaluator by
+   splitting any remaining shared low-level expansion helpers.
+8. Add an explicit direct-spec order assertion. Full byte comparison already detects
+   ordering differences, but the source language does not expose order as an assertion.
+9. Harden artifact-path handling against concurrent symlink replacement and other
+   filesystem races beyond the current path and staging checks.
+10. Expand the Pareto report with compiled-plan bytes, timing, and mutation-coverage
+    metrics if those measurements become useful for selecting later AI candidates.
+
+## Explicit boundaries
+
+- The supervisor cannot infer incorrectly approved or unstated intent; external approval
+  or a task oracle remains necessary.
+- The current proof is for the declared finite-state model and tested runtime boundary,
+  not a claim against arbitrary operating-system, hardware, or privileged-adversary faults.
+- The deterministic comparison establishes correctness and interface size only. It does
+  not establish model-token or cost savings without a later controlled AI evaluation.
+

labs/20-deltawire/README.md

@@ -1,3 +1,22 @@
 # DeltaWire
 
 In an exploratory 24-run pilot, the DeltaWire treatment used 24.2% fewer aggregate input-plus-output tokens than control. Both arms received full reward under the pilot’s existing verifiers. Semantic oracles, per-run pairing, and treatment-compliance evidence are being hardened before the confirmatory run.
+
+## Supervised direct specifications
+
+`deltawire supervise` accepts a strict, non-executable
+`deltawire.direct.v1` YAML specification, compiles it to the existing JSON
+plan, renders only in private staging, and independently interprets the direct
+specification. The synthesized finite-state supervisor is the sole authority
+that can atomically publish matching output:
+
+```text
+deltawire supervise --repo . dataset.dw.yaml
+deltawire supervise --repo . --dry-run dataset.dw.yaml
+```
+
+Failures and declared irregular workloads preserve the previous output and
+produce an abstained or failed-safe receipt under
+`.deltawire/supervision/runs/`. Existing JSON plan commands remain unchanged.
+See [the supervisory boundary](docs/12-supervisory-generation-control.md) for
+the state model, hazards, and deterministic framework-admission rules.

labs/20-deltawire/docs/12-supervisory-generation-control.md

@@ -0,0 +1,76 @@
+# DeltaWire supervisory generation control
+
+## Boundary
+
+The agent-facing mechanism proposes a finite dataset specification. It never
+owns the final output path. `deltawire supervise` freezes the specification,
+compiles it to the unchanged `deltawire.plan.v1` IR, renders in a private
+staging repository, and compares that result byte-for-byte with a separately
+implemented reference evaluator. Only the synthesized supervisor can enable
+atomic promotion.
+
+The finite plant is modeled as a discrete-event system. Synthesis removes
+forbidden states, states with unsafe uncontrollable successors, and states that
+cannot reach a marked terminal. The resulting policy is maximally permissive
+within the declared model: a controllable event is disabled only when it leaves
+the safe, nonblocking region.
+
+This proves artifact-release safety relative to an approved specification. It
+cannot prove that an approved specification captures an unstated human intent.
+That residual risk requires external approval, examples, or a task oracle.
+
+## Modes and safe terminals
+
+- `direct_spec`: parse and normalize the strict YAML proposal.
+- `json_ir`: execute the compiled, stable plan-v1 representation.
+- `recovery`: retain evidence and expose only safe recovery actions.
+- `abstained`: preserve the previous output when no admissible path succeeds.
+- `failed_safe`: preserve the previous output after an authority/evidence fault.
+
+`released`, `abstained`, and `failed_safe` are marked terminal states. Unknown,
+missing, stale, replayed, or hash-inconsistent evidence can never enable
+publication.
+
+## Hazard/control record v1
+
+| Hazard | Prevention | Monitor and isolation | Reconfiguration / safe state | Executable evidence |
+| --- | --- | --- | --- | --- |
+| Incorrect approved intent | Make invocation approve an exact canonical spec hash | Report the residual-risk boundary; task oracles remain independent | Require a corrected, newly hashed spec | framework corpus and task-family oracles |
+| Ambiguous specification | Strict typed YAML; finite domains; mutually exclusive and exhaustive rules | Parser/evaluator diagnostics | abstained | direct-spec rejection tests |
+| Contract/schema tampering | Freeze canonical spec and schema hashes before compilation | Recheck hashes in the receipt chain | failed-safe | transition-chain and staging tests |
+| Compiler defect | Reference evaluator does not import compiler or DeltaWire engine | Byte comparison plus task oracle | abstained | differential fixtures and mutation tests |
+| DeltaWire runtime defect | Keep plan-v1 lifecycle validation and limits | Regeneration/check hash and reference comparison | abstained | lifecycle integration tests |
+| Common-mode verifier defect | Separate plan execution, direct evaluator, and evaluation-family oracle | Disagreement is isolated as reference failure | abstained | canonical family corpus |
+| Path escape or symlink | Existing safe-path rules and private fixed supervision root | Path resolution and regular-directory checks | failed-safe | store/path tests and CLI integration |
+| TOCTOU / partial write | Stage privately; promote by same-filesystem atomic rename | Post-promotion hash | rollback or failed-safe | publication fault injection |
+| Stale evidence | Content-addressed run identity and hash-linked transitions | Sequence, prior hash, policy hash, and state replay | failed-safe | receipt replay tests |
+| Resource exhaustion | Existing plan/schema/record/output limits; finite ranges only | Preflight counts and byte limits | abstained | limit tests |
+| Policy drift | Deterministic synthesized-policy hash in every transition | Replay against the current model | failed-safe | policy determinism tests |
+| Missing observation | No event and no transition authority | Release remains disabled | recovery then abstained | exhaustive state/event tests |
+| Unauthorized publication | Final path is absent from candidate authority; early promotion enters a forbidden state | Synthesized event policy | deny and preserve output | maximal-permissiveness and early-promotion tests |
+| Infrastructure corruption | Boundary failures are distinct from candidate failures | `DW_SUPERVISION_FAULT` and retained receipt | failed-safe | malformed state/evidence tests |
+
+## Strict direct specification v1
+
+The YAML surface supports exactly one primary construction: ordered Cartesian
+product, explicit rows, or base-plus-variants. Products may use finite ordered
+values or bounded integer ranges. Record templates preserve scalar types for an
+exact `${name}` placeholder. Structured rules use only `eq`, `ne`, `lt`, `lte`,
+`gt`, `gte`, `in`, `all`, `any`, or `otherwise`; there is no expression or code
+execution surface. Exceptions match finite domain or record fields and apply
+JSON-pointer `set`/`omit` operations.
+
+Aliases, anchors, custom tags, merge keys, duplicate keys, unknown fields,
+non-string mapping keys, multiple documents, excessive nesting, and oversized
+specifications are rejected. `applicability: should_abstain` deliberately
+selects the safe abstention path.
+
+## Deterministic admission
+
+`eval/supervision/evaluate_frameworks.py` compares strict YAML with a
+hand-authored JSON plan on the same semantic cases. It performs no model or
+provider call. A framework becomes an AI-evaluation candidate only when all
+applicable outputs are exact and deterministic, all supervisory tests pass,
+irregular work abstains, and YAML is smaller than JSON on the high-fit range
+and matrix cases. This is admission to a later experiment, not a token-savings
+claim.

labs/20-deltawire/eval/scripts/v8/verify_manifest.py

@@ -1,8 +1,86 @@
 #!/usr/bin/env python3
-import argparse,hashlib,json,subprocess
+import argparse
+import hashlib
+import json
+import re
+import subprocess
 from pathlib import Path
-ROOT=Path(__file__).resolve().parents[5]
-def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest()
-def main():
-    parser=argparse.ArgumentParser(); parser.add_argument("--manifest",required=True); args=parser.parse_args(); value=json.loads(Path(args.manifest).read_text()); checks={"schema":value.get("schema_version")=="deltawire-v8-campaign-manifest.v1","not_ready":value.get("READY_FOR_72") is False,"attempt_ceiling":value.get("attempt_ceiling")==26,"image_platform_manifest":str(value.get("image_platform_manifest","")).startswith("sha256:") and len(value.get("image_platform_manifest",""))==71,"schedule":value.get("schedule_sha256")==sha(ROOT/"labs/20-deltawire/eval/campaigns/v8/schedule.json"),"ancestor":subprocess.run(["git","merge-base","--is-ancestor",value.get("infrastructure_commit",""),"HEAD"],cwd=ROOT).returncode==0,"inputs":all((ROOT/path).is_file() and sha(ROOT/path)==expected for path,expected in value.get("input_hashes",{}).items()),"evidence":all((ROOT/path).is_file() and sha(ROOT/path)==expected for path,expected in value.get("evidence_hashes",{}).items())}; report={"schema_version":"deltawire-v8-manifest-verification.v1","status":"pass" if all(checks.values()) else "fail","checks":checks}; print(json.dumps(report,indent=2,sort_keys=True)); raise SystemExit(0 if report["status"]=="pass" else 1)
-if __name__=="__main__": main()
+
+
+ROOT = Path(__file__).resolve().parents[5]
+
+
+def sha_bytes(value: bytes) -> str:
+    return hashlib.sha256(value).hexdigest()
+
+
+def sha_file(path: Path) -> str:
+    return sha_bytes(path.read_bytes())
+
+
+def git_blob(commit: str, path: str) -> bytes | None:
+    result = subprocess.run(
+        ["git", "show", f"{commit}:{path}"],
+        cwd=ROOT,
+        check=False,
+        capture_output=True,
+    )
+    return result.stdout if result.returncode == 0 else None
+
+
+def frozen_hashes_match(commit: str, expected: dict[str, str]) -> bool:
+    for path, wanted in expected.items():
+        value = git_blob(commit, path)
+        if value is None or sha_bytes(value) != wanted:
+            return False
+    return True
+
+
+def live_hashes_match(expected: dict[str, str]) -> bool:
+    return all(
+        (ROOT / path).is_file() and sha_file(ROOT / path) == wanted
+        for path, wanted in expected.items()
+    )
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--manifest", required=True)
+    args = parser.parse_args()
+
+    value = json.loads(Path(args.manifest).read_text())
+    commit = value.get("infrastructure_commit", "")
+    commit_valid = bool(re.fullmatch(r"[0-9a-f]{40}", commit))
+    ancestor = (
+        commit_valid
+        and subprocess.run(
+            ["git", "merge-base", "--is-ancestor", commit, "HEAD"],
+            cwd=ROOT,
+            check=False,
+        ).returncode
+        == 0
+    )
+    schedule = git_blob(commit, "labs/20-deltawire/eval/campaigns/v8/schedule.json") if commit_valid else None
+
+    checks = {
+        "schema": value.get("schema_version") == "deltawire-v8-campaign-manifest.v1",
+        "not_ready": value.get("READY_FOR_72") is False,
+        "attempt_ceiling": value.get("attempt_ceiling") == 26,
+        "image_platform_manifest": str(value.get("image_platform_manifest", "")).startswith("sha256:")
+        and len(value.get("image_platform_manifest", "")) == 71,
+        "schedule": schedule is not None and sha_bytes(schedule) == value.get("schedule_sha256"),
+        "ancestor": ancestor,
+        "inputs": commit_valid and frozen_hashes_match(commit, value.get("input_hashes", {})),
+        "evidence": live_hashes_match(value.get("evidence_hashes", {})),
+    }
+    report = {
+        "schema_version": "deltawire-v8-manifest-verification.v1",
+        "status": "pass" if all(checks.values()) else "fail",
+        "checks": checks,
+    }
+    print(json.dumps(report, indent=2, sort_keys=True))
+    return 0 if report["status"] == "pass" else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())