Require binary-observed Pitot agent E2E

.github/scripts/pitot_e2e_report.py

@@ -22,8 +22,10 @@
 
 
 MANIFEST = json.loads((ROOT / "labs/15-pitot/adapter-verification.json").read_text(encoding="utf-8"))
+ENDPOINT_PROVENANCE = json.loads((ROOT / "labs/15-pitot/tests/endpoint-provenance.json").read_text(encoding="utf-8"))
 AGENTS = tuple(agent["id"] for agent in MANIFEST["agents"])
 AGENT_LABELS = {agent["id"]: agent["label"] for agent in MANIFEST["agents"]}
+AGENT_RECORDS = {agent["id"]: agent for agent in MANIFEST["agents"]}
 PLATFORMS = tuple(platform["id"] for platform in MANIFEST["platforms"])
 MAX_ARTIFACT_BYTES = 1_000_000
 
@@ -59,28 +61,64 @@ def validate_result(
         "status",
         "verification_mode",
         "evidence",
+        "cli",
+        "protocol",
+        "endpoint",
+        "prompt_hash",
+        "receipts",
+        "hook",
         "commit_sha",
         "run_url",
     }
     if set(value) != required:
         raise ValueError("result fields do not match schema")
-    if value["schema_version"] != 1 or value["agent"] != agent or value["platform"] != platform:
+    if value["schema_version"] != 2 or value["agent"] != agent or value["platform"] != platform:
         raise ValueError("result identity does not match artifact")
     if value["status"] not in {"pass", "fail"}:
         raise ValueError("invalid result status")
     mode = value["verification_mode"]
-    if value["status"] == "pass" and mode not in {"real_cli", "hook_subprocess"}:
-        raise ValueError("passing result requires a verification mode")
+    if value["status"] == "pass" and mode != "real_cli":
+        raise ValueError("passing result requires real_cli verification")
     if value["status"] == "fail" and mode is not None:
         raise ValueError("failed result cannot claim a verification mode")
-    allowed_evidence = {
-        "real host CLI completed the Pitot integration path",
-        "active hook subprocess produced a normalized Pitot action",
-        "E2E command failed",
-        "E2E command returned without one valid result marker",
-    }
+    allowed_evidence = {"binary-observed request, accepted response, hook, canary, and final receipts", "real-agent evidence contract failed"}
     if value["evidence"] not in allowed_evidence:
         raise ValueError("invalid evidence summary")
+    evidence_fields = ("cli", "protocol", "endpoint", "prompt_hash", "receipts", "hook")
+    if value["status"] == "fail":
+        if any(value[field] is not None for field in evidence_fields):
+            raise ValueError("failed result cannot carry passing evidence")
+    else:
+        record = AGENT_RECORDS[agent]
+        cli = value["cli"]
+        if not isinstance(cli, dict) or set(cli) != {"version", "executable", "executable_sha256", "installer", "runtime"}:
+            raise ValueError("invalid CLI installation receipt")
+        if cli["version"] != record["version"] or cli["installer"] != record["installer"]["kind"] or cli["runtime"] != record["runtime"][platform]:
+            raise ValueError("CLI receipt does not match supervised manifest")
+        cells = [cell for cell in ENDPOINT_PROVENANCE["cells"] if cell["agent"] == agent and cell["platform"] == platform]
+        if len(cells) != 1:
+            raise ValueError("missing supervised platform provenance")
+        fixture = cells[0]
+        if cli["executable_sha256"] != fixture["executable_sha256"]:
+            raise ValueError("CLI executable digest does not match binary capture")
+        if value["protocol"] != fixture["dialect"] or not re.fullmatch(r"[0-9a-f]{64}", str(value["prompt_hash"])):
+            raise ValueError("invalid prompt/protocol receipt")
+        expected_endpoint = {
+            "fixture": f"tests/endpoint-provenance.json#{agent}/{platform}",
+            "fixture_sha256": fixture["capture_sha256"],
+            "provenance": "pinned_real_cli_capture",
+            "dialect": fixture["dialect"],
+            "request": fixture["request"],
+            "response": fixture["response"],
+            "executable_sha256": fixture["executable_sha256"],
+        }
+        if value["endpoint"] != expected_endpoint:
+            raise ValueError("endpoint receipt does not match pinned real-CLI provenance")
+        receipt_fields = {"initial_prompt_observed", "tool_call_response_emitted", "tool_result_observed", "final_response_emitted", "hook_observed", "canary_result_observed", "cli_exit_zero"}
+        if not isinstance(value["receipts"], dict) or set(value["receipts"]) != receipt_fields or not all(item is True for item in value["receipts"].values()):
+            raise ValueError("incomplete causal receipts")
+        if value["hook"] != {"host": agent, "action_kind": "shell", "pitot_exit": 0}:
+            raise ValueError("invalid Pitot hook receipt")
     if not isinstance(value["commit_sha"], str) or not re.fullmatch(r"[0-9a-f]{40}", value["commit_sha"]):
         raise ValueError("invalid commit_sha")
     if not isinstance(value["run_url"], str) or not re.fullmatch(
@@ -117,18 +155,26 @@ def result_cell(result: dict[str, object] | None) -> str:
         return "⏳ Pending"
     if result["status"] == "fail":
         return "❌ Failed"
-    mode = "real CLI" if result["verification_mode"] == "real_cli" else "hook verified"
-    return f"✅ Pass · {mode}"
+    cli = result["cli"]
+    runtime = "WSL" if cli["runtime"] == "wsl" else "native"
+    dialect = result["endpoint"]["dialect"].replace("_", " ")
+    return f"✅ Pass · real CLI {cli['version']} · {runtime} · binary-observed {dialect}"
 
 
 def failed_result(agent: str, platform: str, head_sha: str, run_url: str) -> dict[str, object]:
     return {
-        "schema_version": 1,
+        "schema_version": 2,
         "agent": agent,
         "platform": platform,
         "status": "fail",
         "verification_mode": None,
-        "evidence": "E2E command failed",
+        "evidence": "real-agent evidence contract failed",
+        "cli": None,
+        "protocol": None,
+        "endpoint": None,
+        "prompt_hash": None,
+        "receipts": None,
+        "hook": None,
         "commit_sha": head_sha,
         "run_url": run_url,
     }
@@ -205,6 +251,14 @@ def download(self, url: str) -> bytes:
             raise ValueError("artifact exceeds size limit")
         return data
 
+    def list_run_artifacts(self, run_id: int) -> list[dict[str, object]]:
+        value = self.request("GET", f"/actions/runs/{run_id}/artifacts?per_page=100")
+        artifacts = value.get("artifacts") if isinstance(value, dict) else None
+        total = value.get("total_count") if isinstance(value, dict) else None
+        if not isinstance(artifacts, list) or not isinstance(total, int) or total != len(artifacts):
+            raise ValueError("unified E2E artifact listing is truncated or malformed")
+        return artifacts
+
 
 def load_artifact(
     github: GitHub,
@@ -294,7 +348,7 @@ def main() -> int:
     head_sha = workflow_run["head_sha"]
     run_url = workflow_run["html_url"]
 
-    artifacts = github.request("GET", f"/actions/runs/{workflow_run['id']}/artifacts")["artifacts"]
+    artifacts = github.list_run_artifacts(int(workflow_run["id"]))
     by_name = {artifact["name"]: artifact for artifact in artifacts if not artifact.get("expired")}
     inventory_artifact = by_name.get("pitot-e2e-inventory")
     if inventory_artifact is None:

.github/workflows/pitot-e2e-agent.yml

@@ -15,13 +15,14 @@ on:
         description: GitHub-hosted runner selected by the supervisor
         required: true
         type: string
+      capture:
+        description: Emit a redacted binary-observed candidate fixture
+        required: true
+        type: boolean
 
 permissions:
   contents: read
 
-env:
-  CLAUDE_CLI_VERSION: 2.1.217
-
 jobs:
   e2e:
     name: ${{ inputs.agent }} (${{ inputs.platform }})
@@ -34,10 +35,38 @@ jobs:
           cache-dependency-path: labs/15-pitot/pitot/go.mod
       - uses: actions/setup-node@v4
         with:
-          node-version: "20"
+          node-version: "22"
       - uses: actions/setup-python@v6
         with:
           python-version: "3.11"
+      - name: Enable supported Cursor WSL runtime
+        if: inputs.agent == 'cursor' && runner.os == 'Windows'
+        shell: powershell
+        run: |
+          wsl.exe --status
+          if (-not ((wsl.exe --list --quiet) -replace "`0", "" | Where-Object { $_.Trim() -eq "Ubuntu" })) {
+            wsl.exe --install --distribution Ubuntu --no-launch --web-download
+          }
+          $installScript = @'
+            set -euo pipefail
+            node_version=22.23.1
+            archive="node-v${node_version}-linux-x64.tar.xz"
+            curl -fsSLO "https://nodejs.org/dist/v${node_version}/${archive}"
+            curl -fsSLO "https://nodejs.org/dist/v${node_version}/SHASUMS256.txt"
+            grep " ${archive}$" SHASUMS256.txt | sha256sum --check --strict
+            tar -xJf "${archive}" -C /usr/local --strip-components=1
+            test "$(node --version)" = "v${node_version}"
+            uname -a
+            command -v python3
+          '@
+          $installPath = Join-Path $env:RUNNER_TEMP "pitot-install-wsl-node.sh"
+          $utf8NoBom = New-Object System.Text.UTF8Encoding($false)
+          [System.IO.File]::WriteAllText($installPath, ($installScript -replace "`r`n", "`n"), $utf8NoBom)
+          $drive = $installPath.Substring(0, 1).ToLowerInvariant()
+          $tail = $installPath.Substring(2).Replace("\", "/")
+          $wslInstallPath = "/mnt/$drive$tail"
+          wsl.exe --distribution Ubuntu -- bash $wslInstallPath
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
       - name: Resolve Bash runtime
         shell: bash
         run: |
@@ -46,35 +75,30 @@ jobs:
             bash_path="$(cygpath -m "$bash_path")"
           fi
           echo "PITOT_BASH=$bash_path" >> "$GITHUB_ENV"
-      - name: Install Claude CLI
-        if: inputs.agent == 'claude'
+      - name: Install and attest pinned real agent
         shell: bash
-        run: |
-          npm install -g "@anthropic-ai/claude-code@$CLAUDE_CLI_VERSION"
-          claude_bin="$(command -v claude)"
-          [[ -n "$claude_bin" ]]
-          installed_version="$("$claude_bin" --version)"
-          echo "Using $claude_bin ($installed_version)"
-          [[ "$installed_version" == "$CLAUDE_CLI_VERSION"* ]]
-          echo "CLAUDE_PATH=$claude_bin" >> "$GITHUB_ENV"
-      - name: Install Codex CLI
-        if: inputs.agent == 'codex'
-        run: npm install -g @openai/codex
-      - name: Install Cursor CLI
-        if: inputs.agent == 'cursor' && runner.os != 'Windows'
-        run: curl https://cursor.com/install -fsS | bash
+        run: >-
+          python3 labs/15-pitot/tests/install_real_agent.py
+          --agent "${{ inputs.agent }}"
+          --platform "${{ inputs.platform }}"
+          --output "${{ runner.temp }}/pitot-e2e/install.json"
       - name: Run ${{ inputs.agent }} E2E
         id: e2e
         continue-on-error: true
         timeout-minutes: 10
         shell: bash
         env:
           PITOT_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+          PITOT_E2E_PLATFORM: ${{ inputs.platform }}
+          PITOT_INSTALL_RECEIPT: ${{ runner.temp }}/pitot-e2e/install.json
+          PITOT_E2E_EVIDENCE: ${{ runner.temp }}/pitot-e2e/evidence.json
+          PITOT_CAPTURE_OUTPUT: ${{ inputs.capture && format('{0}/pitot-e2e/capture.json', runner.temp) || '' }}
         run: >-
           python labs/15-pitot/tests/run_e2e_report.py
           --agent "${{ inputs.agent }}"
           --platform "${{ inputs.platform }}"
           --output "${{ runner.temp }}/pitot-e2e/result.json"
+          --evidence "${{ runner.temp }}/pitot-e2e/evidence.json"
           -- "$PITOT_BASH" "labs/15-pitot/tests/e2e_${{ inputs.agent }}_cli_test.sh"
           < /dev/null
       - name: Upload structured E2E result
@@ -85,6 +109,14 @@ jobs:
           path: ${{ runner.temp }}/pitot-e2e/result.json
           if-no-files-found: error
           retention-days: 14
+      - name: Upload endpoint provenance candidate
+        if: always() && inputs.capture
+        uses: actions/upload-artifact@v4
+        with:
+          name: pitot-endpoint-capture-${{ inputs.agent }}-${{ inputs.platform }}
+          path: ${{ runner.temp }}/pitot-e2e/capture.json
+          if-no-files-found: error
+          retention-days: 14
       - name: Enforce E2E result
         if: always() && steps.e2e.outcome != 'success'
         shell: bash

.github/workflows/pitot-e2e.yml

@@ -12,6 +12,12 @@ on:
     branches: [main]
     paths: *pitot_paths
   workflow_dispatch:
+    inputs:
+      capture_provenance:
+        description: Capture redacted binary-observed candidates instead of trusting committed fixtures
+        required: false
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -30,7 +36,11 @@ jobs:
       - id: supervisor
         shell: bash
         run: |
-          matrix="$(python3 labs/15-pitot/scripts/pitot_adapter_supervisor.py matrix)"
+          operation="matrix"
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.capture_provenance }}" == "true" ]]; then
+            operation="capture"
+          fi
+          matrix="$(python3 labs/15-pitot/scripts/pitot_adapter_supervisor.py "$operation")"
           echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
       - name: Upload supervised adapter inventory
         uses: actions/upload-artifact@v4
@@ -50,3 +60,4 @@ jobs:
       agent: ${{ matrix.agent }}
       platform: ${{ matrix.platform }}
       runner: ${{ matrix.runner }}
+      capture: ${{ matrix.capture == 'true' }}

labs/15-pitot/adapter-verification.json

@@ -1,20 +1,20 @@
 {
-  "schema_version": 1,
+  "schema_version": 4,
   "platforms": [
     {"id": "ubuntu", "runner": "ubuntu-latest"},
     {"id": "macos", "runner": "macos-latest"},
     {"id": "windows", "runner": "windows-latest"}
   ],
   "agents": [
-    {"id": "claude", "label": "Claude"},
-    {"id": "cline", "label": "Cline"},
-    {"id": "cursor", "label": "Cursor"},
-    {"id": "codex", "label": "Codex"},
-    {"id": "copilot", "label": "GitHub Copilot CLI"},
-    {"id": "gemini", "label": "Gemini"},
-    {"id": "kimi", "label": "Kimi Code"},
-    {"id": "opencode", "label": "OpenCode"},
-    {"id": "pi", "label": "Pi"},
-    {"id": "qwen", "label": "Qwen Code"}
+    {"id": "claude", "label": "Claude", "version": "2.1.217", "executable": "claude", "installer": {"kind": "npm", "package": "@anthropic-ai/claude-code"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"},
+    {"id": "cline", "label": "Cline", "version": "3.0.46", "executable": "cline", "installer": {"kind": "npm", "package": "cline"}, "integration": "cline_bridge", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"},
+    {"id": "cursor", "label": "Cursor", "version": "2026.07.20-8cc9c0b", "executable": "agent", "installer": {"kind": "cursor_release", "package": "https://downloads.cursor.com/lab/2026.07.20-8cc9c0b"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "wsl"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"},
+    {"id": "codex", "label": "Codex", "version": "0.145.0", "executable": "codex", "installer": {"kind": "npm", "package": "@openai/codex"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"},
+    {"id": "copilot", "label": "GitHub Copilot CLI", "version": "1.0.73", "executable": "copilot", "installer": {"kind": "npm", "package": "@github/copilot"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"},
+    {"id": "gemini", "label": "Gemini", "version": "0.51.0", "executable": "gemini", "installer": {"kind": "npm", "package": "@google/gemini-cli"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"},
+    {"id": "kimi", "label": "Kimi Code", "version": "0.29.0", "executable": "kimi", "installer": {"kind": "kimi_release", "package": "https://code.kimi.com/kimi-code"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"},
+    {"id": "opencode", "label": "OpenCode", "version": "1.18.4", "executable": "opencode", "installer": {"kind": "npm", "package": "opencode-ai"}, "integration": "opencode_plugin", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"},
+    {"id": "pi", "label": "Pi", "version": "0.81.1", "executable": "pi", "installer": {"kind": "npm", "package": "@earendil-works/pi-coding-agent"}, "integration": "pi_extension", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"},
+    {"id": "qwen", "label": "Qwen Code", "version": "0.20.1", "executable": "qwen", "installer": {"kind": "npm", "package": "@qwen-code/qwen-code"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}
   ]
 }

labs/15-pitot/integrations/cline/PreToolUse

@@ -3,7 +3,7 @@ set -uo pipefail
 
 PITOT_COMMAND="${PITOT_BIN:-pitot}"
 PAYLOAD=$(cat)
-if ! TOOL=$(printf '%s' "$PAYLOAD" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("preToolUse", {}).get("tool", ""))' 2>/dev/null); then
+if ! TOOL=$(printf '%s' "$PAYLOAD" | python3 -c 'import json,sys; event=json.load(sys.stdin).get("preToolUse", {}); print(event.get("toolName") or event.get("tool", ""))' 2>/dev/null); then
   printf '{"cancel":true,"errorMessage":"Pitot received a malformed Cline hook payload"}\n'
   exit 0
 fi
@@ -15,7 +15,7 @@ fi
 
 # Cline has no matcher at this boundary. Non-shell tools are outside Pitot's
 # controllable partition and must pass through unchanged.
-if [ "$TOOL" != "execute_command" ]; then
+if [ "$TOOL" != "execute_command" ] && [ "$TOOL" != "run_commands" ]; then
   printf '{"cancel":false}\n'
   exit 0
 fi

labs/15-pitot/integrations/cline/PreToolUse.ps1

@@ -6,11 +6,12 @@ try {
     @{ cancel = $true; errorMessage = "Pitot received a malformed Cline hook payload" } | ConvertTo-Json -Compress
     exit 0
 }
-if (-not $event.preToolUse.tool) {
+$tool = if ($event.preToolUse.toolName) { $event.preToolUse.toolName } else { $event.preToolUse.tool }
+if (-not $tool) {
     @{ cancel = $true; errorMessage = "Pitot received a malformed Cline hook payload" } | ConvertTo-Json -Compress
     exit 0
 }
-if ($event.preToolUse.tool -ne "execute_command") {
+if ($tool -ne "execute_command" -and $tool -ne "run_commands") {
     @{ cancel = $false } | ConvertTo-Json -Compress
     exit 0
 }

labs/15-pitot/integrations/opencode/pitot.ts

@@ -0,0 +1,25 @@
+import { spawnSync } from "node:child_process";
+
+// OpenCode runs plugins in-process. This is the genuine synchronous
+// tool.execute.before boundary; it is not a Claude PreToolUse simulation.
+export const PitotPlugin = async () => ({
+  "tool.execute.before": async (input, output) => {
+    if (input.tool !== "bash") return;
+    const command = output.args?.command;
+    const payload = JSON.stringify({
+      hook_event_name: "PreToolUse",
+      tool_name: "Bash",
+      tool_input: { command: typeof command === "string" ? command : "" },
+    });
+    const result = spawnSync(process.env.PITOT_BIN || "pitot", ["hook", "opencode"], {
+      input: payload,
+      encoding: "utf8",
+      maxBuffer: 1024 * 1024,
+    });
+    if (result.status !== 0) {
+      throw new Error((result.stderr || "Pitot rejected the shell request").trim());
+    }
+  },
+});
+
+export default PitotPlugin;