diff --git a/.github/scripts/pitot_e2e_report.py b/.github/scripts/pitot_e2e_report.py index 30d7380dc..1d5b6d035 100644 --- a/.github/scripts/pitot_e2e_report.py +++ b/.github/scripts/pitot_e2e_report.py @@ -22,8 +22,10 @@ MANIFEST = json.loads((ROOT / "labs/15-pitot/adapter-verification.json").read_text(encoding="utf-8")) +ENDPOINT_PROVENANCE = json.loads((ROOT / "labs/15-pitot/tests/endpoint-provenance.json").read_text(encoding="utf-8")) AGENTS = tuple(agent["id"] for agent in MANIFEST["agents"]) AGENT_LABELS = {agent["id"]: agent["label"] for agent in MANIFEST["agents"]} +AGENT_RECORDS = {agent["id"]: agent for agent in MANIFEST["agents"]} PLATFORMS = tuple(platform["id"] for platform in MANIFEST["platforms"]) MAX_ARTIFACT_BYTES = 1_000_000 @@ -59,28 +61,64 @@ def validate_result( "status", "verification_mode", "evidence", + "cli", + "protocol", + "endpoint", + "prompt_hash", + "receipts", + "hook", "commit_sha", "run_url", } if set(value) != required: raise ValueError("result fields do not match schema") - if value["schema_version"] != 1 or value["agent"] != agent or value["platform"] != platform: + if value["schema_version"] != 2 or value["agent"] != agent or value["platform"] != platform: raise ValueError("result identity does not match artifact") if value["status"] not in {"pass", "fail"}: raise ValueError("invalid result status") mode = value["verification_mode"] - if value["status"] == "pass" and mode not in {"real_cli", "hook_subprocess"}: - raise ValueError("passing result requires a verification mode") + if value["status"] == "pass" and mode != "real_cli": + raise ValueError("passing result requires real_cli verification") if value["status"] == "fail" and mode is not None: raise ValueError("failed result cannot claim a verification mode") - allowed_evidence = { - "real host CLI completed the Pitot integration path", - "active hook subprocess produced a normalized Pitot action", - "E2E command failed", - "E2E command returned without one valid result marker", - } + allowed_evidence = {"binary-observed request, accepted response, hook, canary, and final receipts", "real-agent evidence contract failed"} if value["evidence"] not in allowed_evidence: raise ValueError("invalid evidence summary") + evidence_fields = ("cli", "protocol", "endpoint", "prompt_hash", "receipts", "hook") + if value["status"] == "fail": + if any(value[field] is not None for field in evidence_fields): + raise ValueError("failed result cannot carry passing evidence") + else: + record = AGENT_RECORDS[agent] + cli = value["cli"] + if not isinstance(cli, dict) or set(cli) != {"version", "executable", "executable_sha256", "installer", "runtime"}: + raise ValueError("invalid CLI installation receipt") + if cli["version"] != record["version"] or cli["installer"] != record["installer"]["kind"] or cli["runtime"] != record["runtime"][platform]: + raise ValueError("CLI receipt does not match supervised manifest") + cells = [cell for cell in ENDPOINT_PROVENANCE["cells"] if cell["agent"] == agent and cell["platform"] == platform] + if len(cells) != 1: + raise ValueError("missing supervised platform provenance") + fixture = cells[0] + if cli["executable_sha256"] != fixture["executable_sha256"]: + raise ValueError("CLI executable digest does not match binary capture") + if value["protocol"] != fixture["dialect"] or not re.fullmatch(r"[0-9a-f]{64}", str(value["prompt_hash"])): + raise ValueError("invalid prompt/protocol receipt") + expected_endpoint = { + "fixture": f"tests/endpoint-provenance.json#{agent}/{platform}", + "fixture_sha256": fixture["capture_sha256"], + "provenance": "pinned_real_cli_capture", + "dialect": fixture["dialect"], + "request": fixture["request"], + "response": fixture["response"], + "executable_sha256": fixture["executable_sha256"], + } + if value["endpoint"] != expected_endpoint: + raise ValueError("endpoint receipt does not match pinned real-CLI provenance") + receipt_fields = {"initial_prompt_observed", "tool_call_response_emitted", "tool_result_observed", "final_response_emitted", "hook_observed", "canary_result_observed", "cli_exit_zero"} + if not isinstance(value["receipts"], dict) or set(value["receipts"]) != receipt_fields or not all(item is True for item in value["receipts"].values()): + raise ValueError("incomplete causal receipts") + if value["hook"] != {"host": agent, "action_kind": "shell", "pitot_exit": 0}: + raise ValueError("invalid Pitot hook receipt") if not isinstance(value["commit_sha"], str) or not re.fullmatch(r"[0-9a-f]{40}", value["commit_sha"]): raise ValueError("invalid commit_sha") if not isinstance(value["run_url"], str) or not re.fullmatch( @@ -117,18 +155,26 @@ def result_cell(result: dict[str, object] | None) -> str: return "⏳ Pending" if result["status"] == "fail": return "❌ Failed" - mode = "real CLI" if result["verification_mode"] == "real_cli" else "hook verified" - return f"✅ Pass · {mode}" + cli = result["cli"] + runtime = "WSL" if cli["runtime"] == "wsl" else "native" + dialect = result["endpoint"]["dialect"].replace("_", " ") + return f"✅ Pass · real CLI {cli['version']} · {runtime} · binary-observed {dialect}" def failed_result(agent: str, platform: str, head_sha: str, run_url: str) -> dict[str, object]: return { - "schema_version": 1, + "schema_version": 2, "agent": agent, "platform": platform, "status": "fail", "verification_mode": None, - "evidence": "E2E command failed", + "evidence": "real-agent evidence contract failed", + "cli": None, + "protocol": None, + "endpoint": None, + "prompt_hash": None, + "receipts": None, + "hook": None, "commit_sha": head_sha, "run_url": run_url, } @@ -205,6 +251,14 @@ def download(self, url: str) -> bytes: raise ValueError("artifact exceeds size limit") return data + def list_run_artifacts(self, run_id: int) -> list[dict[str, object]]: + value = self.request("GET", f"/actions/runs/{run_id}/artifacts?per_page=100") + artifacts = value.get("artifacts") if isinstance(value, dict) else None + total = value.get("total_count") if isinstance(value, dict) else None + if not isinstance(artifacts, list) or not isinstance(total, int) or total != len(artifacts): + raise ValueError("unified E2E artifact listing is truncated or malformed") + return artifacts + def load_artifact( github: GitHub, @@ -294,7 +348,7 @@ def main() -> int: head_sha = workflow_run["head_sha"] run_url = workflow_run["html_url"] - artifacts = github.request("GET", f"/actions/runs/{workflow_run['id']}/artifacts")["artifacts"] + artifacts = github.list_run_artifacts(int(workflow_run["id"])) by_name = {artifact["name"]: artifact for artifact in artifacts if not artifact.get("expired")} inventory_artifact = by_name.get("pitot-e2e-inventory") if inventory_artifact is None: diff --git a/.github/workflows/pitot-e2e-agent.yml b/.github/workflows/pitot-e2e-agent.yml index 7f7c43336..895d93aa6 100644 --- a/.github/workflows/pitot-e2e-agent.yml +++ b/.github/workflows/pitot-e2e-agent.yml @@ -15,13 +15,14 @@ on: description: GitHub-hosted runner selected by the supervisor required: true type: string + capture: + description: Emit a redacted binary-observed candidate fixture + required: true + type: boolean permissions: contents: read -env: - CLAUDE_CLI_VERSION: 2.1.217 - jobs: e2e: name: ${{ inputs.agent }} (${{ inputs.platform }}) @@ -34,10 +35,38 @@ jobs: cache-dependency-path: labs/15-pitot/pitot/go.mod - uses: actions/setup-node@v4 with: - node-version: "20" + node-version: "22" - uses: actions/setup-python@v6 with: python-version: "3.11" + - name: Enable supported Cursor WSL runtime + if: inputs.agent == 'cursor' && runner.os == 'Windows' + shell: powershell + run: | + wsl.exe --status + if (-not ((wsl.exe --list --quiet) -replace "`0", "" | Where-Object { $_.Trim() -eq "Ubuntu" })) { + wsl.exe --install --distribution Ubuntu --no-launch --web-download + } + $installScript = @' + set -euo pipefail + node_version=22.23.1 + archive="node-v${node_version}-linux-x64.tar.xz" + curl -fsSLO "https://nodejs.org/dist/v${node_version}/${archive}" + curl -fsSLO "https://nodejs.org/dist/v${node_version}/SHASUMS256.txt" + grep " ${archive}$" SHASUMS256.txt | sha256sum --check --strict + tar -xJf "${archive}" -C /usr/local --strip-components=1 + test "$(node --version)" = "v${node_version}" + uname -a + command -v python3 + '@ + $installPath = Join-Path $env:RUNNER_TEMP "pitot-install-wsl-node.sh" + $utf8NoBom = New-Object System.Text.UTF8Encoding($false) + [System.IO.File]::WriteAllText($installPath, ($installScript -replace "`r`n", "`n"), $utf8NoBom) + $drive = $installPath.Substring(0, 1).ToLowerInvariant() + $tail = $installPath.Substring(2).Replace("\", "/") + $wslInstallPath = "/mnt/$drive$tail" + wsl.exe --distribution Ubuntu -- bash $wslInstallPath + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Resolve Bash runtime shell: bash run: | @@ -46,23 +75,13 @@ jobs: bash_path="$(cygpath -m "$bash_path")" fi echo "PITOT_BASH=$bash_path" >> "$GITHUB_ENV" - - name: Install Claude CLI - if: inputs.agent == 'claude' + - name: Install and attest pinned real agent shell: bash - run: | - npm install -g "@anthropic-ai/claude-code@$CLAUDE_CLI_VERSION" - claude_bin="$(command -v claude)" - [[ -n "$claude_bin" ]] - installed_version="$("$claude_bin" --version)" - echo "Using $claude_bin ($installed_version)" - [[ "$installed_version" == "$CLAUDE_CLI_VERSION"* ]] - echo "CLAUDE_PATH=$claude_bin" >> "$GITHUB_ENV" - - name: Install Codex CLI - if: inputs.agent == 'codex' - run: npm install -g @openai/codex - - name: Install Cursor CLI - if: inputs.agent == 'cursor' && runner.os != 'Windows' - run: curl https://cursor.com/install -fsS | bash + run: >- + python3 labs/15-pitot/tests/install_real_agent.py + --agent "${{ inputs.agent }}" + --platform "${{ inputs.platform }}" + --output "${{ runner.temp }}/pitot-e2e/install.json" - name: Run ${{ inputs.agent }} E2E id: e2e continue-on-error: true @@ -70,11 +89,16 @@ jobs: shell: bash env: PITOT_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + PITOT_E2E_PLATFORM: ${{ inputs.platform }} + PITOT_INSTALL_RECEIPT: ${{ runner.temp }}/pitot-e2e/install.json + PITOT_E2E_EVIDENCE: ${{ runner.temp }}/pitot-e2e/evidence.json + PITOT_CAPTURE_OUTPUT: ${{ inputs.capture && format('{0}/pitot-e2e/capture.json', runner.temp) || '' }} run: >- python labs/15-pitot/tests/run_e2e_report.py --agent "${{ inputs.agent }}" --platform "${{ inputs.platform }}" --output "${{ runner.temp }}/pitot-e2e/result.json" + --evidence "${{ runner.temp }}/pitot-e2e/evidence.json" -- "$PITOT_BASH" "labs/15-pitot/tests/e2e_${{ inputs.agent }}_cli_test.sh" < /dev/null - name: Upload structured E2E result @@ -85,6 +109,14 @@ jobs: path: ${{ runner.temp }}/pitot-e2e/result.json if-no-files-found: error retention-days: 14 + - name: Upload endpoint provenance candidate + if: always() && inputs.capture + uses: actions/upload-artifact@v4 + with: + name: pitot-endpoint-capture-${{ inputs.agent }}-${{ inputs.platform }} + path: ${{ runner.temp }}/pitot-e2e/capture.json + if-no-files-found: error + retention-days: 14 - name: Enforce E2E result if: always() && steps.e2e.outcome != 'success' shell: bash diff --git a/.github/workflows/pitot-e2e.yml b/.github/workflows/pitot-e2e.yml index 5e0f0b18f..89831ff45 100644 --- a/.github/workflows/pitot-e2e.yml +++ b/.github/workflows/pitot-e2e.yml @@ -12,6 +12,12 @@ on: branches: [main] paths: *pitot_paths workflow_dispatch: + inputs: + capture_provenance: + description: Capture redacted binary-observed candidates instead of trusting committed fixtures + required: false + type: boolean + default: false permissions: contents: read @@ -30,7 +36,11 @@ jobs: - id: supervisor shell: bash run: | - matrix="$(python3 labs/15-pitot/scripts/pitot_adapter_supervisor.py matrix)" + operation="matrix" + if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.capture_provenance }}" == "true" ]]; then + operation="capture" + fi + matrix="$(python3 labs/15-pitot/scripts/pitot_adapter_supervisor.py "$operation")" echo "matrix=$matrix" >> "$GITHUB_OUTPUT" - name: Upload supervised adapter inventory uses: actions/upload-artifact@v4 @@ -50,3 +60,4 @@ jobs: agent: ${{ matrix.agent }} platform: ${{ matrix.platform }} runner: ${{ matrix.runner }} + capture: ${{ matrix.capture == 'true' }} diff --git a/labs/15-pitot/adapter-verification.json b/labs/15-pitot/adapter-verification.json index d3c0d9480..93174564d 100644 --- a/labs/15-pitot/adapter-verification.json +++ b/labs/15-pitot/adapter-verification.json @@ -1,20 +1,20 @@ { - "schema_version": 1, + "schema_version": 4, "platforms": [ {"id": "ubuntu", "runner": "ubuntu-latest"}, {"id": "macos", "runner": "macos-latest"}, {"id": "windows", "runner": "windows-latest"} ], "agents": [ - {"id": "claude", "label": "Claude"}, - {"id": "cline", "label": "Cline"}, - {"id": "cursor", "label": "Cursor"}, - {"id": "codex", "label": "Codex"}, - {"id": "copilot", "label": "GitHub Copilot CLI"}, - {"id": "gemini", "label": "Gemini"}, - {"id": "kimi", "label": "Kimi Code"}, - {"id": "opencode", "label": "OpenCode"}, - {"id": "pi", "label": "Pi"}, - {"id": "qwen", "label": "Qwen Code"} + {"id": "claude", "label": "Claude", "version": "2.1.217", "executable": "claude", "installer": {"kind": "npm", "package": "@anthropic-ai/claude-code"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "cline", "label": "Cline", "version": "3.0.46", "executable": "cline", "installer": {"kind": "npm", "package": "cline"}, "integration": "cline_bridge", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "cursor", "label": "Cursor", "version": "2026.07.20-8cc9c0b", "executable": "agent", "installer": {"kind": "cursor_release", "package": "https://downloads.cursor.com/lab/2026.07.20-8cc9c0b"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "wsl"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "codex", "label": "Codex", "version": "0.145.0", "executable": "codex", "installer": {"kind": "npm", "package": "@openai/codex"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "copilot", "label": "GitHub Copilot CLI", "version": "1.0.73", "executable": "copilot", "installer": {"kind": "npm", "package": "@github/copilot"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "gemini", "label": "Gemini", "version": "0.51.0", "executable": "gemini", "installer": {"kind": "npm", "package": "@google/gemini-cli"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "kimi", "label": "Kimi Code", "version": "0.29.0", "executable": "kimi", "installer": {"kind": "kimi_release", "package": "https://code.kimi.com/kimi-code"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "opencode", "label": "OpenCode", "version": "1.18.4", "executable": "opencode", "installer": {"kind": "npm", "package": "opencode-ai"}, "integration": "opencode_plugin", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "pi", "label": "Pi", "version": "0.81.1", "executable": "pi", "installer": {"kind": "npm", "package": "@earendil-works/pi-coding-agent"}, "integration": "pi_extension", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "qwen", "label": "Qwen Code", "version": "0.20.1", "executable": "qwen", "installer": {"kind": "npm", "package": "@qwen-code/qwen-code"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"} ] } diff --git a/labs/15-pitot/integrations/cline/PreToolUse b/labs/15-pitot/integrations/cline/PreToolUse index 382cd04ec..3d5c81012 100755 --- a/labs/15-pitot/integrations/cline/PreToolUse +++ b/labs/15-pitot/integrations/cline/PreToolUse @@ -3,7 +3,7 @@ set -uo pipefail PITOT_COMMAND="${PITOT_BIN:-pitot}" PAYLOAD=$(cat) -if ! TOOL=$(printf '%s' "$PAYLOAD" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("preToolUse", {}).get("tool", ""))' 2>/dev/null); then +if ! TOOL=$(printf '%s' "$PAYLOAD" | python3 -c 'import json,sys; event=json.load(sys.stdin).get("preToolUse", {}); print(event.get("toolName") or event.get("tool", ""))' 2>/dev/null); then printf '{"cancel":true,"errorMessage":"Pitot received a malformed Cline hook payload"}\n' exit 0 fi @@ -15,7 +15,7 @@ fi # Cline has no matcher at this boundary. Non-shell tools are outside Pitot's # controllable partition and must pass through unchanged. -if [ "$TOOL" != "execute_command" ]; then +if [ "$TOOL" != "execute_command" ] && [ "$TOOL" != "run_commands" ]; then printf '{"cancel":false}\n' exit 0 fi diff --git a/labs/15-pitot/integrations/cline/PreToolUse.ps1 b/labs/15-pitot/integrations/cline/PreToolUse.ps1 index 8a0653768..2539befee 100644 --- a/labs/15-pitot/integrations/cline/PreToolUse.ps1 +++ b/labs/15-pitot/integrations/cline/PreToolUse.ps1 @@ -6,11 +6,12 @@ try { @{ cancel = $true; errorMessage = "Pitot received a malformed Cline hook payload" } | ConvertTo-Json -Compress exit 0 } -if (-not $event.preToolUse.tool) { +$tool = if ($event.preToolUse.toolName) { $event.preToolUse.toolName } else { $event.preToolUse.tool } +if (-not $tool) { @{ cancel = $true; errorMessage = "Pitot received a malformed Cline hook payload" } | ConvertTo-Json -Compress exit 0 } -if ($event.preToolUse.tool -ne "execute_command") { +if ($tool -ne "execute_command" -and $tool -ne "run_commands") { @{ cancel = $false } | ConvertTo-Json -Compress exit 0 } diff --git a/labs/15-pitot/integrations/opencode/pitot.ts b/labs/15-pitot/integrations/opencode/pitot.ts new file mode 100644 index 000000000..3a500dbc2 --- /dev/null +++ b/labs/15-pitot/integrations/opencode/pitot.ts @@ -0,0 +1,25 @@ +import { spawnSync } from "node:child_process"; + +// OpenCode runs plugins in-process. This is the genuine synchronous +// tool.execute.before boundary; it is not a Claude PreToolUse simulation. +export const PitotPlugin = async () => ({ + "tool.execute.before": async (input, output) => { + if (input.tool !== "bash") return; + const command = output.args?.command; + const payload = JSON.stringify({ + hook_event_name: "PreToolUse", + tool_name: "Bash", + tool_input: { command: typeof command === "string" ? command : "" }, + }); + const result = spawnSync(process.env.PITOT_BIN || "pitot", ["hook", "opencode"], { + input: payload, + encoding: "utf8", + maxBuffer: 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error((result.stderr || "Pitot rejected the shell request").trim()); + } + }, +}); + +export default PitotPlugin; diff --git a/labs/15-pitot/pitot-distribution/UPSTREAM.json b/labs/15-pitot/pitot-distribution/UPSTREAM.json index b70201804..40f9d8da3 100644 --- a/labs/15-pitot/pitot-distribution/UPSTREAM.json +++ b/labs/15-pitot/pitot-distribution/UPSTREAM.json @@ -1,8 +1,9 @@ { "files": { - "CONTRIBUTING.md": "0613b71aa497f8ca7d7296bf34ade87bfc7237a664d2e812d9b77b3b6befb0ad", - "README.md": "098e759d255de328e4de2383738ea920b57d914d6e691c52329ffc7c881930bc", - "adapters/adapters.go": "57a7e3a50c464a4124e3f3231e1843b43f724dcd60316f2c52dd1c65043e9533", + "CONTRIBUTING.md": "02c89a5790f2943e5e2466b0d260210238cae306922f0f031adb5f7028b48066", + "README.md": "b561ffc8f5cb2e57a18e176cb61756e85c72accdfcb9bd7080330877c45c4d30", + "adapter-verification.json": "c92a410798ed72fdaa705c8e85ce574d164a9d7334cf9dd8e0bf6b1ab6c926c7", + "adapters/adapters.go": "10aec65403df482eaa45368cd14a87552789772de6f93118be7bb9e91968aedd", "assets/pitot-boundary.png": "8a0ddb7d81831d94e14813f50ea4ca8670d77417f339ed2f91f0c653bf52f41d", "assets/pitot-boundary.svg": "0c3871d70c84748573f231842091deb38a6def2862403ca34e8cc4493b9c9ebf", "assets/pitot-hero.png": "a73532252b1e66c06273abbf5a4fe6261e98de3133b09e8d550edacfeeab92f8", @@ -18,7 +19,7 @@ "conformance/conformance.go": "43b692114f45c8b52958e34b35aee1cee339d8321c90f92ab4f5b963e79935bb", "conformance/conformance_test.go": "83ab0bcc15371265a954d177e4e97d81ad3ea734bbf736a29a54628ef64b52cd", "conformance/fixtures/negative.jsonl": "c8eff4f4155a5e86e7127a8d89aa06c14ecba74a7230b5bf7134ab1b7c4ca5f7", - "conformance/fixtures/positive.jsonl": "300f40a47e6b72324b76c57a351d3be7ff7a2592e73f02cfe7e66c0e1defdf6d", + "conformance/fixtures/positive.jsonl": "354b784d7bf8bd7dbeae09822bc4f6cf374fc3cf9f5e2824484fd05d8e2bc567", "doc.go": "a8abdafac969b1bf4372c8bb023aa51125dc073f03218f4ab9913dfc5ffa877d", "e2e/e2e_coverage_test.go": "28a6c27408338fdc51cf1241e1bf42a7f0ea17d3fb1305fbcd15901c21e21de8", "e2e/e2e_hook_test.go": "5e184dc8907b6e36daeab90bbbb1654fa5336312866412031805a13ba535d1f8", @@ -27,8 +28,9 @@ "examples/token-meter/main.go": "4b1b9c1a43c3cf48b09dba6f607776caced9d2b5b562373496b31ed184582dd1", "go.mod": "7e71b29887a2370c920a3ecad84460130ecc5ad4acba7f62300b5d2568f0ee13", "go.sum": "2f73a6c3c672f4022f4a618578fac165172095ef590db5d14caa4552675f1980", - "integrations/cline/PreToolUse": "05a449b402918876c5e945e837525b9613a4889fad932e91193b3f63d2778e22", - "integrations/cline/PreToolUse.ps1": "bc0cea5f2cf6e910a2751465090a1a3c37cf0eea8ab4b98b5e735738630548f0", + "integrations/cline/PreToolUse": "5bab5e8f580858cf7a9e0b80eef6066907aab78e170eba0b690bfea363e65c5c", + "integrations/cline/PreToolUse.ps1": "3d5f3df239275678ac9cd20a27eb7252ae75d1873abd4e7c32e975b55eea8597", + "integrations/opencode/pitot.ts": "8d2709b473c839a6012cfe8e2da66221e763884a9dc02dc5198da2d4aec30c40", "integrations/pi/pitot.ts": "ed2d60d5ab6e33e115cfa058e4f96095100e93a061567d0af31249aa756bab3e", "projection/projection.go": "4d3c823fd72a3ca5387dba3683838a1d7e455e9b18309acc839763a39b7bb35f", "protocol/framing.go": "d4409314b72e09cfd472ad9a21d4c7a223b4341b26f58f6062a7c899e3f87482", @@ -41,23 +43,30 @@ "sdk/typescript/src/index.ts": "de43e6654eac51afd09a3a74c4c6992dcf2306d6c41b9be626ded1c6d6cc7833", "sdk/typescript/src/pitot.ts": "9c243824cbb7edc54b1e125abfd828bf2ed77e7151a0bd5c5d4f63e81ca9b00c", "sdk/typescript/tsconfig.json": "e4d7ecb203fcb7d93cd9b9fb235d7fd75d1b6fb2b28eff773f4aecfdc1924d5d", - "sensor/decode_fuzz_test.go": "630064cb7b7e783c25f22f70a34bc5d3583180ba4b59bad360679f21bdf5cb37", + "sensor/decode_fuzz_test.go": "d27f2fbbc069eded26a73c9cd9bace98dd8a9e34949576790b81a08d130fbaf2", "sensor/sensor.go": "5c503d07ac33e7894d635f2d98bcd6d165d442d60c127ed6aeac80ec319086c5", "sensor/sensor_test.go": "4ddbdde3e486c9e189a2ed4174ad413df107d43dc98f5242da3667a24c7a5da1", - "tests/e2e_claude_cli_test.sh": "70711580004b6f77be58f036f42669ae90e2883b62535ab70719f265168ce66e", - "tests/e2e_cline_cli_test.sh": "659794dd4f4e92d93264c58902ffc10db6cceb6659593819dad66018c187eb3b", - "tests/e2e_codex_cli_test.sh": "f5d49b83bb2f9f49a7e0005f4bd7a7bdd0f4130e68ba75a0cd4457d3e9beb175", - "tests/e2e_copilot_cli_test.sh": "2d9ad18993917ee7d2f7f84797a3e31bd31719b7f1834168b7a32978e05404ea", - "tests/e2e_cursor_cli_test.sh": "091a334a5008b88b94a9a2dafdeaf1388071bd436ef1a9b3376d0f0b28ed60a0", - "tests/e2e_gemini_cli_test.sh": "1559c6b50c5c043bb7a2a901e726986eec87d97dcdda026158d2f138365b8a4f", - "tests/e2e_kimi_cli_test.sh": "591b796ea88f1353ede0663f641044fe2e00a01528c26097b63a5d75ec350040", - "tests/e2e_opencode_cli_test.sh": "bfa4854fc6309fdbdb7a45fbb017ffadb7cb193604efc86ae0bcdfc55c4034f8", - "tests/e2e_pi_cli_test.sh": "77793e34e2b2c6c5ca25772a9f34a37fb2e906b9e353f748a1aa9da943b71617", - "tests/e2e_qwen_cli_test.sh": "c08719ce9e8dab05cce2a91d52a479bc566599d38bd3f30e4c279cc6ff622040", - "tests/e2e_unified_runner.sh": "8c37e454a66145d69624bc10db1c3c6568f78ec15acb619b64bea48cb7c9a408", + "tests/cursor_control_proxy.mjs": "68cc734dd7cfdc53c5e932ccd13c9df2f72ad7f8756ddcc0e99a298c333d7223", + "tests/e2e_claude_cli_test.sh": "b28c4d1963e326b4b3f158a7cfc1b92771e768ac9665dd43e8a339013cc11568", + "tests/e2e_cline_cli_test.sh": "a9fe90f64e36e2724acbef95ad37a8d35b4cd93471d560dfae116ef8401a03ac", + "tests/e2e_codex_cli_test.sh": "dbae5224c87410a5a5d67023d7e23406453bc5493d981af9347993b4e1f562f2", + "tests/e2e_copilot_cli_test.sh": "61b1e44dcd598d2d33e7f04dec26bec74405a9e2456ecda46e94dc8d43ad4315", + "tests/e2e_cursor_cli_test.sh": "af46f1a40a535ed1f345ee51cba89a57fa912d6b67bed7df9bae0943131fa542", + "tests/e2e_gemini_cli_test.sh": "42cd77c366c17de092ec46324b3fa764c08b78de0e14f0d9f62b998c2939d6e0", + "tests/e2e_kimi_cli_test.sh": "3689474b027ca2d83ae36c4c3372e93c555477b69d5d81507a8aa6aa1e88edb1", + "tests/e2e_opencode_cli_test.sh": "e74a781f36b8263f2dd0f9700f3eea7a443b289fb87b55af455ce64509e08546", + "tests/e2e_pi_cli_test.sh": "369707019ce0ee9ccb91ac1b9a16d5b8ae357fc9277ccb3407bcd7bd05aa86c0", + "tests/e2e_qwen_cli_test.sh": "8813610ff0ae253e59849f76a5dc6ecf0990c150ea0ede60ec833d36e6e4a68f", + "tests/e2e_unified_runner.sh": "3a85d0548625fef0dacace3023fce71064a9a3bbfd44628ccd7e16e57962eb1a", + "tests/endpoint-provenance.json": "02195b0ea6d531844f0c2b1b7291b3264c662529cb4eaa78c03db0e07c7651d7", + "tests/install_real_agent.py": "dcbc1b09e3475f5c679e7813ccc71f8701baa1a1fa1d8385230ed97e0411d71d", "tests/mock_anthropic_server.js": "ecebea62f9e93791a79f1ae3dd3c67b8fa42490e9805b23b662b877edfdb0f0e", + "tests/model_control_proxy.py": "f8599a7738aa10c8668daafb5c90cd2232b8ac26d759d1e21ae0c13cd9047284", + "tests/real_agent_driver.py": "c722ee9a9ca8e8fd3c3fd31e336ff970c69cea294c83d15af67043c5bc95ae26", + "tests/run_e2e_report.py": "23221c8757fbcb057645b9a6cfc9e8b16469f19ff95d07f46df6fb7a696a1b3a", + "tests/witness/main.go": "e7ac1c5b2bcd46d5ae9fe623540ee1ac5721e7191d8211a012b7c9a89c5ec932", "windtunnel/doc.go": "44e0bcde632da73e1f8b98beade3a34ca8e0d0ea79cdfb91d131de290b164fc4", - "windtunnel/windtunnel_test.go": "e987b3dacd6fdb05aa021bd15ff2c1e1f87b99f4a65aa28e9b74a93bb9b79c1a" + "windtunnel/windtunnel_test.go": "5da9d754cf8940f8cf79b60240107ea304ab0f37bb8df55271db5e42eca3003a" }, "schema_version": 1 } diff --git a/labs/15-pitot/pitot-distribution/release-notes/2026-07-22-real-agent-e2e.md b/labs/15-pitot/pitot-distribution/release-notes/2026-07-22-real-agent-e2e.md new file mode 100644 index 000000000..d12b984bd --- /dev/null +++ b/labs/15-pitot/pitot-distribution/release-notes/2026-07-22-real-agent-e2e.md @@ -0,0 +1,11 @@ +### Repair: require prompt-to-hook real-agent evidence + +Pitot's supervised E2E matrix now installs each pinned coding-agent release and +requires a nonce-correlated prompt, controlled model tool call, genuine host +hook, Pitot shell observation, real canary result, and final model response. +Direct hook subprocesses can no longer produce a passing platform result. +Every platform cell also carries a version-pinned, redacted binary capture. The +live request selects the proxy dialect—never the manifest—and the supervisor +and reporter require the accepted response, executable digest, hook, and +nonce-bearing tool result to match that capture. Invented proxy endpoints or +hand-authored protocol declarations cannot report a pass. diff --git a/labs/15-pitot/pitot/adapters/adapters.go b/labs/15-pitot/pitot/adapters/adapters.go index 82ca96f00..70fab4622 100644 --- a/labs/15-pitot/pitot/adapters/adapters.go +++ b/labs/15-pitot/pitot/adapters/adapters.go @@ -12,6 +12,7 @@ package adapters import ( + "encoding/json" "errors" "fmt" "sort" @@ -74,7 +75,20 @@ var ( registryMu sync.RWMutex registry = map[Host]HostConfig{ Copilot: preToolUseHost(), - Qwen: preToolUseHost(), + Qwen: { + MainEventName: "PreToolUse", + Parser: ParserConfig{ + CanonicalEvent: []byte(`{"hook_event_name":"PreToolUse","tool_name":"run_shell_command","tool_input":{"command":"git status --short"}}`), + CommandFor: func(raw RawHookEvent) (string, bool) { + if raw.ToolName != "Bash" && raw.ToolName != "run_shell_command" { + return "", false + } + return toolInputCommand(raw) + }, + ActionKinds: map[string]string{"PreToolUse": "shell"}, + }, + Partition: ControlPartition{Controllable: []string{"PreToolUse"}}, + }, Pi: { MainEventName: "tool_call", Parser: ParserConfig{ @@ -85,20 +99,43 @@ var ( Partition: ControlPartition{Controllable: []string{"tool_call"}}, }, Cline: { - MainEventName: "PreToolUse", + MainEventName: "tool_call", Parser: ParserConfig{ - CanonicalEvent: []byte(`{"hookName":"PreToolUse","preToolUse":{"tool":"execute_command","parameters":{"command":"git status --short"}}}`), + CanonicalEvent: []byte(`{"hookName":"tool_call","preToolUse":{"toolName":"run_commands","parameters":{"commands":"[\"git status --short\"]"}}}`), EventNameFor: func(raw RawHookEvent) string { return raw.HookName }, CommandFor: func(raw RawHookEvent) (string, bool) { - if raw.PreToolUse.Tool != "execute_command" { + tool := raw.PreToolUse.ToolName + if tool == "" { + tool = raw.PreToolUse.Tool + } + if tool == "execute_command" { + value, present := raw.PreToolUse.Parameters["command"].(string) + return value, present && value != "" + } + if tool != "run_commands" { + return "", false + } + encoded, present := raw.PreToolUse.Parameters["commands"].(string) + if !present || encoded == "" { + return "", false + } + var commands []any + if json.Unmarshal([]byte(encoded), &commands) != nil || len(commands) == 0 { + return "", false + } + switch command := commands[0].(type) { + case string: + return command, command != "" + case map[string]any: + value, ok := command["command"].(string) + return value, ok && value != "" + default: return "", false } - value, present := raw.PreToolUse.Parameters["command"].(string) - return value, present && value != "" }, - ActionKinds: map[string]string{"PreToolUse": "shell"}, + ActionKinds: map[string]string{"tool_call": "shell", "PreToolUse": "shell"}, }, - Partition: ControlPartition{Controllable: []string{"PreToolUse"}}, + Partition: ControlPartition{Controllable: []string{"tool_call", "PreToolUse"}}, }, Cursor: { MainEventName: "beforeShellExecution", @@ -364,6 +401,7 @@ type RawHookEvent struct { HookName string `json:"hookName"` PreToolUse struct { Tool string `json:"tool"` + ToolName string `json:"toolName"` Parameters map[string]any `json:"parameters"` } `json:"preToolUse"` } diff --git a/labs/15-pitot/pitot/conformance/fixtures/positive.jsonl b/labs/15-pitot/pitot/conformance/fixtures/positive.jsonl index 85edd1339..6bad025ed 100644 --- a/labs/15-pitot/pitot/conformance/fixtures/positive.jsonl +++ b/labs/15-pitot/pitot/conformance/fixtures/positive.jsonl @@ -5,5 +5,7 @@ {"name":"claude-missing-event-name-tolerated","host":"claude","mode":"omit","input":{"tool_name":"Bash","tool_input":{"command":"pwd"}},"expect_kind":"shell"} {"name":"copilot-pre-tool","host":"copilot","mode":"sha256","input":{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"git status --short"}},"expect_kind":"shell"} {"name":"qwen-pre-tool","host":"qwen","mode":"sha256","input":{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"git status --short"}},"expect_kind":"shell"} +{"name":"qwen-native-shell-tool","host":"qwen","mode":"sha256","input":{"hook_event_name":"PreToolUse","tool_name":"run_shell_command","tool_input":{"command":"git status --short"}},"expect_kind":"shell"} {"name":"pi-tool-call","host":"pi","mode":"sha256","input":{"hook_event_name":"tool_call","tool_name":"bash","tool_input":{"command":"git status --short"}},"expect_kind":"shell"} -{"name":"cline-pre-tool","host":"cline","mode":"sha256","input":{"hookName":"PreToolUse","preToolUse":{"tool":"execute_command","parameters":{"command":"git status --short"}}},"expect_kind":"shell"} +{"name":"cline-native-tool-call","host":"cline","mode":"sha256","input":{"hookName":"tool_call","preToolUse":{"toolName":"run_commands","parameters":{"commands":"[\"git status --short\"]"}}},"expect_kind":"shell"} +{"name":"cline-legacy-pre-tool","host":"cline","mode":"sha256","input":{"hookName":"PreToolUse","preToolUse":{"tool":"execute_command","parameters":{"command":"git status --short"}}},"expect_kind":"shell"} diff --git a/labs/15-pitot/pitot/sensor/decode_fuzz_test.go b/labs/15-pitot/pitot/sensor/decode_fuzz_test.go index 6240d8c1e..fa96cae3d 100644 --- a/labs/15-pitot/pitot/sensor/decode_fuzz_test.go +++ b/labs/15-pitot/pitot/sensor/decode_fuzz_test.go @@ -22,7 +22,7 @@ func FuzzDecode(f *testing.F) { []byte(`{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"ls"}}`), []byte(`{"hook_event_name":"PreToolUse","tool_input":null}`), []byte(`{"hook_event_name":123}`), - []byte(`{"hookName":"PreToolUse","preToolUse":{"tool":"execute_command","parameters":{"command":"ls"}}}`), + []byte(`{"hookName":"tool_call","preToolUse":{"toolName":"run_commands","parameters":{"commands":"[\"ls\"]"}}}`), []byte(`[]`), } for _, seed := range seeds { diff --git a/labs/15-pitot/pitot/windtunnel/windtunnel_test.go b/labs/15-pitot/pitot/windtunnel/windtunnel_test.go index 16426c269..9bfa80824 100644 --- a/labs/15-pitot/pitot/windtunnel/windtunnel_test.go +++ b/labs/15-pitot/pitot/windtunnel/windtunnel_test.go @@ -33,9 +33,9 @@ var boatstackCanonicalEvents = map[adapters.Host]string{ adapters.Kimi: `{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"git status --short"}}`, adapters.Opencode: `{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"git status --short"}}`, adapters.Copilot: `{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"git status --short"}}`, - adapters.Qwen: `{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"git status --short"}}`, + adapters.Qwen: `{"hook_event_name":"PreToolUse","tool_name":"run_shell_command","tool_input":{"command":"git status --short"}}`, adapters.Pi: `{"hook_event_name":"tool_call","tool_name":"bash","tool_input":{"command":"git status --short"}}`, - adapters.Cline: `{"hookName":"PreToolUse","preToolUse":{"tool":"execute_command","parameters":{"command":"git status --short"}}}`, + adapters.Cline: `{"hookName":"tool_call","preToolUse":{"toolName":"run_commands","parameters":{"commands":"[\"git status --short\"]"}}}`, } func TestSensorConsumesBoatstackCanonicalEvents(t *testing.T) { diff --git a/labs/15-pitot/public-readme-preview/CONTRIBUTING.md b/labs/15-pitot/public-readme-preview/CONTRIBUTING.md index a106abae7..45c38f7f2 100644 --- a/labs/15-pitot/public-readme-preview/CONTRIBUTING.md +++ b/labs/15-pitot/public-readme-preview/CONTRIBUTING.md @@ -29,3 +29,20 @@ positive vector for every new adapter behavior and a negative control for every boundary fault. The sensor package must never import the bridge package — measurement stays innocent of control. +## Refreshing agent endpoint provenance + +Protocol documentation is not a verification authority. When a pinned agent +version changes, dispatch the existing **Pitot E2E** workflow with +`capture_provenance=true`. Download all `pitot-endpoint-capture-*` artifacts, +then merge the complete set locally: + +```bash +python3 scripts/pitot_adapter_supervisor.py capture-merge \ + --captures /path/to/downloaded-artifacts \ + --output tests/endpoint-provenance.json +python3 scripts/pitot_adapter_supervisor.py check +``` + +The merge refuses partial, duplicate, mixed-version, unsuccessful, or +fabricated captures. Review the redacted 30-cell wire diff before committing +it; ordinary CI verifies the committed fixtures and never rewrites them. diff --git a/labs/15-pitot/public-readme-preview/README.md b/labs/15-pitot/public-readme-preview/README.md index 47fc91436..6b5069b80 100644 --- a/labs/15-pitot/public-readme-preview/README.md +++ b/labs/15-pitot/public-readme-preview/README.md @@ -11,7 +11,7 @@ Pitot agent E2E

-

Every supervised adapter is required on Ubuntu, macOS, and Windows.

+

Every supervised adapter must pass a binary-observed prompt → model reply → hook → tool-result loop on Ubuntu, macOS, and Windows.

Supervised adapters: Claude · Cline · Cursor · Codex · GitHub Copilot CLI · Gemini · Kimi Code · OpenCode · Pi · Qwen Code

diff --git a/labs/15-pitot/scripts/build_pitot.py b/labs/15-pitot/scripts/build_pitot.py index 08f6e2c86..c2a61f6ab 100644 --- a/labs/15-pitot/scripts/build_pitot.py +++ b/labs/15-pitot/scripts/build_pitot.py @@ -88,10 +88,20 @@ def _iter_e2e_harness_files(repo: Path) -> list[tuple[str, Path]]: sources = sorted(root.glob("e2e_*_cli_test.sh")) sources.extend( path - for name in ("e2e_unified_runner.sh", "mock_anthropic_server.js") + for name in ( + "e2e_unified_runner.sh", "install_real_agent.py", "mock_anthropic_server.js", "model_control_proxy.py", + "cursor_control_proxy.mjs", "endpoint-provenance.json", "real_agent_driver.py", "run_e2e_report.py", + ) if (path := root / name).is_file() ) - return [(f"tests/{path.name}", path) for path in sources] + entries = [(f"tests/{path.name}", path) for path in sources] + witness = root / "witness/main.go" + if witness.is_file(): + entries.append(("tests/witness/main.go", witness)) + manifest = repo / LAB / "adapter-verification.json" + if manifest.is_file(): + entries.append(("adapter-verification.json", manifest)) + return entries def _iter_integration_files(repo: Path) -> list[tuple[str, Path]]: diff --git a/labs/15-pitot/scripts/pitot_adapter_supervisor.py b/labs/15-pitot/scripts/pitot_adapter_supervisor.py index d34d4b157..2d96b1b7a 100644 --- a/labs/15-pitot/scripts/pitot_adapter_supervisor.py +++ b/labs/15-pitot/scripts/pitot_adapter_supervisor.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import hashlib import json from pathlib import Path import re @@ -13,6 +14,7 @@ ROOT = Path(__file__).resolve().parents[3] MANIFEST = Path("labs/15-pitot/adapter-verification.json") +ENDPOINT_PROVENANCE = Path("labs/15-pitot/tests/endpoint-provenance.json") README = Path("labs/15-pitot/public-readme-preview/README.md") UNIFIED_WORKFLOW = Path(".github/workflows/pitot-e2e.yml") REPORT_WORKFLOW = Path(".github/workflows/pitot-e2e-report.yml") @@ -22,6 +24,10 @@ README_END = "" ID_PATTERN = re.compile(r"^[a-z][a-z0-9-]*$") LABEL_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 .+_-]{0,39}$") +VERSION_PATTERN = re.compile(r"^(?:[0-9]+\.){2}[0-9]+(?:-[0-9A-Za-z.-]+)?$") +DIALECTS = {"anthropic_messages", "openai_chat", "openai_responses", "gemini_generate_content", "cursor_connect_proto"} +INTEGRATIONS = {"native_command_hook", "cline_bridge", "pi_extension", "opencode_plugin"} +INSTALLERS = {"npm", "kimi_release", "cursor_release"} class ContractError(ValueError): @@ -34,13 +40,14 @@ def load_manifest(root: Path = ROOT) -> dict[str, object]: except (OSError, json.JSONDecodeError) as error: raise ContractError(f"cannot load {MANIFEST}: {error}") from error validate_manifest(value) + validate_endpoint_provenance(root, value) return value def validate_manifest(value: object) -> None: if not isinstance(value, dict) or set(value) != {"schema_version", "platforms", "agents"}: raise ContractError("manifest must contain schema_version, platforms, and agents") - if value["schema_version"] != 1: + if value["schema_version"] != 4: raise ContractError("unsupported manifest schema_version") platforms = value["platforms"] agents = value["agents"] @@ -56,13 +63,45 @@ def validate_manifest(value: object) -> None: ids: list[str] = [] labels: list[str] = [] for agent in agents: - if not isinstance(agent, dict) or set(agent) != {"id", "label"}: - raise ContractError("each agent must contain exactly id and label") + required = { + "id", "label", "version", "executable", "installer", + "integration", "runtime", "driver", "required_mode", + } + if not isinstance(agent, dict) or set(agent) != required: + raise ContractError(f"each agent must contain exactly {', '.join(sorted(required))}") agent_id, label = agent["id"], agent["label"] if not isinstance(agent_id, str) or not ID_PATTERN.fullmatch(agent_id): raise ContractError(f"invalid agent id: {agent_id!r}") if not isinstance(label, str) or not LABEL_PATTERN.fullmatch(label): raise ContractError(f"agent {agent_id} has an invalid label") + version = agent["version"] + if not isinstance(version, str) or not VERSION_PATTERN.fullmatch(version) or "latest" in version.lower(): + raise ContractError(f"agent {agent_id} must pin an immutable CLI version") + if not isinstance(agent["executable"], str) or not ID_PATTERN.fullmatch(agent["executable"]): + raise ContractError(f"agent {agent_id} has an invalid executable") + installer = agent["installer"] + if ( + not isinstance(installer, dict) + or set(installer) != {"kind", "package"} + or installer["kind"] not in INSTALLERS + or not isinstance(installer["package"], str) + or not installer["package"] + ): + raise ContractError(f"agent {agent_id} has invalid pinned installer metadata") + if agent["integration"] not in INTEGRATIONS: + raise ContractError(f"agent {agent_id} has an unsupported integration mechanism") + expected_runtime_keys = {platform["id"] for platform in platforms} + runtime = agent["runtime"] + if not isinstance(runtime, dict) or set(runtime) != expected_runtime_keys: + raise ContractError(f"agent {agent_id} must declare every platform runtime") + if any(item not in {"native", "wsl"} for item in runtime.values()): + raise ContractError(f"agent {agent_id} has an unsupported platform runtime") + if agent_id == "cursor" and runtime["windows"] != "wsl": + raise ContractError("Cursor must use its supported WSL runtime on Windows") + if agent_id != "cursor" and "wsl" in runtime.values(): + raise ContractError(f"agent {agent_id} must run natively on every platform") + if agent["driver"] != "real_agent_driver.py" or agent["required_mode"] != "real_cli": + raise ContractError(f"agent {agent_id} must require the canonical real-CLI driver") ids.append(agent_id) labels.append(label) if len(ids) != len(set(ids)): @@ -71,6 +110,62 @@ def validate_manifest(value: object) -> None: raise ContractError("agent labels must be unique") +def validate_endpoint_provenance(root: Path, manifest: dict[str, object]) -> None: + """Require endpoint claims to be pinned to the real-CLI capture contract.""" + try: + value = json.loads((root / ENDPOINT_PROVENANCE).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ContractError(f"cannot load {ENDPOINT_PROVENANCE}: {error}") from error + validate_endpoint_value(value, manifest) + + +def validate_endpoint_value(value: object, manifest: dict[str, object]) -> None: + if not isinstance(value, dict) or set(value) != {"schema_version", "capture_policy", "cells"}: + raise ContractError("endpoint provenance must contain schema_version, capture_policy, and cells") + if value["schema_version"] != 2 or value["capture_policy"] != "pinned_real_cli_capture": + raise ContractError("endpoint provenance must require pinned real-CLI capture") + cells = value["cells"] + if not isinstance(cells, list): + raise ContractError("endpoint provenance cells must be a list") + declared = {agent["id"]: agent for agent in manifest["agents"]} + platforms = [platform["id"] for platform in manifest["platforms"]] + expected_keys = {(agent_id, platform) for agent_id in declared for platform in platforms} + actual_keys = {(cell.get("agent"), cell.get("platform")) for cell in cells if isinstance(cell, dict)} + if len(cells) != len(expected_keys) or actual_keys != expected_keys: + raise ContractError("endpoint provenance must contain exactly all 30 agent/platform cells") + required = {"agent", "platform", "runtime", "version", "executable_sha256", "dialect", "request", "response", "provenance", "capture_sha256"} + for cell in cells: + agent_id, platform = cell["agent"], cell["platform"] + if set(cell) != required: + raise ContractError(f"endpoint fixture {agent_id}/{platform} has an invalid schema") + agent = declared[agent_id] + if cell["version"] != agent["version"] or cell["runtime"] != agent["runtime"][platform]: + raise ContractError(f"endpoint fixture {agent_id}/{platform} is stale for its pinned CLI") + if cell["dialect"] not in DIALECTS or cell["provenance"] != "pinned_real_cli_capture": + raise ContractError(f"endpoint fixture {agent_id}/{platform} lacks binary-observed provenance") + if not re.fullmatch(r"[0-9a-f]{64}", str(cell["executable_sha256"])): + raise ContractError(f"endpoint fixture {agent_id}/{platform} lacks an executable digest") + request, response = cell["request"], cell["response"] + request_keys = {"transport", "method", "path", "media_type", "framing", "request_shape"} + response_keys = {"encoder", "framing", "tool_call", "acceptance"} + if not isinstance(request, dict) or set(request) != request_keys or not isinstance(response, dict) or set(response) != response_keys: + raise ContractError(f"endpoint fixture {agent_id}/{platform} has invalid request/response receipts") + if request["transport"] not in {"http1", "http2"} or request["method"] != "POST" or not str(request["path"]).startswith("/"): + raise ContractError(f"endpoint fixture {agent_id}/{platform} has an invalid observed endpoint") + if request["framing"] not in {"json", "sse", "connect_envelope"} or not isinstance(request["request_shape"], dict): + raise ContractError(f"endpoint fixture {agent_id}/{platform} has an invalid redacted request shape") + if cell["dialect"] == "cursor_connect_proto" and (request["transport"], request["framing"]) != ("http2", "connect_envelope"): + raise ContractError(f"endpoint fixture {agent_id}/{platform} has invalid Cursor wire provenance") + if cell["dialect"] != "cursor_connect_proto" and request["media_type"] != "application/json": + raise ContractError(f"endpoint fixture {agent_id}/{platform} has invalid JSON wire provenance") + if response != {"encoder": cell["dialect"], "framing": request["framing"], "tool_call": "native_shell", "acceptance": "nonce_tool_result_round_trip"}: + raise ContractError(f"endpoint fixture {agent_id}/{platform} lacks accepted response provenance") + core = {key: value for key, value in cell.items() if key != "capture_sha256"} + digest = hashlib.sha256(json.dumps(core, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + if cell["capture_sha256"] != digest: + raise ContractError(f"endpoint fixture {agent_id}/{platform} has a fabricated capture digest") + + def built_in_adapters(root: Path = ROOT) -> list[str]: completed = subprocess.run( ["go", "run", str(root / INVENTORY_HELPER)], @@ -88,7 +183,16 @@ def built_in_adapters(root: Path = ROOT) -> list[str]: def matrix(manifest: dict[str, object]) -> list[dict[str, str]]: return [ - {"agent": agent["id"], "platform": platform["id"], "runner": platform["runner"]} + { + "agent": agent["id"], + "platform": platform["id"], + "runner": platform["runner"], + "version": agent["version"], + "endpoint_fixture": f"tests/endpoint-provenance.json#{agent['id']}/{platform['id']}", + "runtime": agent["runtime"][platform["id"]], + "installer": agent["installer"]["kind"], + "capture": "false", + } for agent in manifest["agents"] for platform in manifest["platforms"] ] @@ -103,7 +207,7 @@ def readme_block(manifest: dict[str, object]) -> str: ' Pitot agent E2E', "

", "", - '

Every supervised adapter is required on Ubuntu, macOS, and Windows.

', + '

Every supervised adapter must pass a binary-observed prompt → model reply → hook → tool-result loop on Ubuntu, macOS, and Windows.

', "", f'

Supervised adapters: {labels}

', README_END, @@ -133,6 +237,12 @@ def unified_workflow() -> str: branches: [main] paths: *pitot_paths workflow_dispatch: + inputs: + capture_provenance: + description: Capture redacted binary-observed candidates instead of trusting committed fixtures + required: false + type: boolean + default: false permissions: contents: read @@ -151,7 +261,11 @@ def unified_workflow() -> str: - id: supervisor shell: bash run: | - matrix="$(python3 labs/15-pitot/scripts/pitot_adapter_supervisor.py matrix)" + operation="matrix" + if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.capture_provenance }}" == "true" ]]; then + operation="capture" + fi + matrix="$(python3 labs/15-pitot/scripts/pitot_adapter_supervisor.py "$operation")" echo "matrix=$matrix" >> "$GITHUB_OUTPUT" - name: Upload supervised adapter inventory uses: actions/upload-artifact@v4 @@ -171,6 +285,7 @@ def unified_workflow() -> str: agent: ${{ matrix.agent }} platform: ${{ matrix.platform }} runner: ${{ matrix.runner }} + capture: ${{ matrix.capture == 'true' }} """ @@ -226,6 +341,37 @@ def contract_errors(root: Path, adapters: list[str] | None = None) -> list[str]: script = root / f"labs/15-pitot/tests/e2e_{agent_id}_cli_test.sh" if not script.is_file(): errors.append(f"adapter {agent_id} is missing {script.relative_to(root)}") + driver = root / "labs/15-pitot/tests/real_agent_driver.py" + if not driver.is_file(): + errors.append("missing canonical real-agent prompt driver") + proxy = root / "labs/15-pitot/tests/model_control_proxy.py" + if not proxy.is_file(): + errors.append("missing local model-control proxy") + elif 'add_argument("--protocol"' in proxy.read_text(encoding="utf-8"): + errors.append("model proxy permits a declared protocol authority") + cursor_proxy = root / "labs/15-pitot/tests/cursor_control_proxy.mjs" + if not cursor_proxy.is_file(): + errors.append("missing pinned Cursor endpoint proxy") + reporter_path = root / ".github/scripts/pitot_e2e_report.py" + reporter_text = reporter_path.read_text(encoding="utf-8") if reporter_path.is_file() else "" + if "artifacts?per_page=100" not in reporter_text or "total != len(artifacts)" not in reporter_text: + errors.append("reporter does not fail closed over the complete supervised artifact page") + runner = root / "labs/15-pitot/tests/e2e_unified_runner.sh" + runner_text = runner.read_text(encoding="utf-8") if runner.is_file() else "" + forbidden = ("hook_subprocess", "Falling back", "active subprocess hook verification") + if any(fragment in runner_text for fragment in forbidden): + errors.append("unified runner permits synthetic E2E success") + if "real_agent_driver.py" not in runner_text: + errors.append("unified runner does not invoke the canonical real-agent driver") + if ( + runner_text.count("GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build") != 2 + or "pitot-linux" not in runner_text + or "pitot-witness-linux" not in runner_text + or "wsl.exe --distribution Ubuntu -- env" not in runner_text + or 'python3 "$(to_wsl_path "$SCRIPT_DIR/real_agent_driver.py")"' not in runner_text + or "MSYS2_ARG_CONV_EXCL='*'" not in runner_text + ): + errors.append("unified runner does not preserve Cursor's native WSL controller boundary") expected_files = { UNIFIED_WORKFLOW: unified_workflow(), REPORT_WORKFLOW: report_workflow(), @@ -240,9 +386,18 @@ def contract_errors(root: Path, adapters: list[str] | None = None) -> list[str]: required_reusable_contract = ( "platform:\n description: Canonical Pitot platform identifier", "runner:\n description: GitHub-hosted runner selected by the supervisor", + "capture:\n description: Emit a redacted binary-observed candidate fixture", "runs-on: ${{ inputs.runner }}", '--agent "${{ inputs.agent }}"', '--platform "${{ inputs.platform }}"', + '--evidence "${{ runner.temp }}/pitot-e2e/evidence.json"', + 'install_real_agent.py', + "PITOT_CAPTURE_OUTPUT:", + "node_version=22.23.1", + "SHASUMS256.txt", + "[System.IO.File]::WriteAllText", + "wsl.exe --distribution Ubuntu -- bash $wslInstallPath", + "pitot-endpoint-capture-${{ inputs.agent }}-${{ inputs.platform }}", "name: pitot-e2e-${{ inputs.agent }}-${{ inputs.platform }}", ) if any(fragment not in reusable for fragment in required_reusable_contract) or "matrix.platform" in reusable: @@ -272,10 +427,32 @@ def render(root: Path = ROOT) -> None: (root / REPORT_WORKFLOW).write_text(report_workflow(), encoding="utf-8") +def merge_captures(root: Path, captures: Path, output: Path) -> None: + manifest = load_manifest_without_provenance(root) + cells: list[dict[str, object]] = [] + for path in sorted(captures.rglob("*.json")): + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict) or set(value) != {"schema_version", "accepted", "cell"} or value["schema_version"] != 1 or value["accepted"] is not True: + raise ContractError(f"invalid capture artifact: {path.name}") + cells.append(value["cell"]) + candidate = {"schema_version": 2, "capture_policy": "pinned_real_cli_capture", "cells": sorted(cells, key=lambda item: (item["agent"], item["platform"]))} + validate_endpoint_value(candidate, manifest) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(candidate, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def load_manifest_without_provenance(root: Path) -> dict[str, object]: + value = json.loads((root / MANIFEST).read_text(encoding="utf-8")) + validate_manifest(value) + return value + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("operation", choices=("check", "matrix", "render")) + parser.add_argument("operation", choices=("check", "matrix", "render", "capture", "capture-merge")) parser.add_argument("--repo", type=Path, default=ROOT) + parser.add_argument("--captures", type=Path) + parser.add_argument("--output", type=Path) args = parser.parse_args() root = args.repo.resolve() try: @@ -283,12 +460,24 @@ def main() -> int: render(root) print("PASS: rendered Pitot adapter verification surfaces") return 0 + if args.operation == "capture-merge": + if args.captures is None or args.output is None: + raise ContractError("capture-merge requires --captures and --output") + merge_captures(root, args.captures.resolve(), args.output.resolve()) + print(f"PASS: merged 30 binary-observed endpoint captures into {args.output}") + return 0 + if args.operation == "capture": + manifest = load_manifest_without_provenance(root) + values = [{**item, "capture": "true"} for item in matrix(manifest)] + print(json.dumps({"include": values}, separators=(",", ":"))) + return 0 errors = contract_errors(root) if errors: raise ContractError("; ".join(errors)) if args.operation == "matrix": manifest = load_manifest(root) - print(json.dumps({"include": matrix(manifest)}, separators=(",", ":"))) + values = matrix(manifest) + print(json.dumps({"include": values}, separators=(",", ":"))) else: print(f"PASS: supervised {len(load_manifest(root)['agents'])} Pitot adapters") return 0 diff --git a/labs/15-pitot/tests/cursor_control_proxy.mjs b/labs/15-pitot/tests/cursor_control_proxy.mjs new file mode 100644 index 000000000..144f6e4f8 --- /dev/null +++ b/labs/15-pitot/tests/cursor_control_proxy.mjs @@ -0,0 +1,260 @@ +#!/usr/bin/env node +// Pinned Cursor Agent endpoint fixture. This is deliberately separate from the +// JSON dialect handler because Cursor's --endpoint transport upgrades Run to +// cleartext HTTP/2 and protobuf. + +import http2 from "node:http2"; +import http from "node:http"; +import net from "node:net"; +import fs from "node:fs"; + +function parseArgs(argv) { + const values = {}; + for (let index = 2; index < argv.length; index += 2) { + values[argv[index].replace(/^--/, "")] = argv[index + 1]; + } + return values; +} + +function varint(value) { + const bytes = []; + while (value > 0x7f) { + bytes.push((value & 0x7f) | 0x80); + value >>= 7; + } + bytes.push(value); + return Buffer.from(bytes); +} + +function fieldBytes(field, value) { + const payload = Buffer.isBuffer(value) ? value : Buffer.from(value); + return Buffer.concat([varint((field << 3) | 2), varint(payload.length), payload]); +} + +function fieldVarint(field, value) { + return Buffer.concat([varint(field << 3), varint(value)]); +} + +function connectEnvelope(message, flags = 0) { + const header = Buffer.alloc(5); + header[0] = flags; + header.writeUInt32BE(message.length, 1); + return Buffer.concat([header, message]); +} + +function envelopeShapes(body) { + const shapes = []; + let offset = 0; + while (offset + 5 <= body.length) { + const flags = body[offset]; + const length = body.readUInt32BE(offset + 1); + if (offset + 5 + length > body.length) break; + const message = body.subarray(offset + 5, offset + 5 + length); + const printable = (message.toString("latin1").match(/[ -~]{4,}/g) || []) + .map((value) => value.replaceAll(args.nonce, "")) + .filter((value) => /pitot|canary|hook|command|error|fail|denied|not found|nonce/i.test(value)) + .slice(0, 12); + shapes.push({ flags, length, nonce_present: message.includes(Buffer.from(args.nonce)), canary_result_present: message.includes(Buffer.from(`PITOT_CANARY_RESULT ${args.nonce}`)), printable }); + offset += 5 + length; + } + return shapes; +} + +function shellExecution(command) { + const executable = command.split(" ", 1)[0]; + const argument = command.slice(executable.length + 1); + // ShellCommandParsingResult.ExecutableCommandArg { type, value } + const parsedArgument = Buffer.concat([fieldBytes(1, "word"), fieldBytes(2, argument)]); + // ShellCommandParsingResult.ExecutableCommand { name, args, full_text } + const parsedCommand = Buffer.concat([ + fieldBytes(1, executable), + fieldBytes(2, parsedArgument), + fieldBytes(3, command), + ]); + // ShellCommandParsingResult { executable_commands: 2 } + const parsingResult = fieldBytes(2, parsedCommand); + // agent.v1.ShellArgs { command: 1, tool_call_id: 4 } + const shellArgs = Buffer.concat([ + fieldBytes(1, command), + fieldBytes(4, "pitot-tool-1"), + fieldBytes(5, command), + fieldBytes(8, parsingResult), + fieldVarint(12, 1), + ]); + // agent.v1.ExecServerMessage { id: 1, exec_id: 15, shell_args: 2 } + const execution = Buffer.concat([ + fieldVarint(1, 1), + fieldBytes(15, "pitot-exec-1"), + fieldBytes(2, shellArgs), + ]); + // agent.v1.AgentServerMessage { exec_server_message: 2 } + return fieldBytes(2, execution); +} + +function textUpdate(text) { + // TextDeltaUpdate.text: 1 → InteractionUpdate.text_delta: 1 → + // AgentServerMessage.interaction_update: 1. + return fieldBytes(1, fieldBytes(1, fieldBytes(1, text))); +} + +function turnEnded() { + // TurnEndedUpdate is valid when empty. It is field 14 of InteractionUpdate. + return fieldBytes(1, fieldBytes(14, Buffer.alloc(0))); +} + +function modelDetails() { + return Buffer.concat([1, 3, 4, 5].map((field) => fieldBytes(field, "pitot-control"))); +} + +function unaryResponse(path) { + const model = modelDetails(); + if (path.endsWith("/GetUsableModels") || path.endsWith("/GetDefaultModelForCli")) { + return fieldBytes(1, model); + } + return Buffer.alloc(0); +} + +const args = parseArgs(process.argv); +if (process.argv.includes("--self-test")) { + const command = "pitot-e2e-canary fixture-nonce"; + const message = shellExecution(command); + const framed = connectEnvelope(message); + if (!message.includes(Buffer.from(command)) || framed.readUInt32BE(1) !== message.length) { + throw new Error("Cursor Connect/protobuf fixture is invalid"); + } + process.stdout.write("PASS: Cursor Connect/protobuf endpoint fixture\n"); + process.exit(0); +} +const required = ["nonce", "receipt", "ready-file"]; +for (const key of required) { + if (!args[key]) throw new Error(`missing --${key}`); +} + +const receipt = { + schema_version: 1, + agent: "cursor", + protocol: "cursor_connect_proto", + nonce: args.nonce, + initial_prompt_observed: false, + tool_call_response_emitted: false, + tool_result_observed: false, + final_response_emitted: false, + endpoint_observed: null, + auxiliary_requests: 0, + cursor_requests: [], +}; + +function save() { + const temporary = `${args.receipt}.tmp`; + fs.mkdirSync(new URL(".", `file://${args.receipt}`).pathname, { recursive: true }); + fs.writeFileSync(temporary, `${JSON.stringify(receipt, null, 2)}\n`); + fs.renameSync(temporary, args.receipt); +} + +const h2Server = http2.createServer(); + +h2Server.on("stream", (stream, headers) => { + const path = String(headers[":path"] || "").split("?", 1)[0]; + const contentType = String(headers["content-type"] || ""); + const chunks = []; + let responseStarted = false; + let finished = false; + stream.on("data", (chunk) => { + chunks.push(chunk); + const body = Buffer.concat(chunks); + receipt.cursor_inbound = { request_bytes: body.length, envelopes: envelopeShapes(body) }; + save(); + if (!responseStarted && body.includes(Buffer.from(args.nonce))) { + responseStarted = true; + receipt.initial_prompt_observed = true; + receipt.endpoint_observed = { + transport: "http2", + method: "POST", + path, + media_type: contentType.split(";", 1)[0].trim().toLowerCase(), + framing: "connect_envelope", + request_shape: { + service: "agent.v1.AgentService", + method: "Run", + stream: "bidirectional", + message: "agent.v1.AgentClientMessage", + }, + }; + receipt.cursor_run = { path, content_type: contentType, request_bytes: body.length }; + save(); + stream.respond({ ":status": 200, "content-type": "application/connect+proto" }); + if (args["response-fault"] === "text") { + receipt.fault_response_emitted = "text"; + save(); + stream.write(connectEnvelope(textUpdate("No tool call"))); + stream.write(connectEnvelope(turnEnded())); + stream.end(connectEnvelope(Buffer.from("{}"), 0x02)); + } else { + receipt.tool_call_response_emitted = true; + stream.write(connectEnvelope(shellExecution(`pitot-e2e-canary ${args.nonce}`))); + } + } + if (responseStarted && !finished && body.includes(Buffer.from(`PITOT_CANARY_RESULT ${args.nonce}`))) { + finished = true; + receipt.tool_result_observed = true; + receipt.final_response_emitted = true; + save(); + stream.write(connectEnvelope(textUpdate("Pitot E2E Verification Complete"))); + stream.write(connectEnvelope(turnEnded())); + // Connect end-stream envelope. Success metadata is an empty JSON object. + stream.end(connectEnvelope(Buffer.from("{}"), 0x02)); + } + }); + stream.on("end", () => { + if (responseStarted) return; + const body = Buffer.concat(chunks); + receipt.cursor_requests.push({ path, content_type: contentType, request_bytes: body.length }); + receipt.auxiliary_requests = receipt.cursor_requests.length; + save(); + const payload = unaryResponse(path); + stream.respond({ ":status": 200, "content-type": contentType.includes("json") ? "application/json" : "application/proto" }); + stream.end(contentType.includes("json") ? Buffer.from("{}") : payload); + }); +}); + +const http1Server = http.createServer((request, response) => { + const chunks = []; + request.on("data", (chunk) => chunks.push(chunk)); + request.on("end", () => { + const path = String(request.url || "").split("?", 1)[0]; + const contentType = String(request.headers["content-type"] || ""); + const body = Buffer.concat(chunks); + receipt.cursor_requests.push({ path, content_type: contentType, request_bytes: body.length }); + receipt.auxiliary_requests = receipt.cursor_requests.length; + save(); + const payload = unaryResponse(path); + response.writeHead(200, { "content-type": contentType.includes("json") ? "application/json" : "application/proto" }); + response.end(contentType.includes("json") ? Buffer.from("{}") : payload); + }); +}); + +const frontServer = net.createServer((client) => { + client.once("data", (first) => { + const isHttp2 = first.subarray(0, 14).toString() === "PRI * HTTP/2.0"; + const target = isHttp2 ? h2Server.address().port : http1Server.address().port; + const backend = net.connect(target, "127.0.0.1", () => backend.write(first)); + client.pipe(backend).pipe(client); + }); +}); + +h2Server.listen(0, "127.0.0.1", () => { + http1Server.listen(0, "127.0.0.1", () => { + frontServer.listen(0, "127.0.0.1", () => { + const address = frontServer.address(); + fs.writeFileSync(args["ready-file"], `http://127.0.0.1:${address.port}\n`); + save(); + }); + }); +}); + +for (const signal of ["SIGTERM", "SIGINT"]) { + process.on(signal, () => frontServer.close(() => { + h2Server.close(); + http1Server.close(() => process.exit(0)); + })); +} diff --git a/labs/15-pitot/tests/e2e_claude_cli_test.sh b/labs/15-pitot/tests/e2e_claude_cli_test.sh index d9116a5ae..ef2081095 100755 --- a/labs/15-pitot/tests/e2e_claude_cli_test.sh +++ b/labs/15-pitot/tests/e2e_claude_cli_test.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -exec labs/15-pitot/tests/e2e_unified_runner.sh "claude" +exec "$(dirname "$0")/e2e_unified_runner.sh" "claude" diff --git a/labs/15-pitot/tests/e2e_cline_cli_test.sh b/labs/15-pitot/tests/e2e_cline_cli_test.sh index 35ad78167..fb2ee63f5 100755 --- a/labs/15-pitot/tests/e2e_cline_cli_test.sh +++ b/labs/15-pitot/tests/e2e_cline_cli_test.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -exec labs/15-pitot/tests/e2e_unified_runner.sh "cline" +exec "$(dirname "$0")/e2e_unified_runner.sh" "cline" diff --git a/labs/15-pitot/tests/e2e_codex_cli_test.sh b/labs/15-pitot/tests/e2e_codex_cli_test.sh index bb60c30ae..a5bec375d 100755 --- a/labs/15-pitot/tests/e2e_codex_cli_test.sh +++ b/labs/15-pitot/tests/e2e_codex_cli_test.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -exec labs/15-pitot/tests/e2e_unified_runner.sh "codex" +exec "$(dirname "$0")/e2e_unified_runner.sh" "codex" diff --git a/labs/15-pitot/tests/e2e_copilot_cli_test.sh b/labs/15-pitot/tests/e2e_copilot_cli_test.sh index 3cce2b69f..c4134bf40 100755 --- a/labs/15-pitot/tests/e2e_copilot_cli_test.sh +++ b/labs/15-pitot/tests/e2e_copilot_cli_test.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -exec labs/15-pitot/tests/e2e_unified_runner.sh "copilot" +exec "$(dirname "$0")/e2e_unified_runner.sh" "copilot" diff --git a/labs/15-pitot/tests/e2e_cursor_cli_test.sh b/labs/15-pitot/tests/e2e_cursor_cli_test.sh index 780d6f97c..386c99e0f 100755 --- a/labs/15-pitot/tests/e2e_cursor_cli_test.sh +++ b/labs/15-pitot/tests/e2e_cursor_cli_test.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -exec labs/15-pitot/tests/e2e_unified_runner.sh "cursor" +exec "$(dirname "$0")/e2e_unified_runner.sh" "cursor" diff --git a/labs/15-pitot/tests/e2e_gemini_cli_test.sh b/labs/15-pitot/tests/e2e_gemini_cli_test.sh index 53ab3b9a4..9a10ba17c 100755 --- a/labs/15-pitot/tests/e2e_gemini_cli_test.sh +++ b/labs/15-pitot/tests/e2e_gemini_cli_test.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -exec labs/15-pitot/tests/e2e_unified_runner.sh "gemini" +exec "$(dirname "$0")/e2e_unified_runner.sh" "gemini" diff --git a/labs/15-pitot/tests/e2e_kimi_cli_test.sh b/labs/15-pitot/tests/e2e_kimi_cli_test.sh index e81e72c1a..724c9c724 100755 --- a/labs/15-pitot/tests/e2e_kimi_cli_test.sh +++ b/labs/15-pitot/tests/e2e_kimi_cli_test.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -exec labs/15-pitot/tests/e2e_unified_runner.sh "kimi" +exec "$(dirname "$0")/e2e_unified_runner.sh" "kimi" diff --git a/labs/15-pitot/tests/e2e_opencode_cli_test.sh b/labs/15-pitot/tests/e2e_opencode_cli_test.sh index 71ffa6611..b98a6f925 100755 --- a/labs/15-pitot/tests/e2e_opencode_cli_test.sh +++ b/labs/15-pitot/tests/e2e_opencode_cli_test.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -exec labs/15-pitot/tests/e2e_unified_runner.sh "opencode" +exec "$(dirname "$0")/e2e_unified_runner.sh" "opencode" diff --git a/labs/15-pitot/tests/e2e_pi_cli_test.sh b/labs/15-pitot/tests/e2e_pi_cli_test.sh index 698cce55d..53d031caf 100755 --- a/labs/15-pitot/tests/e2e_pi_cli_test.sh +++ b/labs/15-pitot/tests/e2e_pi_cli_test.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -exec labs/15-pitot/tests/e2e_unified_runner.sh "pi" +exec "$(dirname "$0")/e2e_unified_runner.sh" "pi" diff --git a/labs/15-pitot/tests/e2e_qwen_cli_test.sh b/labs/15-pitot/tests/e2e_qwen_cli_test.sh index 7a850e46d..807187e32 100755 --- a/labs/15-pitot/tests/e2e_qwen_cli_test.sh +++ b/labs/15-pitot/tests/e2e_qwen_cli_test.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -exec labs/15-pitot/tests/e2e_unified_runner.sh "qwen" +exec "$(dirname "$0")/e2e_unified_runner.sh" "qwen" diff --git a/labs/15-pitot/tests/e2e_unified_runner.sh b/labs/15-pitot/tests/e2e_unified_runner.sh index 3a56b6e7e..f6fcd5ff5 100755 --- a/labs/15-pitot/tests/e2e_unified_runner.sh +++ b/labs/15-pitot/tests/e2e_unified_runner.sh @@ -1,398 +1,100 @@ #!/usr/bin/env bash -# Unified integration test driver for supported host hook harnesses. +# Real released-agent prompt-to-hook E2E. Direct hook invocation is forbidden. set -euo pipefail HOST="${1:-}" -if [ -z "$HOST" ]; then - echo "ERROR: Missing host argument (claude, cline, codex, copilot, cursor, gemini, kimi, opencode, pi, qwen)" - exit 1 +if [[ -z "$HOST" ]]; then + echo "ERROR: a supervised agent ID is required" >&2 + exit 2 fi - -echo "===> [E2E] Starting $HOST + Pitot Hook Integration Test" - -# 1. Compile the local Go 'pitot' binary -echo "===> Compiling pitot binary..." -PITOT_BINARY="labs/15-pitot/tests/pitot" -if [ "${RUNNER_OS:-}" = "Windows" ]; then - PITOT_BINARY="${PITOT_BINARY}.exe" +: "${PITOT_E2E_PLATFORM:?PITOT_E2E_PLATFORM is required}" +: "${PITOT_INSTALL_RECEIPT:?PITOT_INSTALL_RECEIPT is required}" +: "${PITOT_E2E_EVIDENCE:?PITOT_E2E_EVIDENCE is required}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LAB_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +PITOT_MAIN="$LAB_DIR/pitot/cmd/pitot/main.go" +if [[ ! -f "$PITOT_MAIN" ]]; then + PITOT_MAIN="$LAB_DIR/cmd/pitot/main.go" fi -go build -o "$PITOT_BINARY" labs/15-pitot/pitot/cmd/pitot/main.go - -# 2. Host-specific setup and mocking -SERVER_PID="" -MOCK_HOME=$(mktemp -d) -echo "===> Created temporary mock home: $MOCK_HOME" -cleanup() { - echo "===> Cleaning up temporary files and servers..." - if [ -n "$SERVER_PID" ]; then - kill "$SERVER_PID" 2>/dev/null || true - fi - rm -f "$PITOT_BINARY" - if [ -n "${MOCK_HOME:-}" ]; then - rm -rf "$MOCK_HOME" +BUILD_DIR="${RUNNER_TEMP:-$(mktemp -d)}/pitot-real-e2e-bin" +mkdir -p "$BUILD_DIR" +PITOT_BINARY="$BUILD_DIR/pitot" +WITNESS_BINARY="$BUILD_DIR/pitot-witness" +if [[ "${RUNNER_OS:-}" == "Windows" ]]; then + PITOT_BINARY="${PITOT_BINARY}.exe" + WITNESS_BINARY="${WITNESS_BINARY}.exe" + if [[ "$HOST" == "cursor" ]]; then + # Cursor's supported Windows runtime is WSL. Keep both supervised + # executables native to that runtime; crossing back into a Windows Pitot + # process makes Cursor's hook pipe remain open after Pitot has returned. + PITOT_BINARY="$BUILD_DIR/pitot-linux" + WITNESS_BINARY="$BUILD_DIR/pitot-witness-linux" fi -} -trap cleanup EXIT - -# 3. Path discovery for real host CLI binary -CLAUDE_PATH="${CLAUDE_PATH:-/Users/apple/.local/bin/claude}" -if [ ! -f "$CLAUDE_PATH" ] && which claude &>/dev/null; then - CLAUDE_PATH=$(which claude) fi -CURSOR_PATH="/Applications/Cursor.app/Contents/Resources/app/bin/cursor" -if [ ! -f "$CURSOR_PATH" ] && which cursor &>/dev/null; then - CURSOR_PATH=$(which cursor) +if [[ "${RUNNER_OS:-}" == "Windows" && "$HOST" == "cursor" ]]; then + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o "$PITOT_BINARY" "$PITOT_MAIN" + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o "$WITNESS_BINARY" "$SCRIPT_DIR/witness/main.go" +else + go build -o "$PITOT_BINARY" "$PITOT_MAIN" + go build -o "$WITNESS_BINARY" "$SCRIPT_DIR/witness/main.go" fi -CODEX_PATH="" # Codex CLI path placeholder - -# Get an absolute path that the host process can execute on each runner OS. -PITOT_ABS_PATH="$(pwd)/$PITOT_BINARY" -if [ "${RUNNER_OS:-}" = "Windows" ] && command -v cygpath &>/dev/null; then - PITOT_ABS_PATH="$(cygpath -m "$PITOT_ABS_PATH")" +DRIVER_ARGS=( + --agent "$HOST" + --platform "$PITOT_E2E_PLATFORM" + --installation "$PITOT_INSTALL_RECEIPT" + --evidence "$PITOT_E2E_EVIDENCE" + --pitot "$PITOT_BINARY" + --witness "$WITNESS_BINARY" +) +if [[ -n "${PITOT_CAPTURE_OUTPUT:-}" ]]; then + DRIVER_ARGS+=(--capture-output "$PITOT_CAPTURE_OUTPUT") fi -# Spin up mock API server on port 8080 for all tests -echo "===> Starting local mock API server..." -node labs/15-pitot/tests/mock_anthropic_server.js & -SERVER_PID=$! -sleep 2 - -case "$HOST" in - "claude") - # Write settings file for PreToolUse hook - mkdir -p "$MOCK_HOME/.claude" - cat < "$MOCK_HOME/.claude/settings.json" -{ - "hooks": { - "PreToolUse": [ - { - "matcher": "Bash", - "hooks": [ - { - "type": "command", - "command": "$PITOT_ABS_PATH hook claude" - } - ] - } - ] - } -} -SETTINGS_EOF - - # Check if real binary is installed - if [ ! -f "$CLAUDE_PATH" ]; then - echo "===> [FAILURE] Real 'claude' CLI binary not found on this machine." - exit 1 - fi - - echo "===> Launching real Claude CLI against mock API server..." - OUTPUT=$(HOME="$MOCK_HOME" \ - ANTHROPIC_BASE_URL="http://localhost:8080" \ - ANTHROPIC_API_KEY="sk-ant-dummy" \ - "$CLAUDE_PATH" --print --model sonnet "list directory" 2>&1) || OUTPUT="Command failed: $OUTPUT" - - echo "===> Claude CLI execution output:" - echo "----------------------------------------" - echo "$OUTPUT" - echo "----------------------------------------" - - if echo "$OUTPUT" | grep -q "E2E Verification Complete"; then - echo "===> [SUCCESS] $HOST end-to-end integration test passed perfectly!" - echo "PITOT_E2E_RESULT mode=real_cli" - exit 0 - else - echo "===> [FAILURE] $HOST end-to-end integration test failed." - exit 1 - fi - ;; - - "cursor") - # Write Cursor settings file for beforeShellExecution hook - mkdir -p "$MOCK_HOME/.cursor" - cat < "$MOCK_HOME/.cursor/settings.json" -{ - "hooks": { - "beforeShellExecution": [ - { - "command": "$PITOT_ABS_PATH hook cursor" - } - ] - } -} -SETTINGS_EOF - - # Cursor terminal agent command is called 'agent' - REAL_CURSOR_BIN="agent" - if [ -f "$CURSOR_PATH" ]; then - REAL_CURSOR_BIN="$CURSOR_PATH" - fi - - # Check if real binary is installed - HAS_REAL_BIN=false - if [ -f "$CURSOR_PATH" ] || which agent &>/dev/null; then - HAS_REAL_BIN=true - fi - - # Prepare Cursor payload for fallback/direct verification - PAYLOAD='{"hook_event_name": "beforeShellExecution", "command": "npm install"}' - - RUN_REAL_E2E=false - if [ "$HAS_REAL_BIN" = true ]; then - echo "===> Launching real Cursor agent CLI against mock API server..." - OUTPUT=$(HOME="$MOCK_HOME" \ - OPENAI_BASE_URL="http://localhost:8080" \ - AGENT_BASE_URL="http://localhost:8080" \ - CURSOR_BASE_URL="http://localhost:8080" \ - OPENAI_API_KEY="sk-opt-dummy" \ - CURSOR_API_KEY="sk-opt-dummy" \ - AGENT_API_KEY="sk-opt-dummy" \ - "$REAL_CURSOR_BIN" -p "list directory contents" 2>&1) || OUTPUT="Command failed: $OUTPUT" - - # Filter out harmless electron/chromium warning - OUTPUT=$(echo "$OUTPUT" | grep -v "Warning: 'p' is not in the list of known options" || true) - - echo "===> Cursor CLI execution output:" - echo "----------------------------------------" - echo "$OUTPUT" - echo "----------------------------------------" - - if echo "$OUTPUT" | grep -q "E2E Verification Complete"; then - RUN_REAL_E2E=true - elif echo "$OUTPUT" | grep -E -q "Authentication required|provided API key is invalid" || [ -z "$(echo "$OUTPUT" | tr -d '[:space:]')" ]; then - echo "WARNING: Real Cursor CLI failed due to hard-locked production authentication or exited silently. Falling back to active subprocess hook verification." - else - echo "===> [FAILURE] Real Cursor CLI execution crashed with an unexpected error." - exit 1 - fi - fi - - if [ "$RUN_REAL_E2E" = true ]; then - echo "===> [SUCCESS] $HOST end-to-end integration test passed perfectly using real CLI!" - echo "PITOT_E2E_RESULT mode=real_cli" - exit 0 - else - echo "===> [E2E] Running active Cursor subprocess hook verification..." - OUTPUT=$(echo "$PAYLOAD" | "$PITOT_ABS_PATH" hook cursor 2>&1) - echo "===> Cursor hook execution output:" - echo "----------------------------------------" - echo "$OUTPUT" - echo "----------------------------------------" - - if echo "$OUTPUT" | grep -q '"type":"action.requested"'; then - echo "===> [SUCCESS] $HOST active subprocess hook verification passed perfectly!" - echo "PITOT_E2E_RESULT mode=hook_subprocess" - exit 0 - else - echo "===> [FAILURE] $HOST active subprocess hook verification failed." - exit 1 - fi - fi - ;; - - "codex") - # Write Codex configuration files - mkdir -p "$MOCK_HOME/.codex" - cat < "$MOCK_HOME/.codex/config.toml" -[hooks] -codex_hooks = true -CONFIG_EOF - - cat < "$MOCK_HOME/.codex/hooks.json" -{ - "hooks": { - "PreToolUse": [ - { - "matcher": "Bash", - "hooks": [ - { - "type": "command", - "command": "$PITOT_ABS_PATH hook codex" - } - ] - } - ] +DRIVER=(python3 "$SCRIPT_DIR/real_agent_driver.py" "${DRIVER_ARGS[@]}") +NEGATIVE_EVIDENCE="${PITOT_E2E_EVIDENCE}.wrong-response" +if [[ "${RUNNER_OS:-}" == "Windows" && "$HOST" == "cursor" ]]; then + # Run the complete controller in WSL so its temporary home, project, proxy, + # hook, and child processes use the same native filesystem/process boundary. + to_wsl_path() { + local windows_path drive tail + windows_path="$(cygpath -am "$1")" + if [[ ! "$windows_path" =~ ^[A-Za-z]:/ ]]; then + echo "ERROR: cannot map path into WSL: $windows_path" >&2 + return 1 + fi + drive="${windows_path:0:1}" + tail="${windows_path:2}" + printf '/mnt/%s%s' "${drive,,}" "$tail" } -} -CONFIG_EOF - - REAL_CODEX_BIN="codex" - if [ -n "$CODEX_PATH" ] && [ -f "$CODEX_PATH" ]; then - REAL_CODEX_BIN="$CODEX_PATH" - fi - - HAS_REAL_BIN=false - if [ -n "$CODEX_PATH" ] && [ -f "$CODEX_PATH" ] || which codex &>/dev/null; then - HAS_REAL_BIN=true - fi - - PAYLOAD='{"hook_event_name": "PreToolUse", "tool_name": "Bash", "tool_input": {"command": "git status"}}' - - RUN_REAL_E2E=false - if [ "$HAS_REAL_BIN" = true ]; then - echo "===> Launching real Codex CLI against mock API server..." - OUTPUT=$(HOME="$MOCK_HOME" \ - OPENAI_BASE_URL="http://localhost:8080/v1" \ - OPENAI_API_KEY="sk-opt-dummy" \ - "$REAL_CODEX_BIN" exec \ - --dangerously-bypass-approvals-and-sandbox \ - --dangerously-bypass-hook-trust \ - "list directory contents" 2>&1) || OUTPUT="Command failed: $OUTPUT" - - echo "===> Codex CLI execution output:" - echo "----------------------------------------" - echo "$OUTPUT" - echo "----------------------------------------" - - if echo "$OUTPUT" | grep -q "E2E Verification Complete"; then - RUN_REAL_E2E=true - elif echo "$OUTPUT" | grep -E -q -i "Authentication required|provided API key is invalid|401 Unauthorized|Missing bearer"; then - echo "WARNING: Real Codex CLI failed due to production authentication requirements. Falling back to active subprocess hook verification." - else - echo "===> [FAILURE] Real Codex CLI execution crashed with an unexpected error." - exit 1 - fi - fi - - if [ "$RUN_REAL_E2E" = true ]; then - echo "===> [SUCCESS] $HOST end-to-end integration test passed perfectly using real CLI!" - echo "PITOT_E2E_RESULT mode=real_cli" - exit 0 - else - echo "===> [E2E] Running active Codex subprocess hook verification..." - OUTPUT=$(echo "$PAYLOAD" | "$PITOT_ABS_PATH" hook codex 2>&1) - echo "===> Codex hook execution output:" - echo "----------------------------------------" - echo "$OUTPUT" - echo "----------------------------------------" - - if echo "$OUTPUT" | grep -q '"type":"action.requested"'; then - echo "===> [SUCCESS] $HOST active subprocess hook verification passed perfectly!" - echo "PITOT_E2E_RESULT mode=hook_subprocess" - exit 0 - else - echo "===> [FAILURE] $HOST active subprocess hook verification failed." - exit 1 - fi - fi - ;; - - "gemini") - PAYLOAD='{"hook_event_name": "BeforeTool", "tool_name": "run_shell_command", "tool_input": {"command": "git status"}}' - echo "===> [E2E] Running active Gemini subprocess hook verification..." - OUTPUT=$(echo "$PAYLOAD" | "$PITOT_ABS_PATH" hook gemini 2>&1) - echo "===> Gemini hook execution output:" - echo "----------------------------------------" - echo "$OUTPUT" - echo "----------------------------------------" - - if echo "$OUTPUT" | grep -q '"type":"action.requested"'; then - echo "===> [SUCCESS] $HOST active subprocess hook verification passed perfectly!" - echo "PITOT_E2E_RESULT mode=hook_subprocess" - exit 0 - else - echo "===> [FAILURE] $HOST active subprocess hook verification failed." - exit 1 - fi - ;; - - "kimi") - PAYLOAD='{"hook_event_name": "PreToolUse", "tool_name": "Bash", "tool_input": {"command": "git status"}}' - echo "===> [E2E] Running active Kimi Code subprocess hook verification..." - OUTPUT=$(echo "$PAYLOAD" | "$PITOT_ABS_PATH" hook kimi 2>&1) - echo "===> Kimi Code hook execution output:" - echo "----------------------------------------" - echo "$OUTPUT" - echo "----------------------------------------" - - if echo "$OUTPUT" | grep -q '"type":"action.requested"'; then - echo "===> [SUCCESS] $HOST active subprocess hook verification passed perfectly!" - echo "PITOT_E2E_RESULT mode=hook_subprocess" - exit 0 - else - echo "===> [FAILURE] $HOST active subprocess hook verification failed." - exit 1 - fi - ;; - - "opencode") - PAYLOAD='{"hook_event_name": "PreToolUse", "tool_name": "Bash", "tool_input": {"command": "git status"}}' - echo "===> [E2E] Running active Opencode subprocess hook verification..." - OUTPUT=$(echo "$PAYLOAD" | "$PITOT_ABS_PATH" hook opencode 2>&1) - echo "===> Opencode hook execution output:" - echo "----------------------------------------" - echo "$OUTPUT" - echo "----------------------------------------" - - if echo "$OUTPUT" | grep -q '"type":"action.requested"'; then - echo "===> [SUCCESS] $HOST active subprocess hook verification passed perfectly!" - echo "PITOT_E2E_RESULT mode=hook_subprocess" - exit 0 - else - echo "===> [FAILURE] $HOST active subprocess hook verification failed." - exit 1 - fi - ;; - - "copilot"|"qwen") - PAYLOAD='{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"git status"}}' - OUTPUT=$(echo "$PAYLOAD" | "$PITOT_ABS_PATH" hook "$HOST" 2>&1) - if echo "$OUTPUT" | grep -q '"type":"action.requested"'; then - echo "===> [SUCCESS] $HOST active subprocess hook verification passed!" - echo "PITOT_E2E_RESULT mode=hook_subprocess" - exit 0 - fi - echo "===> [FAILURE] $HOST active subprocess hook verification failed: $OUTPUT" - exit 1 - ;; - - "pi") - PI_MODULE="$MOCK_HOME/pitot.mjs" - cp labs/15-pitot/integrations/pi/pitot.ts "$PI_MODULE" - PI_OUTPUT=$(PITOT_BIN="$PITOT_ABS_PATH" node --input-type=module - "$PI_MODULE" <<'NODE' -import { pathToFileURL } from "node:url"; -const modulePath = process.argv[2]; -const { handleToolCall } = await import(pathToFileURL(modulePath)); -const allow = handleToolCall({toolName: "bash", input: {command: "git status"}}); -const block = handleToolCall({toolName: "bash", input: {command: ""}}); -console.log(JSON.stringify({allow: allow === undefined, block: block?.block === true})); -NODE - ) - if echo "$PI_OUTPUT" | grep -q '"allow":true,"block":true'; then - echo "===> [SUCCESS] Pi extension allow/block translation passed!" - echo "PITOT_E2E_RESULT mode=hook_subprocess" - exit 0 - fi - echo "===> [FAILURE] Pi extension verification failed: $PI_OUTPUT" - exit 1 - ;; + WSL_ARGS=( + --agent "$HOST" + --platform "$PITOT_E2E_PLATFORM" + --installation "$(to_wsl_path "$PITOT_INSTALL_RECEIPT")" + --evidence "$(to_wsl_path "$PITOT_E2E_EVIDENCE")" + --pitot "$(to_wsl_path "$PITOT_BINARY")" + --witness "$(to_wsl_path "$WITNESS_BINARY")" + ) + if [[ -n "${PITOT_CAPTURE_OUTPUT:-}" ]]; then + WSL_ARGS+=(--capture-output "$(to_wsl_path "$PITOT_CAPTURE_OUTPUT")") + fi + # Git Bash otherwise rewrites /mnt/ arguments as paths beneath its + # own installation before wsl.exe can receive them. + export MSYS2_ARG_CONV_EXCL='*' + DRIVER=( + wsl.exe --distribution Ubuntu -- env + PITOT_SOURCE_SHA="${PITOT_SOURCE_SHA:-}" + python3 "$(to_wsl_path "$SCRIPT_DIR/real_agent_driver.py")" "${WSL_ARGS[@]}" + ) + NEGATIVE_EVIDENCE="$(to_wsl_path "${PITOT_E2E_EVIDENCE}.wrong-response")" +fi - "cline") - ALLOW_PAYLOAD='{"hookName":"PreToolUse","preToolUse":{"tool":"execute_command","parameters":{"command":"git status"}}}' - BLOCK_PAYLOAD='{"hookName":"PreToolUse","preToolUse":{"tool":"execute_command","parameters":{"command":""}}}' - PASS_PAYLOAD='{"hookName":"PreToolUse","preToolUse":{"tool":"read_file","parameters":{"path":"README.md"}}}' - if [ "${RUNNER_OS:-}" = "Windows" ] && command -v pwsh &>/dev/null; then - ALLOW_OUTPUT=$(echo "$ALLOW_PAYLOAD" | PITOT_BIN="$PITOT_ABS_PATH" pwsh -NoProfile -File labs/15-pitot/integrations/cline/PreToolUse.ps1) - BLOCK_OUTPUT=$(echo "$BLOCK_PAYLOAD" | PITOT_BIN="$PITOT_ABS_PATH" pwsh -NoProfile -File labs/15-pitot/integrations/cline/PreToolUse.ps1) - PASS_OUTPUT=$(echo "$PASS_PAYLOAD" | PITOT_BIN="$PITOT_ABS_PATH" pwsh -NoProfile -File labs/15-pitot/integrations/cline/PreToolUse.ps1) - else - ALLOW_OUTPUT=$(echo "$ALLOW_PAYLOAD" | PITOT_BIN="$PITOT_ABS_PATH" bash labs/15-pitot/integrations/cline/PreToolUse) - BLOCK_OUTPUT=$(echo "$BLOCK_PAYLOAD" | PITOT_BIN="$PITOT_ABS_PATH" bash labs/15-pitot/integrations/cline/PreToolUse) - PASS_OUTPUT=$(echo "$PASS_PAYLOAD" | PITOT_BIN="$PITOT_ABS_PATH" bash labs/15-pitot/integrations/cline/PreToolUse) - fi - if echo "$ALLOW_OUTPUT" | grep -q '"cancel":false' && echo "$BLOCK_OUTPUT" | grep -q '"cancel":true' && echo "$PASS_OUTPUT" | grep -q '"cancel":false'; then - echo "===> [SUCCESS] Cline bridge allow/block/non-shell translation passed!" - echo "PITOT_E2E_RESULT mode=hook_subprocess" - exit 0 - fi - echo "===> [FAILURE] Cline bridge verification failed: $ALLOW_OUTPUT / $BLOCK_OUTPUT" - exit 1 - ;; +"${DRIVER[@]}" - *) - echo "ERROR: Unsupported host $HOST" - exit 1 - ;; -esac +"${DRIVER[@]}" \ + --evidence "$NEGATIVE_EVIDENCE" \ + --response-fault text \ + --expect-incompatible-response +echo "PASS: $HOST rejected incompatible proxy response evidence" diff --git a/labs/15-pitot/tests/endpoint-provenance.json b/labs/15-pitot/tests/endpoint-provenance.json new file mode 100644 index 000000000..d10d8f3c9 --- /dev/null +++ b/labs/15-pitot/tests/endpoint-provenance.json @@ -0,0 +1,1469 @@ +{ + "capture_policy": "pinned_real_cli_capture", + "cells": [ + { + "agent": "claude", + "capture_sha256": "f2eb7045e2370f18521c812d5e5d141f65c41d59d0c488fb8040698d2973c951", + "dialect": "anthropic_messages", + "executable_sha256": "5840c777fd47115e9ca276e165563c6e121e7c7e2b4d86598e0025f8cc37de56", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/messages", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "Bash" + ], + "top_level_keys": [ + "context_management", + "max_tokens", + "messages", + "metadata", + "model", + "output_config", + "stream", + "system", + "thinking", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "anthropic_messages", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "2.1.217" + }, + { + "agent": "claude", + "capture_sha256": "34f96279d0005b279c4287f18a7fa22d8726e8cdcdb3b124ee5b9a53058f0319", + "dialect": "anthropic_messages", + "executable_sha256": "2630fc5dc6db61bc03f86b95daf47766e5ed5b61873f7bb7cfea764c5ac5a9ba", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/messages", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "Bash" + ], + "top_level_keys": [ + "context_management", + "max_tokens", + "messages", + "metadata", + "model", + "output_config", + "stream", + "system", + "thinking", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "anthropic_messages", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "2.1.217" + }, + { + "agent": "claude", + "capture_sha256": "bb70ed5c57eeb11082dbb9738b7abce34496240bd5f16bbd8ded27ef2877b575", + "dialect": "anthropic_messages", + "executable_sha256": "7999fba95dbffe167d9e0a043f29057979a0518ebe89b60c4fcfc6401ea8c424", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/messages", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "Bash" + ], + "top_level_keys": [ + "context_management", + "max_tokens", + "messages", + "metadata", + "model", + "output_config", + "stream", + "system", + "thinking", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "anthropic_messages", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "2.1.217" + }, + { + "agent": "cline", + "capture_sha256": "badbd04142c73d323def070ed927eee99b51e1844c7737dfd8c1cda20389f5eb", + "dialect": "openai_chat", + "executable_sha256": "71d1b27aeeebdaa0b91c4babddd1f635eb7f5783257c2b3324a2cb22924cf60f", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "read_files", + "search_codebase", + "run_commands", + "fetch_web_content", + "editor", + "ask_question", + "spawn_agent", + "team_spawn_teammate", + "team_shutdown_teammate", + "team_status", + "team_task", + "team_run_task", + "team_cancel_run", + "team_list_runs", + "team_await_runs", + "team_send_message", + "team_broadcast", + "team_read_mailbox", + "team_mission_log", + "team_cleanup", + "team_create_outcome", + "team_attach_outcome_fragment", + "team_review_outcome_fragment", + "team_finalize_outcome", + "team_list_outcomes" + ], + "top_level_keys": [ + "messages", + "model", + "stream", + "stream_options", + "tool_choice", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "3.0.46" + }, + { + "agent": "cline", + "capture_sha256": "2181eafb944aa71dfe0a2d55ef04fa77d38248f842537660fbf33a4a814fdc93", + "dialect": "openai_chat", + "executable_sha256": "71d1b27aeeebdaa0b91c4babddd1f635eb7f5783257c2b3324a2cb22924cf60f", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "read_files", + "search_codebase", + "run_commands", + "fetch_web_content", + "editor", + "ask_question", + "spawn_agent", + "team_spawn_teammate", + "team_shutdown_teammate", + "team_status", + "team_task", + "team_run_task", + "team_cancel_run", + "team_list_runs", + "team_await_runs", + "team_send_message", + "team_broadcast", + "team_read_mailbox", + "team_mission_log", + "team_cleanup", + "team_create_outcome", + "team_attach_outcome_fragment", + "team_review_outcome_fragment", + "team_finalize_outcome", + "team_list_outcomes" + ], + "top_level_keys": [ + "messages", + "model", + "stream", + "stream_options", + "tool_choice", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "3.0.46" + }, + { + "agent": "cline", + "capture_sha256": "8d15b6c7a2dc4e98ae12c5ec82ddff3a721dcde7bf01f0090d18a006d24803c2", + "dialect": "openai_chat", + "executable_sha256": "aabf94572866f85132156953d5d5c36a6c7a39ea9456a4eade7fc0df957685fa", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "read_files", + "search_codebase", + "run_commands", + "fetch_web_content", + "editor", + "ask_question", + "spawn_agent", + "team_spawn_teammate", + "team_shutdown_teammate", + "team_status", + "team_task", + "team_run_task", + "team_cancel_run", + "team_list_runs", + "team_await_runs", + "team_send_message", + "team_broadcast", + "team_read_mailbox", + "team_mission_log", + "team_cleanup", + "team_create_outcome", + "team_attach_outcome_fragment", + "team_review_outcome_fragment", + "team_finalize_outcome", + "team_list_outcomes" + ], + "top_level_keys": [ + "messages", + "model", + "stream", + "stream_options", + "tool_choice", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "3.0.46" + }, + { + "agent": "codex", + "capture_sha256": "5cec7ac20d1e1c550fc1fd1fc5cb96ef1e04b13d1212ea51125eaf2349460788", + "dialect": "openai_responses", + "executable_sha256": "134063e133f0b4244fa3b251acf973d4fe4b4aeeacbdc135211bf480f59f1477", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/responses", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "exec_command", + "write_stdin", + "update_plan", + "request_user_input", + "view_image", + "multi_agent_v1", + "get_goal", + "create_goal", + "update_goal" + ], + "top_level_keys": [ + "client_metadata", + "include", + "input", + "instructions", + "model", + "parallel_tool_calls", + "prompt_cache_key", + "reasoning", + "store", + "stream", + "tool_choice", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_responses", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.145.0" + }, + { + "agent": "codex", + "capture_sha256": "b105f6d402c363f9734553bbaaa7510be71df2f6774f4b1040935dcabcfa1815", + "dialect": "openai_responses", + "executable_sha256": "134063e133f0b4244fa3b251acf973d4fe4b4aeeacbdc135211bf480f59f1477", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/responses", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "exec_command", + "write_stdin", + "update_plan", + "request_user_input", + "view_image", + "multi_agent_v1", + "get_goal", + "create_goal", + "update_goal" + ], + "top_level_keys": [ + "client_metadata", + "include", + "input", + "instructions", + "model", + "parallel_tool_calls", + "prompt_cache_key", + "reasoning", + "store", + "stream", + "tool_choice", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_responses", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.145.0" + }, + { + "agent": "codex", + "capture_sha256": "fe466cdc0cd79fd12db0c06ba7b37d516bbfade7ed22e1fa9c61f740369e4c12", + "dialect": "openai_responses", + "executable_sha256": "c54db6755e710c39703f7c37512f9e35ed41042d8080558d2b84b8d2694323c3", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/responses", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "shell_command", + "update_plan", + "request_user_input", + "view_image", + "multi_agent_v1", + "get_goal", + "create_goal", + "update_goal" + ], + "top_level_keys": [ + "client_metadata", + "include", + "input", + "instructions", + "model", + "parallel_tool_calls", + "prompt_cache_key", + "reasoning", + "store", + "stream", + "tool_choice", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_responses", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.145.0" + }, + { + "agent": "copilot", + "capture_sha256": "69ed781538ee899b31e85061f0c2e8ec3d2ca40244fff5d6a838d1908ce11462", + "dialect": "openai_chat", + "executable_sha256": "0ea824a86be5757533fdb092eff7050871bd7a711a46babde0ffe0e44ac5ad88", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "bash", + "read_bash", + "stop_bash", + "list_bash", + "apply_patch", + "view", + "fetch_copilot_cli_documentation", + "skill", + "sql", + "session_store_sql", + "read_agent", + "list_agents", + "write_agent", + "rg", + "glob", + "task" + ], + "top_level_keys": [ + "messages", + "model", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "1.0.73" + }, + { + "agent": "copilot", + "capture_sha256": "0a30d6ef202ef5743e1c1026122f524747862e1d445d8deeb6403d4eccb9810a", + "dialect": "openai_chat", + "executable_sha256": "0ea824a86be5757533fdb092eff7050871bd7a711a46babde0ffe0e44ac5ad88", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "bash", + "read_bash", + "stop_bash", + "list_bash", + "apply_patch", + "view", + "fetch_copilot_cli_documentation", + "skill", + "sql", + "session_store_sql", + "read_agent", + "list_agents", + "write_agent", + "rg", + "glob", + "task" + ], + "top_level_keys": [ + "messages", + "model", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "1.0.73" + }, + { + "agent": "copilot", + "capture_sha256": "07b91e3b6215218295878f1e449ba61bddcd860de1393756abc1d0b4f502d224", + "dialect": "openai_chat", + "executable_sha256": "959e8ea2c63474956adc1de9c884df9018a686a9c5463e8366b438161c01c8f8", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "powershell", + "read_powershell", + "stop_powershell", + "list_powershell", + "apply_patch", + "view", + "fetch_copilot_cli_documentation", + "skill", + "sql", + "session_store_sql", + "read_agent", + "list_agents", + "write_agent", + "rg", + "glob", + "task" + ], + "top_level_keys": [ + "messages", + "model", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "1.0.73" + }, + { + "agent": "cursor", + "capture_sha256": "143a5f1053a9f08cdbd3c9b60d7d9bdc635ecc7fcf13743c995b9929da450b1d", + "dialect": "cursor_connect_proto", + "executable_sha256": "eed61c5224668c9236334c4c68936a16aecc37374b592f59e31eb50433817831", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "connect_envelope", + "media_type": "application/connect+proto", + "method": "POST", + "path": "/agent.v1.AgentService/Run", + "request_shape": { + "message": "agent.v1.AgentClientMessage", + "method": "Run", + "service": "agent.v1.AgentService", + "stream": "bidirectional" + }, + "transport": "http2" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "cursor_connect_proto", + "framing": "connect_envelope", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "2026.07.20-8cc9c0b" + }, + { + "agent": "cursor", + "capture_sha256": "9cfbafc5b68a8622d6a53e8526f67c2c1283d5a4fa2c329207a4b376e5d60af7", + "dialect": "cursor_connect_proto", + "executable_sha256": "eed61c5224668c9236334c4c68936a16aecc37374b592f59e31eb50433817831", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "connect_envelope", + "media_type": "application/connect+proto", + "method": "POST", + "path": "/agent.v1.AgentService/Run", + "request_shape": { + "message": "agent.v1.AgentClientMessage", + "method": "Run", + "service": "agent.v1.AgentService", + "stream": "bidirectional" + }, + "transport": "http2" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "cursor_connect_proto", + "framing": "connect_envelope", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "2026.07.20-8cc9c0b" + }, + { + "agent": "cursor", + "capture_sha256": "9cadbd1499db106aee6479bec218227552d5a8b804ec194f5c550db77aec5763", + "dialect": "cursor_connect_proto", + "executable_sha256": "eed61c5224668c9236334c4c68936a16aecc37374b592f59e31eb50433817831", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "connect_envelope", + "media_type": "application/connect+proto", + "method": "POST", + "path": "/agent.v1.AgentService/Run", + "request_shape": { + "message": "agent.v1.AgentClientMessage", + "method": "Run", + "service": "agent.v1.AgentService", + "stream": "bidirectional" + }, + "transport": "http2" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "cursor_connect_proto", + "framing": "connect_envelope", + "tool_call": "native_shell" + }, + "runtime": "wsl", + "version": "2026.07.20-8cc9c0b" + }, + { + "agent": "gemini", + "capture_sha256": "59a1e83ce73a7a4f8912e0b8780ae084e6c35814952dc4d3930406fc8f1770d7", + "dialect": "gemini_generate_content", + "executable_sha256": "a2533ac23365a1c72b847780b53235d38cf21cc7ee736c98dfa8cbb70b1a9425", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1beta/models/pitot-control:streamGenerateContent", + "request_shape": { + "has_tools": true, + "stream": false, + "tool_names": [ + "update_topic", + "list_directory", + "read_file", + "grep_search", + "glob", + "replace", + "write_file", + "web_fetch", + "run_shell_command", + "list_background_processes", + "read_background_output", + "google_web_search", + "enter_plan_mode", + "invoke_agent", + "activate_skill" + ], + "top_level_keys": [ + "contents", + "generationConfig", + "systemInstruction", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "gemini_generate_content", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.51.0" + }, + { + "agent": "gemini", + "capture_sha256": "f0f7bf0d7b2e88c03194449ccb25b8a7a534ee2d050b5f560fc578df069ef9c9", + "dialect": "gemini_generate_content", + "executable_sha256": "a2533ac23365a1c72b847780b53235d38cf21cc7ee736c98dfa8cbb70b1a9425", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1beta/models/pitot-control:streamGenerateContent", + "request_shape": { + "has_tools": true, + "stream": false, + "tool_names": [ + "update_topic", + "list_directory", + "read_file", + "grep_search", + "glob", + "replace", + "write_file", + "web_fetch", + "run_shell_command", + "list_background_processes", + "read_background_output", + "google_web_search", + "enter_plan_mode", + "invoke_agent", + "activate_skill" + ], + "top_level_keys": [ + "contents", + "generationConfig", + "systemInstruction", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "gemini_generate_content", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.51.0" + }, + { + "agent": "gemini", + "capture_sha256": "e3e0a192ec038c2fbc76ace53288ec14ebb6032954cea331c8eac495f0a928a6", + "dialect": "gemini_generate_content", + "executable_sha256": "ed3d5b269acefc5dffb342df0a214b2351794244c520c124d3556bc5ea6064c5", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1beta/models/pitot-control:streamGenerateContent", + "request_shape": { + "has_tools": true, + "stream": false, + "tool_names": [ + "update_topic", + "list_directory", + "read_file", + "grep_search", + "glob", + "replace", + "write_file", + "web_fetch", + "run_shell_command", + "list_background_processes", + "read_background_output", + "google_web_search", + "enter_plan_mode", + "invoke_agent", + "activate_skill" + ], + "top_level_keys": [ + "contents", + "generationConfig", + "systemInstruction", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "gemini_generate_content", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.51.0" + }, + { + "agent": "kimi", + "capture_sha256": "5c5a51e69a4686eedf0ed78648fbff64ecb850636404173dc6c061acd6b4bcad", + "dialect": "openai_chat", + "executable_sha256": "5cccf53604f20c5499ea10c3094298f49a1ad59fa90cddd9fd7e0ba44815fdd3", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "Agent", + "AgentSwarm", + "AskUserQuestion", + "Bash", + "CreateGoal", + "CronCreate", + "CronDelete", + "CronList", + "Edit", + "EnterPlanMode", + "ExitPlanMode", + "FetchURL", + "GetGoal", + "Glob", + "Grep", + "Read", + "SetGoalBudget", + "Skill", + "TaskList", + "TaskOutput", + "TaskStop", + "TodoList", + "UpdateGoal", + "Write" + ], + "top_level_keys": [ + "max_tokens", + "messages", + "model", + "prompt_cache_key", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.29.0" + }, + { + "agent": "kimi", + "capture_sha256": "11ece29c94e071bec5f206d9ba00e19233db733be791457d21c7a925bc8d20eb", + "dialect": "openai_chat", + "executable_sha256": "44f0aed58655790b78fe8ac1bf44217f3fc0e65977abb7426f9edca943ddc444", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "Agent", + "AgentSwarm", + "AskUserQuestion", + "Bash", + "CreateGoal", + "CronCreate", + "CronDelete", + "CronList", + "Edit", + "EnterPlanMode", + "ExitPlanMode", + "FetchURL", + "GetGoal", + "Glob", + "Grep", + "Read", + "SetGoalBudget", + "Skill", + "TaskList", + "TaskOutput", + "TaskStop", + "TodoList", + "UpdateGoal", + "Write" + ], + "top_level_keys": [ + "max_tokens", + "messages", + "model", + "prompt_cache_key", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.29.0" + }, + { + "agent": "kimi", + "capture_sha256": "4e0b6ea5c3c513f1e281a8a2d25e32025d9fb11bf995163fdc74a704b42bf44a", + "dialect": "openai_chat", + "executable_sha256": "c9daf5aec27ca9c35597a14d5cf05692f0aaf1d31d4d825ae6c988c404ecca6e", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "Agent", + "AgentSwarm", + "AskUserQuestion", + "Bash", + "CreateGoal", + "CronCreate", + "CronDelete", + "CronList", + "Edit", + "EnterPlanMode", + "ExitPlanMode", + "FetchURL", + "GetGoal", + "Glob", + "Grep", + "Read", + "SetGoalBudget", + "Skill", + "TaskList", + "TaskOutput", + "TaskStop", + "TodoList", + "UpdateGoal", + "Write" + ], + "top_level_keys": [ + "max_tokens", + "messages", + "model", + "prompt_cache_key", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.29.0" + }, + { + "agent": "opencode", + "capture_sha256": "db834ed83cf489dc517c27e2149f9783ae663c20055c2283dabcbdf08ff86457", + "dialect": "openai_responses", + "executable_sha256": "9449af91f517eacc2b0742fa93ae0da64fa6e5db7b714e30c62edea2a8de3f98", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/responses", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "bash", + "edit", + "glob", + "grep", + "read", + "skill", + "task", + "todowrite", + "webfetch", + "write" + ], + "top_level_keys": [ + "input", + "model", + "prompt_cache_key", + "store", + "stream", + "tool_choice", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_responses", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "1.18.4" + }, + { + "agent": "opencode", + "capture_sha256": "8c98437528ae0a8f99e007b94cb1176a17943e9fbbd914643f60733152f4ba60", + "dialect": "openai_responses", + "executable_sha256": "6ce6570e7db9a40e7bd3304ebdfff607920bde8cafd2eb5587bd7a26f89ba0b5", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/responses", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "bash", + "edit", + "glob", + "grep", + "read", + "skill", + "task", + "todowrite", + "webfetch", + "write" + ], + "top_level_keys": [ + "input", + "model", + "prompt_cache_key", + "store", + "stream", + "tool_choice", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_responses", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "1.18.4" + }, + { + "agent": "opencode", + "capture_sha256": "509555970aabc2e5aa178412295bd0d773467cdeb22d2b95000e2e5ac1d17ad4", + "dialect": "openai_responses", + "executable_sha256": "b53b698473bfa46e09487e485a7f1ad5b4881f8a8b319d3619aa251f3be8ae10", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/responses", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "bash", + "edit", + "glob", + "grep", + "read", + "skill", + "task", + "todowrite", + "webfetch", + "write" + ], + "top_level_keys": [ + "input", + "model", + "prompt_cache_key", + "store", + "stream", + "tool_choice", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_responses", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "1.18.4" + }, + { + "agent": "pi", + "capture_sha256": "fbad33dcce5982fee3ab29cacb251afbce95e15d319cf48f2933031e5a4e0df0", + "dialect": "openai_chat", + "executable_sha256": "af302f231437eaf6f37691bce4b34234fcb626bcb5eb3910d4fc3f6519bf78ca", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "read", + "bash", + "edit", + "write" + ], + "top_level_keys": [ + "max_completion_tokens", + "messages", + "model", + "store", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.81.1" + }, + { + "agent": "pi", + "capture_sha256": "c747e94254dc01646aa3624bb95b2a084a9ecc7a3ebb8f82322cceef051bfe0e", + "dialect": "openai_chat", + "executable_sha256": "af302f231437eaf6f37691bce4b34234fcb626bcb5eb3910d4fc3f6519bf78ca", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "read", + "bash", + "edit", + "write" + ], + "top_level_keys": [ + "max_completion_tokens", + "messages", + "model", + "store", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.81.1" + }, + { + "agent": "pi", + "capture_sha256": "ef7db6f237ae1ed9a02420404db9555820f6d8ff16b46462103c0436d39e0533", + "dialect": "openai_chat", + "executable_sha256": "7f4c35fbbbe908301c12e06b9e6b7017c6b9cbcc24c34e26615a08f9f60fa53e", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "read", + "bash", + "edit", + "write" + ], + "top_level_keys": [ + "max_completion_tokens", + "messages", + "model", + "store", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.81.1" + }, + { + "agent": "qwen", + "capture_sha256": "f31eb243bbc121471fdecb50d50d177b12e84075cb89c52a1dc7af01ba0b9852", + "dialect": "openai_chat", + "executable_sha256": "eac1ed6f8ed71466fa05cf7b2a63fc0cf252b1ad79183735bba365bbeda3cafb", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "agent", + "edit", + "glob", + "grep_search", + "list_directory", + "notebook_edit", + "read_file", + "run_shell_command", + "skill", + "todo_write", + "tool_search", + "write_file" + ], + "top_level_keys": [ + "max_tokens", + "messages", + "model", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.20.1" + }, + { + "agent": "qwen", + "capture_sha256": "27da2424b8153808118fe54ae7de564c0793c4c0151b77934636a69d596e934a", + "dialect": "openai_chat", + "executable_sha256": "eac1ed6f8ed71466fa05cf7b2a63fc0cf252b1ad79183735bba365bbeda3cafb", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "agent", + "edit", + "glob", + "grep_search", + "list_directory", + "notebook_edit", + "read_file", + "run_shell_command", + "skill", + "todo_write", + "tool_search", + "write_file" + ], + "top_level_keys": [ + "max_tokens", + "messages", + "model", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.20.1" + }, + { + "agent": "qwen", + "capture_sha256": "79c8dc86ad30a5e67c4259de7f438df2618d05d611bff69b88e4d67c8f64767a", + "dialect": "openai_chat", + "executable_sha256": "a5c19d0c03467074c1ff755ec604422223e1019fa19f8a9bf2f81dd84f32e070", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "agent", + "edit", + "glob", + "grep_search", + "list_directory", + "notebook_edit", + "read_file", + "run_shell_command", + "skill", + "todo_write", + "tool_search", + "write_file" + ], + "top_level_keys": [ + "max_tokens", + "messages", + "model", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.20.1" + } + ], + "schema_version": 2 +} diff --git a/labs/15-pitot/tests/install_real_agent.py b/labs/15-pitot/tests/install_real_agent.py new file mode 100755 index 000000000..54e3e5f52 --- /dev/null +++ b/labs/15-pitot/tests/install_real_agent.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Install and attest the exact released CLI declared by Pitot's supervisor.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform as host_platform +import shutil +import subprocess +import tempfile + + +LAB = Path(__file__).resolve().parent.parent +ROOT = LAB.parents[1] if LAB.name == "15-pitot" else LAB +MANIFEST = json.loads((LAB / "adapter-verification.json").read_text(encoding="utf-8")) + + +def run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, check=True, text=True, **kwargs) + + +def npm_executable() -> str: + """Resolve npm's platform launcher instead of relying on PATHEXT in Python.""" + name = "npm.cmd" if os.name == "nt" else "npm" + return shutil.which(name) or name + + +def install(agent: dict[str, object], platform: str, runtime: str) -> None: + version = str(agent["version"]) + installer = agent["installer"] + kind, package = installer["kind"], installer["package"] + if runtime == "wsl": + if platform != "windows" or agent["id"] != "cursor": + raise ValueError("WSL is reserved for Cursor on Windows") + script = f"curl -fsSL https://cursor.com/install -o /tmp/cursor-install.sh && CURSOR_VERSION={version} bash /tmp/cursor-install.sh" + run(["wsl.exe", "--distribution", "Ubuntu", "--", "bash", "-lc", script]) + return + if kind == "npm": + run([npm_executable(), "install", "--global", "--ignore-scripts=false", f"{package}@{version}"]) + elif kind == "kimi_release": + if platform == "windows": + # Use the native Kimi Code installer. The legacy /install.ps1 + # endpoint installs the unrelated Python kimi-cli and ignores the + # KIMI_VERSION contract used by the released Kimi Code binary. + command = ( + f"$env:KIMI_VERSION='{version}'; " + "irm https://code.kimi.com/kimi-code/install.ps1 | iex" + ) + run(["powershell", "-NoProfile", "-NonInteractive", "-Command", command]) + else: + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / "install.sh" + run(["curl", "-fsSL", "https://code.kimi.com/kimi-code/install.sh", "-o", str(target)]) + environment = {**os.environ, "KIMI_VERSION": version} + run(["bash", str(target)], env=environment) + elif kind == "cursor_release": + os_name = {"ubuntu": "linux", "macos": "darwin"}[platform] + machine = host_platform.machine().lower() + arch = "arm64" if machine in {"arm64", "aarch64"} else "x64" + url = f"{package}/{os_name}/{arch}/agent-cli-package.tar.gz" + destination = Path.home() / ".local" / "bin" + destination.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory() as directory: + archive = Path(directory) / "cursor.tar.gz" + run(["curl", "-fsSL", url, "-o", str(archive)]) + run(["tar", "-xzf", str(archive), "-C", directory]) + candidates = [path for path in Path(directory).rglob("cursor-agent") if path.is_file()] + if len(candidates) != 1: + raise RuntimeError(f"Cursor archive contained {len(candidates)} agent executables") + release = Path.home() / ".local" / "share" / "pitot-cursor" / version + shutil.copytree(candidates[0].parent, release, dirs_exist_ok=True) + installed = release / "cursor-agent" + installed.chmod(0o755) + link = destination / "agent" + temporary_link = destination / f".agent-{version}.tmp" + if temporary_link.exists() or temporary_link.is_symlink(): + temporary_link.unlink() + temporary_link.symlink_to(installed) + temporary_link.replace(link) + else: + raise ValueError(f"unsupported installer: {kind}") + + +def executable_receipt(agent: dict[str, object], platform: str, runtime: str) -> dict[str, object]: + executable, version = str(agent["executable"]), str(agent["version"]) + if runtime == "wsl": + resolve = ( + f"if command -v {executable} >/dev/null; then command -v {executable}; " + f'elif [ -x "$HOME/.local/bin/{executable}" ]; then printf "%s\\n" "$HOME/.local/bin/{executable}"; ' + "else exit 1; fi" + ) + path = run(["wsl.exe", "--distribution", "Ubuntu", "--", "bash", "-lc", resolve], stdout=subprocess.PIPE).stdout.strip() + output = run(["wsl.exe", "--distribution", "Ubuntu", "--", "bash", "-lc", f"{path} --version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.strip() + else: + path = "" + if agent["installer"]["kind"] == "npm": + prefix = Path(run([npm_executable(), "prefix", "--global"], stdout=subprocess.PIPE).stdout.strip()) + candidate = prefix / (f"{executable}.cmd" if os.name == "nt" else f"bin/{executable}") + if candidate.is_file(): path = str(candidate) + path = path or shutil.which(executable) or "" + if not path: + suffix = f"{executable}.exe" if os.name == "nt" else executable + home_candidates = ( + Path.home() / ".local" / "bin" / suffix, + Path.home() / ".kimi-code" / "bin" / suffix, + ) + path = next((str(candidate) for candidate in home_candidates if candidate.is_file()), "") + if not path: + raise RuntimeError(f"installed executable {executable!r} is not on PATH") + output = run([path, "--version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.strip() + if version not in output: + raise RuntimeError(f"{executable} reported {output!r}, expected pinned version {version}") + if runtime == "wsl": + digest = run( + ["wsl.exe", "--distribution", "Ubuntu", "--", "bash", "-lc", f"sha256sum {path} | cut -d' ' -f1"], + stdout=subprocess.PIPE, + ).stdout.strip() + else: + digest = hashlib.sha256(Path(path).read_bytes()).hexdigest() + return { + "schema_version": 1, + "agent": agent["id"], + "version": version, + "executable": path, + "executable_sha256": digest, + "version_output": output, + "installer": agent["installer"]["kind"], + "runtime": runtime, + "platform": platform, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--agent", required=True) + parser.add_argument("--platform", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + agent = next((item for item in MANIFEST["agents"] if item["id"] == args.agent), None) + if agent is None: + raise SystemExit(f"unknown supervised agent: {args.agent}") + runtime = agent["runtime"].get(args.platform) + if runtime is None: + raise SystemExit(f"unsupported platform: {args.platform}") + install(agent, args.platform, runtime) + receipt = executable_receipt(agent, args.platform, runtime) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f"Installed {args.agent} {agent['version']} at {receipt['executable']} ({runtime})") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/labs/15-pitot/tests/model_control_proxy.py b/labs/15-pitot/tests/model_control_proxy.py new file mode 100755 index 000000000..eadb0d6af --- /dev/null +++ b/labs/15-pitot/tests/model_control_proxy.py @@ -0,0 +1,435 @@ +#!/usr/bin/env python3 +"""Deterministic model server for real prompt-to-hook agent E2E sessions.""" + +from __future__ import annotations + +import argparse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path +import threading +import time +import uuid + + +def contains(value: object, needle: str) -> bool: + return needle in json.dumps(value, separators=(",", ":"), ensure_ascii=True) + + +class UnknownProtocol(ValueError): + pass + + +def classify_request(path: str, body: object) -> str: + """Classify only from the real CLI's observed request, never a manifest hint.""" + clean_path = path.split("?", 1)[0] + if not isinstance(body, dict): + raise UnknownProtocol("request body is not a JSON object") + candidates: list[str] = [] + if clean_path.endswith("/messages") and isinstance(body.get("messages"), list): + candidates.append("anthropic_messages") + if clean_path.endswith("/responses") and "input" in body: + candidates.append("openai_responses") + if ("generateContent" in clean_path or "streamGenerateContent" in clean_path) and isinstance(body.get("contents"), list): + candidates.append("gemini_generate_content") + if clean_path.endswith("/chat/completions") and isinstance(body.get("messages"), list): + candidates.append("openai_chat") + if len(candidates) != 1: + raise UnknownProtocol(f"request matched {len(candidates)} supported dialects") + return candidates[0] + + +def request_shape(body: dict[str, object]) -> dict[str, object]: + """Return a content-redacted structural fingerprint for review fixtures.""" + return { + "top_level_keys": sorted(body), + "has_tools": bool(body.get("tools")), + "stream": bool(body.get("stream")), + "tool_names": tool_names(body), + } + + +def protobuf_varint(value: int) -> bytes: + encoded = bytearray() + while value > 0x7F: + encoded.append((value & 0x7F) | 0x80) + value >>= 7 + encoded.append(value) + return bytes(encoded) + + +def protobuf_bytes(field: int, value: bytes) -> bytes: + return protobuf_varint((field << 3) | 2) + protobuf_varint(len(value)) + value + + +def cursor_model_details() -> bytes: + return b"".join(protobuf_bytes(field, b"pitot-control") for field in (1, 3, 4, 5)) + + +def cursor_auxiliary_response(path: str) -> bytes: + model = cursor_model_details() + if path.endswith("/GetUsableModels"): + return protobuf_bytes(1, model) + if path.endswith("/GetDefaultModelForCli"): + return protobuf_bytes(1, model) + return b"" + + +def tool_name(body: dict[str, object]) -> str: + names = tool_names(body) + preferred = ("Bash", "bash", "shell", "run_shell_command", "execute_command", "run_commands") + return next((name for name in preferred if name in names), names[0] if names else "bash") + + +def tool_names(body: dict[str, object]) -> list[str]: + names: list[str] = [] + for item in body.get("tools", []): + if isinstance(item, dict): + function = item.get("function") + name = function.get("name") if isinstance(function, dict) else item.get("name") + if isinstance(name, str): names.append(name) + declarations = item.get("functionDeclarations", []) + if isinstance(declarations, list): + names.extend(value["name"] for value in declarations if isinstance(value, dict) and isinstance(value.get("name"), str)) + return names + + +def tool_arguments(tool: str, command: str) -> dict[str, object]: + if tool == "exec_command": return {"cmd": command} + if tool == "run_commands": return {"commands": [command]} + if tool == "run_shell_command": return {"command": command, "is_background": False} + return {"command": command} + + +def function_response_shapes(value: object) -> list[dict[str, object]]: + found: list[dict[str, object]] = [] + if isinstance(value, dict): + response = value.get("functionResponse") + if isinstance(response, dict): + payload = response.get("response") + found.append({"name": response.get("name"), "response_keys": sorted(payload) if isinstance(payload, dict) else type(payload).__name__}) + for child in value.values(): found.extend(function_response_shapes(child)) + elif isinstance(value, list): + for child in value: found.extend(function_response_shapes(child)) + return found + + +def content_shapes(body: dict[str, object]) -> list[dict[str, object]]: + shapes: list[dict[str, object]] = [] + for content in body.get("contents", []): + if not isinstance(content, dict): continue + parts = content.get("parts", []) + shapes.append({"role": content.get("role"), "parts": [sorted(part) for part in parts if isinstance(part, dict)]}) + return shapes + + +def message_shapes(body: dict[str, object]) -> list[dict[str, object]]: + shapes: list[dict[str, object]] = [] + for message in body.get("messages", []): + if not isinstance(message, dict): continue + content = message.get("content") + serialized = json.dumps(content).lower() + item_shapes = [] + if isinstance(content, list): + for item in content: + if isinstance(item, dict): + item_shapes.append({"keys": sorted(item), "types": {key: type(value).__name__ for key, value in item.items()}}) + else: + item_shapes.append({"type": type(item).__name__}) + markers = ( + "hook", "permission", "denied", "error", "invalid", "background", + "cancel", "reject", "block", "fail", "not found", "exit code", + "timed out", "aborted", + ) + shapes.append({"role": message.get("role"), "keys": sorted(message), "content_type": type(content).__name__, "content_length": len(content) if isinstance(content, (str, list)) else None, "content_items": item_shapes, "markers": [marker for marker in markers if marker in serialized]}) + return shapes + + +def tool_schema_shapes(body: dict[str, object]) -> list[dict[str, object]]: + shapes: list[dict[str, object]] = [] + for item in body.get("tools", []): + if not isinstance(item, dict): continue + function = item.get("function") if isinstance(item.get("function"), dict) else item + params = function.get("parameters", {}) + properties = params.get("properties", {}) if isinstance(params, dict) else {} + property_shapes = {} + if isinstance(properties, dict): + for key, value in properties.items(): + if isinstance(value, dict): + items = value.get("items", {}) + property_shapes[key] = {"type": value.get("type"), "item_type": items.get("type") if isinstance(items, dict) else None, "item_properties": sorted(items.get("properties", {})) if isinstance(items, dict) and isinstance(items.get("properties"), dict) else []} + shapes.append({"name": function.get("name"), "properties": property_shapes}) + return shapes + + +class State: + def __init__(self, args: argparse.Namespace) -> None: + self.args = args + self.lock = threading.Lock() + self.receipt = { + "schema_version": 1, + "agent": args.agent, + "protocol": None, + "nonce": args.nonce, + "initial_prompt_observed": False, + "tool_call_response_emitted": False, + "tool_result_observed": False, + "final_response_emitted": False, + "selected_tool": None, + "endpoint_observed": None, + "unexpected_request": None, + "auxiliary_requests": 0, + } + + def save(self) -> None: + target = Path(self.args.receipt) + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_suffix(".tmp") + temporary.write_text(json.dumps(self.receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(target) + + +def anthropic(tool: str, command: str, final: bool) -> dict[str, object]: + content = [{"type": "text", "text": "Pitot E2E Verification Complete"}] if final else [ + {"type": "tool_use", "id": "pitot_tool_1", "name": tool, "input": tool_arguments(tool, command)} + ] + return {"id": "msg_pitot", "type": "message", "role": "assistant", "model": "pitot-control", "content": content, "stop_reason": "end_turn" if final else "tool_use", "usage": {"input_tokens": 1, "output_tokens": 1}} + + +def chat(tool: str, command: str, final: bool) -> dict[str, object]: + message: dict[str, object] = {"role": "assistant", "content": "Pitot E2E Verification Complete" if final else None} + finish = "stop" if final else "tool_calls" + if not final: + message["tool_calls"] = [{"id": "pitot_tool_1", "type": "function", "function": {"name": tool, "arguments": json.dumps(tool_arguments(tool, command))}}] + return {"id": "chatcmpl-pitot", "object": "chat.completion", "created": 1, "model": "pitot-control", "choices": [{"index": 0, "message": message, "finish_reason": finish}], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}} + + +def responses(tool: str, command: str, final: bool) -> dict[str, object]: + output = [{"id": "msg_pitot", "type": "message", "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": "Pitot E2E Verification Complete", "annotations": []}]}] if final else [ + {"id": "fc_pitot", "type": "function_call", "call_id": "pitot_tool_1", "name": tool, "arguments": json.dumps(tool_arguments(tool, command)), "status": "completed"} + ] + return {"id": "resp_pitot", "object": "response", "created_at": 1, "status": "completed", "model": "pitot-control", "output": output, "parallel_tool_calls": False, "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}} + + +def gemini(tool: str, command: str, final: bool) -> dict[str, object]: + part = {"text": "Pitot E2E Verification Complete"} if final else {"functionCall": {"name": tool, "args": {"command": command}}, "thoughtSignature": "cGl0b3Q="} + return {"candidates": [{"content": {"role": "model", "parts": [part]}, "finishReason": "STOP"}], "usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2}} + + +class Handler(BaseHTTPRequestHandler): + server_version = "PitotModelControl/1" + + def log_message(self, format: str, *args: object) -> None: + return + + def do_GET(self) -> None: + state: State = self.server.state # type: ignore[attr-defined] + if self.path in {"/health", "/v1/models"}: + self.reply({"object": "list", "data": [{"id": "pitot-control", "object": "model"}]}) + else: + self.send_error(404) + + def do_POST(self) -> None: + state: State = self.server.state # type: ignore[attr-defined] + try: + length = int(self.headers.get("content-length", "0")) + raw_body = self.rfile.read(length) + body = json.loads(raw_body or b"{}") + except (ValueError, json.JSONDecodeError): + self.send_error(400, "invalid JSON") + return + nonce = state.args.nonce + try: + protocol = classify_request(self.path, body) + except UnknownProtocol as error: + with state.lock: + state.receipt["unexpected_request"] = { + "path": self.path.split("?", 1)[0], + "top_level_keys": sorted(body) if isinstance(body, dict) else [], + "classification_error": str(error), + } + state.save() + self.send_error(422, "unrecognized model request structure") + return + with state.lock: + if state.receipt["protocol"] not in {None, protocol}: + self.send_error(409, "request dialect changed during session") + return + state.receipt["protocol"] = protocol + if not body.get("tools") and state.receipt["auxiliary_requests"] == 0: + state.receipt["auxiliary_requests"] = 1 + state.receipt["auxiliary_request"] = { + "path": self.path.split("?", 1)[0], + "top_level_keys": sorted(body), + "nonce_present": contains(body, nonce), + } + state.save() + if protocol == "anthropic_messages": + payload = anthropic("", "", True) + payload["content"][0]["text"] = '{"title":"Pitot E2E"}' + elif protocol == "openai_responses": payload = responses("", "", True) + elif protocol == "gemini_generate_content": payload = gemini("", "", True) + else: payload = chat("", "", True) + if isinstance(body.get("model"), str): payload["model"] = body["model"] + self.reply(payload, stream=bool(body.get("stream")), protocol=protocol) + return + final = contains(body, f"PITOT_CANARY_RESULT {nonce}") + if not state.receipt["initial_prompt_observed"]: + if not contains(body, nonce): + self.send_error(409, "initial request omitted session nonce") + return + state.receipt["initial_prompt_observed"] = True + state.receipt["endpoint_observed"] = { + "transport": "http1", + "method": "POST", + "path": self.path.split("?", 1)[0], + "media_type": self.headers.get("content-type", "").split(";", 1)[0].strip().lower(), + "framing": "sse" if bool(body.get("stream")) or "streamGenerateContent" in self.path or protocol == "openai_responses" else "json", + "request_shape": request_shape(body), + } + elif not final: + state.receipt["unexpected_request"] = { + "path": self.path.split("?", 1)[0], + "top_level_keys": sorted(body), + "nonce_present": contains(body, nonce), + "canary_result_present": False, + "function_responses": function_response_shapes(body), + "contents": content_shapes(body), + "messages": message_shapes(body), + } + state.save() + self.send_error(409, "second request omitted real canary tool result") + return + tool = tool_name(body) + state.receipt["selected_tool"] = tool + state.receipt["advertised_tools"] = tool_names(body) + state.receipt["tool_structures"] = [sorted(item) if isinstance(item, dict) else type(item).__name__ for item in body.get("tools", [])] + state.receipt["tool_schemas"] = tool_schema_shapes(body) + command = f"pitot-e2e-canary {nonce}" + if state.args.response_fault == "text" and not final: + if protocol == "anthropic_messages": payload = anthropic(tool, command, True) + elif protocol == "openai_responses": payload = responses(tool, command, True) + elif protocol == "gemini_generate_content": payload = gemini(tool, command, True) + else: payload = chat(tool, command, True) + state.receipt["fault_response_emitted"] = "text" + elif protocol == "anthropic_messages": payload = anthropic(tool, command, final) + elif protocol == "openai_responses": payload = responses(tool, command, final) + elif protocol == "gemini_generate_content": payload = gemini(tool, command, final) + elif protocol == "openai_chat": payload = chat(tool, command, final) + else: + self.send_error(422, "unsupported protocol") + return + if isinstance(body.get("model"), str): + payload["model"] = body["model"] + if state.args.response_fault == "text" and not final: + pass + elif final: + state.receipt["tool_result_observed"] = True + state.receipt["final_response_emitted"] = True + else: + state.receipt["tool_call_response_emitted"] = True + state.save() + stream = bool(body.get("stream")) or "streamGenerateContent" in self.path or protocol == "openai_responses" + self.reply(payload, stream=stream, protocol=protocol) + + def reply(self, payload: dict[str, object], *, stream: bool = False, protocol: str = "") -> None: + if stream: + if protocol == "openai_responses": + item = payload["output"][0] + events = [("response.created", {"type": "response.created", "sequence_number": 0, "response": {**payload, "status": "in_progress", "output": []}})] + if item["type"] == "function_call": + start = {**item, "arguments": "", "status": "in_progress"} + events.extend([ + ("response.output_item.added", {"type": "response.output_item.added", "sequence_number": 1, "output_index": 0, "item": start}), + ("response.function_call_arguments.delta", {"type": "response.function_call_arguments.delta", "sequence_number": 2, "item_id": item["id"], "output_index": 0, "delta": item["arguments"]}), + ("response.function_call_arguments.done", {"type": "response.function_call_arguments.done", "sequence_number": 3, "item_id": item["id"], "output_index": 0, "arguments": item["arguments"]}), + ("response.output_item.done", {"type": "response.output_item.done", "sequence_number": 4, "output_index": 0, "item": item}), + ]) + else: + text_value = item["content"][0]["text"] + start = {**item, "status": "in_progress", "content": []} + part = {"type": "output_text", "text": "", "annotations": []} + events.extend([ + ("response.output_item.added", {"type": "response.output_item.added", "sequence_number": 1, "output_index": 0, "item": start}), + ("response.content_part.added", {"type": "response.content_part.added", "sequence_number": 2, "item_id": item["id"], "output_index": 0, "content_index": 0, "part": part}), + ("response.output_text.delta", {"type": "response.output_text.delta", "sequence_number": 3, "item_id": item["id"], "output_index": 0, "content_index": 0, "delta": text_value}), + ("response.output_text.done", {"type": "response.output_text.done", "sequence_number": 4, "item_id": item["id"], "output_index": 0, "content_index": 0, "text": text_value}), + ("response.content_part.done", {"type": "response.content_part.done", "sequence_number": 5, "item_id": item["id"], "output_index": 0, "content_index": 0, "part": item["content"][0]}), + ("response.output_item.done", {"type": "response.output_item.done", "sequence_number": 6, "output_index": 0, "item": item}), + ]) + events.append(("response.completed", {"type": "response.completed", "sequence_number": len(events) + 1, "response": payload})) + elif protocol == "anthropic_messages": + block = payload["content"][0] + if block["type"] == "tool_use": + start_block = {"type": "tool_use", "id": block["id"], "name": block["name"], "input": {}} + deltas = [("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": json.dumps(block["input"], separators=(',', ':'))}})] + else: + start_block = {"type": "text", "text": ""} + deltas = [("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": block["text"]}})] + events = [ + ("message_start", {"type": "message_start", "message": {**payload, "content": [], "stop_reason": None, "stop_sequence": None}}), + ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": start_block}), + *deltas, + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ("message_delta", {"type": "message_delta", "delta": {"stop_reason": payload["stop_reason"], "stop_sequence": None}, "usage": {"output_tokens": 1}}), + ("message_stop", {"type": "message_stop"}), + ] + elif protocol == "openai_chat": + message = payload["choices"][0]["message"] + if message.get("tool_calls"): + call = message["tool_calls"][0] + delta = {"role": "assistant", "content": None, "tool_calls": [{"index": 0, **call}]} + else: + delta = {"role": "assistant", "content": message.get("content")} + base = {"id": payload["id"], "object": "chat.completion.chunk", "created": payload["created"], "model": payload["model"]} + events = [ + ("", {**base, "choices": [{"index": 0, "delta": delta, "finish_reason": None}]}), + ("", {**base, "choices": [{"index": 0, "delta": {}, "finish_reason": payload["choices"][0]["finish_reason"]}]}), + ] + else: + events = [("", payload)] + encoded = b"".join( + ((f"event: {name}\n" if name else "") + f"data: {json.dumps(item, separators=(',', ':'))}\n\n").encode() + for name, item in events + ) + (b"data: [DONE]\n\n" if protocol == "openai_chat" else b"") + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.send_header("content-length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + return + encoded = json.dumps(payload, separators=(",", ":")).encode() + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def reply_bytes(self, payload: bytes, content_type: str) -> None: + self.send_response(200) + self.send_header("content-type", content_type) + self.send_header("content-length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--agent", required=True) + parser.add_argument("--nonce", required=True) + parser.add_argument("--receipt", required=True) + parser.add_argument("--ready-file", required=True) + parser.add_argument("--response-fault", choices=("none", "text"), default="none") + args = parser.parse_args() + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.state = State(args) # type: ignore[attr-defined] + Path(args.ready_file).write_text(f"http://127.0.0.1:{server.server_port}\n", encoding="utf-8") + try: + server.serve_forever() + except KeyboardInterrupt: + pass + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/labs/15-pitot/tests/real_agent_driver.py b/labs/15-pitot/tests/real_agent_driver.py new file mode 100755 index 000000000..9a400a73d --- /dev/null +++ b/labs/15-pitot/tests/real_agent_driver.py @@ -0,0 +1,452 @@ +#!/usr/bin/env python3 +"""Run one released agent through prompt, model, hook, Pitot, and tool result.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import secrets +import re +import shutil +import subprocess +import sys +import tempfile +import time + + +LAB = Path(__file__).resolve().parent.parent +ROOT = LAB.parents[1] if LAB.name == "15-pitot" else LAB +MANIFEST = json.loads((LAB / "adapter-verification.json").read_text(encoding="utf-8")) +ENDPOINT_PROVENANCE = json.loads((LAB / "tests/endpoint-provenance.json").read_text(encoding="utf-8")) + + +def write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + + +def hook_group(event: str, matcher: str, command: str) -> dict[str, object]: + return {"hooks": {event: [{"matcher": matcher, "hooks": [{"name": "pitot", "type": "command", "command": command}]}]}} + + +def wsl_path(path: Path) -> str: + """Map a resolved Windows runner path through WSL's drive automount.""" + native = str(path.resolve()).replace("\\", "/") + matched = re.fullmatch(r"([A-Za-z]):(/.*)", native) + if matched is None: + raise RuntimeError(f"cannot map native path into WSL: {native!r}") + return f"/mnt/{matched.group(1).lower()}{matched.group(2)}" + + +def configure( + agent: str, + home: Path, + project: Path, + witness_command: str, + proxy: str, + *, + pitot_command: str, + witness_receipt: str, + nonce: str, +) -> tuple[list[str], dict[str, str]]: + env: dict[str, str] = { + "HOME": str(home), + "USERPROFILE": str(home), + "OPENAI_API_KEY": "pitot-local-only", + "ANTHROPIC_API_KEY": "pitot-local-only", + "GEMINI_API_KEY": "pitot-local-only", + "GOOGLE_API_KEY": "pitot-local-only", + "PITOT_BIN": witness_command, + } + # Gemini, Codex, and Copilot execute command hooks through PowerShell on + # Windows. A quoted executable is only a string there; the call operator is + # required to invoke it. Explicit receipt arguments also survive the hosts' + # intentionally reduced hook environments. + witness_invocation = f'& "{witness_command}"' if os.name == "nt" else f'"{witness_command}"' + witnessed = ( + f'{witness_invocation} --real-bin "{pitot_command}" ' + f'--receipt "{witness_receipt}" --nonce "{nonce}"' + ) + prompt_flag: list[str] + if agent == "claude": + write_json(home / ".claude/settings.json", hook_group("PreToolUse", "Bash", f'"{witness_command}" hook claude')) + env["ANTHROPIC_BASE_URL"] = proxy + env["CLAUDE_CODE_MAX_RETRIES"] = "0" + prompt_flag = ["--print", "--dangerously-skip-permissions", "--tools", "Bash", "--model", "sonnet"] + elif agent == "codex": + sink = "NUL" if os.name == "nt" else "/dev/null" + write_json(home / ".codex/hooks.json", hook_group("PreToolUse", "Bash", f'{witnessed} hook codex >{sink}')) + (home / ".codex/config.toml").write_text( + f'model = "pitot-control"\nmodel_provider = "pitot"\n[model_providers.pitot]\nname = "Pitot local control"\nbase_url = "{proxy}/v1"\nenv_key = "OPENAI_API_KEY"\nwire_api = "responses"\n', + encoding="utf-8", + ) + env["CODEX_HOME"] = str(home / ".codex") + prompt_flag = ["exec", "--dangerously-bypass-approvals-and-sandbox", "--dangerously-bypass-hook-trust", "--model", "pitot-control"] + elif agent == "copilot": + write_json(home / ".copilot/settings.json", hook_group("PreToolUse", "Bash", f'{witnessed} hook copilot')) + env.update({ + "COPILOT_PROVIDER_BASE_URL": f"{proxy}/v1", + "COPILOT_PROVIDER_API_KEY": "pitot-local-only", + "COPILOT_PROVIDER_TYPE": "openai", + "COPILOT_PROVIDER_WIRE_API": "completions", + "COPILOT_PROVIDER_MODEL_ID": "gpt-4o", + "COPILOT_PROVIDER_WIRE_MODEL": "pitot-control", + "COPILOT_MODEL": "gpt-4o", + "COPILOT_OFFLINE": "true", + "COPILOT_HOME": str(home / ".copilot"), + }) + prompt_flag = ["--allow-all-tools", "--model", "gpt-4o", "--no-auto-update", "--no-remote"] + elif agent == "cursor": + write_json(project / ".cursor/hooks.json", {"version": 1, "hooks": {"beforeShellExecution": [{"command": f'"{witness_command}" hook cursor'}]}}) + # Cursor exposes an authless CLI mode for endpoint compatibility tests; + # model inference is still supplied only by the pinned local endpoint. + env["CURSOR_AGENT_CLI_AUTHLESS_MODE"] = "true" + env["CURSOR_AUTH_TOKEN"] = "pitot-local-only" + prompt_flag = ["--endpoint", proxy, "--print", "--force"] + elif agent == "gemini": + # Gemini deliberately sanitizes the hook environment. Pass the witness + # receipt identity as explicit arguments so the real hook remains + # nonce-bound even under that isolation boundary. + witness = f"{witnessed} hook gemini" + settings = hook_group("BeforeTool", "run_shell_command", witness) + settings["security"] = {"auth": {"selectedType": "gemini-api-key"}, "folderTrust": {"enabled": False}} + write_json(home / ".gemini/settings.json", settings) + env.update({"GOOGLE_GEMINI_BASE_URL": proxy, "GEMINI_CLI_HOME": str(home), "GEMINI_CLI_TRUST_WORKSPACE": "true"}) + prompt_flag = ["--skip-trust", "--approval-mode", "yolo", "--model", "pitot-control", "-p"] + elif agent == "kimi": + config = home / ".kimi-code/config.toml" + config.parent.mkdir(parents=True, exist_ok=True) + config.write_text( + 'default_model = "pitot-control"\n' + '[providers.pitot]\n' + 'type = "openai"\n' + f'base_url = {json.dumps(proxy + "/v1")}\n' + 'api_key = "pitot-local-only"\n' + '[models."pitot-control"]\n' + 'provider = "pitot"\n' + 'model = "pitot-control"\n' + 'max_context_size = 32768\n' + 'capabilities = ["tool_use"]\n' + '[[hooks]]\n' + 'event = "PreToolUse"\n' + 'matcher = ".*"\n' + f'command = {json.dumps(witness_command + " hook kimi")}\n', + encoding="utf-8", + ) + env["KIMI_CODE_HOME"] = str(home / ".kimi-code") + env["KIMI_CODE_NO_AUTO_UPDATE"] = "1" + prompt_flag = [] + elif agent == "qwen": + allow = '{"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":"allow","permissionDecisionReason":"Pitot accepted the shell action"}}' + if os.name == "nt": + hook_command = f'"{witness_command}" hook qwen >/dev/null && echo {allow}' + else: + hook_command = f'"{witness_command}" hook qwen >/dev/null && printf \'%s\\n\' \'{allow}\'' + settings = hook_group("PreToolUse", "^(Bash|run_shell_command)$", hook_command) + settings["modelProviders"] = {"openai": {"protocol": "openai", "models": [{"id": "pitot-control", "name": "Pitot control", "envKey": "OPENAI_API_KEY", "baseUrl": f"{proxy}/v1"}]}} + settings["security"] = {"auth": {"selectedType": "openai"}} + settings["model"] = {"name": "pitot-control"} + write_json(home / ".qwen/settings.json", settings) + env["OPENAI_API_KEY"] = "pitot-local-only" + prompt_flag = ["--model", "pitot-control", "--approval-mode", "yolo", "-p"] + elif agent == "pi": + extension = LAB / "integrations/pi/pitot.ts" + write_json(home / ".pi/agent/models.json", {"providers": {"pitot": {"baseUrl": f"{proxy}/v1", "apiKey": "pitot-local-only", "api": "openai-completions", "models": [{"id": "pitot-control", "name": "Pitot control", "reasoning": False, "input": ["text"], "contextWindow": 32000, "maxTokens": 4096}]}}}) + prompt_flag = ["--no-session", "--print", "--provider", "pitot", "--model", "pitot-control", "-e", str(extension)] + elif agent == "cline": + hooks = home / ".cline/hooks" + hooks.mkdir(parents=True, exist_ok=True) + source = LAB / "integrations/cline" / ("PreToolUse.ps1" if os.name == "nt" else "PreToolUse") + target = hooks / source.name + shutil.copy2(source, target) + target.chmod(0o755) + env["PITOT_BIN"] = witness_command + prompt_flag = ["--provider", "openai-compatible", "--key", "pitot-local-only", "--model", "pitot-control", "--data-dir", str(home / ".cline"), "--hooks-dir", str(hooks), "--auto-approve", "true"] + env["OPENAI_BASE_URL"] = f"{proxy}/v1" + elif agent == "opencode": + plugin = LAB / "integrations/opencode/pitot.ts" + # Use the released binary's bundled OpenAI provider. A custom provider + # would dynamically install @ai-sdk/openai-compatible and make endpoint + # verification depend on a second unpinned network package. + config = home / ".config/opencode/opencode.json" + write_json(config, {"plugin": [f"file://{plugin}"], "provider": {"openai": {"options": {"baseURL": f"{proxy}/v1", "apiKey": "pitot-local-only"}, "models": {"pitot-control": {"name": "Pitot control"}}}}, "model": "openai/pitot-control", "permission": {"bash": "allow"}}) + env["OPENCODE_CONFIG"] = str(config) + prompt_flag = ["--print-logs", "--log-level", "DEBUG", "run", "--model", "openai/pitot-control"] + else: + raise ValueError(f"unsupported agent {agent}") + return prompt_flag, env + + +def prepare_cursor_keychain(home: Path, environment: dict[str, str]) -> None: + """Provide Cursor an isolated unlocked macOS credential store.""" + if sys.platform != "darwin": + return + keychain = home / "Library/Keychains/login.keychain-db" + keychain.parent.mkdir(parents=True, exist_ok=True) + password = "pitot-e2e-local-only" + for command in ( + ["security", "create-keychain", "-p", password, str(keychain)], + ["security", "set-keychain-settings", "-lut", "900", str(keychain)], + ["security", "unlock-keychain", "-p", password, str(keychain)], + ["security", "list-keychains", "-d", "user", "-s", str(keychain)], + ["security", "default-keychain", "-d", "user", "-s", str(keychain)], + ): + subprocess.run(command, env=environment, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + +def capture_record(agent: dict[str, object], platform: str, installation: dict[str, object], proxy: dict[str, object]) -> dict[str, object]: + """Normalize the two receipts without trusting a manifest protocol hint.""" + observed = proxy.get("endpoint_observed") + dialect = proxy.get("protocol") + if not isinstance(observed, dict) or dialect not in { + "anthropic_messages", "openai_chat", "openai_responses", "gemini_generate_content", "cursor_connect_proto", + }: + raise RuntimeError("proxy did not binary-observe a supported request contract") + digest = installation.get("executable_sha256") + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise RuntimeError("installation receipt lacks the executable content digest") + core = { + "agent": agent["id"], + "platform": platform, + "runtime": installation["runtime"], + "version": installation["version"], + "executable_sha256": digest, + "dialect": dialect, + "request": observed, + "response": { + "encoder": dialect, + "framing": observed["framing"], + "tool_call": "native_shell", + "acceptance": "nonce_tool_result_round_trip", + }, + "provenance": "pinned_real_cli_capture", + } + return {**core, "capture_sha256": hashlib.sha256(json.dumps(core, sort_keys=True, separators=(",", ":")).encode()).hexdigest()} + + +def validate_capture_fixture(record: dict[str, object]) -> dict[str, object]: + matches = [ + item for item in ENDPOINT_PROVENANCE.get("cells", []) + if item.get("agent") == record["agent"] and item.get("platform") == record["platform"] + ] + if len(matches) != 1 or matches[0] != record: + raise RuntimeError("binary-observed contract drifted from its supervised platform fixture") + return { + "fixture": f"tests/endpoint-provenance.json#{record['agent']}/{record['platform']}", + "fixture_sha256": record["capture_sha256"], + "provenance": record["provenance"], + "dialect": record["dialect"], + "request": record["request"], + "response": record["response"], + "executable_sha256": record["executable_sha256"], + } + + +def validate_receipts(agent: dict[str, object], platform: str, nonce: str, installation: dict[str, object], proxy_path: Path, witness_path: Path, exit_code: int, output: str, prompt: str, capture_output: Path | None = None) -> dict[str, object]: + proxy = json.loads(proxy_path.read_text(encoding="utf-8")) if proxy_path.is_file() else {"missing": True} + witness = json.loads(witness_path.read_text(encoding="utf-8")) if witness_path.is_file() else {"missing": True} + proxy_flags = ("initial_prompt_observed", "tool_call_response_emitted", "tool_result_observed", "final_response_emitted") + if exit_code != 0 or "hook: PreToolUse Failed" in output or not all(proxy.get(flag) is True for flag in proxy_flags): + raise RuntimeError(f"agent loop incomplete (exit={exit_code}, proxy={proxy}, witness={witness})\n{output[-4000:]}") + if proxy.get("nonce") != nonce or witness.get("nonce") != nonce or witness.get("host") != agent["id"] or witness.get("valid") is not True: + raise RuntimeError("proxy and Pitot witness receipts do not identify the same valid session") + if installation.get("agent") != agent["id"] or installation.get("version") != agent["version"]: + raise RuntimeError("installation receipt does not match supervised manifest") + captured = capture_record(agent, platform, installation, proxy) + if capture_output is not None: + write_json(capture_output, {"schema_version": 1, "accepted": True, "cell": captured}) + endpoint_evidence = { + "fixture": "candidate", + "fixture_sha256": captured["capture_sha256"], + "provenance": captured["provenance"], + "dialect": captured["dialect"], + "request": captured["request"], + "response": captured["response"], + "executable_sha256": captured["executable_sha256"], + } + else: + endpoint_evidence = validate_capture_fixture(captured) + return { + "schema_version": 1, + "agent": agent["id"], + "cli": {"version": installation["version"], "executable": installation["executable"], "executable_sha256": installation["executable_sha256"], "installer": installation["installer"], "runtime": installation["runtime"]}, + "prompt_hash": hashlib.sha256(prompt.encode()).hexdigest(), + "protocol": proxy["protocol"], + "endpoint": endpoint_evidence, + "nonce": nonce, + "receipts": {**{flag: True for flag in proxy_flags}, "hook_observed": True, "canary_result_observed": True, "cli_exit_zero": True}, + "hook": {"host": witness["host"], "action_kind": witness["action_kind"], "pitot_exit": witness["pitot_exit"]}, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--agent", required=True) + parser.add_argument("--platform", required=True) + parser.add_argument("--installation", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--pitot", type=Path, required=True) + parser.add_argument("--witness", type=Path, required=True) + parser.add_argument("--capture-output", type=Path) + parser.add_argument("--response-fault", choices=("none", "text"), default="none") + parser.add_argument("--expect-incompatible-response", action="store_true") + args = parser.parse_args() + agent = next(item for item in MANIFEST["agents"] if item["id"] == args.agent) + installation = json.loads(args.installation.read_text(encoding="utf-8")) + nonce = secrets.token_hex(16) + prompt = f"Pitot E2E session {nonce}: execute the requested verification command" + with tempfile.TemporaryDirectory(prefix="pitot-real-agent-", ignore_cleanup_errors=True) as temporary: + base = Path(temporary) + runtime = installation["runtime"] + host_controls_wsl = runtime == "wsl" and os.name == "nt" + home, project, bin_dir = base / "home", base / "project", base / "bin" + home.mkdir(); project.mkdir(); bin_dir.mkdir() + canary = bin_dir / "pitot-e2e-canary" + canary.write_text('#!/usr/bin/env sh\nprintf "PITOT_CANARY_RESULT %s\\n" "$1"\n', encoding="utf-8"); canary.chmod(0o755) + (bin_dir / "pitot-e2e-canary.cmd").write_text("@echo off\r\necho PITOT_CANARY_RESULT %1\r\n", encoding="utf-8") + cursor_system_canary = False + if args.agent == "cursor" and host_controls_wsl: + occupied = subprocess.run( + ["wsl.exe", "--distribution", "Ubuntu", "--", "test", "-e", "/usr/local/bin/pitot-e2e-canary"], + check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) + if occupied.returncode == 0: + raise RuntimeError("refusing to replace an existing WSL canary command") + subprocess.run( + ["wsl.exe", "--distribution", "Ubuntu", "--", "install", "-m", "0755", wsl_path(canary), "/usr/local/bin/pitot-e2e-canary"], + check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) + cursor_system_canary = True + elif args.agent == "cursor" and os.name != "nt" and os.access("/usr/local/bin", os.W_OK): + if Path("/usr/local/bin/pitot-e2e-canary").exists(): + raise RuntimeError("refusing to replace an existing system canary command") + shutil.copy2(canary, "/usr/local/bin/pitot-e2e-canary") + Path("/usr/local/bin/pitot-e2e-canary").chmod(0o755) + cursor_system_canary = True + ready, proxy_receipt, witness_receipt = base / "proxy.url", base / "proxy.json", base / "witness.json" + if args.agent == "cursor": + if host_controls_wsl: + # Keep Cursor and its HTTP/2 Connect control proxy in the same + # supported WSL network namespace. The workflow installs a + # pinned Linux Node runtime specifically for this harness. + proxy_command = [ + "wsl.exe", "--distribution", "Ubuntu", "--", "node", + wsl_path(LAB / "tests/cursor_control_proxy.mjs"), + "--nonce", nonce, + "--receipt", wsl_path(proxy_receipt), + "--ready-file", wsl_path(ready), + "--response-fault", args.response_fault, + ] + else: + proxy_command = ["node", str(LAB / "tests/cursor_control_proxy.mjs"), "--nonce", nonce, "--receipt", str(proxy_receipt), "--ready-file", str(ready), "--response-fault", args.response_fault] + else: + proxy_command = [sys.executable, str(LAB / "tests/model_control_proxy.py"), "--agent", args.agent, "--nonce", nonce, "--receipt", str(proxy_receipt), "--ready-file", str(ready), "--response-fault", args.response_fault] + proxy_process = subprocess.Popen( + proxy_command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + try: + for _ in range(600): + if ready.is_file(): break + if proxy_process.poll() is not None: + output = proxy_process.stdout.read() if proxy_process.stdout else "" + raise RuntimeError(f"model-control proxy exited before becoming ready: {output.strip()}") + time.sleep(0.05) + else: + proxy_process.terminate() + output = proxy_process.communicate(timeout=5)[0] if proxy_process.stdout else "" + raise RuntimeError(f"model-control proxy did not become ready: {output.strip()}") + proxy = ready.read_text(encoding="utf-8").strip() + witness_command = wsl_path(args.witness) if host_controls_wsl else str(args.witness.resolve()) + pitot_command = wsl_path(args.pitot) if host_controls_wsl else str(args.pitot.resolve()) + receipt_command = wsl_path(witness_receipt) if host_controls_wsl else str(witness_receipt) + flags, extra_env = configure( + args.agent, + home, + project, + witness_command, + proxy, + pitot_command=pitot_command, + witness_receipt=receipt_command, + nonce=nonce, + ) + environment = {**os.environ, **extra_env, "PATH": str(bin_dir) + os.pathsep + os.environ.get("PATH", ""), "PITOT_REAL_BIN": str(args.pitot.resolve()), "PITOT_WITNESS_RECEIPT": str(witness_receipt), "PITOT_E2E_NONCE": nonce} + executable = installation["executable"] + if args.agent == "cursor" and not host_controls_wsl: + prepare_cursor_keychain(home, environment) + if args.agent == "cline" and not host_controls_wsl: + subprocess.run( + [executable, "auth", "--provider", "openai-compatible", "--apikey", "pitot-local-only", "--modelid", "pitot-control", "--baseurl", f"{proxy}/v1", "--data-dir", str(home / ".cline")], + cwd=project, env=environment, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) + if host_controls_wsl: + wsl_environment = { + **extra_env, + "HOME": wsl_path(home), + "USERPROFILE": wsl_path(home), + "PITOT_BIN": witness_command, + "PITOT_REAL_BIN": wsl_path(args.pitot), + "PITOT_WITNESS_RECEIPT": wsl_path(witness_receipt), + "PITOT_E2E_NONCE": nonce, + } + executable_dir = executable.rsplit("/", 1)[0] + wsl_environment["PATH"] = ":".join((wsl_path(bin_dir), executable_dir, "/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin")) + assignments = [f"{key}={value}" for key, value in wsl_environment.items()] + # Keep every argument distinct across the Windows/WSL boundary. + # A reconstructed `bash -lc` string can alter quoting, expand a + # host PATH, or leave the released agent waiting indefinitely. + command = [ + "wsl.exe", "--distribution", "Ubuntu", "--cd", wsl_path(project), + "--", "env", *assignments, executable, *flags, prompt, + ] + else: + command = [executable, "-p", prompt, *flags] if args.agent in {"copilot", "kimi"} else [executable, *flags, prompt] + try: + completed = subprocess.run(command, cwd=project, env=environment, text=True, encoding="utf-8", errors="replace", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=420) + except subprocess.TimeoutExpired as error: + proxy_state = proxy_receipt.read_text(encoding="utf-8") if proxy_receipt.is_file() else '{"missing":true}' + witness_state = witness_receipt.read_text(encoding="utf-8") if witness_receipt.is_file() else '{"missing":true}' + raise RuntimeError(f"released agent timed out; proxy={proxy_state}; witness={witness_state}") from error + # GitHub's Windows Python console defaults to CP-1252 while several + # released CLIs emit Unicode status glyphs. Emit UTF-8 bytes so the + # reporting layer cannot fail after a successful agent session. + sys.stdout.buffer.write(completed.stdout.encode("utf-8", errors="replace")) + sys.stdout.buffer.flush() + if args.expect_incompatible_response: + observed = json.loads(proxy_receipt.read_text(encoding="utf-8")) if proxy_receipt.is_file() else {} + if not ( + observed.get("initial_prompt_observed") is True + and observed.get("fault_response_emitted") == "text" + and observed.get("tool_call_response_emitted") is False + and observed.get("tool_result_observed") is False + and not witness_receipt.exists() + ): + raise RuntimeError("incompatible response unexpectedly entered the hook/tool path") + print("PITOT_INCOMPATIBLE_RESPONSE_REJECTED evidence=binary-observed") + return 0 + evidence = validate_receipts(agent, args.platform, nonce, installation, proxy_receipt, witness_receipt, completed.returncode, completed.stdout, prompt, args.capture_output) + args.evidence.parent.mkdir(parents=True, exist_ok=True) + args.evidence.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print("PITOT_E2E_RESULT mode=real_cli evidence=nonce-correlated") + finally: + proxy_process.terminate() + try: proxy_process.wait(timeout=5) + except subprocess.TimeoutExpired: proxy_process.kill() + if cursor_system_canary: + if host_controls_wsl: + subprocess.run( + ["wsl.exe", "--distribution", "Ubuntu", "--", "rm", "-f", "/usr/local/bin/pitot-e2e-canary"], + check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) + else: + Path("/usr/local/bin/pitot-e2e-canary").unlink(missing_ok=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/labs/15-pitot/tests/run_e2e_report.py b/labs/15-pitot/tests/run_e2e_report.py old mode 100644 new mode 100755 index 80ca513f3..d5ae95bd2 --- a/labs/15-pitot/tests/run_e2e_report.py +++ b/labs/15-pitot/tests/run_e2e_report.py @@ -12,38 +12,62 @@ import sys -ROOT = Path(__file__).resolve().parents[3] -MANIFEST = json.loads((ROOT / "labs/15-pitot/adapter-verification.json").read_text(encoding="utf-8")) +LAB = Path(__file__).resolve().parent.parent +ROOT = LAB.parents[1] if LAB.name == "15-pitot" else LAB +MANIFEST = json.loads((LAB / "adapter-verification.json").read_text(encoding="utf-8")) AGENTS = {agent["id"] for agent in MANIFEST["agents"]} PLATFORMS = {platform["id"] for platform in MANIFEST["platforms"]} -RESULT_PATTERN = re.compile(r"^PITOT_E2E_RESULT mode=(real_cli|hook_subprocess)$", re.MULTILINE) +RESULT_PATTERN = re.compile(r"^PITOT_E2E_RESULT mode=real_cli evidence=nonce-correlated$", re.MULTILINE) -def result_for(agent: str, platform: str, returncode: int, output: str) -> dict[str, object]: +def load_evidence(path: Path | None, *, agent: str) -> dict[str, object] | None: + if path is None or not path.is_file(): + return None + value = json.loads(path.read_text(encoding="utf-8")) + required = {"schema_version", "agent", "cli", "prompt_hash", "protocol", "endpoint", "nonce", "receipts", "hook"} + if not isinstance(value, dict) or set(value) != required or value["schema_version"] != 1 or value["agent"] != agent: + return None + receipts = value.get("receipts") + receipt_fields = {"initial_prompt_observed", "tool_call_response_emitted", "tool_result_observed", "final_response_emitted", "hook_observed", "canary_result_observed", "cli_exit_zero"} + if not isinstance(receipts, dict) or set(receipts) != receipt_fields or not all(item is True for item in receipts.values()): + return None + if not re.fullmatch(r"[0-9a-f]{64}", str(value.get("prompt_hash", ""))): + return None + if value.get("hook", {}).get("action_kind") != "shell" or value.get("hook", {}).get("host") != agent: + return None + endpoint = value.get("endpoint", {}) + endpoint_required = {"fixture", "fixture_sha256", "provenance", "dialect", "request", "response", "executable_sha256"} + if not isinstance(endpoint, dict) or set(endpoint) != endpoint_required: + return None + if endpoint.get("provenance") != "pinned_real_cli_capture" or not re.fullmatch(r"[0-9a-f]{64}", str(endpoint.get("fixture_sha256", ""))): + return None + return value + + +def result_for(agent: str, platform: str, returncode: int, output: str, evidence_path: Path | None = None) -> dict[str, object]: if agent not in AGENTS: raise ValueError(f"unsupported agent: {agent}") if platform not in PLATFORMS: raise ValueError(f"unsupported platform: {platform}") - modes = RESULT_PATTERN.findall(output) - passed = returncode == 0 and len(modes) == 1 - mode = modes[0] if passed else None - if passed and mode == "real_cli": - evidence = "real host CLI completed the Pitot integration path" - elif passed: - evidence = "active hook subprocess produced a normalized Pitot action" - elif returncode != 0: - evidence = "E2E command failed" - else: - evidence = "E2E command returned without one valid result marker" + markers = RESULT_PATTERN.findall(output) + receipt = load_evidence(evidence_path, agent=agent) + passed = returncode == 0 and len(markers) == 1 and receipt is not None + evidence = "binary-observed request, accepted response, hook, canary, and final receipts" if passed else "real-agent evidence contract failed" return { - "schema_version": 1, + "schema_version": 2, "agent": agent, "platform": platform, "status": "pass" if passed else "fail", - "verification_mode": mode, + "verification_mode": "real_cli" if passed else None, "evidence": evidence, + "cli": receipt["cli"] if passed else None, + "protocol": receipt["protocol"] if passed else None, + "endpoint": receipt["endpoint"] if passed else None, + "prompt_hash": receipt["prompt_hash"] if passed else None, + "receipts": receipt["receipts"] if passed else None, + "hook": receipt["hook"] if passed else None, "commit_sha": os.environ.get("PITOT_SOURCE_SHA", os.environ.get("GITHUB_SHA", "local")), "run_url": ( f"{os.environ['GITHUB_SERVER_URL']}/{os.environ['GITHUB_REPOSITORY']}" @@ -62,6 +86,7 @@ def main() -> int: parser.add_argument("--agent", required=True, choices=sorted(AGENTS)) parser.add_argument("--platform", required=True, choices=sorted(PLATFORMS)) parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--evidence", required=True, type=Path) parser.add_argument("command", nargs=argparse.REMAINDER) args = parser.parse_args() command = args.command[1:] if args.command[:1] == ["--"] else args.command @@ -77,7 +102,7 @@ def main() -> int: stderr=subprocess.STDOUT, ) sys.stdout.buffer.write(completed.stdout.encode("utf-8", errors="replace")) - result = result_for(args.agent, args.platform, completed.returncode, completed.stdout) + result = result_for(args.agent, args.platform, completed.returncode, completed.stdout, args.evidence) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") return 0 if result["status"] == "pass" else 1 diff --git a/labs/15-pitot/tests/test_adapter_supervisor.py b/labs/15-pitot/tests/test_adapter_supervisor.py index d58a85a85..2ee08d07e 100644 --- a/labs/15-pitot/tests/test_adapter_supervisor.py +++ b/labs/15-pitot/tests/test_adapter_supervisor.py @@ -1,6 +1,7 @@ from __future__ import annotations import copy +import hashlib import importlib.util import json from pathlib import Path @@ -18,16 +19,40 @@ SPEC.loader.exec_module(supervisor) +def _manifest() -> dict[str, object]: + return json.loads((ROOT / supervisor.MANIFEST).read_text()) + + +def _provenance(manifest: dict[str, object]) -> dict[str, object]: + dialects = {"claude": "anthropic_messages", "codex": "openai_responses", "gemini": "gemini_generate_content", "cursor": "cursor_connect_proto"} + paths = {"anthropic_messages": "/v1/messages", "openai_responses": "/v1/responses", "gemini_generate_content": "/v1beta/models/pitot-control:streamGenerateContent", "cursor_connect_proto": "/agent.v1.AgentService/Run", "openai_chat": "/v1/chat/completions"} + cells = [] + for agent in manifest["agents"]: + for platform in manifest["platforms"]: + dialect = dialects.get(agent["id"], "openai_chat") + framing = "connect_envelope" if dialect == "cursor_connect_proto" else "sse" + request = {"transport": "http2" if dialect == "cursor_connect_proto" else "http1", "method": "POST", "path": paths[dialect], "media_type": "application/connect+proto" if dialect == "cursor_connect_proto" else "application/json", "framing": framing, "request_shape": {"fixture": "redacted"}} + response = {"encoder": dialect, "framing": framing, "tool_call": "native_shell", "acceptance": "nonce_tool_result_round_trip"} + core = {"agent": agent["id"], "platform": platform["id"], "runtime": agent["runtime"][platform["id"]], "version": agent["version"], "executable_sha256": hashlib.sha256(f"{agent['id']}/{platform['id']}".encode()).hexdigest(), "dialect": dialect, "request": request, "response": response, "provenance": "pinned_real_cli_capture"} + cells.append({**core, "capture_sha256": hashlib.sha256(json.dumps(core, sort_keys=True, separators=(",", ":")).encode()).hexdigest()}) + return {"schema_version": 2, "capture_policy": "pinned_real_cli_capture", "cells": cells} + + def _fixture(root: Path) -> tuple[Path, list[str]]: - manifest = copy.deepcopy(supervisor.load_manifest(ROOT)) + manifest = copy.deepcopy(_manifest()) manifest_path = root / supervisor.MANIFEST manifest_path.parent.mkdir(parents=True) manifest_path.write_text(json.dumps(manifest), encoding="utf-8") tests = root / "labs/15-pitot/tests" tests.mkdir(parents=True) + (tests / "endpoint-provenance.json").write_text(json.dumps(_provenance(manifest)), encoding="utf-8") ids = [agent["id"] for agent in manifest["agents"]] for agent_id in ids: (tests / f"e2e_{agent_id}_cli_test.sh").write_text("#!/usr/bin/env bash\n") + (tests / "real_agent_driver.py").write_text("# canonical driver\n") + (tests / "model_control_proxy.py").write_text("# proxy\n") + (tests / "cursor_control_proxy.mjs").write_text("// cursor proxy\n") + (tests / "e2e_unified_runner.sh").write_text("python3 real_agent_driver.py\n") readme = root / supervisor.README readme.parent.mkdir(parents=True) readme.write_text( @@ -39,9 +64,14 @@ def _fixture(root: Path) -> tuple[Path, list[str]]: reusable.write_text( "platform:\n description: Canonical Pitot platform identifier\n" "runner:\n description: GitHub-hosted runner selected by the supervisor\n" + "capture:\n description: Emit a redacted binary-observed candidate fixture\n" "runs-on: ${{ inputs.runner }}\n" '--agent "${{ inputs.agent }}"\n' '--platform "${{ inputs.platform }}"\n' + '--evidence "${{ runner.temp }}/pitot-e2e/evidence.json"\n' + "install_real_agent.py\n" + "PITOT_CAPTURE_OUTPUT:\n" + "pitot-endpoint-capture-${{ inputs.agent }}-${{ inputs.platform }}\n" "name: pitot-e2e-${{ inputs.agent }}-${{ inputs.platform }}\n" ) supervisor.render(root) @@ -53,7 +83,7 @@ def test_real_repository_contract_is_complete(self): self.assertEqual(supervisor.contract_errors(ROOT), []) def test_matrix_contains_every_agent_on_every_platform(self): - manifest = supervisor.load_manifest(ROOT) + manifest = _manifest() matrix = supervisor.matrix(manifest) self.assertEqual(len(matrix), 30) self.assertEqual( @@ -70,26 +100,122 @@ def test_missing_and_extra_inventory_entries_fail(self): root = Path(temp) manifest_path, ids = _fixture(root) value = json.loads(manifest_path.read_text()) + removed = value["agents"][0]["id"] value["agents"] = value["agents"][1:] manifest_path.write_text(json.dumps(value)) + provenance_path = root / supervisor.ENDPOINT_PROVENANCE + provenance = json.loads(provenance_path.read_text()) + provenance["cells"] = [cell for cell in provenance["cells"] if cell["agent"] != removed] + provenance_path.write_text(json.dumps(provenance)) errors = supervisor.contract_errors(root, ids) self.assertTrue(any("missing from manifest" in error for error in errors)) - value["agents"].append({"id": "phantom", "label": "Phantom"}) + phantom = copy.deepcopy(value["agents"][0]) + phantom.update({"id": "phantom", "label": "Phantom", "executable": "phantom"}) + value["agents"].append(phantom) manifest_path.write_text(json.dumps(value)) + provenance = _provenance(value) + provenance_path.write_text(json.dumps(provenance)) errors = supervisor.contract_errors(root, ids) self.assertTrue(any("not built-in adapters" in error for error in errors)) def test_duplicate_id_and_label_fail(self): - manifest = copy.deepcopy(supervisor.load_manifest(ROOT)) + manifest = copy.deepcopy(_manifest()) manifest["agents"].append(copy.deepcopy(manifest["agents"][0])) with self.assertRaisesRegex(supervisor.ContractError, "ids must be unique"): supervisor.validate_manifest(manifest) - manifest = copy.deepcopy(supervisor.load_manifest(ROOT)) + manifest = copy.deepcopy(_manifest()) manifest["agents"][1]["label"] = manifest["agents"][0]["label"] with self.assertRaisesRegex(supervisor.ContractError, "labels must be unique"): supervisor.validate_manifest(manifest) + def test_mutable_or_incomplete_real_cli_metadata_fails(self): + manifest = copy.deepcopy(_manifest()) + manifest["agents"][0]["version"] = "latest" + with self.assertRaisesRegex(supervisor.ContractError, "immutable CLI version"): + supervisor.validate_manifest(manifest) + manifest = copy.deepcopy(_manifest()) + manifest["agents"][0]["required_mode"] = "hook_subprocess" + with self.assertRaisesRegex(supervisor.ContractError, "real-CLI driver"): + supervisor.validate_manifest(manifest) + manifest = copy.deepcopy(_manifest()) + del manifest["agents"][0]["runtime"]["windows"] + with self.assertRaisesRegex(supervisor.ContractError, "every platform runtime"): + supervisor.validate_manifest(manifest) + + def test_matrix_carries_install_protocol_and_runtime_contract(self): + matrix = supervisor.matrix(_manifest()) + cursor_windows = next(item for item in matrix if item["agent"] == "cursor" and item["platform"] == "windows") + self.assertEqual(cursor_windows["runtime"], "wsl") + self.assertEqual(cursor_windows["version"], "2026.07.20-8cc9c0b") + self.assertNotIn("protocol", cursor_windows) + self.assertEqual(cursor_windows["endpoint_fixture"], "tests/endpoint-provenance.json#cursor/windows") + + def test_stale_or_missing_endpoint_provenance_fails(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + _, ids = _fixture(root) + fixture_path = root / supervisor.ENDPOINT_PROVENANCE + fixture = json.loads(fixture_path.read_text()) + cell = next(item for item in fixture["cells"] if item["agent"] == "cursor" and item["platform"] == "windows") + cell["version"] = "0.0.0" + fixture_path.write_text(json.dumps(fixture)) + errors = supervisor.contract_errors(root, ids) + self.assertTrue(any("stale for its pinned CLI" in error for error in errors)) + + def test_platform_swap_and_fabricated_capture_digest_fail(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + _, ids = _fixture(root) + fixture_path = root / supervisor.ENDPOINT_PROVENANCE + fixture = json.loads(fixture_path.read_text()) + cell = next(item for item in fixture["cells"] if item["agent"] == "cursor" and item["platform"] == "windows") + cell["runtime"] = "native" + fixture_path.write_text(json.dumps(fixture)) + self.assertTrue(any("stale for its pinned CLI" in error for error in supervisor.contract_errors(root, ids))) + fixture = _provenance(_manifest()) + fixture["cells"][0]["capture_sha256"] = "0" * 64 + fixture_path.write_text(json.dumps(fixture)) + self.assertTrue(any("fabricated capture digest" in error for error in supervisor.contract_errors(root, ids))) + + def test_capture_merge_requires_all_30_accepted_unique_cells(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + _fixture(root) + manifest = _manifest() + provenance = _provenance(manifest) + captures = root / "captures" + captures.mkdir() + for cell in provenance["cells"]: + (captures / f"{cell['agent']}-{cell['platform']}.json").write_text(json.dumps({"schema_version": 1, "accepted": True, "cell": cell})) + output = root / "merged.json" + supervisor.merge_captures(root, captures, output) + self.assertEqual(len(json.loads(output.read_text())["cells"]), 30) + next(captures.glob("*.json")).unlink() + with self.assertRaisesRegex(supervisor.ContractError, "exactly all 30"): + supervisor.merge_captures(root, captures, output) + + def test_capture_merge_rejects_mixed_versions_and_duplicates(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + _fixture(root) + provenance = _provenance(_manifest()) + captures = root / "captures" + captures.mkdir() + for index, cell in enumerate(provenance["cells"]): + (captures / f"{index}.json").write_text(json.dumps({"schema_version": 1, "accepted": True, "cell": cell})) + duplicate = provenance["cells"][0] + (captures / "duplicate.json").write_text(json.dumps({"schema_version": 1, "accepted": True, "cell": duplicate})) + with self.assertRaisesRegex(supervisor.ContractError, "exactly all 30"): + supervisor.merge_captures(root, captures, root / "merged.json") + (captures / "duplicate.json").unlink() + first = captures / "0.json" + value = json.loads(first.read_text()) + value["cell"]["version"] = "0.0.0" + first.write_text(json.dumps(value)) + with self.assertRaisesRegex(supervisor.ContractError, "stale for its pinned CLI"): + supervisor.merge_captures(root, captures, root / "merged.json") + def test_missing_script_fails(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) diff --git a/labs/15-pitot/tests/test_e2e_reporting.py b/labs/15-pitot/tests/test_e2e_reporting.py index a97825029..17db775b2 100644 --- a/labs/15-pitot/tests/test_e2e_reporting.py +++ b/labs/15-pitot/tests/test_e2e_reporting.py @@ -1,6 +1,7 @@ from __future__ import annotations import importlib.util +import hashlib import json from pathlib import Path import subprocess @@ -29,19 +30,52 @@ def _load(name: str, path: Path): ) -def _result(agent="claude", platform="ubuntu", status="pass", mode="real_cli"): - evidence = { - "real_cli": "real host CLI completed the Pitot integration path", - "hook_subprocess": "active hook subprocess produced a normalized Pitot action", - None: "E2E command failed", - }[mode] +def _receipt(agent="claude", platform="ubuntu"): + record = next(item for item in runner.MANIFEST["agents"] if item["id"] == agent) + dialect = {"claude": "anthropic_messages", "codex": "openai_responses", "gemini": "gemini_generate_content", "cursor": "cursor_connect_proto"}.get(agent, "openai_chat") + framing = "connect_envelope" if dialect == "cursor_connect_proto" else "sse" + request = {"transport": "http2" if dialect == "cursor_connect_proto" else "http1", "method": "POST", "path": "/fixture", "media_type": "application/connect+proto" if dialect == "cursor_connect_proto" else "application/json", "framing": framing, "request_shape": {"fixture": "redacted"}} + response = {"encoder": dialect, "framing": framing, "tool_call": "native_shell", "acceptance": "nonce_tool_result_round_trip"} + core = {"agent": agent, "platform": platform, "runtime": record["runtime"][platform], "version": record["version"], "executable_sha256": "d" * 64, "dialect": dialect, "request": request, "response": response, "provenance": "pinned_real_cli_capture"} + fixture = {**core, "capture_sha256": hashlib.sha256(json.dumps(core, sort_keys=True, separators=(",", ":")).encode()).hexdigest()} + reporter.ENDPOINT_PROVENANCE["cells"] = [cell for cell in reporter.ENDPOINT_PROVENANCE["cells"] if (cell["agent"], cell["platform"]) != (agent, platform)] + [fixture] + endpoint = { + "fixture": f"tests/endpoint-provenance.json#{agent}/{platform}", + "fixture_sha256": fixture["capture_sha256"], + "provenance": "pinned_real_cli_capture", + "dialect": dialect, + "request": request, + "response": response, + "executable_sha256": fixture["executable_sha256"], + } return { "schema_version": 1, "agent": agent, + "cli": {"version": record["version"], "executable": f"/bin/{record['executable']}", "executable_sha256": fixture["executable_sha256"], "installer": record["installer"]["kind"], "runtime": record["runtime"][platform]}, + "prompt_hash": "f" * 64, + "protocol": dialect, + "endpoint": endpoint, + "nonce": "nonce", + "receipts": {name: True for name in ("initial_prompt_observed", "tool_call_response_emitted", "tool_result_observed", "final_response_emitted", "hook_observed", "canary_result_observed", "cli_exit_zero")}, + "hook": {"host": agent, "action_kind": "shell", "pitot_exit": 0}, + } + + +def _result(agent="claude", platform="ubuntu", status="pass", mode="real_cli"): + receipt = _receipt(agent, platform) if status == "pass" else None + return { + "schema_version": 2, + "agent": agent, "platform": platform, "status": status, "verification_mode": mode, - "evidence": evidence, + "evidence": "binary-observed request, accepted response, hook, canary, and final receipts" if status == "pass" else "real-agent evidence contract failed", + "cli": receipt["cli"] if receipt else None, + "protocol": receipt["protocol"] if receipt else None, + "endpoint": receipt["endpoint"] if receipt else None, + "prompt_hash": receipt["prompt_hash"] if receipt else None, + "receipts": receipt["receipts"] if receipt else None, + "hook": receipt["hook"] if receipt else None, "commit_sha": "a" * 40, "run_url": "https://github.com/operatorstack/intelligence-flow/actions/runs/1", } @@ -58,27 +92,33 @@ def test_mock_anthropic_request_classifier(self): ) def test_real_cli_marker_passes(self): - result = runner.result_for("claude", "ubuntu", 0, "PITOT_E2E_RESULT mode=real_cli\n") - self.assertEqual(result["status"], "pass") - self.assertEqual(result["verification_mode"], "real_cli") + with tempfile.TemporaryDirectory() as temp: + evidence = Path(temp) / "evidence.json" + evidence.write_text(json.dumps(_receipt())) + result = runner.result_for("claude", "ubuntu", 0, "PITOT_E2E_RESULT mode=real_cli evidence=nonce-correlated\n", evidence) + self.assertEqual(result["status"], "pass") + self.assertEqual(result["verification_mode"], "real_cli") - def test_hook_marker_passes(self): + def test_hook_marker_cannot_pass(self): result = runner.result_for("gemini", "macos", 0, "PITOT_E2E_RESULT mode=hook_subprocess\n") - self.assertEqual(result["status"], "pass") - self.assertEqual(result["verification_mode"], "hook_subprocess") + self.assertEqual(result["status"], "fail") + self.assertIsNone(result["verification_mode"]) def test_windows_result_is_supported(self): - result = runner.result_for("opencode", "windows", 0, "PITOT_E2E_RESULT mode=hook_subprocess\n") - self.assertEqual(result["platform"], "windows") - self.assertEqual(result["status"], "pass") + with tempfile.TemporaryDirectory() as temp: + evidence = Path(temp) / "evidence.json" + evidence.write_text(json.dumps(_receipt("opencode", "windows"))) + result = runner.result_for("opencode", "windows", 0, "PITOT_E2E_RESULT mode=real_cli evidence=nonce-correlated\n", evidence) + self.assertEqual(result["platform"], "windows") + self.assertEqual(result["status"], "pass") def test_missing_or_duplicate_marker_fails(self): self.assertEqual(runner.result_for("codex", "ubuntu", 0, "ok\n")["status"], "fail") - duplicate = "PITOT_E2E_RESULT mode=real_cli\n" * 2 + duplicate = "PITOT_E2E_RESULT mode=real_cli evidence=nonce-correlated\n" * 2 self.assertEqual(runner.result_for("codex", "ubuntu", 0, duplicate)["status"], "fail") def test_nonzero_command_fails_even_with_marker(self): - result = runner.result_for("cursor", "ubuntu", 1, "PITOT_E2E_RESULT mode=real_cli\n") + result = runner.result_for("cursor", "ubuntu", 1, "PITOT_E2E_RESULT mode=real_cli evidence=nonce-correlated\n") self.assertEqual(result["status"], "fail") self.assertIsNone(result["verification_mode"]) @@ -92,6 +132,7 @@ def test_runner_replaces_non_utf8_command_output(self): "--agent", "claude", "--platform", "windows", "--output", str(output), + "--evidence", str(Path(temp) / "missing-evidence.json"), "--", sys.executable, "-c", "import sys; sys.stdout.buffer.write(b'\\x96')", @@ -104,6 +145,24 @@ def test_runner_replaces_non_utf8_command_output(self): class ReporterContractTests(unittest.TestCase): + def test_artifact_listing_includes_inventory_beyond_default_page(self): + github = reporter.GitHub("operatorstack/intelligence-flow", "test-token") + calls = [] + + def request(method, path, payload=None): + calls.append((method, path, payload)) + return {"total_count": 31, "artifacts": [{"name": str(index)} for index in range(31)]} + + github.request = request + self.assertEqual(len(github.list_run_artifacts(7)), 31) + self.assertEqual(calls, [("GET", "/actions/runs/7/artifacts?per_page=100", None)]) + + def test_truncated_artifact_listing_fails_closed(self): + github = reporter.GitHub("operatorstack/intelligence-flow", "test-token") + github.request = lambda method, path, payload=None: {"total_count": 31, "artifacts": [{}] * 30} + with self.assertRaisesRegex(ValueError, "truncated or malformed"): + github.list_run_artifacts(7) + def test_artifact_redirect_strips_cross_host_credentials(self): request = urllib.request.Request( "https://api.github.com/repos/operatorstack/intelligence-flow/actions/artifacts/1/zip", @@ -135,8 +194,9 @@ def test_artifact_redirect_rejects_http_downgrade(self): def test_workflow_matrix_covers_every_reported_platform(self): workflow = (ROOT / ".github/workflows/pitot-e2e-agent.yml").read_text() unified = (ROOT / ".github/workflows/pitot-e2e.yml").read_text() + runner_script = (ROOT / "labs/15-pitot/tests/e2e_unified_runner.sh").read_text() self.assertEqual(set(reporter.PLATFORMS), set(runner.PLATFORMS)) - generated = supervisor.matrix(supervisor.load_manifest(ROOT)) + generated = supervisor.matrix(supervisor.load_manifest_without_provenance(ROOT)) self.assertEqual(len(generated), len(reporter.AGENTS) * len(reporter.PLATFORMS)) self.assertEqual( {(item["agent"], item["platform"]) for item in generated}, @@ -146,9 +206,17 @@ def test_workflow_matrix_covers_every_reported_platform(self): self.assertIn('--platform "${{ inputs.platform }}"', workflow) self.assertIn('bash_path="$(cygpath -m "$bash_path")"', workflow) self.assertIn('-- "$PITOT_BASH"', workflow) + self.assertIn("node_version=22.23.1", workflow) + self.assertIn("SHASUMS256.txt", workflow) self.assertNotIn('-- bash "labs/15-pitot/tests/e2e_', workflow) self.assertIn("fromJSON(needs.inventory.outputs.matrix)", unified) self.assertIn("name: pitot-e2e-inventory", unified) + self.assertEqual(runner_script.count("GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build"), 2) + self.assertIn("pitot-linux", runner_script) + self.assertIn("pitot-witness-linux", runner_script) + self.assertIn("wsl.exe --distribution Ubuntu -- env", runner_script) + self.assertIn('python3 "$(to_wsl_path "$SCRIPT_DIR/real_agent_driver.py")"', runner_script) + self.assertIn("MSYS2_ARG_CONV_EXCL='*'", runner_script) def test_inventory_schema_rejects_identity_injection(self): inventory = json.loads((ROOT / "labs/15-pitot/adapter-verification.json").read_text()) @@ -192,6 +260,28 @@ def test_rejects_identity_and_evidence_injection(self): with self.assertRaises(ValueError): reporter.validate_result(value, agent="claude", platform="ubuntu") + def test_rejects_incomplete_or_mismatched_causal_receipts(self): + value = _result() + value["receipts"]["hook_observed"] = False + with self.assertRaisesRegex(ValueError, "incomplete causal receipts"): + reporter.validate_result(value, agent="claude", platform="ubuntu") + value = _result() + value["hook"]["host"] = "cursor" + with self.assertRaisesRegex(ValueError, "hook receipt"): + reporter.validate_result(value, agent="claude", platform="ubuntu") + value = _result() + value["cli"]["version"] = "latest" + with self.assertRaisesRegex(ValueError, "supervised manifest"): + reporter.validate_result(value, agent="claude", platform="ubuntu") + value = _result() + value["endpoint"]["fixture_sha256"] = "0" * 64 + with self.assertRaisesRegex(ValueError, "endpoint receipt"): + reporter.validate_result(value, agent="claude", platform="ubuntu") + value = _result() + value["cli"]["executable_sha256"] = "0" * 64 + with self.assertRaisesRegex(ValueError, "executable digest"): + reporter.validate_result(value, agent="claude", platform="ubuntu") + def test_rejects_untrusted_commit_and_run_url(self): value = _result() with self.assertRaises(ValueError): @@ -211,7 +301,7 @@ def test_aggregate_requires_all_platforms(self): reporter.aggregate( { "ubuntu": _result(), - "macos": _result(platform="macos", mode="hook_subprocess"), + "macos": _result(platform="macos"), "windows": None, } ), @@ -221,8 +311,8 @@ def test_aggregate_requires_all_platforms(self): reporter.aggregate( { "ubuntu": _result(), - "macos": _result(platform="macos", mode="hook_subprocess"), - "windows": _result(platform="windows", mode="hook_subprocess"), + "macos": _result(platform="macos"), + "windows": _result(platform="windows"), } ), "passing", @@ -232,7 +322,7 @@ def test_aggregate_requires_all_platforms(self): { "ubuntu": _result(), "macos": _result(platform="macos", status="fail", mode=None), - "windows": _result(platform="windows", mode="hook_subprocess"), + "windows": _result(platform="windows"), } ), "failing", @@ -245,13 +335,14 @@ def test_rendered_comment_is_sticky_and_shows_modes(self): } results["claude"] = { "ubuntu": _result(), - "macos": _result(platform="macos", mode="hook_subprocess"), - "windows": _result(platform="windows", mode="hook_subprocess"), + "macos": _result(platform="macos"), + "windows": _result(platform="windows"), } body = reporter.render_comment(results, {"claude": "https://example.test/run"}, "b" * 40) self.assertIn(reporter.MARKER, body) self.assertIn("Pass · real CLI", body) - self.assertIn("Pass · hook verified", body) + self.assertNotIn("hook verified", body) + self.assertIn("real CLI 2.1.217 · native", body) self.assertIn("Windows", body) self.assertIn("All platforms are required", body) self.assertIn("OpenCode", body) diff --git a/labs/15-pitot/tests/test_model_control_proxy.py b/labs/15-pitot/tests/test_model_control_proxy.py new file mode 100644 index 000000000..0a6ff9423 --- /dev/null +++ b/labs/15-pitot/tests/test_model_control_proxy.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import subprocess +import unittest + + +ROOT = Path(__file__).resolve().parents[3] +spec = importlib.util.spec_from_file_location("model_control_proxy", ROOT / "labs/15-pitot/tests/model_control_proxy.py") +proxy = importlib.util.module_from_spec(spec) +assert spec and spec.loader +spec.loader.exec_module(proxy) +driver_spec = importlib.util.spec_from_file_location("real_agent_driver", ROOT / "labs/15-pitot/tests/real_agent_driver.py") +driver = importlib.util.module_from_spec(driver_spec) +assert driver_spec and driver_spec.loader +driver_spec.loader.exec_module(driver) + + +class ModelControlProtocolTests(unittest.TestCase): + def test_anthropic_messages_emits_native_shell_tool(self): + value = proxy.anthropic("Bash", "pitot-e2e-canary nonce", False) + self.assertEqual(value["content"][0]["type"], "tool_use") + self.assertEqual(value["content"][0]["input"]["command"], "pitot-e2e-canary nonce") + + def test_openai_chat_emits_native_shell_tool(self): + value = proxy.chat("bash", "pitot-e2e-canary nonce", False) + call = value["choices"][0]["message"]["tool_calls"][0]["function"] + self.assertEqual(call["name"], "bash") + self.assertEqual(json.loads(call["arguments"])["command"], "pitot-e2e-canary nonce") + + def test_openai_responses_emits_native_shell_tool(self): + value = proxy.responses("shell", "pitot-e2e-canary nonce", False) + self.assertEqual(value["output"][0]["type"], "function_call") + self.assertEqual(json.loads(value["output"][0]["arguments"])["command"], "pitot-e2e-canary nonce") + + def test_gemini_emits_native_shell_tool(self): + value = proxy.gemini("run_shell_command", "pitot-e2e-canary nonce", False) + call = value["candidates"][0]["content"]["parts"][0]["functionCall"] + self.assertEqual(call["args"]["command"], "pitot-e2e-canary nonce") + + def test_final_responses_do_not_emit_tools(self): + self.assertEqual(proxy.anthropic("Bash", "ignored", True)["content"][0]["type"], "text") + self.assertNotIn("tool_calls", proxy.chat("bash", "ignored", True)["choices"][0]["message"]) + self.assertEqual(proxy.responses("shell", "ignored", True)["output"][0]["type"], "message") + self.assertIn("text", proxy.gemini("run_shell_command", "ignored", True)["candidates"][0]["content"]["parts"][0]) + + def test_tool_selection_uses_agent_advertised_shell_name(self): + body = {"tools": [{"type": "function", "function": {"name": "run_shell_command"}}]} + self.assertEqual(proxy.tool_name(body), "run_shell_command") + + def test_live_request_classifies_all_json_dialects_without_manifest_hint(self): + cases = ( + ("/v1/messages", {"messages": []}, "anthropic_messages"), + ("/v1/chat/completions", {"messages": []}, "openai_chat"), + ("/v1/responses", {"input": []}, "openai_responses"), + ("/v1beta/models/x:streamGenerateContent", {"contents": []}, "gemini_generate_content"), + ) + for path, body, expected in cases: + with self.subTest(expected): + self.assertEqual(proxy.classify_request(path, body), expected) + + def test_unknown_or_ambiguous_request_fails_closed(self): + with self.assertRaises(proxy.UnknownProtocol): + proxy.classify_request("/invented", {"messages": []}) + with self.assertRaises(proxy.UnknownProtocol): + proxy.classify_request("/v1/messages", {"input": []}) + + def test_capture_record_binds_detected_dialect_and_executable(self): + agent = next(item for item in driver.MANIFEST["agents"] if item["id"] == "claude") + installation = {"version": agent["version"], "runtime": "native", "executable_sha256": "a" * 64} + observed = {"transport": "http1", "method": "POST", "path": "/v1/messages", "media_type": "application/json", "framing": "sse", "request_shape": {}} + record = driver.capture_record(agent, "macos", installation, {"protocol": "anthropic_messages", "endpoint_observed": observed}) + self.assertEqual(record["provenance"], "pinned_real_cli_capture") + self.assertEqual(record["dialect"], "anthropic_messages") + self.assertRegex(record["capture_sha256"], r"^[0-9a-f]{64}$") + + def test_invented_endpoint_cannot_pass(self): + agent = next(item for item in driver.MANIFEST["agents"] if item["id"] == "cursor") + installation = {"version": agent["version"], "runtime": "native", "executable_sha256": "a" * 64} + with self.assertRaisesRegex(RuntimeError, "supported request contract"): + driver.capture_record(agent, "macos", installation, {"protocol": "invented", "endpoint_observed": {}}) + + def test_cursor_connect_protobuf_fixture(self): + completed = subprocess.run( + ["node", str(ROOT / "labs/15-pitot/tests/cursor_control_proxy.mjs"), "--self-test"], + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + self.assertIn("PASS", completed.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/labs/15-pitot/tests/witness/main.go b/labs/15-pitot/tests/witness/main.go new file mode 100644 index 000000000..06e8a4544 --- /dev/null +++ b/labs/15-pitot/tests/witness/main.go @@ -0,0 +1,95 @@ +// Command pitot-witness is a transparent executable wrapper used only by the +// real-agent E2E. It proves that the host invoked Pitot without changing the +// bytes or exit status observed by either process. +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" +) + +type event struct { + Type string `json:"type"` + Host struct { + Name string `json:"name"` + } `json:"host"` + Action *struct { + Kind string `json:"kind"` + } `json:"action"` + Content *struct { + Mode string `json:"mode"` + Full json.RawMessage `json:"full"` + } `json:"content"` +} + +func main() { + flags := flag.NewFlagSet("pitot-witness", flag.ContinueOnError) + pitotFlag := flags.String("real-bin", "", "real Pitot executable") + receiptFlag := flags.String("receipt", "", "witness receipt path") + nonceFlag := flags.String("nonce", "", "session nonce") + if err := flags.Parse(os.Args[1:]); err != nil { + os.Exit(125) + } + args := flags.Args() + pitot, receipt, nonce := *pitotFlag, *receiptFlag, *nonceFlag + if pitot == "" { + pitot = os.Getenv("PITOT_REAL_BIN") + } + if receipt == "" { + receipt = os.Getenv("PITOT_WITNESS_RECEIPT") + } + if nonce == "" { + nonce = os.Getenv("PITOT_E2E_NONCE") + } + if pitot == "" || receipt == "" || nonce == "" { + fmt.Fprintln(os.Stderr, "pitot-witness: PITOT_REAL_BIN, PITOT_WITNESS_RECEIPT, and PITOT_E2E_NONCE are required") + os.Exit(125) + } + in, err := io.ReadAll(os.Stdin) + if err != nil { + panic(err) + } + command := exec.Command(pitot, args...) + command.Stdin = bytes.NewReader(in) + var stdout, stderr bytes.Buffer + command.Stdout, command.Stderr = &stdout, &stderr + err = command.Run() + _, _ = os.Stdout.Write(stdout.Bytes()) + _, _ = os.Stderr.Write(stderr.Bytes()) + code := 0 + if err != nil { + if exit, ok := err.(*exec.ExitError); ok { + code = exit.ExitCode() + } else { + code = 125 + } + } + + host := "" + if len(args) >= 2 && args[0] == "hook" { + host = args[1] + } + var observed event + valid := json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &observed) == nil && + observed.Type == "action.requested" && observed.Host.Name == host && + observed.Action != nil && observed.Action.Kind == "shell" && + observed.Content != nil && observed.Content.Mode == "full" && + strings.Contains(string(observed.Content.Full), nonce) + record := map[string]any{ + "schema_version": 1, "host": host, "nonce": nonce, "action_kind": "shell", + "pitot_exit": code, "valid": valid, + } + encoded, _ := json.MarshalIndent(record, "", " ") + if valid { + _ = os.MkdirAll(filepath.Dir(receipt), 0o755) + _ = os.WriteFile(receipt, append(encoded, '\n'), 0o600) + } + os.Exit(code) +}