diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..c35f4a980 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +labs/15-pitot/integrations/cursor/beforeShellExecution text eol=lf diff --git a/.github/scripts/pitot_e2e_report.py b/.github/scripts/pitot_e2e_report.py index 1d5b6d035..9e3247ac2 100644 --- a/.github/scripts/pitot_e2e_report.py +++ b/.github/scripts/pitot_e2e_report.py @@ -65,8 +65,13 @@ def validate_result( "protocol", "endpoint", "prompt_hash", + "nonce", "receipts", - "hook", + "runtime", + "hooks", + "controller", + "consumer", + "canary", "commit_sha", "run_url", } @@ -81,10 +86,10 @@ def validate_result( raise ValueError("passing result requires real_cli verification") if value["status"] == "fail" and mode is not None: raise ValueError("failed result cannot claim a verification mode") - allowed_evidence = {"binary-observed request, accepted response, hook, canary, and final receipts", "real-agent evidence contract failed"} + allowed_evidence = {"binary-observed request, real hook control, projected Consumer, allow/deny canary, and final receipts", "real-agent control evidence contract failed"} if value["evidence"] not in allowed_evidence: raise ValueError("invalid evidence summary") - evidence_fields = ("cli", "protocol", "endpoint", "prompt_hash", "receipts", "hook") + evidence_fields = ("cli", "protocol", "endpoint", "prompt_hash", "nonce", "receipts", "runtime", "hooks", "controller", "consumer", "canary") if value["status"] == "fail": if any(value[field] is not None for field in evidence_fields): raise ValueError("failed result cannot carry passing evidence") @@ -103,6 +108,9 @@ def validate_result( raise ValueError("CLI executable digest does not match binary capture") if value["protocol"] != fixture["dialect"] or not re.fullmatch(r"[0-9a-f]{64}", str(value["prompt_hash"])): raise ValueError("invalid prompt/protocol receipt") + nonce = value["nonce"] + if not isinstance(nonce, str) or not re.fullmatch(r"[0-9a-f]{32}", nonce): + raise ValueError("invalid session nonce") expected_endpoint = { "fixture": f"tests/endpoint-provenance.json#{agent}/{platform}", "fixture_sha256": fixture["capture_sha256"], @@ -114,11 +122,28 @@ def validate_result( } if value["endpoint"] != expected_endpoint: raise ValueError("endpoint receipt does not match pinned real-CLI provenance") - receipt_fields = {"initial_prompt_observed", "tool_call_response_emitted", "tool_result_observed", "final_response_emitted", "hook_observed", "canary_result_observed", "cli_exit_zero"} + receipt_fields = { + "initial_prompt_observed", "allow_tool_call_response_emitted", "allow_tool_result_observed", + "deny_tool_call_response_emitted", "denied_result_observed", "final_response_emitted", + "consumer_observed", "controller_allow_observed", "controller_deny_observed", + "deny_canary_absent", "final_output_observed", "cli_exit_zero", + } if not isinstance(value["receipts"], dict) or set(value["receipts"]) != receipt_fields or not all(item is True for item in value["receipts"].values()): raise ValueError("incomplete causal receipts") - if value["hook"] != {"host": agent, "action_kind": "shell", "pitot_exit": 0}: - raise ValueError("invalid Pitot hook receipt") + hooks = value["hooks"] + if not isinstance(hooks, list) or len(hooks) != 2 or [item.get("pitot_exit") for item in hooks] != [0, 2] or any(item.get("host") != agent or item.get("action_kind") != "shell" or item.get("nonce") != nonce for item in hooks): + raise ValueError("invalid Pitot allow/deny hook receipts") + action_ids = [item.get("action_id") for item in hooks] + if len(set(action_ids)) != 2 or value["controller"] != {"id": "e2e-shell-controller", "action_ids": action_ids, "outcomes": ["allow", "deny"]}: + raise ValueError("invalid Controller receipts") + if value["consumer"] != {"id": "e2e-audit", "action_ids": action_ids, "projection": "sha256"}: + raise ValueError("invalid Consumer receipts") + canary = value["canary"] + if not isinstance(canary, dict) or canary.get("denied_executions") != 0 or canary.get("executions") != [f"PITOT_ALLOW {nonce}"]: + raise ValueError("invalid canary execution receipts") + runtime = value["runtime"] + if not isinstance(runtime, dict) or set(runtime) != {"schema_version", "instance_id", "pid", "endpoint", "config_sha256"} or runtime["schema_version"] != 1: + raise ValueError("invalid authenticated runtime receipt") if not isinstance(value["commit_sha"], str) or not re.fullmatch(r"[0-9a-f]{40}", value["commit_sha"]): raise ValueError("invalid commit_sha") if not isinstance(value["run_url"], str) or not re.fullmatch( @@ -141,6 +166,42 @@ def validate_inventory(value: object) -> dict[str, object]: return value +def validate_runtime_result( + value: object, *, platform: str, expected_sha: str | None = None, expected_run_url: str | None = None, +) -> dict[str, object]: + required = {"schema_version", "capability", "platform", "status", "verification_mode", "evidence", "nonce", "runtime", "controller", "receipts", "commit_sha", "run_url"} + if not isinstance(value, dict) or set(value) != required: + raise ValueError("runtime result fields do not match schema") + if value["schema_version"] != 2 or value["capability"] != "explicit_request" or value["platform"] != platform: + raise ValueError("runtime result identity does not match artifact") + if value["status"] not in {"pass", "fail"}: + raise ValueError("invalid runtime result status") + if value["status"] == "pass": + if value["verification_mode"] != "real_runtime" or value["evidence"] != "real request CLI, authenticated runtime, and correlated allow/deny Controller receipts": + raise ValueError("passing runtime result lacks real evidence") + if not isinstance(value["nonce"], str) or not re.fullmatch(r"[0-9a-f]{32}", value["nonce"]): + raise ValueError("invalid runtime session nonce") + controller = value["controller"] + if not isinstance(controller, dict) or controller.get("outcomes") != ["allow", "deny"] or not isinstance(controller.get("action_ids"), list) or len(controller["action_ids"]) != 2: + raise ValueError("runtime Controller receipt is incomplete") + if not isinstance(value["receipts"], dict) or not all(item is True for item in value["receipts"].values()): + raise ValueError("runtime causal receipts are incomplete") + runtime = value["runtime"] + if not isinstance(runtime, dict) or set(runtime) != {"schema_version", "instance_id", "pid", "endpoint", "config_sha256"} or runtime["schema_version"] != 1: + raise ValueError("runtime identity receipt is invalid") + elif value["verification_mode"] is not None or any(value[field] is not None for field in ("nonce", "runtime", "controller", "receipts")): + raise ValueError("failed runtime result cannot carry passing evidence") + if not isinstance(value["commit_sha"], str) or not re.fullmatch(r"[0-9a-f]{40}", value["commit_sha"]): + raise ValueError("invalid runtime commit_sha") + if not isinstance(value["run_url"], str) or not re.fullmatch(r"https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/actions/runs/[0-9]+", value["run_url"]): + raise ValueError("invalid runtime run_url") + if expected_sha is not None and value["commit_sha"] != expected_sha: + raise ValueError("runtime result commit does not match workflow run") + if expected_run_url is not None and value["run_url"] != expected_run_url: + raise ValueError("runtime result URL does not match workflow run") + return value + + def aggregate( platform_results: dict[str, dict[str, object] | None], platforms: tuple[str, ...] = PLATFORMS, @@ -168,13 +229,18 @@ def failed_result(agent: str, platform: str, head_sha: str, run_url: str) -> dic "platform": platform, "status": "fail", "verification_mode": None, - "evidence": "real-agent evidence contract failed", + "evidence": "real-agent control evidence contract failed", "cli": None, "protocol": None, "endpoint": None, "prompt_hash": None, + "nonce": None, "receipts": None, - "hook": None, + "runtime": None, + "hooks": None, + "controller": None, + "consumer": None, + "canary": None, "commit_sha": head_sha, "run_url": run_url, } @@ -186,6 +252,7 @@ def render_comment( head_sha: str, agent_records: tuple[dict[str, str], ...] | None = None, platforms: tuple[str, ...] = PLATFORMS, + runtime_results: dict[str, dict[str, object] | None] | None = None, ) -> str: lines = [ MARKER, @@ -211,6 +278,23 @@ def render_comment( f"{result_cell(platform_results.get('macos'))} | {result_cell(platform_results.get('windows'))} | " f"{icons[status]} | {evidence} |" ) + lines.extend([ + "", + "### Runtime capabilities", + "", + "| Capability | Ubuntu | macOS | Windows | Result |", + "|---|---|---|---|---|", + ]) + capability_results = runtime_results or {} + runtime_status = aggregate(capability_results, platforms) + def runtime_cell(result: dict[str, object] | None) -> str: + if result is None: + return "⏳ Pending" + return "✅ Pass · real runtime allow/deny" if result["status"] == "pass" else "❌ Failed" + lines.append( + f"| `pitot request` | {runtime_cell(capability_results.get('ubuntu'))} | " + f"{runtime_cell(capability_results.get('macos'))} | {runtime_cell(capability_results.get('windows'))} | {icons[runtime_status]} |" + ) lines.extend(["", f"Source commit: `{head_sha[:12]}`", ""]) return "\n".join(lines) @@ -293,6 +377,18 @@ def load_inventory(github: GitHub, artifact: dict[str, object]) -> dict[str, obj return validate_inventory(json.loads(bundle.read(names[0]))) +def load_runtime_artifact( + github: GitHub, artifact: dict[str, object], *, platform: str, expected_sha: str, expected_run_url: str, +) -> dict[str, object]: + archive = github.download(str(artifact["archive_download_url"])) + with zipfile.ZipFile(io.BytesIO(archive)) as bundle: + names = bundle.namelist() + if names != ["runtime-result.json"] or bundle.getinfo(names[0]).file_size > MAX_ARTIFACT_BYTES: + raise ValueError("runtime artifact must contain one bounded runtime-result.json") + value = json.loads(bundle.read(names[0])) + return validate_runtime_result(value, platform=platform, expected_sha=expected_sha, expected_run_url=expected_run_url) + + def collect_results( github: GitHub, by_name: dict[str, dict[str, object]], @@ -323,32 +419,69 @@ def collect_results( return results +def failed_runtime_result(platform: str, head_sha: str, run_url: str) -> dict[str, object]: + return { + "schema_version": 2, "capability": "explicit_request", "platform": platform, + "status": "fail", "verification_mode": None, "evidence": "explicit request evidence contract failed", + "nonce": None, "runtime": None, "controller": None, "receipts": None, + "commit_sha": head_sha, "run_url": run_url, + } + + +def collect_runtime_results( + github: GitHub, by_name: dict[str, dict[str, object]], platforms: tuple[str, ...], head_sha: str, run_url: str, +) -> dict[str, dict[str, object]]: + results: dict[str, dict[str, object]] = {} + for platform in platforms: + artifact = by_name.get(f"pitot-e2e-runtime-{platform}") + if artifact is not None: + try: + results[platform] = load_runtime_artifact( + github, artifact, platform=platform, expected_sha=head_sha, expected_run_url=run_url, + ) + continue + except (ValueError, json.JSONDecodeError, zipfile.BadZipFile): + pass + results[platform] = failed_runtime_result(platform, head_sha, run_url) + return results + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--event", type=Path, required=True) parser.add_argument("--repository", required=True) args = parser.parse_args() event = json.loads(args.event.read_text(encoding="utf-8")) - workflow_run = event["workflow_run"] token = os.environ.get("GITHUB_TOKEN") if not token: raise SystemExit("GITHUB_TOKEN is required") github = GitHub(args.repository, token) - pull_requests = workflow_run.get("pull_requests", []) - if not pull_requests: - owner = workflow_run.get("head_repository", {}).get("owner", {}).get("login") - branch = workflow_run.get("head_branch") - if owner and branch: - head = urllib.parse.quote(f"{owner}:{branch}", safe=":") - candidates = github.request("GET", f"/pulls?state=open&head={head}&per_page=20") - pull_requests = [pr for pr in candidates if pr.get("head", {}).get("sha") == workflow_run["head_sha"]] + if "pull_request" in event: + pull_requests = [event["pull_request"]] + head_sha = event["pull_request"]["head"]["sha"] + run_id = int(os.environ["GITHUB_RUN_ID"]) + server_url = os.environ.get("GITHUB_SERVER_URL", "https://github.com") + run_url = f"{server_url}/{args.repository}/actions/runs/{run_id}" + elif "workflow_run" in event: + workflow_run = event["workflow_run"] + pull_requests = workflow_run.get("pull_requests", []) if not pull_requests: - print("No pull request is associated with this workflow run; nothing to report.") - return 0 - head_sha = workflow_run["head_sha"] - run_url = workflow_run["html_url"] + owner = workflow_run.get("head_repository", {}).get("owner", {}).get("login") + branch = workflow_run.get("head_branch") + if owner and branch: + head = urllib.parse.quote(f"{owner}:{branch}", safe=":") + candidates = github.request("GET", f"/pulls?state=open&head={head}&per_page=20") + pull_requests = [pr for pr in candidates if pr.get("head", {}).get("sha") == workflow_run["head_sha"]] + if not pull_requests: + print("No pull request is associated with this workflow run; nothing to report.") + return 0 + head_sha = workflow_run["head_sha"] + run_url = workflow_run["html_url"] + run_id = int(workflow_run["id"]) + else: + raise ValueError("reporter requires a pull_request or workflow_run event") - artifacts = github.list_run_artifacts(int(workflow_run["id"])) + artifacts = github.list_run_artifacts(run_id) by_name = {artifact["name"]: artifact for artifact in artifacts if not artifact.get("expired")} inventory_artifact = by_name.get("pitot-e2e-inventory") if inventory_artifact is None: @@ -359,8 +492,9 @@ def main() -> int: platforms = tuple(platform["id"] for platform in inventory["platforms"]) run_urls = {agent: run_url for agent in agent_ids} results = collect_results(github, by_name, inventory, head_sha, run_url) + runtime_results = collect_runtime_results(github, by_name, platforms, head_sha, run_url) - body = render_comment(results, run_urls, head_sha, agent_records, platforms) + body = render_comment(results, run_urls, head_sha, agent_records, platforms, runtime_results) issue_number = pull_requests[0]["number"] comments = github.request("GET", f"/issues/{issue_number}/comments?per_page=100") existing = next( diff --git a/.github/workflows/pitot-e2e-report.yml b/.github/workflows/pitot-e2e-report.yml deleted file mode 100644 index df03f5958..000000000 --- a/.github/workflows/pitot-e2e-report.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Generated by pitot_adapter_supervisor.py. Do not edit directly. -name: Report Pitot E2E results - -on: - workflow_run: - workflows: [Pitot E2E] - types: [completed] - -permissions: - actions: read - contents: read - pull-requests: write - -concurrency: - group: pitot-e2e-report-${{ github.event.workflow_run.head_sha }} - cancel-in-progress: true - -jobs: - report: - if: github.event.workflow_run.event == 'pull_request' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - name: Update sticky PR report - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: >- - python3 .github/scripts/pitot_e2e_report.py - --event "$GITHUB_EVENT_PATH" - --repository "$GITHUB_REPOSITORY" diff --git a/.github/workflows/pitot-e2e.yml b/.github/workflows/pitot-e2e.yml index 89831ff45..099ceeda8 100644 --- a/.github/workflows/pitot-e2e.yml +++ b/.github/workflows/pitot-e2e.yml @@ -20,13 +20,16 @@ on: default: false permissions: + actions: read contents: read + pull-requests: write jobs: inventory: runs-on: ubuntu-latest outputs: - matrix: ${{ steps.supervisor.outputs.matrix }} + agent_matrix: ${{ steps.supervisor.outputs.agent_matrix }} + runtime_matrix: ${{ steps.supervisor.outputs.runtime_matrix }} steps: - uses: actions/checkout@v7 - uses: actions/setup-go@v7 @@ -40,8 +43,10 @@ jobs: if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.capture_provenance }}" == "true" ]]; then operation="capture" fi - matrix="$(python3 labs/15-pitot/scripts/pitot_adapter_supervisor.py "$operation")" - echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + agent_matrix="$(python3 labs/15-pitot/scripts/pitot_adapter_supervisor.py "$operation")" + runtime_matrix="$(python3 labs/15-pitot/scripts/pitot_adapter_supervisor.py runtime-matrix)" + echo "agent_matrix=$agent_matrix" >> "$GITHUB_OUTPUT" + echo "runtime_matrix=$runtime_matrix" >> "$GITHUB_OUTPUT" - name: Upload supervised adapter inventory uses: actions/upload-artifact@v4 with: @@ -50,14 +55,81 @@ jobs: if-no-files-found: error retention-days: 14 - verify: + verify-agents: needs: inventory strategy: fail-fast: false - matrix: ${{ fromJSON(needs.inventory.outputs.matrix) }} + matrix: ${{ fromJSON(needs.inventory.outputs.agent_matrix) }} uses: ./.github/workflows/pitot-e2e-agent.yml with: agent: ${{ matrix.agent }} platform: ${{ matrix.platform }} runner: ${{ matrix.runner }} capture: ${{ matrix.capture == 'true' }} + + verify-runtime: + needs: inventory + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.inventory.outputs.runtime_matrix) }} + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: labs/15-pitot/pitot/go.mod + cache-dependency-path: labs/15-pitot/pitot/go.mod + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - name: Resolve Bash runtime + shell: bash + run: | + bash_path="$(command -v bash)" + if [[ "${{ runner.os }}" == "Windows" ]]; then + bash_path="$(cygpath -m "$bash_path")" + fi + echo "PITOT_BASH=$bash_path" >> "$GITHUB_ENV" + - name: Run explicit request runtime E2E + id: e2e + continue-on-error: true + timeout-minutes: 5 + shell: bash + env: + PITOT_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + PITOT_E2E_PLATFORM: ${{ matrix.platform }} + PITOT_E2E_EVIDENCE: ${{ runner.temp }}/pitot-e2e/runtime-evidence.json + run: >- + python labs/15-pitot/tests/run_e2e_report.py + --capability explicit_request + --platform "${{ matrix.platform }}" + --output "${{ runner.temp }}/pitot-e2e/runtime-result.json" + --evidence "${{ runner.temp }}/pitot-e2e/runtime-evidence.json" + -- "$PITOT_BASH" labs/15-pitot/tests/e2e_runtime_cli_test.sh + < /dev/null + - name: Upload structured runtime result + if: always() + uses: actions/upload-artifact@v4 + with: + name: pitot-e2e-runtime-${{ matrix.platform }} + path: ${{ runner.temp }}/pitot-e2e/runtime-result.json + if-no-files-found: error + retention-days: 14 + - name: Enforce runtime E2E result + if: always() && steps.e2e.outcome != 'success' + shell: bash + run: exit 1 + + report: + if: ${{ always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + needs: [inventory, verify-agents, verify-runtime] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Update sticky PR report + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: >- + python3 .github/scripts/pitot_e2e_report.py + --event "$GITHUB_EVENT_PATH" + --repository "$GITHUB_REPOSITORY" diff --git a/go.work.sum b/go.work.sum index 0e46a4809..488d913ee 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,5 +1,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/labs/15-pitot/adapter-verification.json b/labs/15-pitot/adapter-verification.json index 93174564d..86ac44bfe 100644 --- a/labs/15-pitot/adapter-verification.json +++ b/labs/15-pitot/adapter-verification.json @@ -1,20 +1,24 @@ { - "schema_version": 4, + "schema_version": 6, + "capabilities": [ + {"id": "hook_control", "matrix": "agent_platform"}, + {"id": "consumer_delivery", "matrix": "agent_platform"}, + {"id": "explicit_request", "matrix": "platform"} + ], "platforms": [ {"id": "ubuntu", "runner": "ubuntu-latest"}, {"id": "macos", "runner": "macos-latest"}, {"id": "windows", "runner": "windows-latest"} ], "agents": [ - {"id": "claude", "label": "Claude", "version": "2.1.217", "executable": "claude", "installer": {"kind": "npm", "package": "@anthropic-ai/claude-code"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, - {"id": "cline", "label": "Cline", "version": "3.0.46", "executable": "cline", "installer": {"kind": "npm", "package": "cline"}, "integration": "cline_bridge", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, - {"id": "cursor", "label": "Cursor", "version": "2026.07.20-8cc9c0b", "executable": "agent", "installer": {"kind": "cursor_release", "package": "https://downloads.cursor.com/lab/2026.07.20-8cc9c0b"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "wsl"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, - {"id": "codex", "label": "Codex", "version": "0.145.0", "executable": "codex", "installer": {"kind": "npm", "package": "@openai/codex"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, - {"id": "copilot", "label": "GitHub Copilot CLI", "version": "1.0.73", "executable": "copilot", "installer": {"kind": "npm", "package": "@github/copilot"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, - {"id": "gemini", "label": "Gemini", "version": "0.51.0", "executable": "gemini", "installer": {"kind": "npm", "package": "@google/gemini-cli"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, - {"id": "kimi", "label": "Kimi Code", "version": "0.29.0", "executable": "kimi", "installer": {"kind": "kimi_release", "package": "https://code.kimi.com/kimi-code"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, - {"id": "opencode", "label": "OpenCode", "version": "1.18.4", "executable": "opencode", "installer": {"kind": "npm", "package": "opencode-ai"}, "integration": "opencode_plugin", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, - {"id": "pi", "label": "Pi", "version": "0.81.1", "executable": "pi", "installer": {"kind": "npm", "package": "@earendil-works/pi-coding-agent"}, "integration": "pi_extension", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, - {"id": "qwen", "label": "Qwen Code", "version": "0.20.1", "executable": "qwen", "installer": {"kind": "npm", "package": "@qwen-code/qwen-code"}, "integration": "native_command_hook", "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"} + {"id": "claude", "label": "Claude", "version": "2.1.217", "executable": "claude", "installer": {"kind": "npm", "package": "@anthropic-ai/claude-code"}, "integration": "native_command_hook", "artifacts": [], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "cursor", "label": "Cursor", "version": "2026.07.20-8cc9c0b", "executable": "agent", "installer": {"kind": "cursor_release", "package": "https://downloads.cursor.com/lab/2026.07.20-8cc9c0b"}, "integration": "native_command_hook", "artifacts": ["integrations/cursor/beforeShellExecution"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "wsl"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "codex", "label": "Codex", "version": "0.145.0", "executable": "codex", "installer": {"kind": "npm", "package": "@openai/codex"}, "integration": "native_command_hook", "artifacts": ["integrations/codex/PreToolUse.ps1"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "copilot", "label": "GitHub Copilot CLI", "version": "1.0.73", "executable": "copilot", "installer": {"kind": "npm", "package": "@github/copilot"}, "integration": "native_command_hook", "artifacts": ["integrations/copilot/PreToolUse", "integrations/copilot/PreToolUse.ps1"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "gemini", "label": "Gemini", "version": "0.51.0", "executable": "gemini", "installer": {"kind": "npm", "package": "@google/gemini-cli"}, "integration": "native_command_hook", "artifacts": ["integrations/gemini/BeforeTool", "integrations/gemini/BeforeTool.ps1"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "kimi", "label": "Kimi Code", "version": "0.29.0", "executable": "kimi", "installer": {"kind": "kimi_release", "package": "https://code.kimi.com/kimi-code"}, "integration": "native_command_hook", "artifacts": [], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "opencode", "label": "OpenCode", "version": "1.18.4", "executable": "opencode", "installer": {"kind": "npm", "package": "opencode-ai"}, "integration": "opencode_plugin", "artifacts": ["integrations/opencode/pitot.ts"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "pi", "label": "Pi", "version": "0.81.1", "executable": "pi", "installer": {"kind": "npm", "package": "@earendil-works/pi-coding-agent"}, "integration": "pi_extension", "artifacts": ["integrations/pi/pitot.ts"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "qwen", "label": "Qwen Code", "version": "0.20.1", "executable": "qwen", "installer": {"kind": "npm", "package": "@qwen-code/qwen-code"}, "integration": "native_command_hook", "artifacts": ["integrations/qwen/PreToolUse", "integrations/qwen/PreToolUse.cjs"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"} ] } diff --git a/labs/15-pitot/integrations/cline/PreToolUse b/labs/15-pitot/integrations/cline/PreToolUse deleted file mode 100755 index 3d5c81012..000000000 --- a/labs/15-pitot/integrations/cline/PreToolUse +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -set -uo pipefail - -PITOT_COMMAND="${PITOT_BIN:-pitot}" -PAYLOAD=$(cat) -if ! TOOL=$(printf '%s' "$PAYLOAD" | python3 -c 'import json,sys; event=json.load(sys.stdin).get("preToolUse", {}); print(event.get("toolName") or event.get("tool", ""))' 2>/dev/null); then - printf '{"cancel":true,"errorMessage":"Pitot received a malformed Cline hook payload"}\n' - exit 0 -fi - -if [ -z "$TOOL" ]; then - printf '{"cancel":true,"errorMessage":"Pitot received a malformed Cline hook payload"}\n' - exit 0 -fi - -# Cline has no matcher at this boundary. Non-shell tools are outside Pitot's -# controllable partition and must pass through unchanged. -if [ "$TOOL" != "execute_command" ] && [ "$TOOL" != "run_commands" ]; then - printf '{"cancel":false}\n' - exit 0 -fi - -if printf '%s' "$PAYLOAD" | "$PITOT_COMMAND" hook cline >/dev/null; then - printf '{"cancel":false}\n' - exit 0 -fi - -printf '{"cancel":true,"errorMessage":"Pitot rejected the shell request"}\n' -exit 0 diff --git a/labs/15-pitot/integrations/cline/PreToolUse.ps1 b/labs/15-pitot/integrations/cline/PreToolUse.ps1 deleted file mode 100644 index 2539befee..000000000 --- a/labs/15-pitot/integrations/cline/PreToolUse.ps1 +++ /dev/null @@ -1,24 +0,0 @@ -$payload = [Console]::In.ReadToEnd() -$pitot = if ($env:PITOT_BIN) { $env:PITOT_BIN } else { "pitot" } -try { - $event = $payload | ConvertFrom-Json -ErrorAction Stop -} catch { - @{ cancel = $true; errorMessage = "Pitot received a malformed Cline hook payload" } | ConvertTo-Json -Compress - exit 0 -} -$tool = if ($event.preToolUse.toolName) { $event.preToolUse.toolName } else { $event.preToolUse.tool } -if (-not $tool) { - @{ cancel = $true; errorMessage = "Pitot received a malformed Cline hook payload" } | ConvertTo-Json -Compress - exit 0 -} -if ($tool -ne "execute_command" -and $tool -ne "run_commands") { - @{ cancel = $false } | ConvertTo-Json -Compress - exit 0 -} -$payload | & $pitot hook cline *> $null -if ($LASTEXITCODE -eq 0) { - @{ cancel = $false } | ConvertTo-Json -Compress -} else { - @{ cancel = $true; errorMessage = "Pitot rejected the shell request" } | ConvertTo-Json -Compress -} -exit 0 diff --git a/labs/15-pitot/integrations/codex/PreToolUse.ps1 b/labs/15-pitot/integrations/codex/PreToolUse.ps1 new file mode 100644 index 000000000..2d3f16a3d --- /dev/null +++ b/labs/15-pitot/integrations/codex/PreToolUse.ps1 @@ -0,0 +1,39 @@ +param( + [Parameter(Mandatory = $true)][string]$Pitot, + [Parameter(Mandatory = $true)][string]$RealBin, + [Parameter(Mandatory = $true)][string]$Receipt, + [Parameter(Mandatory = $true)][string]$Nonce, + [Parameter(Mandatory = $true)][string]$Runtime +) + +$payload = [Console]::In.ReadToEnd() +$OutputEncoding = [Console]::OutputEncoding = [Text.UTF8Encoding]::new($false) +$arguments = @("--real-bin", $RealBin, "--receipt", $Receipt, "--nonce", $Nonce, "hook", "codex", "--runtime", $Runtime) +$startInfo = New-Object System.Diagnostics.ProcessStartInfo +$startInfo.FileName = $Pitot +$startInfo.Arguments = (($arguments | ForEach-Object { '"' + $_.Replace('"', '\"') + '"' }) -join ' ') +$startInfo.UseShellExecute = $false +$startInfo.CreateNoWindow = $true +$startInfo.RedirectStandardInput = $true +$startInfo.StandardInputEncoding = [Text.UTF8Encoding]::new($false) +$startInfo.RedirectStandardOutput = $true +$startInfo.RedirectStandardError = $true +$process = New-Object System.Diagnostics.Process +$process.StartInfo = $startInfo +[void]$process.Start() +$payloadBytes = [Text.UTF8Encoding]::new($false).GetBytes($payload) +$stdin = $process.StandardInput.BaseStream +$stdin.Write($payloadBytes, 0, $payloadBytes.Length) +$stdin.Close() +$stdout = $process.StandardOutput.ReadToEnd() +$stderr = $process.StandardError.ReadToEnd() +$process.WaitForExit() +$pitotOutput = ($stdout + $stderr).Trim() +if ($process.ExitCode -eq 0) { + exit 0 +} + +if (-not $pitotOutput) { $pitotOutput = "Pitot rejected the shell request" } +if ($pitotOutput.Length -gt 1024) { $pitotOutput = $pitotOutput.Substring(0, 1024) } +@{ decision = "block"; reason = $pitotOutput } | ConvertTo-Json -Compress +exit 0 diff --git a/labs/15-pitot/integrations/copilot/PreToolUse b/labs/15-pitot/integrations/copilot/PreToolUse new file mode 100755 index 000000000..30af5f7db --- /dev/null +++ b/labs/15-pitot/integrations/copilot/PreToolUse @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -uo pipefail + +PITOT_COMMAND="${PITOT_BIN:-pitot}" +PAYLOAD=$(cat) +if PITOT_ERROR=$(printf '%s' "$PAYLOAD" | "$PITOT_COMMAND" hook copilot 2>&1 >/dev/null); then + printf '%s\n' '{"permissionDecision":"allow","permissionDecisionReason":"Pitot accepted the shell action"}' + exit 0 +fi + +python3 -c 'import json,sys; print(json.dumps({"permissionDecision":"deny","permissionDecisionReason":(sys.argv[1] or "Pitot rejected the shell request")[:1024]},separators=(",",":")))' "$PITOT_ERROR" +exit 0 diff --git a/labs/15-pitot/integrations/copilot/PreToolUse.ps1 b/labs/15-pitot/integrations/copilot/PreToolUse.ps1 new file mode 100644 index 000000000..1bc5e6704 --- /dev/null +++ b/labs/15-pitot/integrations/copilot/PreToolUse.ps1 @@ -0,0 +1,31 @@ +$payload = [Console]::In.ReadToEnd() +$OutputEncoding = [Console]::OutputEncoding = [Text.UTF8Encoding]::new($false) +$pitot = if ($env:PITOT_BIN) { $env:PITOT_BIN } else { "pitot" } +$startInfo = New-Object System.Diagnostics.ProcessStartInfo +$startInfo.FileName = $pitot +$startInfo.Arguments = '"hook" "copilot"' +$startInfo.UseShellExecute = $false +$startInfo.CreateNoWindow = $true +$startInfo.RedirectStandardInput = $true +$startInfo.StandardInputEncoding = [Text.UTF8Encoding]::new($false) +$startInfo.RedirectStandardOutput = $true +$startInfo.RedirectStandardError = $true +$process = New-Object System.Diagnostics.Process +$process.StartInfo = $startInfo +[void]$process.Start() +$payloadBytes = [Text.UTF8Encoding]::new($false).GetBytes($payload) +$stdin = $process.StandardInput.BaseStream +$stdin.Write($payloadBytes, 0, $payloadBytes.Length) +$stdin.Close() +$stdout = $process.StandardOutput.ReadToEnd() +$stderr = $process.StandardError.ReadToEnd() +$process.WaitForExit() +$pitotOutput = ($stdout + $stderr).Trim() +if ($process.ExitCode -eq 0) { + @{ permissionDecision = "allow"; permissionDecisionReason = "Pitot accepted the shell action" } | ConvertTo-Json -Compress +} else { + if (-not $pitotOutput) { $pitotOutput = "Pitot rejected the shell request" } + if ($pitotOutput.Length -gt 1024) { $pitotOutput = $pitotOutput.Substring(0, 1024) } + @{ permissionDecision = "deny"; permissionDecisionReason = $pitotOutput } | ConvertTo-Json -Compress +} +exit 0 diff --git a/labs/15-pitot/integrations/cursor/beforeShellExecution b/labs/15-pitot/integrations/cursor/beforeShellExecution new file mode 100755 index 000000000..70320efa9 --- /dev/null +++ b/labs/15-pitot/integrations/cursor/beforeShellExecution @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -uo pipefail + +PITOT_COMMAND="${1:-${PITOT_BIN:-pitot}}" +PITOT_ARGUMENTS=() +if [ "$#" -ge 5 ]; then + PITOT_ARGUMENTS=(--real-bin "$2" --receipt "$3" --nonce "$4") + RUNTIME_ARGUMENTS=(--runtime "$5") +else + RUNTIME_ARGUMENTS=() +fi +PAYLOAD=$(cat) +if PITOT_ERROR=$(printf '%s' "$PAYLOAD" | "$PITOT_COMMAND" "${PITOT_ARGUMENTS[@]}" hook cursor "${RUNTIME_ARGUMENTS[@]}" 2>&1 >/dev/null); then + printf '%s\n' '{"continue":true,"permission":"allow"}' + exit 0 +fi + +python3 -c 'import json,sys; reason=(sys.argv[1] or "Pitot rejected the shell request")[:1024]; print(json.dumps({"continue":True,"permission":"deny","user_message":reason,"agent_message":reason},separators=(",",":")))' "$PITOT_ERROR" +exit 0 diff --git a/labs/15-pitot/integrations/gemini/BeforeTool b/labs/15-pitot/integrations/gemini/BeforeTool new file mode 100755 index 000000000..3d9c6d7cc --- /dev/null +++ b/labs/15-pitot/integrations/gemini/BeforeTool @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -uo pipefail + +PITOT_COMMAND="${1:-${PITOT_BIN:-pitot}}" +PITOT_ARGUMENTS=() +if [ "$#" -ge 5 ]; then + PITOT_ARGUMENTS=(--real-bin "$2" --receipt "$3" --nonce "$4") + RUNTIME_ARGUMENTS=(--runtime "$5") +else + RUNTIME_ARGUMENTS=() +fi +PAYLOAD=$(cat) +if PITOT_ERROR=$(printf '%s' "$PAYLOAD" | "$PITOT_COMMAND" "${PITOT_ARGUMENTS[@]}" hook gemini "${RUNTIME_ARGUMENTS[@]}" 2>&1 >/dev/null); then + printf '%s\n' '{"decision":"allow"}' + exit 0 +fi + +python3 -c 'import json,sys; print(json.dumps({"decision":"deny","reason":(sys.argv[1] or "Pitot rejected the shell request")[:1024]},separators=(",",":")))' "$PITOT_ERROR" +exit 0 diff --git a/labs/15-pitot/integrations/gemini/BeforeTool.ps1 b/labs/15-pitot/integrations/gemini/BeforeTool.ps1 new file mode 100644 index 000000000..6110b1c8b --- /dev/null +++ b/labs/15-pitot/integrations/gemini/BeforeTool.ps1 @@ -0,0 +1,41 @@ +param( + [string]$Pitot = $(if ($env:PITOT_BIN) { $env:PITOT_BIN } else { "pitot" }), + [string]$RealBin = "", + [string]$Receipt = "", + [string]$Nonce = "", + [string]$Runtime = "" +) +$payload = [Console]::In.ReadToEnd() +$OutputEncoding = [Console]::OutputEncoding = [Text.UTF8Encoding]::new($false) +$arguments = @() +if ($RealBin) { $arguments += @("--real-bin", $RealBin, "--receipt", $Receipt, "--nonce", $Nonce) } +$arguments += @("hook", "gemini") +if ($Runtime) { $arguments += @("--runtime", $Runtime) } +$startInfo = New-Object System.Diagnostics.ProcessStartInfo +$startInfo.FileName = $Pitot +$startInfo.Arguments = (($arguments | ForEach-Object { '"' + $_.Replace('"', '\"') + '"' }) -join ' ') +$startInfo.UseShellExecute = $false +$startInfo.CreateNoWindow = $true +$startInfo.RedirectStandardInput = $true +$startInfo.StandardInputEncoding = [Text.UTF8Encoding]::new($false) +$startInfo.RedirectStandardOutput = $true +$startInfo.RedirectStandardError = $true +$process = New-Object System.Diagnostics.Process +$process.StartInfo = $startInfo +[void]$process.Start() +$payloadBytes = [Text.UTF8Encoding]::new($false).GetBytes($payload) +$stdin = $process.StandardInput.BaseStream +$stdin.Write($payloadBytes, 0, $payloadBytes.Length) +$stdin.Close() +$stdout = $process.StandardOutput.ReadToEnd() +$stderr = $process.StandardError.ReadToEnd() +$process.WaitForExit() +$pitotOutput = ($stdout + $stderr).Trim() +if ($process.ExitCode -eq 0) { + @{ decision = "allow" } | ConvertTo-Json -Compress +} else { + if (-not $pitotOutput) { $pitotOutput = "Pitot rejected the shell request" } + if ($pitotOutput.Length -gt 1024) { $pitotOutput = $pitotOutput.Substring(0, 1024) } + @{ decision = "deny"; reason = $pitotOutput } | ConvertTo-Json -Compress +} +exit 0 diff --git a/labs/15-pitot/integrations/qwen/PreToolUse b/labs/15-pitot/integrations/qwen/PreToolUse new file mode 100755 index 000000000..0d2161634 --- /dev/null +++ b/labs/15-pitot/integrations/qwen/PreToolUse @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -uo pipefail + +PITOT_COMMAND="${1:-${PITOT_BIN:-pitot}}" +PITOT_ARGUMENTS=() +if [ "$#" -ge 5 ]; then + PITOT_ARGUMENTS=(--real-bin "$2" --receipt "$3" --nonce "$4") + RUNTIME_ARGUMENTS=(--runtime "$5") +else + RUNTIME_ARGUMENTS=() +fi +PAYLOAD=$(cat) +if PITOT_ERROR=$(printf '%s' "$PAYLOAD" | "$PITOT_COMMAND" "${PITOT_ARGUMENTS[@]}" hook qwen "${RUNTIME_ARGUMENTS[@]}" 2>&1 >/dev/null); then + printf '%s\n' '{"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":"allow","permissionDecisionReason":"Pitot accepted the shell action"}}' + exit 0 +fi + +python3 -c 'import json,sys; print(json.dumps({"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":"deny","permissionDecisionReason":(sys.argv[1] or "Pitot rejected the shell request")[:1024]}},separators=(",",":")))' "$PITOT_ERROR" +exit 0 diff --git a/labs/15-pitot/integrations/qwen/PreToolUse.cjs b/labs/15-pitot/integrations/qwen/PreToolUse.cjs new file mode 100644 index 000000000..1644042cc --- /dev/null +++ b/labs/15-pitot/integrations/qwen/PreToolUse.cjs @@ -0,0 +1,27 @@ +#!/usr/bin/env node +"use strict"; + +const { spawnSync } = require("node:child_process"); +const fs = require("node:fs"); + +const supplied = process.argv.slice(2); +const pitot = supplied[0] || process.env.PITOT_BIN || "pitot"; +const pitotArgs = supplied.length >= 5 + ? ["--real-bin", supplied[1], "--receipt", supplied[2], "--nonce", supplied[3], "hook", "qwen", "--runtime", supplied[4]] + : ["hook", "qwen"]; +const payload = fs.readFileSync(0); +const result = spawnSync(pitot, pitotArgs, { input: payload, encoding: "utf8", windowsHide: true }); +const detail = `${result.stdout || ""}${result.stderr || ""}`.trim(); +const allowed = result.status === 0 && !result.error; +const reason = allowed + ? "Pitot accepted the shell action" + : (detail || result.error?.message || "Pitot rejected the shell request").slice(0, 1024); + +process.stdout.write(JSON.stringify({ + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: allowed ? "allow" : "deny", + permissionDecisionReason: reason, + }, +}) + "\n"); + diff --git a/labs/15-pitot/pitot-distribution/PUBLIC_SURFACE.md b/labs/15-pitot/pitot-distribution/PUBLIC_SURFACE.md index 7f2462306..f963cc565 100644 --- a/labs/15-pitot/pitot-distribution/PUBLIC_SURFACE.md +++ b/labs/15-pitot/pitot-distribution/PUBLIC_SURFACE.md @@ -19,6 +19,9 @@ fails. The byte-level sync to the public repo is performed downstream by | `labs/15-pitot/public-readme-preview/CONTRIBUTING.md` | `CONTRIBUTING.md` | | `labs/15-pitot/public-readme-preview/assets/**` | `assets/**` | | Required local host-hook harness files from `labs/15-pitot/tests/` | `tests/` | +| `labs/15-pitot/integrations/**` | `integrations/**` | +| `labs/15-pitot/pitot-distribution/sdk/**` | `sdk/**` | +| `labs/15-pitot/adapter-verification.json` | `adapter-verification.json` | Excluded from the surface: `testdata/` directories and local build artifacts. diff --git a/labs/15-pitot/pitot-distribution/UPSTREAM.json b/labs/15-pitot/pitot-distribution/UPSTREAM.json index 40f9d8da3..9b0284a21 100644 --- a/labs/15-pitot/pitot-distribution/UPSTREAM.json +++ b/labs/15-pitot/pitot-distribution/UPSTREAM.json @@ -1,40 +1,57 @@ { "files": { - "CONTRIBUTING.md": "02c89a5790f2943e5e2466b0d260210238cae306922f0f031adb5f7028b48066", - "README.md": "b561ffc8f5cb2e57a18e176cb61756e85c72accdfcb9bd7080330877c45c4d30", - "adapter-verification.json": "c92a410798ed72fdaa705c8e85ce574d164a9d7334cf9dd8e0bf6b1ab6c926c7", - "adapters/adapters.go": "10aec65403df482eaa45368cd14a87552789772de6f93118be7bb9e91968aedd", + "CONTRIBUTING.md": "23728d8a132d62b8adfb2e5c3eb9d9bfcf8a4d04543765b1e22ad8d55424af8f", + "README.md": "091e47c27d0ee66a533b9aa9f3046a1bc9b84c0db5021926c6ab573d465ca57a", + "adapter-verification.json": "f8ad4e206571650f698826a8b66d8c00822be425e8d2de8ae98d98239e575eb4", + "adapters/adapters.go": "1b46ba131fa3b2c93eed23526330275a3506451ba4bbd4f497e5378dfab2b6a8", "assets/pitot-boundary.png": "8a0ddb7d81831d94e14813f50ea4ca8670d77417f339ed2f91f0c653bf52f41d", "assets/pitot-boundary.svg": "0c3871d70c84748573f231842091deb38a6def2862403ca34e8cc4493b9c9ebf", "assets/pitot-hero.png": "a73532252b1e66c06273abbf5a4fe6261e98de3133b09e8d550edacfeeab92f8", "assets/pitot-hero.svg": "226206e254252e2e7111a49d650d12cf5b32ceedd9db2adf06d0598be44809d6", "assets/pitot-mark.svg": "cacd728b4d4da45000ccde15d905314f2d92eb47b5e40a5f86a9e24ad671a003", "assets/pitot-two-roles.png": "9093368b30b7a0b704358343e78df2f937af21f2263b50e8da21a2946306255a", - "assets/pitot-two-roles.svg": "05a740b169110b2d6865fe4ecc9bd9e20c8a6c0ee00ceaf948f2c811b9dce0c5", - "bridge/bridge.go": "79ac2e025e16782f3c283b43cbea5b9ba4f837446583864df8f57d6346cae816", - "bridge/bridge_test.go": "23a19b7580d4b1e826224ec8322208ccca44ddd9a97b150efe15cafecc53e47f", + "assets/pitot-two-roles.svg": "528edf57c3eddb4432e119b9aa80a47452d47eb428e0e77a6f8623f244334475", + "bridge/bridge.go": "5adfcd3f743cae46e4446a6e030d53464ada97de0261a8588fa2a9fcd62136b8", + "bridge/bridge_test.go": "6dcc6d05f2b39c25955fc0b2d21d3d148dd9d77600fb12799941f86bdb1acb61", "cmd/generate-schema/main.go": "6e9d0030290d99e36967433f96e38385a122974f899ad9421aac1ef7e50d8fcb", - "cmd/pitot/main.go": "aa921d65706ea77b2c04aeddf6a482886ecf25f2c119eb0922edb4278aa53740", - "cmd/pitot/main_test.go": "377406a4c1479b4017505540164795dd224cba64c77a5cfa0297a9517194d940", + "cmd/pitot/main.go": "c3820dab52c790d35742a145ce81bf014480bbbbe7fbd670d2fe6d6ffc5ca1a6", + "cmd/pitot/main_test.go": "544997295e0c4b75ef8f3d698b3de0883153f671b6b8f62057cc6e3452d6dc93", + "config/config.go": "e6666567d0c0cca41de69361e8f1243adda1ec0a54a9300b39a84d2290bff319", + "config/config_test.go": "87d3e5ddc4a3b43c736070de671d03e03ffe29cdd759771526ad27fd9bc0034c", "conformance/conformance.go": "43b692114f45c8b52958e34b35aee1cee339d8321c90f92ab4f5b963e79935bb", "conformance/conformance_test.go": "83ab0bcc15371265a954d177e4e97d81ad3ea734bbf736a29a54628ef64b52cd", - "conformance/fixtures/negative.jsonl": "c8eff4f4155a5e86e7127a8d89aa06c14ecba74a7230b5bf7134ab1b7c4ca5f7", - "conformance/fixtures/positive.jsonl": "354b784d7bf8bd7dbeae09822bc4f6cf374fc3cf9f5e2824484fd05d8e2bc567", + "conformance/fixtures/negative.jsonl": "503ea76988df595d96ebf695f991b8ea6c892be4a578522dff4ddb0d39b647e4", + "conformance/fixtures/positive.jsonl": "23010bb2306f90fec40dc870cd922089550dbdfc977f778561c81032f4912a4c", "doc.go": "a8abdafac969b1bf4372c8bb023aa51125dc073f03218f4ab9913dfc5ffa877d", - "e2e/e2e_coverage_test.go": "28a6c27408338fdc51cf1241e1bf42a7f0ea17d3fb1305fbcd15901c21e21de8", + "e2e/e2e_coverage_test.go": "6235bd1df7212e4e229be324ac50f592aef70dce8855519d02e6b843656ea109", "e2e/e2e_hook_test.go": "5e184dc8907b6e36daeab90bbbb1654fa5336312866412031805a13ba535d1f8", "examples/doc.go": "58f3f9eb7d272d7b6eecdb05f43e1613d5e3ef92d15d97c5440bd4b6990c26f9", "examples/local-approval/main.go": "51386af324cd7d3bb07fe3ace53503884b02714b96b83073342fde81ce3b83a5", "examples/token-meter/main.go": "4b1b9c1a43c3cf48b09dba6f607776caced9d2b5b562373496b31ed184582dd1", - "go.mod": "7e71b29887a2370c920a3ecad84460130ecc5ad4acba7f62300b5d2568f0ee13", - "go.sum": "2f73a6c3c672f4022f4a618578fac165172095ef590db5d14caa4552675f1980", - "integrations/cline/PreToolUse": "5bab5e8f580858cf7a9e0b80eef6066907aab78e170eba0b690bfea363e65c5c", - "integrations/cline/PreToolUse.ps1": "3d5f3df239275678ac9cd20a27eb7252ae75d1873abd4e7c32e975b55eea8597", + "go.mod": "5b5f818251bbfb9dc3c066b70c3c33512ed18486d5713ba60e7f3141e23139fe", + "go.sum": "6e1aefcb6cb7d180f4b9f3eec64b547945cbebf660764b2bc3818e69bb7c8827", + "integrations/codex/PreToolUse.ps1": "a032bdefcaa6bdabc240ea5a91b19caa9804e976697ae67cbbb9c3a407fa4450", + "integrations/copilot/PreToolUse": "1a487ca7dfaaa21cbc7db5cec0091fed8ce9337bc116b9a6dec0a109ac5802a5", + "integrations/copilot/PreToolUse.ps1": "da5d113b9fd9441e8646414c3a1eb4754b415f40db7d06cbce111b123cdb2c88", + "integrations/cursor/beforeShellExecution": "57e0bb29a8ec64876f390d619dc8591e21d277ea0a9ba81aa1ef732ce82485a6", + "integrations/gemini/BeforeTool": "1291377cf2acdc8d1d938484c584f881ee7bc2d8e3dabecc645a18addc637aad", + "integrations/gemini/BeforeTool.ps1": "8673ad9da68cf6459eb2443ac71a1b528e0c6e3d26ef20d850a45d733f181986", "integrations/opencode/pitot.ts": "8d2709b473c839a6012cfe8e2da66221e763884a9dc02dc5198da2d4aec30c40", "integrations/pi/pitot.ts": "ed2d60d5ab6e33e115cfa058e4f96095100e93a061567d0af31249aa756bab3e", + "integrations/qwen/PreToolUse": "95c358620f2f882bb8680e6aa9639f3b36a8567a3fd60cf14a6cdf3b3fbcf78b", + "integrations/qwen/PreToolUse.cjs": "e9bf00bbbee5c15f01ea203d0992b34f8754b75a9c9889eecb16baeb10fcce6c", + "internal/testrole/main.go": "6d657eb85d8ddaff0ae5a3da281aa7aa0a4bdda860d179e99508110033765787", + "internal/testrole/main_test.go": "c7fbc4905bcef7d662c9e162a4c32c8a86d5c1a97820e9f4ee48f643c4e47a38", "projection/projection.go": "4d3c823fd72a3ca5387dba3683838a1d7e455e9b18309acc839763a39b7bb35f", "protocol/framing.go": "d4409314b72e09cfd472ad9a21d4c7a223b4341b26f58f6062a7c899e3f87482", "protocol/framing_test.go": "9fcb13767fdcc7129d2c87bc2133a5800c69d6ac0166016dc266dca5fcfaa1c5", + "runtime/capabilities.go": "76c27bdd7ddfa11a1d639915ac2eb2d573b7ccdd687a5f548b3731b7c1e1828f", + "runtime/descriptor_unix.go": "df41b6867e9840933f186c93b6bc61861e7ea5252a365455686ea0414bfa0044", + "runtime/descriptor_windows.go": "2d9ffefe3af0154fa8042de6b67460d4e86dd3f4cdd9e986f180f7d0c535c9a5", + "runtime/runtime.go": "693189eadd040629b616614e25dcdebb954e9c3546c25d064d0b8c3a7d01394a", + "runtime/runtime_test.go": "afd78d122af20148bf30d0db873ff002544189df0dfec0f6167b8cf5cd0d42b1", + "runtime/transport.go": "83e2218fb28474e875dafa6943bc5b665acef0565aaf5955fa88b1b4fd21614e", + "runtime/transport_test.go": "9b69f590f1e258470adea249b3ac6d4a00f1001f10bb08dfa7b56c6e2d6709ae", "schema/schema.go": "fd5c3b76979c88aeed75fadb7e3c94abcad62e067d77595f021fb422a60f2211", "sdk/python/pitot/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "sdk/python/pitot/types.py": "c9a7221f1ad6627f152f26148155d3c74f249c52262c73a515251f469e8eb4ad", @@ -46,9 +63,8 @@ "sensor/decode_fuzz_test.go": "d27f2fbbc069eded26a73c9cd9bace98dd8a9e34949576790b81a08d130fbaf2", "sensor/sensor.go": "5c503d07ac33e7894d635f2d98bcd6d165d442d60c127ed6aeac80ec319086c5", "sensor/sensor_test.go": "4ddbdde3e486c9e189a2ed4174ad413df107d43dc98f5242da3667a24c7a5da1", - "tests/cursor_control_proxy.mjs": "68cc734dd7cfdc53c5e932ccd13c9df2f72ad7f8756ddcc0e99a298c333d7223", + "tests/cursor_control_proxy.mjs": "ab532aa56a9299f497f3ceeedb4b6a0beb26089b7c022ed774385dd149ea56e6", "tests/e2e_claude_cli_test.sh": "b28c4d1963e326b4b3f158a7cfc1b92771e768ac9665dd43e8a339013cc11568", - "tests/e2e_cline_cli_test.sh": "a9fe90f64e36e2724acbef95ad37a8d35b4cd93471d560dfae116ef8401a03ac", "tests/e2e_codex_cli_test.sh": "dbae5224c87410a5a5d67023d7e23406453bc5493d981af9347993b4e1f562f2", "tests/e2e_copilot_cli_test.sh": "61b1e44dcd598d2d33e7f04dec26bec74405a9e2456ecda46e94dc8d43ad4315", "tests/e2e_cursor_cli_test.sh": "af46f1a40a535ed1f345ee51cba89a57fa912d6b67bed7df9bae0943131fa542", @@ -57,16 +73,18 @@ "tests/e2e_opencode_cli_test.sh": "e74a781f36b8263f2dd0f9700f3eea7a443b289fb87b55af455ce64509e08546", "tests/e2e_pi_cli_test.sh": "369707019ce0ee9ccb91ac1b9a16d5b8ae357fc9277ccb3407bcd7bd05aa86c0", "tests/e2e_qwen_cli_test.sh": "8813610ff0ae253e59849f76a5dc6ecf0990c150ea0ede60ec833d36e6e4a68f", - "tests/e2e_unified_runner.sh": "3a85d0548625fef0dacace3023fce71064a9a3bbfd44628ccd7e16e57962eb1a", - "tests/endpoint-provenance.json": "02195b0ea6d531844f0c2b1b7291b3264c662529cb4eaa78c03db0e07c7651d7", - "tests/install_real_agent.py": "dcbc1b09e3475f5c679e7813ccc71f8701baa1a1fa1d8385230ed97e0411d71d", + "tests/e2e_runtime_cli_test.sh": "54f64c3ef21eb62aa08dd7cb11d288be805d14a248c5baf7c172babd0957b4c9", + "tests/e2e_unified_runner.sh": "b9dc001971ab0e27154c16455e75b6eddf82f7b528c3e7352a160571c254ff2c", + "tests/endpoint-provenance.json": "3b9c5a2acab964d49a0c534e5dba01258e7f56708920a6463cf521b0000a446e", + "tests/install_real_agent.py": "999cb3326468998f0474431afe6b5b5dbd21752f76e68600bc03adb0d49484b9", "tests/mock_anthropic_server.js": "ecebea62f9e93791a79f1ae3dd3c67b8fa42490e9805b23b662b877edfdb0f0e", - "tests/model_control_proxy.py": "f8599a7738aa10c8668daafb5c90cd2232b8ac26d759d1e21ae0c13cd9047284", - "tests/real_agent_driver.py": "c722ee9a9ca8e8fd3c3fd31e336ff970c69cea294c83d15af67043c5bc95ae26", - "tests/run_e2e_report.py": "23221c8757fbcb057645b9a6cfc9e8b16469f19ff95d07f46df6fb7a696a1b3a", - "tests/witness/main.go": "e7ac1c5b2bcd46d5ae9fe623540ee1ac5721e7191d8211a012b7c9a89c5ec932", + "tests/model_control_proxy.py": "cd4733ab20c16770fe3bf6ac477935f7dc285cd89be020848ae4798cb509e5f2", + "tests/real_agent_driver.py": "8c57b50823be6bbf58a39624ef5661a4270a04d82b5ee7909280c9efa10344fd", + "tests/run_e2e_report.py": "e75da4aac49b70832e6ea98f295d6ebd9ef909538614136665c74f085220b1a5", + "tests/runtime_capability_driver.py": "54a485fc4f16981f2542d0dbd903a12b80f31f071908a378607f26678aeea07f", + "tests/witness/main.go": "cd56bbd00aa44cc5baf6426c8461a8ebca4a8391518f6acfa2301ac36add7c5f", "windtunnel/doc.go": "44e0bcde632da73e1f8b98beade3a34ca8e0d0ea79cdfb91d131de290b164fc4", - "windtunnel/windtunnel_test.go": "5da9d754cf8940f8cf79b60240107ea304ab0f37bb8df55271db5e42eca3003a" + "windtunnel/windtunnel_test.go": "d34929ffdb1927b2ee27cc79640b333b0571659d231977b91c3a8898cf79bc42" }, "schema_version": 1 } diff --git a/labs/15-pitot/pitot-distribution/release-notes/2026-07-22-complete-runtime-capabilities.md b/labs/15-pitot/pitot-distribution/release-notes/2026-07-22-complete-runtime-capabilities.md new file mode 100644 index 000000000..de7ae907a --- /dev/null +++ b/labs/15-pitot/pitot-distribution/release-notes/2026-07-22-complete-runtime-capabilities.md @@ -0,0 +1,22 @@ +### Complete the supervised Pitot runtime + +Pitot now runs passive Consumers and synchronous Controllers behind one +authenticated local runtime. Agent hooks can deliver projected events and apply +correlated allow or deny decisions, while `pitot request KIND` uses the same +Controller bridge for explicit non-hook requests. + +The unified supervisor verifies `hook_control`, `consumer_delivery`, and +`explicit_request` as one shipped capability inventory. Real-agent evidence now +requires both allowed and denied actions plus Consumer and Controller receipts; +separate runtime cells verify explicit requests on Ubuntu, macOS, and Windows. + +The previously advertised tenth adapter is removed because its pinned headless +CLI terminates on a hook denial instead of returning the denied tool result to +the model. Pitot now claims only the nine adapters that can complete the +supervised causal loop. + +Cursor's binary-observed Connect proxy also preserves the outstanding control +phase across transient stream reconnections, so a transport reset cannot turn +successful response emission into a false or hanging verification result. Its +WSL hook bridge is pinned to LF and canonicalized at installation so Windows +checkout settings cannot corrupt the executable shebang. diff --git a/labs/15-pitot/pitot/adapters/adapters.go b/labs/15-pitot/pitot/adapters/adapters.go index 70fab4622..b31fc631a 100644 --- a/labs/15-pitot/pitot/adapters/adapters.go +++ b/labs/15-pitot/pitot/adapters/adapters.go @@ -12,7 +12,6 @@ package adapters import ( - "encoding/json" "errors" "fmt" "sort" @@ -33,7 +32,6 @@ const ( Copilot Host = "copilot" Qwen Host = "qwen" Pi Host = "pi" - Cline Host = "cline" ) // AdapterVersion is the semantic version stamped onto normalized events so @@ -98,45 +96,6 @@ var ( }, Partition: ControlPartition{Controllable: []string{"tool_call"}}, }, - Cline: { - MainEventName: "tool_call", - Parser: ParserConfig{ - CanonicalEvent: []byte(`{"hookName":"tool_call","preToolUse":{"toolName":"run_commands","parameters":{"commands":"[\"git status --short\"]"}}}`), - EventNameFor: func(raw RawHookEvent) string { return raw.HookName }, - CommandFor: func(raw RawHookEvent) (string, bool) { - tool := raw.PreToolUse.ToolName - if tool == "" { - tool = raw.PreToolUse.Tool - } - if tool == "execute_command" { - value, present := raw.PreToolUse.Parameters["command"].(string) - return value, present && value != "" - } - if tool != "run_commands" { - return "", false - } - encoded, present := raw.PreToolUse.Parameters["commands"].(string) - if !present || encoded == "" { - return "", false - } - var commands []any - if json.Unmarshal([]byte(encoded), &commands) != nil || len(commands) == 0 { - return "", false - } - switch command := commands[0].(type) { - case string: - return command, command != "" - case map[string]any: - value, ok := command["command"].(string) - return value, ok && value != "" - default: - return "", false - } - }, - ActionKinds: map[string]string{"tool_call": "shell", "PreToolUse": "shell"}, - }, - Partition: ControlPartition{Controllable: []string{"tool_call", "PreToolUse"}}, - }, Cursor: { MainEventName: "beforeShellExecution", Parser: ParserConfig{ @@ -397,13 +356,6 @@ type RawHookEvent struct { // ToolName / ToolInput are populated by PreToolUse-style hosts and Gemini. ToolName string `json:"tool_name"` ToolInput map[string]any `json:"tool_input"` - // HookName / PreToolUse preserve Cline's native nested hook shape. - HookName string `json:"hookName"` - PreToolUse struct { - Tool string `json:"tool"` - ToolName string `json:"toolName"` - Parameters map[string]any `json:"parameters"` - } `json:"preToolUse"` } // EventNameFor extracts the host-specific event discriminator. diff --git a/labs/15-pitot/pitot/bridge/bridge.go b/labs/15-pitot/pitot/bridge/bridge.go index d508fb211..1c2b7e6ed 100644 --- a/labs/15-pitot/pitot/bridge/bridge.go +++ b/labs/15-pitot/pitot/bridge/bridge.go @@ -13,6 +13,7 @@ import ( "errors" "fmt" "sort" + "sync" "github.com/operatorstack/pitot/schema" ) @@ -49,15 +50,19 @@ func (r Registration) validate() error { // Router holds at most one Controller registration per request kind. type Router struct { registrations map[string]Registration + resolved map[string]struct{} + mu sync.Mutex } // NewRouter returns an empty Router. func NewRouter() *Router { - return &Router{registrations: map[string]Registration{}} + return &Router{registrations: map[string]Registration{}, resolved: map[string]struct{}{}} } // Register records reg, enforcing the exactly-one-Controller-per-kind rule. func (r *Router) Register(reg Registration) error { + r.mu.Lock() + defer r.mu.Unlock() if err := reg.validate(); err != nil { return err } @@ -70,6 +75,8 @@ func (r *Router) Register(reg Registration) error { // Kinds returns the registered request kinds in stable order, for diagnostics. func (r *Router) Kinds() []string { + r.mu.Lock() + defer r.mu.Unlock() kinds := make([]string, 0, len(r.registrations)) for kind := range r.registrations { kinds = append(kinds, kind) @@ -80,6 +87,8 @@ func (r *Router) Kinds() []string { // Registration returns the registration for kind, if any. func (r *Router) Registration(kind string) (Registration, bool) { + r.mu.Lock() + defer r.mu.Unlock() reg, ok := r.registrations[kind] return reg, ok } @@ -98,10 +107,19 @@ var ( // candidate means the Controller was unavailable; a candidate that fails // correlation is rejected and the declared default applies. func (r *Router) Resolve(req schema.ControlRequested, candidate *schema.ControlResponse) (schema.ControlResponse, error) { + r.mu.Lock() + defer r.mu.Unlock() + if err := validateRequest(req); err != nil { + return schema.ControlResponse{}, err + } + if _, resolved := r.resolved[req.ActionID]; resolved { + return schema.ControlResponse{}, ErrDuplicate + } reg, ok := r.registrations[req.Kind] if !ok { return schema.ControlResponse{}, ErrNoController } + r.resolved[req.ActionID] = struct{}{} if candidate == nil { return r.defaultResponse(reg, req, reg.OnUnavailable), nil } @@ -123,13 +141,35 @@ func (r *Router) Resolve(req schema.ControlRequested, candidate *schema.ControlR // TimeoutResponse returns the declared default resolution when the deadline // elapsed before the Controller answered. func (r *Router) TimeoutResponse(req schema.ControlRequested) (schema.ControlResponse, error) { + r.mu.Lock() + defer r.mu.Unlock() + if err := validateRequest(req); err != nil { + return schema.ControlResponse{}, err + } + if _, resolved := r.resolved[req.ActionID]; resolved { + return schema.ControlResponse{}, ErrDuplicate + } reg, ok := r.registrations[req.Kind] if !ok { return schema.ControlResponse{}, ErrNoController } + r.resolved[req.ActionID] = struct{}{} return r.defaultResponse(reg, req, reg.OnTimeout), nil } +func validateRequest(req schema.ControlRequested) error { + if req.PitotVersion != schema.Version { + return fmt.Errorf("pitot: request has unsupported version %q", req.PitotVersion) + } + if req.Type != schema.TypeControlRequested { + return fmt.Errorf("pitot: request has unexpected type %q", req.Type) + } + if req.Kind == "" || req.ActionID == "" { + return errors.New("pitot: request requires kind and action id") + } + return nil +} + func (r *Router) defaultResponse(reg Registration, req schema.ControlRequested, outcome string) schema.ControlResponse { return schema.ControlResponse{ PitotVersion: schema.Version, diff --git a/labs/15-pitot/pitot/bridge/bridge_test.go b/labs/15-pitot/pitot/bridge/bridge_test.go index 69732793d..7e5bc4ccf 100644 --- a/labs/15-pitot/pitot/bridge/bridge_test.go +++ b/labs/15-pitot/pitot/bridge/bridge_test.go @@ -119,3 +119,29 @@ func TestResolveNoControllerFails(t *testing.T) { t.Fatalf("err = %v, want ErrNoController", err) } } + +func TestResolveRejectsDuplicateTerminalResponse(t *testing.T) { + r := NewRouter() + _ = r.Register(reg()) + request := request() + response := &schema.ControlResponse{ + PitotVersion: schema.Version, Type: schema.TypeControlResponse, + ControllerID: "local-approval", ActionID: request.ActionID, Outcome: schema.OutcomeAllow, + } + if _, err := r.Resolve(request, response); err != nil { + t.Fatal(err) + } + if _, err := r.Resolve(request, response); err != ErrDuplicate { + t.Fatalf("err = %v, want ErrDuplicate", err) + } +} + +func TestResolveRejectsMalformedRequest(t *testing.T) { + r := NewRouter() + _ = r.Register(reg()) + malformed := request() + malformed.PitotVersion = "future" + if _, err := r.Resolve(malformed, nil); err == nil { + t.Fatal("expected malformed request to fail") + } +} diff --git a/labs/15-pitot/pitot/cmd/pitot/main.go b/labs/15-pitot/pitot/cmd/pitot/main.go index 3320dbd60..94ab54ab4 100644 --- a/labs/15-pitot/pitot/cmd/pitot/main.go +++ b/labs/15-pitot/pitot/cmd/pitot/main.go @@ -1,27 +1,30 @@ -// Command pitot is the reference Go executable for the Pitot sensor and control -// transport. In v1 Pitot supervises local processes: it starts declared -// Consumers and Controllers itself, projects content before bytes enter a child -// pipe, and exposes no unauthenticated local socket. -// -// This skeleton implements the `doctor` boundary inspection and the `run` -// configuration boundary; supervised delivery lands with the first buildable -// release. +// Command pitot is the reference executable for Pitot's sensor and control transport. package main import ( + "context" "encoding/json" + "errors" "fmt" "io" "os" + "os/signal" + "syscall" "github.com/operatorstack/pitot/adapters" + "github.com/operatorstack/pitot/config" + "github.com/operatorstack/pitot/runtime" "github.com/operatorstack/pitot/schema" "github.com/operatorstack/pitot/sensor" ) +var errBlocked = errors.New("pitot: block") + func main() { - if err := run(os.Args[1:], os.Stdout, os.Stderr); err != nil { - if err.Error() == "pitot: block" { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := runWithIO(ctx, os.Args[1:], os.Stdin, os.Stdout, os.Stderr); err != nil { + if errors.Is(err, errBlocked) { os.Exit(2) } fmt.Fprintln(os.Stderr, err) @@ -30,6 +33,10 @@ func main() { } func run(args []string, stdout, stderr io.Writer) error { + return runWithIO(context.Background(), args, os.Stdin, stdout, stderr) +} + +func runWithIO(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) error { if len(args) == 0 { return usageError() } @@ -37,9 +44,11 @@ func run(args []string, stdout, stderr io.Writer) error { case "doctor": return doctor(stdout) case "run": - return runSupervisor(args[1:], stdout) + return runRuntime(ctx, args[1:], stdout, stderr) case "hook": - return runHook(args[1:], stdout, stderr) + return runHook(ctx, args[1:], stdin, stdout, stderr) + case "request": + return runRequest(ctx, args[1:], stdout) case "-h", "--help", "help": fmt.Fprint(stdout, usage()) return nil @@ -49,47 +58,145 @@ func run(args []string, stdout, stderr io.Writer) error { } } -// runHook implements the direct host CLI hook interface. It reads the raw hook payload -// from stdin, normalizes it, and exits with 0 (allow) or 2 (block/deny). -func runHook(args []string, stdout, stderr io.Writer) error { +// runHook preserves observation-only behavior unless an authenticated runtime is selected. +func runHook(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) error { if len(args) == 0 { - return fmt.Errorf("pitot: hook requires a host identifier (claude, cline, codex, copilot, cursor, gemini, kimi, opencode, pi, qwen)") + return fmt.Errorf("pitot: hook requires a host identifier (claude, codex, copilot, cursor, gemini, kimi, opencode, pi, qwen)") } host := adapters.Host(args[0]) if !adapters.IsSupported(host) { return fmt.Errorf("pitot: unsupported hook host %q", host) } - - // Read raw payload from stdin - payload, err := io.ReadAll(os.Stdin) + runtimePath, err := parseRuntimeFlag(args[1:], false) + if err != nil { + return err + } + payload, err := io.ReadAll(io.LimitReader(stdin, 4<<20+1)) if err != nil { return fmt.Errorf("pitot: read stdin: %w", err) } - - // In this reference hook implementation, we decode with "full" projection + if len(payload) > 4<<20 { + fmt.Fprintln(stderr, "pitot: hook payload exceeds 4 MiB") + return errBlocked + } event, err := sensor.Decode(host, payload, "full") if err != nil { - // Serialize content-safe boundary fault to stderr - if fault, ok := sensor.AsFault(err, "act_hook"); ok { + actionID, idErr := runtime.NewActionID() + if idErr != nil { + actionID = "act_hook" + } + if fault, ok := sensor.AsFault(err, actionID); ok { _ = json.NewEncoder(stderr).Encode(fault) } else { fmt.Fprintln(stderr, err.Error()) } - // Return specific error to trigger exit code 2 in main() - return fmt.Errorf("pitot: block") + return errBlocked } + actionID, err := runtime.NewActionID() + if err != nil { + return err + } + event.Action.ID = actionID + if err := json.NewEncoder(stdout).Encode(event); err != nil { + return fmt.Errorf("pitot: emit normalized event: %w", err) + } + if runtimePath == "" { + return nil + } + client, err := runtime.OpenClient(runtimePath) + if err != nil { + fmt.Fprintln(stderr, err) + return errBlocked + } + response, err := client.DeliverEvent(ctx, event) + if err != nil { + fmt.Fprintln(stderr, err) + return errBlocked + } + if response == nil || response.Outcome == schema.OutcomeAllow { + return nil + } + if response.Outcome != schema.OutcomeDeny || response.ActionID != actionID { + fmt.Fprintln(stderr, "pitot: invalid controller resolution") + return errBlocked + } + if response.Message != "" { + fmt.Fprintln(stderr, response.Message) + } else { + fmt.Fprintln(stderr, "Pitot Controller denied the shell request") + } + return errBlocked +} - // Print the normalized event to stdout (useful for logging/consumers) - _ = json.NewEncoder(stdout).Encode(event) +func runRequest(ctx context.Context, args []string, stdout io.Writer) error { + if len(args) == 0 { + return errors.New("pitot: request requires a kind") + } + kind := args[0] + runtimePath := "" + data := json.RawMessage(`{}`) + for i := 1; i < len(args); i++ { + switch args[i] { + case "--runtime": + if i+1 >= len(args) { + return errors.New("pitot: --runtime requires a path") + } + runtimePath = args[i+1] + i++ + case "--data": + if i+1 >= len(args) { + return errors.New("pitot: --data requires JSON") + } + data = json.RawMessage(args[i+1]) + i++ + default: + return fmt.Errorf("pitot: unexpected request argument %q", args[i]) + } + } + if runtimePath == "" { + runtimePath = os.Getenv("PITOT_RUNTIME") + } + if runtimePath == "" { + return errors.New("pitot: request requires --runtime PATH or PITOT_RUNTIME") + } + if !json.Valid(data) { + return errors.New("pitot: --data must be valid JSON") + } + actionID, err := runtime.NewActionID() + if err != nil { + return err + } + client, err := runtime.OpenClient(runtimePath) + if err != nil { + return err + } + response, err := client.Request(ctx, schema.ControlRequested{ + PitotVersion: schema.Version, + Type: schema.TypeControlRequested, + Kind: kind, + ActionID: actionID, + Data: data, + }) + if err != nil { + return err + } + if err := json.NewEncoder(stdout).Encode(response); err != nil { + return err + } + if response.Outcome == schema.OutcomeDeny { + return errBlocked + } + if response.Outcome != schema.OutcomeAllow { + return errors.New("pitot: invalid controller outcome") + } return nil } -// doctor inspects the effective local boundary and proves the decoder against -// each host's canonical read-only probe, mirroring Boatstack's DiagnoseHook. func doctor(stdout io.Writer) error { fmt.Fprintf(stdout, "Pitot %s — local boundary\n", schema.Version) fmt.Fprintf(stdout, "adapter version: %s\n", adapters.AdapterVersion) fmt.Fprintln(stdout, "unauthenticated local socket: none") + fmt.Fprintln(stdout, "runtime capabilities: hook_control consumer_delivery explicit_request") fmt.Fprintln(stdout, "hosts:") for _, host := range adapters.Supported() { probe, err := adapters.CanonicalHookEvent(host) @@ -105,44 +212,78 @@ func doctor(stdout io.Writer) error { return nil } -// runSupervisor validates the configuration boundary. Actual supervised delivery -// is not enabled in this skeleton; it never opens a socket. -func runSupervisor(args []string, stdout io.Writer) error { - config := "" +func runRuntime(ctx context.Context, args []string, stdout, stderr io.Writer) error { + configPath := "" + runtimePath := "" for i := 0; i < len(args); i++ { switch args[i] { case "--config": if i+1 >= len(args) { - return fmt.Errorf("pitot: --config requires a path") + return errors.New("pitot: --config requires a path") + } + configPath = args[i+1] + i++ + case "--runtime": + if i+1 >= len(args) { + return errors.New("pitot: --runtime requires a path") } - config = args[i+1] + runtimePath = args[i+1] i++ default: return fmt.Errorf("pitot: unexpected argument %q", args[i]) } } - if config == "" { - return fmt.Errorf("pitot: run requires --config ") + if configPath == "" { + return errors.New("pitot: run requires --config PATH") } - if _, err := os.Stat(config); err != nil { - return fmt.Errorf("pitot: cannot read config %q: %w", config, err) + if runtimePath == "" { + runtimePath = os.Getenv("PITOT_RUNTIME") } - fmt.Fprintf(stdout, "Pitot %s — configuration boundary\n", schema.Version) - fmt.Fprintf(stdout, "config: %s\n", config) - fmt.Fprintln(stdout, "supervised delivery: not enabled in this build") - return nil + if runtimePath == "" { + return errors.New("pitot: run requires --runtime PATH or PITOT_RUNTIME") + } + loaded, err := config.Load(configPath) + if err != nil { + return err + } + manager, err := runtime.Start(ctx, loaded.Config, stderr) + if err != nil { + return err + } + defer manager.Close() + return runtime.NewServer(manager, loaded.SHA256, runtimePath, stdout, stderr).Serve(ctx) +} + +func parseRuntimeFlag(args []string, required bool) (string, error) { + runtimePath := "" + for i := 0; i < len(args); i++ { + if args[i] != "--runtime" { + return "", fmt.Errorf("pitot: unexpected hook argument %q", args[i]) + } + if i+1 >= len(args) { + return "", errors.New("pitot: --runtime requires a path") + } + runtimePath = args[i+1] + i++ + } + if runtimePath == "" { + runtimePath = os.Getenv("PITOT_RUNTIME") + } + if required && runtimePath == "" { + return "", errors.New("pitot: runtime is required") + } + return runtimePath, nil } func usage() string { return `pitot — the open sensor and control transport for coding-agent tooling usage: - pitot doctor inspect the effective local boundary - pitot run --config PATH start Pitot with repository-owned configuration - pitot hook HOST direct integration interface for host CLI hook payloads (reads stdin) + pitot doctor + pitot run --config PATH --runtime PATH + pitot hook HOST [--runtime PATH] + pitot request KIND [--data JSON] --runtime PATH ` } -func usageError() error { - return fmt.Errorf("%s", usage()) -} +func usageError() error { return errors.New(usage()) } diff --git a/labs/15-pitot/pitot/cmd/pitot/main_test.go b/labs/15-pitot/pitot/cmd/pitot/main_test.go index 53e705015..939e99559 100644 --- a/labs/15-pitot/pitot/cmd/pitot/main_test.go +++ b/labs/15-pitot/pitot/cmd/pitot/main_test.go @@ -2,19 +2,43 @@ package main import ( "bytes" + "context" + "errors" + "fmt" "os" + "os/exec" "path/filepath" + goruntime "runtime" "strings" + "sync" "testing" + "time" ) +type lockedBuffer struct { + mu sync.Mutex + b bytes.Buffer +} + +func (b *lockedBuffer) Write(value []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.b.Write(value) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.b.String() +} + func TestDoctorReportsBoundary(t *testing.T) { var stdout, stderr bytes.Buffer if err := run([]string{"doctor"}, &stdout, &stderr); err != nil { t.Fatalf("doctor: %v", err) } out := stdout.String() - for _, want := range []string{"local boundary", "claude", "cline", "codex", "copilot", "cursor", "gemini", "kimi", "opencode", "pi", "qwen", "decoder=PASS", "unauthenticated local socket: none"} { + for _, want := range []string{"local boundary", "claude", "codex", "copilot", "cursor", "gemini", "kimi", "opencode", "pi", "qwen", "decoder=PASS", "unauthenticated local socket: none", "hook_control consumer_delivery explicit_request"} { if !strings.Contains(out, want) { t.Errorf("doctor output missing %q\n%s", want, out) } @@ -28,18 +52,79 @@ func TestRunRequiresConfig(t *testing.T) { } } -func TestRunValidatesConfigPath(t *testing.T) { +func buildTestRole(t *testing.T) string { + t.Helper() + name := "pitot-testrole" + if goruntime.GOOS == "windows" { + name += ".exe" + } + path := filepath.Join(t.TempDir(), name) + command := exec.Command("go", "build", "-o", path, "../../internal/testrole") + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("build test role: %v\n%s", err, output) + } + return path +} + +func TestRuntimeBacksHookAndExplicitRequestCommands(t *testing.T) { + t.Setenv("PITOT_RUNTIME", "") dir := t.TempDir() config := filepath.Join(dir, ".pitot.yaml") - if err := os.WriteFile(config, []byte("consumers: []\n"), 0o644); err != nil { + runtimePath := filepath.Join(dir, "runtime.json") + helper := buildTestRole(t) + raw := fmt.Sprintf(`consumers: + - id: audit + command: [%q, "--role", "consumer", "--receipt", %q] + events: ["action.requested"] + projection: {content: omit} +controllers: + shell: + id: shell-policy + command: [%q, "--role", "controller", "--id", "shell-policy", "--nonce", "cli"] + deadline_ms: 2000 + on_timeout: deny + on_unavailable: deny + release.approval: + id: release-policy + command: [%q, "--role", "controller", "--id", "release-policy", "--nonce", "cli"] + deadline_ms: 2000 + on_timeout: deny + on_unavailable: deny +`, helper, filepath.Join(dir, "consumer.jsonl"), helper, helper) + if err := os.WriteFile(config, []byte(raw), 0o600); err != nil { t.Fatal(err) } - var stdout, stderr bytes.Buffer - if err := run([]string{"run", "--config", config}, &stdout, &stderr); err != nil { - t.Fatalf("run: %v", err) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var runtimeOut bytes.Buffer + var runtimeErr lockedBuffer + done := make(chan error, 1) + go func() { + done <- runWithIO(ctx, []string{"run", "--config", config, "--runtime", runtimePath}, strings.NewReader(""), &runtimeOut, &runtimeErr) + }() + for i := 0; i < 250; i++ { + if _, err := os.Stat(runtimePath); err == nil { + break + } + time.Sleep(20 * time.Millisecond) } - if !strings.Contains(stdout.String(), "configuration boundary") { - t.Errorf("unexpected output: %s", stdout.String()) + if _, err := os.Stat(runtimePath); err != nil { + t.Fatalf("runtime did not become ready: %v\n%s", err, runtimeErr.String()) + } + var requestOut bytes.Buffer + err := runWithIO(context.Background(), []string{"request", "release.approval", "--data", `{"phase":"PITOT_DENY"}`, "--runtime", runtimePath}, strings.NewReader(""), &requestOut, &bytes.Buffer{}) + if !errors.Is(err, errBlocked) || !strings.Contains(requestOut.String(), `"outcome":"deny"`) { + t.Fatalf("request err=%v output=%s", err, requestOut.String()) + } + payload := `{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"PITOT_DENY cli"}}` + var hookOut, hookErr bytes.Buffer + err = runWithIO(context.Background(), []string{"hook", "claude", "--runtime", runtimePath}, strings.NewReader(payload), &hookOut, &hookErr) + if !errors.Is(err, errBlocked) || !strings.Contains(hookErr.String(), "PITOT_CONTROLLER_DENY cli") || !strings.Contains(hookOut.String(), `"type":"action.requested"`) { + t.Fatalf("hook err=%v stdout=%s stderr=%s", err, hookOut.String(), hookErr.String()) + } + cancel() + if err := <-done; err != nil { + t.Fatal(err) } } @@ -51,6 +136,7 @@ func TestUnknownCommandFails(t *testing.T) { } func TestHookCommandSubprocessBehavior(t *testing.T) { + t.Setenv("PITOT_RUNTIME", "") // 1. Test successful hook execution (allow) t.Run("allow", func(t *testing.T) { rawPayload := `{"hook_event_name":"beforeShellExecution","command":"git status"}` diff --git a/labs/15-pitot/pitot/config/config.go b/labs/15-pitot/pitot/config/config.go new file mode 100644 index 000000000..be2e2804c --- /dev/null +++ b/labs/15-pitot/pitot/config/config.go @@ -0,0 +1,165 @@ +// Package config defines and validates Pitot's repository-owned runtime configuration. +package config + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "sort" + + "github.com/operatorstack/pitot/projection" + "github.com/operatorstack/pitot/schema" + "go.yaml.in/yaml/v4" +) + +// Config is the complete v1 process-delivery boundary. +type Config struct { + Consumers []ConsumerConfig `yaml:"consumers,omitempty"` + Controllers map[string]ControllerConfig `yaml:"controllers,omitempty"` +} + +// ConsumerConfig declares a passive JSON-Lines event sink. +type ConsumerConfig struct { + ID string `yaml:"id"` + Command []string `yaml:"command"` + Events []string `yaml:"events"` + Projection ProjectionConfig `yaml:"projection"` +} + +// ProjectionConfig controls what event content crosses a Consumer's pipe. +type ProjectionConfig struct { + Content projection.Mode `yaml:"content"` +} + +// ControllerConfig declares one synchronous decision process for a request kind. +type ControllerConfig struct { + ID string `yaml:"id"` + Command []string `yaml:"command"` + DeadlineMS int `yaml:"deadline_ms"` + OnTimeout string `yaml:"on_timeout"` + OnUnavailable string `yaml:"on_unavailable"` +} + +// Loaded preserves the validated config and the digest bound into its runtime descriptor. +type Loaded struct { + Config Config + SHA256 string +} + +// Load reads exactly one strict YAML document and validates its complete process surface. +func Load(path string) (Loaded, error) { + raw, err := os.ReadFile(path) + if err != nil { + return Loaded{}, fmt.Errorf("pitot: read config %q: %w", path, err) + } + dec := yaml.NewDecoder(bytes.NewReader(raw)) + dec.KnownFields(true) + var cfg Config + if err := dec.Decode(&cfg); err != nil { + return Loaded{}, fmt.Errorf("pitot: decode config %q: %w", path, err) + } + var trailing any + if err := dec.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return Loaded{}, fmt.Errorf("pitot: config %q must contain exactly one YAML document", path) + } + return Loaded{}, fmt.Errorf("pitot: decode trailing config %q: %w", path, err) + } + if err := cfg.Validate(); err != nil { + return Loaded{}, err + } + digest := sha256.Sum256(raw) + return Loaded{Config: cfg, SHA256: hex.EncodeToString(digest[:])}, nil +} + +// Validate enforces role separation and deterministic registration. +func (c Config) Validate() error { + if len(c.Consumers) == 0 && len(c.Controllers) == 0 { + return errors.New("pitot: config requires at least one consumer or controller") + } + consumerIDs := map[string]struct{}{} + for i, consumer := range c.Consumers { + if consumer.ID == "" { + return fmt.Errorf("pitot: consumer %d requires an id", i) + } + if _, exists := consumerIDs[consumer.ID]; exists { + return fmt.Errorf("pitot: duplicate consumer id %q", consumer.ID) + } + consumerIDs[consumer.ID] = struct{}{} + if err := validateCommand("consumer "+consumer.ID, consumer.Command); err != nil { + return err + } + if len(consumer.Events) == 0 { + return fmt.Errorf("pitot: consumer %q requires at least one event", consumer.ID) + } + seenEvents := map[string]struct{}{} + for _, event := range consumer.Events { + if event != schema.TypeActionRequested { + return fmt.Errorf("pitot: consumer %q has unsupported event %q", consumer.ID, event) + } + if _, exists := seenEvents[event]; exists { + return fmt.Errorf("pitot: consumer %q repeats event %q", consumer.ID, event) + } + seenEvents[event] = struct{}{} + } + if !consumer.Projection.Content.Valid() { + return fmt.Errorf("pitot: consumer %q has invalid content projection %q", consumer.ID, consumer.Projection.Content) + } + } + controllerIDs := map[string]string{} + for _, kind := range sortedControllerKinds(c.Controllers) { + controller := c.Controllers[kind] + if kind == "" { + return errors.New("pitot: controller request kind cannot be empty") + } + if controller.ID == "" { + return fmt.Errorf("pitot: controller for %q requires an id", kind) + } + if other, exists := controllerIDs[controller.ID]; exists { + return fmt.Errorf("pitot: controller id %q is registered for both %q and %q", controller.ID, other, kind) + } + controllerIDs[controller.ID] = kind + if err := validateCommand("controller "+controller.ID, controller.Command); err != nil { + return err + } + if controller.DeadlineMS <= 0 { + return fmt.Errorf("pitot: controller %q requires a positive deadline_ms", controller.ID) + } + if !validOutcome(controller.OnTimeout) { + return fmt.Errorf("pitot: controller %q has invalid on_timeout %q", controller.ID, controller.OnTimeout) + } + if !validOutcome(controller.OnUnavailable) { + return fmt.Errorf("pitot: controller %q has invalid on_unavailable %q", controller.ID, controller.OnUnavailable) + } + } + return nil +} + +func validateCommand(role string, command []string) error { + if len(command) == 0 || command[0] == "" { + return fmt.Errorf("pitot: %s requires a command", role) + } + for _, argument := range command { + if argument == "" { + return fmt.Errorf("pitot: %s command contains an empty argument", role) + } + } + return nil +} + +func validOutcome(value string) bool { + return value == schema.OutcomeAllow || value == schema.OutcomeDeny +} + +func sortedControllerKinds(values map[string]ControllerConfig) []string { + kinds := make([]string, 0, len(values)) + for kind := range values { + kinds = append(kinds, kind) + } + sort.Strings(kinds) + return kinds +} diff --git a/labs/15-pitot/pitot/config/config_test.go b/labs/15-pitot/pitot/config/config_test.go new file mode 100644 index 000000000..57691acce --- /dev/null +++ b/labs/15-pitot/pitot/config/config_test.go @@ -0,0 +1,58 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestLoadStrictCompleteConfig(t *testing.T) { + path := filepath.Join(t.TempDir(), ".pitot.yaml") + raw := `consumers: + - id: audit + command: ["audit"] + events: ["action.requested"] + projection: + content: sha256 +controllers: + shell: + id: policy + command: ["policy", "--jsonl"] + deadline_ms: 2000 + on_timeout: deny + on_unavailable: deny +` + if err := os.WriteFile(path, []byte(raw), 0o600); err != nil { + t.Fatal(err) + } + loaded, err := Load(path) + if err != nil { + t.Fatal(err) + } + if loaded.SHA256 == "" || len(loaded.Config.Consumers) != 1 || len(loaded.Config.Controllers) != 1 { + t.Fatalf("unexpected loaded config: %+v", loaded) + } +} + +func TestConfigRejectsUnknownAndAmbiguousRoles(t *testing.T) { + tests := []struct{ name, raw, want string }{ + {"unknown", "mystery: true\n", "field mystery not found"}, + {"empty", "consumers: []\ncontrollers: {}\n", "requires at least one"}, + {"duplicate consumer", "consumers:\n- {id: audit, command: [x], events: [action.requested], projection: {content: omit}}\n- {id: audit, command: [x], events: [action.requested], projection: {content: omit}}\n", "duplicate consumer"}, + {"duplicate controller id", "controllers:\n shell: {id: policy, command: [x], deadline_ms: 1, on_timeout: deny, on_unavailable: deny}\n release: {id: policy, command: [x], deadline_ms: 1, on_timeout: deny, on_unavailable: deny}\n", "registered for both"}, + {"unsupported event", "consumers:\n- {id: audit, command: [x], events: [model.usage], projection: {content: omit}}\n", "unsupported event"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), ".pitot.yaml") + if err := os.WriteFile(path, []byte(test.raw), 0o600); err != nil { + t.Fatal(err) + } + _, err := Load(path) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("err = %v, want substring %q", err, test.want) + } + }) + } +} diff --git a/labs/15-pitot/pitot/conformance/fixtures/negative.jsonl b/labs/15-pitot/pitot/conformance/fixtures/negative.jsonl index 1fdc37b45..3fa4d78da 100644 --- a/labs/15-pitot/pitot/conformance/fixtures/negative.jsonl +++ b/labs/15-pitot/pitot/conformance/fixtures/negative.jsonl @@ -5,6 +5,4 @@ {"name":"claude-null-tool-input","host":"claude","mode":"omit","input":{"hook_event_name":"PreToolUse","tool_name":"Bash"},"reason":"empty-command"} {"name":"mismatched-event-name","host":"claude","mode":"omit","input":{"hook_event_name":"PostToolUse","tool_name":"Bash","tool_input":{"command":"ls"}},"reason":"malformed-event"} {"name":"unsupported-host","host":"aider","mode":"omit","input":{"hook_event_name":"PreToolUse"},"reason":"unsupported-host"} -{"name":"cline-mismatched-event","host":"cline","mode":"omit","input":{"hookName":"PostToolUse","preToolUse":{"tool":"execute_command","parameters":{"command":"ls"}}},"reason":"malformed-event"} -{"name":"cline-non-shell-tool","host":"cline","mode":"omit","input":{"hookName":"PreToolUse","preToolUse":{"tool":"read_file","parameters":{"command":"ls"}}},"reason":"empty-command"} {"name":"qwen-non-shell-tool","host":"qwen","mode":"omit","input":{"hook_event_name":"PreToolUse","tool_name":"ReadFile","tool_input":{"command":"ls"}},"reason":"empty-command"} diff --git a/labs/15-pitot/pitot/conformance/fixtures/positive.jsonl b/labs/15-pitot/pitot/conformance/fixtures/positive.jsonl index 6bad025ed..562d9e57b 100644 --- a/labs/15-pitot/pitot/conformance/fixtures/positive.jsonl +++ b/labs/15-pitot/pitot/conformance/fixtures/positive.jsonl @@ -7,5 +7,3 @@ {"name":"qwen-pre-tool","host":"qwen","mode":"sha256","input":{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"git status --short"}},"expect_kind":"shell"} {"name":"qwen-native-shell-tool","host":"qwen","mode":"sha256","input":{"hook_event_name":"PreToolUse","tool_name":"run_shell_command","tool_input":{"command":"git status --short"}},"expect_kind":"shell"} {"name":"pi-tool-call","host":"pi","mode":"sha256","input":{"hook_event_name":"tool_call","tool_name":"bash","tool_input":{"command":"git status --short"}},"expect_kind":"shell"} -{"name":"cline-native-tool-call","host":"cline","mode":"sha256","input":{"hookName":"tool_call","preToolUse":{"toolName":"run_commands","parameters":{"commands":"[\"git status --short\"]"}}},"expect_kind":"shell"} -{"name":"cline-legacy-pre-tool","host":"cline","mode":"sha256","input":{"hookName":"PreToolUse","preToolUse":{"tool":"execute_command","parameters":{"command":"git status --short"}}},"expect_kind":"shell"} diff --git a/labs/15-pitot/pitot/e2e/e2e_coverage_test.go b/labs/15-pitot/pitot/e2e/e2e_coverage_test.go index 7f67b2068..649c4e816 100644 --- a/labs/15-pitot/pitot/e2e/e2e_coverage_test.go +++ b/labs/15-pitot/pitot/e2e/e2e_coverage_test.go @@ -24,4 +24,4 @@ func TestAllAdaptersHaveE2EScripts(t *testing.T) { } }) } -} \ No newline at end of file +} diff --git a/labs/15-pitot/pitot/go.mod b/labs/15-pitot/pitot/go.mod index e75d4b443..72c1e793d 100644 --- a/labs/15-pitot/pitot/go.mod +++ b/labs/15-pitot/pitot/go.mod @@ -2,10 +2,14 @@ module github.com/operatorstack/pitot go 1.26 +require ( + github.com/invopop/jsonschema v0.14.0 + go.yaml.in/yaml/v4 v4.0.0-rc.2 + golang.org/x/sys v0.42.0 +) + require ( github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect - github.com/invopop/jsonschema v0.14.0 // indirect github.com/pb33f/ordered-map/v2 v2.3.1 // indirect - go.yaml.in/yaml/v4 v4.0.0-rc.2 // indirect ) diff --git a/labs/15-pitot/pitot/go.sum b/labs/15-pitot/pitot/go.sum index afe512a42..c9dfa4b3a 100644 --- a/labs/15-pitot/pitot/go.sum +++ b/labs/15-pitot/pitot/go.sum @@ -2,9 +2,19 @@ github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPn github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/invopop/jsonschema v0.14.0 h1:MHQqLhvpNUZfw+hM3AZDYK7jxO8FZoQeQM77g8iyZjg= github.com/invopop/jsonschema v0.14.0/go.mod h1:ygm6C2EaVNMBDPpaPlnOA2pFAxBnxGjFlMZABxm9n2I= github.com/pb33f/ordered-map/v2 v2.3.1 h1:5319HDO0aw4DA4gzi+zv4FXU9UlSs3xGZ40wcP1nBjY= github.com/pb33f/ordered-map/v2 v2.3.1/go.mod h1:qxFQgd0PkVUtOMCkTapqotNgzRhMPL7VvaHKbd1HnmQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.yaml.in/yaml/v4 v4.0.0-rc.2 h1:/FrI8D64VSr4HtGIlUtlFMGsm7H7pWTbj6vOLVZcA6s= go.yaml.in/yaml/v4 v4.0.0-rc.2/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/labs/15-pitot/pitot/internal/testrole/main.go b/labs/15-pitot/pitot/internal/testrole/main.go new file mode 100644 index 000000000..3ae07cb38 --- /dev/null +++ b/labs/15-pitot/pitot/internal/testrole/main.go @@ -0,0 +1,115 @@ +// Command testrole is a language-neutral-process fixture for Pitot runtime tests. +package main + +import ( + "bufio" + "encoding/json" + "flag" + "fmt" + "os" + "strings" + "time" + + "github.com/operatorstack/pitot/schema" +) + +func main() { + role := flag.String("role", "", "controller or consumer") + id := flag.String("id", "test-controller", "controller identity") + receipt := flag.String("receipt", "", "append-only receipt path") + nonce := flag.String("nonce", "", "E2E nonce") + mode := flag.String("mode", "route", "route, timeout, malformed, mismatch, allow, or deny") + flag.Parse() + if *role == "canary" { + if err := canary(*receipt, flag.Args()); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + return + } + if *role == "consumer" { + copyLines(*receipt) + return + } + if *role != "controller" { + fmt.Fprintln(os.Stderr, "testrole: --role is required") + os.Exit(2) + } + controller(*id, *receipt, *nonce, *mode) +} + +func canary(receipt string, args []string) error { + if receipt == "" || len(args) != 2 { + return fmt.Errorf("testrole: canary requires --receipt, marker, and nonce") + } + appendLine(receipt, args[0]+" "+args[1]) + fmt.Printf("PITOT_CANARY_RESULT %s %s\n", args[0], args[1]) + return nil +} + +func copyLines(receipt string) { + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + appendLine(receipt, scanner.Text()) + } +} + +func controller(id, receipt, nonce, mode string) { + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + var request schema.ControlRequested + if err := json.Unmarshal(scanner.Bytes(), &request); err != nil { + continue + } + appendJSON(receipt, map[string]any{"receipt_type": "request", "value": request}) + switch mode { + case "timeout": + time.Sleep(10 * time.Second) + continue + case "malformed": + fmt.Println(`{"not":"a control response"}`) + continue + } + outcome := schema.OutcomeAllow + if mode == "deny" || (mode == "route" && strings.Contains(string(request.Data), "PITOT_DENY")) { + outcome = schema.OutcomeDeny + } + actionID := request.ActionID + if mode == "mismatch" { + actionID = "act_mismatch" + } + message := "" + if outcome == schema.OutcomeDeny { + message = "PITOT_CONTROLLER_DENY " + nonce + } + response := schema.ControlResponse{ + PitotVersion: schema.Version, + Type: schema.TypeControlResponse, + ControllerID: id, + ActionID: actionID, + Outcome: outcome, + Message: message, + } + appendJSON(receipt, map[string]any{"receipt_type": "response", "value": response}) + _ = json.NewEncoder(os.Stdout).Encode(response) + } +} + +func appendJSON(path string, value any) { + encoded, err := json.Marshal(value) + if err == nil { + appendLine(path, string(encoded)) + } +} + +func appendLine(path, line string) { + if path == "" { + return + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) + if err != nil { + return + } + defer file.Close() + _, _ = fmt.Fprintln(file, line) +} diff --git a/labs/15-pitot/pitot/internal/testrole/main_test.go b/labs/15-pitot/pitot/internal/testrole/main_test.go new file mode 100644 index 000000000..4d28e78be --- /dev/null +++ b/labs/15-pitot/pitot/internal/testrole/main_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestCanaryWritesNonceBoundReceipt(t *testing.T) { + path := filepath.Join(t.TempDir(), "canary.jsonl") + if err := canary(path, []string{"PITOT_ALLOW", "nonce"}); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(raw) != "PITOT_ALLOW nonce\n" { + t.Fatalf("receipt = %q", raw) + } +} + +func TestCanaryRejectsIncompleteInvocation(t *testing.T) { + if err := canary("", []string{"PITOT_ALLOW"}); err == nil { + t.Fatal("expected incomplete canary invocation to fail") + } +} diff --git a/labs/15-pitot/pitot/runtime/capabilities.go b/labs/15-pitot/pitot/runtime/capabilities.go new file mode 100644 index 000000000..c775e5057 --- /dev/null +++ b/labs/15-pitot/pitot/runtime/capabilities.go @@ -0,0 +1,15 @@ +package runtime + +// Capability identifies a shipped runtime behavior supervised by repository CI. +type Capability string + +const ( + CapabilityHookControl Capability = "hook_control" + CapabilityConsumerDelivery Capability = "consumer_delivery" + CapabilityExplicitRequest Capability = "explicit_request" +) + +// Capabilities returns the canonical ordered shipped runtime inventory. +func Capabilities() []Capability { + return []Capability{CapabilityHookControl, CapabilityConsumerDelivery, CapabilityExplicitRequest} +} diff --git a/labs/15-pitot/pitot/runtime/descriptor_unix.go b/labs/15-pitot/pitot/runtime/descriptor_unix.go new file mode 100644 index 000000000..a51a480fd --- /dev/null +++ b/labs/15-pitot/pitot/runtime/descriptor_unix.go @@ -0,0 +1,42 @@ +//go:build !windows + +package runtime + +import ( + "fmt" + "os" + "syscall" +) + +func writeSecureDescriptorFile(path string, contents []byte) error { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + if _, err = file.Write(contents); err == nil { + err = file.Sync() + } + if closeErr := file.Close(); err == nil { + err = closeErr + } + if err != nil { + _ = os.Remove(path) + return err + } + return validateDescriptorSecurity(path) +} + +func validateDescriptorSecurity(path string) error { + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("pitot: inspect runtime descriptor: %w", err) + } + if info.Mode().Perm()&0o077 != 0 { + return fmt.Errorf("pitot: runtime descriptor %q is accessible by another user", path) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if ok && int(stat.Uid) != os.Geteuid() { + return fmt.Errorf("pitot: runtime descriptor %q is not owned by the current user", path) + } + return nil +} diff --git a/labs/15-pitot/pitot/runtime/descriptor_windows.go b/labs/15-pitot/pitot/runtime/descriptor_windows.go new file mode 100644 index 000000000..2767bbe69 --- /dev/null +++ b/labs/15-pitot/pitot/runtime/descriptor_windows.go @@ -0,0 +1,116 @@ +//go:build windows + +package runtime + +import ( + "fmt" + "os" + "unsafe" + + "golang.org/x/sys/windows" +) + +func currentUserSID() (*windows.SID, error) { + token, err := windows.OpenCurrentProcessToken() + if err != nil { + return nil, err + } + defer token.Close() + user, err := token.GetTokenUser() + if err != nil { + return nil, err + } + return user.User.Sid, nil +} + +func ownerOnlySecurityDescriptor() (*windows.SECURITY_DESCRIPTOR, error) { + sid, err := currentUserSID() + if err != nil { + return nil, fmt.Errorf("pitot: identify runtime descriptor owner: %w", err) + } + sddl := fmt.Sprintf("O:%sD:P(A;;GA;;;%s)", sid.String(), sid.String()) + descriptor, err := windows.SecurityDescriptorFromString(sddl) + if err != nil { + return nil, fmt.Errorf("pitot: build owner-only runtime descriptor: %w", err) + } + return descriptor, nil +} + +func writeSecureDescriptorFile(path string, contents []byte) error { + descriptor, err := ownerOnlySecurityDescriptor() + if err != nil { + return err + } + name, err := windows.UTF16PtrFromString(path) + if err != nil { + return err + } + attributes := windows.SecurityAttributes{ + Length: uint32(unsafe.Sizeof(windows.SecurityAttributes{})), + SecurityDescriptor: descriptor, + } + handle, err := windows.CreateFile( + name, + windows.GENERIC_WRITE, + 0, + &attributes, + windows.CREATE_NEW, + windows.FILE_ATTRIBUTE_NORMAL, + 0, + ) + if err != nil { + return err + } + file := os.NewFile(uintptr(handle), path) + if file == nil { + windows.CloseHandle(handle) + return fmt.Errorf("pitot: open secure runtime descriptor %q", path) + } + if _, err = file.Write(contents); err == nil { + err = file.Sync() + } + if closeErr := file.Close(); err == nil { + err = closeErr + } + if err != nil { + _ = os.Remove(path) + return err + } + return validateDescriptorSecurity(path) +} + +func validateDescriptorSecurity(path string) error { + expected, err := currentUserSID() + if err != nil { + return err + } + descriptor, err := windows.GetNamedSecurityInfo( + path, + windows.SE_FILE_OBJECT, + windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION, + ) + if err != nil { + return fmt.Errorf("pitot: inspect runtime descriptor ACL: %w", err) + } + owner, _, err := descriptor.Owner() + if err != nil || owner == nil || !owner.Equals(expected) { + return fmt.Errorf("pitot: runtime descriptor %q is not owned by the current user", path) + } + dacl, _, err := descriptor.DACL() + if err != nil || dacl == nil || dacl.AceCount != 1 { + return fmt.Errorf("pitot: runtime descriptor %q does not have an owner-only ACL", path) + } + control, _, err := descriptor.Control() + if err != nil || control&windows.SE_DACL_PROTECTED == 0 { + return fmt.Errorf("pitot: runtime descriptor %q permits inherited ACL entries", path) + } + var ace *windows.ACCESS_ALLOWED_ACE + if err := windows.GetAce(dacl, 0, &ace); err != nil || ace == nil || ace.Header.AceType != windows.ACCESS_ALLOWED_ACE_TYPE { + return fmt.Errorf("pitot: runtime descriptor %q has an invalid owner ACL", path) + } + trustee := (*windows.SID)(unsafe.Pointer(&ace.SidStart)) + if !trustee.Equals(expected) { + return fmt.Errorf("pitot: runtime descriptor %q grants access outside the current user", path) + } + return nil +} diff --git a/labs/15-pitot/pitot/runtime/runtime.go b/labs/15-pitot/pitot/runtime/runtime.go new file mode 100644 index 000000000..48290a293 --- /dev/null +++ b/labs/15-pitot/pitot/runtime/runtime.go @@ -0,0 +1,388 @@ +// Package runtime owns Pitot's local child-process delivery boundary. +// It transports observations and decisions but contains no policy engine. +package runtime + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os/exec" + "sort" + "sync" + "time" + + "github.com/operatorstack/pitot/bridge" + "github.com/operatorstack/pitot/config" + "github.com/operatorstack/pitot/projection" + "github.com/operatorstack/pitot/protocol" + "github.com/operatorstack/pitot/schema" +) + +const consumerQueueSize = 128 + +// Manager starts configured role processes and exposes their shared delivery path. +type Manager struct { + ctx context.Context + cancel context.CancelFunc + stderr io.Writer + controllers map[string]*controllerWorker + consumers []*consumerWorker +} + +// Start creates the complete configured process boundary. A child that cannot +// start remains unavailable so the declared default can resolve requests. +func Start(parent context.Context, cfg config.Config, stderr io.Writer) (*Manager, error) { + ctx, cancel := context.WithCancel(parent) + manager := &Manager{ + ctx: ctx, + cancel: cancel, + stderr: stderr, + controllers: map[string]*controllerWorker{}, + } + for _, kind := range sortedKinds(cfg.Controllers) { + declared := cfg.Controllers[kind] + registration := bridge.Registration{ + Kind: kind, + ControllerID: declared.ID, + DeadlineMS: declared.DeadlineMS, + OnTimeout: declared.OnTimeout, + OnUnavailable: declared.OnUnavailable, + } + router := bridge.NewRouter() + if err := router.Register(registration); err != nil { + cancel() + return nil, err + } + worker := newControllerWorker(ctx, router, registration, declared.Command, stderr) + manager.controllers[kind] = worker + if worker.startErr != nil { + fmt.Fprintf(stderr, "pitot: controller %q unavailable: %v\n", declared.ID, worker.startErr) + } + } + for _, declared := range cfg.Consumers { + worker := newConsumerWorker(ctx, declared, stderr) + manager.consumers = append(manager.consumers, worker) + if worker.startErr != nil { + fmt.Fprintf(stderr, "pitot: consumer %q unavailable: %v\n", declared.ID, worker.startErr) + } + } + return manager, nil +} + +// Close stops every child process owned by the runtime. +func (m *Manager) Close() { + m.cancel() + for _, consumer := range m.consumers { + consumer.close() + } + for _, controller := range m.controllers { + controller.close() + } +} + +// DeliverEvent fans an observation to Consumers and, when registered, resolves +// the synchronous action through its Controller. A nil response means the +// action kind is observation-only. +func (m *Manager) DeliverEvent(ctx context.Context, event schema.Event) (*schema.ControlResponse, error) { + for _, consumer := range m.consumers { + if err := consumer.offer(event); err != nil { + fmt.Fprintf(m.stderr, "pitot: consumer %q delivery fault: %v\n", consumer.id, err) + } + } + if event.Action == nil { + return nil, errors.New("pitot: controllable event requires an action") + } + worker, exists := m.controllers[event.Action.Kind] + if !exists { + return nil, nil + } + data, err := json.Marshal(event) + if err != nil { + return nil, fmt.Errorf("pitot: encode controller event: %w", err) + } + response, resolveErr := worker.resolve(ctx, schema.ControlRequested{ + PitotVersion: schema.Version, + Type: schema.TypeControlRequested, + Kind: event.Action.Kind, + ActionID: event.Action.ID, + Data: data, + }) + return &response, resolveErr +} + +// Request routes an explicit request through the same Controller worker used by hooks. +func (m *Manager) Request(ctx context.Context, request schema.ControlRequested) (schema.ControlResponse, error) { + worker, exists := m.controllers[request.Kind] + if !exists { + return schema.ControlResponse{}, bridge.ErrNoController + } + return worker.resolve(ctx, request) +} + +type controllerResult struct { + response schema.ControlResponse + err error +} + +type controllerWorker struct { + ctx context.Context + router *bridge.Router + registration bridge.Registration + stdin io.WriteCloser + responses chan controllerResult + done chan error + startErr error + mu sync.Mutex + resolved map[string]struct{} + resolvedFIFO []string + closeOnce sync.Once +} + +func newControllerWorker(ctx context.Context, router *bridge.Router, registration bridge.Registration, command []string, stderr io.Writer) *controllerWorker { + worker := &controllerWorker{ + ctx: ctx, + router: router, + registration: registration, + responses: make(chan controllerResult, 16), + done: make(chan error, 1), + resolved: map[string]struct{}{}, + } + cmd := exec.CommandContext(ctx, command[0], command[1:]...) + cmd.Stderr = stderr + stdout, err := cmd.StdoutPipe() + if err != nil { + worker.startErr = err + return worker + } + stdin, err := cmd.StdinPipe() + if err != nil { + worker.startErr = err + return worker + } + if err := cmd.Start(); err != nil { + worker.startErr = err + return worker + } + worker.stdin = stdin + go worker.readResponses(stdout) + go func() { worker.done <- cmd.Wait() }() + return worker +} + +func (w *controllerWorker) readResponses(reader io.Reader) { + scanner := protocol.NewReader(reader) + for scanner.Scan() { + var response schema.ControlResponse + if err := protocol.DecodeLine(scanner.Bytes(), &response); err != nil { + w.responses <- controllerResult{err: err} + return + } + select { + case w.responses <- controllerResult{response: response}: + case <-w.ctx.Done(): + return + } + } + if err := scanner.Err(); err != nil { + select { + case w.responses <- controllerResult{err: err}: + case <-w.ctx.Done(): + } + } +} + +func (w *controllerWorker) resolve(ctx context.Context, request schema.ControlRequested) (schema.ControlResponse, error) { + w.mu.Lock() + defer w.mu.Unlock() + if w.startErr != nil || w.stdin == nil { + response, err := w.router.Resolve(request, nil) + return response, errors.Join(w.startErr, err) + } + if err := protocol.WriteLine(w.stdin, request); err != nil { + w.startErr = err + response, resolveErr := w.router.Resolve(request, nil) + return response, errors.Join(err, resolveErr) + } + deadline := time.NewTimer(time.Duration(w.registration.DeadlineMS) * time.Millisecond) + defer deadline.Stop() + for { + select { + case result := <-w.responses: + if result.err != nil { + w.startErr = result.err + response, resolveErr := w.router.Resolve(request, nil) + w.remember(request.ActionID) + return response, errors.Join(result.err, resolveErr) + } + if result.response.ActionID != request.ActionID { + if _, stale := w.resolved[result.response.ActionID]; stale { + continue + } + response, err := w.router.Resolve(request, &result.response) + w.remember(request.ActionID) + return response, err + } + response, err := w.router.Resolve(request, &result.response) + w.remember(request.ActionID) + return response, err + case err := <-w.done: + if err == nil { + err = errors.New("controller exited") + } + w.startErr = err + response, resolveErr := w.router.Resolve(request, nil) + w.remember(request.ActionID) + return response, errors.Join(err, resolveErr) + case <-deadline.C: + response, err := w.router.TimeoutResponse(request) + w.remember(request.ActionID) + return response, err + case <-ctx.Done(): + response, err := w.router.TimeoutResponse(request) + w.remember(request.ActionID) + return response, errors.Join(ctx.Err(), err) + case <-w.ctx.Done(): + response, err := w.router.Resolve(request, nil) + w.remember(request.ActionID) + return response, errors.Join(w.ctx.Err(), err) + } + } +} + +func (w *controllerWorker) remember(actionID string) { + w.resolved[actionID] = struct{}{} + w.resolvedFIFO = append(w.resolvedFIFO, actionID) + if len(w.resolvedFIFO) > 1024 { + delete(w.resolved, w.resolvedFIFO[0]) + w.resolvedFIFO = w.resolvedFIFO[1:] + } +} + +func (w *controllerWorker) close() { + w.closeOnce.Do(func() { + if w.stdin != nil { + _ = w.stdin.Close() + } + }) +} + +type consumerWorker struct { + id string + events map[string]struct{} + projection projection.Mode + queue chan schema.Event + stdin io.WriteCloser + startErr error + dead chan struct{} + closeOnce sync.Once +} + +func newConsumerWorker(ctx context.Context, declared config.ConsumerConfig, stderr io.Writer) *consumerWorker { + worker := &consumerWorker{ + id: declared.ID, + events: map[string]struct{}{}, + projection: declared.Projection.Content, + queue: make(chan schema.Event, consumerQueueSize), + dead: make(chan struct{}), + } + for _, event := range declared.Events { + worker.events[event] = struct{}{} + } + cmd := exec.CommandContext(ctx, declared.Command[0], declared.Command[1:]...) + cmd.Stdout = stderr + cmd.Stderr = stderr + stdin, err := cmd.StdinPipe() + if err != nil { + worker.startErr = err + close(worker.dead) + return worker + } + if err := cmd.Start(); err != nil { + worker.startErr = err + close(worker.dead) + return worker + } + worker.stdin = stdin + go worker.deliver(ctx) + go func() { + _ = cmd.Wait() + worker.closeOnce.Do(func() { close(worker.dead) }) + }() + return worker +} + +func (w *consumerWorker) offer(event schema.Event) error { + if _, subscribed := w.events[event.Type]; !subscribed { + return nil + } + projected, err := projectEvent(event, w.projection) + if err != nil { + return err + } + select { + case <-w.dead: + return errors.New("consumer process is unavailable") + case w.queue <- projected: + return nil + default: + return errors.New("consumer queue is full") + } +} + +func (w *consumerWorker) deliver(ctx context.Context) { + for { + select { + case event := <-w.queue: + if err := protocol.WriteLine(w.stdin, event); err != nil { + w.closeOnce.Do(func() { close(w.dead) }) + return + } + case <-ctx.Done(): + return + case <-w.dead: + return + } + } +} + +func (w *consumerWorker) close() { + if w.stdin != nil { + _ = w.stdin.Close() + } +} + +func projectEvent(event schema.Event, mode projection.Mode) (schema.Event, error) { + copyEvent := event + if event.Content == nil { + return copyEvent, nil + } + if event.Content.Mode != schema.ContentFull || len(event.Content.Full) == 0 { + if mode == projection.Mode(event.Content.Mode) { + content := *event.Content + copyEvent.Content = &content + return copyEvent, nil + } + return schema.Event{}, errors.New("pitot: cannot widen an already projected event") + } + var raw string + if err := json.Unmarshal(event.Content.Full, &raw); err != nil { + return schema.Event{}, fmt.Errorf("pitot: decode full event content: %w", err) + } + content, err := projection.Apply(mode, []byte(raw)) + if err != nil { + return schema.Event{}, err + } + copyEvent.Content = &content + return copyEvent, nil +} + +func sortedKinds(values map[string]config.ControllerConfig) []string { + kinds := make([]string, 0, len(values)) + for kind := range values { + kinds = append(kinds, kind) + } + sort.Strings(kinds) + return kinds +} diff --git a/labs/15-pitot/pitot/runtime/runtime_test.go b/labs/15-pitot/pitot/runtime/runtime_test.go new file mode 100644 index 000000000..5f20bf8a1 --- /dev/null +++ b/labs/15-pitot/pitot/runtime/runtime_test.go @@ -0,0 +1,186 @@ +package runtime + +import ( + "context" + "encoding/json" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/operatorstack/pitot/config" + "github.com/operatorstack/pitot/projection" + "github.com/operatorstack/pitot/schema" +) + +func buildTestRole(t *testing.T) string { + t.Helper() + name := "pitot-testrole" + if runtime.GOOS == "windows" { + name += ".exe" + } + path := filepath.Join(t.TempDir(), name) + command := exec.Command("go", "build", "-o", path, "../internal/testrole") + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("build test role: %v\n%s", err, output) + } + return path +} + +func actionEvent(t *testing.T, id, command string) schema.Event { + t.Helper() + content, err := projection.Apply(projection.Full, []byte(command)) + if err != nil { + t.Fatal(err) + } + return schema.Event{ + PitotVersion: schema.Version, + Type: schema.TypeActionRequested, + Host: schema.Host{Name: "claude"}, + Action: &schema.Action{ID: id, Kind: "shell"}, + Content: &content, + Observation: schema.Observation{Source: schema.SourceHostHook, Fidelity: schema.FidelityDirect}, + } +} + +func waitFor(t *testing.T, path, contains string) string { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + raw, _ := os.ReadFile(path) + if strings.Contains(string(raw), contains) { + return string(raw) + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("%s did not contain %q", path, contains) + return "" +} + +func TestRuntimeSharesHookConsumerAndRequestBoundary(t *testing.T) { + helper := buildTestRole(t) + dir := t.TempDir() + consumerReceipt := filepath.Join(dir, "consumer.jsonl") + controllerReceipt := filepath.Join(dir, "controller.jsonl") + requestReceipt := filepath.Join(dir, "request.jsonl") + cfg := config.Config{ + Consumers: []config.ConsumerConfig{{ + ID: "audit", Command: []string{helper, "--role", "consumer", "--receipt", consumerReceipt}, + Events: []string{schema.TypeActionRequested}, Projection: config.ProjectionConfig{Content: projection.SHA256}, + }}, + Controllers: map[string]config.ControllerConfig{ + "shell": {ID: "shell-policy", Command: []string{helper, "--role", "controller", "--id", "shell-policy", "--receipt", controllerReceipt, "--nonce", "abc"}, DeadlineMS: 2000, OnTimeout: schema.OutcomeDeny, OnUnavailable: schema.OutcomeDeny}, + "release.approval": {ID: "release-policy", Command: []string{helper, "--role", "controller", "--id", "release-policy", "--receipt", requestReceipt, "--nonce", "abc"}, DeadlineMS: 2000, OnTimeout: schema.OutcomeDeny, OnUnavailable: schema.OutcomeDeny}, + }, + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + manager, err := Start(ctx, cfg, io.Discard) + if err != nil { + t.Fatal(err) + } + defer manager.Close() + runtimePath := filepath.Join(dir, "runtime.json") + serveErr := make(chan error, 1) + go func() { + serveErr <- NewServer(manager, strings.Repeat("a", 64), runtimePath, io.Discard, io.Discard).Serve(ctx) + }() + for i := 0; i < 250; i++ { + if _, err := os.Stat(runtimePath); err == nil { + break + } + time.Sleep(20 * time.Millisecond) + } + client, err := OpenClient(runtimePath) + if err != nil { + t.Fatal(err) + } + allow, err := client.DeliverEvent(ctx, actionEvent(t, "act_allow", "PITOT_ALLOW abc")) + if err != nil || allow == nil || allow.Outcome != schema.OutcomeAllow { + t.Fatalf("allow = %+v, err = %v", allow, err) + } + deny, err := client.DeliverEvent(ctx, actionEvent(t, "act_deny", "PITOT_DENY abc")) + if err != nil || deny == nil || deny.Outcome != schema.OutcomeDeny || !strings.Contains(deny.Message, "abc") { + t.Fatalf("deny = %+v, err = %v", deny, err) + } + consumer := waitFor(t, consumerReceipt, "act_deny") + if strings.Contains(consumer, "PITOT_ALLOW") || !strings.Contains(consumer, `"mode":"sha256"`) { + t.Fatalf("consumer projection leaked content or missed digest: %s", consumer) + } + response, err := client.Request(ctx, schema.ControlRequested{PitotVersion: schema.Version, Type: schema.TypeControlRequested, Kind: "release.approval", ActionID: "act_request", Data: json.RawMessage(`{"release":"PITOT_DENY abc"}`)}) + if err != nil || response.Outcome != schema.OutcomeDeny || response.ControllerID != "release-policy" { + t.Fatalf("request response = %+v, err = %v", response, err) + } + waitFor(t, requestReceipt, "act_request") + cancel() + if err := <-serveErr; err != nil { + t.Fatal(err) + } + if _, err := os.Stat(runtimePath); !os.IsNotExist(err) { + t.Fatalf("runtime descriptor was not removed: %v", err) + } +} + +func TestUnavailableAndTimeoutUseDeclaredDefaults(t *testing.T) { + helper := buildTestRole(t) + tests := []struct { + name string + command []string + }{ + {"unavailable", []string{filepath.Join(t.TempDir(), "missing-controller")}}, + {"timeout", []string{helper, "--role", "controller", "--mode", "timeout"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + manager, err := Start(ctx, config.Config{Controllers: map[string]config.ControllerConfig{ + "shell": {ID: "policy", Command: test.command, DeadlineMS: 30, OnTimeout: schema.OutcomeDeny, OnUnavailable: schema.OutcomeDeny}, + }}, io.Discard) + if err != nil { + t.Fatal(err) + } + defer manager.Close() + response, _ := manager.DeliverEvent(ctx, actionEvent(t, "act_default", "echo ok")) + if response == nil || response.Outcome != schema.OutcomeDeny { + t.Fatalf("response = %+v", response) + } + }) + } +} + +func TestProjectEventModesAndCannotWiden(t *testing.T) { + event := actionEvent(t, "act_projection", "secret") + for _, mode := range []projection.Mode{projection.Full, projection.SHA256, projection.Omit} { + projected, err := projectEvent(event, mode) + if err != nil || projected.Content.Mode != string(mode) { + t.Fatalf("mode %s: %+v, %v", mode, projected.Content, err) + } + } + hashed, _ := projectEvent(event, projection.SHA256) + if _, err := projectEvent(hashed, projection.Full); err == nil { + t.Fatal("expected widening a hash projection to fail") + } +} + +func TestConsumerFilterAndBoundedQueue(t *testing.T) { + worker := &consumerWorker{ + id: "audit", events: map[string]struct{}{schema.TypeActionRequested: {}}, + projection: projection.Omit, queue: make(chan schema.Event, 1), dead: make(chan struct{}), + } + ignored := actionEvent(t, "act_ignored", "secret") + ignored.Type = "other.event" + if err := worker.offer(ignored); err != nil || len(worker.queue) != 0 { + t.Fatalf("filtered event err=%v queue=%d", err, len(worker.queue)) + } + if err := worker.offer(actionEvent(t, "act_one", "secret")); err != nil { + t.Fatal(err) + } + if err := worker.offer(actionEvent(t, "act_two", "secret")); err == nil || !strings.Contains(err.Error(), "queue is full") { + t.Fatalf("full queue error = %v", err) + } +} diff --git a/labs/15-pitot/pitot/runtime/transport.go b/labs/15-pitot/pitot/runtime/transport.go new file mode 100644 index 000000000..97e436319 --- /dev/null +++ b/labs/15-pitot/pitot/runtime/transport.go @@ -0,0 +1,388 @@ +package runtime + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/operatorstack/pitot/bridge" + "github.com/operatorstack/pitot/schema" +) + +const ( + descriptorVersion = 1 + maxRequestBytes = 4 << 20 +) + +// Descriptor is the owner-only capability used by local Pitot clients. +type Descriptor struct { + SchemaVersion int `json:"schema_version"` + InstanceID string `json:"instance_id"` + PID int `json:"pid"` + Endpoint string `json:"endpoint"` + Token string `json:"token"` + ConfigSHA256 string `json:"config_sha256"` +} + +// Server exposes one authenticated loopback ingress for hooks and explicit requests. +type Server struct { + manager *Manager + configSHA string + runtimePath string + stdout io.Writer + stderr io.Writer +} + +// NewServer builds a local transport around manager. +func NewServer(manager *Manager, configSHA, runtimePath string, stdout, stderr io.Writer) *Server { + return &Server{manager: manager, configSHA: configSHA, runtimePath: runtimePath, stdout: stdout, stderr: stderr} +} + +// Serve publishes the runtime descriptor only after the authenticated endpoint is ready. +func (s *Server) Serve(ctx context.Context) error { + if err := prepareRuntimePath(s.runtimePath); err != nil { + return err + } + listener, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + return fmt.Errorf("pitot: bind loopback runtime: %w", err) + } + defer listener.Close() + instanceID, err := randomID(16) + if err != nil { + return err + } + token, err := randomID(32) + if err != nil { + return err + } + descriptor := Descriptor{ + SchemaVersion: descriptorVersion, + InstanceID: instanceID, + PID: os.Getpid(), + Endpoint: "http://" + listener.Addr().String(), + Token: token, + ConfigSHA256: s.configSHA, + } + if err := writeDescriptor(s.runtimePath, descriptor); err != nil { + return err + } + defer removeDescriptorIfOwned(s.runtimePath, instanceID) + + mux := http.NewServeMux() + mux.HandleFunc("GET /v1/health", s.authorize(token, func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{"schema_version": descriptorVersion, "instance_id": instanceID}) + })) + mux.HandleFunc("POST /v1/events", s.authorize(token, s.handleEvent)) + mux.HandleFunc("POST /v1/requests", s.authorize(token, s.handleRequest)) + server := &http.Server{ + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 30 * time.Second, + } + serveErr := make(chan error, 1) + go func() { + err := server.Serve(listener) + if errors.Is(err, http.ErrServerClosed) { + err = nil + } + serveErr <- err + }() + fmt.Fprintf(s.stdout, "Pitot %s — runtime ready\n", schema.Version) + fmt.Fprintf(s.stdout, "runtime: %s\n", s.runtimePath) + fmt.Fprintf(s.stdout, "config sha256: %s\n", s.configSHA) + select { + case err := <-serveErr: + return err + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := server.Shutdown(shutdownCtx); err != nil { + return fmt.Errorf("pitot: shut down runtime: %w", err) + } + return nil + } +} + +func (s *Server) authorize(token string, next http.HandlerFunc) http.HandlerFunc { + expected := sha256.Sum256([]byte("Bearer " + token)) + return func(w http.ResponseWriter, request *http.Request) { + provided := sha256.Sum256([]byte(request.Header.Get("Authorization"))) + if subtle.ConstantTimeCompare(provided[:], expected[:]) != 1 { + writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"}) + return + } + next(w, request) + } +} + +func (s *Server) handleEvent(w http.ResponseWriter, request *http.Request) { + var event schema.Event + if err := decodeRequest(w, request, &event); err != nil { + return + } + if event.PitotVersion != schema.Version || event.Type != schema.TypeActionRequested || event.Action == nil || event.Action.ID == "" || event.Action.Kind == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid action event"}) + return + } + response, err := s.manager.DeliverEvent(request.Context(), event) + if err != nil { + fmt.Fprintf(s.stderr, "pitot: action %s resolved with boundary fault: %v\n", event.Action.ID, err) + } + if response == nil { + w.WriteHeader(http.StatusNoContent) + return + } + writeJSON(w, http.StatusOK, response) +} + +func (s *Server) handleRequest(w http.ResponseWriter, request *http.Request) { + var control schema.ControlRequested + if err := decodeRequest(w, request, &control); err != nil { + return + } + if control.PitotVersion != schema.Version || control.Type != schema.TypeControlRequested || control.Kind == "" || control.ActionID == "" || !validJSON(control.Data) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid control request"}) + return + } + response, err := s.manager.Request(request.Context(), control) + if errors.Is(err, bridge.ErrNoController) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "no controller registered"}) + return + } + if err != nil { + fmt.Fprintf(s.stderr, "pitot: request %s resolved with boundary fault: %v\n", control.ActionID, err) + } + writeJSON(w, http.StatusOK, response) +} + +func decodeRequest(w http.ResponseWriter, request *http.Request, target any) error { + request.Body = http.MaxBytesReader(w, request.Body, maxRequestBytes) + decoder := json.NewDecoder(request.Body) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request"}) + return err + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "trailing request data"}) + return errors.New("trailing request data") + } + return nil +} + +func validJSON(value json.RawMessage) bool { + return len(value) == 0 || json.Valid(value) +} + +func writeJSON(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(value) +} + +// Client uses a runtime capability without exposing its token to child roles. +type Client struct { + descriptor Descriptor + http *http.Client +} + +// OpenClient validates and loads an owner-only runtime descriptor. +func OpenClient(path string) (*Client, error) { + descriptor, err := loadDescriptor(path) + if err != nil { + return nil, err + } + return &Client{descriptor: descriptor, http: &http.Client{Timeout: 35 * time.Second}}, nil +} + +// Health proves the descriptor still identifies its live runtime. +func (c *Client) Health(ctx context.Context) error { + var response struct { + SchemaVersion int `json:"schema_version"` + InstanceID string `json:"instance_id"` + } + status, err := c.call(ctx, http.MethodGet, "/v1/health", nil, &response) + if err != nil { + return err + } + if status != http.StatusOK || response.SchemaVersion != descriptorVersion || response.InstanceID != c.descriptor.InstanceID { + return errors.New("pitot: runtime descriptor identity mismatch") + } + return nil +} + +// DeliverEvent returns nil when no Controller is registered for the action kind. +func (c *Client) DeliverEvent(ctx context.Context, event schema.Event) (*schema.ControlResponse, error) { + var response schema.ControlResponse + status, err := c.call(ctx, http.MethodPost, "/v1/events", event, &response) + if err != nil { + return nil, err + } + if status == http.StatusNoContent { + return nil, nil + } + if status != http.StatusOK { + return nil, fmt.Errorf("pitot: runtime rejected event with HTTP %d", status) + } + return &response, nil +} + +// Request sends an explicit correlated control request. +func (c *Client) Request(ctx context.Context, request schema.ControlRequested) (schema.ControlResponse, error) { + var response schema.ControlResponse + status, err := c.call(ctx, http.MethodPost, "/v1/requests", request, &response) + if err != nil { + return schema.ControlResponse{}, err + } + if status == http.StatusNotFound { + return schema.ControlResponse{}, bridge.ErrNoController + } + if status != http.StatusOK { + return schema.ControlResponse{}, fmt.Errorf("pitot: runtime rejected request with HTTP %d", status) + } + return response, nil +} + +func (c *Client) call(ctx context.Context, method, path string, input, output any) (int, error) { + var body io.Reader + if input != nil { + encoded, err := json.Marshal(input) + if err != nil { + return 0, err + } + body = bytes.NewReader(encoded) + } + request, err := http.NewRequestWithContext(ctx, method, c.descriptor.Endpoint+path, body) + if err != nil { + return 0, err + } + request.Header.Set("Authorization", "Bearer "+c.descriptor.Token) + if input != nil { + request.Header.Set("Content-Type", "application/json") + } + response, err := c.http.Do(request) + if err != nil { + return 0, fmt.Errorf("pitot: contact runtime: %w", err) + } + defer response.Body.Close() + if response.StatusCode == http.StatusNoContent { + return response.StatusCode, nil + } + if response.StatusCode == http.StatusOK && output != nil { + decoder := json.NewDecoder(io.LimitReader(response.Body, maxRequestBytes)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(output); err != nil { + return response.StatusCode, fmt.Errorf("pitot: decode runtime response: %w", err) + } + } + return response.StatusCode, nil +} + +// NewActionID returns an unpredictable correlation identifier. +func NewActionID() (string, error) { + value, err := randomID(16) + if err != nil { + return "", err + } + return "act_" + value, nil +} + +func randomID(size int) (string, error) { + raw := make([]byte, size) + if _, err := rand.Read(raw); err != nil { + return "", fmt.Errorf("pitot: generate runtime identity: %w", err) + } + return hex.EncodeToString(raw), nil +} + +func writeDescriptor(path string, descriptor Descriptor) error { + if path == "" { + return errors.New("pitot: runtime descriptor path is required") + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return fmt.Errorf("pitot: create runtime directory: %w", err) + } + encoded, err := json.MarshalIndent(descriptor, "", " ") + if err != nil { + return err + } + suffix, err := randomID(8) + if err != nil { + return err + } + temporary := path + "." + suffix + ".tmp" + defer os.Remove(temporary) + if err := writeSecureDescriptorFile(temporary, append(encoded, '\n')); err != nil { + return fmt.Errorf("pitot: write runtime descriptor: %w", err) + } + if err := os.Rename(temporary, path); err != nil { + return fmt.Errorf("pitot: publish runtime descriptor: %w", err) + } + return validateDescriptorSecurity(path) +} + +func loadDescriptor(path string) (Descriptor, error) { + if err := validateDescriptorSecurity(path); err != nil { + return Descriptor{}, err + } + raw, err := os.ReadFile(path) + if err != nil { + return Descriptor{}, fmt.Errorf("pitot: read runtime descriptor: %w", err) + } + var descriptor Descriptor + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&descriptor); err != nil { + return Descriptor{}, fmt.Errorf("pitot: decode runtime descriptor: %w", err) + } + if descriptor.SchemaVersion != descriptorVersion || descriptor.InstanceID == "" || descriptor.Endpoint == "" || descriptor.Token == "" || descriptor.ConfigSHA256 == "" { + return Descriptor{}, errors.New("pitot: invalid runtime descriptor") + } + return descriptor, nil +} + +func prepareRuntimePath(path string) error { + if path == "" { + return errors.New("pitot: runtime descriptor path is required") + } + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return fmt.Errorf("pitot: inspect runtime descriptor: %w", err) + } + client, err := OpenClient(path) + if err != nil { + return fmt.Errorf("pitot: refusing to replace an invalid runtime descriptor: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + if client.Health(ctx) == nil { + return errors.New("pitot: runtime descriptor already belongs to a live instance") + } + if err := os.Remove(path); err != nil { + return fmt.Errorf("pitot: remove stale runtime descriptor: %w", err) + } + return nil +} + +func removeDescriptorIfOwned(path, instanceID string) { + descriptor, err := loadDescriptor(path) + if err == nil && descriptor.InstanceID == instanceID { + _ = os.Remove(path) + } +} diff --git a/labs/15-pitot/pitot/runtime/transport_test.go b/labs/15-pitot/pitot/runtime/transport_test.go new file mode 100644 index 000000000..ad9451bec --- /dev/null +++ b/labs/15-pitot/pitot/runtime/transport_test.go @@ -0,0 +1,129 @@ +package runtime + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/operatorstack/pitot/config" +) + +func startTestServer(t *testing.T) (string, *Client, context.CancelFunc) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + manager, err := Start(ctx, config.Config{}, io.Discard) + if err != nil { + cancel() + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "runtime.json") + done := make(chan error, 1) + go func() { done <- NewServer(manager, strings.Repeat("a", 64), path, io.Discard, io.Discard).Serve(ctx) }() + t.Cleanup(func() { + cancel() + manager.Close() + if err := <-done; err != nil { + t.Errorf("serve runtime: %v", err) + } + }) + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); { + client, openErr := OpenClient(path) + if openErr == nil { + return path, client, cancel + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("runtime descriptor was not published") + return "", nil, cancel +} + +func TestDescriptorIsOwnerOnlyAndStrict(t *testing.T) { + path := filepath.Join(t.TempDir(), "runtime.json") + descriptor := Descriptor{SchemaVersion: 1, InstanceID: "instance", PID: os.Getpid(), Endpoint: "http://127.0.0.1:1", Token: "secret", ConfigSHA256: strings.Repeat("a", 64)} + if err := writeDescriptor(path, descriptor); err != nil { + t.Fatal(err) + } + if runtime.GOOS != "windows" { + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("descriptor mode = %v", info.Mode().Perm()) + } + if err := os.Chmod(path, 0o644); err != nil { + t.Fatal(err) + } + if _, err := OpenClient(path); err == nil { + t.Fatal("expected group-readable descriptor to be rejected") + } + } + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var value map[string]any + if err := json.Unmarshal(raw, &value); err != nil || value["token"] != "secret" { + t.Fatalf("descriptor = %s, err = %v", raw, err) + } +} + +func TestTransportRejectsWrongTokenAndIdentity(t *testing.T) { + _, client, _ := startTestServer(t) + wrongToken := *client + wrongToken.descriptor.Token = "wrong" + if err := wrongToken.Health(context.Background()); err == nil { + t.Fatal("wrong token unexpectedly authenticated") + } + wrongIdentity := *client + wrongIdentity.descriptor.InstanceID = "stale-instance" + if err := wrongIdentity.Health(context.Background()); err == nil || !strings.Contains(err.Error(), "identity mismatch") { + t.Fatalf("identity error = %v", err) + } +} + +func TestTransportRejectsMalformedOversizedAndMissingControllerRequests(t *testing.T) { + _, client, _ := startTestServer(t) + request, err := http.NewRequest(http.MethodPost, client.descriptor.Endpoint+"/v1/requests", bytes.NewBufferString(`{"unknown":true}`)) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Authorization", "Bearer "+client.descriptor.Token) + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusBadRequest { + t.Fatalf("malformed status = %d", response.StatusCode) + } + + oversized, err := http.NewRequest(http.MethodPost, client.descriptor.Endpoint+"/v1/events", strings.NewReader(strings.Repeat("x", maxRequestBytes+1))) + if err != nil { + t.Fatal(err) + } + oversized.Header.Set("Authorization", "Bearer "+client.descriptor.Token) + response, err = http.DefaultClient.Do(oversized) + if err != nil { + t.Fatal(err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusBadRequest { + t.Fatalf("oversized status = %d", response.StatusCode) + } +} + +func TestLiveDescriptorCannotBeReplaced(t *testing.T) { + path, _, _ := startTestServer(t) + if err := prepareRuntimePath(path); err == nil || !strings.Contains(err.Error(), "live instance") { + t.Fatalf("prepare live descriptor = %v", err) + } +} diff --git a/labs/15-pitot/pitot/windtunnel/windtunnel_test.go b/labs/15-pitot/pitot/windtunnel/windtunnel_test.go index 9bfa80824..8dda1f946 100644 --- a/labs/15-pitot/pitot/windtunnel/windtunnel_test.go +++ b/labs/15-pitot/pitot/windtunnel/windtunnel_test.go @@ -35,7 +35,6 @@ var boatstackCanonicalEvents = map[adapters.Host]string{ adapters.Copilot: `{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"git status --short"}}`, adapters.Qwen: `{"hook_event_name":"PreToolUse","tool_name":"run_shell_command","tool_input":{"command":"git status --short"}}`, adapters.Pi: `{"hook_event_name":"tool_call","tool_name":"bash","tool_input":{"command":"git status --short"}}`, - adapters.Cline: `{"hookName":"tool_call","preToolUse":{"toolName":"run_commands","parameters":{"commands":"[\"git status --short\"]"}}}`, } func TestSensorConsumesBoatstackCanonicalEvents(t *testing.T) { diff --git a/labs/15-pitot/public-readme-preview/CONTRIBUTING.md b/labs/15-pitot/public-readme-preview/CONTRIBUTING.md index 45c38f7f2..b9b856efa 100644 --- a/labs/15-pitot/public-readme-preview/CONTRIBUTING.md +++ b/labs/15-pitot/public-readme-preview/CONTRIBUTING.md @@ -44,5 +44,5 @@ python3 scripts/pitot_adapter_supervisor.py check ``` The merge refuses partial, duplicate, mixed-version, unsuccessful, or -fabricated captures. Review the redacted 30-cell wire diff before committing +fabricated captures. Review the redacted 27-cell wire diff before committing it; ordinary CI verifies the committed fixtures and never rewrites them. diff --git a/labs/15-pitot/public-readme-preview/README.md b/labs/15-pitot/public-readme-preview/README.md index 6b5069b80..01be6c6eb 100644 --- a/labs/15-pitot/public-readme-preview/README.md +++ b/labs/15-pitot/public-readme-preview/README.md @@ -11,9 +11,11 @@ Pitot agent E2E

-

Every supervised adapter must pass a binary-observed prompt → model reply → hook → tool-result loop on Ubuntu, macOS, and Windows.

+

Every supervised adapter must pass a binary-observed prompt → real hook → projected Consumer → Controller allow/deny → tool-result loop on Ubuntu, macOS, and Windows.

-

Supervised adapters: Claude · Cline · Cursor · Codex · GitHub Copilot CLI · Gemini · Kimi Code · OpenCode · Pi · Qwen Code

+

Supervised adapters: Claude · Cursor · Codex · GitHub Copilot CLI · Gemini · Kimi Code · OpenCode · Pi · Qwen Code

+ +

Supervised runtime capabilities: hook control · Consumer delivery · explicit request

@@ -65,25 +67,25 @@ If you want to see concrete integration ideas, start with the **Use-Cases Grid** This gallery shows practical ways teams can compose Consumers and Controllers without forcing each workflow into the host or into a single monolithic runtime. -It includes both engineering patterns (token metering, approvals, audit hooks) and +It includes both engineering patterns (action auditing, approvals, audit hooks) and non-coding workflows (email triage, file movement, and local automation), so you can quickly evaluate where Pitot helps before building. ## Two small programs

- A token meter consumes events while an approval controller receives a request and returns a response + An action auditor consumes events while an approval controller receives a request and returns a response

-### 1. Count tokens without recording prompts +### 1. Audit shell requests without recording commands Configure a Consumer with an `omit` content projection: ```yaml consumers: - - id: token-meter - command: ["python3", "./examples/token-meter.py"] - events: ["model.usage"] + - id: action-audit + command: ["python3", "./examples/action-audit.py"] + events: ["action.requested"] projection: content: omit ``` @@ -91,7 +93,7 @@ consumers: Pitot writes newline-delimited JSON to the program's standard input: ```json -{"pitot_version":"1","type":"model.usage","session_id":"sess_42","model":"gpt-5","usage":{"input_tokens":1240,"output_tokens":380},"observation":{"source":"host_event","fidelity":"direct"}} +{"pitot_version":"1","type":"action.requested","host":{"name":"claude"},"action":{"id":"act_7f2","kind":"shell"},"content":{"mode":"omit"},"observation":{"source":"host_hook","fidelity":"direct"}} ``` The Consumer is ordinary Python—no Pitot SDK required: @@ -100,26 +102,24 @@ The Consumer is ordinary Python—no Pitot SDK required: import json import sys -total = 0 - for line in sys.stdin: event = json.loads(line) - usage = event.get("usage", {}) - total += usage.get("input_tokens", 0) - total += usage.get("output_tokens", 0) - print(json.dumps({"session_tokens": total}), file=sys.stderr) + print(json.dumps({ + "host": event["host"]["name"], + "action_id": event["action"]["id"], + "kind": event["action"]["kind"], + }), file=sys.stderr) ``` -Pitot preserves the quality of the measurement. Provider-reported usage is -marked `direct`; tokenizer-derived usage is `estimated`; unavailable usage is -never silently manufactured. +The command is projected out before bytes enter the Consumer pipe. Consumer +failure cannot allow or deny the waiting host action. ### 2. Let a skill request approval A coding-agent skill can make a synchronous request: ```bash -pitot request release.approval --data '{"release":"v1.4.0"}' +pitot request release.approval --data '{"release":"v1.4.0"}' --runtime "$PITOT_RUNTIME" ``` Register one Controller for that request kind: @@ -164,10 +164,25 @@ Inspect the effective local boundary: pitot doctor ``` -Start Pitot with repository-owned configuration: +Start Pitot with repository-owned configuration and an owner-only runtime +descriptor: ```bash -pitot run --config .pitot.yaml +export PITOT_RUNTIME="${XDG_RUNTIME_DIR:-$TMPDIR}/pitot/project.json" +pitot run --config .pitot.yaml --runtime "$PITOT_RUNTIME" +``` + +Start coding-agent CLIs from the same environment. Their `pitot hook HOST` +commands discover the authenticated runtime through `PITOT_RUNTIME`. Without +that variable or `--runtime PATH`, hooks remain observation-only for backwards +compatibility. Once a runtime is explicitly selected, transport or +authentication failure blocks the controllable action. + +On Windows, set the descriptor in the launching PowerShell session: + +```powershell +$env:PITOT_RUNTIME = Join-Path $env:LOCALAPPDATA "Pitot\project.json" +pitot run --config .pitot.yaml --runtime $env:PITOT_RUNTIME ``` ### Kimi Code @@ -197,7 +212,8 @@ command = "pitot hook kimi" ``` Kimi sends the hook payload to Pitot on standard input. Pitot exits `0` when -the request is accepted and `2` when a malformed request must be blocked. Check +the request is accepted and `2` when input is malformed or the configured +Controller denies the action. Check the non-interactive Kimi CLI after configuration with: ```bash @@ -209,40 +225,62 @@ authentication, configuration, and hook behavior. ### GitHub Copilot CLI -Add the following Claude-compatible hook to `~/.copilot/settings.json`: +Copy `integrations/copilot/PreToolUse` to a stable executable path (or use +`PreToolUse.ps1` on Windows), then add the following Claude-compatible hook to +`~/.copilot/settings.json`: ```json { "hooks": { "PreToolUse": [{ "matcher": "Bash", - "hooks": [{"type": "command", "command": "pitot hook copilot"}] + "hooks": [{"type": "command", "command": "/path/to/PreToolUse"}] }] } } ``` -The PascalCase event keeps the blocking payload on Pitot's standard -`hook_event_name` and `tool_input.command` boundary. See the official +The bridge keeps the blocking payload on Pitot's standard `hook_event_name` +and `tool_input.command` boundary and returns Copilot's structured native deny +reason when a Controller rejects the command. See the official [Copilot CLI hooks reference](https://docs.github.com/en/copilot/reference/hooks-reference). +### Cursor + +Copy `integrations/cursor/beforeShellExecution` into the repository and point +`.cursor/hooks.json` at it with `failClosed: true`. The bridge returns Cursor's +native `permission: "deny"` envelope, including the Controller message, while +the runtime remains available through `PITOT_RUNTIME`. See Cursor's +[hooks documentation](https://cursor.com/docs/agent/hooks). + +### Gemini + +Copy `integrations/gemini/BeforeTool` to an executable path (or use +`BeforeTool.ps1` on Windows) and register it as a `BeforeTool` command hook for +`run_shell_command`. The bridge translates Pitot rejection into Gemini's +structured `decision: "deny"` and `reason` response so the model receives the +blocked tool result. See the [Gemini CLI hooks reference](https://geminicli.com/docs/hooks/reference/). + ### Qwen Code -Add this command hook to `~/.qwen/settings.json`: +Copy `integrations/qwen/PreToolUse` to a stable executable path (or use the +Node-based `PreToolUse.cjs` bridge on Windows), then add this command hook to +`~/.qwen/settings.json`: ```json { "hooks": { "PreToolUse": [{ "matcher": "^Bash$", - "hooks": [{"type": "command", "command": "pitot hook qwen"}] + "hooks": [{"type": "command", "command": "/path/to/PreToolUse"}] }] } } ``` -Qwen sends the native JSON payload on standard input and honors Pitot's -blocking exit status. See the official +Qwen sends the native JSON payload on standard input. The bridge returns its +native structured allow or deny decision and preserves the Controller reason. +See the official [Qwen Code hooks guide](https://qwenlm.github.io/qwen-code-docs/en/users/features/hooks/). ### Pi @@ -253,19 +291,10 @@ blocking `tool_call` event into Pitot's stable envelope and returns Pi's native `block` response when Pitot rejects the request. See the official [Pi extensions documentation](https://pi.dev/docs/latest/extensions). -### Cline - -Copy `integrations/cline/PreToolUse` to `~/Documents/Cline/Hooks/PreToolUse` on -macOS or Linux and make it executable. On Windows, copy `PreToolUse.ps1` into -that directory instead. Enable hooks in Cline's Hooks tab or run -`cline config set hooks-enabled=true`. These -bridges pass Cline's native nested payload to `pitot hook cline` and translate -the result into Cline's `cancel` response. See the official -[Cline hooks documentation](https://docs.cline.bot/customization/hooks). - Pitot uses supervised local processes in v1. It starts declared Consumers and Controllers itself, applies each projection before bytes enter the child pipe, -and exposes no unauthenticated local socket. +and exposes only a loopback endpoint authenticated by the owner-only runtime +descriptor. The capability token is never passed to child processes or logs. ## Language-neutral by design @@ -399,12 +428,14 @@ decides.** pitot/ ├── schema/ public event and response schemas ├── protocol/ framing and state-machine specifications -├── adapters/ Claude Code, Cursor, and Codex boundaries +├── adapters/ built-in coding-agent hook boundaries ├── sensor/ normalization and observation pipeline ├── bridge/ controller routing and response transport ├── projection/ full, sha256, and omit policies +├── config/ strict Consumer and Controller declarations +├── runtime/ authenticated ingress and local process delivery ├── conformance/ language-neutral fixtures and negative controls -├── examples/ token-meter and local-approval +├── examples/ local approval and Consumer examples └── cmd/pitot/ reference Go executable ``` diff --git a/labs/15-pitot/public-readme-preview/assets/pitot-two-roles.svg b/labs/15-pitot/public-readme-preview/assets/pitot-two-roles.svg index 355e15d6f..88b4a9503 100644 --- a/labs/15-pitot/public-readme-preview/assets/pitot-two-roles.svg +++ b/labs/15-pitot/public-readme-preview/assets/pitot-two-roles.svg @@ -7,7 +7,7 @@ 01 / CONSUMER - Token meter + Action audit Counts usage. Receives no prompt or reply channel. model.usage @@ -35,4 +35,3 @@ Consumer: Event → observe Controller: Request → Response - diff --git a/labs/15-pitot/scripts/build_pitot.py b/labs/15-pitot/scripts/build_pitot.py index c2a61f6ab..78b5f3891 100644 --- a/labs/15-pitot/scripts/build_pitot.py +++ b/labs/15-pitot/scripts/build_pitot.py @@ -91,6 +91,7 @@ def _iter_e2e_harness_files(repo: Path) -> list[tuple[str, Path]]: for name in ( "e2e_unified_runner.sh", "install_real_agent.py", "mock_anthropic_server.js", "model_control_proxy.py", "cursor_control_proxy.mjs", "endpoint-provenance.json", "real_agent_driver.py", "run_e2e_report.py", + "runtime_capability_driver.py", ) if (path := root / name).is_file() ) diff --git a/labs/15-pitot/scripts/pitot_adapter_inventory.go b/labs/15-pitot/scripts/pitot_adapter_inventory.go index 9e43f6c6e..c2d17139e 100644 --- a/labs/15-pitot/scripts/pitot_adapter_inventory.go +++ b/labs/15-pitot/scripts/pitot_adapter_inventory.go @@ -8,6 +8,7 @@ import ( "os" "github.com/operatorstack/pitot/adapters" + pitotruntime "github.com/operatorstack/pitot/runtime" ) func main() { @@ -16,7 +17,12 @@ func main() { for index, host := range hosts { values[index] = string(host) } - if err := json.NewEncoder(os.Stdout).Encode(values); err != nil { + capabilities := pitotruntime.Capabilities() + capabilityValues := make([]string, len(capabilities)) + for index, capability := range capabilities { + capabilityValues[index] = string(capability) + } + if err := json.NewEncoder(os.Stdout).Encode(map[string]any{"adapters": values, "capabilities": capabilityValues}); err != nil { panic(err) } } diff --git a/labs/15-pitot/scripts/pitot_adapter_supervisor.py b/labs/15-pitot/scripts/pitot_adapter_supervisor.py index 2d96b1b7a..86aacd4b7 100644 --- a/labs/15-pitot/scripts/pitot_adapter_supervisor.py +++ b/labs/15-pitot/scripts/pitot_adapter_supervisor.py @@ -17,7 +17,6 @@ ENDPOINT_PROVENANCE = Path("labs/15-pitot/tests/endpoint-provenance.json") README = Path("labs/15-pitot/public-readme-preview/README.md") UNIFIED_WORKFLOW = Path(".github/workflows/pitot-e2e.yml") -REPORT_WORKFLOW = Path(".github/workflows/pitot-e2e-report.yml") REUSABLE_WORKFLOW = Path(".github/workflows/pitot-e2e-agent.yml") INVENTORY_HELPER = Path("labs/15-pitot/scripts/pitot_adapter_inventory.go") README_START = "" @@ -26,7 +25,7 @@ LABEL_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 .+_-]{0,39}$") VERSION_PATTERN = re.compile(r"^(?:[0-9]+\.){2}[0-9]+(?:-[0-9A-Za-z.-]+)?$") DIALECTS = {"anthropic_messages", "openai_chat", "openai_responses", "gemini_generate_content", "cursor_connect_proto"} -INTEGRATIONS = {"native_command_hook", "cline_bridge", "pi_extension", "opencode_plugin"} +INTEGRATIONS = {"native_command_hook", "pi_extension", "opencode_plugin"} INSTALLERS = {"npm", "kimi_release", "cursor_release"} @@ -45,10 +44,17 @@ def load_manifest(root: Path = ROOT) -> dict[str, object]: def validate_manifest(value: object) -> None: - if not isinstance(value, dict) or set(value) != {"schema_version", "platforms", "agents"}: - raise ContractError("manifest must contain schema_version, platforms, and agents") - if value["schema_version"] != 4: + if not isinstance(value, dict) or set(value) != {"schema_version", "capabilities", "platforms", "agents"}: + raise ContractError("manifest must contain schema_version, capabilities, platforms, and agents") + if value["schema_version"] != 6: raise ContractError("unsupported manifest schema_version") + expected_capabilities = [ + {"id": "hook_control", "matrix": "agent_platform"}, + {"id": "consumer_delivery", "matrix": "agent_platform"}, + {"id": "explicit_request", "matrix": "platform"}, + ] + if value["capabilities"] != expected_capabilities: + raise ContractError("capabilities must exactly supervise hook control, Consumer delivery, and explicit request") platforms = value["platforms"] agents = value["agents"] if not isinstance(platforms, list) or not isinstance(agents, list) or not agents: @@ -65,7 +71,7 @@ def validate_manifest(value: object) -> None: for agent in agents: required = { "id", "label", "version", "executable", "installer", - "integration", "runtime", "driver", "required_mode", + "integration", "artifacts", "runtime", "driver", "required_mode", } if not isinstance(agent, dict) or set(agent) != required: raise ContractError(f"each agent must contain exactly {', '.join(sorted(required))}") @@ -90,6 +96,14 @@ def validate_manifest(value: object) -> None: raise ContractError(f"agent {agent_id} has invalid pinned installer metadata") if agent["integration"] not in INTEGRATIONS: raise ContractError(f"agent {agent_id} has an unsupported integration mechanism") + artifacts = agent["artifacts"] + artifact_prefix = f"integrations/{agent_id}/" + if ( + not isinstance(artifacts, list) + or len(artifacts) != len(set(artifacts)) + or any(not isinstance(path, str) or not path.startswith(artifact_prefix) for path in artifacts) + ): + raise ContractError(f"agent {agent_id} has invalid supervised integration artifacts") expected_runtime_keys = {platform["id"] for platform in platforms} runtime = agent["runtime"] if not isinstance(runtime, dict) or set(runtime) != expected_runtime_keys: @@ -132,7 +146,9 @@ def validate_endpoint_value(value: object, manifest: dict[str, object]) -> None: expected_keys = {(agent_id, platform) for agent_id in declared for platform in platforms} actual_keys = {(cell.get("agent"), cell.get("platform")) for cell in cells if isinstance(cell, dict)} if len(cells) != len(expected_keys) or actual_keys != expected_keys: - raise ContractError("endpoint provenance must contain exactly all 30 agent/platform cells") + raise ContractError( + f"endpoint provenance must contain exactly all {len(expected_keys)} agent/platform cells" + ) required = {"agent", "platform", "runtime", "version", "executable_sha256", "dialect", "request", "response", "provenance", "capture_sha256"} for cell in cells: agent_id, platform = cell["agent"], cell["platform"] @@ -166,7 +182,7 @@ def validate_endpoint_value(value: object, manifest: dict[str, object]) -> None: raise ContractError(f"endpoint fixture {agent_id}/{platform} has a fabricated capture digest") -def built_in_adapters(root: Path = ROOT) -> list[str]: +def built_in_inventory(root: Path = ROOT) -> dict[str, list[str]]: completed = subprocess.run( ["go", "run", str(root / INVENTORY_HELPER)], cwd=root, @@ -176,11 +192,22 @@ def built_in_adapters(root: Path = ROOT) -> list[str]: stderr=subprocess.PIPE, ) value = json.loads(completed.stdout) - if not isinstance(value, list) or not all(isinstance(item, str) for item in value): - raise ContractError("Go adapter inventory returned an invalid result") + if not isinstance(value, dict) or set(value) != {"adapters", "capabilities"}: + raise ContractError("Go runtime inventory returned an invalid result") + for name in ("adapters", "capabilities"): + if not isinstance(value[name], list) or not all(isinstance(item, str) for item in value[name]): + raise ContractError(f"Go {name} inventory returned an invalid result") return value +def built_in_adapters(root: Path = ROOT) -> list[str]: + return built_in_inventory(root)["adapters"] + + +def built_in_capabilities(root: Path = ROOT) -> list[str]: + return built_in_inventory(root)["capabilities"] + + def matrix(manifest: dict[str, object]) -> list[dict[str, str]]: return [ { @@ -198,6 +225,13 @@ def matrix(manifest: dict[str, object]) -> list[dict[str, str]]: ] +def runtime_matrix(manifest: dict[str, object]) -> list[dict[str, str]]: + return [ + {"capability": "explicit_request", "platform": platform["id"], "runner": platform["runner"]} + for platform in manifest["platforms"] + ] + + def readme_block(manifest: dict[str, object]) -> str: labels = " · ".join(agent["label"] for agent in manifest["agents"]) return "\n".join( @@ -207,9 +241,11 @@ def readme_block(manifest: dict[str, object]) -> str: ' Pitot agent E2E', "

", "", - '

Every supervised adapter must pass a binary-observed prompt → model reply → hook → tool-result loop on Ubuntu, macOS, and Windows.

', + '

Every supervised adapter must pass a binary-observed prompt → real hook → projected Consumer → Controller allow/deny → tool-result loop on Ubuntu, macOS, and Windows.

', "", f'

Supervised adapters: {labels}

', + "", + '

Supervised runtime capabilities: hook control · Consumer delivery · explicit request

', README_END, ] ) @@ -245,13 +281,16 @@ def unified_workflow() -> str: default: false permissions: + actions: read contents: read + pull-requests: write jobs: inventory: runs-on: ubuntu-latest outputs: - matrix: ${{ steps.supervisor.outputs.matrix }} + agent_matrix: ${{ steps.supervisor.outputs.agent_matrix }} + runtime_matrix: ${{ steps.supervisor.outputs.runtime_matrix }} steps: - uses: actions/checkout@v7 - uses: actions/setup-go@v7 @@ -265,8 +304,10 @@ def unified_workflow() -> str: if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.capture_provenance }}" == "true" ]]; then operation="capture" fi - matrix="$(python3 labs/15-pitot/scripts/pitot_adapter_supervisor.py "$operation")" - echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + agent_matrix="$(python3 labs/15-pitot/scripts/pitot_adapter_supervisor.py "$operation")" + runtime_matrix="$(python3 labs/15-pitot/scripts/pitot_adapter_supervisor.py runtime-matrix)" + echo "agent_matrix=$agent_matrix" >> "$GITHUB_OUTPUT" + echo "runtime_matrix=$runtime_matrix" >> "$GITHUB_OUTPUT" - name: Upload supervised adapter inventory uses: actions/upload-artifact@v4 with: @@ -275,41 +316,74 @@ def unified_workflow() -> str: if-no-files-found: error retention-days: 14 - verify: + verify-agents: needs: inventory strategy: fail-fast: false - matrix: ${{ fromJSON(needs.inventory.outputs.matrix) }} + matrix: ${{ fromJSON(needs.inventory.outputs.agent_matrix) }} uses: ./.github/workflows/pitot-e2e-agent.yml with: agent: ${{ matrix.agent }} platform: ${{ matrix.platform }} runner: ${{ matrix.runner }} capture: ${{ matrix.capture == 'true' }} -""" - - -def report_workflow() -> str: - return """# Generated by pitot_adapter_supervisor.py. Do not edit directly. -name: Report Pitot E2E results - -on: - workflow_run: - workflows: [Pitot E2E] - types: [completed] - -permissions: - actions: read - contents: read - pull-requests: write -concurrency: - group: pitot-e2e-report-${{ github.event.workflow_run.head_sha }} - cancel-in-progress: true + verify-runtime: + needs: inventory + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.inventory.outputs.runtime_matrix) }} + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: labs/15-pitot/pitot/go.mod + cache-dependency-path: labs/15-pitot/pitot/go.mod + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - name: Resolve Bash runtime + shell: bash + run: | + bash_path="$(command -v bash)" + if [[ "${{ runner.os }}" == "Windows" ]]; then + bash_path="$(cygpath -m "$bash_path")" + fi + echo "PITOT_BASH=$bash_path" >> "$GITHUB_ENV" + - name: Run explicit request runtime E2E + id: e2e + continue-on-error: true + timeout-minutes: 5 + shell: bash + env: + PITOT_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + PITOT_E2E_PLATFORM: ${{ matrix.platform }} + PITOT_E2E_EVIDENCE: ${{ runner.temp }}/pitot-e2e/runtime-evidence.json + run: >- + python labs/15-pitot/tests/run_e2e_report.py + --capability explicit_request + --platform "${{ matrix.platform }}" + --output "${{ runner.temp }}/pitot-e2e/runtime-result.json" + --evidence "${{ runner.temp }}/pitot-e2e/runtime-evidence.json" + -- "$PITOT_BASH" labs/15-pitot/tests/e2e_runtime_cli_test.sh + < /dev/null + - name: Upload structured runtime result + if: always() + uses: actions/upload-artifact@v4 + with: + name: pitot-e2e-runtime-${{ matrix.platform }} + path: ${{ runner.temp }}/pitot-e2e/runtime-result.json + if-no-files-found: error + retention-days: 14 + - name: Enforce runtime E2E result + if: always() && steps.e2e.outcome != 'success' + shell: bash + run: exit 1 -jobs: report: - if: github.event.workflow_run.event == 'pull_request' + if: ${{ always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + needs: [inventory, verify-agents, verify-runtime] runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 @@ -329,7 +403,8 @@ def contract_errors(root: Path, adapters: list[str] | None = None) -> list[str]: except ContractError as error: return [str(error)] errors: list[str] = [] - inventory = adapters if adapters is not None else built_in_adapters(root) + go_inventory = None if adapters is not None else built_in_inventory(root) + inventory = adapters if adapters is not None else go_inventory["adapters"] declared = [agent["id"] for agent in manifest["agents"]] missing = sorted(set(inventory) - set(declared)) extra = sorted(set(declared) - set(inventory)) @@ -337,13 +412,28 @@ def contract_errors(root: Path, adapters: list[str] | None = None) -> list[str]: errors.append(f"built-in adapters missing from manifest: {', '.join(missing)}") if extra: errors.append(f"manifest agents are not built-in adapters: {', '.join(extra)}") + declared_capabilities = [item["id"] for item in manifest["capabilities"]] + capability_inventory = declared_capabilities if adapters is not None else go_inventory["capabilities"] + if declared_capabilities != capability_inventory: + errors.append("manifest capabilities do not exactly match the Go runtime inventory") for agent_id in declared: script = root / f"labs/15-pitot/tests/e2e_{agent_id}_cli_test.sh" if not script.is_file(): errors.append(f"adapter {agent_id} is missing {script.relative_to(root)}") + for agent in manifest["agents"]: + for artifact in agent["artifacts"]: + path = root / "labs/15-pitot" / artifact + if not path.is_file(): + errors.append(f"adapter {agent['id']} is missing supervised artifact {artifact}") driver = root / "labs/15-pitot/tests/real_agent_driver.py" if not driver.is_file(): errors.append("missing canonical real-agent prompt driver") + elif any(fragment not in driver.read_text(encoding="utf-8") for fragment in ("consumer_observed", "controller_deny_observed", "deny_canary_absent", "final_output_observed")): + errors.append("real-agent driver does not require the complete runtime capability receipts") + runtime_driver = root / "labs/15-pitot/tests/runtime_capability_driver.py" + runtime_script = root / "labs/15-pitot/tests/e2e_runtime_cli_test.sh" + if not runtime_driver.is_file() or not runtime_script.is_file(): + errors.append("explicit request capability is missing its canonical runtime driver") proxy = root / "labs/15-pitot/tests/model_control_proxy.py" if not proxy.is_file(): errors.append("missing local model-control proxy") @@ -361,12 +451,13 @@ def contract_errors(root: Path, adapters: list[str] | None = None) -> list[str]: forbidden = ("hook_subprocess", "Falling back", "active subprocess hook verification") if any(fragment in runner_text for fragment in forbidden): errors.append("unified runner permits synthetic E2E success") - if "real_agent_driver.py" not in runner_text: + if "real_agent_driver.py" not in runner_text or "--test-role" not in runner_text: errors.append("unified runner does not invoke the canonical real-agent driver") if ( - runner_text.count("GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build") != 2 + runner_text.count("GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build") != 3 or "pitot-linux" not in runner_text or "pitot-witness-linux" not in runner_text + or "pitot-testrole-linux" not in runner_text or "wsl.exe --distribution Ubuntu -- env" not in runner_text or 'python3 "$(to_wsl_path "$SCRIPT_DIR/real_agent_driver.py")"' not in runner_text or "MSYS2_ARG_CONV_EXCL='*'" not in runner_text @@ -374,13 +465,15 @@ def contract_errors(root: Path, adapters: list[str] | None = None) -> list[str]: errors.append("unified runner does not preserve Cursor's native WSL controller boundary") expected_files = { UNIFIED_WORKFLOW: unified_workflow(), - REPORT_WORKFLOW: report_workflow(), } for relative, expected in expected_files.items(): path = root / relative actual = path.read_text(encoding="utf-8") if path.is_file() else "" if actual != expected: errors.append(f"generated surface drifted: {relative}") + legacy_report = root / ".github/workflows/pitot-e2e-report.yml" + if legacy_report.exists(): + errors.append("legacy Pitot report workflow escaped the unified supervisor") reusable_path = root / REUSABLE_WORKFLOW reusable = reusable_path.read_text(encoding="utf-8") if reusable_path.is_file() else "" required_reusable_contract = ( @@ -424,7 +517,6 @@ def render(root: Path = ROOT) -> None: encoding="utf-8", ) (root / UNIFIED_WORKFLOW).write_text(unified_workflow(), encoding="utf-8") - (root / REPORT_WORKFLOW).write_text(report_workflow(), encoding="utf-8") def merge_captures(root: Path, captures: Path, output: Path) -> None: @@ -449,7 +541,7 @@ def load_manifest_without_provenance(root: Path) -> dict[str, object]: def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("operation", choices=("check", "matrix", "render", "capture", "capture-merge")) + parser.add_argument("operation", choices=("check", "matrix", "runtime-matrix", "render", "capture", "capture-merge")) parser.add_argument("--repo", type=Path, default=ROOT) parser.add_argument("--captures", type=Path) parser.add_argument("--output", type=Path) @@ -464,7 +556,11 @@ def main() -> int: if args.captures is None or args.output is None: raise ContractError("capture-merge requires --captures and --output") merge_captures(root, args.captures.resolve(), args.output.resolve()) - print(f"PASS: merged 30 binary-observed endpoint captures into {args.output}") + manifest = load_manifest_without_provenance(root) + print( + f"PASS: merged {len(matrix(manifest))} binary-observed endpoint captures " + f"into {args.output}" + ) return 0 if args.operation == "capture": manifest = load_manifest_without_provenance(root) @@ -478,6 +574,9 @@ def main() -> int: manifest = load_manifest(root) values = matrix(manifest) print(json.dumps({"include": values}, separators=(",", ":"))) + elif args.operation == "runtime-matrix": + manifest = load_manifest(root) + print(json.dumps({"include": runtime_matrix(manifest)}, separators=(",", ":"))) else: print(f"PASS: supervised {len(load_manifest(root)['agents'])} Pitot adapters") return 0 diff --git a/labs/15-pitot/tests/cursor_control_proxy.mjs b/labs/15-pitot/tests/cursor_control_proxy.mjs index 144f6e4f8..e032241d6 100644 --- a/labs/15-pitot/tests/cursor_control_proxy.mjs +++ b/labs/15-pitot/tests/cursor_control_proxy.mjs @@ -60,7 +60,7 @@ function envelopeShapes(body) { return shapes; } -function shellExecution(command) { +function shellExecution(command, phase = "allow") { const executable = command.split(" ", 1)[0]; const argument = command.slice(executable.length + 1); // ShellCommandParsingResult.ExecutableCommandArg { type, value } @@ -76,7 +76,7 @@ function shellExecution(command) { // agent.v1.ShellArgs { command: 1, tool_call_id: 4 } const shellArgs = Buffer.concat([ fieldBytes(1, command), - fieldBytes(4, "pitot-tool-1"), + fieldBytes(4, `pitot-tool-${phase}`), fieldBytes(5, command), fieldBytes(8, parsingResult), fieldVarint(12, 1), @@ -84,7 +84,7 @@ function shellExecution(command) { // agent.v1.ExecServerMessage { id: 1, exec_id: 15, shell_args: 2 } const execution = Buffer.concat([ fieldVarint(1, 1), - fieldBytes(15, "pitot-exec-1"), + fieldBytes(15, `pitot-exec-${phase}`), fieldBytes(2, shellArgs), ]); // agent.v1.AgentServerMessage { exec_server_message: 2 } @@ -114,6 +114,13 @@ function unaryResponse(path) { return Buffer.alloc(0); } +function outstandingPhase(state, sent) { + if (!state.allow_tool_result_observed && !sent.allow) return "allow"; + if (state.allow_tool_result_observed && !state.denied_result_observed && !sent.deny) return "deny"; + if (state.denied_result_observed && !sent.final) return "final"; + return null; +} + const args = parseArgs(process.argv); if (process.argv.includes("--self-test")) { const command = "pitot-e2e-canary fixture-nonce"; @@ -122,10 +129,26 @@ if (process.argv.includes("--self-test")) { if (!message.includes(Buffer.from(command)) || framed.readUInt32BE(1) !== message.length) { throw new Error("Cursor Connect/protobuf fixture is invalid"); } + const state = { allow_tool_result_observed: false, denied_result_observed: false }; + const firstStream = { allow: false, deny: false, final: false }; + if (outstandingPhase(state, firstStream) !== "allow") throw new Error("allow phase was not selected"); + firstStream.allow = true; + if (outstandingPhase(state, firstStream) !== null) throw new Error("allow phase duplicated on one stream"); + if (outstandingPhase(state, { allow: false, deny: false, final: false }) !== "allow") { + throw new Error("allow phase was not recovered on a replacement stream"); + } + state.allow_tool_result_observed = true; + if (outstandingPhase(state, { allow: false, deny: false, final: false }) !== "deny") { + throw new Error("deny phase was not selected after the allow receipt"); + } + state.denied_result_observed = true; + if (outstandingPhase(state, { allow: false, deny: false, final: false }) !== "final") { + throw new Error("final phase was not selected after the denial receipt"); + } process.stdout.write("PASS: Cursor Connect/protobuf endpoint fixture\n"); process.exit(0); } -const required = ["nonce", "receipt", "ready-file"]; +const required = ["nonce", "receipt", "ready-file", "canary-command"]; for (const key of required) { if (!args[key]) throw new Error(`missing --${key}`); } @@ -138,10 +161,16 @@ const receipt = { initial_prompt_observed: false, tool_call_response_emitted: false, tool_result_observed: false, + allow_tool_call_response_emitted: false, + allow_tool_result_observed: false, + deny_tool_call_response_emitted: false, + denied_result_observed: false, final_response_emitted: false, endpoint_observed: null, auxiliary_requests: 0, cursor_requests: [], + response_attempts: { allow: 0, deny: 0, final: 0 }, + transport_errors: [], }; function save() { @@ -153,12 +182,24 @@ function save() { const h2Server = http2.createServer(); +function recordTransportError(scope, error) { + receipt.transport_errors.push({ + scope, + code: String(error?.code || "unknown"), + }); + receipt.transport_errors = receipt.transport_errors.slice(-20); + save(); +} + h2Server.on("stream", (stream, headers) => { const path = String(headers[":path"] || "").split("?", 1)[0]; const contentType = String(headers["content-type"] || ""); const chunks = []; let responseStarted = false; - let finished = false; + let allowSent = false; + let denySent = false; + let finalSent = false; + stream.on("error", (error) => recordTransportError("run_stream", error)); stream.on("data", (chunk) => { chunks.push(chunk); const body = Buffer.concat(chunks); @@ -189,17 +230,49 @@ h2Server.on("stream", (stream, headers) => { stream.write(connectEnvelope(textUpdate("No tool call"))); stream.write(connectEnvelope(turnEnded())); stream.end(connectEnvelope(Buffer.from("{}"), 0x02)); - } else { - receipt.tool_call_response_emitted = true; - stream.write(connectEnvelope(shellExecution(`pitot-e2e-canary ${args.nonce}`))); } } - if (responseStarted && !finished && body.includes(Buffer.from(`PITOT_CANARY_RESULT ${args.nonce}`))) { - finished = true; + if (!responseStarted || args["response-fault"] === "text" || receipt.final_response_emitted) return; + // A released Cursor CLI may reconnect its bidirectional Run stream after + // a transient reset. Delivery is only proven by the matching client tool + // result, so resend the outstanding phase once on each replacement stream. + let phase = outstandingPhase(receipt, { allow: allowSent, deny: denySent, final: finalSent }); + if (phase === "allow") { + allowSent = true; + receipt.tool_call_response_emitted = true; + receipt.allow_tool_call_response_emitted = true; + receipt.response_attempts.allow += 1; + save(); + stream.write(connectEnvelope(shellExecution(`${args["canary-command"]} PITOT_ALLOW ${args.nonce}`, "allow"))); + return; + } + if (!receipt.allow_tool_result_observed && body.includes(Buffer.from(`PITOT_CANARY_RESULT PITOT_ALLOW ${args.nonce}`))) { receipt.tool_result_observed = true; + receipt.allow_tool_result_observed = true; + save(); + } + phase = outstandingPhase(receipt, { allow: allowSent, deny: denySent, final: finalSent }); + if (phase === "deny") { + denySent = true; + receipt.deny_tool_call_response_emitted = true; + receipt.response_attempts.deny += 1; + save(); + stream.write(connectEnvelope(shellExecution(`${args["canary-command"]} PITOT_DENY ${args.nonce}`, "deny"))); + return; + } + const nativeDenial = body.includes(Buffer.from(`PITOT_DENY ${args.nonce}`)) && + body.includes(Buffer.from("blocked by a hook")); + if (receipt.deny_tool_call_response_emitted && !receipt.denied_result_observed && nativeDenial) { + receipt.denied_result_observed = true; receipt.final_response_emitted = true; save(); - stream.write(connectEnvelope(textUpdate("Pitot E2E Verification Complete"))); + } + phase = outstandingPhase(receipt, { allow: allowSent, deny: denySent, final: finalSent }); + if (phase === "final") { + finalSent = true; + receipt.response_attempts.final += 1; + save(); + stream.write(connectEnvelope(textUpdate(`PITOT_E2E_COMPLETE ${args.nonce}`))); stream.write(connectEnvelope(turnEnded())); // Connect end-stream envelope. Success metadata is an empty JSON object. stream.end(connectEnvelope(Buffer.from("{}"), 0x02)); @@ -216,6 +289,8 @@ h2Server.on("stream", (stream, headers) => { stream.end(contentType.includes("json") ? Buffer.from("{}") : payload); }); }); +h2Server.on("sessionError", (error) => recordTransportError("h2_session", error)); +h2Server.on("error", (error) => recordTransportError("h2_server", error)); const http1Server = http.createServer((request, response) => { const chunks = []; @@ -232,15 +307,26 @@ const http1Server = http.createServer((request, response) => { response.end(contentType.includes("json") ? Buffer.from("{}") : payload); }); }); +http1Server.on("clientError", (error, socket) => { + recordTransportError("http1_client", error); + socket.destroy(); +}); +http1Server.on("error", (error) => recordTransportError("http1_server", error)); const frontServer = net.createServer((client) => { + client.on("error", (error) => recordTransportError("front_client", error)); client.once("data", (first) => { const isHttp2 = first.subarray(0, 14).toString() === "PRI * HTTP/2.0"; const target = isHttp2 ? h2Server.address().port : http1Server.address().port; const backend = net.connect(target, "127.0.0.1", () => backend.write(first)); + backend.on("error", (error) => { + recordTransportError("front_backend", error); + client.destroy(); + }); client.pipe(backend).pipe(client); }); }); +frontServer.on("error", (error) => recordTransportError("front_server", error)); h2Server.listen(0, "127.0.0.1", () => { http1Server.listen(0, "127.0.0.1", () => { diff --git a/labs/15-pitot/tests/e2e_cline_cli_test.sh b/labs/15-pitot/tests/e2e_cline_cli_test.sh deleted file mode 100755 index fb2ee63f5..000000000 --- a/labs/15-pitot/tests/e2e_cline_cli_test.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -exec "$(dirname "$0")/e2e_unified_runner.sh" "cline" diff --git a/labs/15-pitot/tests/e2e_runtime_cli_test.sh b/labs/15-pitot/tests/e2e_runtime_cli_test.sh new file mode 100755 index 000000000..de2cde6ed --- /dev/null +++ b/labs/15-pitot/tests/e2e_runtime_cli_test.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${PITOT_E2E_PLATFORM:?PITOT_E2E_PLATFORM is required}" +: "${PITOT_E2E_EVIDENCE:?PITOT_E2E_EVIDENCE is required}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LAB_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +BUILD_DIR="${RUNNER_TEMP:-$(mktemp -d)}/pitot-runtime-e2e-bin" +mkdir -p "$BUILD_DIR" +PITOT_BINARY="$BUILD_DIR/pitot" +TESTROLE_BINARY="$BUILD_DIR/pitot-testrole" +if [[ "${RUNNER_OS:-}" == "Windows" ]]; then + PITOT_BINARY="${PITOT_BINARY}.exe" + TESTROLE_BINARY="${TESTROLE_BINARY}.exe" +fi +go build -o "$PITOT_BINARY" "$LAB_DIR/pitot/cmd/pitot" +go build -o "$TESTROLE_BINARY" "$LAB_DIR/pitot/internal/testrole" +PYTHON_COMMAND="python3" +if [[ "${RUNNER_OS:-}" == "Windows" ]]; then + PYTHON_COMMAND="python" +fi +"$PYTHON_COMMAND" "$SCRIPT_DIR/runtime_capability_driver.py" \ + --pitot "$PITOT_BINARY" \ + --test-role "$TESTROLE_BINARY" \ + --platform "$PITOT_E2E_PLATFORM" \ + --evidence "$PITOT_E2E_EVIDENCE" diff --git a/labs/15-pitot/tests/e2e_unified_runner.sh b/labs/15-pitot/tests/e2e_unified_runner.sh index f6fcd5ff5..8419ea7a2 100755 --- a/labs/15-pitot/tests/e2e_unified_runner.sh +++ b/labs/15-pitot/tests/e2e_unified_runner.sh @@ -21,24 +21,29 @@ BUILD_DIR="${RUNNER_TEMP:-$(mktemp -d)}/pitot-real-e2e-bin" mkdir -p "$BUILD_DIR" PITOT_BINARY="$BUILD_DIR/pitot" WITNESS_BINARY="$BUILD_DIR/pitot-witness" +TESTROLE_BINARY="$BUILD_DIR/pitot-testrole" if [[ "${RUNNER_OS:-}" == "Windows" ]]; then PITOT_BINARY="${PITOT_BINARY}.exe" WITNESS_BINARY="${WITNESS_BINARY}.exe" + TESTROLE_BINARY="${TESTROLE_BINARY}.exe" if [[ "$HOST" == "cursor" ]]; then # Cursor's supported Windows runtime is WSL. Keep both supervised # executables native to that runtime; crossing back into a Windows Pitot # process makes Cursor's hook pipe remain open after Pitot has returned. PITOT_BINARY="$BUILD_DIR/pitot-linux" WITNESS_BINARY="$BUILD_DIR/pitot-witness-linux" + TESTROLE_BINARY="$BUILD_DIR/pitot-testrole-linux" fi fi if [[ "${RUNNER_OS:-}" == "Windows" && "$HOST" == "cursor" ]]; then GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o "$PITOT_BINARY" "$PITOT_MAIN" GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o "$WITNESS_BINARY" "$SCRIPT_DIR/witness/main.go" + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o "$TESTROLE_BINARY" "$LAB_DIR/pitot/internal/testrole" else go build -o "$PITOT_BINARY" "$PITOT_MAIN" go build -o "$WITNESS_BINARY" "$SCRIPT_DIR/witness/main.go" + go build -o "$TESTROLE_BINARY" "$LAB_DIR/pitot/internal/testrole" fi DRIVER_ARGS=( @@ -48,6 +53,7 @@ DRIVER_ARGS=( --evidence "$PITOT_E2E_EVIDENCE" --pitot "$PITOT_BINARY" --witness "$WITNESS_BINARY" + --test-role "$TESTROLE_BINARY" ) if [[ -n "${PITOT_CAPTURE_OUTPUT:-}" ]]; then DRIVER_ARGS+=(--capture-output "$PITOT_CAPTURE_OUTPUT") @@ -76,6 +82,7 @@ if [[ "${RUNNER_OS:-}" == "Windows" && "$HOST" == "cursor" ]]; then --evidence "$(to_wsl_path "$PITOT_E2E_EVIDENCE")" --pitot "$(to_wsl_path "$PITOT_BINARY")" --witness "$(to_wsl_path "$WITNESS_BINARY")" + --test-role "$(to_wsl_path "$TESTROLE_BINARY")" ) if [[ -n "${PITOT_CAPTURE_OUTPUT:-}" ]]; then WSL_ARGS+=(--capture-output "$(to_wsl_path "$PITOT_CAPTURE_OUTPUT")") diff --git a/labs/15-pitot/tests/endpoint-provenance.json b/labs/15-pitot/tests/endpoint-provenance.json index d10d8f3c9..960e7c764 100644 --- a/labs/15-pitot/tests/endpoint-provenance.json +++ b/labs/15-pitot/tests/endpoint-provenance.json @@ -127,192 +127,6 @@ "runtime": "native", "version": "2.1.217" }, - { - "agent": "cline", - "capture_sha256": "badbd04142c73d323def070ed927eee99b51e1844c7737dfd8c1cda20389f5eb", - "dialect": "openai_chat", - "executable_sha256": "71d1b27aeeebdaa0b91c4babddd1f635eb7f5783257c2b3324a2cb22924cf60f", - "platform": "macos", - "provenance": "pinned_real_cli_capture", - "request": { - "framing": "sse", - "media_type": "application/json", - "method": "POST", - "path": "/v1/chat/completions", - "request_shape": { - "has_tools": true, - "stream": true, - "tool_names": [ - "read_files", - "search_codebase", - "run_commands", - "fetch_web_content", - "editor", - "ask_question", - "spawn_agent", - "team_spawn_teammate", - "team_shutdown_teammate", - "team_status", - "team_task", - "team_run_task", - "team_cancel_run", - "team_list_runs", - "team_await_runs", - "team_send_message", - "team_broadcast", - "team_read_mailbox", - "team_mission_log", - "team_cleanup", - "team_create_outcome", - "team_attach_outcome_fragment", - "team_review_outcome_fragment", - "team_finalize_outcome", - "team_list_outcomes" - ], - "top_level_keys": [ - "messages", - "model", - "stream", - "stream_options", - "tool_choice", - "tools" - ] - }, - "transport": "http1" - }, - "response": { - "acceptance": "nonce_tool_result_round_trip", - "encoder": "openai_chat", - "framing": "sse", - "tool_call": "native_shell" - }, - "runtime": "native", - "version": "3.0.46" - }, - { - "agent": "cline", - "capture_sha256": "2181eafb944aa71dfe0a2d55ef04fa77d38248f842537660fbf33a4a814fdc93", - "dialect": "openai_chat", - "executable_sha256": "71d1b27aeeebdaa0b91c4babddd1f635eb7f5783257c2b3324a2cb22924cf60f", - "platform": "ubuntu", - "provenance": "pinned_real_cli_capture", - "request": { - "framing": "sse", - "media_type": "application/json", - "method": "POST", - "path": "/v1/chat/completions", - "request_shape": { - "has_tools": true, - "stream": true, - "tool_names": [ - "read_files", - "search_codebase", - "run_commands", - "fetch_web_content", - "editor", - "ask_question", - "spawn_agent", - "team_spawn_teammate", - "team_shutdown_teammate", - "team_status", - "team_task", - "team_run_task", - "team_cancel_run", - "team_list_runs", - "team_await_runs", - "team_send_message", - "team_broadcast", - "team_read_mailbox", - "team_mission_log", - "team_cleanup", - "team_create_outcome", - "team_attach_outcome_fragment", - "team_review_outcome_fragment", - "team_finalize_outcome", - "team_list_outcomes" - ], - "top_level_keys": [ - "messages", - "model", - "stream", - "stream_options", - "tool_choice", - "tools" - ] - }, - "transport": "http1" - }, - "response": { - "acceptance": "nonce_tool_result_round_trip", - "encoder": "openai_chat", - "framing": "sse", - "tool_call": "native_shell" - }, - "runtime": "native", - "version": "3.0.46" - }, - { - "agent": "cline", - "capture_sha256": "8d15b6c7a2dc4e98ae12c5ec82ddff3a721dcde7bf01f0090d18a006d24803c2", - "dialect": "openai_chat", - "executable_sha256": "aabf94572866f85132156953d5d5c36a6c7a39ea9456a4eade7fc0df957685fa", - "platform": "windows", - "provenance": "pinned_real_cli_capture", - "request": { - "framing": "sse", - "media_type": "application/json", - "method": "POST", - "path": "/v1/chat/completions", - "request_shape": { - "has_tools": true, - "stream": true, - "tool_names": [ - "read_files", - "search_codebase", - "run_commands", - "fetch_web_content", - "editor", - "ask_question", - "spawn_agent", - "team_spawn_teammate", - "team_shutdown_teammate", - "team_status", - "team_task", - "team_run_task", - "team_cancel_run", - "team_list_runs", - "team_await_runs", - "team_send_message", - "team_broadcast", - "team_read_mailbox", - "team_mission_log", - "team_cleanup", - "team_create_outcome", - "team_attach_outcome_fragment", - "team_review_outcome_fragment", - "team_finalize_outcome", - "team_list_outcomes" - ], - "top_level_keys": [ - "messages", - "model", - "stream", - "stream_options", - "tool_choice", - "tools" - ] - }, - "transport": "http1" - }, - "response": { - "acceptance": "nonce_tool_result_round_trip", - "encoder": "openai_chat", - "framing": "sse", - "tool_call": "native_shell" - }, - "runtime": "native", - "version": "3.0.46" - }, { "agent": "codex", "capture_sha256": "5cec7ac20d1e1c550fc1fd1fc5cb96ef1e04b13d1212ea51125eaf2349460788", diff --git a/labs/15-pitot/tests/install_real_agent.py b/labs/15-pitot/tests/install_real_agent.py index 54e3e5f52..a8b1170c9 100755 --- a/labs/15-pitot/tests/install_real_agent.py +++ b/labs/15-pitot/tests/install_real_agent.py @@ -36,8 +36,16 @@ def install(agent: dict[str, object], platform: str, runtime: str) -> None: if runtime == "wsl": if platform != "windows" or agent["id"] != "cursor": raise ValueError("WSL is reserved for Cursor on Windows") - script = f"curl -fsSL https://cursor.com/install -o /tmp/cursor-install.sh && CURSOR_VERSION={version} bash /tmp/cursor-install.sh" - run(["wsl.exe", "--distribution", "Ubuntu", "--", "bash", "-lc", script]) + url = f"{package}/linux/x64/agent-cli-package.tar.gz" + archive = f"/tmp/pitot-cursor-{version}.tar.gz" + release = f"/usr/local/share/pitot-cursor/{version}" + wsl = ["wsl.exe", "--distribution", "Ubuntu", "--"] + run([*wsl, "curl", "-fsSL", url, "-o", archive]) + run([*wsl, "mkdir", "-p", release, "/usr/local/bin"]) + run([*wsl, "tar", "-xzf", archive, "-C", release, "--strip-components=1"]) + run([*wsl, "chmod", "0755", f"{release}/cursor-agent"]) + run([*wsl, "ln", "-sfn", f"{release}/cursor-agent", "/usr/local/bin/agent"]) + run([*wsl, "/usr/local/bin/agent", "--version"]) return if kind == "npm": run([npm_executable(), "install", "--global", "--ignore-scripts=false", f"{package}@{version}"]) diff --git a/labs/15-pitot/tests/model_control_proxy.py b/labs/15-pitot/tests/model_control_proxy.py index eadb0d6af..2d7ab3609 100755 --- a/labs/15-pitot/tests/model_control_proxy.py +++ b/labs/15-pitot/tests/model_control_proxy.py @@ -16,6 +16,38 @@ def contains(value: object, needle: str) -> bool: return needle in json.dumps(value, separators=(",", ":"), ensure_ascii=True) +def tool_result_contains(protocol: str, body: dict[str, object], needle: str) -> bool: + """Search only native tool-result fields, never echoed tool-call arguments.""" + if protocol == "gemini_generate_content": + values = [] + for content in body.get("contents", []): + if not isinstance(content, dict): + continue + for part in content.get("parts", []): + if isinstance(part, dict) and isinstance(part.get("functionResponse"), dict): + values.append(part["functionResponse"].get("response")) + return any(contains(value, needle) for value in values) + if protocol in {"anthropic_messages", "openai_chat"}: + values = [] + for message in body.get("messages", []): + if not isinstance(message, dict): + continue + if protocol == "openai_chat" and message.get("role") == "tool": + values.append(message.get("content")) + for item in message.get("content", []) if isinstance(message.get("content"), list) else []: + if isinstance(item, dict) and item.get("type") == "tool_result": + values.append(item.get("content")) + return any(contains(value, needle) for value in values) + if protocol == "openai_responses": + values = [ + item.get("output") + for item in body.get("input", []) + if isinstance(item, dict) and item.get("type") == "function_call_output" + ] + return any(contains(value, needle) for value in values) + return False + + class UnknownProtocol(ValueError): pass @@ -174,6 +206,10 @@ def __init__(self, args: argparse.Namespace) -> None: "initial_prompt_observed": False, "tool_call_response_emitted": False, "tool_result_observed": False, + "allow_tool_call_response_emitted": False, + "allow_tool_result_observed": False, + "deny_tool_call_response_emitted": False, + "denied_result_observed": False, "final_response_emitted": False, "selected_tool": None, "endpoint_observed": None, @@ -189,30 +225,30 @@ def save(self) -> None: temporary.replace(target) -def anthropic(tool: str, command: str, final: bool) -> dict[str, object]: - content = [{"type": "text", "text": "Pitot E2E Verification Complete"}] if final else [ - {"type": "tool_use", "id": "pitot_tool_1", "name": tool, "input": tool_arguments(tool, command)} +def anthropic(tool: str, command: str, final: bool, nonce: str = "", phase: str = "allow") -> dict[str, object]: + content = [{"type": "text", "text": f"PITOT_E2E_COMPLETE {nonce}"}] if final else [ + {"type": "tool_use", "id": f"pitot_tool_{phase}", "name": tool, "input": tool_arguments(tool, command)} ] return {"id": "msg_pitot", "type": "message", "role": "assistant", "model": "pitot-control", "content": content, "stop_reason": "end_turn" if final else "tool_use", "usage": {"input_tokens": 1, "output_tokens": 1}} -def chat(tool: str, command: str, final: bool) -> dict[str, object]: - message: dict[str, object] = {"role": "assistant", "content": "Pitot E2E Verification Complete" if final else None} +def chat(tool: str, command: str, final: bool, nonce: str = "", phase: str = "allow") -> dict[str, object]: + message: dict[str, object] = {"role": "assistant", "content": f"PITOT_E2E_COMPLETE {nonce}" if final else None} finish = "stop" if final else "tool_calls" if not final: - message["tool_calls"] = [{"id": "pitot_tool_1", "type": "function", "function": {"name": tool, "arguments": json.dumps(tool_arguments(tool, command))}}] + message["tool_calls"] = [{"id": f"pitot_tool_{phase}", "type": "function", "function": {"name": tool, "arguments": json.dumps(tool_arguments(tool, command))}}] return {"id": "chatcmpl-pitot", "object": "chat.completion", "created": 1, "model": "pitot-control", "choices": [{"index": 0, "message": message, "finish_reason": finish}], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}} -def responses(tool: str, command: str, final: bool) -> dict[str, object]: - output = [{"id": "msg_pitot", "type": "message", "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": "Pitot E2E Verification Complete", "annotations": []}]}] if final else [ - {"id": "fc_pitot", "type": "function_call", "call_id": "pitot_tool_1", "name": tool, "arguments": json.dumps(tool_arguments(tool, command)), "status": "completed"} +def responses(tool: str, command: str, final: bool, nonce: str = "", phase: str = "allow") -> dict[str, object]: + output = [{"id": "msg_pitot", "type": "message", "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": f"PITOT_E2E_COMPLETE {nonce}", "annotations": []}]}] if final else [ + {"id": f"fc_pitot_{phase}", "type": "function_call", "call_id": f"pitot_tool_{phase}", "name": tool, "arguments": json.dumps(tool_arguments(tool, command)), "status": "completed"} ] return {"id": "resp_pitot", "object": "response", "created_at": 1, "status": "completed", "model": "pitot-control", "output": output, "parallel_tool_calls": False, "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}} -def gemini(tool: str, command: str, final: bool) -> dict[str, object]: - part = {"text": "Pitot E2E Verification Complete"} if final else {"functionCall": {"name": tool, "args": {"command": command}}, "thoughtSignature": "cGl0b3Q="} +def gemini(tool: str, command: str, final: bool, nonce: str = "", phase: str = "allow") -> dict[str, object]: + part = {"text": f"PITOT_E2E_COMPLETE {nonce}"} if final else {"functionCall": {"name": tool, "args": tool_arguments(tool, command)}, "thoughtSignature": "cGl0b3Q="} return {"candidates": [{"content": {"role": "model", "parts": [part]}, "finishReason": "STOP"}], "usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2}} @@ -273,7 +309,10 @@ def do_POST(self) -> None: if isinstance(body.get("model"), str): payload["model"] = body["model"] self.reply(payload, stream=bool(body.get("stream")), protocol=protocol) return - final = contains(body, f"PITOT_CANARY_RESULT {nonce}") + allow_result = tool_result_contains(protocol, body, f"PITOT_CANARY_RESULT PITOT_ALLOW {nonce}") + denied_result = tool_result_contains(protocol, body, f"PITOT_CONTROLLER_DENY {nonce}") + phase = "allow" + final = False if not state.receipt["initial_prompt_observed"]: if not contains(body, nonce): self.send_error(409, "initial request omitted session nonce") @@ -287,35 +326,44 @@ def do_POST(self) -> None: "framing": "sse" if bool(body.get("stream")) or "streamGenerateContent" in self.path or protocol == "openai_responses" else "json", "request_shape": request_shape(body), } - elif not final: + elif allow_result and not state.receipt["allow_tool_result_observed"]: + state.receipt["allow_tool_result_observed"] = True + state.receipt["tool_result_observed"] = True + phase = "deny" + elif denied_result and state.receipt["deny_tool_call_response_emitted"]: + state.receipt["denied_result_observed"] = True + phase = "final" + final = True + else: state.receipt["unexpected_request"] = { "path": self.path.split("?", 1)[0], "top_level_keys": sorted(body), "nonce_present": contains(body, nonce), - "canary_result_present": False, + "allow_result_present": allow_result, + "denied_result_present": denied_result, "function_responses": function_response_shapes(body), "contents": content_shapes(body), "messages": message_shapes(body), } state.save() - self.send_error(409, "second request omitted real canary tool result") + self.send_error(409, "agent request did not advance the supervised allow/deny trajectory") return tool = tool_name(body) state.receipt["selected_tool"] = tool state.receipt["advertised_tools"] = tool_names(body) state.receipt["tool_structures"] = [sorted(item) if isinstance(item, dict) else type(item).__name__ for item in body.get("tools", [])] state.receipt["tool_schemas"] = tool_schema_shapes(body) - command = f"pitot-e2e-canary {nonce}" + command = f"{state.args.canary_command} PITOT_{phase.upper()} {nonce}" if state.args.response_fault == "text" and not final: - if protocol == "anthropic_messages": payload = anthropic(tool, command, True) - elif protocol == "openai_responses": payload = responses(tool, command, True) - elif protocol == "gemini_generate_content": payload = gemini(tool, command, True) - else: payload = chat(tool, command, True) + if protocol == "anthropic_messages": payload = anthropic(tool, command, True, nonce, phase) + elif protocol == "openai_responses": payload = responses(tool, command, True, nonce, phase) + elif protocol == "gemini_generate_content": payload = gemini(tool, command, True, nonce, phase) + else: payload = chat(tool, command, True, nonce, phase) state.receipt["fault_response_emitted"] = "text" - elif protocol == "anthropic_messages": payload = anthropic(tool, command, final) - elif protocol == "openai_responses": payload = responses(tool, command, final) - elif protocol == "gemini_generate_content": payload = gemini(tool, command, final) - elif protocol == "openai_chat": payload = chat(tool, command, final) + elif protocol == "anthropic_messages": payload = anthropic(tool, command, final, nonce, phase) + elif protocol == "openai_responses": payload = responses(tool, command, final, nonce, phase) + elif protocol == "gemini_generate_content": payload = gemini(tool, command, final, nonce, phase) + elif protocol == "openai_chat": payload = chat(tool, command, final, nonce, phase) else: self.send_error(422, "unsupported protocol") return @@ -324,10 +372,12 @@ def do_POST(self) -> None: if state.args.response_fault == "text" and not final: pass elif final: - state.receipt["tool_result_observed"] = True state.receipt["final_response_emitted"] = True + elif phase == "deny": + state.receipt["deny_tool_call_response_emitted"] = True else: state.receipt["tool_call_response_emitted"] = True + state.receipt["allow_tool_call_response_emitted"] = True state.save() stream = bool(body.get("stream")) or "streamGenerateContent" in self.path or protocol == "openai_responses" self.reply(payload, stream=stream, protocol=protocol) @@ -419,6 +469,7 @@ def main() -> int: parser.add_argument("--nonce", required=True) parser.add_argument("--receipt", required=True) parser.add_argument("--ready-file", required=True) + parser.add_argument("--canary-command", default="pitot-e2e-canary") parser.add_argument("--response-fault", choices=("none", "text"), default="none") args = parser.parse_args() server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) diff --git a/labs/15-pitot/tests/real_agent_driver.py b/labs/15-pitot/tests/real_agent_driver.py index 9a400a73d..28590db2d 100755 --- a/labs/15-pitot/tests/real_agent_driver.py +++ b/labs/15-pitot/tests/real_agent_driver.py @@ -10,6 +10,7 @@ from pathlib import Path import secrets import re +import shlex import shutil import subprocess import sys @@ -41,6 +42,18 @@ def wsl_path(path: Path) -> str: return f"/mnt/{matched.group(1).lower()}{matched.group(2)}" +def windows_host() -> bool: + return os.name == "nt" + + +def render_canary_command(executable: str, receipt: str, native_windows: bool) -> str: + if native_windows: + executable = executable.replace("\\", "/") + receipt = receipt.replace("\\", "/") + return subprocess.list2cmdline([executable, "--role", "canary", "--receipt", receipt]) + return shlex.join([executable, "--role", "canary", "--receipt", receipt]) + + def configure( agent: str, home: Path, @@ -51,6 +64,7 @@ def configure( pitot_command: str, witness_receipt: str, nonce: str, + runtime_path: str, ) -> tuple[list[str], dict[str, str]]: env: dict[str, str] = { "HOME": str(home), @@ -60,25 +74,38 @@ def configure( "GEMINI_API_KEY": "pitot-local-only", "GOOGLE_API_KEY": "pitot-local-only", "PITOT_BIN": witness_command, + "PITOT_RUNTIME": runtime_path, } # Gemini, Codex, and Copilot execute command hooks through PowerShell on # Windows. A quoted executable is only a string there; the call operator is # required to invoke it. Explicit receipt arguments also survive the hosts' # intentionally reduced hook environments. - witness_invocation = f'& "{witness_command}"' if os.name == "nt" else f'"{witness_command}"' + witness_invocation = f'& "{witness_command}"' if windows_host() else f'"{witness_command}"' witnessed = ( f'{witness_invocation} --real-bin "{pitot_command}" ' f'--receipt "{witness_receipt}" --nonce "{nonce}"' ) + witnessed_direct = ( + f'"{witness_command}" --real-bin "{pitot_command}" ' + f'--receipt "{witness_receipt}" --nonce "{nonce}"' + ) prompt_flag: list[str] if agent == "claude": - write_json(home / ".claude/settings.json", hook_group("PreToolUse", "Bash", f'"{witness_command}" hook claude')) + write_json(home / ".claude/settings.json", hook_group("PreToolUse", "Bash", f'{witnessed_direct} hook claude --runtime "{runtime_path}"')) env["ANTHROPIC_BASE_URL"] = proxy env["CLAUDE_CODE_MAX_RETRIES"] = "0" prompt_flag = ["--print", "--dangerously-skip-permissions", "--tools", "Bash", "--model", "sonnet"] elif agent == "codex": - sink = "NUL" if os.name == "nt" else "/dev/null" - write_json(home / ".codex/hooks.json", hook_group("PreToolUse", "Bash", f'{witnessed} hook codex >{sink}')) + if windows_host(): + bridge = LAB / "integrations/codex/PreToolUse.ps1" + hook_command = ( + f'& "{bridge}" ' + f'-Pitot "{witness_command}" -RealBin "{pitot_command}" ' + f'-Receipt "{witness_receipt}" -Nonce "{nonce}" -Runtime "{runtime_path}"' + ) + else: + hook_command = f'{witnessed} hook codex --runtime "{runtime_path}" >/dev/null' + write_json(home / ".codex/hooks.json", hook_group("PreToolUse", "Bash", hook_command)) (home / ".codex/config.toml").write_text( f'model = "pitot-control"\nmodel_provider = "pitot"\n[model_providers.pitot]\nname = "Pitot local control"\nbase_url = "{proxy}/v1"\nenv_key = "OPENAI_API_KEY"\nwire_api = "responses"\n', encoding="utf-8", @@ -86,7 +113,15 @@ def configure( env["CODEX_HOME"] = str(home / ".codex") prompt_flag = ["exec", "--dangerously-bypass-approvals-and-sandbox", "--dangerously-bypass-hook-trust", "--model", "pitot-control"] elif agent == "copilot": - write_json(home / ".copilot/settings.json", hook_group("PreToolUse", "Bash", f'{witnessed} hook copilot')) + hooks = home / ".copilot/hooks" + hooks.mkdir(parents=True, exist_ok=True) + source = LAB / "integrations/copilot" / ("PreToolUse.ps1" if windows_host() else "PreToolUse") + target = hooks / source.name + shutil.copy2(source, target) + target.chmod(0o755) + hook_command = f'powershell -NoProfile -NonInteractive -File "{target}"' if windows_host() else str(target) + write_json(home / ".copilot/settings.json", hook_group("PreToolUse", "Bash", hook_command)) + env["PITOT_BIN"] = witness_command env.update({ "COPILOT_PROVIDER_BASE_URL": f"{proxy}/v1", "COPILOT_PROVIDER_API_KEY": "pitot-local-only", @@ -100,21 +135,46 @@ def configure( }) prompt_flag = ["--allow-all-tools", "--model", "gpt-4o", "--no-auto-update", "--no-remote"] elif agent == "cursor": - write_json(project / ".cursor/hooks.json", {"version": 1, "hooks": {"beforeShellExecution": [{"command": f'"{witness_command}" hook cursor'}]}}) + source = LAB / "integrations/cursor/beforeShellExecution" + target = project / ".cursor/hooks/beforeShellExecution" + target.parent.mkdir(parents=True, exist_ok=True) + # Cursor runs in WSL on the Windows cell. A Windows checkout may expose + # CRLF bytes through /mnt/, which turns the bridge shebang into + # `bash\r` and makes Cursor fail the hook closed before Pitot can run. + target.write_bytes(source.read_bytes().replace(b"\r\n", b"\n")) + target.chmod(0o755) + hook_command = ( + f'"{target}" "{witness_command}" "{pitot_command}" ' + f'"{witness_receipt}" "{nonce}" "{runtime_path}"' + ) + write_json(project / ".cursor/hooks.json", {"version": 1, "hooks": {"beforeShellExecution": [{"command": hook_command, "failClosed": True}]}}) # Cursor exposes an authless CLI mode for endpoint compatibility tests; # model inference is still supplied only by the pinned local endpoint. env["CURSOR_AGENT_CLI_AUTHLESS_MODE"] = "true" env["CURSOR_AUTH_TOKEN"] = "pitot-local-only" prompt_flag = ["--endpoint", proxy, "--print", "--force"] elif agent == "gemini": - # Gemini deliberately sanitizes the hook environment. Pass the witness - # receipt identity as explicit arguments so the real hook remains - # nonce-bound even under that isolation boundary. - witness = f"{witnessed} hook gemini" - settings = hook_group("BeforeTool", "run_shell_command", witness) + hooks = home / ".gemini/hooks" + hooks.mkdir(parents=True, exist_ok=True) + source = LAB / "integrations/gemini" / ("BeforeTool.ps1" if windows_host() else "BeforeTool") + target = hooks / source.name + shutil.copy2(source, target) + target.chmod(0o755) + if windows_host(): + hook_command = ( + f'& "{target}" ' + f'-Pitot "{witness_command}" -RealBin "{pitot_command}" ' + f'-Receipt "{witness_receipt}" -Nonce "{nonce}" -Runtime "{runtime_path}"' + ) + else: + hook_command = ( + f'"{target}" "{witness_command}" "{pitot_command}" ' + f'"{witness_receipt}" "{nonce}" "{runtime_path}"' + ) + settings = hook_group("BeforeTool", "run_shell_command", hook_command) settings["security"] = {"auth": {"selectedType": "gemini-api-key"}, "folderTrust": {"enabled": False}} write_json(home / ".gemini/settings.json", settings) - env.update({"GOOGLE_GEMINI_BASE_URL": proxy, "GEMINI_CLI_HOME": str(home), "GEMINI_CLI_TRUST_WORKSPACE": "true"}) + env.update({"GOOGLE_GEMINI_BASE_URL": proxy, "GEMINI_CLI_HOME": str(home), "GEMINI_CLI_TRUST_WORKSPACE": "true", "PITOT_BIN": witness_command}) prompt_flag = ["--skip-trust", "--approval-mode", "yolo", "--model", "pitot-control", "-p"] elif agent == "kimi": config = home / ".kimi-code/config.toml" @@ -133,39 +193,40 @@ def configure( '[[hooks]]\n' 'event = "PreToolUse"\n' 'matcher = ".*"\n' - f'command = {json.dumps(witness_command + " hook kimi")}\n', + f'command = {json.dumps(witnessed_direct + " hook kimi --runtime " + runtime_path)}\n', encoding="utf-8", ) env["KIMI_CODE_HOME"] = str(home / ".kimi-code") env["KIMI_CODE_NO_AUTO_UPDATE"] = "1" prompt_flag = [] elif agent == "qwen": - allow = '{"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":"allow","permissionDecisionReason":"Pitot accepted the shell action"}}' - if os.name == "nt": - hook_command = f'"{witness_command}" hook qwen >/dev/null && echo {allow}' + hooks = home / ".qwen/hooks" + hooks.mkdir(parents=True, exist_ok=True) + source = LAB / "integrations/qwen" / ("PreToolUse.cjs" if windows_host() else "PreToolUse") + target = hooks / source.name + shutil.copy2(source, target) + target.chmod(0o755) + if windows_host(): + hook_command = ( + f'node "{target}" "{witness_command}" "{pitot_command}" ' + f'"{witness_receipt}" "{nonce}" "{runtime_path}"' + ) else: - hook_command = f'"{witness_command}" hook qwen >/dev/null && printf \'%s\\n\' \'{allow}\'' + hook_command = ( + f'"{target}" "{witness_command}" "{pitot_command}" ' + f'"{witness_receipt}" "{nonce}" "{runtime_path}"' + ) settings = hook_group("PreToolUse", "^(Bash|run_shell_command)$", hook_command) settings["modelProviders"] = {"openai": {"protocol": "openai", "models": [{"id": "pitot-control", "name": "Pitot control", "envKey": "OPENAI_API_KEY", "baseUrl": f"{proxy}/v1"}]}} settings["security"] = {"auth": {"selectedType": "openai"}} settings["model"] = {"name": "pitot-control"} write_json(home / ".qwen/settings.json", settings) - env["OPENAI_API_KEY"] = "pitot-local-only" - prompt_flag = ["--model", "pitot-control", "--approval-mode", "yolo", "-p"] + env.update({"OPENAI_API_KEY": "pitot-local-only", "PITOT_BIN": witness_command}) + prompt_flag = ["--model", "pitot-control", "-y", "-p"] elif agent == "pi": extension = LAB / "integrations/pi/pitot.ts" write_json(home / ".pi/agent/models.json", {"providers": {"pitot": {"baseUrl": f"{proxy}/v1", "apiKey": "pitot-local-only", "api": "openai-completions", "models": [{"id": "pitot-control", "name": "Pitot control", "reasoning": False, "input": ["text"], "contextWindow": 32000, "maxTokens": 4096}]}}}) prompt_flag = ["--no-session", "--print", "--provider", "pitot", "--model", "pitot-control", "-e", str(extension)] - elif agent == "cline": - hooks = home / ".cline/hooks" - hooks.mkdir(parents=True, exist_ok=True) - source = LAB / "integrations/cline" / ("PreToolUse.ps1" if os.name == "nt" else "PreToolUse") - target = hooks / source.name - shutil.copy2(source, target) - target.chmod(0o755) - env["PITOT_BIN"] = witness_command - prompt_flag = ["--provider", "openai-compatible", "--key", "pitot-local-only", "--model", "pitot-control", "--data-dir", str(home / ".cline"), "--hooks-dir", str(hooks), "--auto-approve", "true"] - env["OPENAI_BASE_URL"] = f"{proxy}/v1" elif agent == "opencode": plugin = LAB / "integrations/opencode/pitot.ts" # Use the released binary's bundled OpenAI provider. A custom provider @@ -245,14 +306,56 @@ def validate_capture_fixture(record: dict[str, object]) -> dict[str, object]: } -def validate_receipts(agent: dict[str, object], platform: str, nonce: str, installation: dict[str, object], proxy_path: Path, witness_path: Path, exit_code: int, output: str, prompt: str, capture_output: Path | None = None) -> dict[str, object]: +def json_lines(path: Path) -> list[dict[str, object]]: + if not path.is_file(): + return [] + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def validate_receipts( + agent: dict[str, object], platform: str, nonce: str, installation: dict[str, object], + proxy_path: Path, witness_path: Path, controller_path: Path, consumer_path: Path, + canary_path: Path, runtime_identity: dict[str, object], exit_code: int, output: str, + prompt: str, capture_output: Path | None = None, +) -> dict[str, object]: proxy = json.loads(proxy_path.read_text(encoding="utf-8")) if proxy_path.is_file() else {"missing": True} - witness = json.loads(witness_path.read_text(encoding="utf-8")) if witness_path.is_file() else {"missing": True} - proxy_flags = ("initial_prompt_observed", "tool_call_response_emitted", "tool_result_observed", "final_response_emitted") - if exit_code != 0 or "hook: PreToolUse Failed" in output or not all(proxy.get(flag) is True for flag in proxy_flags): - raise RuntimeError(f"agent loop incomplete (exit={exit_code}, proxy={proxy}, witness={witness})\n{output[-4000:]}") - if proxy.get("nonce") != nonce or witness.get("nonce") != nonce or witness.get("host") != agent["id"] or witness.get("valid") is not True: - raise RuntimeError("proxy and Pitot witness receipts do not identify the same valid session") + witnesses = json_lines(witness_path) + controller = json_lines(controller_path) + consumers = json_lines(consumer_path) + canary = canary_path.read_text(encoding="utf-8").splitlines() if canary_path.is_file() else [] + proxy_flags = ( + "initial_prompt_observed", "allow_tool_call_response_emitted", "allow_tool_result_observed", + "deny_tool_call_response_emitted", "denied_result_observed", "final_response_emitted", + ) + final_marker = f"PITOT_E2E_COMPLETE {nonce}" + if exit_code != 0 or "hook: PreToolUse Failed" in output or final_marker not in output or not all(proxy.get(flag) is True for flag in proxy_flags): + raise RuntimeError(f"agent loop incomplete (exit={exit_code}, proxy={proxy}, witnesses={witnesses})\n{output[-4000:]}") + if proxy.get("nonce") != nonce or len(witnesses) != 2: + raise RuntimeError("proxy and Pitot witness receipts do not identify exactly two hook actions") + if any(item.get("nonce") != nonce or item.get("host") != agent["id"] or item.get("valid") is not True for item in witnesses): + raise RuntimeError("Pitot hook witnesses escaped the nonce-bound real-agent session") + if [item.get("pitot_exit") for item in witnesses] != [0, 2]: + raise RuntimeError(f"real hook did not carry one allow and one deny: {witnesses}") + action_ids = [item.get("action_id") for item in witnesses] + if len(set(action_ids)) != 2 or not all(isinstance(item, str) and item.startswith("act_") for item in action_ids): + raise RuntimeError("hook actions lack unique Pitot correlation ids") + requests = [item.get("value") for item in controller if item.get("receipt_type") == "request"] + responses = [item.get("value") for item in controller if item.get("receipt_type") == "response"] + if len(requests) != 2 or len(responses) != 2: + raise RuntimeError(f"Controller did not receive and resolve both hook actions: {controller}") + if [item.get("action_id") for item in requests] != action_ids or [item.get("action_id") for item in responses] != action_ids: + raise RuntimeError("Controller correlation ids do not match the real hook actions") + if [item.get("outcome") for item in responses] != ["allow", "deny"] or responses[1].get("message") != f"PITOT_CONTROLLER_DENY {nonce}": + raise RuntimeError("external Controller did not produce the nonce-bound allow/deny trajectory") + if len(consumers) != 2 or [item.get("action", {}).get("id") for item in consumers] != action_ids: + raise RuntimeError(f"passive Consumer did not receive both real hook observations: {consumers}") + if any(item.get("content", {}).get("mode") != "sha256" or "full" in item.get("content", {}) for item in consumers): + raise RuntimeError("Consumer projection did not remove full command content") + if canary != [f"PITOT_ALLOW {nonce}"]: + raise RuntimeError(f"canary execution count violated allow/deny control: {canary}") + runtime_public = {key: runtime_identity.get(key) for key in ("schema_version", "instance_id", "pid", "endpoint", "config_sha256")} + if runtime_public["schema_version"] != 1 or not all(runtime_public.get(key) for key in ("instance_id", "endpoint", "config_sha256")): + raise RuntimeError("authenticated Pitot runtime identity is incomplete") if installation.get("agent") != agent["id"] or installation.get("version") != agent["version"]: raise RuntimeError("installation receipt does not match supervised manifest") captured = capture_record(agent, platform, installation, proxy) @@ -270,15 +373,19 @@ def validate_receipts(agent: dict[str, object], platform: str, nonce: str, insta else: endpoint_evidence = validate_capture_fixture(captured) return { - "schema_version": 1, + "schema_version": 2, "agent": agent["id"], "cli": {"version": installation["version"], "executable": installation["executable"], "executable_sha256": installation["executable_sha256"], "installer": installation["installer"], "runtime": installation["runtime"]}, "prompt_hash": hashlib.sha256(prompt.encode()).hexdigest(), "protocol": proxy["protocol"], "endpoint": endpoint_evidence, "nonce": nonce, - "receipts": {**{flag: True for flag in proxy_flags}, "hook_observed": True, "canary_result_observed": True, "cli_exit_zero": True}, - "hook": {"host": witness["host"], "action_kind": witness["action_kind"], "pitot_exit": witness["pitot_exit"]}, + "receipts": {**{flag: True for flag in proxy_flags}, "consumer_observed": True, "controller_allow_observed": True, "controller_deny_observed": True, "deny_canary_absent": True, "final_output_observed": True, "cli_exit_zero": True}, + "runtime": runtime_public, + "hooks": [{"host": item["host"], "action_kind": item["action_kind"], "action_id": item["action_id"], "pitot_exit": item["pitot_exit"], "nonce": item["nonce"]} for item in witnesses], + "controller": {"id": "e2e-shell-controller", "action_ids": action_ids, "outcomes": ["allow", "deny"]}, + "consumer": {"id": "e2e-audit", "action_ids": action_ids, "projection": "sha256"}, + "canary": {"executions": canary, "denied_executions": 0}, } @@ -290,6 +397,7 @@ def main() -> int: parser.add_argument("--evidence", type=Path, required=True) parser.add_argument("--pitot", type=Path, required=True) parser.add_argument("--witness", type=Path, required=True) + parser.add_argument("--test-role", type=Path, required=True) parser.add_argument("--capture-output", type=Path) parser.add_argument("--response-fault", choices=("none", "text"), default="none") parser.add_argument("--expect-incompatible-response", action="store_true") @@ -304,9 +412,8 @@ def main() -> int: host_controls_wsl = runtime == "wsl" and os.name == "nt" home, project, bin_dir = base / "home", base / "project", base / "bin" home.mkdir(); project.mkdir(); bin_dir.mkdir() - canary = bin_dir / "pitot-e2e-canary" - canary.write_text('#!/usr/bin/env sh\nprintf "PITOT_CANARY_RESULT %s\\n" "$1"\n', encoding="utf-8"); canary.chmod(0o755) - (bin_dir / "pitot-e2e-canary.cmd").write_text("@echo off\r\necho PITOT_CANARY_RESULT %1\r\n", encoding="utf-8") + canary_receipt = base / "canary.jsonl" + canary_executable = args.test_role.resolve() cursor_system_canary = False if args.agent == "cursor" and host_controls_wsl: occupied = subprocess.run( @@ -316,17 +423,62 @@ def main() -> int: if occupied.returncode == 0: raise RuntimeError("refusing to replace an existing WSL canary command") subprocess.run( - ["wsl.exe", "--distribution", "Ubuntu", "--", "install", "-m", "0755", wsl_path(canary), "/usr/local/bin/pitot-e2e-canary"], + ["wsl.exe", "--distribution", "Ubuntu", "--", "install", "-m", "0755", wsl_path(canary_executable), "/usr/local/bin/pitot-e2e-canary"], check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) + canary_executable = Path("/usr/local/bin/pitot-e2e-canary") cursor_system_canary = True elif args.agent == "cursor" and os.name != "nt" and os.access("/usr/local/bin", os.W_OK): if Path("/usr/local/bin/pitot-e2e-canary").exists(): raise RuntimeError("refusing to replace an existing system canary command") - shutil.copy2(canary, "/usr/local/bin/pitot-e2e-canary") + shutil.copy2(canary_executable, "/usr/local/bin/pitot-e2e-canary") Path("/usr/local/bin/pitot-e2e-canary").chmod(0o755) + canary_executable = Path("/usr/local/bin/pitot-e2e-canary") cursor_system_canary = True - ready, proxy_receipt, witness_receipt = base / "proxy.url", base / "proxy.json", base / "witness.json" + receipt_argument = wsl_path(canary_receipt) if host_controls_wsl else str(canary_receipt) + # Forward slashes keep the same native Windows command valid in + # PowerShell, cmd.exe, and the Unix-like shells selected by hosts. + canary_command = render_canary_command( + str(canary_executable), receipt_argument, windows_host() and not host_controls_wsl, + ) + ready, proxy_receipt, witness_receipt = base / "proxy.url", base / "proxy.json", base / "witness.jsonl" + runtime_descriptor = base / "runtime.json" + runtime_config = base / "pitot.json" + consumer_receipt = base / "consumer.jsonl" + controller_receipt = base / "controller.jsonl" + runtime_config.write_text(json.dumps({ + "consumers": [{ + "id": "e2e-audit", + "command": [str(args.test_role.resolve()), "--role", "consumer", "--receipt", str(consumer_receipt)], + "events": ["action.requested"], + "projection": {"content": "sha256"}, + }], + "controllers": { + "shell": { + "id": "e2e-shell-controller", + "command": [str(args.test_role.resolve()), "--role", "controller", "--id", "e2e-shell-controller", "--receipt", str(controller_receipt), "--nonce", nonce], + "deadline_ms": 5000, + "on_timeout": "deny", + "on_unavailable": "deny", + } + }, + }, indent=2) + "\n", encoding="utf-8") + runtime_process = subprocess.Popen( + [str(args.pitot.resolve()), "run", "--config", str(runtime_config), "--runtime", str(runtime_descriptor)], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + for _ in range(600): + if runtime_descriptor.is_file(): break + if runtime_process.poll() is not None: + output = runtime_process.stdout.read() if runtime_process.stdout else "" + raise RuntimeError(f"Pitot runtime exited before becoming ready: {output.strip()}") + time.sleep(0.05) + else: + runtime_process.terminate() + raise RuntimeError("Pitot runtime did not publish its authenticated descriptor") + runtime_identity = json.loads(runtime_descriptor.read_text(encoding="utf-8")) if args.agent == "cursor": if host_controls_wsl: # Keep Cursor and its HTTP/2 Connect control proxy in the same @@ -338,12 +490,13 @@ def main() -> int: "--nonce", nonce, "--receipt", wsl_path(proxy_receipt), "--ready-file", wsl_path(ready), + "--canary-command", canary_command, "--response-fault", args.response_fault, ] else: - proxy_command = ["node", str(LAB / "tests/cursor_control_proxy.mjs"), "--nonce", nonce, "--receipt", str(proxy_receipt), "--ready-file", str(ready), "--response-fault", args.response_fault] + proxy_command = ["node", str(LAB / "tests/cursor_control_proxy.mjs"), "--nonce", nonce, "--receipt", str(proxy_receipt), "--ready-file", str(ready), "--canary-command", canary_command, "--response-fault", args.response_fault] else: - proxy_command = [sys.executable, str(LAB / "tests/model_control_proxy.py"), "--agent", args.agent, "--nonce", nonce, "--receipt", str(proxy_receipt), "--ready-file", str(ready), "--response-fault", args.response_fault] + proxy_command = [sys.executable, str(LAB / "tests/model_control_proxy.py"), "--agent", args.agent, "--nonce", nonce, "--receipt", str(proxy_receipt), "--ready-file", str(ready), "--canary-command", canary_command, "--response-fault", args.response_fault] proxy_process = subprocess.Popen( proxy_command, stdout=subprocess.PIPE, @@ -374,16 +527,12 @@ def main() -> int: pitot_command=pitot_command, witness_receipt=receipt_command, nonce=nonce, + runtime_path=str(runtime_descriptor), ) - environment = {**os.environ, **extra_env, "PATH": str(bin_dir) + os.pathsep + os.environ.get("PATH", ""), "PITOT_REAL_BIN": str(args.pitot.resolve()), "PITOT_WITNESS_RECEIPT": str(witness_receipt), "PITOT_E2E_NONCE": nonce} + environment = {**os.environ, **extra_env, "PATH": str(bin_dir) + os.pathsep + os.environ.get("PATH", ""), "PITOT_REAL_BIN": str(args.pitot.resolve()), "PITOT_WITNESS_RECEIPT": str(witness_receipt), "PITOT_E2E_NONCE": nonce, "PITOT_RUNTIME": str(runtime_descriptor)} executable = installation["executable"] if args.agent == "cursor" and not host_controls_wsl: prepare_cursor_keychain(home, environment) - if args.agent == "cline" and not host_controls_wsl: - subprocess.run( - [executable, "auth", "--provider", "openai-compatible", "--apikey", "pitot-local-only", "--modelid", "pitot-control", "--baseurl", f"{proxy}/v1", "--data-dir", str(home / ".cline")], - cwd=project, env=environment, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - ) if host_controls_wsl: wsl_environment = { **extra_env, @@ -393,6 +542,7 @@ def main() -> int: "PITOT_REAL_BIN": wsl_path(args.pitot), "PITOT_WITNESS_RECEIPT": wsl_path(witness_receipt), "PITOT_E2E_NONCE": nonce, + "PITOT_RUNTIME": wsl_path(runtime_descriptor), } executable_dir = executable.rsplit("/", 1)[0] wsl_environment["PATH"] = ":".join((wsl_path(bin_dir), executable_dir, "/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin")) @@ -411,7 +561,11 @@ def main() -> int: except subprocess.TimeoutExpired as error: proxy_state = proxy_receipt.read_text(encoding="utf-8") if proxy_receipt.is_file() else '{"missing":true}' witness_state = witness_receipt.read_text(encoding="utf-8") if witness_receipt.is_file() else '{"missing":true}' - raise RuntimeError(f"released agent timed out; proxy={proxy_state}; witness={witness_state}") from error + partial_output = error.stdout or "" + raise RuntimeError( + f"released agent timed out; proxy={proxy_state}; witness={witness_state}; " + f"output={partial_output[-4000:]}" + ) from error # GitHub's Windows Python console defaults to CP-1252 while several # released CLIs emit Unicode status glyphs. Emit UTF-8 bytes so the # reporting layer cannot fail after a successful agent session. @@ -429,7 +583,11 @@ def main() -> int: raise RuntimeError("incompatible response unexpectedly entered the hook/tool path") print("PITOT_INCOMPATIBLE_RESPONSE_REJECTED evidence=binary-observed") return 0 - evidence = validate_receipts(agent, args.platform, nonce, installation, proxy_receipt, witness_receipt, completed.returncode, completed.stdout, prompt, args.capture_output) + evidence = validate_receipts( + agent, args.platform, nonce, installation, proxy_receipt, witness_receipt, + controller_receipt, consumer_receipt, canary_receipt, runtime_identity, + completed.returncode, completed.stdout, prompt, args.capture_output, + ) args.evidence.parent.mkdir(parents=True, exist_ok=True) args.evidence.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8") print("PITOT_E2E_RESULT mode=real_cli evidence=nonce-correlated") @@ -437,6 +595,9 @@ def main() -> int: proxy_process.terminate() try: proxy_process.wait(timeout=5) except subprocess.TimeoutExpired: proxy_process.kill() + runtime_process.terminate() + try: runtime_process.wait(timeout=5) + except subprocess.TimeoutExpired: runtime_process.kill() if cursor_system_canary: if host_controls_wsl: subprocess.run( diff --git a/labs/15-pitot/tests/run_e2e_report.py b/labs/15-pitot/tests/run_e2e_report.py index d5ae95bd2..5e34938d7 100755 --- a/labs/15-pitot/tests/run_e2e_report.py +++ b/labs/15-pitot/tests/run_e2e_report.py @@ -18,22 +18,32 @@ AGENTS = {agent["id"] for agent in MANIFEST["agents"]} PLATFORMS = {platform["id"] for platform in MANIFEST["platforms"]} RESULT_PATTERN = re.compile(r"^PITOT_E2E_RESULT mode=real_cli evidence=nonce-correlated$", re.MULTILINE) +RUNTIME_RESULT_PATTERN = re.compile(r"^PITOT_RUNTIME_E2E_RESULT capability=explicit_request evidence=nonce-correlated$", re.MULTILINE) def load_evidence(path: Path | None, *, agent: str) -> dict[str, object] | None: if path is None or not path.is_file(): return None value = json.loads(path.read_text(encoding="utf-8")) - required = {"schema_version", "agent", "cli", "prompt_hash", "protocol", "endpoint", "nonce", "receipts", "hook"} - if not isinstance(value, dict) or set(value) != required or value["schema_version"] != 1 or value["agent"] != agent: + required = {"schema_version", "agent", "cli", "prompt_hash", "protocol", "endpoint", "nonce", "receipts", "runtime", "hooks", "controller", "consumer", "canary"} + if not isinstance(value, dict) or set(value) != required or value["schema_version"] != 2 or value["agent"] != agent: return None receipts = value.get("receipts") - receipt_fields = {"initial_prompt_observed", "tool_call_response_emitted", "tool_result_observed", "final_response_emitted", "hook_observed", "canary_result_observed", "cli_exit_zero"} + receipt_fields = { + "initial_prompt_observed", "allow_tool_call_response_emitted", "allow_tool_result_observed", + "deny_tool_call_response_emitted", "denied_result_observed", "final_response_emitted", + "consumer_observed", "controller_allow_observed", "controller_deny_observed", + "deny_canary_absent", "final_output_observed", "cli_exit_zero", + } if not isinstance(receipts, dict) or set(receipts) != receipt_fields or not all(item is True for item in receipts.values()): return None if not re.fullmatch(r"[0-9a-f]{64}", str(value.get("prompt_hash", ""))): return None - if value.get("hook", {}).get("action_kind") != "shell" or value.get("hook", {}).get("host") != agent: + nonce = value.get("nonce") + if not isinstance(nonce, str) or not re.fullmatch(r"[0-9a-f]{32}", nonce): + return None + hooks = value.get("hooks") + if not isinstance(hooks, list) or len(hooks) != 2 or [item.get("pitot_exit") for item in hooks] != [0, 2] or any(item.get("action_kind") != "shell" or item.get("host") != agent or item.get("nonce") != nonce for item in hooks): return None endpoint = value.get("endpoint", {}) endpoint_required = {"fixture", "fixture_sha256", "provenance", "dialect", "request", "response", "executable_sha256"} @@ -44,6 +54,24 @@ def load_evidence(path: Path | None, *, agent: str) -> dict[str, object] | None: return value +def load_runtime_evidence(path: Path | None, *, platform: str) -> dict[str, object] | None: + if path is None or not path.is_file(): + return None + value = json.loads(path.read_text(encoding="utf-8")) + required = {"schema_version", "capability", "platform", "nonce", "runtime", "controller", "receipts", "commit_sha"} + if not isinstance(value, dict) or set(value) != required or value["schema_version"] != 1 or value["capability"] != "explicit_request" or value["platform"] != platform: + return None + if not isinstance(value["nonce"], str) or not re.fullmatch(r"[0-9a-f]{32}", value["nonce"]): + return None + receipts = value.get("receipts") + if not isinstance(receipts, dict) or set(receipts) != {"request_allow_observed", "request_deny_observed", "correlation_observed"} or not all(item is True for item in receipts.values()): + return None + controller = value.get("controller") + if not isinstance(controller, dict) or controller.get("outcomes") != ["allow", "deny"] or len(controller.get("action_ids", [])) != 2: + return None + return value + + def result_for(agent: str, platform: str, returncode: int, output: str, evidence_path: Path | None = None) -> dict[str, object]: if agent not in AGENTS: raise ValueError(f"unsupported agent: {agent}") @@ -53,7 +81,7 @@ def result_for(agent: str, platform: str, returncode: int, output: str, evidence markers = RESULT_PATTERN.findall(output) receipt = load_evidence(evidence_path, agent=agent) passed = returncode == 0 and len(markers) == 1 and receipt is not None - evidence = "binary-observed request, accepted response, hook, canary, and final receipts" if passed else "real-agent evidence contract failed" + evidence = "binary-observed request, real hook control, projected Consumer, allow/deny canary, and final receipts" if passed else "real-agent control evidence contract failed" return { "schema_version": 2, @@ -66,8 +94,13 @@ def result_for(agent: str, platform: str, returncode: int, output: str, evidence "protocol": receipt["protocol"] if passed else None, "endpoint": receipt["endpoint"] if passed else None, "prompt_hash": receipt["prompt_hash"] if passed else None, + "nonce": receipt["nonce"] if passed else None, "receipts": receipt["receipts"] if passed else None, - "hook": receipt["hook"] if passed else None, + "runtime": receipt["runtime"] if passed else None, + "hooks": receipt["hooks"] if passed else None, + "controller": receipt["controller"] if passed else None, + "consumer": receipt["consumer"] if passed else None, + "canary": receipt["canary"] if passed else None, "commit_sha": os.environ.get("PITOT_SOURCE_SHA", os.environ.get("GITHUB_SHA", "local")), "run_url": ( f"{os.environ['GITHUB_SERVER_URL']}/{os.environ['GITHUB_REPOSITORY']}" @@ -81,9 +114,38 @@ def result_for(agent: str, platform: str, returncode: int, output: str, evidence } +def result_for_runtime(platform: str, returncode: int, output: str, evidence_path: Path | None = None) -> dict[str, object]: + if platform not in PLATFORMS: + raise ValueError(f"unsupported platform: {platform}") + markers = RUNTIME_RESULT_PATTERN.findall(output) + receipt = load_runtime_evidence(evidence_path, platform=platform) + passed = returncode == 0 and len(markers) == 1 and receipt is not None + return { + "schema_version": 2, + "capability": "explicit_request", + "platform": platform, + "status": "pass" if passed else "fail", + "verification_mode": "real_runtime" if passed else None, + "evidence": "real request CLI, authenticated runtime, and correlated allow/deny Controller receipts" if passed else "explicit request evidence contract failed", + "nonce": receipt["nonce"] if passed else None, + "runtime": receipt["runtime"] if passed else None, + "controller": receipt["controller"] if passed else None, + "receipts": receipt["receipts"] if passed else None, + "commit_sha": os.environ.get("PITOT_SOURCE_SHA", os.environ.get("GITHUB_SHA", "local")), + "run_url": ( + f"{os.environ['GITHUB_SERVER_URL']}/{os.environ['GITHUB_REPOSITORY']}" + f"/actions/runs/{os.environ['GITHUB_RUN_ID']}" + if all(key in os.environ for key in ("GITHUB_SERVER_URL", "GITHUB_REPOSITORY", "GITHUB_RUN_ID")) + else "local" + ), + } + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--agent", required=True, choices=sorted(AGENTS)) + target = parser.add_mutually_exclusive_group(required=True) + target.add_argument("--agent", choices=sorted(AGENTS)) + target.add_argument("--capability", choices=("explicit_request",)) parser.add_argument("--platform", required=True, choices=sorted(PLATFORMS)) parser.add_argument("--output", required=True, type=Path) parser.add_argument("--evidence", required=True, type=Path) @@ -102,7 +164,10 @@ def main() -> int: stderr=subprocess.STDOUT, ) sys.stdout.buffer.write(completed.stdout.encode("utf-8", errors="replace")) - result = result_for(args.agent, args.platform, completed.returncode, completed.stdout, args.evidence) + result = ( + result_for(args.agent, args.platform, completed.returncode, completed.stdout, args.evidence) + if args.agent else result_for_runtime(args.platform, completed.returncode, completed.stdout, args.evidence) + ) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") return 0 if result["status"] == "pass" else 1 diff --git a/labs/15-pitot/tests/runtime_capability_driver.py b/labs/15-pitot/tests/runtime_capability_driver.py new file mode 100644 index 000000000..8ad322726 --- /dev/null +++ b/labs/15-pitot/tests/runtime_capability_driver.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Prove the real Pitot request CLI against its authenticated runtime.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import secrets +import subprocess +import tempfile +import time + + +def lines(path: Path) -> list[dict[str, object]]: + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--pitot", type=Path, required=True) + parser.add_argument("--test-role", type=Path, required=True) + parser.add_argument("--platform", required=True) + parser.add_argument("--evidence", type=Path, required=True) + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix="pitot-request-e2e-") as temporary: + nonce = secrets.token_hex(16) + base = Path(temporary) + runtime_path = base / "runtime.json" + receipt = base / "controller.jsonl" + config = base / "pitot.json" + config.write_text(json.dumps({"controllers": {"release.approval": { + "id": "e2e-release-controller", + "command": [str(args.test_role.resolve()), "--role", "controller", "--id", "e2e-release-controller", "--receipt", str(receipt), "--nonce", nonce], + "deadline_ms": 2000, + "on_timeout": "deny", + "on_unavailable": "deny", + }}}) + "\n", encoding="utf-8") + process = subprocess.Popen( + [str(args.pitot.resolve()), "run", "--config", str(config), "--runtime", str(runtime_path)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + ) + try: + for _ in range(300): + if runtime_path.is_file(): break + if process.poll() is not None: + raise RuntimeError(f"runtime exited: {process.stdout.read() if process.stdout else ''}") + time.sleep(0.02) + else: + raise RuntimeError("runtime descriptor was not published") + runtime = json.loads(runtime_path.read_text(encoding="utf-8")) + allow = subprocess.run( + [str(args.pitot.resolve()), "request", "release.approval", "--data", json.dumps({"phase": "PITOT_ALLOW", "nonce": nonce}), "--runtime", str(runtime_path)], + text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) + deny = subprocess.run( + [str(args.pitot.resolve()), "request", "release.approval", "--data", json.dumps({"phase": "PITOT_DENY", "nonce": nonce}), "--runtime", str(runtime_path)], + text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) + allow_value = json.loads(allow.stdout) + deny_value = json.loads(deny.stdout) + controller = lines(receipt) + requests = [item["value"] for item in controller if item.get("receipt_type") == "request"] + responses = [item["value"] for item in controller if item.get("receipt_type") == "response"] + if allow.returncode != 0 or deny.returncode != 2: + raise RuntimeError(f"request exits did not encode allow/deny: {allow.returncode}, {deny.returncode}") + if [allow_value.get("outcome"), deny_value.get("outcome")] != ["allow", "deny"]: + raise RuntimeError("request output did not carry Controller outcomes") + action_ids = [item.get("action_id") for item in requests] + if len(requests) != 2 or len(responses) != 2 or [item.get("action_id") for item in responses] != action_ids: + raise RuntimeError("request and response correlation receipts disagree") + if any(nonce not in json.dumps(item.get("data"), sort_keys=True) for item in requests): + raise RuntimeError("request receipts are not bound to the session nonce") + public_runtime = {key: runtime[key] for key in ("schema_version", "instance_id", "pid", "endpoint", "config_sha256")} + evidence = { + "schema_version": 1, + "capability": "explicit_request", + "platform": args.platform, + "nonce": nonce, + "runtime": public_runtime, + "controller": {"id": "e2e-release-controller", "action_ids": action_ids, "outcomes": ["allow", "deny"]}, + "receipts": {"request_allow_observed": True, "request_deny_observed": True, "correlation_observed": True}, + "commit_sha": os.environ.get("PITOT_SOURCE_SHA", os.environ.get("GITHUB_SHA", "local")), + } + args.evidence.parent.mkdir(parents=True, exist_ok=True) + args.evidence.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print("PITOT_RUNTIME_E2E_RESULT capability=explicit_request evidence=nonce-correlated") + finally: + process.terminate() + try: process.wait(timeout=5) + except subprocess.TimeoutExpired: process.kill() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/labs/15-pitot/tests/test_adapter_supervisor.py b/labs/15-pitot/tests/test_adapter_supervisor.py index 2ee08d07e..2cc15fa67 100644 --- a/labs/15-pitot/tests/test_adapter_supervisor.py +++ b/labs/15-pitot/tests/test_adapter_supervisor.py @@ -49,6 +49,11 @@ def _fixture(root: Path) -> tuple[Path, list[str]]: ids = [agent["id"] for agent in manifest["agents"]] for agent_id in ids: (tests / f"e2e_{agent_id}_cli_test.sh").write_text("#!/usr/bin/env bash\n") + for agent in manifest["agents"]: + for artifact in agent["artifacts"]: + path = root / "labs/15-pitot" / artifact + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("# supervised integration artifact\n") (tests / "real_agent_driver.py").write_text("# canonical driver\n") (tests / "model_control_proxy.py").write_text("# proxy\n") (tests / "cursor_control_proxy.mjs").write_text("// cursor proxy\n") @@ -85,16 +90,46 @@ def test_real_repository_contract_is_complete(self): def test_matrix_contains_every_agent_on_every_platform(self): manifest = _manifest() matrix = supervisor.matrix(manifest) - self.assertEqual(len(matrix), 30) + self.assertEqual(len(matrix), 27) self.assertEqual( {(item["agent"], item["platform"]) for item in matrix if item["agent"] == "kimi"}, {("kimi", "ubuntu"), ("kimi", "macos"), ("kimi", "windows")}, ) self.assertEqual( - {(item["agent"], item["platform"]) for item in matrix if item["agent"] in {"cline", "copilot", "pi", "qwen"}}, - {(agent, platform) for agent in {"cline", "copilot", "pi", "qwen"} for platform in {"ubuntu", "macos", "windows"}}, + {(item["agent"], item["platform"]) for item in matrix if item["agent"] in {"copilot", "pi", "qwen"}}, + {(agent, platform) for agent in {"copilot", "pi", "qwen"} for platform in {"ubuntu", "macos", "windows"}}, + ) + runtime = supervisor.runtime_matrix(manifest) + self.assertEqual( + runtime, + [ + {"capability": "explicit_request", "platform": "ubuntu", "runner": "ubuntu-latest"}, + {"capability": "explicit_request", "platform": "macos", "runner": "macos-latest"}, + {"capability": "explicit_request", "platform": "windows", "runner": "windows-latest"}, + ], ) + def test_capability_inventory_is_exact(self): + manifest = copy.deepcopy(_manifest()) + manifest["capabilities"] = manifest["capabilities"][:-1] + with self.assertRaisesRegex(supervisor.ContractError, "capabilities must exactly supervise"): + supervisor.validate_manifest(manifest) + manifest = copy.deepcopy(_manifest()) + manifest["capabilities"].append({"id": "phantom", "matrix": "platform"}) + with self.assertRaisesRegex(supervisor.ContractError, "capabilities must exactly supervise"): + supervisor.validate_manifest(manifest) + + def test_missing_integration_artifact_fails(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + _, ids = _fixture(root) + manifest = json.loads((root / supervisor.MANIFEST).read_text()) + agent = next(item for item in manifest["agents"] if item["artifacts"]) + artifact = agent["artifacts"][0] + (root / "labs/15-pitot" / artifact).unlink() + errors = supervisor.contract_errors(root, ids) + self.assertIn(f"adapter {agent['id']} is missing supervised artifact {artifact}", errors) + def test_missing_and_extra_inventory_entries_fail(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) @@ -111,7 +146,13 @@ def test_missing_and_extra_inventory_entries_fail(self): self.assertTrue(any("missing from manifest" in error for error in errors)) phantom = copy.deepcopy(value["agents"][0]) - phantom.update({"id": "phantom", "label": "Phantom", "executable": "phantom"}) + phantom.update({ + "id": "phantom", + "label": "Phantom", + "executable": "phantom", + "artifacts": [], + "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, + }) value["agents"].append(phantom) manifest_path.write_text(json.dumps(value)) provenance = _provenance(value) @@ -178,7 +219,7 @@ def test_platform_swap_and_fabricated_capture_digest_fail(self): fixture_path.write_text(json.dumps(fixture)) self.assertTrue(any("fabricated capture digest" in error for error in supervisor.contract_errors(root, ids))) - def test_capture_merge_requires_all_30_accepted_unique_cells(self): + def test_capture_merge_requires_all_27_accepted_unique_cells(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) _fixture(root) @@ -190,9 +231,9 @@ def test_capture_merge_requires_all_30_accepted_unique_cells(self): (captures / f"{cell['agent']}-{cell['platform']}.json").write_text(json.dumps({"schema_version": 1, "accepted": True, "cell": cell})) output = root / "merged.json" supervisor.merge_captures(root, captures, output) - self.assertEqual(len(json.loads(output.read_text())["cells"]), 30) + self.assertEqual(len(json.loads(output.read_text())["cells"]), 27) next(captures.glob("*.json")).unlink() - with self.assertRaisesRegex(supervisor.ContractError, "exactly all 30"): + with self.assertRaisesRegex(supervisor.ContractError, "exactly all 27"): supervisor.merge_captures(root, captures, output) def test_capture_merge_rejects_mixed_versions_and_duplicates(self): @@ -206,7 +247,7 @@ def test_capture_merge_rejects_mixed_versions_and_duplicates(self): (captures / f"{index}.json").write_text(json.dumps({"schema_version": 1, "accepted": True, "cell": cell})) duplicate = provenance["cells"][0] (captures / "duplicate.json").write_text(json.dumps({"schema_version": 1, "accepted": True, "cell": duplicate})) - with self.assertRaisesRegex(supervisor.ContractError, "exactly all 30"): + with self.assertRaisesRegex(supervisor.ContractError, "exactly all 27"): supervisor.merge_captures(root, captures, root / "merged.json") (captures / "duplicate.json").unlink() first = captures / "0.json" diff --git a/labs/15-pitot/tests/test_e2e_reporting.py b/labs/15-pitot/tests/test_e2e_reporting.py index 17db775b2..aead5a61b 100644 --- a/labs/15-pitot/tests/test_e2e_reporting.py +++ b/labs/15-pitot/tests/test_e2e_reporting.py @@ -49,15 +49,22 @@ def _receipt(agent="claude", platform="ubuntu"): "executable_sha256": fixture["executable_sha256"], } return { - "schema_version": 1, + "schema_version": 2, "agent": agent, "cli": {"version": record["version"], "executable": f"/bin/{record['executable']}", "executable_sha256": fixture["executable_sha256"], "installer": record["installer"]["kind"], "runtime": record["runtime"][platform]}, "prompt_hash": "f" * 64, "protocol": dialect, "endpoint": endpoint, - "nonce": "nonce", - "receipts": {name: True for name in ("initial_prompt_observed", "tool_call_response_emitted", "tool_result_observed", "final_response_emitted", "hook_observed", "canary_result_observed", "cli_exit_zero")}, - "hook": {"host": agent, "action_kind": "shell", "pitot_exit": 0}, + "nonce": "e" * 32, + "receipts": {name: True for name in ("initial_prompt_observed", "allow_tool_call_response_emitted", "allow_tool_result_observed", "deny_tool_call_response_emitted", "denied_result_observed", "final_response_emitted", "consumer_observed", "controller_allow_observed", "controller_deny_observed", "deny_canary_absent", "final_output_observed", "cli_exit_zero")}, + "runtime": {"schema_version": 1, "instance_id": "instance", "pid": 1, "endpoint": "http://127.0.0.1:1", "config_sha256": "c" * 64}, + "hooks": [ + {"host": agent, "action_kind": "shell", "action_id": "act_allow", "pitot_exit": 0, "nonce": "e" * 32}, + {"host": agent, "action_kind": "shell", "action_id": "act_deny", "pitot_exit": 2, "nonce": "e" * 32}, + ], + "controller": {"id": "e2e-shell-controller", "action_ids": ["act_allow", "act_deny"], "outcomes": ["allow", "deny"]}, + "consumer": {"id": "e2e-audit", "action_ids": ["act_allow", "act_deny"], "projection": "sha256"}, + "canary": {"executions": ["PITOT_ALLOW " + "e" * 32], "denied_executions": 0}, } @@ -69,18 +76,36 @@ def _result(agent="claude", platform="ubuntu", status="pass", mode="real_cli"): "platform": platform, "status": status, "verification_mode": mode, - "evidence": "binary-observed request, accepted response, hook, canary, and final receipts" if status == "pass" else "real-agent evidence contract failed", + "evidence": "binary-observed request, real hook control, projected Consumer, allow/deny canary, and final receipts" if status == "pass" else "real-agent control evidence contract failed", "cli": receipt["cli"] if receipt else None, "protocol": receipt["protocol"] if receipt else None, "endpoint": receipt["endpoint"] if receipt else None, "prompt_hash": receipt["prompt_hash"] if receipt else None, + "nonce": receipt["nonce"] if receipt else None, "receipts": receipt["receipts"] if receipt else None, - "hook": receipt["hook"] if receipt else None, + "runtime": receipt["runtime"] if receipt else None, + "hooks": receipt["hooks"] if receipt else None, + "controller": receipt["controller"] if receipt else None, + "consumer": receipt["consumer"] if receipt else None, + "canary": receipt["canary"] if receipt else None, "commit_sha": "a" * 40, "run_url": "https://github.com/operatorstack/intelligence-flow/actions/runs/1", } +def _runtime_receipt(platform="ubuntu"): + return { + "schema_version": 1, + "capability": "explicit_request", + "platform": platform, + "nonce": "e" * 32, + "runtime": {"schema_version": 1, "instance_id": "instance", "pid": 1, "endpoint": "http://127.0.0.1:1", "config_sha256": "c" * 64}, + "controller": {"id": "e2e-release-controller", "action_ids": ["act_allow", "act_deny"], "outcomes": ["allow", "deny"]}, + "receipts": {"request_allow_observed": True, "request_deny_observed": True, "correlation_observed": True}, + "commit_sha": "a" * 40, + } + + class ResultGenerationTests(unittest.TestCase): def test_mock_anthropic_request_classifier(self): subprocess.run( @@ -117,6 +142,18 @@ def test_missing_or_duplicate_marker_fails(self): duplicate = "PITOT_E2E_RESULT mode=real_cli evidence=nonce-correlated\n" * 2 self.assertEqual(runner.result_for("codex", "ubuntu", 0, duplicate)["status"], "fail") + def test_explicit_request_runtime_marker_passes(self): + with tempfile.TemporaryDirectory() as temp: + evidence = Path(temp) / "runtime-evidence.json" + evidence.write_text(json.dumps(_runtime_receipt())) + result = runner.result_for_runtime( + "ubuntu", 0, + "PITOT_RUNTIME_E2E_RESULT capability=explicit_request evidence=nonce-correlated\n", + evidence, + ) + self.assertEqual(result["status"], "pass") + self.assertEqual(result["verification_mode"], "real_runtime") + def test_nonzero_command_fails_even_with_marker(self): result = runner.result_for("cursor", "ubuntu", 1, "PITOT_E2E_RESULT mode=real_cli evidence=nonce-correlated\n") self.assertEqual(result["status"], "fail") @@ -209,11 +246,15 @@ def test_workflow_matrix_covers_every_reported_platform(self): self.assertIn("node_version=22.23.1", workflow) self.assertIn("SHASUMS256.txt", workflow) self.assertNotIn('-- bash "labs/15-pitot/tests/e2e_', workflow) - self.assertIn("fromJSON(needs.inventory.outputs.matrix)", unified) + self.assertIn("fromJSON(needs.inventory.outputs.agent_matrix)", unified) + self.assertIn("fromJSON(needs.inventory.outputs.runtime_matrix)", unified) + self.assertIn("pitot-e2e-runtime-${{ matrix.platform }}", unified) + self.assertIn('-- "$PITOT_BASH" labs/15-pitot/tests/e2e_runtime_cli_test.sh', unified) self.assertIn("name: pitot-e2e-inventory", unified) - self.assertEqual(runner_script.count("GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build"), 2) + self.assertEqual(runner_script.count("GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build"), 3) self.assertIn("pitot-linux", runner_script) self.assertIn("pitot-witness-linux", runner_script) + self.assertIn("pitot-testrole-linux", runner_script) self.assertIn("wsl.exe --distribution Ubuntu -- env", runner_script) self.assertIn('python3 "$(to_wsl_path "$SCRIPT_DIR/real_agent_driver.py")"', runner_script) self.assertIn("MSYS2_ARG_CONV_EXCL='*'", runner_script) @@ -236,7 +277,7 @@ def test_missing_artifacts_fail_every_inventory_cell(self): "a" * 40, "https://github.com/operatorstack/intelligence-flow/actions/runs/1", ) - self.assertEqual(len(results), 10) + self.assertEqual(len(results), 9) self.assertTrue( all( result["status"] == "fail" @@ -262,11 +303,11 @@ def test_rejects_identity_and_evidence_injection(self): def test_rejects_incomplete_or_mismatched_causal_receipts(self): value = _result() - value["receipts"]["hook_observed"] = False + value["receipts"]["controller_deny_observed"] = False with self.assertRaisesRegex(ValueError, "incomplete causal receipts"): reporter.validate_result(value, agent="claude", platform="ubuntu") value = _result() - value["hook"]["host"] = "cursor" + value["hooks"][0]["host"] = "cursor" with self.assertRaisesRegex(ValueError, "hook receipt"): reporter.validate_result(value, agent="claude", platform="ubuntu") value = _result() @@ -345,6 +386,8 @@ def test_rendered_comment_is_sticky_and_shows_modes(self): self.assertIn("real CLI 2.1.217 · native", body) self.assertIn("Windows", body) self.assertIn("All platforms are required", body) + self.assertIn("Runtime capabilities", body) + self.assertIn("`pitot request`", body) self.assertIn("OpenCode", body) self.assertIn("Kimi Code", body) diff --git a/labs/15-pitot/tests/test_model_control_proxy.py b/labs/15-pitot/tests/test_model_control_proxy.py index 0a6ff9423..c8ba9e6f3 100644 --- a/labs/15-pitot/tests/test_model_control_proxy.py +++ b/labs/15-pitot/tests/test_model_control_proxy.py @@ -4,7 +4,9 @@ import json from pathlib import Path import subprocess +import tempfile import unittest +from unittest import mock ROOT = Path(__file__).resolve().parents[3] @@ -19,6 +21,69 @@ class ModelControlProtocolTests(unittest.TestCase): + def test_windows_codex_uses_blocking_json_bridge(self): + with tempfile.TemporaryDirectory() as temporary, mock.patch.object(driver, "windows_host", return_value=True): + base = Path(temporary) + driver.configure( + "codex", base / "home", base / "project", "witness.exe", "http://127.0.0.1:1", + pitot_command="pitot.exe", witness_receipt="witness.jsonl", nonce="nonce", + runtime_path="runtime.json", + ) + hooks = json.loads((base / "home/.codex/hooks.json").read_text(encoding="utf-8")) + command = hooks["hooks"]["PreToolUse"][0]["hooks"][0]["command"] + self.assertIn("integrations/codex/PreToolUse.ps1", command.replace("\\", "/")) + self.assertTrue(command.startswith('& "')) + self.assertNotIn("powershell -NoProfile", command) + + def test_windows_qwen_uses_node_bridge_without_shell_override(self): + with tempfile.TemporaryDirectory() as temporary, mock.patch.object(driver, "windows_host", return_value=True): + base = Path(temporary) + driver.configure( + "qwen", base / "home", base / "project", "witness.exe", "http://127.0.0.1:1", + pitot_command="pitot.exe", witness_receipt="witness.jsonl", nonce="nonce", + runtime_path="runtime.json", + ) + settings = json.loads((base / "home/.qwen/settings.json").read_text(encoding="utf-8")) + hook = settings["hooks"]["PreToolUse"][0]["hooks"][0] + self.assertNotIn("shell", hook) + self.assertIn("PreToolUse.cjs", hook["command"]) + self.assertTrue(hook["command"].startswith('node "')) + + def test_qwen_uses_cross_platform_yolo_flag(self): + with tempfile.TemporaryDirectory() as temporary: + base = Path(temporary) + flags, _ = driver.configure( + "qwen", base / "home", base / "project", "witness", "http://127.0.0.1:1", + pitot_command="pitot", witness_receipt="witness.jsonl", nonce="nonce", + runtime_path="runtime.json", + ) + self.assertIn("-y", flags) + self.assertNotIn("--approval-mode", flags) + + def test_cursor_bridge_is_canonicalized_for_wsl(self): + with tempfile.TemporaryDirectory() as temporary: + base = Path(temporary) + lab = base / "lab" + source = lab / "integrations/cursor/beforeShellExecution" + source.parent.mkdir(parents=True) + source.write_bytes(b"#!/usr/bin/env bash\r\necho hook\r\n") + with mock.patch.object(driver, "LAB", lab): + driver.configure( + "cursor", base / "home", base / "project", "witness", "http://127.0.0.1:1", + pitot_command="pitot", witness_receipt="witness.jsonl", nonce="nonce", + runtime_path="runtime.json", + ) + bridge = (base / "project/.cursor/hooks/beforeShellExecution").read_bytes() + self.assertEqual(bridge, b"#!/usr/bin/env bash\necho hook\n") + + def test_windows_canary_command_is_shell_portable(self): + command = driver.render_canary_command( + r"D:\a\temp\pitot-testrole.exe", r"D:\a\temp\canary.jsonl", True, + ) + self.assertIn("D:/a/temp/pitot-testrole.exe", command) + self.assertIn("D:/a/temp/canary.jsonl", command) + self.assertNotIn("\\", command) + def test_anthropic_messages_emits_native_shell_tool(self): value = proxy.anthropic("Bash", "pitot-e2e-canary nonce", False) self.assertEqual(value["content"][0]["type"], "tool_use") @@ -67,6 +132,30 @@ def test_unknown_or_ambiguous_request_fails_closed(self): with self.assertRaises(proxy.UnknownProtocol): proxy.classify_request("/v1/messages", {"input": []}) + def test_tool_result_detection_rejects_echoed_tool_call_arguments(self): + marker = "PITOT_CANARY_RESULT PITOT_ALLOW nonce" + bodies = { + "anthropic_messages": {"messages": [{"role": "assistant", "content": [{"type": "tool_use", "input": {"command": marker}}]}]}, + "openai_chat": {"messages": [{"role": "assistant", "tool_calls": [{"function": {"arguments": marker}}]}]}, + "openai_responses": {"input": [{"type": "function_call", "arguments": marker}]}, + "gemini_generate_content": {"contents": [{"role": "model", "parts": [{"functionCall": {"args": {"command": marker}}}]}]}, + } + for dialect, body in bodies.items(): + with self.subTest(dialect): + self.assertFalse(proxy.tool_result_contains(dialect, body, marker)) + + def test_tool_result_detection_accepts_only_native_result_fields(self): + marker = "PITOT_CANARY_RESULT PITOT_ALLOW nonce" + bodies = { + "anthropic_messages": {"messages": [{"role": "user", "content": [{"type": "tool_result", "content": marker}]}]}, + "openai_chat": {"messages": [{"role": "tool", "content": marker}]}, + "openai_responses": {"input": [{"type": "function_call_output", "output": marker}]}, + "gemini_generate_content": {"contents": [{"role": "user", "parts": [{"functionResponse": {"response": {"output": marker}}}]}]}, + } + for dialect, body in bodies.items(): + with self.subTest(dialect): + self.assertTrue(proxy.tool_result_contains(dialect, body, marker)) + def test_capture_record_binds_detected_dialect_and_executable(self): agent = next(item for item in driver.MANIFEST["agents"] if item["id"] == "claude") installation = {"version": agent["version"], "runtime": "native", "executable_sha256": "a" * 64} diff --git a/labs/15-pitot/tests/test_pitot_harness.py b/labs/15-pitot/tests/test_pitot_harness.py index 6b2315c27..4ead5e444 100644 --- a/labs/15-pitot/tests/test_pitot_harness.py +++ b/labs/15-pitot/tests/test_pitot_harness.py @@ -49,9 +49,12 @@ def _fake_repo(root: Path) -> Path: integrations = root / "labs/15-pitot/integrations/pi" integrations.mkdir(parents=True) (integrations / "pitot.ts").write_text("// bridge\n") - cline = root / "labs/15-pitot/integrations/cline" - cline.mkdir(parents=True) - (cline / "PreToolUse").write_text("#!/usr/bin/env bash\n") + copilot = root / "labs/15-pitot/integrations/copilot" + copilot.mkdir(parents=True) + (copilot / "PreToolUse").write_text("#!/usr/bin/env bash\n") + codex = root / "labs/15-pitot/integrations/codex" + codex.mkdir(parents=True) + (codex / "PreToolUse.ps1").write_text("# bridge\n") return root @@ -72,7 +75,8 @@ def test_manifest_covers_surface_and_excludes_testdata(self): self.assertIn("tests/e2e_unified_runner.sh", manifest) self.assertIn("tests/mock_anthropic_server.js", manifest) self.assertIn("integrations/pi/pitot.ts", manifest) - self.assertIn("integrations/cline/PreToolUse", manifest) + self.assertIn("integrations/copilot/PreToolUse", manifest) + self.assertIn("integrations/codex/PreToolUse.ps1", manifest) self.assertNotIn("tests/test_internal.py", manifest) self.assertNotIn("testdata/ignored.bin", manifest) @@ -121,6 +125,32 @@ def test_committed_manifest_matches_sources(self): repo = Path(__file__).resolve().parents[3] build_pitot.check_manifest(repo) # the real repo must be drift-free + def test_windows_bridges_preserve_raw_hook_stdin(self): + lab = Path(__file__).resolve().parents[1] + bridges = ( + lab / "integrations/codex/PreToolUse.ps1", + lab / "integrations/copilot/PreToolUse.ps1", + lab / "integrations/gemini/BeforeTool.ps1", + ) + for bridge in bridges: + text = bridge.read_text(encoding="utf-8") + self.assertIn("RedirectStandardInput = $true", text) + self.assertIn("StandardInputEncoding = [Text.UTF8Encoding]::new($false)", text) + self.assertIn("StandardInput.BaseStream", text) + self.assertIn("GetBytes($payload)", text) + self.assertNotIn("$payload | &", text) + + qwen = (lab / "integrations/qwen/PreToolUse.cjs").read_text(encoding="utf-8") + self.assertIn("fs.readFileSync(0)", qwen) + self.assertIn("spawnSync(pitot", qwen) + + def test_cursor_wsl_install_does_not_depend_on_host_home(self): + installer = (Path(__file__).with_name("install_real_agent.py")).read_text(encoding="utf-8") + self.assertIn('/usr/local/share/pitot-cursor/{version}', installer) + self.assertIn('run([*wsl, "/usr/local/bin/agent", "--version"])', installer) + self.assertNotIn('"bash", "-lc", script', installer) + self.assertNotIn('release="$HOME/.local/share/pitot-cursor/{version}"', installer) + class ReleaseNoteTests(unittest.TestCase): def test_valid_note_passes(self): diff --git a/labs/15-pitot/tests/witness/main.go b/labs/15-pitot/tests/witness/main.go index 06e8a4544..f54da7cfa 100644 --- a/labs/15-pitot/tests/witness/main.go +++ b/labs/15-pitot/tests/witness/main.go @@ -21,6 +21,7 @@ type event struct { Name string `json:"name"` } `json:"host"` Action *struct { + ID string `json:"id"` Kind string `json:"kind"` } `json:"action"` Content *struct { @@ -82,14 +83,22 @@ func main() { observed.Action != nil && observed.Action.Kind == "shell" && observed.Content != nil && observed.Content.Mode == "full" && strings.Contains(string(observed.Content.Full), nonce) + actionID := "" + if observed.Action != nil { + actionID = observed.Action.ID + } record := map[string]any{ "schema_version": 1, "host": host, "nonce": nonce, "action_kind": "shell", - "pitot_exit": code, "valid": valid, + "action_id": actionID, "pitot_exit": code, "valid": valid, } - encoded, _ := json.MarshalIndent(record, "", " ") + encoded, _ := json.Marshal(record) if valid { _ = os.MkdirAll(filepath.Dir(receipt), 0o755) - _ = os.WriteFile(receipt, append(encoded, '\n'), 0o600) + file, openErr := os.OpenFile(receipt, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) + if openErr == nil { + _, _ = file.Write(append(encoded, '\n')) + _ = file.Close() + } } os.Exit(code) }