From 843528d0724d6bb26c6487c9db349960d2964467 Mon Sep 17 00:00:00 2001 From: bigboateng Date: Fri, 31 Jul 2026 10:06:15 +0100 Subject: [PATCH 1/5] feat(pitot): add Devin ACP adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disclosure-Reviewed: reviewed — public-safe only, private facet kept out of this commit --- .github/scripts/pitot_e2e_report.py | 17 +- labs/15-pitot/adapter-verification.json | 5 +- .../15-pitot/pitot-distribution/UPSTREAM.json | 44 +- .../2026-07-31-devin-acp-support.md | 14 + labs/15-pitot/pitot/adapters/adapters.go | 106 ++++- labs/15-pitot/pitot/cmd/pitot/devin_acp.go | 114 +++++ .../pitot/cmd/pitot/devin_acp_test.go | 168 ++++++++ labs/15-pitot/pitot/cmd/pitot/doctor_host.go | 35 ++ labs/15-pitot/pitot/cmd/pitot/main.go | 8 +- labs/15-pitot/pitot/cmd/pitot/main_test.go | 2 +- labs/15-pitot/pitot/cmd/pitot/wire.go | 9 +- labs/15-pitot/pitot/cmd/pitot/workbench.go | 28 +- .../pitot/docs/devin-adapter-research.md | 128 ++++++ .../docs/evidence/devin-acp-3000.3.22.json | 47 +++ .../pitot/internal/devinacp/client.go | 397 ++++++++++++++++++ .../pitot/internal/devinacp/client_test.go | 330 +++++++++++++++ labs/15-pitot/pitot/runtime/capabilities.go | 6 +- labs/15-pitot/pitot/sensor/sensor.go | 14 +- labs/15-pitot/pitot/sensor/sensor_test.go | 6 +- .../pitot/windtunnel/windtunnel_test.go | 30 +- .../public-readme-preview/CONTRIBUTING.md | 2 +- labs/15-pitot/public-readme-preview/README.md | 27 +- labs/15-pitot/publish.config.json | 3 +- labs/15-pitot/scripts/build_pitot.py | 2 +- .../scripts/pitot_adapter_supervisor.py | 25 +- labs/15-pitot/tests/devin_control_proxy.py | 179 ++++++++ labs/15-pitot/tests/e2e_devin_cli_test.sh | 2 + labs/15-pitot/tests/e2e_unified_runner.sh | 2 +- labs/15-pitot/tests/endpoint-provenance.json | 87 ++++ labs/15-pitot/tests/install_real_agent.py | 48 +++ labs/15-pitot/tests/real_agent_driver.py | 73 +++- labs/15-pitot/tests/run_e2e_report.py | 12 +- .../15-pitot/tests/test_adapter_supervisor.py | 19 +- labs/15-pitot/tests/test_e2e_reporting.py | 22 +- .../15-pitot/tests/test_install_real_agent.py | 108 +++++ .../tests/test_model_control_proxy.py | 25 ++ 36 files changed, 2016 insertions(+), 128 deletions(-) create mode 100644 labs/15-pitot/pitot-distribution/release-notes/2026-07-31-devin-acp-support.md create mode 100644 labs/15-pitot/pitot/cmd/pitot/devin_acp.go create mode 100644 labs/15-pitot/pitot/cmd/pitot/devin_acp_test.go create mode 100644 labs/15-pitot/pitot/docs/devin-adapter-research.md create mode 100644 labs/15-pitot/pitot/docs/evidence/devin-acp-3000.3.22.json create mode 100644 labs/15-pitot/pitot/internal/devinacp/client.go create mode 100644 labs/15-pitot/pitot/internal/devinacp/client_test.go create mode 100644 labs/15-pitot/tests/devin_control_proxy.py create mode 100755 labs/15-pitot/tests/e2e_devin_cli_test.sh create mode 100644 labs/15-pitot/tests/test_install_real_agent.py diff --git a/.github/scripts/pitot_e2e_report.py b/.github/scripts/pitot_e2e_report.py index 9e3247ac2..7551d1f4e 100644 --- a/.github/scripts/pitot_e2e_report.py +++ b/.github/scripts/pitot_e2e_report.py @@ -68,7 +68,7 @@ def validate_result( "nonce", "receipts", "runtime", - "hooks", + "boundaries", "controller", "consumer", "canary", @@ -86,10 +86,10 @@ def validate_result( raise ValueError("passing result requires real_cli verification") if value["status"] == "fail" and mode is not None: raise ValueError("failed result cannot claim a verification mode") - allowed_evidence = {"binary-observed request, real hook control, projected Consumer, allow/deny canary, and final receipts", "real-agent control evidence contract failed"} + allowed_evidence = {"binary-observed request, real action control, projected Consumer, allow/deny canary, and final receipts", "real-agent control evidence contract failed"} if value["evidence"] not in allowed_evidence: raise ValueError("invalid evidence summary") - evidence_fields = ("cli", "protocol", "endpoint", "prompt_hash", "nonce", "receipts", "runtime", "hooks", "controller", "consumer", "canary") + evidence_fields = ("cli", "protocol", "endpoint", "prompt_hash", "nonce", "receipts", "runtime", "boundaries", "controller", "consumer", "canary") if value["status"] == "fail": if any(value[field] is not None for field in evidence_fields): raise ValueError("failed result cannot carry passing evidence") @@ -130,10 +130,11 @@ def validate_result( } if not isinstance(value["receipts"], dict) or set(value["receipts"]) != receipt_fields or not all(item is True for item in value["receipts"].values()): raise ValueError("incomplete causal receipts") - hooks = value["hooks"] - if not isinstance(hooks, list) or len(hooks) != 2 or [item.get("pitot_exit") for item in hooks] != [0, 2] or any(item.get("host") != agent or item.get("action_kind") != "shell" or item.get("nonce") != nonce for item in hooks): - raise ValueError("invalid Pitot allow/deny hook receipts") - action_ids = [item.get("action_id") for item in hooks] + boundaries = value["boundaries"] + expected_transport = "acp" if record["integration"] == "acp_client" else "hook" + if not isinstance(boundaries, list) or len(boundaries) != 2 or [item.get("decision") for item in boundaries] != ["allow", "deny"] or any(item.get("host") != agent or item.get("transport") != expected_transport or item.get("action_kind") != "shell" or item.get("nonce") != nonce for item in boundaries): + raise ValueError("invalid Pitot allow/deny boundary receipts") + action_ids = [item.get("action_id") for item in boundaries] if len(set(action_ids)) != 2 or value["controller"] != {"id": "e2e-shell-controller", "action_ids": action_ids, "outcomes": ["allow", "deny"]}: raise ValueError("invalid Controller receipts") if value["consumer"] != {"id": "e2e-audit", "action_ids": action_ids, "projection": "sha256"}: @@ -237,7 +238,7 @@ def failed_result(agent: str, platform: str, head_sha: str, run_url: str) -> dic "nonce": None, "receipts": None, "runtime": None, - "hooks": None, + "boundaries": None, "controller": None, "consumer": None, "canary": None, diff --git a/labs/15-pitot/adapter-verification.json b/labs/15-pitot/adapter-verification.json index 86ac44bfe..648907f91 100644 --- a/labs/15-pitot/adapter-verification.json +++ b/labs/15-pitot/adapter-verification.json @@ -1,7 +1,7 @@ { - "schema_version": 6, + "schema_version": 7, "capabilities": [ - {"id": "hook_control", "matrix": "agent_platform"}, + {"id": "action_control", "matrix": "agent_platform"}, {"id": "consumer_delivery", "matrix": "agent_platform"}, {"id": "explicit_request", "matrix": "platform"} ], @@ -14,6 +14,7 @@ {"id": "claude", "label": "Claude", "version": "2.1.217", "executable": "claude", "installer": {"kind": "npm", "package": "@anthropic-ai/claude-code"}, "integration": "native_command_hook", "artifacts": [], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, {"id": "cursor", "label": "Cursor", "version": "2026.07.20-8cc9c0b", "executable": "agent", "installer": {"kind": "cursor_release", "package": "https://downloads.cursor.com/lab/2026.07.20-8cc9c0b"}, "integration": "native_command_hook", "artifacts": ["integrations/cursor/beforeShellExecution"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "wsl"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, {"id": "codex", "label": "Codex", "version": "0.145.0", "executable": "codex", "installer": {"kind": "npm", "package": "@openai/codex"}, "integration": "native_command_hook", "artifacts": ["integrations/codex/PreToolUse.ps1"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "devin", "label": "Devin", "version": "3000.3.22", "executable": "devin", "installer": {"kind": "devin_release", "package": "https://static.devin.ai/cli"}, "integration": "acp_client", "artifacts": [], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, {"id": "copilot", "label": "GitHub Copilot CLI", "version": "1.0.73", "executable": "copilot", "installer": {"kind": "npm", "package": "@github/copilot"}, "integration": "native_command_hook", "artifacts": ["integrations/copilot/PreToolUse", "integrations/copilot/PreToolUse.ps1"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, {"id": "gemini", "label": "Gemini", "version": "0.51.0", "executable": "gemini", "installer": {"kind": "npm", "package": "@google/gemini-cli"}, "integration": "native_command_hook", "artifacts": ["integrations/gemini/BeforeTool", "integrations/gemini/BeforeTool.ps1"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, {"id": "kimi", "label": "Kimi Code", "version": "0.29.0", "executable": "kimi", "installer": {"kind": "kimi_release", "package": "https://code.kimi.com/kimi-code"}, "integration": "native_command_hook", "artifacts": [], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, diff --git a/labs/15-pitot/pitot-distribution/UPSTREAM.json b/labs/15-pitot/pitot-distribution/UPSTREAM.json index d31a1190a..1028ccaec 100644 --- a/labs/15-pitot/pitot-distribution/UPSTREAM.json +++ b/labs/15-pitot/pitot-distribution/UPSTREAM.json @@ -1,11 +1,11 @@ { "files": { ".goreleaser.yml": "d375f187624219e9420acd2a40cfae95772ef1fee3abda3e729d6a16ba612032", - "CONTRIBUTING.md": "23728d8a132d62b8adfb2e5c3eb9d9bfcf8a4d04543765b1e22ad8d55424af8f", + "CONTRIBUTING.md": "02c89a5790f2943e5e2466b0d260210238cae306922f0f031adb5f7028b48066", "LICENSE": "bd17d47aefaa7406616179288058001b6532881a2517254fe668d7e9c4965cfb", - "README.md": "dc3adbcf60ed946537353c1b873a8270c67dde11d7f71cf4cd361d13544f909e", - "adapter-verification.json": "f8ad4e206571650f698826a8b66d8c00822be425e8d2de8ae98d98239e575eb4", - "adapters/adapters.go": "1b46ba131fa3b2c93eed23526330275a3506451ba4bbd4f497e5378dfab2b6a8", + "README.md": "27df9b71b14ac474de3d03f2049eabb7d1c2c35514b7d351940683b0651d1984", + "adapter-verification.json": "fedd1624e01d19739114203674281296de779097e45fbb9e497047805b7d9250", + "adapters/adapters.go": "13cc5d94ea7c4ae50eccfd27d63e0d91b38cf76ee20aa9737a7edc9bd47d13b2", "assets/pitot-boundary.png": "8a0ddb7d81831d94e14813f50ea4ca8670d77417f339ed2f91f0c653bf52f41d", "assets/pitot-boundary.svg": "0c3871d70c84748573f231842091deb38a6def2862403ca34e8cc4493b9c9ebf", "assets/pitot-hero.png": "a73532252b1e66c06273abbf5a4fe6261e98de3133b09e8d550edacfeeab92f8", @@ -16,20 +16,22 @@ "bridge/bridge.go": "5adfcd3f743cae46e4446a6e030d53464ada97de0261a8588fa2a9fcd62136b8", "bridge/bridge_test.go": "6dcc6d05f2b39c25955fc0b2d21d3d148dd9d77600fb12799941f86bdb1acb61", "cmd/generate-schema/main.go": "6e9d0030290d99e36967433f96e38385a122974f899ad9421aac1ef7e50d8fcb", - "cmd/pitot/doctor_host.go": "3eb21fb3a9b42fa7f2a33ed2ed82ae6de68eb2d084ecef4a9e0dce36e37d8a31", + "cmd/pitot/devin_acp.go": "b10962cc21239d0b3955d2521f1f4d940c5de34624d14ce89ef0e40e1fbc1138", + "cmd/pitot/devin_acp_test.go": "158a9ee019e8368b9f9434000eea8cbeee98201203213ef83797102549e1dd0a", + "cmd/pitot/doctor_host.go": "fa7819448218c3dd92d3393df9a0ac6e5aa70803348898eec10220bc7b3867a4", "cmd/pitot/doctor_host_test.go": "4e6e327f6cf27cf94a0a608e10eb6790d6c11fcd53e6dfd7370007190749952f", "cmd/pitot/install.go": "46cab87809566f8da47c4d71b8a87618d06cb0829bdabe1c8f734faf53a9f46c", "cmd/pitot/kimi_control_test.go": "27b38867d4799636a664e3b1726ef55568f5dfd6a9be11f0ce5eca9931d759a4", "cmd/pitot/kimi_smoke_test.go": "01cbf18312902cac42ec1f2547d35362c2bc920938acea508ba7c7f0638a9473", - "cmd/pitot/main.go": "be069ed59ffe5e8cde26e3b88640dd8ae876fa0c266281ab9ac206be3e8944f1", - "cmd/pitot/main_test.go": "6a97e930a83d15a51713530c092a7e7146c4cb4e4d7eb9f58b97d1577c986c6b", + "cmd/pitot/main.go": "9b7851b4e91c3eef08d6dac82939c1842f8e1a5237bf603e5a6ccc51a50561ec", + "cmd/pitot/main_test.go": "35c17c8de7caf13d0ed399c0b67903236cd3c64faa1ea134c5445d00887a32a5", "cmd/pitot/shim.go": "3a7a4be5bd769010ad947de78188dd17c729388dbf74153d4dbd51fba2121ab1", "cmd/pitot/substrate_test.go": "b4e7d83aa592f7b5bf4ec059ec552abd62473e51a6634a26428448168898d0a3", "cmd/pitot/upgrade.go": "2ad6f581318c91c2617e1a65409458cd4de6641c940554b232dbdfd65e0fd221", "cmd/pitot/version.go": "60ddbabcc8985c5137d5b73e3ba1be0f62fbb9db2a6612027e08314dfd82d618", - "cmd/pitot/wire.go": "56c4d410eab4974952c5d7b7c37ad739341ebf292bb21de6143011c0bb1b1f48", + "cmd/pitot/wire.go": "e9181ef4e7133b35e263db69baac4be6ee33d6ee1ea36ab9749a2c630e5e2259", "cmd/pitot/wire_test.go": "66a3c3302780145ebf12da3df1390f61a8d9b82f30b03f6332dd958799e70f7f", - "cmd/pitot/workbench.go": "aa0a270829cdaa2bc7bf84d39a88f703088a9dc1c55057f7a2fd369ba98c9597", + "cmd/pitot/workbench.go": "614ec10f70542c5bbf65583e8ee94bb236e9757ad73f1d5cb46cdca08511182d", "cmd/pitot/workbench_build_test.go": "8d5c5c35e8cbd59e21cfdd7e206d6b5b769892ab27a99fe5d87b13b1a31d0714", "cmd/pitot/workbench_contract_test.go": "5e465f3d3f8b93ffaacfb4738279d369b13514bf7613b8611952f3cd26896586", "cmd/pitot/workbench_dev_test.go": "abffe81e1a25f086d7f3c2f1c32986ee93618bbfc87220bdfda840ec6c6466bf", @@ -43,6 +45,8 @@ "conformance/fixtures/negative.jsonl": "503ea76988df595d96ebf695f991b8ea6c892be4a578522dff4ddb0d39b647e4", "conformance/fixtures/positive.jsonl": "881efdf58b66ee7d03171c6b4410bf1bce9e1b8db5c8bf969d0e8ec467420c3f", "doc.go": "a8abdafac969b1bf4372c8bb023aa51125dc073f03218f4ab9913dfc5ffa877d", + "docs/devin-adapter-research.md": "687993d733b3f57a3be539fb468e12a79f851d3039958369b97bd68d375ad7e1", + "docs/evidence/devin-acp-3000.3.22.json": "4b4f0a386b56e586eeef5a95226a9a15420b2550893d3ba7a777714fdb7705c7", "e2e/e2e_coverage_test.go": "09a738ca70749be94c462ce9974cdac9a7be0b24b9dc2b20ec0738d4686ebfbf", "e2e/e2e_hook_test.go": "5e184dc8907b6e36daeab90bbbb1654fa5336312866412031805a13ba535d1f8", "examples/doc.go": "58f3f9eb7d272d7b6eecdb05f43e1613d5e3ef92d15d97c5440bd4b6990c26f9", @@ -63,12 +67,14 @@ "integrations/pi/pitot.ts": "ed2d60d5ab6e33e115cfa058e4f96095100e93a061567d0af31249aa756bab3e", "integrations/qwen/PreToolUse": "95c358620f2f882bb8680e6aa9639f3b36a8567a3fd60cf14a6cdf3b3fbcf78b", "integrations/qwen/PreToolUse.cjs": "e9bf00bbbee5c15f01ea203d0992b34f8754b75a9c9889eecb16baeb10fcce6c", + "internal/devinacp/client.go": "e2e70755d5888a2bd34e44b160d1dbf955fe147adf020050aa3fbc68cdbe709d", + "internal/devinacp/client_test.go": "5bfff9b7f29d253450e4424ab44dab97877ddb39b3941607a4e264dd66da32cb", "internal/testrole/main.go": "6d657eb85d8ddaff0ae5a3da281aa7aa0a4bdda860d179e99508110033765787", "internal/testrole/main_test.go": "c7fbc4905bcef7d662c9e162a4c32c8a86d5c1a97820e9f4ee48f643c4e47a38", "projection/projection.go": "4d3c823fd72a3ca5387dba3683838a1d7e455e9b18309acc839763a39b7bb35f", "protocol/framing.go": "d4409314b72e09cfd472ad9a21d4c7a223b4341b26f58f6062a7c899e3f87482", "protocol/framing_test.go": "9fcb13767fdcc7129d2c87bc2133a5800c69d6ac0166016dc266dca5fcfaa1c5", - "runtime/capabilities.go": "76c27bdd7ddfa11a1d639915ac2eb2d573b7ccdd687a5f548b3731b7c1e1828f", + "runtime/capabilities.go": "cf6370334a16a64f2b78b5e9d6b7e1d479c18a8b57610cbf14aa2fd7001b787b", "runtime/descriptor_unix.go": "df41b6867e9840933f186c93b6bc61861e7ea5252a365455686ea0414bfa0044", "runtime/descriptor_windows.go": "2d9ffefe3af0154fa8042de6b67460d4e86dd3f4cdd9e986f180f7d0c535c9a5", "runtime/request.go": "198c44fd6c547022a15b6d0d48e4d0130fa8afb687994365115576e4d874550d", @@ -106,30 +112,32 @@ "sdk/typescript/src/runner.ts": "c58babd3ec3996a05988f3cb7dec061cf7c48f41cd5125625be3408cb9b207f0", "sdk/typescript/tsconfig.json": "de0065da9acf19a93ba8646c90b7101dfd5a8667552d3d3c59e3effd813218aa", "sensor/decode_fuzz_test.go": "d27f2fbbc069eded26a73c9cd9bace98dd8a9e34949576790b81a08d130fbaf2", - "sensor/sensor.go": "5c503d07ac33e7894d635f2d98bcd6d165d442d60c127ed6aeac80ec319086c5", - "sensor/sensor_test.go": "4ddbdde3e486c9e189a2ed4174ad413df107d43dc98f5242da3667a24c7a5da1", + "sensor/sensor.go": "498d4f69c9243a409b25704beab9d441bb312c7813d168138b85a69cff08d8e9", + "sensor/sensor_test.go": "9e0ec50ec3930c37b0e4cc56419be451fb56432f0f22efa145b55ddf6440cd4b", "tests/cursor_control_proxy.mjs": "ab532aa56a9299f497f3ceeedb4b6a0beb26089b7c022ed774385dd149ea56e6", + "tests/devin_control_proxy.py": "0e74ac4c0e7182b8e2e3a89b4a16fbe0d2ffdabb2c40a301aa60176ae74c8d9c", "tests/e2e_claude_cli_test.sh": "b28c4d1963e326b4b3f158a7cfc1b92771e768ac9665dd43e8a339013cc11568", "tests/e2e_codex_cli_test.sh": "dbae5224c87410a5a5d67023d7e23406453bc5493d981af9347993b4e1f562f2", "tests/e2e_copilot_cli_test.sh": "61b1e44dcd598d2d33e7f04dec26bec74405a9e2456ecda46e94dc8d43ad4315", "tests/e2e_cursor_cli_test.sh": "af46f1a40a535ed1f345ee51cba89a57fa912d6b67bed7df9bae0943131fa542", + "tests/e2e_devin_cli_test.sh": "337aa3fb41a9661a6a1a69edbee67f90a7b65f493c52054bbe8c018c342f66fd", "tests/e2e_gemini_cli_test.sh": "42cd77c366c17de092ec46324b3fa764c08b78de0e14f0d9f62b998c2939d6e0", "tests/e2e_kimi_cli_test.sh": "3689474b027ca2d83ae36c4c3372e93c555477b69d5d81507a8aa6aa1e88edb1", "tests/e2e_opencode_cli_test.sh": "e74a781f36b8263f2dd0f9700f3eea7a443b289fb87b55af455ce64509e08546", "tests/e2e_pi_cli_test.sh": "369707019ce0ee9ccb91ac1b9a16d5b8ae357fc9277ccb3407bcd7bd05aa86c0", "tests/e2e_qwen_cli_test.sh": "8813610ff0ae253e59849f76a5dc6ecf0990c150ea0ede60ec833d36e6e4a68f", "tests/e2e_runtime_cli_test.sh": "54f64c3ef21eb62aa08dd7cb11d288be805d14a248c5baf7c172babd0957b4c9", - "tests/e2e_unified_runner.sh": "bcad531c79b0840220d9353dbb64358ecddedd868f5334178944593926550c74", - "tests/endpoint-provenance.json": "3b9c5a2acab964d49a0c534e5dba01258e7f56708920a6463cf521b0000a446e", - "tests/install_real_agent.py": "999cb3326468998f0474431afe6b5b5dbd21752f76e68600bc03adb0d49484b9", + "tests/e2e_unified_runner.sh": "c9c79e27f8daa4ff15399ed7ee892d821b4f4aa34af91dbccf57bf9031210b75", + "tests/endpoint-provenance.json": "6d817e51680eb833ec41e21982a6931393ff76df06551cf9b47880cc5b1f1796", + "tests/install_real_agent.py": "a8dc79bf0914ed5ab2a82c4f2471e805f24eef2adde2261fee22dbd5aabeb985", "tests/mock_anthropic_server.js": "ecebea62f9e93791a79f1ae3dd3c67b8fa42490e9805b23b662b877edfdb0f0e", "tests/model_control_proxy.py": "cd4733ab20c16770fe3bf6ac477935f7dc285cd89be020848ae4798cb509e5f2", - "tests/real_agent_driver.py": "8c57b50823be6bbf58a39624ef5661a4270a04d82b5ee7909280c9efa10344fd", - "tests/run_e2e_report.py": "e75da4aac49b70832e6ea98f295d6ebd9ef909538614136665c74f085220b1a5", + "tests/real_agent_driver.py": "24bec3016f5ddee69bf56df255613223fa51444c7c53ecccc9689e4bb4673f1f", + "tests/run_e2e_report.py": "a72cbc486c100b75fccf6ac787da1736e13426b44a15d9d5dddcb8177bbaf596", "tests/runtime_capability_driver.py": "54a485fc4f16981f2542d0dbd903a12b80f31f071908a378607f26678aeea07f", "tests/witness/main.go": "cd56bbd00aa44cc5baf6426c8461a8ebca4a8391518f6acfa2301ac36add7c5f", "windtunnel/doc.go": "44e0bcde632da73e1f8b98beade3a34ca8e0d0ea79cdfb91d131de290b164fc4", - "windtunnel/windtunnel_test.go": "d34929ffdb1927b2ee27cc79640b333b0571659d231977b91c3a8898cf79bc42", + "windtunnel/windtunnel_test.go": "30f9d99d8f0bb47439585cb1db8c576b724e940c58a9fb95a0fff735a0c930d8", "wiring/wiring.go": "2cdf286fb26def02c3bad1d6c7e7e4419d366c732faf55ad3bb02a4ae127acdf", "wiring/wiring_test.go": "ade8b847ecdcbee17439e27d186c7078f39490716df5b56badd43a2a05f01141" }, diff --git a/labs/15-pitot/pitot-distribution/release-notes/2026-07-31-devin-acp-support.md b/labs/15-pitot/pitot-distribution/release-notes/2026-07-31-devin-acp-support.md new file mode 100644 index 000000000..0b05a711b --- /dev/null +++ b/labs/15-pitot/pitot-distribution/release-notes/2026-07-31-devin-acp-support.md @@ -0,0 +1,14 @@ +### Devin ACP support + +Pitot adds Devin CLI `3000.3.22` as its tenth supervised adapter. Devin uses a +stateful ACP v1 client boundary instead of lifecycle hooks, while normalized +events and Controller responses remain Pitot protocol v1. + +Use `pitot dev --host devin -- devin -p ""`, or attach to a manually +managed runtime with `pitot acp devin --runtime PATH --prompt TEXT`. Initial +support is intentionally single-prompt and selects only one-shot ACP permission +options. + +The verification manifest moves to schema v7, renames `hook_control` to +transport-neutral `action_control`, and retains the Go +`CapabilityHookControl` identifier as a deprecated alias. diff --git a/labs/15-pitot/pitot/adapters/adapters.go b/labs/15-pitot/pitot/adapters/adapters.go index b31fc631a..9f886c346 100644 --- a/labs/15-pitot/pitot/adapters/adapters.go +++ b/labs/15-pitot/pitot/adapters/adapters.go @@ -1,5 +1,5 @@ // Package adapters declares the coding-agent host boundaries Pitot supports and -// the host-specific hook shapes it understands. +// the host-specific event shapes it understands. // // The host mapping is seeded from the Boatstack safety-hook adapters // (labs/12-product-engineering-loop/product-engineering-loop/hooks.go: @@ -32,18 +32,27 @@ const ( Copilot Host = "copilot" Qwen Host = "qwen" Pi Host = "pi" + Devin Host = "devin" ) // AdapterVersion is the semantic version stamped onto normalized events so // consumers can reason about host-compatibility differences. const AdapterVersion = "0.1.0" +// Transport identifies how a host exposes its controllable boundary. +type Transport string + +const ( + TransportHook Transport = "hook" + TransportACP Transport = "acp" +) + // ParserConfig handles regular coding/parsing chores, converting raw host fields // into normalized action types and command strings. type ParserConfig struct { CanonicalEvent []byte - EventNameFor func(raw RawHookEvent) string - CommandFor func(raw RawHookEvent) (string, bool) + EventNameFor func(raw RawBoundaryEvent) string + CommandFor func(raw RawBoundaryEvent) (string, bool) ActionKinds map[string]string // hook event name -> normalized action kind ("shell", "mcp") } @@ -67,11 +76,27 @@ type HostConfig struct { MainEventName string Parser ParserConfig Partition ControlPartition + Transport Transport } var ( registryMu sync.RWMutex registry = map[Host]HostConfig{ + Devin: { + MainEventName: "session/request_permission", + Transport: TransportACP, + Parser: ParserConfig{ + CanonicalEvent: []byte(`{"hook_event_name":"session/request_permission","tool_name":"exec","tool_input":{"command":"git status --short"}}`), + CommandFor: func(raw RawHookEvent) (string, bool) { + if raw.ToolName != "exec" { + return "", false + } + return toolInputCommand(raw) + }, + ActionKinds: map[string]string{"session/request_permission": "shell"}, + }, + Partition: ControlPartition{Controllable: []string{"session/request_permission"}}, + }, Copilot: preToolUseHost(), Qwen: { MainEventName: "PreToolUse", @@ -259,6 +284,12 @@ func RegisterHost(h Host, config HostConfig) error { if config.Parser.CommandFor == nil { return fmt.Errorf("pitot: host %q must declare a CommandFor extractor", h) } + if config.Transport == "" { + config.Transport = TransportHook + } + if config.Transport != TransportHook && config.Transport != TransportACP { + return fmt.Errorf("pitot: host %q declares unsupported transport %q", h, config.Transport) + } // --- Control Laws Verification --- // To safely supervise a host (maintain nonblocking control under K), @@ -307,8 +338,8 @@ func IsSupported(h Host) bool { return exists } -// HookEvents returns the host hook event names Pitot attaches to. -func HookEvents(h Host) []string { +// BoundaryEvents returns the host event names Pitot observes or controls. +func BoundaryEvents(h Host) []string { registryMu.RLock() defer registryMu.RUnlock() @@ -321,19 +352,25 @@ func HookEvents(h Host) []string { return nil } -// HookEventName is the field each host uses to name its hook event. -func HookEventName(h Host) (string, error) { +// HookEvents is the compatibility name for BoundaryEvents. +func HookEvents(h Host) []string { return BoundaryEvents(h) } + +// BoundaryEventName is the field each host uses to name its boundary event. +func BoundaryEventName(h Host) (string, error) { registryMu.RLock() defer registryMu.RUnlock() if config, exists := registry[h]; exists { return config.MainEventName, nil } - return "", fmt.Errorf("pitot: unsupported hook host %q", h) + return "", fmt.Errorf("pitot: unsupported host %q", h) } -// CanonicalHookEvent returns a canonical, read-only probe payload for a host. -func CanonicalHookEvent(h Host) ([]byte, error) { +// HookEventName is the compatibility name for BoundaryEventName. +func HookEventName(h Host) (string, error) { return BoundaryEventName(h) } + +// CanonicalBoundaryEvent returns a canonical, read-only probe payload for a host. +func CanonicalBoundaryEvent(h Host) ([]byte, error) { registryMu.RLock() defer registryMu.RUnlock() @@ -342,13 +379,30 @@ func CanonicalHookEvent(h Host) ([]byte, error) { copy(payload, config.Parser.CanonicalEvent) return payload, nil } - return nil, fmt.Errorf("pitot: unsupported hook host %q", h) + return nil, fmt.Errorf("pitot: unsupported host %q", h) +} + +// CanonicalHookEvent is the compatibility name for CanonicalBoundaryEvent. +func CanonicalHookEvent(h Host) ([]byte, error) { return CanonicalBoundaryEvent(h) } + +// BoundaryTransport returns the host's controllable transport. +func BoundaryTransport(h Host) (Transport, error) { + registryMu.RLock() + defer registryMu.RUnlock() + config, exists := registry[h] + if !exists { + return "", fmt.Errorf("pitot: unsupported host %q", h) + } + if config.Transport == "" { + return TransportHook, nil + } + return config.Transport, nil } -// RawHookEvent is the union of fields Pitot reads from a host hook payload. The -// sensor decodes into this shape before normalizing; adapters preserve host -// capability differences rather than inventing missing fields. -type RawHookEvent struct { +// RawBoundaryEvent is the union of fields Pitot reads from a host-controlled +// action boundary. Adapters preserve host capability differences rather than +// inventing missing fields. +type RawBoundaryEvent struct { // HookEventName is the host's event discriminator. HookEventName string `json:"hook_event_name"` // Command is populated by Cursor's beforeShellExecution. @@ -358,8 +412,12 @@ type RawHookEvent struct { ToolInput map[string]any `json:"tool_input"` } +// RawHookEvent is retained as a source-compatible alias. +// Deprecated: use RawBoundaryEvent. +type RawHookEvent = RawBoundaryEvent + // EventNameFor extracts the host-specific event discriminator. -func (h Host) EventNameFor(raw RawHookEvent) string { +func (h Host) EventNameFor(raw RawBoundaryEvent) string { registryMu.RLock() defer registryMu.RUnlock() if config, exists := registry[h]; exists && config.Parser.EventNameFor != nil { @@ -371,7 +429,7 @@ func (h Host) EventNameFor(raw RawHookEvent) string { // CommandFor extracts the shell command a raw hook event describes, per host // shape. It returns ok=false when the host omitted the command entirely, so the // sensor can raise a boundary fault instead of manufacturing an empty command. -func (h Host) CommandFor(raw RawHookEvent) (command string, ok bool) { +func (h Host) CommandFor(raw RawBoundaryEvent) (command string, ok bool) { registryMu.RLock() defer registryMu.RUnlock() @@ -381,8 +439,8 @@ func (h Host) CommandFor(raw RawHookEvent) (command string, ok bool) { return "", false } -// HasHookEvent reports whether the host supports the given hook event name. -func (h Host) HasHookEvent(name string) bool { +// HasBoundaryEvent reports whether the host supports the given boundary event. +func (h Host) HasBoundaryEvent(name string) bool { registryMu.RLock() defer registryMu.RUnlock() @@ -401,13 +459,17 @@ func (h Host) HasHookEvent(name string) bool { return false } -// ActionKind returns the normalized action kind (e.g., "shell", "mcp") for a given hook event name. -func (h Host) ActionKind(hookEventName string) string { +// HasHookEvent is retained as a source-compatible alias. +// Deprecated: use HasBoundaryEvent. +func (h Host) HasHookEvent(name string) bool { return h.HasBoundaryEvent(name) } + +// ActionKind returns the normalized action kind for a boundary event. +func (h Host) ActionKind(boundaryEventName string) string { registryMu.RLock() defer registryMu.RUnlock() if config, exists := registry[h]; exists { - if kind, ok := config.Parser.ActionKinds[hookEventName]; ok { + if kind, ok := config.Parser.ActionKinds[boundaryEventName]; ok { return kind } } diff --git a/labs/15-pitot/pitot/cmd/pitot/devin_acp.go b/labs/15-pitot/pitot/cmd/pitot/devin_acp.go new file mode 100644 index 000000000..5e7db9321 --- /dev/null +++ b/labs/15-pitot/pitot/cmd/pitot/devin_acp.go @@ -0,0 +1,114 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/operatorstack/pitot/internal/devinacp" +) + +var runDevinACP = devinacp.Run + +func runACP(ctx context.Context, args []string, stdout, stderr io.Writer) error { + if len(args) == 0 || args[0] != "devin" { + return errors.New("pitot acp: requires host devin") + } + options := devinacp.Options{Executable: "devin", Cwd: ".", Stdout: stdout, Stderr: stderr} + options.Runtime = os.Getenv("PITOT_RUNTIME") + for i := 1; i < len(args); i++ { + switch args[i] { + case "--runtime": + value, next, err := flagValue(args, i, "--runtime") + if err != nil { + return err + } + options.Runtime, i = value, next + case "--prompt": + value, next, err := flagValue(args, i, "--prompt") + if err != nil { + return err + } + options.Prompt, i = value, next + case "--exec": + value, next, err := flagValue(args, i, "--exec") + if err != nil { + return err + } + options.Executable, i = value, next + case "--model": + value, next, err := flagValue(args, i, "--model") + if err != nil { + return err + } + options.Model, i = value, next + case "--agent-type": + value, next, err := flagValue(args, i, "--agent-type") + if err != nil { + return err + } + options.AgentType, i = value, next + case "--cwd": + value, next, err := flagValue(args, i, "--cwd") + if err != nil { + return err + } + options.Cwd, i = value, next + default: + return fmt.Errorf("pitot acp: unsupported argument %q (single-prompt Devin ACP supports --runtime, --prompt, --exec, --model, --agent-type, --cwd)", args[i]) + } + } + if options.Runtime == "" { + return errors.New("pitot acp: --runtime is required (or set PITOT_RUNTIME)") + } + return runDevinACP(ctx, options) +} + +func flagValue(args []string, index int, name string) (string, int, error) { + if index+1 >= len(args) || args[index+1] == "" { + return "", index, fmt.Errorf("pitot acp: %s requires a value", name) + } + return args[index+1], index + 1, nil +} + +func devinOptionsFromAgent(program string, args []string, runtimePath string, stdout, stderr io.Writer) (devinacp.Options, error) { + name := strings.TrimSuffix(strings.ToLower(filepath.Base(program)), ".exe") + if name != "devin" { + return devinacp.Options{}, fmt.Errorf("pitot dev: host devin requires a Devin executable, got %q", program) + } + options := devinacp.Options{ + Executable: program, + Cwd: ".", + Runtime: runtimePath, + Stdout: stdout, + Stderr: stderr, + } + for i := 0; i < len(args); i++ { + switch { + case args[i] == "-p" || args[i] == "--print": + if i+1 >= len(args) { + return devinacp.Options{}, errors.New("pitot dev: Devin -p/--print requires a prompt") + } + options.Prompt = args[i+1] + i++ + case strings.HasPrefix(args[i], "--print="): + options.Prompt = strings.TrimPrefix(args[i], "--print=") + case args[i] == "--model": + if i+1 >= len(args) { + return devinacp.Options{}, errors.New("pitot dev: Devin --model requires a value") + } + options.Model = args[i+1] + i++ + default: + return devinacp.Options{}, fmt.Errorf("pitot dev: unsupported Devin argument %q; initial ACP support accepts -p/--print and --model", args[i]) + } + } + if options.Prompt == "" { + return devinacp.Options{}, errors.New("pitot dev: Devin ACP support requires one -p/--print prompt") + } + return options, nil +} diff --git a/labs/15-pitot/pitot/cmd/pitot/devin_acp_test.go b/labs/15-pitot/pitot/cmd/pitot/devin_acp_test.go new file mode 100644 index 000000000..ec51ed17b --- /dev/null +++ b/labs/15-pitot/pitot/cmd/pitot/devin_acp_test.go @@ -0,0 +1,168 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "strings" + "testing" + + "github.com/operatorstack/pitot/internal/devinacp" +) + +func TestRunACPParsesSinglePromptOptions(t *testing.T) { + original := runDevinACP + t.Cleanup(func() { runDevinACP = original }) + var captured devinacp.Options + runDevinACP = func(_ context.Context, options devinacp.Options) error { + captured = options + return nil + } + var stdout, stderr bytes.Buffer + err := runACP(context.Background(), []string{ + "devin", "--runtime", "/tmp/runtime.json", "--prompt", "fix it", + "--exec", "/opt/devin", "--model", "opus", "--agent-type", "review", "--cwd", "/tmp/project", + }, &stdout, &stderr) + if err != nil { + t.Fatal(err) + } + if captured.Runtime != "/tmp/runtime.json" || captured.Prompt != "fix it" || captured.Executable != "/opt/devin" || + captured.Model != "opus" || captured.AgentType != "review" || captured.Cwd != "/tmp/project" { + t.Fatalf("captured options = %+v", captured) + } +} + +func TestRunACPRejectsUnsupportedAndMissingArguments(t *testing.T) { + tests := [][]string{ + {}, + {"claude"}, + {"devin", "--runtime", "r"}, + {"devin", "--runtime"}, + {"devin", "--runtime", "r", "--prompt", "p", "--resume", "x"}, + } + for _, args := range tests { + if err := runACP(context.Background(), args, io.Discard, io.Discard); err == nil { + t.Errorf("runACP(%v) succeeded", args) + } + } +} + +func TestDevinOptionsFromAgent(t *testing.T) { + options, err := devinOptionsFromAgent("/opt/devin", []string{"--model", "opus", "-p", "fix it"}, "runtime.json", io.Discard, io.Discard) + if err != nil { + t.Fatal(err) + } + if options.Executable != "/opt/devin" || options.Model != "opus" || options.Prompt != "fix it" || options.Runtime != "runtime.json" { + t.Fatalf("options = %+v", options) + } + if _, err := devinOptionsFromAgent("claude", []string{"-p", "x"}, "r", io.Discard, io.Discard); err == nil { + t.Fatal("non-Devin executable accepted") + } + if _, err := devinOptionsFromAgent("devin", []string{"--resume", "x"}, "r", io.Discard, io.Discard); err == nil || !strings.Contains(err.Error(), "unsupported Devin argument") { + t.Fatalf("unsupported flag error = %v", err) + } +} + +func TestDevTranslatesDevinPrintInvocationToACP(t *testing.T) { + original := runDevinACP + t.Cleanup(func() { runDevinACP = original }) + var captured devinacp.Options + runDevinACP = func(_ context.Context, options devinacp.Options) error { + captured = options + return nil + } + project := t.TempDir() + role := buildTestRole(t) + writeFragment(t, project, "consumer", fmt.Sprintf(`consumers: + - id: test-consumer + command: [%q, "--role", "consumer"] + events: ["action.requested"] + projection: {content: omit} +`, role)) + t.Setenv("PITOT_RUNTIME", "") + t.Chdir(project) + var stdout, stderr lockedBuffer + if err := runDev(context.Background(), []string{ + "--host", "devin", "--", "devin", "--model", "swe-1.6", "-p", "fix it", + }, &stdout, &stderr); err != nil { + t.Fatalf("pitot dev: %v\n%s", err, stderr.String()) + } + if captured.Executable != "devin" || captured.Prompt != "fix it" || captured.Model != "swe-1.6" || + !strings.Contains(captured.Runtime, "pitot-dev-") { + t.Fatalf("ACP options = %+v", captured) + } +} + +func TestDevinInitAndHookGuidance(t *testing.T) { + t.Chdir(t.TempDir()) + out, err := runInitHost(t, "devin") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "stateful ACP boundary") || !strings.Contains(out, "pitot dev --host devin") { + t.Fatalf("guidance = %s", out) + } + var stdout, stderr bytes.Buffer + err = runHook(context.Background(), []string{"devin"}, strings.NewReader(`{}`), &stdout, &stderr) + if err == nil || !strings.Contains(err.Error(), "uses ACP") { + t.Fatalf("hook error = %v", err) + } +} + +func TestDoctorDevinChecksPinnedACPBinary(t *testing.T) { + originalLookPath, originalCommand := doctorLookPath, doctorCommand + t.Cleanup(func() { + doctorLookPath, doctorCommand = originalLookPath, originalCommand + }) + doctorLookPath = func(string) (string, error) { return os.Args[0], nil } + doctorCommand = func(_ string, args ...string) *exec.Cmd { + scenario := strings.Join(args, " ") + command := exec.Command(os.Args[0], "-test.run=TestDoctorDevinHelperProcess", "--") + command.Env = append(os.Environ(), "PITOT_DOCTOR_DEVIN_HELPER="+scenario) + return command + } + var stdout, stderr bytes.Buffer + if err := doctorHost("devin", &stdout, &stderr); err != nil { + t.Fatal(err) + } + if output := stdout.String(); !strings.Contains(output, "3000.3.22") || !strings.Contains(output, "ACP v1 boundary: FOUND") { + t.Fatalf("doctor output = %s", output) + } +} + +func TestDoctorDevinRejectsUnsupportedVersion(t *testing.T) { + originalLookPath, originalCommand := doctorLookPath, doctorCommand + t.Cleanup(func() { + doctorLookPath, doctorCommand = originalLookPath, originalCommand + }) + doctorLookPath = func(string) (string, error) { return os.Args[0], nil } + doctorCommand = func(_ string, args ...string) *exec.Cmd { + command := exec.Command(os.Args[0], "-test.run=TestDoctorDevinHelperProcess", "--") + command.Env = append(os.Environ(), "PITOT_DOCTOR_DEVIN_HELPER=old "+strings.Join(args, " ")) + return command + } + if err := doctorHost("devin", io.Discard, io.Discard); err == nil || !strings.Contains(err.Error(), "unsupported") { + t.Fatalf("doctor error = %v", err) + } +} + +func TestDoctorDevinHelperProcess(t *testing.T) { + scenario := os.Getenv("PITOT_DOCTOR_DEVIN_HELPER") + if scenario == "" { + return + } + switch { + case strings.Contains(scenario, "--version") && strings.HasPrefix(scenario, "old "): + _, _ = io.WriteString(os.Stdout, "devin 2999.0.0 (old)\n") + case strings.Contains(scenario, "--version"): + _, _ = io.WriteString(os.Stdout, "devin 3000.3.22 (test)\n") + case strings.Contains(scenario, "acp --help"): + _, _ = io.WriteString(os.Stdout, "Run an Agent Client Protocol server\n") + default: + os.Exit(2) + } + os.Exit(0) +} diff --git a/labs/15-pitot/pitot/cmd/pitot/doctor_host.go b/labs/15-pitot/pitot/cmd/pitot/doctor_host.go index 6d66501e9..aa7cdfb03 100644 --- a/labs/15-pitot/pitot/cmd/pitot/doctor_host.go +++ b/labs/15-pitot/pitot/cmd/pitot/doctor_host.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "io" "os" @@ -45,6 +46,9 @@ var hostProbes = map[adapters.Host]hostProbe{ }, } +var doctorLookPath = exec.LookPath +var doctorCommand = exec.Command + // doctorHost reports whether a host is configured to route its blocking shell // boundary to Pitot. It never edits configuration without an explicit --fix — // plain doctor only inspects and reports, returning a non-nil error when a @@ -54,6 +58,9 @@ func doctorHost(host adapters.Host, stdout, stderr io.Writer) error { return fmt.Errorf("pitot doctor: unsupported host %q (want one of: %s)", host, hostList()) } fmt.Fprintf(stdout, "Pitot %s — host check: %s\n", adapters.AdapterVersion, host) + if host == adapters.Devin { + return doctorDevin(stdout, stderr) + } // Repo-wireable hosts report their wiring state against the fragment // witness (drift-is-named-not-silently-fixed). @@ -139,3 +146,31 @@ func doctorHost(host adapters.Host, stdout, stderr io.Writer) error { fmt.Fprintf(stdout, " ready: %s can route its shell boundary to Pitot\n", host) return nil } + +func doctorDevin(stdout, stderr io.Writer) error { + const supportedVersion = "3000.3.22" + path, err := doctorLookPath("devin") + if err != nil { + fmt.Fprintln(stdout, " binary on PATH: NOT FOUND (devin)") + return errors.New("pitot doctor: devin host check found 1 issue: \"devin\" not on PATH") + } + fmt.Fprintf(stdout, " binary on PATH: %s\n", path) + version, versionErr := doctorCommand(path, "--version").CombinedOutput() + if versionErr != nil { + fmt.Fprintf(stderr, " devin --version: %s\n", strings.TrimSpace(string(version))) + return fmt.Errorf("pitot doctor: devin --version failed: %w", versionErr) + } + versionText := strings.TrimSpace(string(version)) + fmt.Fprintf(stdout, " version: %s\n", versionText) + if !strings.Contains(versionText, supportedVersion) { + return fmt.Errorf("pitot doctor: Devin version %q is unsupported; install %s", versionText, supportedVersion) + } + help, helpErr := doctorCommand(path, "acp", "--help").CombinedOutput() + if helpErr != nil || !strings.Contains(string(help), "Agent Client Protocol") { + return errors.New("pitot doctor: installed Devin does not advertise the required ACP server") + } + fmt.Fprintln(stdout, " ACP v1 boundary: FOUND (`devin acp`)") + fmt.Fprintln(stdout, " wiring: none required; use `pitot dev --host devin -- devin -p \"\"`") + fmt.Fprintln(stdout, " ready: Devin can route its exec permission boundary to Pitot") + return nil +} diff --git a/labs/15-pitot/pitot/cmd/pitot/main.go b/labs/15-pitot/pitot/cmd/pitot/main.go index 9d9b7b9a5..167ce733f 100644 --- a/labs/15-pitot/pitot/cmd/pitot/main.go +++ b/labs/15-pitot/pitot/cmd/pitot/main.go @@ -45,6 +45,8 @@ func runWithIO(ctx context.Context, args []string, stdin io.Reader, stdout, stde return runInit(args[1:], stdin, stdout, stderr) case "dev": return runDev(ctx, args[1:], stdout, stderr) + case "acp": + return runACP(ctx, args[1:], stdout, stderr) case "doctor": return doctor(args[1:], stdout, stderr) case "run": @@ -77,6 +79,9 @@ func runHook(ctx context.Context, args []string, stdin io.Reader, stdout, stderr if !adapters.IsSupported(host) { return fmt.Errorf("pitot: unsupported hook host %q", host) } + if transport, _ := adapters.BoundaryTransport(host); transport == adapters.TransportACP { + return fmt.Errorf("pitot: host %q uses ACP, not lifecycle hooks; use `pitot acp devin` or `pitot dev --host devin`", host) + } runtimePath, err := parseRuntimeFlag(args[1:], false) if err != nil { return err @@ -235,7 +240,7 @@ func doctor(args []string, stdout, stderr io.Writer) error { fmt.Fprintf(stdout, "binary version: %s (%s)\n", releaseVersion(), buildCommit()) fmt.Fprintf(stdout, "adapter version: %s\n", adapters.AdapterVersion) fmt.Fprintln(stdout, "unauthenticated local socket: none") - fmt.Fprintln(stdout, "runtime capabilities: hook_control consumer_delivery explicit_request") + fmt.Fprintln(stdout, "runtime capabilities: action_control consumer_delivery explicit_request") printHydrationStatus(stdout) fmt.Fprintln(stdout, "hosts:") for _, host := range adapters.Supported() { @@ -332,6 +337,7 @@ usage: pitot init [--language python|typescript|go|rust] [--role consumer|controller] [--template shell-policy|release-approval|blank-controller|blank-consumer] [--dir PATH] [--fragment NAME] [--force] pitot init --host HOST [--force] pitot dev --host HOST -- AGENT [ARGS...] + pitot acp devin --runtime PATH --prompt TEXT [--exec PATH] [--model MODEL] [--agent-type TYPE] [--cwd PATH] pitot doctor [--host HOST] [--fix] pitot run [--config PATH] --runtime PATH pitot hook HOST [--runtime PATH] diff --git a/labs/15-pitot/pitot/cmd/pitot/main_test.go b/labs/15-pitot/pitot/cmd/pitot/main_test.go index f252e71bc..118b75884 100644 --- a/labs/15-pitot/pitot/cmd/pitot/main_test.go +++ b/labs/15-pitot/pitot/cmd/pitot/main_test.go @@ -38,7 +38,7 @@ func TestDoctorReportsBoundary(t *testing.T) { t.Fatalf("doctor: %v", err) } out := stdout.String() - for _, want := range []string{"local boundary", "claude", "codex", "copilot", "cursor", "gemini", "kimi", "opencode", "pi", "qwen", "decoder=PASS", "unauthenticated local socket: none", "hook_control consumer_delivery explicit_request"} { + for _, want := range []string{"local boundary", "claude", "codex", "copilot", "cursor", "devin", "gemini", "kimi", "opencode", "pi", "qwen", "decoder=PASS", "unauthenticated local socket: none", "action_control consumer_delivery explicit_request"} { if !strings.Contains(out, want) { t.Errorf("doctor output missing %q\n%s", want, out) } diff --git a/labs/15-pitot/pitot/cmd/pitot/wire.go b/labs/15-pitot/pitot/cmd/pitot/wire.go index a4c6f61a1..808c3eddd 100644 --- a/labs/15-pitot/pitot/cmd/pitot/wire.go +++ b/labs/15-pitot/pitot/cmd/pitot/wire.go @@ -104,6 +104,7 @@ func runWireHost(host string, force bool, stdout io.Writer) error { // documented wiring instead of editing $HOME. func printWiringGuidance(host string, stdout io.Writer) error { guidance := map[string]string{ + "devin": "no hook configuration is required. Run:\n\n pitot dev --host devin -- devin -p \"\"\n\nFor a separately managed runtime, use:\n\n pitot acp devin --runtime \"$PITOT_RUNTIME\" --prompt \"\"", "kimi": "add to ~/.kimi-code/config.toml (or $KIMI_CODE_HOME/config.toml):\n\n [[hooks]]\n event = \"PreToolUse\"\n matcher = \"Bash\"\n command = \"pitot hook kimi\"", "qwen": "add to ~/.qwen/settings.json under hooks.PreToolUse:\n\n {\"matcher\": \"^Bash$\", \"hooks\": [{\"type\": \"command\", \"command\": \"\"}]}", "copilot": "add to ~/.copilot/settings.json under hooks.PreToolUse:\n\n {\"matcher\": \"Bash\", \"hooks\": [{\"type\": \"command\", \"command\": \"\"}]}", @@ -112,9 +113,13 @@ func printWiringGuidance(host string, stdout io.Writer) error { } text, known := guidance[host] if !known { - return fmt.Errorf("pitot init: unsupported host %q (want one of: %s, or a guidance-only host: kimi, qwen, copilot, opencode, pi)", host, joinSorted(wiring.RepoHosts())) + return fmt.Errorf("pitot init: unsupported host %q (want one of: %s, or a guidance-only host: devin, kimi, qwen, copilot, opencode, pi)", host, joinSorted(wiring.RepoHosts())) + } + if host == "devin" { + fmt.Fprintln(stdout, "devin uses a stateful ACP boundary; Pitot does not create lifecycle-hook configuration.") + } else { + fmt.Fprintf(stdout, "%s is configured at user level; Pitot does not edit files outside the repository.\n", host) } - fmt.Fprintf(stdout, "%s is configured at user level; Pitot does not edit files outside the repository.\n", host) fmt.Fprintf(stdout, "To wire it, %s\n\nThen verify: pitot doctor --host %s\n", text, host) return nil } diff --git a/labs/15-pitot/pitot/cmd/pitot/workbench.go b/labs/15-pitot/pitot/cmd/pitot/workbench.go index c447bd88d..d04746549 100644 --- a/labs/15-pitot/pitot/cmd/pitot/workbench.go +++ b/labs/15-pitot/pitot/cmd/pitot/workbench.go @@ -14,6 +14,7 @@ import ( "github.com/operatorstack/pitot/adapters" "github.com/operatorstack/pitot/config" + "github.com/operatorstack/pitot/internal/devinacp" "github.com/operatorstack/pitot/runtime" ) @@ -820,6 +821,14 @@ func runDev(ctx context.Context, args []string, stdout, stderr io.Writer) error program = fields[0] programArgs = fields[1:] } + var devinOptions *devinacp.Options + if adapters.Host(host) == adapters.Devin { + options, optionsErr := devinOptionsFromAgent(program, programArgs, "", stdout, stderr) + if optionsErr != nil { + return optionsErr + } + devinOptions = &options + } root, err := config.FindRoot(".") if err != nil { @@ -876,13 +885,18 @@ func runDev(ctx context.Context, args []string, stdout, stderr io.Writer) error fmt.Fprintf(stdout, "Runtime ready. Starting agent: %s %s\n", program, strings.Join(programArgs, " ")) fmt.Fprintln(stdout, "Decisions:") - cmd := exec.CommandContext(devCtx, program, programArgs...) - cmd.Env = append(os.Environ(), "PITOT_RUNTIME="+runtimePath) - cmd.Stdout = stdout - cmd.Stderr = stderr - cmd.Stdin = os.Stdin - - runErr := cmd.Run() + var runErr error + if devinOptions != nil { + devinOptions.Runtime = runtimePath + runErr = runDevinACP(devCtx, *devinOptions) + } else { + cmd := exec.CommandContext(devCtx, program, programArgs...) + cmd.Env = append(os.Environ(), "PITOT_RUNTIME="+runtimePath) + cmd.Stdout = stdout + cmd.Stderr = stderr + cmd.Stdin = os.Stdin + runErr = cmd.Run() + } // Stop the runtime and reap its goroutine before returning. cancel() diff --git a/labs/15-pitot/pitot/docs/devin-adapter-research.md b/labs/15-pitot/pitot/docs/devin-adapter-research.md new file mode 100644 index 000000000..2624f4973 --- /dev/null +++ b/labs/15-pitot/pitot/docs/devin-adapter-research.md @@ -0,0 +1,128 @@ +# Devin CLI adapter research + +Research date: 2026-07-31. + +## Question + +Pitot support requires a synchronous shell-control boundary that carries a +proposed command to a Controller, applies allow or deny before execution, and +lets the agent continue from the resulting tool outcome. The investigation +tested Devin's hook and ACP surfaces separately rather than assuming every host +must use Pitot's existing one-shot hook transport. + +## Released binary + +The stable manifest at + selected version +`3000.3.22`. The pinned `aarch64-apple-darwin` archive had the published +SHA-256 `eefe1f3c970c06d58b5ec2612ac5b36cd350635e0a2e59eb4d4f5baf43427662`. +The extracted binary reported `devin 3000.3.22 (d5152ff5)`. + +## Surface 1: lifecycle hooks + +Devin documents project hooks in `.devin/hooks.v1.json`. `PreToolUse` runs +before a tool executes, shell calls use tool name `exec`, and the command is +available as `tool_input.command`. A command hook can block with exit `2` or a +JSON block decision. + +A deterministic Connect/protobuf endpoint drove the pinned binary through a +real `exec` call in noninteractive `--print` mode. Pitot's hook, Consumer, and +Controller all observed the request; denial prevented the canary command from +executing. Devin then ended the turn instead of returning the denied tool +outcome to the model. The documented JSON block response behaved the same way. + +`PermissionRequest` was also tested in normal noninteractive mode. Devin +rejected the confirmation-required tool call itself and did not invoke the +configured permission hook. Therefore neither hook variant completes Pitot's +causal loop in `--print` mode. + +Hooks remain unsuitable for a Devin adapter. The candidate `.devin` wiring is +not shipped. + +## Surface 2: Agent Client Protocol + +`devin acp` runs Devin as an ACP server over stdio JSON-RPC. ACP is +bidirectional: the agent sends `session/request_permission`, and the client +selects a typed permission option. Official ACP clients may enforce policy by +automatically selecting an allow or reject option. + +The pinned Devin binary was launched through the official Python ACP SDK +(`agent-client-protocol`, protocol version 1) against the same deterministic +model boundary. The model proposed: + +```text +touch /tmp/PITOT_ACP_DENY_CANARY +``` + +The observed sequence was: + +1. Devin emitted a `tool_call` update with tool-call ID `pitot_acp_deny`, + kind `execute`, inference tool `exec`, and direct + `rawInput.command`. +2. Devin sent `session/request_permission` for the same tool-call ID with + `allow_once` and `reject_once` options. +3. The client selected `reject_once`. +4. The canary file remained absent. +5. Devin emitted a `tool_call_update` with status `failed`, + `cognition.ai/rejected: true`, and the text + `Tool execution was rejected: User rejected this tool call`. +6. Devin made the next main model request. That request contained the original + tool-call ID and the rejection terms `reject`, `denied`, and `permission`. +7. Devin emitted the model's final response and ended the prompt with + `stopReason: end_turn`. + +The content-safe receipt is +[`evidence/devin-acp-3000.3.22.json`](evidence/devin-acp-3000.3.22.json). + +This is the missing causal behavior. The stable changelog independently notes +that skipping a tool call through an ACP client no longer stops the agent and +that the model sees the rejection and can try an alternative. + +Primary sources: + +- +- +- +- +- + +## SDK and protocol implications + +ACP's official SDKs are Rust, Python, TypeScript, Kotlin, and Java. There is no +official Go SDK. Pitot does not need to change its public event or control +envelopes: the direct command still normalizes to `action.requested`, and the +Controller still returns `allow` or `deny`. + +The shipped implementation is a stateful Devin transport adapter: + +- launch `devin acp` and perform ACP initialization plus session creation; +- correlate each `tool_call` update's `rawInput.command` by `toolCallId`; +- on `session/request_permission`, deliver the normalized shell event to the + existing Pitot runtime; +- map `allow` to ACP `allow_once` and `deny` to `reject_once`; +- preserve ACP session output and lifecycle until `session/prompt` completes. + +The adapter must fail closed when the tool-call ID is unknown, the command was +not directly observed, a required permission option is absent, or the ACP +protocol version is unsupported. It must never select persistent permission +options such as `allow_always` or bypass mode. + +## Revised verdict + +Devin is supportable through ACP, not through lifecycle hooks. This does not +require weakening Pitot's supervised causal-loop requirement or changing +protocol version 1. It does require broadening the adapter architecture from +only one-shot host hooks to include a stateful host transport. + +The two-slice ZCA projection separates: + +1. **Control semantics:** Devin ACP satisfies pre-execution allow/deny and + post-denial continuation. +2. **Integration transport:** Pitot uses a Devin-specific ACP client boundary, + registered as `acp`, while hook-oriented Go APIs remain compatibility + aliases. + +The immediate value is that Pitot supports Devin without treating its existing +hook shape as the protocol itself. The pinned production adapter is supervised +with real-agent allow/deny, Consumer/Controller, canary, continuation, and +incompatible-response checks. diff --git a/labs/15-pitot/pitot/docs/evidence/devin-acp-3000.3.22.json b/labs/15-pitot/pitot/docs/evidence/devin-acp-3000.3.22.json new file mode 100644 index 000000000..d4ebea06b --- /dev/null +++ b/labs/15-pitot/pitot/docs/evidence/devin-acp-3000.3.22.json @@ -0,0 +1,47 @@ +{ + "schema_version": 1, + "observed_at": "2026-07-31", + "cli": { + "name": "devin", + "version": "3000.3.22", + "build": "d5152ff5", + "archive_sha256": "eefe1f3c970c06d58b5ec2612ac5b36cd350635e0a2e59eb4d4f5baf43427662", + "executable_sha256": "1252ab77359e1cc63c23fda7a88ff7711bab5f853c1ab6d3323cb319b71a60ee" + }, + "transport": { + "command": "devin acp", + "protocol": "agent-client-protocol", + "protocol_version": 1, + "sdk": "official Python agent-client-protocol" + }, + "probe": { + "tool_call_id": "pitot_acp_deny", + "tool": "exec", + "command": "touch /tmp/PITOT_ACP_DENY_CANARY", + "permission_options_observed": [ + "allow_once", + "allow_session", + "allow_always", + "allow_always_global", + "switch_bypass", + "reject_once" + ], + "selected_option": "reject_once", + "canary_absent": true, + "rejected_update": { + "status": "failed", + "rejected": true, + "text": "Tool execution was rejected: User rejected this tool call" + }, + "next_main_model_request": { + "tool_call_id_observed": true, + "rejection_terms_observed": [ + "reject", + "denied", + "permission" + ] + }, + "final_agent_message": "PITOT_ACP_DENIAL_CONTINUED", + "stop_reason": "end_turn" + } +} diff --git a/labs/15-pitot/pitot/internal/devinacp/client.go b/labs/15-pitot/pitot/internal/devinacp/client.go new file mode 100644 index 000000000..650a33503 --- /dev/null +++ b/labs/15-pitot/pitot/internal/devinacp/client.go @@ -0,0 +1,397 @@ +// Package devinacp implements Pitot's single-turn Devin ACP transport. +package devinacp + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os/exec" + "path/filepath" + "sync" + "time" + + "github.com/operatorstack/pitot/adapters" + "github.com/operatorstack/pitot/projection" + "github.com/operatorstack/pitot/runtime" + "github.com/operatorstack/pitot/schema" + "github.com/operatorstack/pitot/sensor" +) + +const maxMessageBytes = 4 << 20 + +// DeliverFunc routes a normalized action through the Pitot runtime. +type DeliverFunc func(context.Context, schema.Event) (*schema.ControlResponse, error) + +// Options configures one Devin ACP prompt. +type Options struct { + Executable string + Prompt string + Cwd string + Model string + AgentType string + Runtime string + Stdout io.Writer + Stderr io.Writer + Deliver DeliverFunc + command func(context.Context, string, ...string) *exec.Cmd +} + +type rpcMessage struct { + JSONRPC string `json:"jsonrpc"` + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *rpcError `json:"error,omitempty"` +} + +type rpcError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +type client struct { + ctx context.Context + stdin io.WriteCloser + scanner *bufio.Scanner + stdout io.Writer + deliver DeliverFunc + commands map[string]string + nextID int + writeMu sync.Mutex +} + +// Run launches Devin as an ACP subprocess and completes one prompt turn. +func Run(ctx context.Context, options Options) error { + if options.Executable == "" { + options.Executable = "devin" + } + if options.Prompt == "" { + return errors.New("pitot acp: --prompt is required") + } + if options.Stdout == nil { + options.Stdout = io.Discard + } + if options.Stderr == nil { + options.Stderr = io.Discard + } + if options.Cwd == "" { + options.Cwd = "." + } + deliver := options.Deliver + if deliver == nil { + if options.Runtime == "" { + return errors.New("pitot acp: runtime is required") + } + runtimeClient, err := runtime.OpenClient(options.Runtime) + if err != nil { + return fmt.Errorf("pitot acp: open runtime: %w", err) + } + deliver = runtimeClient.DeliverEvent + } + + args := []string{"acp"} + if options.AgentType != "" { + args = append(args, "--agent-type", options.AgentType) + } + if options.Model != "" { + args = append(args, "--model", options.Model) + } + commandBuilder := options.command + if commandBuilder == nil { + commandBuilder = exec.CommandContext + } + command := commandBuilder(ctx, options.Executable, args...) + command.Dir = options.Cwd + stdin, err := command.StdinPipe() + if err != nil { + return fmt.Errorf("pitot acp: child stdin: %w", err) + } + stdout, err := command.StdoutPipe() + if err != nil { + return fmt.Errorf("pitot acp: child stdout: %w", err) + } + command.Stderr = options.Stderr + if err := command.Start(); err != nil { + return fmt.Errorf("pitot acp: start %s: %w", options.Executable, err) + } + + scanner := bufio.NewScanner(stdout) + scanner.Buffer(make([]byte, 64<<10), maxMessageBytes) + transport := &client{ + ctx: ctx, + stdin: stdin, + scanner: scanner, + stdout: options.Stdout, + deliver: deliver, + commands: map[string]string{}, + } + runErr := transport.run(options) + _ = stdin.Close() + waitErr := waitForChild(command) + if runErr != nil { + return runErr + } + if waitErr != nil { + return fmt.Errorf("pitot acp: Devin exited: %w", waitErr) + } + return nil +} + +func waitForChild(command *exec.Cmd) error { + done := make(chan error, 1) + go func() { done <- command.Wait() }() + select { + case err := <-done: + return err + case <-time.After(2 * time.Second): + _ = command.Process.Kill() + <-done + return nil + } +} + +func (c *client) run(options Options) error { + var initialized struct { + ProtocolVersion int `json:"protocolVersion"` + } + if err := c.call("initialize", map[string]any{ + "protocolVersion": 1, + "clientCapabilities": map[string]any{}, + "clientInfo": map[string]string{ + "name": "pitot", "title": "Pitot", "version": adapters.AdapterVersion, + }, + }, &initialized); err != nil { + return err + } + if initialized.ProtocolVersion != 1 { + return fmt.Errorf("pitot acp: unsupported negotiated protocol version %d", initialized.ProtocolVersion) + } + + absoluteCwd, err := filepath.Abs(options.Cwd) + if err != nil { + return fmt.Errorf("pitot acp: resolve cwd: %w", err) + } + var session struct { + SessionID string `json:"sessionId"` + } + if err := c.call("session/new", map[string]any{"cwd": absoluteCwd, "mcpServers": []any{}}, &session); err != nil { + return err + } + if session.SessionID == "" { + return errors.New("pitot acp: Devin returned an empty session id") + } + var promptResult struct { + StopReason string `json:"stopReason"` + } + if err := c.call("session/prompt", map[string]any{ + "sessionId": session.SessionID, + "prompt": []map[string]string{{"type": "text", "text": options.Prompt}}, + }, &promptResult); err != nil { + return err + } + if promptResult.StopReason == "" { + return errors.New("pitot acp: Devin prompt completed without a stop reason") + } + return nil +} + +func (c *client) call(method string, params any, target any) error { + c.nextID++ + id := c.nextID + if err := c.write(map[string]any{"jsonrpc": "2.0", "id": id, "method": method, "params": params}); err != nil { + return err + } + for c.scanner.Scan() { + var message rpcMessage + if err := json.Unmarshal(c.scanner.Bytes(), &message); err != nil { + return fmt.Errorf("pitot acp: malformed JSON-RPC message: %w", err) + } + if message.JSONRPC != "2.0" { + return fmt.Errorf("pitot acp: unsupported JSON-RPC version %q", message.JSONRPC) + } + if message.Method != "" { + if err := c.handleInbound(message); err != nil { + return err + } + continue + } + var responseID int + if len(message.ID) == 0 || json.Unmarshal(message.ID, &responseID) != nil || responseID != id { + return errors.New("pitot acp: unexpected JSON-RPC response id") + } + if message.Error != nil { + return fmt.Errorf("pitot acp: %s failed (%d): %s", method, message.Error.Code, message.Error.Message) + } + if err := json.Unmarshal(message.Result, target); err != nil { + return fmt.Errorf("pitot acp: decode %s response: %w", method, err) + } + return nil + } + if err := c.scanner.Err(); err != nil { + return fmt.Errorf("pitot acp: read Devin output: %w", err) + } + return errors.New("pitot acp: Devin closed the protocol stream") +} + +func (c *client) handleInbound(message rpcMessage) error { + switch message.Method { + case "session/update": + return c.handleUpdate(message.Params) + case "session/request_permission": + if len(message.ID) == 0 { + return errors.New("pitot acp: permission request omitted its JSON-RPC id") + } + return c.handlePermission(message.ID, message.Params) + default: + if len(message.ID) == 0 { + return nil + } + return c.write(map[string]any{ + "jsonrpc": "2.0", + "id": json.RawMessage(message.ID), + "error": rpcError{Code: -32601, Message: "unsupported ACP client method"}, + }) + } +} + +func (c *client) handleUpdate(raw json.RawMessage) error { + var params struct { + SessionID string `json:"sessionId"` + Update struct { + SessionUpdate string `json:"sessionUpdate"` + ToolCallID string `json:"toolCallId"` + RawInput map[string]any `json:"rawInput"` + Content json.RawMessage `json:"content"` + Meta map[string]any `json:"_meta"` + } `json:"update"` + } + if err := json.Unmarshal(raw, ¶ms); err != nil { + return fmt.Errorf("pitot acp: malformed session update: %w", err) + } + switch params.Update.SessionUpdate { + case "tool_call": + command, _ := params.Update.RawInput["command"].(string) + tool, _ := params.Update.Meta["cognition.ai/inferenceToolName"].(string) + if params.SessionID != "" && params.Update.ToolCallID != "" && tool == "exec" && command != "" { + c.commands[commandKey(params.SessionID, params.Update.ToolCallID)] = command + } + case "agent_message_chunk": + var content struct { + Type string `json:"type"` + Text string `json:"text"` + } + if err := json.Unmarshal(params.Update.Content, &content); err != nil { + return fmt.Errorf("pitot acp: malformed agent message chunk: %w", err) + } + if content.Type == "text" && content.Text != "" { + if _, err := io.WriteString(c.stdout, content.Text); err != nil { + return fmt.Errorf("pitot acp: write agent output: %w", err) + } + } + } + return nil +} + +type permissionOption struct { + OptionID string `json:"optionId"` + Kind string `json:"kind"` +} + +func (c *client) handlePermission(id json.RawMessage, raw json.RawMessage) error { + var optionEnvelope struct { + Options []permissionOption `json:"options"` + } + _ = json.Unmarshal(raw, &optionEnvelope) + var params struct { + SessionID string `json:"sessionId"` + ToolCall struct { + ToolCallID string `json:"toolCallId"` + } `json:"toolCall"` + Options []permissionOption `json:"options"` + } + if err := json.Unmarshal(raw, ¶ms); err != nil { + if reject := optionFor(optionEnvelope.Options, "reject_once"); reject != "" { + _ = c.permissionResponse(id, reject, false) + } + return fmt.Errorf("pitot acp: malformed permission request: %w", err) + } + reject := optionFor(params.Options, "reject_once") + if reject == "" { + _ = c.permissionResponse(id, "", true) + return errors.New("pitot acp: permission request omitted reject_once") + } + command := c.commands[commandKey(params.SessionID, params.ToolCall.ToolCallID)] + if command == "" { + return c.permissionResponse(id, reject, false) + } + + payload, _ := json.Marshal(map[string]any{ + "hook_event_name": "session/request_permission", + "tool_name": "exec", + "tool_input": map[string]string{"command": command}, + }) + event, err := sensor.Decode(adapters.Devin, payload, projection.Full) + if err != nil { + return c.permissionResponse(id, reject, false) + } + actionID, err := runtime.NewActionID() + if err != nil { + return c.permissionResponse(id, reject, false) + } + event.Action.ID = actionID + event.SessionID = params.SessionID + response, deliverErr := c.deliver(c.ctx, event) + allow := deliverErr == nil && (response == nil || response.Outcome == schema.OutcomeAllow) + if response != nil && response.ActionID != actionID { + allow = false + } + if allow { + if option := optionFor(params.Options, "allow_once"); option != "" { + delete(c.commands, commandKey(params.SessionID, params.ToolCall.ToolCallID)) + return c.permissionResponse(id, option, false) + } + } + delete(c.commands, commandKey(params.SessionID, params.ToolCall.ToolCallID)) + return c.permissionResponse(id, reject, false) +} + +func (c *client) permissionResponse(id json.RawMessage, optionID string, cancelled bool) error { + outcome := map[string]any{"outcome": "cancelled"} + if !cancelled { + outcome = map[string]any{"outcome": "selected", "optionId": optionID} + } + return c.write(map[string]any{ + "jsonrpc": "2.0", + "id": json.RawMessage(id), + "result": map[string]any{"outcome": outcome}, + }) +} + +func optionFor(options []permissionOption, kind string) string { + for _, option := range options { + if option.Kind == kind { + return option.OptionID + } + } + return "" +} + +func commandKey(sessionID, toolCallID string) string { return sessionID + "\x00" + toolCallID } + +func (c *client) write(value any) error { + c.writeMu.Lock() + defer c.writeMu.Unlock() + encoded, err := json.Marshal(value) + if err != nil { + return err + } + encoded = append(encoded, '\n') + if _, err := c.stdin.Write(encoded); err != nil { + return fmt.Errorf("pitot acp: write Devin input: %w", err) + } + return nil +} diff --git a/labs/15-pitot/pitot/internal/devinacp/client_test.go b/labs/15-pitot/pitot/internal/devinacp/client_test.go new file mode 100644 index 000000000..18e49c114 --- /dev/null +++ b/labs/15-pitot/pitot/internal/devinacp/client_test.go @@ -0,0 +1,330 @@ +package devinacp + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strings" + "sync" + "testing" + "time" + + "github.com/operatorstack/pitot/schema" +) + +func TestRunAllowDenyAndContinue(t *testing.T) { + var mu sync.Mutex + var events []schema.Event + var calls int + deliver := func(_ context.Context, event schema.Event) (*schema.ControlResponse, error) { + mu.Lock() + defer mu.Unlock() + events = append(events, event) + calls++ + outcome := schema.OutcomeAllow + if calls == 2 { + outcome = schema.OutcomeDeny + } + return &schema.ControlResponse{ActionID: event.Action.ID, Outcome: outcome}, nil + } + var output strings.Builder + err := Run(context.Background(), helperOptions(t, "allow-deny", deliver, &output)) + if err != nil { + t.Fatalf("Run: %v", err) + } + if output.String() != "PITOT_ACP_COMPLETE" { + t.Fatalf("output = %q", output.String()) + } + if len(events) != 2 { + t.Fatalf("events = %d, want 2", len(events)) + } + for index, event := range events { + if event.Host.Name != "devin" || event.Action == nil || !strings.HasPrefix(event.Action.ID, "act_") { + t.Errorf("event %d = %+v", index, event) + } + if event.SessionID != "session-1" || event.Observation.Source != schema.SourceHostEvent { + t.Errorf("event %d boundary metadata = %+v", index, event) + } + } + if string(events[0].Content.Full) != `"echo allow"` || string(events[1].Content.Full) != `"echo deny"` { + t.Errorf("commands = %s, %s", events[0].Content.Full, events[1].Content.Full) + } +} + +func TestRunRejectsUnsupportedPermissionWithoutDelivery(t *testing.T) { + delivered := false + options := helperOptions(t, "unsupported", func(context.Context, schema.Event) (*schema.ControlResponse, error) { + delivered = true + return nil, nil + }, io.Discard) + if err := Run(context.Background(), options); err != nil { + t.Fatalf("Run: %v", err) + } + if delivered { + t.Fatal("unsupported permission reached Pitot runtime") + } +} + +func TestRunFailsClosedWhenControllerFaults(t *testing.T) { + options := helperOptions(t, "controller-fault", func(context.Context, schema.Event) (*schema.ControlResponse, error) { + return nil, errors.New("unavailable") + }, io.Discard) + if err := Run(context.Background(), options); err != nil { + t.Fatalf("Run: %v", err) + } +} + +func TestRunAllowsOnceWithoutController(t *testing.T) { + options := helperOptions(t, "no-controller", func(context.Context, schema.Event) (*schema.ControlResponse, error) { + return nil, nil + }, io.Discard) + if err := Run(context.Background(), options); err != nil { + t.Fatalf("Run: %v", err) + } +} + +func TestRunRejectsMissingToolCorrelationWithoutDelivery(t *testing.T) { + delivered := false + options := helperOptions(t, "missing-correlation", func(context.Context, schema.Event) (*schema.ControlResponse, error) { + delivered = true + return nil, nil + }, io.Discard) + if err := Run(context.Background(), options); err != nil { + t.Fatalf("Run: %v", err) + } + if delivered { + t.Fatal("uncorrelated permission reached Pitot runtime") + } +} + +func TestRunRejectsMissingRejectOption(t *testing.T) { + options := helperOptions(t, "missing-reject", nil, io.Discard) + if err := Run(context.Background(), options); err == nil || !strings.Contains(err.Error(), "omitted reject_once") { + t.Fatalf("error = %v", err) + } +} + +func TestRunRejectsProtocolDrift(t *testing.T) { + options := helperOptions(t, "protocol-drift", nil, io.Discard) + if err := Run(context.Background(), options); err == nil || !strings.Contains(err.Error(), "protocol version 2") { + t.Fatalf("error = %v", err) + } +} + +func TestRunRejectsMalformedJSONRPC(t *testing.T) { + options := helperOptions(t, "malformed", nil, io.Discard) + if err := Run(context.Background(), options); err == nil || !strings.Contains(err.Error(), "malformed JSON-RPC") { + t.Fatalf("error = %v", err) + } +} + +func TestRunRejectsWrongJSONRPCVersion(t *testing.T) { + options := helperOptions(t, "wrong-jsonrpc", nil, io.Discard) + if err := Run(context.Background(), options); err == nil || !strings.Contains(err.Error(), "JSON-RPC version") { + t.Fatalf("error = %v", err) + } +} + +func TestRunRejectsMalformedPermissionBeforeStopping(t *testing.T) { + options := helperOptions(t, "malformed-permission", nil, io.Discard) + if err := Run(context.Background(), options); err == nil || !strings.Contains(err.Error(), "malformed permission") { + t.Fatalf("error = %v", err) + } +} + +func TestRunReportsChildFailure(t *testing.T) { + options := helperOptions(t, "child-failure", nil, io.Discard) + if err := Run(context.Background(), options); err == nil { + t.Fatal("expected child failure") + } +} + +func TestRunCancellationStopsChild(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + options := helperOptions(t, "block", nil, io.Discard) + if err := Run(ctx, options); err == nil { + t.Fatal("expected cancellation error") + } +} + +func helperOptions(t *testing.T, scenario string, deliver DeliverFunc, output io.Writer) Options { + t.Helper() + if deliver == nil { + deliver = func(context.Context, schema.Event) (*schema.ControlResponse, error) { return nil, nil } + } + return Options{ + Executable: os.Args[0], + Prompt: "test prompt", + Cwd: t.TempDir(), + Stdout: output, + Stderr: io.Discard, + Deliver: deliver, + command: func(ctx context.Context, _ string, _ ...string) *exec.Cmd { + command := exec.CommandContext(ctx, os.Args[0], "-test.run=TestACPHelperProcess", "--") + command.Env = append(os.Environ(), "PITOT_ACP_HELPER="+scenario) + return command + }, + } +} + +func TestACPHelperProcess(t *testing.T) { + scenario := os.Getenv("PITOT_ACP_HELPER") + if scenario == "" { + return + } + if scenario == "child-failure" { + os.Exit(17) + } + reader := bufio.NewScanner(os.Stdin) + writer := json.NewEncoder(os.Stdout) + readRequest := func() map[string]any { + if !reader.Scan() { + os.Exit(20) + } + var value map[string]any + if json.Unmarshal(reader.Bytes(), &value) != nil { + os.Exit(21) + } + return value + } + respond := func(request map[string]any, result any) { + _ = writer.Encode(map[string]any{"jsonrpc": "2.0", "id": request["id"], "result": result}) + } + + initialize := readRequest() + if scenario == "malformed" { + fmt.Fprintln(os.Stdout, "{broken") + return + } + if scenario == "wrong-jsonrpc" { + _ = writer.Encode(map[string]any{"jsonrpc": "1.0", "id": initialize["id"], "result": map[string]any{"protocolVersion": 1}}) + return + } + version := 1 + if scenario == "protocol-drift" { + version = 2 + } + respond(initialize, map[string]any{"protocolVersion": version}) + if version != 1 { + return + } + newSession := readRequest() + respond(newSession, map[string]any{"sessionId": "session-1"}) + prompt := readRequest() + if scenario == "block" { + _, _ = io.Copy(io.Discard, os.Stdin) + return + } + + switch scenario { + case "allow-deny": + emitTool(writer, "allow", "exec", "echo allow") + emitPermission(writer, 101, "allow", true) + expectPermission(reader, "allow") + emitTool(writer, "deny", "exec", "echo deny") + emitPermission(writer, 102, "deny", true) + expectPermission(reader, "reject") + _ = writer.Encode(map[string]any{"jsonrpc": "2.0", "method": "session/update", "params": map[string]any{ + "sessionId": "session-1", + "update": map[string]any{ + "sessionUpdate": "agent_message_chunk", + "content": map[string]any{"type": "text", "text": "PITOT_ACP_COMPLETE"}, + }, + }}) + case "unsupported": + emitTool(writer, "other", "browser", "") + emitPermission(writer, 101, "other", true) + expectPermission(reader, "reject") + case "controller-fault": + emitTool(writer, "fault", "exec", "echo fault") + emitPermission(writer, 101, "fault", true) + expectPermission(reader, "reject") + case "no-controller": + emitTool(writer, "allow", "exec", "echo allow") + emitPermission(writer, 101, "allow", true) + expectPermission(reader, "allow") + case "missing-correlation": + emitPermission(writer, 101, "missing", true) + expectPermission(reader, "reject") + case "malformed-permission": + _ = writer.Encode(map[string]any{"jsonrpc": "2.0", "id": 101, "method": "session/request_permission", "params": map[string]any{ + "sessionId": []string{"wrong-type"}, + "options": []map[string]string{ + {"optionId": "allow", "kind": "allow_once"}, + {"optionId": "reject", "kind": "reject_once"}, + }, + }}) + expectPermission(reader, "reject") + return + case "missing-reject": + emitTool(writer, "missing", "exec", "echo missing") + emitPermission(writer, 101, "missing", false) + expectCancelled(reader) + return + } + respond(prompt, map[string]any{"stopReason": "end_turn"}) + _, _ = io.Copy(io.Discard, os.Stdin) +} + +func emitTool(writer *json.Encoder, id, tool, command string) { + rawInput := map[string]any{} + if command != "" { + rawInput["command"] = command + } + _ = writer.Encode(map[string]any{"jsonrpc": "2.0", "method": "session/update", "params": map[string]any{ + "sessionId": "session-1", + "update": map[string]any{ + "sessionUpdate": "tool_call", + "toolCallId": id, + "rawInput": rawInput, + "_meta": map[string]any{"cognition.ai/inferenceToolName": tool}, + }, + }}) +} + +func emitPermission(writer *json.Encoder, requestID int, toolID string, reject bool) { + options := []map[string]string{{"optionId": "allow", "kind": "allow_once"}} + if reject { + options = append(options, map[string]string{"optionId": "reject", "kind": "reject_once"}) + } + _ = writer.Encode(map[string]any{"jsonrpc": "2.0", "id": requestID, "method": "session/request_permission", "params": map[string]any{ + "sessionId": "session-1", + "toolCall": map[string]string{"toolCallId": toolID}, + "options": options, + }}) +} + +func expectPermission(reader *bufio.Scanner, option string) { + value := readHelperResponse(reader) + result := value["result"].(map[string]any) + outcome := result["outcome"].(map[string]any) + if outcome["optionId"] != option { + os.Exit(30) + } +} + +func expectCancelled(reader *bufio.Scanner) { + value := readHelperResponse(reader) + result := value["result"].(map[string]any) + outcome := result["outcome"].(map[string]any) + if outcome["outcome"] != "cancelled" { + os.Exit(31) + } +} + +func readHelperResponse(reader *bufio.Scanner) map[string]any { + if !reader.Scan() { + os.Exit(32) + } + var value map[string]any + if json.Unmarshal(reader.Bytes(), &value) != nil { + os.Exit(33) + } + return value +} diff --git a/labs/15-pitot/pitot/runtime/capabilities.go b/labs/15-pitot/pitot/runtime/capabilities.go index c775e5057..15f25c302 100644 --- a/labs/15-pitot/pitot/runtime/capabilities.go +++ b/labs/15-pitot/pitot/runtime/capabilities.go @@ -4,12 +4,14 @@ package runtime type Capability string const ( - CapabilityHookControl Capability = "hook_control" + CapabilityActionControl Capability = "action_control" + // CapabilityHookControl is retained as a source-compatible alias. + CapabilityHookControl Capability = CapabilityActionControl CapabilityConsumerDelivery Capability = "consumer_delivery" CapabilityExplicitRequest Capability = "explicit_request" ) // Capabilities returns the canonical ordered shipped runtime inventory. func Capabilities() []Capability { - return []Capability{CapabilityHookControl, CapabilityConsumerDelivery, CapabilityExplicitRequest} + return []Capability{CapabilityActionControl, CapabilityConsumerDelivery, CapabilityExplicitRequest} } diff --git a/labs/15-pitot/pitot/sensor/sensor.go b/labs/15-pitot/pitot/sensor/sensor.go index 475ef33e9..0de152ee7 100644 --- a/labs/15-pitot/pitot/sensor/sensor.go +++ b/labs/15-pitot/pitot/sensor/sensor.go @@ -1,4 +1,4 @@ -// Package sensor normalizes raw host hook payloads into Pitot's event envelope. +// Package sensor normalizes raw host boundary payloads into Pitot's event envelope. // // It is the observation pipeline: it reports what a host supplied and marks the // quality of that observation. It never decides what an action means and it must @@ -39,7 +39,7 @@ func (e *FaultError) Fault(actionID string) schema.BoundaryFault { } } -// Decode normalizes a raw host hook payload into an action.requested event with +// Decode normalizes a raw host boundary payload into an action.requested event with // the requested content projection. A malformed payload or an omitted command // yields a *FaultError so the caller can emit a boundary fault rather than a // silently degraded event. @@ -51,14 +51,14 @@ func Decode(host adapters.Host, raw []byte, mode projection.Mode) (schema.Event, return schema.Event{}, fmt.Errorf("pitot: unsupported projection mode %q", mode) } - var event adapters.RawHookEvent + var event adapters.RawBoundaryEvent if err := json.Unmarshal(raw, &event); err != nil { return schema.Event{}, &FaultError{Host: host, Reason: schema.ReasonMalformed} } // An empty or mismatched event name is a malformed boundary, not a decision. eventName := host.EventNameFor(event) - if eventName != "" && !host.HasHookEvent(eventName) { + if eventName != "" && !host.HasBoundaryEvent(eventName) { return schema.Event{}, &FaultError{Host: host, Reason: schema.ReasonMalformed} } @@ -71,6 +71,10 @@ func Decode(host adapters.Host, raw []byte, mode projection.Mode) (schema.Event, if err != nil { return schema.Event{}, err } + source := schema.SourceHostHook + if transport, transportErr := adapters.BoundaryTransport(host); transportErr == nil && transport == adapters.TransportACP { + source = schema.SourceHostEvent + } return schema.Event{ PitotVersion: schema.Version, @@ -82,7 +86,7 @@ func Decode(host adapters.Host, raw []byte, mode projection.Mode) (schema.Event, Action: &schema.Action{Kind: host.ActionKind(eventName)}, Content: &content, Observation: schema.Observation{ - Source: schema.SourceHostHook, + Source: source, Fidelity: schema.FidelityDirect, }, }, nil diff --git a/labs/15-pitot/pitot/sensor/sensor_test.go b/labs/15-pitot/pitot/sensor/sensor_test.go index e31ddc942..3af724354 100644 --- a/labs/15-pitot/pitot/sensor/sensor_test.go +++ b/labs/15-pitot/pitot/sensor/sensor_test.go @@ -28,7 +28,11 @@ func TestDecodeCanonicalEvents(t *testing.T) { if event.Host.Name != string(host) { t.Errorf("%s: host name = %q", host, event.Host.Name) } - if event.Observation.Source != schema.SourceHostHook || event.Observation.Fidelity != schema.FidelityDirect { + wantSource := schema.SourceHostHook + if host == adapters.Devin { + wantSource = schema.SourceHostEvent + } + if event.Observation.Source != wantSource || event.Observation.Fidelity != schema.FidelityDirect { t.Errorf("%s: observation = %+v", host, event.Observation) } if event.Content == nil || event.Content.Mode != schema.ContentSHA256 || event.Content.SHA256 == "" { diff --git a/labs/15-pitot/pitot/windtunnel/windtunnel_test.go b/labs/15-pitot/pitot/windtunnel/windtunnel_test.go index 8dda1f946..86f3c5161 100644 --- a/labs/15-pitot/pitot/windtunnel/windtunnel_test.go +++ b/labs/15-pitot/pitot/windtunnel/windtunnel_test.go @@ -38,11 +38,7 @@ var boatstackCanonicalEvents = map[adapters.Host]string{ } func TestSensorConsumesBoatstackCanonicalEvents(t *testing.T) { - for _, host := range adapters.Supported() { - raw := boatstackCanonicalEvents[host] - if raw == "" { - t.Fatalf("no canonical event registered for host %s", host) - } + for host, raw := range boatstackCanonicalEvents { // The bytes Pitot's own adapter would emit must match what Boatstack emits. probe, err := adapters.CanonicalHookEvent(host) if err != nil { @@ -61,6 +57,30 @@ func TestSensorConsumesBoatstackCanonicalEvents(t *testing.T) { } } +func TestSensorConsumesPitotOnlyTransportBoundaries(t *testing.T) { + var pitotOnly []adapters.Host + for _, host := range adapters.Supported() { + if _, sharedWithBoatstack := boatstackCanonicalEvents[host]; sharedWithBoatstack { + continue + } + pitotOnly = append(pitotOnly, host) + probe, err := adapters.CanonicalBoundaryEvent(host) + if err != nil { + t.Fatalf("canonical boundary for %s: %v", host, err) + } + event, err := sensor.Decode(host, probe, projection.SHA256) + if err != nil { + t.Fatalf("sensor could not consume %s boundary: %v", host, err) + } + if event.Action == nil || event.Action.Kind != "shell" || event.Observation.Source != schema.SourceHostEvent { + t.Fatalf("host %s: unexpected boundary event %+v", host, event) + } + } + if len(pitotOnly) != 1 || pitotOnly[0] != adapters.Devin { + t.Fatalf("unexpected Pitot-only transport inventory: %v", pitotOnly) + } +} + func TestControlRoundTripThroughBridge(t *testing.T) { router := bridge.NewRouter() if err := router.Register(bridge.Registration{ diff --git a/labs/15-pitot/public-readme-preview/CONTRIBUTING.md b/labs/15-pitot/public-readme-preview/CONTRIBUTING.md index b9b856efa..45c38f7f2 100644 --- a/labs/15-pitot/public-readme-preview/CONTRIBUTING.md +++ b/labs/15-pitot/public-readme-preview/CONTRIBUTING.md @@ -44,5 +44,5 @@ python3 scripts/pitot_adapter_supervisor.py check ``` The merge refuses partial, duplicate, mixed-version, unsuccessful, or -fabricated captures. Review the redacted 27-cell wire diff before committing +fabricated captures. Review the redacted 30-cell wire diff before committing it; ordinary CI verifies the committed fixtures and never rewrites them. diff --git a/labs/15-pitot/public-readme-preview/README.md b/labs/15-pitot/public-readme-preview/README.md index 260ee7cef..343313473 100644 --- a/labs/15-pitot/public-readme-preview/README.md +++ b/labs/15-pitot/public-readme-preview/README.md @@ -11,11 +11,11 @@ Pitot agent E2E

-

Every supervised adapter must pass a binary-observed prompt → real hook → projected Consumer → Controller allow/deny → tool-result loop on Ubuntu, macOS, and Windows.

+

Every supervised adapter must pass a binary-observed prompt → real action boundary → projected Consumer → Controller allow/deny → tool-result loop on Ubuntu, macOS, and Windows.

-

Supervised adapters: Claude · Cursor · Codex · GitHub Copilot CLI · Gemini · Kimi Code · OpenCode · Pi · Qwen Code

+

Supervised adapters: Claude · Cursor · Codex · Devin · GitHub Copilot CLI · Gemini · Kimi Code · OpenCode · Pi · Qwen Code

-

Supervised runtime capabilities: hook control · Consumer delivery · explicit request

+

Supervised runtime capabilities: action control · Consumer delivery · explicit request

@@ -78,6 +78,27 @@ Kimi as the blocked tool result. The `shell-policy` sample is a demonstration tripwire, not a general shell-security control; the point is that *your* code made the decision. +### Devin uses ACP + +Devin exposes the same Pitot control semantics through a stateful ACP boundary, +so it does not need lifecycle-hook files: + +```bash +pitot doctor --host devin +pitot dev --host devin -- devin -p "Run: echo hello" +``` + +To attach to an already-running Pitot runtime, use the explicit single-prompt +surface: + +```bash +pitot acp devin --runtime "$PITOT_RUNTIME" --prompt "Run: echo hello" +``` + +Pitot selects only ACP's one-shot `allow_once` or `reject_once` options. +Interactive, resume, and multi-turn Devin sessions are not supported in this +initial adapter. + ## Why Pitot? Building above coding agents usually forces one of two expensive choices: diff --git a/labs/15-pitot/publish.config.json b/labs/15-pitot/publish.config.json index 1822513ba..d62054b35 100644 --- a/labs/15-pitot/publish.config.json +++ b/labs/15-pitot/publish.config.json @@ -82,6 +82,7 @@ "mock_anthropic_server.js", "model_control_proxy.py", "cursor_control_proxy.mjs", + "devin_control_proxy.py", "endpoint-provenance.json", "real_agent_driver.py", "run_e2e_report.py", @@ -109,4 +110,4 @@ "import_boundary": "scripts/check_import_boundary.sh", "extra_lab_jobs": ".labkit/lab-jobs.yml" } -} \ No newline at end of file +} diff --git a/labs/15-pitot/scripts/build_pitot.py b/labs/15-pitot/scripts/build_pitot.py index ca4d2ba0e..39a042fba 100644 --- a/labs/15-pitot/scripts/build_pitot.py +++ b/labs/15-pitot/scripts/build_pitot.py @@ -90,7 +90,7 @@ def _iter_e2e_harness_files(repo: Path) -> list[tuple[str, Path]]: path for name in ( "e2e_unified_runner.sh", "install_real_agent.py", "mock_anthropic_server.js", "model_control_proxy.py", - "cursor_control_proxy.mjs", "endpoint-provenance.json", "real_agent_driver.py", "run_e2e_report.py", + "cursor_control_proxy.mjs", "devin_control_proxy.py", "endpoint-provenance.json", "real_agent_driver.py", "run_e2e_report.py", "runtime_capability_driver.py", ) if (path := root / name).is_file() diff --git a/labs/15-pitot/scripts/pitot_adapter_supervisor.py b/labs/15-pitot/scripts/pitot_adapter_supervisor.py index 86aacd4b7..726649056 100644 --- a/labs/15-pitot/scripts/pitot_adapter_supervisor.py +++ b/labs/15-pitot/scripts/pitot_adapter_supervisor.py @@ -24,9 +24,9 @@ ID_PATTERN = re.compile(r"^[a-z][a-z0-9-]*$") LABEL_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 .+_-]{0,39}$") VERSION_PATTERN = re.compile(r"^(?:[0-9]+\.){2}[0-9]+(?:-[0-9A-Za-z.-]+)?$") -DIALECTS = {"anthropic_messages", "openai_chat", "openai_responses", "gemini_generate_content", "cursor_connect_proto"} -INTEGRATIONS = {"native_command_hook", "pi_extension", "opencode_plugin"} -INSTALLERS = {"npm", "kimi_release", "cursor_release"} +DIALECTS = {"anthropic_messages", "openai_chat", "openai_responses", "gemini_generate_content", "cursor_connect_proto", "devin_connect_proto"} +INTEGRATIONS = {"native_command_hook", "pi_extension", "opencode_plugin", "acp_client"} +INSTALLERS = {"npm", "kimi_release", "cursor_release", "devin_release"} class ContractError(ValueError): @@ -46,15 +46,15 @@ def load_manifest(root: Path = ROOT) -> dict[str, object]: def validate_manifest(value: object) -> None: if not isinstance(value, dict) or set(value) != {"schema_version", "capabilities", "platforms", "agents"}: raise ContractError("manifest must contain schema_version, capabilities, platforms, and agents") - if value["schema_version"] != 6: + if value["schema_version"] != 7: raise ContractError("unsupported manifest schema_version") expected_capabilities = [ - {"id": "hook_control", "matrix": "agent_platform"}, + {"id": "action_control", "matrix": "agent_platform"}, {"id": "consumer_delivery", "matrix": "agent_platform"}, {"id": "explicit_request", "matrix": "platform"}, ] if value["capabilities"] != expected_capabilities: - raise ContractError("capabilities must exactly supervise hook control, Consumer delivery, and explicit request") + raise ContractError("capabilities must exactly supervise action control, Consumer delivery, and explicit request") platforms = value["platforms"] agents = value["agents"] if not isinstance(platforms, list) or not isinstance(agents, list) or not agents: @@ -172,7 +172,11 @@ def validate_endpoint_value(value: object, manifest: dict[str, object]) -> None: raise ContractError(f"endpoint fixture {agent_id}/{platform} has an invalid redacted request shape") if cell["dialect"] == "cursor_connect_proto" and (request["transport"], request["framing"]) != ("http2", "connect_envelope"): raise ContractError(f"endpoint fixture {agent_id}/{platform} has invalid Cursor wire provenance") - if cell["dialect"] != "cursor_connect_proto" and request["media_type"] != "application/json": + if cell["dialect"] == "devin_connect_proto" and (request["transport"], request["framing"]) != ("http1", "connect_envelope"): + raise ContractError(f"endpoint fixture {agent_id}/{platform} has invalid Devin wire provenance") + if cell["dialect"] in {"cursor_connect_proto", "devin_connect_proto"} and request["media_type"] != "application/connect+proto": + raise ContractError(f"endpoint fixture {agent_id}/{platform} has invalid Connect wire provenance") + if cell["dialect"] not in {"cursor_connect_proto", "devin_connect_proto"} and request["media_type"] != "application/json": raise ContractError(f"endpoint fixture {agent_id}/{platform} has invalid JSON wire provenance") if response != {"encoder": cell["dialect"], "framing": request["framing"], "tool_call": "native_shell", "acceptance": "nonce_tool_result_round_trip"}: raise ContractError(f"endpoint fixture {agent_id}/{platform} lacks accepted response provenance") @@ -241,11 +245,11 @@ def readme_block(manifest: dict[str, object]) -> str: ' Pitot agent E2E', "

", "", - '

Every supervised adapter must pass a binary-observed prompt → real hook → projected Consumer → Controller allow/deny → tool-result loop on Ubuntu, macOS, and Windows.

', + '

Every supervised adapter must pass a binary-observed prompt → real action boundary → projected Consumer → Controller allow/deny → tool-result loop on Ubuntu, macOS, and Windows.

', "", f'

Supervised adapters: {labels}

', "", - '

Supervised runtime capabilities: hook control · Consumer delivery · explicit request

', + '

Supervised runtime capabilities: action control · Consumer delivery · explicit request

', README_END, ] ) @@ -442,6 +446,9 @@ def contract_errors(root: Path, adapters: list[str] | None = None) -> list[str]: cursor_proxy = root / "labs/15-pitot/tests/cursor_control_proxy.mjs" if not cursor_proxy.is_file(): errors.append("missing pinned Cursor endpoint proxy") + devin_proxy = root / "labs/15-pitot/tests/devin_control_proxy.py" + if not devin_proxy.is_file(): + errors.append("missing pinned Devin endpoint proxy") reporter_path = root / ".github/scripts/pitot_e2e_report.py" reporter_text = reporter_path.read_text(encoding="utf-8") if reporter_path.is_file() else "" if "artifacts?per_page=100" not in reporter_text or "total != len(artifacts)" not in reporter_text: diff --git a/labs/15-pitot/tests/devin_control_proxy.py b/labs/15-pitot/tests/devin_control_proxy.py new file mode 100644 index 000000000..d4c1c5936 --- /dev/null +++ b/labs/15-pitot/tests/devin_control_proxy.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Deterministic Connect/protobuf model endpoint for released Devin ACP E2E.""" + +from __future__ import annotations + +import argparse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path + + +def varint(value: int) -> bytes: + output = bytearray() + while value > 127: + output.append((value & 127) | 128) + value >>= 7 + output.append(value) + return bytes(output) + + +def field_bytes(field: int, value: bytes) -> bytes: + return varint((field << 3) | 2) + varint(len(value)) + value + + +def field_string(field: int, value: str) -> bytes: + return field_bytes(field, value.encode()) + + +def field_enum(field: int, value: int) -> bytes: + return varint(field << 3) + varint(value) + + +def envelope(payload: bytes, flags: int = 0) -> bytes: + return bytes([flags]) + len(payload).to_bytes(4, "big") + payload + + +def tool_response(call_id: str, command: str) -> bytes: + call = field_string(1, call_id) + field_string(2, "exec") + field_string( + 3, json.dumps({"command": command}, separators=(",", ":")) + ) + message = field_string(1, f"msg-{call_id}") + field_enum(5, 10) + field_bytes(6, call) + return envelope(message) + envelope(b"{}", 2) + + +def text_response(text: str) -> bytes: + message = field_string(1, "pitot-final") + field_string(3, text) + field_enum(5, 2) + return envelope(message) + envelope(b"{}", 2) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--nonce", required=True) + parser.add_argument("--receipt", type=Path, required=True) + parser.add_argument("--ready-file", type=Path, required=True) + parser.add_argument("--canary-command", required=True) + parser.add_argument("--response-fault", choices=("none", "text"), default="none") + args = parser.parse_args() + allow_id = f"pitot_tool_allow_{args.nonce}" + deny_id = f"pitot_tool_deny_{args.nonce}" + receipt: dict[str, object] = { + "schema_version": 1, + "agent": "devin", + "protocol": "devin_connect_proto", + "nonce": args.nonce, + "initial_prompt_observed": False, + "tool_call_response_emitted": False, + "tool_result_observed": False, + "allow_tool_call_response_emitted": False, + "allow_tool_result_observed": False, + "deny_tool_call_response_emitted": False, + "denied_result_observed": False, + "final_response_emitted": False, + "endpoint_observed": None, + "auxiliary_requests": 0, + } + + def save() -> None: + args.receipt.parent.mkdir(parents=True, exist_ok=True) + temporary = args.receipt.with_suffix(".tmp") + temporary.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(args.receipt) + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_: object) -> None: + return + + def do_GET(self) -> None: + self.send_response(404) + self.end_headers() + + def do_POST(self) -> None: + raw = self.rfile.read(int(self.headers.get("content-length", "0"))) + path = self.path.split("?", 1)[0] + content_type = self.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if not path.endswith("/GetChatMessage"): + receipt["auxiliary_requests"] = int(receipt["auxiliary_requests"]) + 1 + save() + self.send_response(200) + self.send_header("content-type", "application/proto") + self.send_header("content-length", "0") + self.end_headers() + return + + lowered = raw.lower() + is_title = b"session title generator" in lowered + allow_result = f"PITOT_CANARY_RESULT PITOT_ALLOW {args.nonce}".encode() in raw + denied_result = deny_id.encode() in raw and any( + marker in lowered for marker in (b"reject", b"denied", b"permission") + ) + if is_title: + payload = text_response("Pitot E2E") + receipt["auxiliary_requests"] = int(receipt["auxiliary_requests"]) + 1 + elif not receipt["initial_prompt_observed"] and args.nonce.encode() in raw: + receipt["initial_prompt_observed"] = True + receipt["endpoint_observed"] = { + "transport": "http1", + "method": "POST", + "path": path, + "media_type": content_type, + "framing": "connect_envelope", + "request_shape": { + "service": "exa.api_server_pb.ApiServerService", + "method": "GetChatMessage", + "stream": "server", + "message": "GetChatMessageRequest", + }, + } + if args.response_fault == "text": + receipt["fault_response_emitted"] = "text" + payload = text_response(f"PITOT_E2E_COMPLETE {args.nonce}") + else: + payload = tool_response( + allow_id, f"{args.canary_command} PITOT_ALLOW {args.nonce}" + ) + receipt["tool_call_response_emitted"] = True + receipt["allow_tool_call_response_emitted"] = True + elif allow_result and not receipt["allow_tool_result_observed"]: + receipt["tool_result_observed"] = True + receipt["allow_tool_result_observed"] = True + receipt["deny_tool_call_response_emitted"] = True + payload = tool_response( + deny_id, f"{args.canary_command} PITOT_DENY {args.nonce}" + ) + elif denied_result and receipt["deny_tool_call_response_emitted"]: + receipt["denied_result_observed"] = True + receipt["final_response_emitted"] = True + payload = text_response(f"PITOT_E2E_COMPLETE {args.nonce}") + else: + receipt["unexpected_request"] = { + "path": path, + "bytes": len(raw), + "allow_result": allow_result, + "deny_call_id": deny_id.encode() in raw, + "rejection_marker": any( + marker in lowered for marker in (b"reject", b"denied", b"permission") + ), + } + save() + self.send_error(409, "request did not advance Devin control trajectory") + return + save() + self.send_response(200) + self.send_header("content-type", "application/connect+proto") + self.send_header("content-length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + args.ready_file.write_text(f"http://127.0.0.1:{server.server_port}\n", encoding="utf-8") + save() + try: + server.serve_forever() + except KeyboardInterrupt: + return 0 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/labs/15-pitot/tests/e2e_devin_cli_test.sh b/labs/15-pitot/tests/e2e_devin_cli_test.sh new file mode 100755 index 000000000..437438365 --- /dev/null +++ b/labs/15-pitot/tests/e2e_devin_cli_test.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exec "$(dirname "$0")/e2e_unified_runner.sh" "devin" diff --git a/labs/15-pitot/tests/e2e_unified_runner.sh b/labs/15-pitot/tests/e2e_unified_runner.sh index 7957d5ca9..bd78dfffd 100755 --- a/labs/15-pitot/tests/e2e_unified_runner.sh +++ b/labs/15-pitot/tests/e2e_unified_runner.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Real released-agent prompt-to-hook E2E. Direct hook invocation is forbidden. +# Real released-agent prompt-to-boundary E2E. Direct boundary invocation is forbidden. set -euo pipefail HOST="${1:-}" diff --git a/labs/15-pitot/tests/endpoint-provenance.json b/labs/15-pitot/tests/endpoint-provenance.json index 960e7c764..50ba0c73d 100644 --- a/labs/15-pitot/tests/endpoint-provenance.json +++ b/labs/15-pitot/tests/endpoint-provenance.json @@ -1277,6 +1277,93 @@ }, "runtime": "native", "version": "0.20.1" + }, + { + "agent": "devin", + "capture_sha256": "4e486e91fb33c8220d0772d0a9dc2552799b5af309c2fb8a73cf72f8cde1902b", + "dialect": "devin_connect_proto", + "executable_sha256": "c303dc3395149d946568f57e00bcea509b269ed361f6845eea38206f59113551", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "connect_envelope", + "media_type": "application/connect+proto", + "method": "POST", + "path": "/exa.api_server_pb.ApiServerService/GetChatMessage", + "request_shape": { + "message": "GetChatMessageRequest", + "method": "GetChatMessage", + "service": "exa.api_server_pb.ApiServerService", + "stream": "server" + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "devin_connect_proto", + "framing": "connect_envelope", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "3000.3.22" + }, + { + "agent": "devin", + "capture_sha256": "16888b9fe29244c7462d2637c55f3b1938375b4cac30bc978236c6726378c3dd", + "dialect": "devin_connect_proto", + "executable_sha256": "1252ab77359e1cc63c23fda7a88ff7711bab5f853c1ab6d3323cb319b71a60ee", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "connect_envelope", + "media_type": "application/connect+proto", + "method": "POST", + "path": "/exa.api_server_pb.ApiServerService/GetChatMessage", + "request_shape": { + "message": "GetChatMessageRequest", + "method": "GetChatMessage", + "service": "exa.api_server_pb.ApiServerService", + "stream": "server" + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "devin_connect_proto", + "framing": "connect_envelope", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "3000.3.22" + }, + { + "agent": "devin", + "capture_sha256": "cf0add8cb81014d2bdefe14ce92e7e4b47b7ae40f22cf65ecff7696137b3411d", + "dialect": "devin_connect_proto", + "executable_sha256": "fd0901491ad7a42d8d2d7e0af6f1b53564bbc4e563a6bd3135111346c78e08f6", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "connect_envelope", + "media_type": "application/connect+proto", + "method": "POST", + "path": "/exa.api_server_pb.ApiServerService/GetChatMessage", + "request_shape": { + "message": "GetChatMessageRequest", + "method": "GetChatMessage", + "service": "exa.api_server_pb.ApiServerService", + "stream": "server" + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "devin_connect_proto", + "framing": "connect_envelope", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "3000.3.22" } ], "schema_version": 2 diff --git a/labs/15-pitot/tests/install_real_agent.py b/labs/15-pitot/tests/install_real_agent.py index a8b1170c9..420073646 100755 --- a/labs/15-pitot/tests/install_real_agent.py +++ b/labs/15-pitot/tests/install_real_agent.py @@ -11,7 +11,10 @@ import platform as host_platform import shutil import subprocess +import tarfile import tempfile +import urllib.request +import zipfile LAB = Path(__file__).resolve().parent.parent @@ -89,6 +92,51 @@ def install(agent: dict[str, object], platform: str, runtime: str) -> None: temporary_link.unlink() temporary_link.symlink_to(installed) temporary_link.replace(link) + elif kind == "devin_release": + manifest_url = f"{package}/{version}/manifest.json" + with urllib.request.urlopen(manifest_url, timeout=30) as response: + manifest = json.loads(response.read()) + if manifest.get("version") != version: + raise RuntimeError("Devin release manifest version does not match the supervised pin") + machine = host_platform.machine().lower() + arch = "aarch64" if machine in {"arm64", "aarch64"} else "x86_64" + suffix = {"ubuntu": "unknown-linux", "macos": "apple-darwin", "windows": "pc-windows"}[platform] + target = f"{arch}-{suffix}" + release = manifest.get("platforms", {}).get(target) + if not isinstance(release, dict) or set(release) != {"url", "sha256"}: + raise RuntimeError(f"Devin release manifest omitted {target}") + with tempfile.TemporaryDirectory() as directory: + archive = Path(directory) / ("devin.zip" if platform == "windows" else "devin.tar.gz") + with urllib.request.urlopen(release["url"], timeout=120) as response: + archive.write_bytes(response.read()) + if hashlib.sha256(archive.read_bytes()).hexdigest() != release["sha256"]: + raise RuntimeError("Devin release archive SHA-256 mismatch") + executable_name = "devin.exe" if platform == "windows" else "devin" + if platform == "windows": + with zipfile.ZipFile(archive) as bundle: + matches = [name for name in bundle.namelist() if Path(name).name == executable_name] + if len(matches) != 1: + raise RuntimeError(f"Devin archive contained {len(matches)} executables") + executable_bytes = bundle.read(matches[0]) + else: + with tarfile.open(archive, "r:gz") as bundle: + matches = [member for member in bundle.getmembers() if member.isfile() and Path(member.name).name == executable_name] + if len(matches) != 1: + raise RuntimeError(f"Devin archive contained {len(matches)} executables") + source = bundle.extractfile(matches[0]) + if source is None: + raise RuntimeError("Devin executable could not be extracted") + executable_bytes = source.read() + installed_dir = Path.home() / ".local" / "share" / "pitot-devin" / version + installed_dir.mkdir(parents=True, exist_ok=True) + installed = installed_dir / executable_name + installed.write_bytes(executable_bytes) + installed.chmod(0o755) + destination = Path.home() / ".local" / "bin" + destination.mkdir(parents=True, exist_ok=True) + launcher = destination / executable_name + shutil.copy2(installed, launcher) + launcher.chmod(0o755) else: raise ValueError(f"unsupported installer: {kind}") diff --git a/labs/15-pitot/tests/real_agent_driver.py b/labs/15-pitot/tests/real_agent_driver.py index 28590db2d..df92ff163 100755 --- a/labs/15-pitot/tests/real_agent_driver.py +++ b/labs/15-pitot/tests/real_agent_driver.py @@ -16,6 +16,7 @@ import sys import tempfile import time +import urllib.parse LAB = Path(__file__).resolve().parent.parent @@ -112,6 +113,24 @@ def configure( ) env["CODEX_HOME"] = str(home / ".codex") prompt_flag = ["exec", "--dangerously-bypass-approvals-and-sandbox", "--dangerously-bypass-hook-trust", "--model", "pitot-control"] + elif agent == "devin": + endpoint = urllib.parse.urlparse(proxy) + credentials = home / ".local/share/devin/credentials.toml" + credentials.parent.mkdir(parents=True, exist_ok=True) + credentials.write_text( + 'windsurf_api_key = "pitot-local-only"\n' + f"api_server_url = {json.dumps(proxy)}\n" + f"devin_webapp_host = {json.dumps(endpoint.netloc)}\n" + f"devin_api_url = {json.dumps(proxy)}\n", + encoding="utf-8", + ) + env.update({ + "WINDSURF_API_KEY": "pitot-local-only", + "WINDSURF_API_SERVER_URL": proxy, + "XDG_CONFIG_HOME": str(home / ".config"), + "XDG_DATA_HOME": str(home / ".local/share"), + }) + prompt_flag = [] elif agent == "copilot": hooks = home / ".copilot/hooks" hooks.mkdir(parents=True, exist_ok=True) @@ -263,7 +282,7 @@ def capture_record(agent: dict[str, object], platform: str, installation: dict[s observed = proxy.get("endpoint_observed") dialect = proxy.get("protocol") if not isinstance(observed, dict) or dialect not in { - "anthropic_messages", "openai_chat", "openai_responses", "gemini_generate_content", "cursor_connect_proto", + "anthropic_messages", "openai_chat", "openai_responses", "gemini_generate_content", "cursor_connect_proto", "devin_connect_proto", }: raise RuntimeError("proxy did not binary-observe a supported request contract") digest = installation.get("executable_sha256") @@ -330,21 +349,29 @@ def validate_receipts( final_marker = f"PITOT_E2E_COMPLETE {nonce}" if exit_code != 0 or "hook: PreToolUse Failed" in output or final_marker not in output or not all(proxy.get(flag) is True for flag in proxy_flags): raise RuntimeError(f"agent loop incomplete (exit={exit_code}, proxy={proxy}, witnesses={witnesses})\n{output[-4000:]}") - if proxy.get("nonce") != nonce or len(witnesses) != 2: - raise RuntimeError("proxy and Pitot witness receipts do not identify exactly two hook actions") - if any(item.get("nonce") != nonce or item.get("host") != agent["id"] or item.get("valid") is not True for item in witnesses): - raise RuntimeError("Pitot hook witnesses escaped the nonce-bound real-agent session") - if [item.get("pitot_exit") for item in witnesses] != [0, 2]: - raise RuntimeError(f"real hook did not carry one allow and one deny: {witnesses}") - action_ids = [item.get("action_id") for item in witnesses] - if len(set(action_ids)) != 2 or not all(isinstance(item, str) and item.startswith("act_") for item in action_ids): - raise RuntimeError("hook actions lack unique Pitot correlation ids") + if proxy.get("nonce") != nonce: + raise RuntimeError("proxy receipt escaped the nonce-bound real-agent session") + if agent["integration"] == "acp_client": + if witnesses: + raise RuntimeError("ACP control unexpectedly entered the lifecycle-hook witness path") + else: + if len(witnesses) != 2: + raise RuntimeError("Pitot witness receipts do not identify exactly two hook actions") + if any(item.get("nonce") != nonce or item.get("host") != agent["id"] or item.get("valid") is not True for item in witnesses): + raise RuntimeError("Pitot hook witnesses escaped the nonce-bound real-agent session") + if [item.get("pitot_exit") for item in witnesses] != [0, 2]: + raise RuntimeError(f"real hook did not carry one allow and one deny: {witnesses}") requests = [item.get("value") for item in controller if item.get("receipt_type") == "request"] responses = [item.get("value") for item in controller if item.get("receipt_type") == "response"] if len(requests) != 2 or len(responses) != 2: - raise RuntimeError(f"Controller did not receive and resolve both hook actions: {controller}") + raise RuntimeError(f"Controller did not receive and resolve both boundary actions: {controller}") + action_ids = [item.get("action_id") for item in requests] + if len(set(action_ids)) != 2 or not all(isinstance(item, str) and item.startswith("act_") for item in action_ids): + raise RuntimeError("boundary actions lack unique Pitot correlation ids") + if witnesses and [item.get("action_id") for item in witnesses] != action_ids: + raise RuntimeError("hook correlation ids do not match the Controller actions") if [item.get("action_id") for item in requests] != action_ids or [item.get("action_id") for item in responses] != action_ids: - raise RuntimeError("Controller correlation ids do not match the real hook actions") + raise RuntimeError("Controller correlation ids do not match the real boundary actions") if [item.get("outcome") for item in responses] != ["allow", "deny"] or responses[1].get("message") != f"PITOT_CONTROLLER_DENY {nonce}": raise RuntimeError("external Controller did not produce the nonce-bound allow/deny trajectory") if len(consumers) != 2 or [item.get("action", {}).get("id") for item in consumers] != action_ids: @@ -382,7 +409,17 @@ def validate_receipts( "nonce": nonce, "receipts": {**{flag: True for flag in proxy_flags}, "consumer_observed": True, "controller_allow_observed": True, "controller_deny_observed": True, "deny_canary_absent": True, "final_output_observed": True, "cli_exit_zero": True}, "runtime": runtime_public, - "hooks": [{"host": item["host"], "action_kind": item["action_kind"], "action_id": item["action_id"], "pitot_exit": item["pitot_exit"], "nonce": item["nonce"]} for item in witnesses], + "boundaries": [ + { + "host": agent["id"], + "transport": "acp" if agent["integration"] == "acp_client" else "hook", + "action_kind": "shell", + "action_id": action_id, + "decision": outcome, + "nonce": nonce, + } + for action_id, outcome in zip(action_ids, ("allow", "deny"), strict=True) + ], "controller": {"id": "e2e-shell-controller", "action_ids": action_ids, "outcomes": ["allow", "deny"]}, "consumer": {"id": "e2e-audit", "action_ids": action_ids, "projection": "sha256"}, "canary": {"executions": canary, "denied_executions": 0}, @@ -495,6 +532,8 @@ def main() -> int: ] else: proxy_command = ["node", str(LAB / "tests/cursor_control_proxy.mjs"), "--nonce", nonce, "--receipt", str(proxy_receipt), "--ready-file", str(ready), "--canary-command", canary_command, "--response-fault", args.response_fault] + elif args.agent == "devin": + proxy_command = [sys.executable, str(LAB / "tests/devin_control_proxy.py"), "--nonce", nonce, "--receipt", str(proxy_receipt), "--ready-file", str(ready), "--canary-command", canary_command, "--response-fault", args.response_fault] else: proxy_command = [sys.executable, str(LAB / "tests/model_control_proxy.py"), "--agent", args.agent, "--nonce", nonce, "--receipt", str(proxy_receipt), "--ready-file", str(ready), "--canary-command", canary_command, "--response-fault", args.response_fault] proxy_process = subprocess.Popen( @@ -554,6 +593,14 @@ def main() -> int: "wsl.exe", "--distribution", "Ubuntu", "--cd", wsl_path(project), "--", "env", *assignments, executable, *flags, prompt, ] + elif args.agent == "devin": + command = [ + str(args.pitot.resolve()), "acp", "devin", + "--runtime", str(runtime_descriptor), + "--prompt", prompt, + "--exec", executable, + "--cwd", str(project), + ] else: command = [executable, "-p", prompt, *flags] if args.agent in {"copilot", "kimi"} else [executable, *flags, prompt] try: diff --git a/labs/15-pitot/tests/run_e2e_report.py b/labs/15-pitot/tests/run_e2e_report.py index 5e34938d7..2e149dc70 100755 --- a/labs/15-pitot/tests/run_e2e_report.py +++ b/labs/15-pitot/tests/run_e2e_report.py @@ -16,6 +16,7 @@ ROOT = LAB.parents[1] if LAB.name == "15-pitot" else LAB MANIFEST = json.loads((LAB / "adapter-verification.json").read_text(encoding="utf-8")) AGENTS = {agent["id"] for agent in MANIFEST["agents"]} +AGENT_RECORDS = {agent["id"]: agent for agent in MANIFEST["agents"]} PLATFORMS = {platform["id"] for platform in MANIFEST["platforms"]} RESULT_PATTERN = re.compile(r"^PITOT_E2E_RESULT mode=real_cli evidence=nonce-correlated$", re.MULTILINE) RUNTIME_RESULT_PATTERN = re.compile(r"^PITOT_RUNTIME_E2E_RESULT capability=explicit_request evidence=nonce-correlated$", re.MULTILINE) @@ -25,7 +26,7 @@ def load_evidence(path: Path | None, *, agent: str) -> dict[str, object] | None: if path is None or not path.is_file(): return None value = json.loads(path.read_text(encoding="utf-8")) - required = {"schema_version", "agent", "cli", "prompt_hash", "protocol", "endpoint", "nonce", "receipts", "runtime", "hooks", "controller", "consumer", "canary"} + required = {"schema_version", "agent", "cli", "prompt_hash", "protocol", "endpoint", "nonce", "receipts", "runtime", "boundaries", "controller", "consumer", "canary"} if not isinstance(value, dict) or set(value) != required or value["schema_version"] != 2 or value["agent"] != agent: return None receipts = value.get("receipts") @@ -42,8 +43,9 @@ def load_evidence(path: Path | None, *, agent: str) -> dict[str, object] | None: nonce = value.get("nonce") if not isinstance(nonce, str) or not re.fullmatch(r"[0-9a-f]{32}", nonce): return None - hooks = value.get("hooks") - if not isinstance(hooks, list) or len(hooks) != 2 or [item.get("pitot_exit") for item in hooks] != [0, 2] or any(item.get("action_kind") != "shell" or item.get("host") != agent or item.get("nonce") != nonce for item in hooks): + boundaries = value.get("boundaries") + expected_transport = "acp" if AGENT_RECORDS[agent]["integration"] == "acp_client" else "hook" + if not isinstance(boundaries, list) or len(boundaries) != 2 or [item.get("decision") for item in boundaries] != ["allow", "deny"] or any(item.get("action_kind") != "shell" or item.get("host") != agent or item.get("nonce") != nonce or item.get("transport") != expected_transport for item in boundaries): return None endpoint = value.get("endpoint", {}) endpoint_required = {"fixture", "fixture_sha256", "provenance", "dialect", "request", "response", "executable_sha256"} @@ -81,7 +83,7 @@ def result_for(agent: str, platform: str, returncode: int, output: str, evidence markers = RESULT_PATTERN.findall(output) receipt = load_evidence(evidence_path, agent=agent) passed = returncode == 0 and len(markers) == 1 and receipt is not None - evidence = "binary-observed request, real hook control, projected Consumer, allow/deny canary, and final receipts" if passed else "real-agent control evidence contract failed" + evidence = "binary-observed request, real action control, projected Consumer, allow/deny canary, and final receipts" if passed else "real-agent control evidence contract failed" return { "schema_version": 2, @@ -97,7 +99,7 @@ def result_for(agent: str, platform: str, returncode: int, output: str, evidence "nonce": receipt["nonce"] if passed else None, "receipts": receipt["receipts"] if passed else None, "runtime": receipt["runtime"] if passed else None, - "hooks": receipt["hooks"] if passed else None, + "boundaries": receipt["boundaries"] if passed else None, "controller": receipt["controller"] if passed else None, "consumer": receipt["consumer"] if passed else None, "canary": receipt["canary"] if passed else None, diff --git a/labs/15-pitot/tests/test_adapter_supervisor.py b/labs/15-pitot/tests/test_adapter_supervisor.py index 2cc15fa67..afd88a1ae 100644 --- a/labs/15-pitot/tests/test_adapter_supervisor.py +++ b/labs/15-pitot/tests/test_adapter_supervisor.py @@ -24,14 +24,14 @@ def _manifest() -> dict[str, object]: def _provenance(manifest: dict[str, object]) -> dict[str, object]: - dialects = {"claude": "anthropic_messages", "codex": "openai_responses", "gemini": "gemini_generate_content", "cursor": "cursor_connect_proto"} - paths = {"anthropic_messages": "/v1/messages", "openai_responses": "/v1/responses", "gemini_generate_content": "/v1beta/models/pitot-control:streamGenerateContent", "cursor_connect_proto": "/agent.v1.AgentService/Run", "openai_chat": "/v1/chat/completions"} + dialects = {"claude": "anthropic_messages", "codex": "openai_responses", "gemini": "gemini_generate_content", "cursor": "cursor_connect_proto", "devin": "devin_connect_proto"} + paths = {"anthropic_messages": "/v1/messages", "openai_responses": "/v1/responses", "gemini_generate_content": "/v1beta/models/pitot-control:streamGenerateContent", "cursor_connect_proto": "/agent.v1.AgentService/Run", "devin_connect_proto": "/connectrpc.devin.api.ModelService/StreamResponse", "openai_chat": "/v1/chat/completions"} cells = [] for agent in manifest["agents"]: for platform in manifest["platforms"]: dialect = dialects.get(agent["id"], "openai_chat") - framing = "connect_envelope" if dialect == "cursor_connect_proto" else "sse" - request = {"transport": "http2" if dialect == "cursor_connect_proto" else "http1", "method": "POST", "path": paths[dialect], "media_type": "application/connect+proto" if dialect == "cursor_connect_proto" else "application/json", "framing": framing, "request_shape": {"fixture": "redacted"}} + framing = "connect_envelope" if dialect in {"cursor_connect_proto", "devin_connect_proto"} else "sse" + request = {"transport": "http2" if dialect == "cursor_connect_proto" else "http1", "method": "POST", "path": paths[dialect], "media_type": "application/connect+proto" if dialect in {"cursor_connect_proto", "devin_connect_proto"} else "application/json", "framing": framing, "request_shape": {"fixture": "redacted"}} response = {"encoder": dialect, "framing": framing, "tool_call": "native_shell", "acceptance": "nonce_tool_result_round_trip"} core = {"agent": agent["id"], "platform": platform["id"], "runtime": agent["runtime"][platform["id"]], "version": agent["version"], "executable_sha256": hashlib.sha256(f"{agent['id']}/{platform['id']}".encode()).hexdigest(), "dialect": dialect, "request": request, "response": response, "provenance": "pinned_real_cli_capture"} cells.append({**core, "capture_sha256": hashlib.sha256(json.dumps(core, sort_keys=True, separators=(",", ":")).encode()).hexdigest()}) @@ -57,6 +57,7 @@ def _fixture(root: Path) -> tuple[Path, list[str]]: (tests / "real_agent_driver.py").write_text("# canonical driver\n") (tests / "model_control_proxy.py").write_text("# proxy\n") (tests / "cursor_control_proxy.mjs").write_text("// cursor proxy\n") + (tests / "devin_control_proxy.py").write_text("# devin proxy\n") (tests / "e2e_unified_runner.sh").write_text("python3 real_agent_driver.py\n") readme = root / supervisor.README readme.parent.mkdir(parents=True) @@ -90,7 +91,7 @@ def test_real_repository_contract_is_complete(self): def test_matrix_contains_every_agent_on_every_platform(self): manifest = _manifest() matrix = supervisor.matrix(manifest) - self.assertEqual(len(matrix), 27) + self.assertEqual(len(matrix), 30) self.assertEqual( {(item["agent"], item["platform"]) for item in matrix if item["agent"] == "kimi"}, {("kimi", "ubuntu"), ("kimi", "macos"), ("kimi", "windows")}, @@ -219,7 +220,7 @@ def test_platform_swap_and_fabricated_capture_digest_fail(self): fixture_path.write_text(json.dumps(fixture)) self.assertTrue(any("fabricated capture digest" in error for error in supervisor.contract_errors(root, ids))) - def test_capture_merge_requires_all_27_accepted_unique_cells(self): + def test_capture_merge_requires_all_30_accepted_unique_cells(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) _fixture(root) @@ -231,9 +232,9 @@ def test_capture_merge_requires_all_27_accepted_unique_cells(self): (captures / f"{cell['agent']}-{cell['platform']}.json").write_text(json.dumps({"schema_version": 1, "accepted": True, "cell": cell})) output = root / "merged.json" supervisor.merge_captures(root, captures, output) - self.assertEqual(len(json.loads(output.read_text())["cells"]), 27) + self.assertEqual(len(json.loads(output.read_text())["cells"]), 30) next(captures.glob("*.json")).unlink() - with self.assertRaisesRegex(supervisor.ContractError, "exactly all 27"): + with self.assertRaisesRegex(supervisor.ContractError, "exactly all 30"): supervisor.merge_captures(root, captures, output) def test_capture_merge_rejects_mixed_versions_and_duplicates(self): @@ -247,7 +248,7 @@ def test_capture_merge_rejects_mixed_versions_and_duplicates(self): (captures / f"{index}.json").write_text(json.dumps({"schema_version": 1, "accepted": True, "cell": cell})) duplicate = provenance["cells"][0] (captures / "duplicate.json").write_text(json.dumps({"schema_version": 1, "accepted": True, "cell": duplicate})) - with self.assertRaisesRegex(supervisor.ContractError, "exactly all 27"): + with self.assertRaisesRegex(supervisor.ContractError, "exactly all 30"): supervisor.merge_captures(root, captures, root / "merged.json") (captures / "duplicate.json").unlink() first = captures / "0.json" diff --git a/labs/15-pitot/tests/test_e2e_reporting.py b/labs/15-pitot/tests/test_e2e_reporting.py index aead5a61b..03be8cf06 100644 --- a/labs/15-pitot/tests/test_e2e_reporting.py +++ b/labs/15-pitot/tests/test_e2e_reporting.py @@ -32,9 +32,9 @@ def _load(name: str, path: Path): def _receipt(agent="claude", platform="ubuntu"): record = next(item for item in runner.MANIFEST["agents"] if item["id"] == agent) - dialect = {"claude": "anthropic_messages", "codex": "openai_responses", "gemini": "gemini_generate_content", "cursor": "cursor_connect_proto"}.get(agent, "openai_chat") - framing = "connect_envelope" if dialect == "cursor_connect_proto" else "sse" - request = {"transport": "http2" if dialect == "cursor_connect_proto" else "http1", "method": "POST", "path": "/fixture", "media_type": "application/connect+proto" if dialect == "cursor_connect_proto" else "application/json", "framing": framing, "request_shape": {"fixture": "redacted"}} + dialect = {"claude": "anthropic_messages", "codex": "openai_responses", "gemini": "gemini_generate_content", "cursor": "cursor_connect_proto", "devin": "devin_connect_proto"}.get(agent, "openai_chat") + framing = "connect_envelope" if dialect in {"cursor_connect_proto", "devin_connect_proto"} else "sse" + request = {"transport": "http2" if dialect == "cursor_connect_proto" else "http1", "method": "POST", "path": "/fixture", "media_type": "application/connect+proto" if dialect in {"cursor_connect_proto", "devin_connect_proto"} else "application/json", "framing": framing, "request_shape": {"fixture": "redacted"}} response = {"encoder": dialect, "framing": framing, "tool_call": "native_shell", "acceptance": "nonce_tool_result_round_trip"} core = {"agent": agent, "platform": platform, "runtime": record["runtime"][platform], "version": record["version"], "executable_sha256": "d" * 64, "dialect": dialect, "request": request, "response": response, "provenance": "pinned_real_cli_capture"} fixture = {**core, "capture_sha256": hashlib.sha256(json.dumps(core, sort_keys=True, separators=(",", ":")).encode()).hexdigest()} @@ -58,9 +58,9 @@ def _receipt(agent="claude", platform="ubuntu"): "nonce": "e" * 32, "receipts": {name: True for name in ("initial_prompt_observed", "allow_tool_call_response_emitted", "allow_tool_result_observed", "deny_tool_call_response_emitted", "denied_result_observed", "final_response_emitted", "consumer_observed", "controller_allow_observed", "controller_deny_observed", "deny_canary_absent", "final_output_observed", "cli_exit_zero")}, "runtime": {"schema_version": 1, "instance_id": "instance", "pid": 1, "endpoint": "http://127.0.0.1:1", "config_sha256": "c" * 64}, - "hooks": [ - {"host": agent, "action_kind": "shell", "action_id": "act_allow", "pitot_exit": 0, "nonce": "e" * 32}, - {"host": agent, "action_kind": "shell", "action_id": "act_deny", "pitot_exit": 2, "nonce": "e" * 32}, + "boundaries": [ + {"host": agent, "transport": "acp" if agent == "devin" else "hook", "action_kind": "shell", "action_id": "act_allow", "decision": "allow", "nonce": "e" * 32}, + {"host": agent, "transport": "acp" if agent == "devin" else "hook", "action_kind": "shell", "action_id": "act_deny", "decision": "deny", "nonce": "e" * 32}, ], "controller": {"id": "e2e-shell-controller", "action_ids": ["act_allow", "act_deny"], "outcomes": ["allow", "deny"]}, "consumer": {"id": "e2e-audit", "action_ids": ["act_allow", "act_deny"], "projection": "sha256"}, @@ -76,7 +76,7 @@ def _result(agent="claude", platform="ubuntu", status="pass", mode="real_cli"): "platform": platform, "status": status, "verification_mode": mode, - "evidence": "binary-observed request, real hook control, projected Consumer, allow/deny canary, and final receipts" if status == "pass" else "real-agent control evidence contract failed", + "evidence": "binary-observed request, real action control, projected Consumer, allow/deny canary, and final receipts" if status == "pass" else "real-agent control evidence contract failed", "cli": receipt["cli"] if receipt else None, "protocol": receipt["protocol"] if receipt else None, "endpoint": receipt["endpoint"] if receipt else None, @@ -84,7 +84,7 @@ def _result(agent="claude", platform="ubuntu", status="pass", mode="real_cli"): "nonce": receipt["nonce"] if receipt else None, "receipts": receipt["receipts"] if receipt else None, "runtime": receipt["runtime"] if receipt else None, - "hooks": receipt["hooks"] if receipt else None, + "boundaries": receipt["boundaries"] if receipt else None, "controller": receipt["controller"] if receipt else None, "consumer": receipt["consumer"] if receipt else None, "canary": receipt["canary"] if receipt else None, @@ -277,7 +277,7 @@ def test_missing_artifacts_fail_every_inventory_cell(self): "a" * 40, "https://github.com/operatorstack/intelligence-flow/actions/runs/1", ) - self.assertEqual(len(results), 9) + self.assertEqual(len(results), 10) self.assertTrue( all( result["status"] == "fail" @@ -307,8 +307,8 @@ def test_rejects_incomplete_or_mismatched_causal_receipts(self): with self.assertRaisesRegex(ValueError, "incomplete causal receipts"): reporter.validate_result(value, agent="claude", platform="ubuntu") value = _result() - value["hooks"][0]["host"] = "cursor" - with self.assertRaisesRegex(ValueError, "hook receipt"): + value["boundaries"][0]["host"] = "cursor" + with self.assertRaisesRegex(ValueError, "boundary receipt"): reporter.validate_result(value, agent="claude", platform="ubuntu") value = _result() value["cli"]["version"] = "latest" diff --git a/labs/15-pitot/tests/test_install_real_agent.py b/labs/15-pitot/tests/test_install_real_agent.py new file mode 100644 index 000000000..219accbb6 --- /dev/null +++ b/labs/15-pitot/tests/test_install_real_agent.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import tarfile +import tempfile +import unittest +from unittest import mock + + +ROOT = Path(__file__).resolve().parents[3] +SPEC = importlib.util.spec_from_file_location( + "install_real_agent", + ROOT / "labs/15-pitot/tests/install_real_agent.py", +) +installer = importlib.util.module_from_spec(SPEC) +assert SPEC and SPEC.loader +SPEC.loader.exec_module(installer) + + +def devin_archive(payload: bytes = b"released-devin") -> bytes: + output = io.BytesIO() + with tarfile.open(fileobj=output, mode="w:gz") as bundle: + info = tarfile.TarInfo("bin/devin") + info.mode = 0o755 + info.size = len(payload) + bundle.addfile(info, io.BytesIO(payload)) + return output.getvalue() + + +class Response: + def __init__(self, payload: bytes): + self.payload = payload + + def __enter__(self) -> "Response": + return self + + def __exit__(self, *_: object) -> None: + return None + + def read(self) -> bytes: + return self.payload + + +class DevinInstallerTests(unittest.TestCase): + def test_installs_attested_version_specific_release(self): + archive = devin_archive() + manifest = { + "version": "3000.3.22", + "platforms": { + "aarch64-unknown-linux": { + "url": "https://example.invalid/devin.tar.gz", + "sha256": hashlib.sha256(archive).hexdigest(), + } + }, + } + responses = [ + Response(json.dumps(manifest).encode()), + Response(archive), + ] + agent = { + "id": "devin", + "version": "3000.3.22", + "executable": "devin", + "installer": {"kind": "devin_release", "package": "https://static.devin.ai/cli"}, + } + with tempfile.TemporaryDirectory() as home, mock.patch.dict(os.environ, {"HOME": home}), \ + mock.patch.object(installer.host_platform, "machine", return_value="arm64"), \ + mock.patch.object(installer.urllib.request, "urlopen", side_effect=responses) as urlopen: + installer.install(agent, "ubuntu", "native") + installed = Path(home) / ".local/bin/devin" + self.assertEqual(installed.read_bytes(), b"released-devin") + self.assertTrue(os.access(installed, os.X_OK)) + self.assertEqual( + urlopen.call_args_list[0].args[0], + "https://static.devin.ai/cli/3000.3.22/manifest.json", + ) + + def test_rejects_archive_digest_mismatch(self): + archive = devin_archive() + manifest = { + "version": "3000.3.22", + "platforms": { + "x86_64-unknown-linux": { + "url": "https://example.invalid/devin.tar.gz", + "sha256": "0" * 64, + } + }, + } + agent = { + "id": "devin", + "version": "3000.3.22", + "executable": "devin", + "installer": {"kind": "devin_release", "package": "https://static.devin.ai/cli"}, + } + responses = [Response(json.dumps(manifest).encode()), Response(archive)] + with mock.patch.object(installer.host_platform, "machine", return_value="AMD64"), \ + mock.patch.object(installer.urllib.request, "urlopen", side_effect=responses): + with self.assertRaisesRegex(RuntimeError, "SHA-256 mismatch"): + installer.install(agent, "ubuntu", "native") + + +if __name__ == "__main__": + unittest.main() diff --git a/labs/15-pitot/tests/test_model_control_proxy.py b/labs/15-pitot/tests/test_model_control_proxy.py index c8ba9e6f3..f829c6dca 100644 --- a/labs/15-pitot/tests/test_model_control_proxy.py +++ b/labs/15-pitot/tests/test_model_control_proxy.py @@ -18,6 +18,10 @@ driver = importlib.util.module_from_spec(driver_spec) assert driver_spec and driver_spec.loader driver_spec.loader.exec_module(driver) +devin_spec = importlib.util.spec_from_file_location("devin_control_proxy", ROOT / "labs/15-pitot/tests/devin_control_proxy.py") +devin_proxy = importlib.util.module_from_spec(devin_spec) +assert devin_spec and devin_spec.loader +devin_spec.loader.exec_module(devin_proxy) class ModelControlProtocolTests(unittest.TestCase): @@ -76,6 +80,27 @@ def test_cursor_bridge_is_canonicalized_for_wsl(self): bridge = (base / "project/.cursor/hooks/beforeShellExecution").read_bytes() self.assertEqual(bridge, b"#!/usr/bin/env bash\necho hook\n") + def test_devin_routes_all_released_client_calls_to_isolated_proxy(self): + with tempfile.TemporaryDirectory() as temporary: + base = Path(temporary) + flags, environment = driver.configure( + "devin", base / "home", base / "project", "witness", + "http://127.0.0.1:9123", pitot_command="pitot", + witness_receipt="witness.jsonl", nonce="nonce", + runtime_path="runtime.json", + ) + self.assertEqual(flags, []) + self.assertEqual(environment["WINDSURF_API_SERVER_URL"], "http://127.0.0.1:9123") + credentials = (base / "home/.local/share/devin/credentials.toml").read_text() + self.assertIn('api_server_url = "http://127.0.0.1:9123"', credentials) + + def test_devin_connect_fixture_carries_direct_exec_call(self): + payload = devin_proxy.tool_response("call-nonce", "echo nonce") + self.assertEqual(payload[0], 0) + self.assertIn(b"call-nonce", payload) + self.assertIn(b"exec", payload) + self.assertIn(b'{"command":"echo nonce"}', payload) + def test_windows_canary_command_is_shell_portable(self): command = driver.render_canary_command( r"D:\a\temp\pitot-testrole.exe", r"D:\a\temp\canary.jsonl", True, From de6e255aa1a68a38ee7adaed0a70fdd367d1b8b0 Mon Sep 17 00:00:00 2001 From: bigboateng Date: Fri, 31 Jul 2026 10:09:51 +0100 Subject: [PATCH 2/5] fix(pitot): isolate Devin Windows config paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disclosure-Reviewed: reviewed — public-safe only, private facet kept out of this commit --- labs/15-pitot/pitot-distribution/UPSTREAM.json | 2 +- labs/15-pitot/tests/real_agent_driver.py | 6 ++++++ labs/15-pitot/tests/test_model_control_proxy.py | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/labs/15-pitot/pitot-distribution/UPSTREAM.json b/labs/15-pitot/pitot-distribution/UPSTREAM.json index 1028ccaec..f2245ad28 100644 --- a/labs/15-pitot/pitot-distribution/UPSTREAM.json +++ b/labs/15-pitot/pitot-distribution/UPSTREAM.json @@ -132,7 +132,7 @@ "tests/install_real_agent.py": "a8dc79bf0914ed5ab2a82c4f2471e805f24eef2adde2261fee22dbd5aabeb985", "tests/mock_anthropic_server.js": "ecebea62f9e93791a79f1ae3dd3c67b8fa42490e9805b23b662b877edfdb0f0e", "tests/model_control_proxy.py": "cd4733ab20c16770fe3bf6ac477935f7dc285cd89be020848ae4798cb509e5f2", - "tests/real_agent_driver.py": "24bec3016f5ddee69bf56df255613223fa51444c7c53ecccc9689e4bb4673f1f", + "tests/real_agent_driver.py": "3037a8238e76451cbb6cd56e3ab4b30c402d0937c794f1522b7040f0b2959f0f", "tests/run_e2e_report.py": "a72cbc486c100b75fccf6ac787da1736e13426b44a15d9d5dddcb8177bbaf596", "tests/runtime_capability_driver.py": "54a485fc4f16981f2542d0dbd903a12b80f31f071908a378607f26678aeea07f", "tests/witness/main.go": "cd56bbd00aa44cc5baf6426c8461a8ebca4a8391518f6acfa2301ac36add7c5f", diff --git a/labs/15-pitot/tests/real_agent_driver.py b/labs/15-pitot/tests/real_agent_driver.py index df92ff163..a2847c5b2 100755 --- a/labs/15-pitot/tests/real_agent_driver.py +++ b/labs/15-pitot/tests/real_agent_driver.py @@ -115,6 +115,10 @@ def configure( prompt_flag = ["exec", "--dangerously-bypass-approvals-and-sandbox", "--dangerously-bypass-hook-trust", "--model", "pitot-control"] elif agent == "devin": endpoint = urllib.parse.urlparse(proxy) + app_data = home / "AppData/Roaming" + local_app_data = home / "AppData/Local" + app_data.mkdir(parents=True, exist_ok=True) + local_app_data.mkdir(parents=True, exist_ok=True) credentials = home / ".local/share/devin/credentials.toml" credentials.parent.mkdir(parents=True, exist_ok=True) credentials.write_text( @@ -129,6 +133,8 @@ def configure( "WINDSURF_API_SERVER_URL": proxy, "XDG_CONFIG_HOME": str(home / ".config"), "XDG_DATA_HOME": str(home / ".local/share"), + "APPDATA": str(app_data), + "LOCALAPPDATA": str(local_app_data), }) prompt_flag = [] elif agent == "copilot": diff --git a/labs/15-pitot/tests/test_model_control_proxy.py b/labs/15-pitot/tests/test_model_control_proxy.py index f829c6dca..c18a32d44 100644 --- a/labs/15-pitot/tests/test_model_control_proxy.py +++ b/labs/15-pitot/tests/test_model_control_proxy.py @@ -91,6 +91,8 @@ def test_devin_routes_all_released_client_calls_to_isolated_proxy(self): ) self.assertEqual(flags, []) self.assertEqual(environment["WINDSURF_API_SERVER_URL"], "http://127.0.0.1:9123") + self.assertEqual(environment["APPDATA"], str(base / "home/AppData/Roaming")) + self.assertEqual(environment["LOCALAPPDATA"], str(base / "home/AppData/Local")) credentials = (base / "home/.local/share/devin/credentials.toml").read_text() self.assertIn('api_server_url = "http://127.0.0.1:9123"', credentials) From d62a86a2868a0fdc6383aebeb3896b0af83b69dc Mon Sep 17 00:00:00 2001 From: bigboateng Date: Fri, 31 Jul 2026 13:05:40 +0100 Subject: [PATCH 3/5] fix(pitot): serve Devin E2E proxy over HTTP/1.1 keep-alive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Devin's HTTP client pools connections; the deterministic Connect proxy spoke HTTP/1.0 and closed the socket after every response. A request written to the stale pooled connection can black-hole until the CLI's 10s team-settings deadline expires, killing session/new before the negative-phase check can observe anything (seen intermittently on the Windows cell; the same run's log shows the fast-fail variant of the race recovered by Devin's 250ms retry at init). Keep-alive removes the stale-close race: protocol_version HTTP/1.1 and an exact content-length on every response path, including the GET 404. Disclosure-Reviewed: reviewed — public-safe only, private facet kept out of this commit --- labs/15-pitot/pitot-distribution/UPSTREAM.json | 2 +- labs/15-pitot/tests/devin_control_proxy.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/labs/15-pitot/pitot-distribution/UPSTREAM.json b/labs/15-pitot/pitot-distribution/UPSTREAM.json index f2245ad28..5c248ab56 100644 --- a/labs/15-pitot/pitot-distribution/UPSTREAM.json +++ b/labs/15-pitot/pitot-distribution/UPSTREAM.json @@ -115,7 +115,7 @@ "sensor/sensor.go": "498d4f69c9243a409b25704beab9d441bb312c7813d168138b85a69cff08d8e9", "sensor/sensor_test.go": "9e0ec50ec3930c37b0e4cc56419be451fb56432f0f22efa145b55ddf6440cd4b", "tests/cursor_control_proxy.mjs": "ab532aa56a9299f497f3ceeedb4b6a0beb26089b7c022ed774385dd149ea56e6", - "tests/devin_control_proxy.py": "0e74ac4c0e7182b8e2e3a89b4a16fbe0d2ffdabb2c40a301aa60176ae74c8d9c", + "tests/devin_control_proxy.py": "21e695dfb60d2449b8cf10c09274d4b4512a9162ccd39b725469fb405a0cd644", "tests/e2e_claude_cli_test.sh": "b28c4d1963e326b4b3f158a7cfc1b92771e768ac9665dd43e8a339013cc11568", "tests/e2e_codex_cli_test.sh": "dbae5224c87410a5a5d67023d7e23406453bc5493d981af9347993b4e1f562f2", "tests/e2e_copilot_cli_test.sh": "61b1e44dcd598d2d33e7f04dec26bec74405a9e2456ecda46e94dc8d43ad4315", diff --git a/labs/15-pitot/tests/devin_control_proxy.py b/labs/15-pitot/tests/devin_control_proxy.py index d4c1c5936..37cc884d0 100644 --- a/labs/15-pitot/tests/devin_control_proxy.py +++ b/labs/15-pitot/tests/devin_control_proxy.py @@ -81,11 +81,20 @@ def save() -> None: temporary.replace(args.receipt) class Handler(BaseHTTPRequestHandler): + # Devin's HTTP client pools connections. Under the default HTTP/1.0 + # the server closes the socket after every response, and a request + # written to the stale pooled connection can black-hole until the + # CLI's 10s team-settings deadline expires (observed on Windows). + # HTTP/1.1 keep-alive keeps pooled connections valid; every response + # below must therefore carry an exact content-length. + protocol_version = "HTTP/1.1" + def log_message(self, *_: object) -> None: return def do_GET(self) -> None: self.send_response(404) + self.send_header("content-length", "0") self.end_headers() def do_POST(self) -> None: From 356114eb3f6eade57ea182e48558cb8dd473a41f Mon Sep 17 00:00:00 2001 From: bigboateng Date: Fri, 31 Jul 2026 13:16:38 +0100 Subject: [PATCH 4/5] fix(pitot): deepen Devin E2E proxy listen backlog and drain request bodies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 2 on the intermittent Windows Devin cell. New evidence from run 30629363835 refutes the stale-connection reading: the same 10s team-settings timeout occurred with keep-alive active, attempt 1 fails instantly on a fresh process during Devin's startup burst, the hang is a single request with no transient retries, and other requests were served concurrently (auxiliary_requests=10). The mechanism consistent with all observations is listen-backlog overflow: socketserver defaults request_queue_size to 5, Devin bursts more concurrent connections than that at startup/session creation, and Windows either RSTs the overflowed SYN (the fast ConnectionFailed) or silently drops it so the client retransmit schedule outlives the 10s deadline (the single hang). Linux/macOS backlog semantics differ, matching the Windows-only intermittency. - request_queue_size=128 (+ daemon_threads) on the serving socket - drain request bodies exactly (content-length and chunked) so a kept-alive connection can never desync on an unconsumed body Disclosure-Reviewed: reviewed — public-safe only, private facet kept out of this commit --- .../15-pitot/pitot-distribution/UPSTREAM.json | 2 +- labs/15-pitot/tests/devin_control_proxy.py | 32 +++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/labs/15-pitot/pitot-distribution/UPSTREAM.json b/labs/15-pitot/pitot-distribution/UPSTREAM.json index 5c248ab56..2ec7ff960 100644 --- a/labs/15-pitot/pitot-distribution/UPSTREAM.json +++ b/labs/15-pitot/pitot-distribution/UPSTREAM.json @@ -115,7 +115,7 @@ "sensor/sensor.go": "498d4f69c9243a409b25704beab9d441bb312c7813d168138b85a69cff08d8e9", "sensor/sensor_test.go": "9e0ec50ec3930c37b0e4cc56419be451fb56432f0f22efa145b55ddf6440cd4b", "tests/cursor_control_proxy.mjs": "ab532aa56a9299f497f3ceeedb4b6a0beb26089b7c022ed774385dd149ea56e6", - "tests/devin_control_proxy.py": "21e695dfb60d2449b8cf10c09274d4b4512a9162ccd39b725469fb405a0cd644", + "tests/devin_control_proxy.py": "04d200d3828fc3766b1f5b6b2f5a9c9b6d39791a9ed4d0706dd5d3a0c073c003", "tests/e2e_claude_cli_test.sh": "b28c4d1963e326b4b3f158a7cfc1b92771e768ac9665dd43e8a339013cc11568", "tests/e2e_codex_cli_test.sh": "dbae5224c87410a5a5d67023d7e23406453bc5493d981af9347993b4e1f562f2", "tests/e2e_copilot_cli_test.sh": "61b1e44dcd598d2d33e7f04dec26bec74405a9e2456ecda46e94dc8d43ad4315", diff --git a/labs/15-pitot/tests/devin_control_proxy.py b/labs/15-pitot/tests/devin_control_proxy.py index 37cc884d0..c70528580 100644 --- a/labs/15-pitot/tests/devin_control_proxy.py +++ b/labs/15-pitot/tests/devin_control_proxy.py @@ -97,8 +97,26 @@ def do_GET(self) -> None: self.send_header("content-length", "0") self.end_headers() + def _read_body(self) -> bytes: + # Drain the request body exactly, including chunked framing: an + # unconsumed body on a kept-alive connection desyncs every later + # request the client sends on it. + if "chunked" in self.headers.get("transfer-encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline(65537).split(b";", 1)[0].strip() + size = int(size_line or b"0", 16) + if size == 0: + while self.rfile.readline(65537) not in (b"\r\n", b"\n", b""): + pass # trailers + break + chunks.append(self.rfile.read(size)) + self.rfile.readline(65537) # CRLF after each chunk + return b"".join(chunks) + return self.rfile.read(int(self.headers.get("content-length", "0"))) + def do_POST(self) -> None: - raw = self.rfile.read(int(self.headers.get("content-length", "0"))) + raw = self._read_body() path = self.path.split("?", 1)[0] content_type = self.headers.get("content-type", "").split(";", 1)[0].strip().lower() if not path.endswith("/GetChatMessage"): @@ -174,7 +192,17 @@ def do_POST(self) -> None: self.end_headers() self.wfile.write(payload) - server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + class BurstTolerantServer(ThreadingHTTPServer): + # Devin bursts many concurrent connections at startup and session + # creation. socketserver's default listen backlog of 5 overflows on + # Windows, where an overflowed SYN is silently dropped and the + # client's retransmit schedule outlives the CLI's 10s team-settings + # deadline (or is RST -> instant ConnectionFailed). A deep backlog + # makes the local fetch path deterministic. + request_queue_size = 128 + daemon_threads = True + + server = BurstTolerantServer(("127.0.0.1", 0), Handler) args.ready_file.write_text(f"http://127.0.0.1:{server.server_port}\n", encoding="utf-8") save() try: From 61a75e2cbbfeb501140c3ba00173221592cf89fb Mon Sep 17 00:00:00 2001 From: bigboateng Date: Fri, 31 Jul 2026 13:26:45 +0100 Subject: [PATCH 5/5] test(pitot): journal Devin proxy connections for Windows-cell triage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-2 evidence eliminated the startup-burst fast-fail (backlog fix held: team settings ready in 8ms) but the session/new refresh still black-holes for exactly 10s as a single request, under both HTTP/1.0 and HTTP/1.1 proxies. The remaining question is binary and only CI can answer it: does that request reach the proxy at all, and on which connection. The proxy now journals CONN open/close and REQ/RES lines (client port, path, framing, duration) to a file, and the driver prints the journal after each Devin phase. Receipt schema unchanged. Disclosure-Reviewed: reviewed — public-safe only, private facet kept out of this commit --- .../15-pitot/pitot-distribution/UPSTREAM.json | 4 +- labs/15-pitot/tests/devin_control_proxy.py | 38 +++++++++++++++++++ labs/15-pitot/tests/real_agent_driver.py | 11 +++++- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/labs/15-pitot/pitot-distribution/UPSTREAM.json b/labs/15-pitot/pitot-distribution/UPSTREAM.json index 2ec7ff960..7f648e9a4 100644 --- a/labs/15-pitot/pitot-distribution/UPSTREAM.json +++ b/labs/15-pitot/pitot-distribution/UPSTREAM.json @@ -115,7 +115,7 @@ "sensor/sensor.go": "498d4f69c9243a409b25704beab9d441bb312c7813d168138b85a69cff08d8e9", "sensor/sensor_test.go": "9e0ec50ec3930c37b0e4cc56419be451fb56432f0f22efa145b55ddf6440cd4b", "tests/cursor_control_proxy.mjs": "ab532aa56a9299f497f3ceeedb4b6a0beb26089b7c022ed774385dd149ea56e6", - "tests/devin_control_proxy.py": "04d200d3828fc3766b1f5b6b2f5a9c9b6d39791a9ed4d0706dd5d3a0c073c003", + "tests/devin_control_proxy.py": "0be724db777bb790d18c49fad6280ddf77c261f7ae3c947323cc0cc20aab0422", "tests/e2e_claude_cli_test.sh": "b28c4d1963e326b4b3f158a7cfc1b92771e768ac9665dd43e8a339013cc11568", "tests/e2e_codex_cli_test.sh": "dbae5224c87410a5a5d67023d7e23406453bc5493d981af9347993b4e1f562f2", "tests/e2e_copilot_cli_test.sh": "61b1e44dcd598d2d33e7f04dec26bec74405a9e2456ecda46e94dc8d43ad4315", @@ -132,7 +132,7 @@ "tests/install_real_agent.py": "a8dc79bf0914ed5ab2a82c4f2471e805f24eef2adde2261fee22dbd5aabeb985", "tests/mock_anthropic_server.js": "ecebea62f9e93791a79f1ae3dd3c67b8fa42490e9805b23b662b877edfdb0f0e", "tests/model_control_proxy.py": "cd4733ab20c16770fe3bf6ac477935f7dc285cd89be020848ae4798cb509e5f2", - "tests/real_agent_driver.py": "3037a8238e76451cbb6cd56e3ab4b30c402d0937c794f1522b7040f0b2959f0f", + "tests/real_agent_driver.py": "789e72797137fedcc7afc61f14a943df3f57e02f0a2329d125beeb8ce9311394", "tests/run_e2e_report.py": "a72cbc486c100b75fccf6ac787da1736e13426b44a15d9d5dddcb8177bbaf596", "tests/runtime_capability_driver.py": "54a485fc4f16981f2542d0dbd903a12b80f31f071908a378607f26678aeea07f", "tests/witness/main.go": "cd56bbd00aa44cc5baf6426c8461a8ebca4a8391518f6acfa2301ac36add7c5f", diff --git a/labs/15-pitot/tests/devin_control_proxy.py b/labs/15-pitot/tests/devin_control_proxy.py index c70528580..64633972e 100644 --- a/labs/15-pitot/tests/devin_control_proxy.py +++ b/labs/15-pitot/tests/devin_control_proxy.py @@ -7,6 +7,8 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json from pathlib import Path +import threading +import time def varint(value: int) -> bytes: @@ -54,7 +56,18 @@ def main() -> int: parser.add_argument("--ready-file", type=Path, required=True) parser.add_argument("--canary-command", required=True) parser.add_argument("--response-fault", choices=("none", "text"), default="none") + parser.add_argument("--request-log", type=Path, default=None) args = parser.parse_args() + log_lock = threading.Lock() + + def rlog(line: str) -> None: + # Connection/request journal for CI triage: written to a file (never + # a pipe nobody drains) and printed by the driver after each phase. + if args.request_log is None: + return + with log_lock, open(args.request_log, "a", encoding="utf-8") as fh: + fh.write(f"{time.time():.6f} {line}\n") + fh.flush() allow_id = f"pitot_tool_allow_{args.nonce}" deny_id = f"pitot_tool_deny_{args.nonce}" receipt: dict[str, object] = { @@ -92,10 +105,20 @@ class Handler(BaseHTTPRequestHandler): def log_message(self, *_: object) -> None: return + def setup(self) -> None: + super().setup() + rlog(f"CONN open client=:{self.client_address[1]}") + + def finish(self) -> None: + rlog(f"CONN close client=:{self.client_address[1]}") + super().finish() + def do_GET(self) -> None: + rlog(f"REQ client=:{self.client_address[1]} GET {self.path}") self.send_response(404) self.send_header("content-length", "0") self.end_headers() + rlog(f"RES client=:{self.client_address[1]} 404 GET {self.path}") def _read_body(self) -> bytes: # Drain the request body exactly, including chunked framing: an @@ -116,6 +139,12 @@ def _read_body(self) -> bytes: return self.rfile.read(int(self.headers.get("content-length", "0"))) def do_POST(self) -> None: + started = time.monotonic() + rlog( + f"REQ client=:{self.client_address[1]} POST {self.path} " + f"cl={self.headers.get('content-length', '-')} " + f"te={self.headers.get('transfer-encoding', '-')}" + ) raw = self._read_body() path = self.path.split("?", 1)[0] content_type = self.headers.get("content-type", "").split(";", 1)[0].strip().lower() @@ -126,6 +155,10 @@ def do_POST(self) -> None: self.send_header("content-type", "application/proto") self.send_header("content-length", "0") self.end_headers() + rlog( + f"RES client=:{self.client_address[1]} 200 {path} " + f"aux {int((time.monotonic() - started) * 1000)}ms" + ) return lowered = raw.lower() @@ -184,6 +217,7 @@ def do_POST(self) -> None: } save() self.send_error(409, "request did not advance Devin control trajectory") + rlog(f"RES client=:{self.client_address[1]} 409 {path}") return save() self.send_response(200) @@ -191,6 +225,10 @@ def do_POST(self) -> None: self.send_header("content-length", str(len(payload))) self.end_headers() self.wfile.write(payload) + rlog( + f"RES client=:{self.client_address[1]} 200 {path} " + f"chat {int((time.monotonic() - started) * 1000)}ms" + ) class BurstTolerantServer(ThreadingHTTPServer): # Devin bursts many concurrent connections at startup and session diff --git a/labs/15-pitot/tests/real_agent_driver.py b/labs/15-pitot/tests/real_agent_driver.py index a2847c5b2..0a49e2994 100755 --- a/labs/15-pitot/tests/real_agent_driver.py +++ b/labs/15-pitot/tests/real_agent_driver.py @@ -485,6 +485,7 @@ def main() -> int: str(canary_executable), receipt_argument, windows_host() and not host_controls_wsl, ) ready, proxy_receipt, witness_receipt = base / "proxy.url", base / "proxy.json", base / "witness.jsonl" + proxy_request_log = base / "proxy-requests.log" runtime_descriptor = base / "runtime.json" runtime_config = base / "pitot.json" consumer_receipt = base / "consumer.jsonl" @@ -539,7 +540,7 @@ def main() -> int: else: proxy_command = ["node", str(LAB / "tests/cursor_control_proxy.mjs"), "--nonce", nonce, "--receipt", str(proxy_receipt), "--ready-file", str(ready), "--canary-command", canary_command, "--response-fault", args.response_fault] elif args.agent == "devin": - proxy_command = [sys.executable, str(LAB / "tests/devin_control_proxy.py"), "--nonce", nonce, "--receipt", str(proxy_receipt), "--ready-file", str(ready), "--canary-command", canary_command, "--response-fault", args.response_fault] + proxy_command = [sys.executable, str(LAB / "tests/devin_control_proxy.py"), "--nonce", nonce, "--receipt", str(proxy_receipt), "--ready-file", str(ready), "--canary-command", canary_command, "--response-fault", args.response_fault, "--request-log", str(proxy_request_log)] else: proxy_command = [sys.executable, str(LAB / "tests/model_control_proxy.py"), "--agent", args.agent, "--nonce", nonce, "--receipt", str(proxy_receipt), "--ready-file", str(ready), "--canary-command", canary_command, "--response-fault", args.response_fault] proxy_process = subprocess.Popen( @@ -624,6 +625,14 @@ def main() -> int: # reporting layer cannot fail after a successful agent session. sys.stdout.buffer.write(completed.stdout.encode("utf-8", errors="replace")) sys.stdout.buffer.flush() + if args.agent == "devin" and proxy_request_log.is_file(): + # Connection-level journal from the proxy: shows whether a + # hanging fetch ever reached the server and on which socket. + journal = proxy_request_log.read_text(encoding="utf-8", errors="replace") + sys.stdout.buffer.write( + ("PITOT_PROXY_REQUEST_LOG begin\n" + journal[-12000:] + "PITOT_PROXY_REQUEST_LOG end\n").encode("utf-8", errors="replace") + ) + sys.stdout.buffer.flush() if args.expect_incompatible_response: observed = json.loads(proxy_receipt.read_text(encoding="utf-8")) if proxy_receipt.is_file() else {} if not (