From 1eb145027e9798e5682916c0cd9d1b84672108e5 Mon Sep 17 00:00:00 2001 From: bigboateng Date: Fri, 31 Jul 2026 15:00:40 +0100 Subject: [PATCH 1/2] harden(pitot): fail-closed boundaries, bounded state, adapter guards + public README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feature-preserving hardening across the Pitot adapter surface, guided by a full Locus analysis (trace-refinement per adapter, supervisory-rw by host class, nonblockingness of the router, root-cause of the two silent degradations, ranked frontier). No adapter, transport, or observe-only mode was removed. Runtime / boundary: - transport: an event-delivery fault now returns 409, not a 204 that read as allow (fail-open closed). - runtime: additive strict mode — require_controller config key and `pitot run --strict` flag — faults instead of silently allowing when a declared kind has no controller; default behavior unchanged, with a once-per-kind observation-only notice otherwise. - hook: one-line stderr notice when no runtime is selected (still exit 0); allow path now requires the response ActionID to match. - sensor/adapters: decode-time tool-name guards per host; ActionKind no longer silently defaults to "shell" for unknown boundary events. - bridge: the router's resolved-action set is now bounded (4096-entry window). - devinacp: one-shot diagnostic when a tool_call update lacks the cognition.ai/inferenceToolName meta (stays fail-closed). - doctor: PITOT_RUNTIME-unset note and a binary-hash drift check for non-repo-wired hosts. Tests: - new Go guards (adapters, sensor, bridge window, runtime strict + transport fault, devinacp meta, hook notice). - new Python coverage: devin control-proxy HTTP behavior, validate_receipts negative branches, the two E2E apparatus binaries, and npm/kimi/cursor installers. Fixed a real devin-proxy save() race surfaced by the burst test. Public README: Devin added to the supported-hosts table with a platforms column, host admission criteria (deny-continuation) section, a threat model that names Kimi's fail-open host semantics plainly, and a contributing guide covering both adapter transport classes. Disclosure-Reviewed: reviewed — public-safe only, private facet kept out of this commit --- .../15-pitot/pitot-distribution/UPSTREAM.json | 33 +-- labs/15-pitot/pitot/adapters/adapters.go | 81 ++++--- labs/15-pitot/pitot/adapters/guard_test.go | 48 +++++ labs/15-pitot/pitot/bridge/bridge.go | 24 ++- labs/15-pitot/pitot/bridge/window_test.go | 42 ++++ labs/15-pitot/pitot/cmd/pitot/doctor_host.go | 48 ++++- .../pitot/cmd/pitot/hook_notice_test.go | 25 +++ labs/15-pitot/pitot/cmd/pitot/main.go | 23 +- labs/15-pitot/pitot/config/config.go | 9 + .../pitot/internal/devinacp/client.go | 12 ++ .../pitot/internal/devinacp/meta_test.go | 44 ++++ labs/15-pitot/pitot/runtime/runtime.go | 40 +++- labs/15-pitot/pitot/runtime/strict_test.go | 53 +++++ labs/15-pitot/pitot/runtime/transport.go | 6 + .../pitot/runtime/transport_fault_test.go | 46 ++++ labs/15-pitot/pitot/sensor/guard_test.go | 47 ++++ labs/15-pitot/pitot/sensor/sensor.go | 9 +- .../public-readme-preview/CONTRIBUTING.md | 26 +++ labs/15-pitot/public-readme-preview/README.md | 132 ++++++++++-- labs/15-pitot/tests/devin_control_proxy.py | 15 +- labs/15-pitot/tests/real_agent_driver.py | 7 + .../tests/test_devin_control_proxy.py | 200 ++++++++++++++++++ labs/15-pitot/tests/test_e2e_binaries.py | 126 +++++++++++ .../15-pitot/tests/test_install_real_agent.py | 82 +++++++ .../15-pitot/tests/test_receipt_validation.py | 167 +++++++++++++++ 25 files changed, 1249 insertions(+), 96 deletions(-) create mode 100644 labs/15-pitot/pitot/adapters/guard_test.go create mode 100644 labs/15-pitot/pitot/bridge/window_test.go create mode 100644 labs/15-pitot/pitot/cmd/pitot/hook_notice_test.go create mode 100644 labs/15-pitot/pitot/internal/devinacp/meta_test.go create mode 100644 labs/15-pitot/pitot/runtime/strict_test.go create mode 100644 labs/15-pitot/pitot/runtime/transport_fault_test.go create mode 100644 labs/15-pitot/pitot/sensor/guard_test.go create mode 100644 labs/15-pitot/tests/test_devin_control_proxy.py create mode 100644 labs/15-pitot/tests/test_e2e_binaries.py create mode 100644 labs/15-pitot/tests/test_receipt_validation.py diff --git a/labs/15-pitot/pitot-distribution/UPSTREAM.json b/labs/15-pitot/pitot-distribution/UPSTREAM.json index 7f648e9a4..23de2e092 100644 --- a/labs/15-pitot/pitot-distribution/UPSTREAM.json +++ b/labs/15-pitot/pitot-distribution/UPSTREAM.json @@ -1,11 +1,12 @@ { "files": { ".goreleaser.yml": "d375f187624219e9420acd2a40cfae95772ef1fee3abda3e729d6a16ba612032", - "CONTRIBUTING.md": "02c89a5790f2943e5e2466b0d260210238cae306922f0f031adb5f7028b48066", + "CONTRIBUTING.md": "a983d968cf228cc60d3f80cecd9a52e5ba8f3235595886f858fc384ed3dd114b", "LICENSE": "bd17d47aefaa7406616179288058001b6532881a2517254fe668d7e9c4965cfb", - "README.md": "27df9b71b14ac474de3d03f2049eabb7d1c2c35514b7d351940683b0651d1984", + "README.md": "dce6eccbb3638c04ea0310a3f6efe103f0181c22e254e79c2bdd91e923656e19", "adapter-verification.json": "fedd1624e01d19739114203674281296de779097e45fbb9e497047805b7d9250", - "adapters/adapters.go": "13cc5d94ea7c4ae50eccfd27d63e0d91b38cf76ee20aa9737a7edc9bd47d13b2", + "adapters/adapters.go": "25ebd930b4e3aa92a02043e75f2a4a6a818c546b479c978f4b75b8c3962c56de", + "adapters/guard_test.go": "cd1c09e2fb3a2ecc7a73a1714d9b5bf976d67ab206ed845027ed34699c8e123d", "assets/pitot-boundary.png": "8a0ddb7d81831d94e14813f50ea4ca8670d77417f339ed2f91f0c653bf52f41d", "assets/pitot-boundary.svg": "0c3871d70c84748573f231842091deb38a6def2862403ca34e8cc4493b9c9ebf", "assets/pitot-hero.png": "a73532252b1e66c06273abbf5a4fe6261e98de3133b09e8d550edacfeeab92f8", @@ -13,17 +14,19 @@ "assets/pitot-mark.svg": "cacd728b4d4da45000ccde15d905314f2d92eb47b5e40a5f86a9e24ad671a003", "assets/pitot-two-roles.png": "9093368b30b7a0b704358343e78df2f937af21f2263b50e8da21a2946306255a", "assets/pitot-two-roles.svg": "528edf57c3eddb4432e119b9aa80a47452d47eb428e0e77a6f8623f244334475", - "bridge/bridge.go": "5adfcd3f743cae46e4446a6e030d53464ada97de0261a8588fa2a9fcd62136b8", + "bridge/bridge.go": "104d515b1cd66ef1e9dac57e5ef25d80b847b7512c44fc6703ad43efa99e59be", "bridge/bridge_test.go": "6dcc6d05f2b39c25955fc0b2d21d3d148dd9d77600fb12799941f86bdb1acb61", + "bridge/window_test.go": "3d9f3f6af95af61a26d99da39f84842db8b71f33c0788f746782e8ec2f216c60", "cmd/generate-schema/main.go": "6e9d0030290d99e36967433f96e38385a122974f899ad9421aac1ef7e50d8fcb", "cmd/pitot/devin_acp.go": "b10962cc21239d0b3955d2521f1f4d940c5de34624d14ce89ef0e40e1fbc1138", "cmd/pitot/devin_acp_test.go": "158a9ee019e8368b9f9434000eea8cbeee98201203213ef83797102549e1dd0a", - "cmd/pitot/doctor_host.go": "fa7819448218c3dd92d3393df9a0ac6e5aa70803348898eec10220bc7b3867a4", + "cmd/pitot/doctor_host.go": "f9d91ae1f0e8cba344f8a6e742467e98e2cf7c82afdb9da3043fba1bc13bbee5", "cmd/pitot/doctor_host_test.go": "4e6e327f6cf27cf94a0a608e10eb6790d6c11fcd53e6dfd7370007190749952f", + "cmd/pitot/hook_notice_test.go": "b6a0c3689a97beecf32e9aced68585715cc125139f1646f8ba90c4e65e6b2fa8", "cmd/pitot/install.go": "46cab87809566f8da47c4d71b8a87618d06cb0829bdabe1c8f734faf53a9f46c", "cmd/pitot/kimi_control_test.go": "27b38867d4799636a664e3b1726ef55568f5dfd6a9be11f0ce5eca9931d759a4", "cmd/pitot/kimi_smoke_test.go": "01cbf18312902cac42ec1f2547d35362c2bc920938acea508ba7c7f0638a9473", - "cmd/pitot/main.go": "9b7851b4e91c3eef08d6dac82939c1842f8e1a5237bf603e5a6ccc51a50561ec", + "cmd/pitot/main.go": "230d3ccc99a244150644602692a5efc0f40cf8e82de840f43cb830db29fd6bd9", "cmd/pitot/main_test.go": "35c17c8de7caf13d0ed399c0b67903236cd3c64faa1ea134c5445d00887a32a5", "cmd/pitot/shim.go": "3a7a4be5bd769010ad947de78188dd17c729388dbf74153d4dbd51fba2121ab1", "cmd/pitot/substrate_test.go": "b4e7d83aa592f7b5bf4ec059ec552abd62473e51a6634a26428448168898d0a3", @@ -36,7 +39,7 @@ "cmd/pitot/workbench_contract_test.go": "5e465f3d3f8b93ffaacfb4738279d369b13514bf7613b8611952f3cd26896586", "cmd/pitot/workbench_dev_test.go": "abffe81e1a25f086d7f3c2f1c32986ee93618bbfc87220bdfda840ec6c6466bf", "cmd/pitot/workbench_test.go": "3c561498dfee4aacf6935fe4b0bfe4449c3961c169c04f1f13e81a38b22d0914", - "config/config.go": "84476c7525833b07222b3b06834ff528f9c496f164bb065d006afad29bbca1fb", + "config/config.go": "d49d274ec7fe16aebb0991b2422095f2f68a28ba818c6a2b2e9826ee30377645", "config/config_test.go": "87d3e5ddc4a3b43c736070de671d03e03ffe29cdd759771526ad27fd9bc0034c", "config/findroot_test.go": "89d8b31f188325a02bde2dcd77e683c06e9dca4f59c872366c027111147ea8a5", "config/merge_test.go": "595d2c96ac879cca7c57b77b99504f34629b8aa55b38d515f35ad76c006ef532", @@ -67,8 +70,9 @@ "integrations/pi/pitot.ts": "ed2d60d5ab6e33e115cfa058e4f96095100e93a061567d0af31249aa756bab3e", "integrations/qwen/PreToolUse": "95c358620f2f882bb8680e6aa9639f3b36a8567a3fd60cf14a6cdf3b3fbcf78b", "integrations/qwen/PreToolUse.cjs": "e9bf00bbbee5c15f01ea203d0992b34f8754b75a9c9889eecb16baeb10fcce6c", - "internal/devinacp/client.go": "e2e70755d5888a2bd34e44b160d1dbf955fe147adf020050aa3fbc68cdbe709d", + "internal/devinacp/client.go": "531bf28ea26c5c4b417c7b6ce2f4b97a41ca760226ca06509412c353aae29131", "internal/devinacp/client_test.go": "5bfff9b7f29d253450e4424ab44dab97877ddb39b3941607a4e264dd66da32cb", + "internal/devinacp/meta_test.go": "0b56f3beb24d4afeff0a6066cb4600bb773976cfa558fea7b963ccda9752c939", "internal/testrole/main.go": "6d657eb85d8ddaff0ae5a3da281aa7aa0a4bdda860d179e99508110033765787", "internal/testrole/main_test.go": "c7fbc4905bcef7d662c9e162a4c32c8a86d5c1a97820e9f4ee48f643c4e47a38", "projection/projection.go": "4d3c823fd72a3ca5387dba3683838a1d7e455e9b18309acc839763a39b7bb35f", @@ -79,9 +83,11 @@ "runtime/descriptor_windows.go": "2d9ffefe3af0154fa8042de6b67460d4e86dd3f4cdd9e986f180f7d0c535c9a5", "runtime/request.go": "198c44fd6c547022a15b6d0d48e4d0130fa8afb687994365115576e4d874550d", "runtime/request_test.go": "86d8a2feb4ec72e8ed675b9567da2d1f5d628950eeec907c10b9cc1675aa1904", - "runtime/runtime.go": "b90072bf119c9121e3d185fa27e8ac372ec9dcb33c9f38fea8050c314b9dd5e4", + "runtime/runtime.go": "36fdf4643c04d058fef380333a0a3942913d1dd60e910b893dc134da4b65d6bf", "runtime/runtime_test.go": "afd78d122af20148bf30d0db873ff002544189df0dfec0f6167b8cf5cd0d42b1", - "runtime/transport.go": "83e2218fb28474e875dafa6943bc5b665acef0565aaf5955fa88b1b4fd21614e", + "runtime/strict_test.go": "d2eb2b78563733e3279289d8b6d17a4276e831ed3c78f8b3ec9d6706992ffaec", + "runtime/transport.go": "671d67be8b8ea0c5c3132cc355054b36b0ecf8001dcc79b4e04be7a584f4bf02", + "runtime/transport_fault_test.go": "1eb58f09a2412ed5411f5d07c4ac62bb8f0fca1ce41201f670f5fe9e98b1abc7", "runtime/transport_test.go": "9b69f590f1e258470adea249b3ac6d4a00f1001f10bb08dfa7b56c6e2d6709ae", "schema/schema.go": "fd5c3b76979c88aeed75fadb7e3c94abcad62e067d77595f021fb422a60f2211", "sdk/csharp/Pitot/Types.cs": "3fecc1eabca6e3ad218c73741d185f815d868a618d1e7ed3b9320eeb6942977d", @@ -112,10 +118,11 @@ "sdk/typescript/src/runner.ts": "c58babd3ec3996a05988f3cb7dec061cf7c48f41cd5125625be3408cb9b207f0", "sdk/typescript/tsconfig.json": "de0065da9acf19a93ba8646c90b7101dfd5a8667552d3d3c59e3effd813218aa", "sensor/decode_fuzz_test.go": "d27f2fbbc069eded26a73c9cd9bace98dd8a9e34949576790b81a08d130fbaf2", - "sensor/sensor.go": "498d4f69c9243a409b25704beab9d441bb312c7813d168138b85a69cff08d8e9", + "sensor/guard_test.go": "bbd112398c9bb61eeb889f5cb5468bd0d4581537b4292edfcb3b6bc0864b7347", + "sensor/sensor.go": "3832eeb1267513f74eefe910394983df847a4c5152029df0e19d170bd05fa059", "sensor/sensor_test.go": "9e0ec50ec3930c37b0e4cc56419be451fb56432f0f22efa145b55ddf6440cd4b", "tests/cursor_control_proxy.mjs": "ab532aa56a9299f497f3ceeedb4b6a0beb26089b7c022ed774385dd149ea56e6", - "tests/devin_control_proxy.py": "0be724db777bb790d18c49fad6280ddf77c261f7ae3c947323cc0cc20aab0422", + "tests/devin_control_proxy.py": "8306a80af890b040bfa7bda43573cd2fec7df207aa653f0b1a970f625a263566", "tests/e2e_claude_cli_test.sh": "b28c4d1963e326b4b3f158a7cfc1b92771e768ac9665dd43e8a339013cc11568", "tests/e2e_codex_cli_test.sh": "dbae5224c87410a5a5d67023d7e23406453bc5493d981af9347993b4e1f562f2", "tests/e2e_copilot_cli_test.sh": "61b1e44dcd598d2d33e7f04dec26bec74405a9e2456ecda46e94dc8d43ad4315", @@ -132,7 +139,7 @@ "tests/install_real_agent.py": "a8dc79bf0914ed5ab2a82c4f2471e805f24eef2adde2261fee22dbd5aabeb985", "tests/mock_anthropic_server.js": "ecebea62f9e93791a79f1ae3dd3c67b8fa42490e9805b23b662b877edfdb0f0e", "tests/model_control_proxy.py": "cd4733ab20c16770fe3bf6ac477935f7dc285cd89be020848ae4798cb509e5f2", - "tests/real_agent_driver.py": "789e72797137fedcc7afc61f14a943df3f57e02f0a2329d125beeb8ce9311394", + "tests/real_agent_driver.py": "33955c673caabcff33f7884a60b87a84e60880cce658e5747a9c939f9f20f4ee", "tests/run_e2e_report.py": "a72cbc486c100b75fccf6ac787da1736e13426b44a15d9d5dddcb8177bbaf596", "tests/runtime_capability_driver.py": "54a485fc4f16981f2542d0dbd903a12b80f31f071908a378607f26678aeea07f", "tests/witness/main.go": "cd56bbd00aa44cc5baf6426c8461a8ebca4a8391518f6acfa2301ac36add7c5f", diff --git a/labs/15-pitot/pitot/adapters/adapters.go b/labs/15-pitot/pitot/adapters/adapters.go index 9f886c346..c24f0b6ad 100644 --- a/labs/15-pitot/pitot/adapters/adapters.go +++ b/labs/15-pitot/pitot/adapters/adapters.go @@ -116,7 +116,7 @@ var ( MainEventName: "tool_call", Parser: ParserConfig{ CanonicalEvent: []byte(`{"hook_event_name":"tool_call","tool_name":"bash","tool_input":{"command":"git status --short"}}`), - CommandFor: toolInputCommand, + CommandFor: shellCommandForTools("bash"), ActionKinds: map[string]string{"tool_call": "shell"}, }, Partition: ControlPartition{Controllable: []string{"tool_call"}}, @@ -142,13 +142,7 @@ var ( MainEventName: "PreToolUse", Parser: ParserConfig{ CanonicalEvent: []byte(`{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"git status --short"}}`), - CommandFor: func(raw RawHookEvent) (string, bool) { - if raw.ToolInput == nil { - return "", false - } - value, present := raw.ToolInput["command"].(string) - return value, present && value != "" - }, + CommandFor: shellCommandForTools("Bash"), ActionKinds: map[string]string{ "PreToolUse": "shell", }, @@ -162,13 +156,7 @@ var ( MainEventName: "PreToolUse", Parser: ParserConfig{ CanonicalEvent: []byte(`{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"git status --short"}}`), - CommandFor: func(raw RawHookEvent) (string, bool) { - if raw.ToolInput == nil { - return "", false - } - value, present := raw.ToolInput["command"].(string) - return value, present && value != "" - }, + CommandFor: shellCommandForTools("Bash"), ActionKinds: map[string]string{ "PreToolUse": "shell", }, @@ -182,13 +170,7 @@ var ( MainEventName: "BeforeTool", Parser: ParserConfig{ CanonicalEvent: []byte(`{"hook_event_name":"BeforeTool","tool_name":"run_shell_command","tool_input":{"command":"git status --short"}}`), - CommandFor: func(raw RawHookEvent) (string, bool) { - if raw.ToolInput == nil { - return "", false - } - value, present := raw.ToolInput["command"].(string) - return value, present && value != "" - }, + CommandFor: shellCommandForTools("run_shell_command"), ActionKinds: map[string]string{ "BeforeTool": "shell", }, @@ -202,13 +184,7 @@ var ( MainEventName: "PreToolUse", Parser: ParserConfig{ CanonicalEvent: []byte(`{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"git status --short"}}`), - CommandFor: func(raw RawHookEvent) (string, bool) { - if raw.ToolInput == nil { - return "", false - } - value, present := raw.ToolInput["command"].(string) - return value, present && value != "" - }, + CommandFor: shellCommandForTools("Bash"), ActionKinds: map[string]string{ "PreToolUse": "shell", }, @@ -222,13 +198,7 @@ var ( MainEventName: "PreToolUse", Parser: ParserConfig{ CanonicalEvent: []byte(`{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"git status --short"}}`), - CommandFor: func(raw RawHookEvent) (string, bool) { - if raw.ToolInput == nil { - return "", false - } - value, present := raw.ToolInput["command"].(string) - return value, present && value != "" - }, + CommandFor: shellCommandForTools("Bash"), ActionKinds: map[string]string{ "PreToolUse": "shell", }, @@ -246,6 +216,24 @@ func toolInputCommand(raw RawHookEvent) (string, bool) { return value, present && value != "" } +// shellCommandForTools returns a CommandFor that accepts tool_input.command +// only when the event's tool_name is one of names. Hosts scope their hook to +// the shell tool in their own config; this guard holds the same line inside +// the decoder, so a widened or drifted host matcher cannot promote another +// tool's input to a supervised shell action. +func shellCommandForTools(names ...string) func(RawBoundaryEvent) (string, bool) { + allowed := make(map[string]struct{}, len(names)) + for _, name := range names { + allowed[name] = struct{}{} + } + return func(raw RawBoundaryEvent) (string, bool) { + if _, ok := allowed[raw.ToolName]; !ok { + return "", false + } + return toolInputCommand(raw) + } +} + func preToolUseHost() HostConfig { return HostConfig{ MainEventName: "PreToolUse", @@ -463,15 +451,24 @@ func (h Host) HasBoundaryEvent(name string) bool { // Deprecated: use HasBoundaryEvent. func (h Host) HasHookEvent(name string) bool { return h.HasBoundaryEvent(name) } -// ActionKind returns the normalized action kind for a boundary event. +// ActionKind returns the normalized action kind for a boundary event. An +// empty boundaryEventName resolves through the host's main boundary event +// (hosts whose payloads omit the discriminator mean their primary boundary). +// An unknown non-empty event returns "" so callers fault instead of silently +// acquiring the shell kind. func (h Host) ActionKind(boundaryEventName string) string { registryMu.RLock() defer registryMu.RUnlock() - if config, exists := registry[h]; exists { - if kind, ok := config.Parser.ActionKinds[boundaryEventName]; ok { - return kind - } + config, exists := registry[h] + if !exists { + return "" + } + if boundaryEventName == "" { + boundaryEventName = config.MainEventName + } + if kind, ok := config.Parser.ActionKinds[boundaryEventName]; ok { + return kind } - return "shell" // fallback default + return "" } diff --git a/labs/15-pitot/pitot/adapters/guard_test.go b/labs/15-pitot/pitot/adapters/guard_test.go new file mode 100644 index 000000000..902e3dd2c --- /dev/null +++ b/labs/15-pitot/pitot/adapters/guard_test.go @@ -0,0 +1,48 @@ +package adapters + +import "testing" + +// The decoder holds the same tool-scope line the host matcher promises, so a +// widened or drifted host config cannot promote another tool's input to a +// supervised shell action. +func TestCommandForRejectsNonShellTools(t *testing.T) { + cases := map[Host]struct { + accepted []string + }{ + Claude: {accepted: []string{"Bash"}}, + Codex: {accepted: []string{"Bash"}}, + Kimi: {accepted: []string{"Bash"}}, + Opencode: {accepted: []string{"Bash"}}, + Gemini: {accepted: []string{"run_shell_command"}}, + Pi: {accepted: []string{"bash"}}, + Qwen: {accepted: []string{"Bash", "run_shell_command"}}, + } + for host, tc := range cases { + for _, tool := range tc.accepted { + raw := RawBoundaryEvent{ToolName: tool, ToolInput: map[string]any{"command": "git status"}} + if command, ok := host.CommandFor(raw); !ok || command != "git status" { + t.Errorf("%s: expected tool %q accepted, got ok=%v", host, tool, ok) + } + } + raw := RawBoundaryEvent{ToolName: "Write", ToolInput: map[string]any{"command": "rm -rf /"}} + if _, ok := host.CommandFor(raw); ok { + t.Errorf("%s: non-shell tool %q must not yield a supervised shell command", host, "Write") + } + } +} + +func TestActionKindNeverSilentlyDefaults(t *testing.T) { + if kind := Claude.ActionKind("SomeFutureEvent"); kind != "" { + t.Fatalf("unknown boundary event must have no kind, got %q", kind) + } + // An omitted discriminator means the host's main boundary event. + if kind := Claude.ActionKind(""); kind != "shell" { + t.Fatalf("empty event name should resolve via the main boundary event, got %q", kind) + } + if kind := Cursor.ActionKind("beforeMCPExecution"); kind != "mcp" { + t.Fatalf("registered kinds must be preserved, got %q", kind) + } + if kind := Host("unregistered").ActionKind("PreToolUse"); kind != "" { + t.Fatalf("unregistered host must have no kind, got %q", kind) + } +} diff --git a/labs/15-pitot/pitot/bridge/bridge.go b/labs/15-pitot/pitot/bridge/bridge.go index 1c2b7e6ed..2634139d1 100644 --- a/labs/15-pitot/pitot/bridge/bridge.go +++ b/labs/15-pitot/pitot/bridge/bridge.go @@ -47,10 +47,18 @@ func (r Registration) validate() error { return nil } +// resolvedWindow bounds the duplicate-detection memory: the Router remembers +// the most recent resolvedWindow resolved action IDs. Action IDs are 16 +// crypto-random bytes minted per action, so a duplicate arriving after 4096 +// newer actions is not a realistic correlation hazard, and the bound keeps +// long-lived runtimes at constant memory. +const resolvedWindow = 4096 + // Router holds at most one Controller registration per request kind. type Router struct { registrations map[string]Registration resolved map[string]struct{} + resolvedOrder []string mu sync.Mutex } @@ -59,6 +67,18 @@ func NewRouter() *Router { return &Router{registrations: map[string]Registration{}, resolved: map[string]struct{}{}} } +// markResolved records actionID in the bounded duplicate-detection window. +// Callers must hold r.mu. +func (r *Router) markResolved(actionID string) { + r.resolved[actionID] = struct{}{} + r.resolvedOrder = append(r.resolvedOrder, actionID) + if len(r.resolvedOrder) > resolvedWindow { + evict := r.resolvedOrder[0] + r.resolvedOrder = r.resolvedOrder[1:] + delete(r.resolved, evict) + } +} + // Register records reg, enforcing the exactly-one-Controller-per-kind rule. func (r *Router) Register(reg Registration) error { r.mu.Lock() @@ -119,7 +139,7 @@ func (r *Router) Resolve(req schema.ControlRequested, candidate *schema.ControlR if !ok { return schema.ControlResponse{}, ErrNoController } - r.resolved[req.ActionID] = struct{}{} + r.markResolved(req.ActionID) if candidate == nil { return r.defaultResponse(reg, req, reg.OnUnavailable), nil } @@ -153,7 +173,7 @@ func (r *Router) TimeoutResponse(req schema.ControlRequested) (schema.ControlRes if !ok { return schema.ControlResponse{}, ErrNoController } - r.resolved[req.ActionID] = struct{}{} + r.markResolved(req.ActionID) return r.defaultResponse(reg, req, reg.OnTimeout), nil } diff --git a/labs/15-pitot/pitot/bridge/window_test.go b/labs/15-pitot/pitot/bridge/window_test.go new file mode 100644 index 000000000..0626a18ef --- /dev/null +++ b/labs/15-pitot/pitot/bridge/window_test.go @@ -0,0 +1,42 @@ +package bridge + +import ( + "fmt" + "testing" + + "github.com/operatorstack/pitot/schema" +) + +// The duplicate-detection window is bounded: long-lived runtimes stay at +// constant memory, and the window semantics (most recent resolvedWindow +// actions) are explicit. +func TestResolvedWindowIsBounded(t *testing.T) { + router := NewRouter() + if err := router.Register(Registration{Kind: "shell", ControllerID: "c1", DeadlineMS: 1000, OnTimeout: schema.OutcomeDeny, OnUnavailable: schema.OutcomeDeny}); err != nil { + t.Fatal(err) + } + request := func(i int) schema.ControlRequested { + return schema.ControlRequested{ + PitotVersion: schema.Version, + Type: schema.TypeControlRequested, + Kind: "shell", + ActionID: fmt.Sprintf("act_%032d", i), + } + } + for i := 0; i < resolvedWindow+10; i++ { + if _, err := router.Resolve(request(i), nil); err != nil { + t.Fatalf("resolve %d: %v", i, err) + } + } + if len(router.resolved) != resolvedWindow { + t.Fatalf("resolved map should be capped at %d, got %d", resolvedWindow, len(router.resolved)) + } + // A duplicate inside the window is still rejected. + if _, err := router.Resolve(request(resolvedWindow+9), nil); err != ErrDuplicate { + t.Fatalf("recent duplicate must be rejected, got %v", err) + } + // The oldest entries were evicted (documented window boundary). + if _, dup := router.resolved[request(0).ActionID]; dup { + t.Fatal("oldest action should have been evicted from the window") + } +} diff --git a/labs/15-pitot/pitot/cmd/pitot/doctor_host.go b/labs/15-pitot/pitot/cmd/pitot/doctor_host.go index aa7cdfb03..3c39706dd 100644 --- a/labs/15-pitot/pitot/cmd/pitot/doctor_host.go +++ b/labs/15-pitot/pitot/cmd/pitot/doctor_host.go @@ -2,6 +2,8 @@ package main import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "io" @@ -69,12 +71,15 @@ func doctorHost(host adapters.Host, stdout, stderr io.Writer) error { if found, err := config.FindRoot("."); err == nil { root = found } - return printWiringStatus(root, string(host), stdout) + err := printWiringStatus(root, string(host), stdout) + doctorCommonNotes(host, stdout) + return err } probe, known := hostProbes[host] if !known { fmt.Fprintf(stdout, " host-config inspection is not implemented for %q in this release; run `pitot doctor` for the decoder status, or `pitot init --host %s` for the wiring snippet\n", host, host) + doctorCommonNotes(host, stdout) return nil } @@ -139,6 +144,7 @@ func doctorHost(host adapters.Host, stdout, stderr io.Writer) error { // Kimi's PreToolUse hook is fail-open on crash/timeout per host semantics. fmt.Fprintf(stdout, " note: %s hooks are fail-open on hook crash or timeout per host semantics; this sample controller is not a security sandbox\n", host) + doctorCommonNotes(host, stdout) if len(problems) > 0 { return fmt.Errorf("pitot doctor: %s host check found %d issue(s): %s", host, len(problems), strings.Join(problems, "; ")) @@ -147,6 +153,46 @@ func doctorHost(host adapters.Host, stdout, stderr io.Writer) error { return nil } +// doctorCommonNotes surfaces the two silent degradations every wired host is +// exposed to: hooks without a selected runtime observe instead of supervise, +// and user-level wrappers resolve `pitot` from PATH, which can drift from the +// binary the operator reviewed. Notes never fail the check; they name the +// state so the operator can decide. +func doctorCommonNotes(host adapters.Host, stdout io.Writer) { + if os.Getenv("PITOT_RUNTIME") == "" { + fmt.Fprintln(stdout, " note: PITOT_RUNTIME is not set in this shell; hooks run observe-only until a runtime is selected (`pitot run` / `pitot dev`), and `pitot run --strict` or `require_controller: true` faults instead of allowing") + } + if wiring.Supported(string(host)) { + return // repo-wired hosts pin .pitot/bin/pitot; PATH is not consulted + } + self, selfErr := os.Executable() + onPath, pathErr := doctorLookPath("pitot") + if selfErr != nil || pathErr != nil { + return + } + selfSum, err1 := fileSHA256(self) + pathSum, err2 := fileSHA256(onPath) + if err1 != nil || err2 != nil { + return + } + if selfSum != pathSum { + fmt.Fprintf(stdout, " warning: `pitot` on PATH (%s) is a different binary than this one (%s); user-level %s wrappers resolve ${PITOT_BIN:-pitot} from PATH, so the drifted binary would do the supervising\n", onPath, self, host) + } +} + +func fileSHA256(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + digest := sha256.New() + if _, err := io.Copy(digest, file); err != nil { + return "", err + } + return hex.EncodeToString(digest.Sum(nil)), nil +} + func doctorDevin(stdout, stderr io.Writer) error { const supportedVersion = "3000.3.22" path, err := doctorLookPath("devin") diff --git a/labs/15-pitot/pitot/cmd/pitot/hook_notice_test.go b/labs/15-pitot/pitot/cmd/pitot/hook_notice_test.go new file mode 100644 index 000000000..a89f0625c --- /dev/null +++ b/labs/15-pitot/pitot/cmd/pitot/hook_notice_test.go @@ -0,0 +1,25 @@ +package main + +import ( + "bytes" + "context" + "strings" + "testing" +) + +// Observation-only stays exit-0 but is never silent (Locus root-cause: +// silent-mode-degradation). +func TestHookWithoutRuntimeAnnouncesObserveOnly(t *testing.T) { + t.Setenv("PITOT_RUNTIME", "") + stdin := strings.NewReader(`{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"git status"}}`) + stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{} + if err := runWithIO(context.Background(), []string{"hook", "claude"}, stdin, stdout, stderr); err != nil { + t.Fatalf("observe-only hook must exit clean: %v", err) + } + if !strings.Contains(stdout.String(), `"action.requested"`) { + t.Fatalf("normalized event missing: %q", stdout.String()) + } + if !strings.Contains(stderr.String(), "observe-only") { + t.Fatalf("observe-only notice missing: %q", stderr.String()) + } +} diff --git a/labs/15-pitot/pitot/cmd/pitot/main.go b/labs/15-pitot/pitot/cmd/pitot/main.go index 167ce733f..e407ceffd 100644 --- a/labs/15-pitot/pitot/cmd/pitot/main.go +++ b/labs/15-pitot/pitot/cmd/pitot/main.go @@ -116,6 +116,9 @@ func runHook(ctx context.Context, args []string, stdin io.Reader, stdout, stderr return fmt.Errorf("pitot: emit normalized event: %w", err) } if runtimePath == "" { + // Observation-only is a feature; silence about it is not (Locus + // root-cause: silent-mode-degradation). Exit 0 is preserved. + fmt.Fprintln(stderr, "pitot: observe-only (no --runtime/PITOT_RUNTIME); this action is recorded, not supervised") return nil } client, err := runtime.OpenClient(runtimePath) @@ -128,7 +131,17 @@ func runHook(ctx context.Context, args []string, stdin io.Reader, stdout, stderr fmt.Fprintln(stderr, err) return errBlocked } - if response == nil || response.Outcome == schema.OutcomeAllow { + if response == nil { + return nil + } + if response.Outcome == schema.OutcomeAllow { + if response.ActionID != actionID { + // An allow that does not correlate to this action is not an + // allow for this action (same nonce binding the ACP client + // enforces before selecting allow_once). + fmt.Fprintln(stderr, "pitot: invalid controller resolution") + return errBlocked + } return nil } if response.Outcome != schema.OutcomeDeny || response.ActionID != actionID { @@ -260,6 +273,7 @@ func doctor(args []string, stdout, stderr io.Writer) error { func runRuntime(ctx context.Context, args []string, stdout, stderr io.Writer) error { configPath := "" runtimePath := "" + strict := false for i := 0; i < len(args); i++ { switch args[i] { case "--config": @@ -274,6 +288,8 @@ func runRuntime(ctx context.Context, args []string, stdout, stderr io.Writer) er } runtimePath = args[i+1] i++ + case "--strict": + strict = true default: return fmt.Errorf("pitot: unexpected argument %q", args[i]) } @@ -301,6 +317,9 @@ func runRuntime(ctx context.Context, args []string, stdout, stderr io.Writer) er if err != nil { return err } + if strict { + loaded.Config.RequireController = true + } manager, err := runtime.Start(ctx, loaded.Config, stderr) if err != nil { return err @@ -339,7 +358,7 @@ usage: pitot dev --host HOST -- AGENT [ARGS...] pitot acp devin --runtime PATH --prompt TEXT [--exec PATH] [--model MODEL] [--agent-type TYPE] [--cwd PATH] pitot doctor [--host HOST] [--fix] - pitot run [--config PATH] --runtime PATH + pitot run [--config PATH] [--strict] --runtime PATH pitot hook HOST [--runtime PATH] pitot request KIND [--data JSON] --runtime PATH pitot version diff --git a/labs/15-pitot/pitot/config/config.go b/labs/15-pitot/pitot/config/config.go index f7faf4bb1..b2a8d2752 100644 --- a/labs/15-pitot/pitot/config/config.go +++ b/labs/15-pitot/pitot/config/config.go @@ -37,6 +37,12 @@ type Config struct { RequiresProtocol string `yaml:"requires_protocol,omitempty"` Consumers []ConsumerConfig `yaml:"consumers,omitempty"` Controllers map[string]ControllerConfig `yaml:"controllers,omitempty"` + // RequireController converts silent observation-only degradation into a + // fault: an action kind delivered without a registered Controller is + // blocked instead of allowed. Additive and off by default — plain + // observation-only operation is unchanged. Any fragment declaring it + // makes the merged config strict. + RequireController bool `yaml:"require_controller,omitempty"` } // ConsumerConfig declares a passive JSON-Lines event sink. @@ -210,6 +216,9 @@ func mergeSources(sources []source) (Loaded, error) { if len(fragment.Consumers) == 0 && len(fragment.Controllers) == 0 { return Loaded{}, fmt.Errorf("pitot: fragment %q declares no consumers or controllers", rel) } + if fragment.RequireController { + merged.RequireController = true + } if fragment.RequiresProtocol != "" && fragment.RequiresProtocol != schema.Version { return Loaded{}, fmt.Errorf("pitot: fragment %q requires protocol %q but this pitot speaks protocol %q", rel, fragment.RequiresProtocol, schema.Version) } diff --git a/labs/15-pitot/pitot/internal/devinacp/client.go b/labs/15-pitot/pitot/internal/devinacp/client.go index 650a33503..d2e0939fb 100644 --- a/labs/15-pitot/pitot/internal/devinacp/client.go +++ b/labs/15-pitot/pitot/internal/devinacp/client.go @@ -58,10 +58,14 @@ type client struct { stdin io.WriteCloser scanner *bufio.Scanner stdout io.Writer + stderr io.Writer deliver DeliverFunc commands map[string]string nextID int writeMu sync.Mutex + // metaAbsenceReported dedupes the once-per-session diagnostic for + // tool_call updates missing the vendor _meta tool name. + metaAbsenceReported bool } // Run launches Devin as an ACP subprocess and completes one prompt turn. @@ -126,6 +130,7 @@ func Run(ctx context.Context, options Options) error { stdin: stdin, scanner: scanner, stdout: options.Stdout, + stderr: options.Stderr, deliver: deliver, commands: map[string]string{}, } @@ -278,6 +283,13 @@ func (c *client) handleUpdate(raw json.RawMessage) error { tool, _ := params.Update.Meta["cognition.ai/inferenceToolName"].(string) if params.SessionID != "" && params.Update.ToolCallID != "" && tool == "exec" && command != "" { c.commands[commandKey(params.SessionID, params.Update.ToolCallID)] = command + } else if tool == "" && command != "" && !c.metaAbsenceReported { + // Fail-closed fallbacks must be observable: without the vendor + // _meta tool name no command is ever cached, so every permission + // request will be rejected. Name protocol drift once so it is + // not mistaken for Controller policy. + c.metaAbsenceReported = true + fmt.Fprintln(c.stderr, "pitot acp: tool_call update lacks _meta cognition.ai/inferenceToolName; commands cannot be attributed and permissions will be rejected (fail-closed)") } case "agent_message_chunk": var content struct { diff --git a/labs/15-pitot/pitot/internal/devinacp/meta_test.go b/labs/15-pitot/pitot/internal/devinacp/meta_test.go new file mode 100644 index 000000000..0061b3e29 --- /dev/null +++ b/labs/15-pitot/pitot/internal/devinacp/meta_test.go @@ -0,0 +1,44 @@ +package devinacp + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +// A tool_call update missing the vendor _meta tool name leaves the command +// cache empty (fail-closed rejections follow). That degradation must be +// named once per session, not mistaken for Controller policy. +func TestMetaAbsenceIsDiagnosedOnce(t *testing.T) { + stderr := &bytes.Buffer{} + transport := &client{stderr: stderr, stdout: &bytes.Buffer{}, commands: map[string]string{}} + update := json.RawMessage(`{"sessionId":"s1","update":{"sessionUpdate":"tool_call","toolCallId":"t1","rawInput":{"command":"git status"}}}`) + for i := 0; i < 2; i++ { + if err := transport.handleUpdate(update); err != nil { + t.Fatalf("update %d: %v", i, err) + } + } + if len(transport.commands) != 0 { + t.Fatalf("no command may be cached without the vendor tool name, got %v", transport.commands) + } + if got := strings.Count(stderr.String(), "cognition.ai/inferenceToolName"); got != 1 { + t.Fatalf("expected exactly one diagnostic, got %d in %q", got, stderr.String()) + } +} + +// The normal path still caches and emits no diagnostic. +func TestMetaPresenceCachesWithoutDiagnostic(t *testing.T) { + stderr := &bytes.Buffer{} + transport := &client{stderr: stderr, stdout: &bytes.Buffer{}, commands: map[string]string{}} + update := json.RawMessage(`{"sessionId":"s1","update":{"sessionUpdate":"tool_call","toolCallId":"t1","rawInput":{"command":"git status"},"_meta":{"cognition.ai/inferenceToolName":"exec"}}}`) + if err := transport.handleUpdate(update); err != nil { + t.Fatal(err) + } + if transport.commands[commandKey("s1", "t1")] != "git status" { + t.Fatalf("command not cached: %v", transport.commands) + } + if stderr.Len() != 0 { + t.Fatalf("unexpected diagnostic: %q", stderr.String()) + } +} diff --git a/labs/15-pitot/pitot/runtime/runtime.go b/labs/15-pitot/pitot/runtime/runtime.go index a466dc709..440eaac3a 100644 --- a/labs/15-pitot/pitot/runtime/runtime.go +++ b/labs/15-pitot/pitot/runtime/runtime.go @@ -34,14 +34,21 @@ type Decision struct { // Manager starts configured role processes and exposes their shared delivery path. type Manager struct { - ctx context.Context - cancel context.CancelFunc - stderr io.Writer - controllers map[string]*controllerWorker - consumers []*consumerWorker - observer func(Decision) + ctx context.Context + cancel context.CancelFunc + stderr io.Writer + controllers map[string]*controllerWorker + consumers []*consumerWorker + observer func(Decision) + requireController bool + observedKindsMu sync.Mutex + observedKinds map[string]struct{} } +// ErrControllerRequired reports a kind delivered without a registered +// Controller while require_controller (or --strict) is active. +var ErrControllerRequired = errors.New("pitot: no controller registered for this action kind and require_controller is active") + // SetDecisionObserver registers a callback invoked for every resolved controller // decision. It is optional; a nil observer disables receipts. Not safe to change // concurrently with active delivery. @@ -66,10 +73,12 @@ func (m *Manager) reportDecision(kind string, response *schema.ControlResponse) func Start(parent context.Context, cfg config.Config, stderr io.Writer) (*Manager, error) { ctx, cancel := context.WithCancel(parent) manager := &Manager{ - ctx: ctx, - cancel: cancel, - stderr: stderr, - controllers: map[string]*controllerWorker{}, + ctx: ctx, + cancel: cancel, + stderr: stderr, + controllers: map[string]*controllerWorker{}, + requireController: cfg.RequireController, + observedKinds: map[string]struct{}{}, } for _, kind := range sortedKinds(cfg.Controllers) { declared := cfg.Controllers[kind] @@ -126,6 +135,17 @@ func (m *Manager) DeliverEvent(ctx context.Context, event schema.Event) (*schema } worker, exists := m.controllers[event.Action.Kind] if !exists { + if m.requireController { + return nil, fmt.Errorf("%w (kind %q)", ErrControllerRequired, event.Action.Kind) + } + // Observation-only is a feature; silence about it is not. Name the + // degradation once per kind on the operator's channel. + m.observedKindsMu.Lock() + if _, seen := m.observedKinds[event.Action.Kind]; !seen { + m.observedKinds[event.Action.Kind] = struct{}{} + fmt.Fprintf(m.stderr, "pitot: kind %q has no controller; observation-only (set require_controller: true or run with --strict to fault instead)\n", event.Action.Kind) + } + m.observedKindsMu.Unlock() return nil, nil } data, err := json.Marshal(event) diff --git a/labs/15-pitot/pitot/runtime/strict_test.go b/labs/15-pitot/pitot/runtime/strict_test.go new file mode 100644 index 000000000..ceff87add --- /dev/null +++ b/labs/15-pitot/pitot/runtime/strict_test.go @@ -0,0 +1,53 @@ +package runtime + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + + "github.com/operatorstack/pitot/config" +) + +// require_controller converts the silent observation-only allow into a fault +// (Locus strict-mode candidate strict-config-v1 / strict-flag-v1). +func TestRequireControllerFaultsInsteadOfAllowing(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + manager, err := Start(ctx, config.Config{RequireController: true}, &bytes.Buffer{}) + if err != nil { + t.Fatal(err) + } + defer manager.Close() + response, err := manager.DeliverEvent(ctx, actionEvent(t, "act_strict_1", "true")) + if response != nil { + t.Fatalf("strict mode must not resolve, got %+v", response) + } + if !errors.Is(err, ErrControllerRequired) { + t.Fatalf("expected ErrControllerRequired, got %v", err) + } +} + +// Without strict mode, observation-only stays allowed but is announced once +// per kind on the operator's channel (never silently). +func TestObservationOnlyIsAnnouncedOncePerKind(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + stderr := &bytes.Buffer{} + manager, err := Start(ctx, config.Config{}, stderr) + if err != nil { + t.Fatal(err) + } + defer manager.Close() + for i, id := range []string{"act_obs_1", "act_obs_2"} { + response, deliverErr := manager.DeliverEvent(ctx, actionEvent(t, id, "true")) + if response != nil || deliverErr != nil { + t.Fatalf("delivery %d: expected observation-only nil/nil, got %+v %v", i, response, deliverErr) + } + } + notice := "observation-only" + if got := strings.Count(stderr.String(), notice); got != 1 { + t.Fatalf("expected exactly one %q notice, got %d in %q", notice, got, stderr.String()) + } +} diff --git a/labs/15-pitot/pitot/runtime/transport.go b/labs/15-pitot/pitot/runtime/transport.go index 97e436319..ae793a654 100644 --- a/labs/15-pitot/pitot/runtime/transport.go +++ b/labs/15-pitot/pitot/runtime/transport.go @@ -142,6 +142,12 @@ func (s *Server) handleEvent(w http.ResponseWriter, request *http.Request) { response, err := s.manager.DeliverEvent(request.Context(), event) if err != nil { fmt.Fprintf(s.stderr, "pitot: action %s resolved with boundary fault: %v\n", event.Action.ID, err) + if response == nil { + // An errored delivery with no resolution must never read as an + // observation-only allow (204): fail closed so the hook blocks. + writeJSON(w, http.StatusConflict, map[string]string{"error": "event delivery faulted"}) + return + } } if response == nil { w.WriteHeader(http.StatusNoContent) diff --git a/labs/15-pitot/pitot/runtime/transport_fault_test.go b/labs/15-pitot/pitot/runtime/transport_fault_test.go new file mode 100644 index 000000000..7f605c1af --- /dev/null +++ b/labs/15-pitot/pitot/runtime/transport_fault_test.go @@ -0,0 +1,46 @@ +package runtime + +import ( + "context" + "io" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/operatorstack/pitot/config" +) + +// An errored delivery with no resolution must never read as an +// observation-only allow: the transport fails closed and the hook blocks. +func TestStrictDeliveryFaultIsNotAnAllow(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + manager, err := Start(ctx, config.Config{RequireController: true}, io.Discard) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "runtime.json") + done := make(chan error, 1) + go func() { done <- NewServer(manager, strings.Repeat("a", 64), path, io.Discard, io.Discard).Serve(ctx) }() + t.Cleanup(func() { + cancel() + manager.Close() + <-done + }) + var client *Client + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); { + if opened, openErr := OpenClient(path); openErr == nil { + client = opened + break + } + time.Sleep(20 * time.Millisecond) + } + if client == nil { + t.Fatal("runtime descriptor was not published") + } + response, err := client.DeliverEvent(ctx, actionEvent(t, "act_strict_http", "true")) + if err == nil { + t.Fatalf("strict fault must surface as an error, got response %+v", response) + } +} diff --git a/labs/15-pitot/pitot/sensor/guard_test.go b/labs/15-pitot/pitot/sensor/guard_test.go new file mode 100644 index 000000000..fd5f77573 --- /dev/null +++ b/labs/15-pitot/pitot/sensor/guard_test.go @@ -0,0 +1,47 @@ +package sensor + +import ( + "errors" + "strings" + "testing" + + "github.com/operatorstack/pitot/adapters" + "github.com/operatorstack/pitot/schema" +) + +// A non-shell tool carrying a command string must fault at the decoder even +// if the host matcher forwarded it (defense in depth for the five hosts that +// previously accepted any tool_input.command). +func TestDecodeRejectsNonShellToolInput(t *testing.T) { + payloads := map[adapters.Host]string{ + adapters.Claude: `{"hook_event_name":"PreToolUse","tool_name":"Write","tool_input":{"command":"rm -rf /"}}`, + adapters.Codex: `{"hook_event_name":"PreToolUse","tool_name":"Edit","tool_input":{"command":"curl evil"}}`, + adapters.Gemini: `{"hook_event_name":"BeforeTool","tool_name":"write_file","tool_input":{"command":"true"}}`, + adapters.Opencode: `{"hook_event_name":"PreToolUse","tool_name":"webfetch","tool_input":{"command":"true"}}`, + adapters.Kimi: `{"hook_event_name":"PreToolUse","tool_name":"Write","tool_input":{"command":"true"}}`, + adapters.Pi: `{"hook_event_name":"tool_call","tool_name":"editor","tool_input":{"command":"true"}}`, + } + for host, payload := range payloads { + _, err := Decode(host, []byte(payload), "full") + var fault *FaultError + if !errors.As(err, &fault) || fault.Reason != schema.ReasonEmptyCommand { + t.Errorf("%s: non-shell tool must fault with empty-command, got %v", host, err) + } + } +} + +// Payloads that omit the event discriminator keep decoding as the host's main +// boundary event — the tolerated-empty-name behavior is a feature and must +// not regress into the removed silent shell fallback. +func TestDecodeEmptyEventNameResolvesMainBoundary(t *testing.T) { + event, err := Decode(adapters.Claude, []byte(`{"tool_name":"Bash","tool_input":{"command":"git status"}}`), "full") + if err != nil { + t.Fatalf("empty event name should decode via the main boundary event: %v", err) + } + if event.Action.Kind != "shell" { + t.Fatalf("expected shell kind, got %q", event.Action.Kind) + } + if !strings.Contains(string(event.Content.Full), "git status") { + t.Fatalf("expected command content, got %s", event.Content.Full) + } +} diff --git a/labs/15-pitot/pitot/sensor/sensor.go b/labs/15-pitot/pitot/sensor/sensor.go index 0de152ee7..2c428cf43 100644 --- a/labs/15-pitot/pitot/sensor/sensor.go +++ b/labs/15-pitot/pitot/sensor/sensor.go @@ -67,6 +67,13 @@ func Decode(host adapters.Host, raw []byte, mode projection.Mode) (schema.Event, return schema.Event{}, &FaultError{Host: host, Reason: schema.ReasonEmptyCommand} } + kind := host.ActionKind(eventName) + if kind == "" { + // No registered kind for this boundary event: a malformed boundary, + // never a silently-defaulted shell action. + return schema.Event{}, &FaultError{Host: host, Reason: schema.ReasonMalformed} + } + content, err := projection.Apply(mode, []byte(command)) if err != nil { return schema.Event{}, err @@ -83,7 +90,7 @@ func Decode(host adapters.Host, raw []byte, mode projection.Mode) (schema.Event, Name: string(host), AdapterVersion: adapters.AdapterVersion, }, - Action: &schema.Action{Kind: host.ActionKind(eventName)}, + Action: &schema.Action{Kind: kind}, Content: &content, Observation: schema.Observation{ Source: source, diff --git a/labs/15-pitot/public-readme-preview/CONTRIBUTING.md b/labs/15-pitot/public-readme-preview/CONTRIBUTING.md index 45c38f7f2..fea906d14 100644 --- a/labs/15-pitot/public-readme-preview/CONTRIBUTING.md +++ b/labs/15-pitot/public-readme-preview/CONTRIBUTING.md @@ -12,6 +12,32 @@ Before proposing an adapter or protocol change: 5. prove passive Consumers cannot reach the response channel; and 6. document any change to request correlation or timeout behavior. +## Adding a host adapter + +A new host is admissible only if its blocking boundary can complete Pitot's +causal loop: a proposed command reaches a Controller, allow/deny applies before +execution, and a denied action returns to the model so the agent continues. If +the host can only abort on denial, it is not yet supervisable — say so in the +research note and stop there (see `docs/devin-adapter-research.md` for a worked +rejection-then-acceptance across two of the same host's surfaces). + +Adapters come in two transport classes: + +- **One-shot hook** (Claude, Codex, Cursor, Copilot, Gemini, Kimi, OpenCode, + Pi, Qwen): the host invokes `pitot hook HOST` at a synchronous pre-execution + boundary, Pitot answers by exit code or a native block envelope, and the + process ends. Add a decoder that normalizes the host payload to a `shell` + action and an encoder for the host's native deny shape. +- **Stateful transport** (Devin, over ACP): Pitot launches and drives a + long-lived host process, correlates each command to a permission request, and + maps allow/deny onto the host's typed one-shot permission options. Never + select persistent or bypass options. + +Either way: add positive and negative conformance fixtures, prove passive +Consumers cannot reach the response channel, and register the adapter in the +supervised inventory so it joins the cross-platform E2E matrix (each adapter is +verified allow-and-deny on Ubuntu, macOS, and Windows before it ships). + ## Development The reference implementation is a Go module (`go 1.26`). From the repository diff --git a/labs/15-pitot/public-readme-preview/README.md b/labs/15-pitot/public-readme-preview/README.md index 343313473..46f727ec3 100644 --- a/labs/15-pitot/public-readme-preview/README.md +++ b/labs/15-pitot/public-readme-preview/README.md @@ -304,9 +304,10 @@ Agent finished. Runtime stopped. ``` `--host` must name a supported agent (`claude`, `codex`, `copilot`, `cursor`, -`gemini`, `kimi`, `opencode`, `pi`, `qwen`), and that agent's host hook must +`devin`, `gemini`, `kimi`, `opencode`, `pi`, `qwen`). Hook-based hosts must already be wired to `pitot hook HOST` (see **Connect your agent** and -`pitot doctor --host HOST`). The runtime descriptor lives in a per-invocation +`pitot doctor --host HOST`); Devin connects over ACP and needs no hook file. +The runtime descriptor lives in a per-invocation temporary path and is removed on exit, so concurrent `pitot dev` sessions never collide. @@ -398,25 +399,47 @@ pitot run --runtime $env:PITOT_RUNTIME Every host below normalizes its native blocking boundary to a `shell` action and passes Pitot's language-neutral decoder conformance suite. The E2E column marks -adapters exercised by the cross-platform agent supervisor (the badge at the top) -on Ubuntu, macOS, and Windows. Kimi additionally has an in-repo, no-model test -that asserts the full allow **and** deny control path end to end. - -| Host | Blocking boundary | Hook wiring | Verified in this repo | -|---|---|---|---| -| Kimi Code | `PreToolUse` / Bash | native `config.toml` | decoder + E2E + allow/deny control test | -| Claude | `PreToolUse` | native settings hook | decoder + E2E | -| Cursor | `beforeShellExecution` | bridge (`integrations/cursor`) | decoder + E2E | -| Codex | `PreToolUse` | bridge (`integrations/codex`) | decoder + E2E | -| GitHub Copilot CLI | `PreToolUse` | bridge (`integrations/copilot`) | decoder + E2E | -| Gemini | `BeforeTool` | bridge (`integrations/gemini`) | decoder + E2E | -| OpenCode | `PreToolUse` | bridge (`integrations/opencode`) | decoder + E2E | -| Pi | `tool_call` | extension (`integrations/pi`) | decoder + E2E | -| Qwen Code | `PreToolUse` | bridge (`integrations/qwen`) | decoder + E2E | +adapters exercised by the cross-platform agent supervisor (the badge at the top); +the Platforms column names the operating systems each one is verified on. Kimi +additionally has an in-repo, no-model test that asserts the full allow **and** +deny control path end to end. + +| Host | Blocking boundary | Hook wiring | Platforms | Verified in this repo | +|---|---|---|---|---| +| Kimi Code | `PreToolUse` / Bash | native `config.toml` | Ubuntu · macOS · Windows | decoder + E2E + allow/deny control test | +| Claude | `PreToolUse` | native settings hook | Ubuntu · macOS · Windows | decoder + E2E | +| Cursor | `beforeShellExecution` | bridge (`integrations/cursor`) | Ubuntu · macOS · Windows (WSL) | decoder + E2E | +| Codex | `PreToolUse` | bridge (`integrations/codex`) | Ubuntu · macOS · Windows | decoder + E2E | +| Devin | `session/request_permission` (ACP) | stateful ACP transport | Ubuntu · macOS · Windows | decoder + E2E | +| GitHub Copilot CLI | `PreToolUse` | bridge (`integrations/copilot`) | Ubuntu · macOS · Windows | decoder + E2E | +| Gemini | `BeforeTool` | bridge (`integrations/gemini`) | Ubuntu · macOS · Windows | decoder + E2E | +| OpenCode | `PreToolUse` | bridge (`integrations/opencode`) | Ubuntu · macOS · Windows | decoder + E2E | +| Pi | `tool_call` | extension (`integrations/pi`) | Ubuntu · macOS · Windows | decoder + E2E | +| Qwen Code | `PreToolUse` | bridge (`integrations/qwen`) | Ubuntu · macOS · Windows | decoder + E2E | "Decoder" means Pitot correctly normalizes that host's payload into the stable event envelope. It does not claim Pitot judges whether any command is safe — that -decision belongs to your Controller. +decision belongs to your Controller. On Windows, Cursor runs under WSL; every +other host runs natively. + +## Host admission criteria + +A host earns a supervised adapter only when its blocking boundary can complete +Pitot's causal loop: a proposed command must reach a Controller, an allow or +deny decision must apply **before** the command runs, and — critically — a +denied action must return to the model so the agent can continue from the +blocked outcome rather than halting. This last requirement, deny-continuation, +is what distinguishes a supervisable boundary from one that can only abort. + +Devin is the worked example. Its lifecycle hooks (`PreToolUse`, +`PermissionRequest`) apply a denial but end the turn instead of handing the +rejected outcome back to the model, so they do not complete the loop in +non-interactive mode. Its Agent Client Protocol surface does: the client +selects `reject_once`, the canary never executes, and Devin makes a follow-up +model request carrying the rejection and continues. Pitot therefore ships Devin +over ACP and does not ship the hook wiring. The full investigation, including +the content-safe evidence receipt, is in +[docs/devin-adapter-research.md](docs/devin-adapter-research.md). ## Connect your agent @@ -568,6 +591,31 @@ blocking `tool_call` event into Pitot's stable envelope and returns Pi's native `block` response when Pitot rejects the request. See the official [Pi extensions documentation](https://pi.dev/docs/latest/extensions). +### Devin + +Devin needs no hook file. Pitot speaks to it over the Agent Client Protocol, +launching `devin acp` as a stdio JSON-RPC server and correlating each +`tool_call` command by its tool-call ID: + +```bash +pitot dev --host devin -- devin -p "Run: echo hello" +``` + +To attach to an already-running runtime, use the explicit single-prompt +surface: + +```bash +pitot acp devin --runtime "$PITOT_RUNTIME" --prompt "Run: echo hello" +``` + +Pitot maps an allow decision to ACP's one-shot `allow_once` and a deny to +`reject_once`; it never selects a persistent option such as `allow_always` or a +bypass mode. Attestation comes from the Controller receipts and canary rather +than a lifecycle-hook witness. This initial adapter is single-prompt; resume and +multi-turn Devin sessions are not yet supported. See +[docs/devin-adapter-research.md](docs/devin-adapter-research.md) for why ACP, +not hooks, is the supervised boundary. + Pitot uses supervised local processes in v1. It starts declared Consumers and Controllers itself, applies each projection before bytes enter the child pipe, and exposes only a loopback endpoint authenticated by the owner-only runtime @@ -685,6 +733,45 @@ Pitot is local and storage-free by default. - custom agent interfaces over existing runtimes; and - new Pitot-compatible coding-agent runtimes. +## Threat model + +Pitot mediates the host's wired boundary and nothing else. Understanding where +that boundary ends is part of using it correctly. + +- **Only the wired boundary is mediated.** Pitot sees an action when the host + actually routes it through the configured hook or ACP transport. An action + the host takes through a path you did not wire is not observed. `pitot doctor + --host HOST` reports whether the boundary is present, and for repo-owned + hosts whether the entry has drifted. +- **User-level hook configs are user-editable.** For hosts wired at user level + (Kimi, Copilot, Qwen) the hook lives in a file the user owns and can change + or remove. Pitot does not police edits outside the repository; it reports the + current state through `pitot doctor`. +- **Without a runtime, hooks only observe.** A `pitot hook` invocation with no + `PITOT_RUNTIME` (or `--runtime`) selected records the action and exits + allowing it — observation-only, for backwards compatibility. It now prints a + one-line notice so this mode is never silent, and `pitot doctor` flags a host + that is wired but has no runtime. Once a runtime is explicitly selected, + transport or authentication failure blocks the controllable action rather + than falling open. +- **Hosts retain their own bypass options.** Some hosts expose persistent or + bypass permissions (for example Devin's ACP `allow_always` and switch-bypass + modes). Pitot never selects them — it uses only the one-shot `allow_once` / + `reject_once` options — but a human operating the host directly still can. +- **Kimi executes the action if its hook process crashes or times out.** This + is Kimi's host semantics, not a Pitot decision, and Pitot cannot override it: + a supervisory-control analysis of the Kimi lifecycle shows a reachable + transition from the pending state straight to an unsupervised execution when + the hook fails open, which no supervisor placed at the boundary can prevent. + Every other supervised host either fails closed or returns the denial to the + model. `pitot doctor --host kimi` states this plainly. If your policy cannot + tolerate fail-open execution, prefer a fail-closed host (such as Cursor, + wired with `failClosed: true`) or Devin's ACP transport. + +None of these change the core contract: within the boundary Pitot mediates, +every pending action receives exactly one terminal resolution. **Pitot reports. +Your controller decides.** + ## What Pitot does not decide Pitot does not define whether: @@ -743,10 +830,13 @@ at the host boundary; interpretation and control belong downstream. Start with the protocol and conformance fixtures. A new adapter should declare its host capabilities, normalize supported events, classify boundary faults without exposing content, encode Controller responses, and pass the shared -positive and negative fixture suite. +positive and negative fixture suite. It must also meet the host admission +criteria above — a boundary that can only abort on denial, rather than return +the outcome to the model, is not yet supervisable. -See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup and compatibility -requirements. +See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup, the two adapter +transport classes (one-shot hook and stateful ACP-style transport), and the +cross-platform verification every adapter must pass. ## License diff --git a/labs/15-pitot/tests/devin_control_proxy.py b/labs/15-pitot/tests/devin_control_proxy.py index 64633972e..cc134096f 100644 --- a/labs/15-pitot/tests/devin_control_proxy.py +++ b/labs/15-pitot/tests/devin_control_proxy.py @@ -87,11 +87,18 @@ def rlog(line: str) -> None: "auxiliary_requests": 0, } + receipt_lock = threading.Lock() + def save() -> None: - args.receipt.parent.mkdir(parents=True, exist_ok=True) - temporary = args.receipt.with_suffix(".tmp") - temporary.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") - temporary.replace(args.receipt) + # Handlers run on concurrent threads (one per connection). The receipt + # mutation and the tmp-file replace must be atomic as a unit: a shared + # tmp name raced by two handlers throws, killing the connection — + # which a bursting Devin client observes as "error sending request". + with receipt_lock: + args.receipt.parent.mkdir(parents=True, exist_ok=True) + temporary = args.receipt.with_suffix(f".{threading.get_ident()}.tmp") + temporary.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(args.receipt) class Handler(BaseHTTPRequestHandler): # Devin's HTTP client pools connections. Under the default HTTP/1.0 diff --git a/labs/15-pitot/tests/real_agent_driver.py b/labs/15-pitot/tests/real_agent_driver.py index 0a49e2994..b2a141107 100755 --- a/labs/15-pitot/tests/real_agent_driver.py +++ b/labs/15-pitot/tests/real_agent_driver.py @@ -360,6 +360,13 @@ def validate_receipts( if agent["integration"] == "acp_client": if witnesses: raise RuntimeError("ACP control unexpectedly entered the lifecycle-hook witness path") + # Observed baseline is ~10 auxiliary (non-chat) requests per session; + # an unexplained surge means the proxy is answering traffic the + # trajectory never accounted for. Test-side ceiling only: the proxy + # itself stays permissive so a chattier Devin build cannot black-hole. + auxiliary = int(proxy.get("auxiliary_requests", 0)) + if auxiliary > 100: + raise RuntimeError(f"devin proxy answered {auxiliary} auxiliary requests; ceiling is 100") else: if len(witnesses) != 2: raise RuntimeError("Pitot witness receipts do not identify exactly two hook actions") diff --git a/labs/15-pitot/tests/test_devin_control_proxy.py b/labs/15-pitot/tests/test_devin_control_proxy.py new file mode 100644 index 000000000..47177ab11 --- /dev/null +++ b/labs/15-pitot/tests/test_devin_control_proxy.py @@ -0,0 +1,200 @@ +"""Unit tests for the deterministic Devin Connect/protobuf proxy's HTTP +behavior — the properties the Windows CI cells depend on (keep-alive reuse, +chunked draining, burst-tolerant backlog, the request journal, and the +fail-closed 409 state machine) exercised without a released Devin binary.""" + +from __future__ import annotations + +import http.client +import json +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from pathlib import Path + +LAB = Path(__file__).resolve().parents[1] +PROXY = LAB / "tests" / "devin_control_proxy.py" +NONCE = "cafe" * 8 + + +class ProxyHarness: + def __init__(self, response_fault: str = "none", request_log: bool = True): + self.workdir = Path(tempfile.mkdtemp(prefix="devin-proxy-test-")) + self.receipt = self.workdir / "receipt.json" + self.ready = self.workdir / "ready.txt" + self.log = self.workdir / "requests.log" if request_log else None + command = [ + sys.executable, str(PROXY), + "--nonce", NONCE, + "--receipt", str(self.receipt), + "--ready-file", str(self.ready), + "--canary-command", "echo PITOT_CANARY", + "--response-fault", response_fault, + ] + if self.log is not None: + command += ["--request-log", str(self.log)] + self.process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + deadline = time.time() + 10 + while time.time() < deadline: + if self.ready.is_file() and self.ready.read_text(encoding="utf-8").strip(): + break + time.sleep(0.02) + else: + raise RuntimeError("proxy did not become ready") + url = self.ready.read_text(encoding="utf-8").strip() + self.host, self.port = url.split("//", 1)[1].split(":") + self.port = int(self.port) + + def receipt_json(self) -> dict: + return json.loads(self.receipt.read_text(encoding="utf-8")) + + def close(self) -> None: + self.process.terminate() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + + +class DevinProxyHTTPTests(unittest.TestCase): + def setUp(self): + self.proxy = ProxyHarness() + self.addCleanup(self.proxy.close) + + def test_keepalive_serves_sequential_requests_on_one_connection(self): + connection = http.client.HTTPConnection(self.proxy.host, self.proxy.port, timeout=5) + self.addCleanup(connection.close) + for i in range(3): + connection.request( + "POST", "/exa.seat_management_pb.SeatManagementService/GetCliTeamSettings", + body=b"x", headers={"Content-Type": "application/proto"}, + ) + response = connection.getresponse() + self.assertEqual(response.status, 200, f"request {i}") + self.assertEqual(response.read(), b"") + self.assertEqual(response.version, 11, "proxy must speak HTTP/1.1") + self.assertNotEqual( + (response.getheader("Connection") or "").lower(), "close", + "keep-alive must not be refused on the auxiliary path", + ) + self.assertEqual(self.proxy.receipt_json()["auxiliary_requests"], 3) + + def test_chunked_request_body_is_drained_exactly(self): + connection = http.client.HTTPConnection(self.proxy.host, self.proxy.port, timeout=5) + self.addCleanup(connection.close) + connection.putrequest("POST", "/aux.Service/Chunked") + connection.putheader("Transfer-Encoding", "chunked") + connection.endheaders() + for chunk in (b"hello ", b"chunked ", b"world"): + connection.send(b"%x\r\n%s\r\n" % (len(chunk), chunk)) + connection.send(b"0\r\n\r\n") + response = connection.getresponse() + self.assertEqual(response.status, 200) + response.read() + # The connection must stay usable: an undrained body would desync it. + connection.request("POST", "/aux.Service/After", body=b"y") + after = connection.getresponse() + self.assertEqual(after.status, 200) + after.read() + self.assertEqual(self.proxy.receipt_json()["auxiliary_requests"], 2) + + def test_connection_burst_within_backlog_all_served(self): + errors: list[str] = [] + def one(i: int) -> None: + try: + connection = http.client.HTTPConnection(self.proxy.host, self.proxy.port, timeout=10) + connection.request("POST", "/aux.Service/Burst", body=b"b") + response = connection.getresponse() + if response.status != 200: + errors.append(f"{i}: status {response.status}") + response.read() + connection.close() + except Exception as error: # noqa: BLE001 - the failure IS the signal + errors.append(f"{i}: {error}") + threads = [threading.Thread(target=one, args=(i,)) for i in range(64)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=15) + self.assertEqual(errors, [], "burst connections must all be served (backlog 128)") + self.assertEqual(self.proxy.receipt_json()["auxiliary_requests"], 64) + + def test_request_journal_records_connections_and_requests(self): + connection = http.client.HTTPConnection(self.proxy.host, self.proxy.port, timeout=5) + connection.request("POST", "/aux.Service/Journal", body=b"j") + connection.getresponse().read() + connection.close() + time.sleep(0.2) + journal = self.proxy.log.read_text(encoding="utf-8") + self.assertIn("CONN open", journal) + self.assertIn("REQ ", journal) + self.assertIn("/aux.Service/Journal", journal) + self.assertIn("RES ", journal) + + def test_get_is_404_with_explicit_length(self): + connection = http.client.HTTPConnection(self.proxy.host, self.proxy.port, timeout=5) + self.addCleanup(connection.close) + connection.request("GET", "/anything") + response = connection.getresponse() + self.assertEqual(response.status, 404) + self.assertEqual(response.getheader("content-length"), "0") + response.read() + # keep-alive survives the 404 + connection.request("POST", "/aux.Service/AfterGet", body=b"z") + self.assertEqual(connection.getresponse().status, 200) + + def test_off_trajectory_chat_request_is_409(self): + connection = http.client.HTTPConnection(self.proxy.host, self.proxy.port, timeout=5) + self.addCleanup(connection.close) + # A GetChatMessage without the nonce is off the control trajectory. + connection.request( + "POST", "/exa.api_server_pb.ApiServerService/GetChatMessage", + body=b"no nonce here", headers={"Content-Type": "application/connect+proto"}, + ) + response = connection.getresponse() + self.assertEqual(response.status, 409) + response.read() + receipt = self.proxy.receipt_json() + self.assertIn("unexpected_request", receipt) + self.assertFalse(receipt["initial_prompt_observed"]) + + def test_initial_prompt_with_nonce_advances_trajectory(self): + connection = http.client.HTTPConnection(self.proxy.host, self.proxy.port, timeout=5) + self.addCleanup(connection.close) + connection.request( + "POST", "/exa.api_server_pb.ApiServerService/GetChatMessage", + body=NONCE.encode(), headers={"Content-Type": "application/connect+proto"}, + ) + response = connection.getresponse() + self.assertEqual(response.status, 200) + body = response.read() + self.assertGreater(len(body), 5, "expected a Connect envelope payload") + receipt = self.proxy.receipt_json() + self.assertTrue(receipt["initial_prompt_observed"]) + self.assertTrue(receipt["tool_call_response_emitted"]) + self.assertEqual(receipt["endpoint_observed"]["framing"], "connect_envelope") + + +class DevinProxyFaultModeTests(unittest.TestCase): + def test_response_fault_text_emits_text_instead_of_tool_call(self): + proxy = ProxyHarness(response_fault="text") + self.addCleanup(proxy.close) + connection = http.client.HTTPConnection(proxy.host, proxy.port, timeout=5) + self.addCleanup(connection.close) + connection.request( + "POST", "/exa.api_server_pb.ApiServerService/GetChatMessage", + body=NONCE.encode(), headers={"Content-Type": "application/connect+proto"}, + ) + response = connection.getresponse() + self.assertEqual(response.status, 200) + response.read() + receipt = proxy.receipt_json() + self.assertEqual(receipt["fault_response_emitted"], "text") + self.assertFalse(receipt["tool_call_response_emitted"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/labs/15-pitot/tests/test_e2e_binaries.py b/labs/15-pitot/tests/test_e2e_binaries.py new file mode 100644 index 000000000..873ff2fa9 --- /dev/null +++ b/labs/15-pitot/tests/test_e2e_binaries.py @@ -0,0 +1,126 @@ +"""Tests for the two E2E apparatus binaries that previously had no coverage +outside live CI cells: the pitot-witness wrapper (byte/exit transparency + +nonce-bound receipts) and the runtime capability driver (explicit_request +allow/deny against a real runtime).""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +LAB = Path(__file__).resolve().parents[1] +NONCE = "feed" * 8 + +FAKE_PITOT = """#!/usr/bin/env python3 +import json, os, sys +sys.stdin.read() +event = { + "pitot_version": "1", "type": "action.requested", + "host": {"name": "claude", "adapter_version": "0.1.0"}, + "action": {"id": "act_" + "a" * 32, "kind": "shell"}, + "content": {"mode": "full", "full": json.dumps("echo %s")}, + "observation": {"source": "host_hook", "fidelity": "direct"}, +} +print(json.dumps(event)) +sys.exit(int(os.environ.get("FAKE_PITOT_EXIT", "0"))) +""" + + +def build(binary: str, source: str, cwd: Path, out_dir: Path) -> Path: + out = out_dir / (binary + (".exe" if os.name == "nt" else "")) + subprocess.run(["go", "build", "-o", str(out), source], cwd=cwd, check=True, capture_output=True) + return out + + +class WitnessTransparencyTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + base = Path(cls.tmp.name) + cls.witness = build("pitot-witness", "./main.go", LAB / "tests" / "witness", base) + # The fake pitot IS the --real-bin the witness execs, so it must be a + # self-contained executable (shebang + mode). The witness prepends it + # and the wrapped argv is exactly the hook invocation (`hook claude`). + cls.fake_pitot = base / "fake-pitot.py" + cls.fake_pitot.write_text(FAKE_PITOT % NONCE, encoding="utf-8") + cls.fake_pitot.chmod(0o755) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def run_witness(self, exit_code: int, nonce: str = NONCE): + # The witness prepends --real-bin, so the wrapped argv is just the + # hook invocation (`hook claude`) — exactly as the real harness calls + # it. The fake pitot IS the real-bin. + receipt = Path(self.tmp.name) / f"receipt-{exit_code}-{nonce[:4]}.jsonl" + environment = {**os.environ, "FAKE_PITOT_EXIT": str(exit_code)} + completed = subprocess.run( + [str(self.witness), "--real-bin", str(self.fake_pitot), "--receipt", str(receipt), "--nonce", nonce, + "--", "hook", "claude"], + input="{}", text=True, capture_output=True, env=environment, + ) + rows = [json.loads(line) for line in receipt.read_text(encoding="utf-8").splitlines()] if receipt.is_file() else [] + return completed, rows + + def test_witness_is_exit_and_byte_transparent_and_validates(self): + completed, rows = self.run_witness(0) + self.assertEqual(completed.returncode, 0) + self.assertIn("action.requested", completed.stdout, "witness must not alter pitot stdout") + self.assertEqual(len(rows), 1) + row = rows[0] + self.assertEqual(row["nonce"], NONCE) + self.assertEqual(row["host"], "claude") + self.assertEqual(row["pitot_exit"], 0) + self.assertTrue(row["valid"], f"witness rejected a valid event: {row}") + + def test_witness_records_deny_exit(self): + completed, rows = self.run_witness(2) + self.assertEqual(completed.returncode, 2, "deny exit must pass through unchanged") + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["pitot_exit"], 2) + + def test_witness_writes_no_receipt_on_nonce_mismatch(self): + # The witness only records a receipt for a valid, nonce-bound event; + # an event without the session nonce leaves no receipt (the driver's + # negative test relies on this absence). + _, rows = self.run_witness(0, nonce="00" * 16) + self.assertEqual(rows, []) + + +class RuntimeCapabilityDriverTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + base = Path(cls.tmp.name) + module = LAB / "pitot" + cls.pitot = build("pitot", "./cmd/pitot", module, base) + cls.testrole = build("pitot-testrole", "./internal/testrole", module, base) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def test_explicit_request_allow_deny_round_trip(self): + evidence = Path(self.tmp.name) / "evidence.json" + completed = subprocess.run( + [sys.executable, str(LAB / "tests" / "runtime_capability_driver.py"), + "--pitot", str(self.pitot), "--test-role", str(self.testrole), + "--platform", "local", "--evidence", str(evidence)], + text=True, capture_output=True, timeout=120, + ) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertIn("PITOT_RUNTIME_E2E_RESULT capability=explicit_request", completed.stdout) + value = json.loads(evidence.read_text(encoding="utf-8")) + self.assertEqual(value["controller"]["outcomes"], ["allow", "deny"]) + self.assertTrue(all(value["receipts"].values())) + self.assertEqual(len(set(value["controller"]["action_ids"])), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/labs/15-pitot/tests/test_install_real_agent.py b/labs/15-pitot/tests/test_install_real_agent.py index 219accbb6..78d86fde4 100644 --- a/labs/15-pitot/tests/test_install_real_agent.py +++ b/labs/15-pitot/tests/test_install_real_agent.py @@ -104,5 +104,87 @@ def test_rejects_archive_digest_mismatch(self): installer.install(agent, "ubuntu", "native") +class NpmInstallerTests(unittest.TestCase): + def test_global_install_pins_version_and_runs_scripts(self): + agent = { + "id": "claude", "version": "2.1.217", "executable": "claude", + "installer": {"kind": "npm", "package": "@anthropic-ai/claude-code"}, + } + with mock.patch.object(installer, "run") as run, \ + mock.patch.object(installer, "npm_executable", return_value="npm"): + installer.install(agent, "ubuntu", "native") + run.assert_called_once_with( + ["npm", "install", "--global", "--ignore-scripts=false", + "@anthropic-ai/claude-code@2.1.217"] + ) + + +class KimiInstallerTests(unittest.TestCase): + def test_posix_install_pins_version_via_env(self): + agent = { + "id": "kimi", "version": "0.29.0", "executable": "kimi", + "installer": {"kind": "kimi_release", "package": "https://code.kimi.com/kimi-code"}, + } + with mock.patch.object(installer, "run") as run: + installer.install(agent, "ubuntu", "native") + commands = [call.args[0] for call in run.call_args_list] + self.assertEqual(commands[0][0], "curl") + self.assertIn("https://code.kimi.com/kimi-code/install.sh", commands[0]) + bash_call = run.call_args_list[1] + self.assertEqual(bash_call.args[0][0], "bash") + self.assertEqual(bash_call.kwargs["env"]["KIMI_VERSION"], "0.29.0") + + def test_windows_install_uses_native_installer_with_version(self): + agent = { + "id": "kimi", "version": "0.29.0", "executable": "kimi", + "installer": {"kind": "kimi_release", "package": "https://code.kimi.com/kimi-code"}, + } + with mock.patch.object(installer, "run") as run: + installer.install(agent, "windows", "native") + command = run.call_args_list[0].args[0] + self.assertEqual(command[0], "powershell") + script = command[-1] + self.assertIn("$env:KIMI_VERSION='0.29.0'", script) + self.assertIn("kimi-code/install.ps1", script) + + +class CursorInstallerTests(unittest.TestCase): + AGENT = { + "id": "cursor", "version": "2026.07.20-8cc9c0b", "executable": "agent", + "installer": {"kind": "cursor_release", + "package": "https://downloads.cursor.com/lab/2026.07.20-8cc9c0b"}, + } + + def _run_factory(self, executables: int): + """Make a fake `run` that materializes the tar extraction so the + single-executable invariant is exercised on real files.""" + def fake_run(command, **_kwargs): + if command[0] == "tar" and "-C" in command: + directory = Path(command[command.index("-C") + 1]) + for index in range(executables): + sub = directory / f"pkg{index}" + sub.mkdir(parents=True, exist_ok=True) + (sub / "cursor-agent").write_bytes(b"cursor") + return None + return fake_run + + def test_extracts_single_executable_and_swaps_symlink(self): + with tempfile.TemporaryDirectory() as home, mock.patch.dict(os.environ, {"HOME": home}), \ + mock.patch.object(installer.host_platform, "machine", return_value="arm64"), \ + mock.patch.object(installer, "run", side_effect=self._run_factory(1)): + installer.install(self.AGENT, "ubuntu", "native") + link = Path(home) / ".local/bin/agent" + self.assertTrue(link.is_symlink()) + self.assertEqual(link.resolve().read_bytes(), b"cursor") + self.assertIn("pitot-cursor/2026.07.20-8cc9c0b", str(link.resolve())) + + def test_rejects_archive_with_multiple_executables(self): + with tempfile.TemporaryDirectory() as home, mock.patch.dict(os.environ, {"HOME": home}), \ + mock.patch.object(installer.host_platform, "machine", return_value="x86_64"), \ + mock.patch.object(installer, "run", side_effect=self._run_factory(2)): + with self.assertRaisesRegex(RuntimeError, "2 agent executables"): + installer.install(self.AGENT, "ubuntu", "native") + + if __name__ == "__main__": unittest.main() diff --git a/labs/15-pitot/tests/test_receipt_validation.py b/labs/15-pitot/tests/test_receipt_validation.py new file mode 100644 index 000000000..0a775f37e --- /dev/null +++ b/labs/15-pitot/tests/test_receipt_validation.py @@ -0,0 +1,167 @@ +"""Negative-branch tests for real_agent_driver.validate_receipts — the +causal checks that previously were enforced only by live CI cells. Each test +mutates one receipt and expects the specific rejection; the baseline is +proven sound by reaching the installation-manifest check (the first gate +after every causal branch under test).""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[3] +LAB = ROOT / "labs/15-pitot" + +driver_spec = importlib.util.spec_from_file_location("real_agent_driver", LAB / "tests/real_agent_driver.py") +driver = importlib.util.module_from_spec(driver_spec) +assert driver_spec and driver_spec.loader +driver_spec.loader.exec_module(driver) + +MANIFEST = json.loads((LAB / "adapter-verification.json").read_text(encoding="utf-8")) +AGENTS = {agent["id"]: agent for agent in MANIFEST["agents"]} +NONCE = "ab" * 16 + + +class ReceiptFixture: + """A synthetic, internally-consistent receipt set for one hook run.""" + + def __init__(self, base: Path, agent_id: str = "claude"): + self.base = base + self.agent = AGENTS[agent_id] + self.action_ids = ["act_" + "1" * 32, "act_" + "2" * 32] + self.proxy = base / "proxy.json" + self.witness = base / "witness.jsonl" + self.controller = base / "controller.jsonl" + self.consumer = base / "consumer.jsonl" + self.canary = base / "canary.txt" + self.proxy_receipt = { + "schema_version": 1, "agent": agent_id, "protocol": "anthropic_messages", "nonce": NONCE, + "initial_prompt_observed": True, "allow_tool_call_response_emitted": True, + "allow_tool_result_observed": True, "deny_tool_call_response_emitted": True, + "denied_result_observed": True, "final_response_emitted": True, + "tool_call_response_emitted": True, "tool_result_observed": True, + "auxiliary_requests": 1, + } + self.witness_rows = [ + {"nonce": NONCE, "host": agent_id, "valid": True, "pitot_exit": exit_code, "action_id": action} + for exit_code, action in zip((0, 2), self.action_ids) + ] + self.controller_rows = ( + [{"receipt_type": "request", "value": {"action_id": action}} for action in self.action_ids] + + [ + {"receipt_type": "response", "value": {"action_id": self.action_ids[0], "outcome": "allow"}}, + {"receipt_type": "response", "value": {"action_id": self.action_ids[1], "outcome": "deny", "message": f"PITOT_CONTROLLER_DENY {NONCE}"}}, + ] + ) + self.consumer_rows = [ + {"action": {"id": action}, "content": {"mode": "sha256", "sha256": "0" * 64}} + for action in self.action_ids + ] + self.canary_lines = [f"PITOT_ALLOW {NONCE}"] + self.runtime_identity = { + "schema_version": 1, "instance_id": "i1", "pid": 42, + "endpoint": "http://127.0.0.1:1/", "config_sha256": "c" * 64, + } + # Deliberately wrong version: validation must run every causal branch + # first and terminate at the installation-manifest gate. + self.installation = {"agent": agent_id, "version": "0.0.0-synthetic"} + self.output = f"PITOT_E2E_COMPLETE {NONCE}\n" + + def write(self) -> None: + self.proxy.write_text(json.dumps(self.proxy_receipt), encoding="utf-8") + self.witness.write_text("\n".join(json.dumps(row) for row in self.witness_rows) + "\n", encoding="utf-8") + self.controller.write_text("\n".join(json.dumps(row) for row in self.controller_rows) + "\n", encoding="utf-8") + self.consumer.write_text("\n".join(json.dumps(row) for row in self.consumer_rows) + "\n", encoding="utf-8") + self.canary.write_text("\n".join(self.canary_lines) + "\n", encoding="utf-8") + + def validate(self): + self.write() + return driver.validate_receipts( + self.agent, "ubuntu", NONCE, self.installation, self.proxy, self.witness, + self.controller, self.consumer, self.canary, self.runtime_identity, + 0, self.output, "prompt", + ) + + +class ValidateReceiptsNegativeBranches(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.fixture = ReceiptFixture(Path(self.tmp.name)) + + def expect(self, fragment: str): + with self.assertRaises(RuntimeError) as caught: + self.fixture.validate() + self.assertIn(fragment, str(caught.exception)) + + def test_baseline_reaches_installation_gate(self): + # Every causal branch passes; the synthetic version stops at the gate. + self.expect("installation receipt does not match supervised manifest") + + def test_witness_order_must_be_allow_then_deny(self): + self.fixture.witness_rows[0]["pitot_exit"] = 2 + self.fixture.witness_rows[1]["pitot_exit"] = 0 + self.expect("one allow and one deny") + + def test_duplicated_canary_execution_is_rejected(self): + self.fixture.canary_lines.append(f"PITOT_ALLOW {NONCE}") + self.expect("canary execution count") + + def test_denied_canary_execution_is_rejected(self): + self.fixture.canary_lines.append(f"PITOT_DENY {NONCE}") + self.expect("canary execution count") + + def test_consumer_projection_leak_is_rejected(self): + self.fixture.consumer_rows[0]["content"] = {"mode": "sha256", "sha256": "0" * 64, "full": "secret"} + self.expect("projection did not remove full command content") + + def test_deny_message_must_be_nonce_bound(self): + self.fixture.controller_rows[3]["value"]["message"] = "PITOT_CONTROLLER_DENY wrong" + self.expect("nonce-bound allow/deny trajectory") + + def test_witness_correlation_must_match_controller(self): + self.fixture.witness_rows[0]["action_id"] = "act_" + "9" * 32 + self.expect("hook correlation ids do not match") + + def test_nonzero_exit_is_agent_loop_incomplete(self): + self.fixture.write() + with self.assertRaises(RuntimeError) as caught: + driver.validate_receipts( + self.fixture.agent, "ubuntu", NONCE, self.fixture.installation, + self.fixture.proxy, self.fixture.witness, self.fixture.controller, + self.fixture.consumer, self.fixture.canary, self.fixture.runtime_identity, + 1, self.fixture.output, "prompt", + ) + self.assertIn("agent loop incomplete", str(caught.exception)) + + +class ValidateReceiptsACPBranches(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.fixture = ReceiptFixture(Path(self.tmp.name), agent_id="devin") + self.fixture.proxy_receipt["protocol"] = "devin_connect_proto" + self.fixture.witness_rows = [] + + def expect(self, fragment: str): + with self.assertRaises(RuntimeError) as caught: + self.fixture.validate() + self.assertIn(fragment, str(caught.exception)) + + def test_acp_baseline_reaches_installation_gate(self): + self.expect("installation receipt does not match supervised manifest") + + def test_acp_with_witness_receipts_is_rejected(self): + self.fixture.witness_rows = [{"nonce": NONCE, "host": "devin", "valid": True, "pitot_exit": 0, "action_id": self.fixture.action_ids[0]}] + self.expect("unexpectedly entered the lifecycle-hook witness path") + + def test_auxiliary_request_ceiling(self): + self.fixture.proxy_receipt["auxiliary_requests"] = 101 + self.expect("ceiling is 100") + + +if __name__ == "__main__": + unittest.main() From 005d8088c38b43aa5fe33f62e34261e4e08bcf3c Mon Sep 17 00:00:00 2001 From: bigboateng Date: Fri, 31 Jul 2026 15:27:18 +0100 Subject: [PATCH 2/2] docs(pitot): add adapter-hardening release note MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Required by the Generated distribution gate: any labs/15-pitot change ships a release note under pitot-distribution/release-notes. Disclosure-Reviewed: reviewed — public-safe only, private facet kept out of this commit --- .../2026-07-31-adapter-hardening.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 labs/15-pitot/pitot-distribution/release-notes/2026-07-31-adapter-hardening.md diff --git a/labs/15-pitot/pitot-distribution/release-notes/2026-07-31-adapter-hardening.md b/labs/15-pitot/pitot-distribution/release-notes/2026-07-31-adapter-hardening.md new file mode 100644 index 000000000..ab9d234ac --- /dev/null +++ b/labs/15-pitot/pitot-distribution/release-notes/2026-07-31-adapter-hardening.md @@ -0,0 +1,20 @@ +### Adapter boundary hardening + +Pitot hardens the supervised boundary without removing any adapter, transport, +or observation mode. An event-delivery fault now returns a distinct conflict +response instead of one that a host could read as an allow, and the ACP +transport emits a one-shot diagnostic when a `tool_call` update omits the +vendor inference-tool metadata (it stays fail-closed either way). + +An additive strict mode — the `require_controller` config key and the +`pitot run --strict` flag — makes a declared action kind with no registered +controller fault instead of allowing. The default remains permissive, now with +an explicit observation-only notice so a downgraded guarantee is never silent. +`pitot hook` prints a notice when no runtime is selected, and `pitot doctor` +gains a binary-hash drift check for hosts wired outside the repository. + +Decode-time tool-name guards were added per host, unknown boundary events no +longer default to `shell`, and the router's resolved-action set is now bounded. +Coverage grows with Go guard tests, Devin control-proxy HTTP tests, receipt +negative-branch tests, the two E2E apparatus binaries, and npm/Kimi/Cursor +installer tests.