From 639228183db533ffa3a2665e6745035b252cb900 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Mon, 17 Aug 2026 18:59:27 -0400 Subject: [PATCH 01/30] Plan plugin foundation migration --- .plans/37-plugin-system-and-filesystem.md | 215 ++++++++++++++++++++++ .plans/38-mcp-plugin.md | 173 +++++++++++++++++ .plans/plugin-foundation-review-packet.md | 10 + docs/behavior.md | 36 +++- 4 files changed, 433 insertions(+), 1 deletion(-) create mode 100644 .plans/37-plugin-system-and-filesystem.md create mode 100644 .plans/38-mcp-plugin.md create mode 100644 .plans/plugin-foundation-review-packet.md diff --git a/.plans/37-plugin-system-and-filesystem.md b/.plans/37-plugin-system-and-filesystem.md new file mode 100644 index 0000000..e4a5a3a --- /dev/null +++ b/.plans/37-plugin-system-and-filesystem.md @@ -0,0 +1,215 @@ +# Plugin system and filesystem plugin — plan v2 + +Build the plugin seam and migrate the filesystem tools through it. This is the first implementation slice. It must land and receive implementation review before the MCP plugin starts. No release is cut between this plan and the MCP plugin plan. + +## Approved decisions + +1. **No implicit filesystem tools.** `Harness(...)` has no tools unless the caller passes `tools=` or `plugins=`. The replacement is `plugins=[FilesystemPlugin()]`. +2. **One canonical root.** `HarnessConfig.root` stays in core as run context. `FilesystemPlugin` uses `PluginContext.root`; it does not accept a second root. +3. **Static contributions are visible before connection.** `FilesystemPlugin` tools appear in `harness.tools` and `tool_schemas()` immediately after construction. Connected plugins can add a second, dynamic contribution during `connect()`. +4. **Direct extension stays simple.** Independent custom tools still use `tools=[ToolSpec(...)]`, and independent hooks still use `hooks=[Hook(...)]`. +5. **Plugin names are unique within one harness.** Names identify contribution origin. `MCPPlugin` therefore groups all servers for a harness. +6. **Plugins connect before `run_start`.** A connected hook applies to the first lazy run. A connection failure happens before a run starts, so `run_start` and `run_end` do not fire for that attempt. +7. **Parallel-LLM policy remains in core temporarily.** Keep `HarnessConfig.read_paths` and `write_paths` until `ParallelLlmPlugin` migrates. + +These are breaking pre-1.0 changes. Do not add compatibility aliases or an implicit default plugin. + +## Goal + +After this plan: + +```python +harness = Harness( + HarnessConfig(root="."), + model=model, + plugins=[ + FilesystemPlugin(tools=["read", "write", "edit", "search", "list", "glob", "jsonl_search"]), + ], + tools=[custom_tool], + hooks=[custom_hook], +) +``` + +The core run loop knows how to compose plugins, but it does not import or construct filesystem tools. + +## Public interface + +Add `thinharness/plugins/base.py` with these concepts. Exact private helper names can change, but the public shape and lifecycle must stay small. + +```python +@dataclass(frozen=True) +class PluginContext: + root: Path + +@dataclass(frozen=True) +class ToolOrigin: + plugin: str + source: str | None = None + attributes: Json = field(default_factory=dict) + +@dataclass(frozen=True) +class PluginContribution: + tools: tuple[ToolSpec, ...] = () + instructions: tuple[str, ...] = () + hooks: tuple[Hook, ...] = () + +@dataclass(frozen=True) +class PluginBinding: + static: PluginContribution = PluginContribution() + connect: Callable[[], AsyncContextManager[PluginContribution]] | None = None + +class Plugin(Protocol): + name: str + def bind(self, context: PluginContext) -> PluginBinding: ... +``` + +Rules: + +- `bind()` is synchronous and does no file or network I/O. +- One plugin object can bind to more than one harness. Each call returns independent binding state. +- Plugin names are non-empty and unique within one harness. Reject duplicates before binding. +- `PluginBinding.static` is validated and installed during harness construction. +- `connect` is optional. ThinHarness enters it lazily during `Harness.connect()` or before `run_start` on the first run. It can return dynamic tools, instructions, and hooks. +- Concurrent `connect()` calls share one connection attempt and cannot enter a binding twice. +- Dynamic contributions go through the full tool, hook-filter, approval, reserved-name, structured-output, callable-handler, and collision validation path. Commit the complete staged set only after every plugin connects successfully. +- A connection failure catches `BaseException`, closes entered bindings in reverse order, leaves no dynamic contribution installed, and allows a later retry. Cancellation propagates after cleanup. +- `Harness.aclose()` closes plugin bindings in reverse order, then closes an owned model. +- `Harness.plugins` is a read-only tuple of configured plugin objects for temporary feature bridges such as subagents. Core composition uses bindings, not plugin-specific inspection. +- Plugins run in-process and are trusted. There is no isolation, entry-point discovery, hot reload, dependency ordering, or plugin-to-plugin lookup. + +Observers are not part of this first interface. Current tracing does not implement a neutral read-only observer seam, and adding an unused interface now would be speculative. A later observability plan can add it with a real adapter. + +## Composition order + +Use one deterministic order: + +1. plugin static contributions in caller plugin order; +2. direct `tools=` and `hooks=` contributions; +3. plugin dynamic contributions in caller plugin order. + +System instructions are assembled as: + +1. `HarnessConfig.system_prompt`; +2. plugin-level instruction strings in contribution order; +3. the current skill summary while skills remain on the transitional built-in path; +4. `ToolSpec.instructions` in final tool order; +5. structured-output instructions through the existing output path. + +Plugin hooks use the existing strict/non-strict behavior and execute in contribution order. Always build a fresh `HookRegistry` from copied caller hooks plus plugin hooks; never mutate a caller-supplied registry. Validate dynamic hook filters before commit. Do not add topological ordering or hook priority. + +## Filesystem plugin + +Add `thinharness/plugins/filesystem.py` with `FilesystemPlugin`. + +- The plugin wraps the existing `FileTools` implementation instead of copying tool logic. +- Its default tool list is `read`, `write`, `edit`, `search`, `list`, and `glob` in that order. +- `jsonl_search` remains opt-in and stays behind this plugin because it shares the root, read policy, ripgrep process, truncation, and spill-output handling. +- `tools=` accepts an ordered sequence, rejects duplicates and unknown names, and preserves caller order. Do not accept a set because set order is not part of the interface. +- The plugin contributes `Workspace root: ` as plugin-level instructions. A harness without this plugin does not claim that it has a model-visible workspace. +- The plugin owns the current filesystem settings: `output_dir`, read and write path policies, read and output limits, search line limit, ripgrep timeout, and search exclusion globs. +- Keep `FileTools` as a public low-level deep module for callers that want direct `ToolSpec` values. Remove only the public `builtin_tools()` helper after all in-repo callers move to `FilesystemPlugin`. +- `bind()` must not create the root. Remove eager root creation from plugin composition. Missing-root read, list, glob, search, and JSONL calls return their normal empty or not-found result; write and spill paths create required parents when used. + +Keep `HarnessConfig.root`. Remove these filesystem-only fields from `HarnessConfig`: + +- `output_dir` +- `max_read_chars` +- `max_read_bytes` +- `max_tool_chars` +- `max_search_line_chars` +- `rg_timeout` +- `search_exclude_globs` + +Keep `read_paths` and `write_paths` temporarily as parallel-LLM policy. `FilesystemPlugin` receives its own path-policy settings explicitly. Document this temporary duplication and remove the core fields when `ParallelLlmPlugin` migrates. + +`Harness` and `FilesystemPlugin` must not create the root during construction or binding. A harness without the plugin has no filesystem side effect and no workspace instruction. + +## Transitional built-ins + +`HarnessConfig.builtin_tools` stays temporarily for skills, subagents, and parallel LLM until their own plugin plans land. It no longer accepts filesystem tool names. `builtin_tools=None` selects no transitional built-ins; every remaining built-in is explicit. An old filesystem name fails with an error that points to `FilesystemPlugin`. + +Update `SubAgentConfig` and `build_child_harness` only as much as needed to preserve current child filesystem choices: + +- add an explicit `plugins` field and count it as a valid tool source in `SubAgentConfig.validate_subagent`; +- pass child plugins through `build_child_harness` into `Harness`; +- convert in-repo child filesystem selections to `FilesystemPlugin`; +- default child inheritance can continue to pass already-resolved non-MCP `ToolSpec` values; +- do not redesign subagent factories or plugin inheritance in this slice. + +Mark the temporary `builtin_tools` path for removal in the later skills, parallel-LLM, and subagent plugin plans. Do not create a second hidden filesystem construction path for children. + +## Implementation steps + +1. Add plugin contracts, exports, and focused contract tests. +2. Refactor harness construction into collection, validation, and final assignment so static plugin contributions are atomic. +3. Add generic plugin connection management. During this unreleased intermediate slice, connect generic plugins first and the existing MCP stack second, both before `run_start`. If either path fails, close and reset both paths and remove all staged dynamic contributions so retry starts clean. Close MCP first, then generic plugins, then an owned model. +4. Add full dynamic contribution staging, validation, rollback, retry, concurrent-connect serialization, and reverse-order close. +5. Add `ToolOrigin` to `ToolSpec`. Stamp plugin tools with their unique plugin name while direct tools can keep `origin=None`. +6. Implement `FilesystemPlugin` on top of `FileTools` and move filesystem configuration except temporary parallel-LLM path policy out of `HarnessConfig`. +7. Migrate unit tests, end-to-end journeys, examples, README, `docs/docs.md`, and the hand-maintained `docs/site/explainer/index.html` to explicit filesystem plugins. +8. Remove core imports of `tools.filesystem`, `DEFAULT_BUILTIN_TOOLS`, and the exported `builtin_tools()` helper. Migrate its direct caller in `tests/unit/test_tool_retry.py`. +9. Add an architecture test that fails if `thinharness/core.py` imports `thinharness.plugins.filesystem` or `thinharness.tools.filesystem`. + +## Behavior contract changes before implementation + +After this plan review and before code changes, update `docs/behavior.md` with: + +- explicit plugin composition and no implicit filesystem tools; +- bind, static contribution, lazy connect, atomic commit, retry, and reverse close rules; +- deterministic tool, instruction, and hook ordering; +- duplicate-name errors and plugin origin; +- one canonical root and the absence of filesystem side effects without `FilesystemPlugin`; +- plugin connection occurring before `run_start`, and the run toolset freeze occurring after both; + +Edit only affected sections. + +## Tests + +Add focused tests for: + +- static plugin tools and instructions visible before `connect()`; +- direct custom tools combined with plugin tools; +- duplicate plugin names, duplicate tools across plugins, and collisions between a plugin and `tools=`; +- copied hook registries, plugin hook ordering, dynamic hook-filter validation, and existing strict-hook behavior; +- lazy connection before `run_start`, one connection across several runs, concurrent connection, reverse close, failed-open rollback, cancellation, and successful retry; +- full dynamic tool validation, including reserved names, structured output, approval policy, and callable handlers; +- no partial dynamic tools, instructions, or hooks after failure; +- toolset freeze after plugin connection and run-start hooks; +- reuse of one plugin object across two harnesses with independent binding state; +- no root creation and no workspace instruction without `FilesystemPlugin`, plus missing-root tool behavior with the plugin; +- filesystem default selection, explicit ordering, empty selection, duplicates, unknown names, path rules, spill files, and opt-in JSONL search; +- approval, resume, streaming, tracing, and subagent behavior with plugin-provided tools. + +Run: + +```bash +uv run pytest tests/unit/test_harness.py tests/unit/test_hooks.py tests/unit/test_streaming.py tests/unit/test_resume.py tests/unit/test_approvals.py +uv run pytest tests/unit/test_file_tools.py tests/unit/test_subagents.py tests/unit/test_tool_retry.py tests/unit/test_parallel_llm.py tests/unit/test_parallel_tools.py tests/unit/test_skills.py +uv run pytest +uv run ruff check . +uv run pyright +``` + +Use the actual filesystem test file names present at implementation time if they differ. + +## Success criteria + +- The plugin seam supports static and connected adapters without exposing resources to callers. +- Filesystem tools are available only through explicit `FilesystemPlugin` use or direct `ToolSpec` registration. +- `jsonl_search` is an option on `FilesystemPlugin`, not a separate plugin. +- Independent custom tools and hooks remain direct constructor inputs. +- Core does not import or construct filesystem implementations. +- Failed plugin connection leaves the harness clean and retryable. +- The full test suite, Ruff, and Pyright pass. + +## Out of scope + +- MCP migration, except for making generic connection management possible. +- Skills, subagents, parallel LLM, providers, or tracing as plugins. +- Package distribution splitting. +- Plugin discovery, package manifests, hot reload, UI contributions, sandboxing, or dependency graphs. +- Dynamic tool-list change notifications after connection. + +## Review record + +One Codex, Claude, and GLM panel round reviewed plan v1 together with the MCP plan. Plan v2 applies the verified findings on parallel-LLM path policy, root side effects, hook registry copying, instruction order, dynamic validation, concurrent connection, temporary MCP coexistence, subagent validation, and missing tests. The approved product decisions are recorded above. No second plan-review round will run. diff --git a/.plans/38-mcp-plugin.md b/.plans/38-mcp-plugin.md new file mode 100644 index 0000000..ffc489c --- /dev/null +++ b/.plans/38-mcp-plugin.md @@ -0,0 +1,173 @@ +# MCP plugin — plan v2 + +Migrate MCP after `.plans/37-plugin-system-and-filesystem.md` lands and its implementation review is complete. This is the second implementation slice and the first connected production plugin. No release is cut between the two slices. + +## Approved decisions + +1. **One MCP plugin per harness.** Plugin names are unique, `MCPPlugin.name` is fixed to `"mcp"`, and one plugin groups every server for that harness. +2. **Keep server adapters.** `MCPServer`, `MCPServerStdio`, `MCPServerSSE`, and `MCPServerStreamableHTTP` remain transport and tool-conversion adapters. `MCPPlugin` owns harness composition and lifecycle. +3. **One discovered snapshot per binding.** MCP tool changes after connection are ignored until a new harness binding is created. Do not implement `notifications/tools/list_changed`. +4. **Subagent MCP settings stay temporarily.** Existing `SubAgentConfig.inherit_mcp_servers` and `mcp_servers` remain until the subagent plugin plan. The bridge moves out of core and builds child `MCPPlugin` values explicitly. +5. **Connection precedes `run_start`.** MCP setup failure occurs before a run starts and does not fire run lifecycle hooks. + +This plan removes `HarnessConfig.mcp_servers`. Do not support both configuration paths. + +## Goal + +After this plan: + +```python +harness = Harness( + HarnessConfig(root="."), + model=model, + plugins=[ + MCPPlugin(servers=[ + MCPServerStdio("python", ["server.py"], tool_prefix="docs"), + ]), + ], +) +``` + +`Harness.connect()` is fully generic. Core does not import MCP classes, resolve MCP server ids, discover MCP tools, or own an MCP-specific exit stack. + +## Plugin behavior + +Add `thinharness/plugins/mcp.py` with `MCPPlugin`. + +- `bind()` resolves and validates configuration without I/O. +- The binding has no static tools. +- Its connector owns a private `AsyncExitStack`, enters servers in caller order, discovers one tool snapshot from each server, and returns one dynamic `PluginContribution`. +- The connector keeps entered servers open until the harness closes. +- Generic harness composition validates the complete discovered set before committing any tool. +- Because an async context manager whose `__aenter__` fails is not owned by the harness stack, the MCP connector itself catches `BaseException`, closes its private stack in reverse order, and re-raises. This covers connection, discovery, schema, collision, and cancellation failures. +- Repeated `connect()` and repeated runs reuse the same binding and discovered tools. +- `Harness.aclose()` closes the plugin once. Reusing the same `MCPServer` wrapper across parent and child bindings keeps the current FastMCP reference-counted session sharing. + +Keep server filtering, prefixing, schema cleanup, error conversion, timeout behavior, optional dependency behavior, and transport ownership inside the existing MCP implementation module. + +## Generic tool origin + +Plan 37 adds `ToolOrigin`. Use it for MCP tools: + +```python +ToolOrigin( + plugin="mcp", + source=resolved_server_id, + attributes={"tool_name": original_tool_name}, +) +``` + +Remove the MCP-specific fields from the core tool contract: + +- remove `McpToolInfo`; +- remove `ToolSpec.mcp`; +- remove `"mcp"` from `ToolKind` after all checks use `ToolOrigin`. + +Keep `ToolKind` only for remaining framework control behavior such as the reserved subagent tool. Do not turn every origin into a new `ToolKind` value. + +`MCPPlugin.name` is fixed to `"mcp"`, so tracing derives `mcp.server.id` and `mcp.tool.name` from `ToolOrigin`, and subagent parent-tool inheritance excludes `origin.plugin == "mcp"` unless MCP inheritance is explicit. Tool result metadata remains unchanged because it is model-visible behavior. + +## Server identity + +Move duplicate server-id resolution from `Harness` into the MCP plugin binding. Preserve the current deterministic base ids and `-2`, `-3` suffix behavior. + +Server identity must be binding-local. Remove `MCPServer._resolved_id` and `resolve_id()` rather than leaving mutable identity on a shared wrapper. A private bound-server adapter carries `(server, resolved_id)` and builds handlers that pass the resolved id into result normalization. Both `ToolOrigin.source` and model-visible `ToolResult.metadata["mcp_server_id"]` use that bound id. Keep the public server wrapper responsible for connection and raw calls. + +Add tests for: + +- duplicate ids inside the one plugin; +- one server object reused by two harnesses with different collision neighbors; +- stable trace and result metadata after both harnesses connect; +- the public base id before binding. + +## Subagent bridge + +The subagent module is not core, so it can keep a temporary MCP-specific bridge until `SubagentsPlugin` exists. + +- Explicit `SubAgentConfig.mcp_servers` values become a child `MCPPlugin`. +- `inherit_mcp_servers=True` finds the fixed-name MCP plugin in the read-only `parent.plugins` tuple, copies its server adapters by identity, then adds explicit child servers without duplicate objects. +- Default parent-tool inheritance still excludes resolved MCP `ToolSpec` values. The child must connect its own MCP plugin so handlers and lifecycle are valid. +- Remove reads of `parent._mcp_servers` and `child.config.mcp_servers`; do not add a replacement MCP field to core. +- Keep current sharing, override, union, failure rollback, tracing, and cleanup behavior. +- Record the bridge for deletion in the later subagent plugin plan; do not add a generic plugin inheritance flag now. + +## Implementation steps + +1. Add `MCPPlugin` and public exports. +2. Remove mutable resolved ids from server wrappers and move resolution plus result attribution into binding-local MCP state. +3. Move server entry and discovery from `Harness._ensure_mcp_connected()` into the plugin connector, with its own failed-entry cleanup. +4. Delete `_mcp_servers`, `_mcp_stack`, `_mcp_connected`, `_resolve_mcp_server_ids()`, and `_ensure_mcp_connected()` from core. +5. Make `Harness.connect()` delegate only to generic plugin connection management from plan 37. +6. Remove `HarnessConfig.mcp_servers` and migrate every caller to `plugins=[MCPPlugin(...)]`. +7. Replace `McpToolInfo` with `ToolOrigin` in MCP conversion, tracing, tests, and subagent inheritance. +8. Implement the temporary subagent bridge without adding MCP knowledge back to core. +9. Add an architecture test that fails if `thinharness/core.py` directly imports MCP modules or contains MCP lifecycle logic. Transitive MCP reachability through the temporary subagent bridge remains until `SubagentsPlugin` migrates. +10. Update README, `docs/docs.md`, end-to-end journeys, examples, exports, changelog, and MCP-1/MCP-5 in `docs/behavior.md`. + +## Behavior contract changes before implementation + +After this plan review and before code changes, update the MCP section in `docs/behavior.md`: + +- replace `HarnessConfig.mcp_servers` with explicit `MCPPlugin` composition; +- state lazy generic plugin connection and one discovered snapshot per binding; +- state binding-local server ids and origin metadata; +- preserve filter, prefix, schema, result, error, timeout, optional dependency, session sharing, cancellation, and bounded cleanup rules; +- preserve explicit subagent inheritance while naming the temporary bridge; +- update MCP-1 server-id ownership and MCP-5 from `kind="mcp"`/`McpToolInfo` to `ToolOrigin`; +- leave the run toolset freeze wording owned by plan 37. + +Edit only affected sections. + +## Tests + +Retain or migrate all existing MCP behavior tests. Add focused coverage for: + +- MCP tools absent before connect and installed atomically after connect; +- explicit `connect()` and first-run lazy connection; +- repeated runs reuse one discovered snapshot; +- a second fixed-name `MCPPlugin` is rejected during harness construction; +- collisions with static plugin tools, direct tools, structured-output tools, and tools from another MCP server; +- failed second-server connection and failed discovery close the first server and allow retry; +- cancellation during connect and close; +- no dynamic contribution after failed connection; +- reverse close order across MCP and other connected plugins; +- binding-local server ids and one server wrapper shared across harnesses; +- tracing attribution after an after-tool hook rewrites output; +- resume and approval flows do not serialize MCP connection details; +- subagent no-inherit, explicit, inherit, union, shared-session, and failure cases, including a child whose second server fails while the parent keeps a shared server live; +- base import and wrapper construction without MCP dependencies. + +Run: + +```bash +uv run pytest tests/unit/test_mcp.py tests/unit/test_mcp_optional_dependency.py +uv run pytest tests/unit/test_subagents.py tests/unit/test_tracing.py tests/unit/test_resume.py tests/unit/test_approvals.py +uv run pytest +uv run ruff check . +uv run pyright +``` + +Run the deterministic MCP end-to-end journey. Report credential-based skips separately and do not count them as passes. + +## Success criteria + +- MCP is enabled only through `MCPPlugin`. +- Core contains no direct MCP imports, state, lifecycle, identity, or discovery logic; the temporary transitive dependency is isolated in the subagent module. +- Generic plugin rollback and cleanup preserve every current MCP lifecycle guarantee. +- `ToolSpec` contains generic origin data instead of MCP-specific fields. +- Parent and child harnesses retain explicit MCP inheritance and safe shared-session behavior. +- Existing MCP transport and result semantics do not change. +- The focused suite, full suite, Ruff, Pyright, and deterministic MCP journey pass. + +## Out of scope + +- Replacing FastMCP or changing its pinned version. +- MCP prompts, resources, sampling, elicitation, OAuth, tasks, server instructions, or provider-native MCP. +- Dynamic MCP tool-list updates. +- Generic plugin inheritance or child-harness factories. +- Subagents as a plugin. +- Package distribution splitting. + +## Review record + +One Codex, Claude, and GLM panel round reviewed plan v1 together with the plugin-system plan. Plan v2 applies the verified findings on unique plugin identity, failed connector entry, binding-local server ids, result metadata, subagent access, architecture-test scope, behavior-document ownership, and combined failure tests. The approved product decisions are recorded above. No second plan-review round will run. diff --git a/.plans/plugin-foundation-review-packet.md b/.plans/plugin-foundation-review-packet.md new file mode 100644 index 0000000..9902fb1 --- /dev/null +++ b/.plans/plugin-foundation-review-packet.md @@ -0,0 +1,10 @@ +# Plugin foundation review packet + +Review these two implementation plans as one ordered migration: + +1. [Plugin system and filesystem plugin](37-plugin-system-and-filesystem.md) +2. [MCP plugin](38-mcp-plugin.md) + +Read both linked files in full. Review each plan and the seam between them. In particular, check whether plan 37 leaves a complete and testable plugin interface, whether plan 38 can use that interface without MCP-specific changes to core, and whether the temporary subagent bridges create avoidable dual systems. + +Return one verdict with findings labeled `Plan 37`, `Plan 38`, or `Cross-plan`. Raise decisions that require product-owner input separately from implementation corrections. diff --git a/docs/behavior.md b/docs/behavior.md index 7ad40ae..0b1078e 100644 --- a/docs/behavior.md +++ b/docs/behavior.md @@ -80,6 +80,40 @@ Built-in provider resume state is a self-contained, provider-agnostic transcript - RESUME-6: A session seeded via `OpenAIResponsesSession.start(prompt, constants, previous_response_id=...)` captures only new transcript entries, so externally seeded prior turns are not present when later resumed from `resume_state`. This is unrelated to reasoning fidelity and is not changed by RESUME-3/RESUME-7. - RESUME-7: For reasoning-capable OpenAI Responses models the harness requests `include=["reasoning.encrypted_content"]` so reasoning survives resume; non-reasoning models are unaffected. Captured `resume_state` therefore contains encrypted reasoning blobs (OpenAI/OpenRouter) and signed thinking (Anthropic) and should be treated as sensitive, consistent with the local-trace sensitivity note. +## Plugin Composition + +### Purpose + +Callers compose optional harness behavior explicitly while independent custom tools and hooks stay direct constructor inputs. + +### Requirements + +- PLUGIN-1: `Harness` accepts plugins in caller order through `plugins=`; no plugin is loaded through entry points, directories, manifests, or implicit defaults. +- PLUGIN-2: Plugin names are non-empty and unique within one harness. A duplicate name fails before either plugin binds. +- PLUGIN-3: Plugin binding is synchronous and performs no file or network I/O. Static tools, instructions, and hooks are validated and visible immediately after harness construction. +- PLUGIN-4: `Harness.connect()` or the first run opens connected plugin bindings once in caller order. Concurrent connection calls share that attempt, and connection completes before `run_start` hooks fire. +- PLUGIN-5: Dynamic tools, instructions, and hooks are staged and receive the same complete validation as static contributions. ThinHarness commits the full dynamic set only after every binding opens successfully. +- PLUGIN-6: A connection failure, including cancellation, closes entered bindings in reverse order, installs no dynamic contribution, and leaves connection retryable. `run_start` and `run_end` do not fire for an attempt that fails during connection. +- PLUGIN-7: Closing a harness closes plugin bindings in reverse order before closing a model owned by the harness. Repeated close calls have no effect. +- PLUGIN-8: Contribution order is plugin static contributions, direct `tools=` and `hooks=`, then plugin dynamic contributions. System instructions are the configured system prompt, plugin instructions, the transitional skill summary, and per-tool instructions; structured-output instructions are added through the existing output path. +- PLUGIN-9: ThinHarness copies caller-supplied hook registries before adding plugin hooks. Plugin composition never mutates a caller-owned registry. +- PLUGIN-10: Plugins are trusted in-process code. ThinHarness does not isolate them or resolve dependencies between them. + +## Filesystem Plugin + +### Purpose + +Callers opt into root-scoped workspace tools without making filesystem behavior part of the core harness. + +### Requirements + +- FILESYSTEM-PLUGIN-1: `Harness` has no implicit filesystem tools. `FilesystemPlugin` provides `read`, `write`, `edit`, `search`, `list`, and `glob` by default; callers select an ordered subset explicitly. +- FILESYSTEM-PLUGIN-2: `jsonl_search` is an opt-in tool of `FilesystemPlugin` and shares its root, read policy, search process, truncation, and spill-output handling. +- FILESYSTEM-PLUGIN-3: `HarnessConfig.root` is the one run root. `FilesystemPlugin` uses that root and cannot configure a different root. +- FILESYSTEM-PLUGIN-4: Harness construction and plugin binding do not create the root. A harness without `FilesystemPlugin` adds no workspace-root instruction and has no filesystem side effect. +- FILESYSTEM-PLUGIN-5: Filesystem limits, output location, search settings, and path policies belong to `FilesystemPlugin`. `HarnessConfig.read_paths` and `write_paths` remain temporarily as parallel-LLM policy and do not configure filesystem plugin tools. +- FILESYSTEM-PLUGIN-6: Independent custom tools continue to use `tools=[ToolSpec(...)]`; callers do not need to wrap one tool in a plugin. + ## Run Toolset Freeze ### Purpose @@ -88,7 +122,7 @@ The set of tools a model can call is fixed when a run starts, so every provider ### Requirements -- TOOLSET-FREEZE-1: The run's tool schemas, system instructions, request metadata, and structured-output request are captured once per run after run-start hooks and MCP connection, and every provider request in that run uses that captured set. +- TOOLSET-FREEZE-1: The run's tool schemas, system instructions, request metadata, and structured-output request are captured once per run after harness connection and run-start hooks, and every provider request in that run uses that captured set. - TOOLSET-FREEZE-2: A tool added with `add_tool` during an in-flight run does not appear in that run's later provider requests; it takes effect on the next run. - TOOLSET-FREEZE-3: The executable tool map is frozen with the schemas: a model call naming a tool added mid-run resolves as an unknown tool for the current run, and approval-required detection uses the same frozen map. From 50602ec0d436c8517e592fb6096f04663cb275bc Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Mon, 17 Aug 2026 19:22:01 -0400 Subject: [PATCH 02/30] Add plugin system and filesystem plugin --- CHANGELOG.md | 8 + README.md | 9 +- docs/behavior.md | 2 +- docs/docs.md | 81 +++-- docs/site/about/index.html | 9 +- docs/site/explainer/index.html | 14 +- examples/web_research_report/agent.py | 15 +- pyproject.toml | 2 +- tests/e2e/anthropic_modernization_journey.py | 5 +- tests/e2e/langfuse_tracing_journey.py | 7 +- tests/e2e/prompt_caching_journey.py | 7 +- tests/e2e/structured_output_journey.py | 5 +- tests/e2e/workspace_tools_journey.py | 5 +- tests/unit/test_harness.py | 53 +++- tests/unit/test_mcp.py | 15 +- tests/unit/test_parallel_tools.py | 4 +- tests/unit/test_plugins.py | 295 +++++++++++++++++++ tests/unit/test_resume.py | 7 +- tests/unit/test_subagents.py | 15 +- tests/unit/test_tool_retry.py | 4 +- tests/unit/test_tracing.py | 13 +- thinharness/__init__.py | 13 +- thinharness/core.py | 206 +++++++++---- thinharness/defaults.py | 11 +- thinharness/plugins/__init__.py | 13 + thinharness/plugins/base.py | 57 ++++ thinharness/plugins/filesystem.py | 68 +++++ thinharness/subagents.py | 10 +- thinharness/tools/__init__.py | 5 +- thinharness/tools/base.py | 10 + thinharness/tools/filesystem.py | 5 - 31 files changed, 806 insertions(+), 167 deletions(-) create mode 100644 tests/unit/test_plugins.py create mode 100644 thinharness/plugins/__init__.py create mode 100644 thinharness/plugins/base.py create mode 100644 thinharness/plugins/filesystem.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c7b30b7..3026192 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## Unreleased + +- Added explicit plugin composition with static and connected contributions, atomic connection rollback, unique plugin names, generic tool origin, and plugin-provided hooks and instructions. +- Added `FilesystemPlugin` for the ordered workspace tool surface; `jsonl_search` remains opt-in through this plugin. +- **Breaking:** `Harness` no longer enables filesystem tools by default. Pass `plugins=[FilesystemPlugin(...)]`; independent custom tools still use `tools=`. +- **Breaking:** Removed filesystem settings from `HarnessConfig` and removed the `builtin_tools()` helper. `read_paths` and `write_paths` remain temporarily for the transitional parallel-LLM built-in. +- Changed connection setup to complete before `run_start` hooks. A connection failure does not fire run lifecycle hooks. + ## 0.6.0 - 2026-08-07 - Added automatic retries with bounded exponential backoff, jitter, and `Retry-After` support for transient OpenAI, Anthropic, and OpenRouter request failures. Configure retries with `HarnessConfig.request_retries` and `request_retry_backoff`. diff --git a/README.md b/README.md index 2017452..e79de50 100644 --- a/README.md +++ b/README.md @@ -241,10 +241,13 @@ Requires Python 3.11+. ```python import asyncio -from thinharness import Harness, HarnessConfig +from thinharness import FilesystemPlugin, Harness, HarnessConfig async def main(): - async with Harness(HarnessConfig(root=".", model="openai:gpt-5.5")) as harness: + async with Harness( + HarnessConfig(root=".", model="openai:gpt-5.5"), + plugins=[FilesystemPlugin(tools=["read"])], + ) as harness: result = await harness.run("Read README.md and summarize it.") print(result.text) @@ -273,7 +276,7 @@ Streaming emits coarse run, model, tool, retry, limit, and subagent events, then ## Features -- **Filesystem tools:** `read`, `write`, batched exact-replacement `edit`, `search`, `list`, and `glob` with root-scoped path policies. +- **Filesystem plugin:** explicit `FilesystemPlugin` composition for `read`, `write`, batched exact-replacement `edit`, `search`, `list`, and `glob` with root-scoped path policies. - **JSONL search:** opt-in `jsonl_search` for structured line-delimited data, with ripgrep prefiltering, field projection, equality/contains/regex/range `where` filters, and field-level snippets from large multiline string values. - **Bash prototype tool:** opt-in `BashTool` for exploratory shell commands. It is lightweight, custom-registration only, and is not included in the default or built-in tool set. - **Provider adapters:** built-in OpenAI, Anthropic, and OpenRouter adapters, plus public model/session protocols for implementing another provider. diff --git a/docs/behavior.md b/docs/behavior.md index 0b1078e..87e86db 100644 --- a/docs/behavior.md +++ b/docs/behavior.md @@ -110,7 +110,7 @@ Callers opt into root-scoped workspace tools without making filesystem behavior - FILESYSTEM-PLUGIN-1: `Harness` has no implicit filesystem tools. `FilesystemPlugin` provides `read`, `write`, `edit`, `search`, `list`, and `glob` by default; callers select an ordered subset explicitly. - FILESYSTEM-PLUGIN-2: `jsonl_search` is an opt-in tool of `FilesystemPlugin` and shares its root, read policy, search process, truncation, and spill-output handling. - FILESYSTEM-PLUGIN-3: `HarnessConfig.root` is the one run root. `FilesystemPlugin` uses that root and cannot configure a different root. -- FILESYSTEM-PLUGIN-4: Harness construction and plugin binding do not create the root. A harness without `FilesystemPlugin` adds no workspace-root instruction and has no filesystem side effect. +- FILESYSTEM-PLUGIN-4: Harness construction and plugin binding do not create the root. A harness without `FilesystemPlugin` has a generic default prompt, adds no workspace-root instruction, and has no filesystem side effect. - FILESYSTEM-PLUGIN-5: Filesystem limits, output location, search settings, and path policies belong to `FilesystemPlugin`. `HarnessConfig.read_paths` and `write_paths` remain temporarily as parallel-LLM policy and do not configure filesystem plugin tools. - FILESYSTEM-PLUGIN-6: Independent custom tools continue to use `tools=[ToolSpec(...)]`; callers do not need to wrap one tool in a plugin. diff --git a/docs/docs.md b/docs/docs.md index f98805b..6c2b097 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -24,11 +24,14 @@ ThinHarness requires Python 3.11+. ```python import asyncio -from thinharness import Harness, HarnessConfig +from thinharness import FilesystemPlugin, Harness, HarnessConfig async def main() -> None: - async with Harness(HarnessConfig(root=".", model="openai:gpt-5.5")) as harness: + async with Harness( + HarnessConfig(root=".", model="openai:gpt-5.5"), + plugins=[FilesystemPlugin(tools=["read"])], + ) as harness: result = await harness.run("Read README.md and summarize it.") print(result.text) @@ -88,23 +91,38 @@ config = HarnessConfig( system_prompt="You are a focused research agent.", max_model_requests=32, max_tool_calls=80, - read_paths=["inputs", "docs"], - write_paths=["outputs"], ) ``` Important groups: -- `root`, `read_paths`, `write_paths`, and `output_dir` define filesystem scope. +- `root` defines the run root. `FilesystemPlugin` owns filesystem paths, limits, search settings, and output location. - `model`, `api_key`, `base_url`, `temperature`, `max_tokens`, `effort`, `extra_body`, `request_timeout`, `request_retries`, and `request_retry_backoff` define provider settings. -- `builtin_tools`, `tools`, `subagents`, `mcp_servers`, and `skills_dir` define the model-callable surface. +- The `Harness` constructor's `plugins=` and `tools=` inputs, plus `builtin_tools`, `subagents`, `mcp_servers`, and `skills_dir`, define the model-callable surface. `builtin_tools` is temporary for features that have not migrated to plugins. - `max_model_requests`, `max_tool_calls`, `output_retries`, and `tool_retries` bound the run. - `output_type` and `output_mode` define structured output. - `tracing`, `local_tracing`, and `local_trace_dir` define observability. -## Built-In Filesystem Tools +## Plugins + +A plugin is a configured bundle of tools, system instructions, and hooks. Pass plugins explicitly in caller order: + +```python +harness = Harness( + HarnessConfig(root="."), + plugins=[FilesystemPlugin()], + tools=[custom_tool], + hooks=[custom_hook], +) +``` + +Independent custom tools and hooks stay direct constructor inputs. Plugin names must be unique within one harness. ThinHarness binds static contributions during construction, then opens connected plugins on `Harness.connect()` or before the first run. Connection is atomic: a failure installs no dynamic contribution, closes opened plugins in reverse order, and allows retry. Closing the harness also closes plugins in reverse order. + +Plugins are trusted in-process code. ThinHarness does not discover them from entry points or directories, isolate them, resolve dependencies between them, or hot reload them. + +## Filesystem Plugin -When `builtin_tools` is omitted, the model gets these filesystem tools: +The core harness has no implicit filesystem tools. Add `FilesystemPlugin()` to get these default tools: - `read`: read bounded UTF-8 file ranges with line numbers. - `write`: create, overwrite, or append UTF-8 files. This tool is sequential. @@ -113,15 +131,13 @@ When `builtin_tools` is omitted, the model gets these filesystem tools: - `list`: list files or directories. - `glob`: find files by glob pattern. -When `builtin_tools` is provided, it is an explicit replacement list. Include every built-in tool the model should see. - -`jsonl_search` is available as an opt-in built-in: +Use the plugin's ordered `tools` list to select a different surface. `jsonl_search` is opt-in: ```python -harness = Harness(HarnessConfig( - root=".", - builtin_tools=["read", "search", "jsonl_search"], -)) +harness = Harness( + HarnessConfig(root="."), + plugins=[FilesystemPlugin(tools=["read", "search", "jsonl_search"])], +) ``` Use `query` as a ripgrep row prefilter, `fields` to project only the values the model needs, and `where` for structured filters over jq-style field paths: @@ -182,11 +198,13 @@ For large multiline string fields, `field_searches` returns matching internal li Filesystem tools enforce the configured read and write policies. Paths must resolve under `root`; escape attempts through absolute paths outside `root`, `..`, or symlinks are rejected. ```python -harness = Harness(HarnessConfig( - root="/repo", - read_paths=["src", "tests"], - write_paths=["outputs"], -)) +harness = Harness( + HarnessConfig(root="/repo"), + plugins=[FilesystemPlugin( + read_paths=["src", "tests"], + write_paths=["outputs"], + )], +) ``` With this configuration, `read` can access `src/app.py` and `tests/test_app.py`, but not `docs/notes.md`. `write` can create or update `outputs/report.md`, but not `src/generated.py`. Omit `read_paths` or `write_paths` to allow that operation anywhere under `root`. @@ -364,12 +382,12 @@ By default, hook exceptions are logged and the run continues. Set `strict_hooks= The `subagent` tool is opt-in. It lets the parent delegate a bounded task to a child harness. Child runs start fresh; they do not inherit the parent provider transcript. ```python -from thinharness import Harness, HarnessConfig, SubAgentConfig +from thinharness import FilesystemPlugin, Harness, HarnessConfig, SubAgentConfig harness = Harness(HarnessConfig( root=".", - builtin_tools=["read", "search", "subagent"], + builtin_tools=["subagent"], subagents=[ SubAgentConfig( name="reviewer", @@ -379,7 +397,7 @@ harness = Harness(HarnessConfig( max_model_requests=12, ) ], -)) +), plugins=[FilesystemPlugin(tools=["read", "search"])]) ``` Calling `subagent` without an `agent` argument uses the framework default subagent, which inherits parent tools except for recursive `subagent` access and MCP-discovered tools. Named subagents use their own `SubAgentConfig`. @@ -387,7 +405,7 @@ Calling `subagent` without an `agent` argument uses the framework default subage Named subagents can: - inherit parent tools with `inherit_parent_tools=True` -- choose explicit `builtin_tools` +- choose explicit `plugins` or transitional `builtin_tools` - receive explicit custom `tools` - opt into MCP with `inherit_mcp_servers=True` or `mcp_servers=[...]` - use their own model, limits, and structured output @@ -459,12 +477,15 @@ When `output_type` is set on a custom `ParallelLlmTool`, successful entries cont Skills are explicit tools, not auto-discovery. Configure `skills_dir`, then expose `skill_read` and/or `skill_run` through `builtin_tools`. ```python -harness = Harness(HarnessConfig( - root=".", - skills_dir="skills", - selected_skills=["invoice-review"], - builtin_tools=["read", "search", "skill_read", "skill_run"], -)) +harness = Harness( + HarnessConfig( + root=".", + skills_dir="skills", + selected_skills=["invoice-review"], + builtin_tools=["skill_read", "skill_run"], + ), + plugins=[FilesystemPlugin(tools=["read", "search"])], +) ``` If skills are configured and skill tools are exposed, the system prompt includes a compact skill summary. The model still has to call `skill_read` to inspect details. diff --git a/docs/site/about/index.html b/docs/site/about/index.html index 17a9e95..9b4ffd3 100644 --- a/docs/site/about/index.html +++ b/docs/site/about/index.html @@ -164,10 +164,13 @@

Install

// use

Use

import asyncio
-from thinharness import Harness, HarnessConfig
+from thinharness import FilesystemPlugin, Harness, HarnessConfig
 
 async def main():
-    async with Harness(HarnessConfig(root=".", model="openai:gpt-5.5")) as harness:
+    async with Harness(
+        HarnessConfig(root=".", model="openai:gpt-5.5"),
+        plugins=[FilesystemPlugin(tools=["read"])],
+    ) as harness:
         result = await harness.run("Read README.md and summarize it.")
         print(result.text)
 
@@ -179,7 +182,7 @@ 

Use

// features

Features

-
Filesystem tools

read, write, batched exact-replacement edit, search, list, and glob with root-scoped path policies.

+
Filesystem plugin

Explicit FilesystemPlugin composition for read, write, batched exact-replacement edit, search, list, and glob with root-scoped path policies.

JSONL search

Opt-in jsonl_search for structured line-delimited data, with ripgrep prefiltering, field projection, equality/contains/regex/range where filters, and field-level snippets from large multiline string values.

Bash prototype tool

Opt-in BashTool for exploratory shell commands. It is lightweight, custom-registration only, and is not included in the default or built-in tool set.

Provider adapters

Built-in OpenAI, Anthropic, and OpenRouter adapters, plus public model/session protocols for implementing another provider.

diff --git a/docs/site/explainer/index.html b/docs/site/explainer/index.html index b40199d..206f905 100644 --- a/docs/site/explainer/index.html +++ b/docs/site/explainer/index.html @@ -257,11 +257,11 @@

Sequential flag

-

Built-in selection

+

Plugin composition

- builtin_tools is a list of tool names. When omitted, ThinHarness exposes the default filesystem - tools. When provided, it selects from built-in candidates such as filesystem tools, skill_read, - skill_run, subagent, and parallel_llm. + ThinHarness has no implicit filesystem tools. plugins=[FilesystemPlugin(...)] contributes an ordered + filesystem tool set, workspace instructions, and plugin origin data. builtin_tools remains temporarily + for skill_read, skill_run, subagent, and parallel_llm.

@@ -275,9 +275,9 @@

Tool surfaces

Filesystem - FileTools - builtin_tools(root, ...) returns FileTools(root).specs(). - Default exposed set is read, write, edit, search, list, glob. Mutating tools are marked sequential=True. + FilesystemPlugin over FileTools + plugins=[FilesystemPlugin(...)] binds static ToolSpec values without importing filesystem code into core. + The default set is read, write, edit, search, list, glob; jsonl_search is opt-in. Mutating tools are sequential. Framework-provided diff --git a/examples/web_research_report/agent.py b/examples/web_research_report/agent.py index 0b4767e..9b7cc6d 100644 --- a/examples/web_research_report/agent.py +++ b/examples/web_research_report/agent.py @@ -16,7 +16,7 @@ import httpx from pydantic import BaseModel, ConfigDict, Field -from thinharness import Harness, HarnessConfig, Hook, ParallelLlmTool, PathPolicy, PathValidationError, SubAgentConfig, ToolResult, ToolSpec +from thinharness import FilesystemPlugin, Harness, HarnessConfig, Hook, ParallelLlmTool, PathPolicy, PathValidationError, SubAgentConfig, ToolResult, ToolSpec REPO_ROOT = Path(__file__).resolve().parents[2] EXAMPLE_ROOT = Path(__file__).resolve().parent @@ -487,7 +487,7 @@ def build_harness(root: Path, *, model: str = DEFAULT_MODEL) -> Harness: root=root, model=model, system_prompt=SYSTEM_PROMPT, - builtin_tools=["read", "write", "edit", "search", "list", "glob", "jsonl_search", "subagent"], + builtin_tools=["subagent"], output_type=ReportReceipt, output_mode=output_mode, output_retries=2, @@ -496,8 +496,6 @@ def build_harness(root: Path, *, model: str = DEFAULT_MODEL) -> Harness: max_tool_calls=96, read_paths=["outputs"], write_paths=["outputs"], - max_read_chars=80_000, - max_tool_chars=80_000, tool_execution="sequential", request_timeout=240, temperature=0, @@ -507,7 +505,7 @@ def build_harness(root: Path, *, model: str = DEFAULT_MODEL) -> Harness: name="citation_critic", description="Citation and evidence critic for saved draft reports.", system_prompt=CITATION_CRITIC_PROMPT, - builtin_tools=["read", "search", "list", "glob"], + plugins=[FilesystemPlugin(tools=["read", "search", "list", "glob"], read_paths=["outputs"])], max_model_requests=10, max_tool_calls=20, output_retries=1, @@ -515,6 +513,13 @@ def build_harness(root: Path, *, model: str = DEFAULT_MODEL) -> Harness: ) ], ), + plugins=[FilesystemPlugin( + tools=["read", "write", "edit", "search", "list", "glob", "jsonl_search"], + read_paths=["outputs"], + write_paths=["outputs"], + max_read_chars=80_000, + max_tool_chars=80_000, + )], tools=[*exa_tools.specs(), parallel_tool], hooks=[Hook("after_tool_call", _source_audit_hook)], ) diff --git a/pyproject.toml b/pyproject.toml index b54c217..0ad22a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "thinharness" version = "0.6.0" -description = "Minimal filesystem agent harness with provider-backed Responses-like models." +description = "Minimal plugin-based agent harness with provider-backed Responses-like models." readme = "README.md" requires-python = ">=3.11" license = "MIT" diff --git a/tests/e2e/anthropic_modernization_journey.py b/tests/e2e/anthropic_modernization_journey.py index 2e4ad71..e93f857 100644 --- a/tests/e2e/anthropic_modernization_journey.py +++ b/tests/e2e/anthropic_modernization_journey.py @@ -12,7 +12,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from thinharness import AnthropicMessagesModel, AnthropicProvider, Harness, HarnessConfig, ModelSettings, ToolSpec +from thinharness import AnthropicMessagesModel, AnthropicProvider, FilesystemPlugin, Harness, HarnessConfig, ModelSettings, ToolSpec MODEL = os.getenv("E2E_ANTHROPIC_MODERNIZATION_MODEL", "anthropic:claude-sonnet-5") @@ -69,12 +69,13 @@ async def _assert_native_structured_output_and_defaults(root: Path, model_name: harness = Harness( HarnessConfig( root=root, - builtin_tools=["read"], + builtin_tools=[], output_type=InventoryAnswer, max_model_requests=4, max_tool_calls=2, ), model=AnthropicMessagesModel(model_name, provider=provider), + plugins=[FilesystemPlugin(tools=["read"])], ) result = await harness.run( diff --git a/tests/e2e/langfuse_tracing_journey.py b/tests/e2e/langfuse_tracing_journey.py index 7446247..a83074a 100644 --- a/tests/e2e/langfuse_tracing_journey.py +++ b/tests/e2e/langfuse_tracing_journey.py @@ -9,7 +9,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from thinharness import Harness, HarnessConfig, SubAgentConfig, TracingOptions, create_otlp_tracing +from thinharness import FilesystemPlugin, Harness, HarnessConfig, SubAgentConfig, TracingOptions, create_otlp_tracing MODEL = os.getenv("E2E_LANGFUSE_TRACING_MODEL", "openrouter:anthropic/claude-haiku-4.5") SYSTEM_PROMPT = "You are a tracing validation parent. Do your own parent checks, then delegate child file work to the named subagent." @@ -52,7 +52,7 @@ def main() -> None: root=root, model=MODEL, system_prompt=SYSTEM_PROMPT, - builtin_tools=["list", "read", "write", "subagent"], + builtin_tools=["subagent"], max_model_requests=40, max_tool_calls=12, local_trace_dir=trace_dir, @@ -60,12 +60,13 @@ def main() -> None: SubAgentConfig( name="writer", description="Creates and revises files for tracing validation.", - builtin_tools=["read", "write", "edit", "list"], + plugins=[FilesystemPlugin(tools=["read", "write", "edit", "list"])], max_model_requests=20, max_tool_calls=8, ) ], ), + plugins=[FilesystemPlugin(tools=["list", "read", "write"])], tracing=[TracingOptions( tracer=tracing.tracer, agent_name="langfuse-parent", diff --git a/tests/e2e/prompt_caching_journey.py b/tests/e2e/prompt_caching_journey.py index b477ef8..c7312fe 100644 --- a/tests/e2e/prompt_caching_journey.py +++ b/tests/e2e/prompt_caching_journey.py @@ -7,7 +7,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from thinharness import Harness, HarnessConfig +from thinharness import FilesystemPlugin, Harness, HarnessConfig MODEL = os.getenv("E2E_PROMPT_CACHING_MODEL", "anthropic:claude-haiku-4-5") @@ -34,10 +34,11 @@ def main() -> None: root=root, model=MODEL, system_prompt=SYSTEM_PROMPT, - builtin_tools=["read"], + builtin_tools=[], max_model_requests=4, max_tool_calls=2, - ) + ), + plugins=[FilesystemPlugin(tools=["read"])], ) # Run diff --git a/tests/e2e/structured_output_journey.py b/tests/e2e/structured_output_journey.py index f650541..aedebb7 100644 --- a/tests/e2e/structured_output_journey.py +++ b/tests/e2e/structured_output_journey.py @@ -9,7 +9,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from thinharness import Harness, HarnessConfig, Hook +from thinharness import FilesystemPlugin, Harness, HarnessConfig, Hook MODEL = os.getenv("E2E_STRUCTURED_MODEL", "openai:gpt-5-mini") SYSTEM_PROMPT = """You are a structured-output extraction agent. Use tools before finalizing.""" @@ -41,12 +41,13 @@ def main() -> None: root=root, model=MODEL, system_prompt=SYSTEM_PROMPT, - builtin_tools=["read"], + builtin_tools=[], output_type=InventoryAnswer, output_mode="native", max_model_requests=4, max_tool_calls=2, ), + plugins=[FilesystemPlugin(tools=["read"])], hooks=[Hook("before_tool_call", lambda ctx: tool_names.append(ctx.tool_name))], ) diff --git a/tests/e2e/workspace_tools_journey.py b/tests/e2e/workspace_tools_journey.py index 4b69970..4c0dc21 100644 --- a/tests/e2e/workspace_tools_journey.py +++ b/tests/e2e/workspace_tools_journey.py @@ -8,7 +8,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from thinharness import Harness, HarnessConfig, Hook +from thinharness import FilesystemPlugin, Harness, HarnessConfig, Hook MODEL = os.getenv("E2E_WORKSPACE_MODEL", "openai:gpt-5.2") SYSTEM_PROMPT = """You are an exacting workspace agent. Use tools when instructed and keep the final answer brief.""" @@ -47,11 +47,12 @@ def main() -> None: root=root, model=MODEL, system_prompt=SYSTEM_PROMPT, - builtin_tools=["read", "write", "edit", "search", "list", "glob", "jsonl_search"], + builtin_tools=[], max_model_requests=30, max_tool_calls=12, local_trace_dir=trace_dir, ), + plugins=[FilesystemPlugin(tools=["read", "write", "edit", "search", "list", "glob", "jsonl_search"])], hooks=[Hook("before_tool_call", lambda ctx: tool_names.append(ctx.tool_name))], ) diff --git a/tests/unit/test_harness.py b/tests/unit/test_harness.py index 233bf8e..acc3ae0 100644 --- a/tests/unit/test_harness.py +++ b/tests/unit/test_harness.py @@ -24,6 +24,7 @@ from thinharness import ( AfterToolCallContext, AnthropicMessagesModel, + FilesystemPlugin, Harness, HarnessConfig, HarnessError, @@ -50,7 +51,11 @@ def test_harness_tool_loop_with_custom_client(tmp_path: Path) -> None: (tmp_path / "hello.txt").write_text("hello", encoding="utf-8") client = FakeClient() - harness = Harness(HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client)) + harness = Harness( + HarnessConfig(root=tmp_path, model="openai:test-model"), + model=_fake_openai(client), + plugins=[FilesystemPlugin(tools=["read"])], + ) result = harness.run_sync("read hello", metadata={"case": "test"}) assert result.text == "done" assert client.payloads[0]["tools"] @@ -288,22 +293,33 @@ def test_custom_tool_invalid_json_is_structured(tmp_path: Path) -> None: assert output["metadata"]["retry"] is True assert "invalid JSON arguments" in output["content"] -def test_builtin_tool_selection_is_explicit(tmp_path: Path) -> None: - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=["read", "search"]), model=_fake_openai(FakeClient())) +def test_filesystem_plugin_tool_selection_is_explicit(tmp_path: Path) -> None: + harness = Harness( + HarnessConfig(root=tmp_path), + model=_fake_openai(FakeClient()), + plugins=[FilesystemPlugin(tools=["read", "search"])], + ) assert [tool["name"] for tool in harness.tool_schemas()] == ["read", "search"] -def test_default_builtin_tools_are_minimal_filesystem_surface(tmp_path: Path) -> None: + +def test_harness_has_no_implicit_filesystem_tools(tmp_path: Path) -> None: harness = Harness(HarnessConfig(root=tmp_path), model=_fake_openai(FakeClient())) - assert [tool["name"] for tool in harness.tool_schemas()] == ["read", "write", "edit", "search", "list", "glob"] + assert harness.tool_schemas() == [] -def test_specialized_builtin_tools_are_explicit_opt_ins(tmp_path: Path) -> None: - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=["jsonl_search", "subagent"]), model=_fake_openai(FakeClient())) + +def test_specialized_filesystem_tools_are_explicit_opt_ins(tmp_path: Path) -> None: + harness = Harness( + HarnessConfig(root=tmp_path, builtin_tools=["subagent"]), + model=_fake_openai(FakeClient()), + plugins=[FilesystemPlugin(tools=["jsonl_search"])], + ) assert [tool["name"] for tool in harness.tool_schemas()] == ["jsonl_search", "subagent"] def test_enabled_tool_instructions_are_appended_after_base_instructions(tmp_path: Path) -> None: harness = Harness( HarnessConfig(root=tmp_path, builtin_tools=["parallel_llm"], system_prompt="Caller instructions."), model=ScriptedModel([]), + plugins=[FilesystemPlugin(tools=[])], ) instructions = harness.system_instructions() @@ -313,7 +329,11 @@ def test_enabled_tool_instructions_are_appended_after_base_instructions(tmp_path assert "It does not inherit the parent system prompt" in instructions def test_disabled_tool_instructions_are_omitted(tmp_path: Path) -> None: - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=["read"]), model=ScriptedModel([])) + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[FilesystemPlugin(tools=["read"])], + ) assert "parallel_llm usage:" not in harness.system_instructions() @@ -336,8 +356,9 @@ def test_tool_instructions_follow_skill_summary(tmp_path: Path) -> None: def test_builtin_tool_instructions_are_appended(tmp_path: Path) -> None: harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=["search"]), + HarnessConfig(root=tmp_path), model=ScriptedModel([]), + plugins=[FilesystemPlugin(tools=["search"])], ) assert DEFAULT_SEARCH_INSTRUCTIONS in harness.system_instructions() @@ -352,7 +373,7 @@ def test_blank_tool_instructions_are_omitted(tmp_path: Path) -> None: ) harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([]), tools=[custom]) - assert harness.system_instructions() == f"{harness.config.system_prompt}\n\nWorkspace root: {tmp_path}" + assert harness.system_instructions() == harness.config.system_prompt def test_tool_instructions_do_not_change_tool_schema(tmp_path: Path) -> None: custom = ToolSpec( @@ -385,7 +406,7 @@ def test_skill_dirs_require_selected_skill_tools(tmp_path: Path) -> None: ) assert "skill_read" in [tool["name"] for tool in harness.tool_schemas()] with pytest.raises(ValueError, match="skill_read or skill_run"): - Harness(HarnessConfig(root=tmp_path, skills_dir=tmp_path / "skills", builtin_tools=["read"]), model=_fake_openai(FakeClient())) + Harness(HarnessConfig(root=tmp_path, skills_dir=tmp_path / "skills", builtin_tools=[]), model=_fake_openai(FakeClient())) def test_skills_are_not_discovered_without_explicit_skills_dir(tmp_path: Path) -> None: skill = tmp_path / ".agents" / "skills" / "demo" @@ -410,9 +431,10 @@ def test_selected_skills_are_exposed_when_skill_tool_is_selected(tmp_path: Path) root=tmp_path, skills_dir=tmp_path / "skills", selected_skills=["demo"], - builtin_tools=["read", "skill_read"], + builtin_tools=["skill_read"], ), model=_fake_openai(FakeClient()), + plugins=[FilesystemPlugin(tools=["read"])], ) assert [tool["name"] for tool in harness.tool_schemas()] == ["read", "skill_read"] @@ -442,7 +464,12 @@ def test_child_harness_tool_surfaces_follow_subagent_policy(tmp_path: Path) -> N def test_duplicate_tool_names_are_rejected(tmp_path: Path) -> None: duplicate = ToolSpec("read", "Duplicate read", {"type": "object", "properties": {}}, lambda args: "ok") with pytest.raises(ValueError, match="duplicate tool name: read"): - Harness(HarnessConfig(root=tmp_path), model=_fake_openai(FakeClient()), tools=[duplicate]) + Harness( + HarnessConfig(root=tmp_path), + model=_fake_openai(FakeClient()), + plugins=[FilesystemPlugin(tools=["read"])], + tools=[duplicate], + ) def test_after_tool_call_fires_for_handler_exception(tmp_path: Path) -> None: client = MultiCallClient([("boom", "{}")]) diff --git a/tests/unit/test_mcp.py b/tests/unit/test_mcp.py index 83e5571..6682d84 100644 --- a/tests/unit/test_mcp.py +++ b/tests/unit/test_mcp.py @@ -15,6 +15,7 @@ from thinharness import ( ApprovalDecision, + FilesystemPlugin, Harness, HarnessConfig, HarnessError, @@ -966,7 +967,11 @@ async def test_mcp_collision_detected_before_model_request(tmp_path, monkeypatch """MCP names collide with existing tools during connect.""" server = scripted_server(monkeypatch, {"read": _schema()}) client = MultiCallClient([]) - harness = Harness(HarnessConfig(root=tmp_path, mcp_servers=[server]), model=_fake_openai(client)) + harness = Harness( + HarnessConfig(root=tmp_path, mcp_servers=[server]), + model=_fake_openai(client), + plugins=[FilesystemPlugin(tools=["read"])], + ) with pytest.raises(HarnessError, match="tool name collision"): await harness.run("go") @@ -1343,8 +1348,8 @@ def rewrite(ctx) -> None: assert tool_span.attributes["mcp.tool.name"] == "remote" -async def test_connection_failure_in_run_fires_run_hooks(tmp_path) -> None: - """MCP connection failures happen inside the normal run lifecycle.""" +async def test_connection_failure_happens_before_run_hooks(tmp_path) -> None: + """Connection failures happen before the normal run lifecycle.""" events = [] tracer = FakeTracer() @@ -1367,5 +1372,5 @@ async def list_tools(self) -> list[ToolSpec]: with pytest.raises(MCPError, match="connect failed"): await harness.run("go") - assert events == ["start", "error"] - assert tracer.spans[0].exceptions + assert events == [] + assert tracer.spans == [] diff --git a/tests/unit/test_parallel_tools.py b/tests/unit/test_parallel_tools.py index 62edb49..478ceb5 100644 --- a/tests/unit/test_parallel_tools.py +++ b/tests/unit/test_parallel_tools.py @@ -13,6 +13,7 @@ ) from thinharness import ( + FilesystemPlugin, Harness, HarnessConfig, ToolSpec, @@ -147,8 +148,9 @@ def test_truncate_spill_files_do_not_collide_under_parallel_reads(tmp_path: Path (tmp_path / "b.txt").write_text(big, encoding="utf-8") client = MultiCallClient([("read", '{"path":"a.txt","max_chars":50}'), ("read", '{"path":"b.txt","max_chars":50}')]) harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", max_tool_chars=50, max_read_chars=50), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), + plugins=[FilesystemPlugin(max_tool_chars=50, max_read_chars=50, tools=["read"])], ) harness.run_sync("go") diff --git a/tests/unit/test_plugins.py b/tests/unit/test_plugins.py new file mode 100644 index 0000000..2f27470 --- /dev/null +++ b/tests/unit/test_plugins.py @@ -0,0 +1,295 @@ +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from pathlib import Path + +import pytest +from fakes import ScriptedModel, ScriptedSession, echo_tool + +from thinharness import ( + FilesystemPlugin, + Harness, + HarnessConfig, + Hook, + HookRegistry, + ModelTurn, + PluginBinding, + PluginContext, + PluginContribution, + ToolOrigin, + ToolSpec, +) + + +def _tool(name: str) -> ToolSpec: + return ToolSpec(name, name, {"type": "object", "properties": {}}, lambda _args: "ok") + + +class StaticPlugin: + def __init__(self, name: str, contribution: PluginContribution | None = None) -> None: + self.name = name + self.contribution = contribution or PluginContribution() + self.bindings = 0 + + def bind(self, context: PluginContext) -> PluginBinding: + self.bindings += 1 + return PluginBinding(static=self.contribution) + + +class InvalidBindingPlugin: + name = "invalid" + + def bind(self, context: PluginContext) -> object: + return object() + + +class ConnectedPlugin: + def __init__(self, name: str, events: list[str], contribution: PluginContribution, *, fail_first: bool = False) -> None: + self.name = name + self.events = events + self.contribution = contribution + self.fail_first = fail_first + self.attempts = 0 + + def bind(self, context: PluginContext) -> PluginBinding: + @asynccontextmanager + async def connect(): + self.attempts += 1 + self.events.append(f"enter:{self.name}") + if self.fail_first and self.attempts == 1: + raise RuntimeError(f"failed:{self.name}") + try: + yield self.contribution + finally: + self.events.append(f"exit:{self.name}") + + return PluginBinding(connect=connect) + + +def test_filesystem_plugin_is_explicit_static_and_ordered(tmp_path: Path) -> None: + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[FilesystemPlugin(tools=["search", "read", "jsonl_search"])], + tools=[echo_tool()], + ) + + assert [tool.name for tool in harness.tools] == ["search", "read", "jsonl_search", "echo"] + assert [tool.origin.plugin if tool.origin else None for tool in harness.tools] == ["filesystem", "filesystem", "filesystem", None] + assert f"Workspace root: {tmp_path.resolve()}" in harness.system_instructions() + + +def test_no_filesystem_plugin_has_no_tools_instruction_or_root_side_effect(tmp_path: Path) -> None: + missing = tmp_path / "missing" + harness = Harness(HarnessConfig(root=missing), model=ScriptedModel([])) + + assert harness.tools == [] + assert "Workspace root:" not in harness.system_instructions() + assert not missing.exists() + + with_plugin = Harness( + HarnessConfig(root=missing), + model=ScriptedModel([]), + plugins=[FilesystemPlugin()], + ) + assert [tool.name for tool in with_plugin.tools] == ["read", "write", "edit", "search", "list", "glob"] + read = next(tool for tool in with_plugin.tools if tool.name == "read") + search = next(tool for tool in with_plugin.tools if tool.name == "search") + read_result = read.handler(read.parse_args({"path": "missing.txt"})) + search_result = search.handler(search.parse_args({"query": "missing"})) + assert read_result.ok is False + assert search_result.ok is True + assert not missing.exists() + + +def test_filesystem_plugin_selection_validation(tmp_path: Path) -> None: + with pytest.raises(TypeError, match="ordered sequence"): + FilesystemPlugin(tools={"read"}) # type: ignore[arg-type] + with pytest.raises(ValueError, match="duplicate"): + FilesystemPlugin(tools=["read", "read"]) + with pytest.raises(ValueError, match="unknown FilesystemPlugin tool"): + Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[FilesystemPlugin(tools=["missing"])]) + + +def test_plugin_names_are_nonempty_and_unique(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="non-empty"): + Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[StaticPlugin("")]) + with pytest.raises(ValueError, match="duplicate plugin name: same"): + Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[StaticPlugin("same"), StaticPlugin("same")], + ) + with pytest.raises(TypeError, match="returned an invalid binding"): + Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[InvalidBindingPlugin()], # type: ignore[list-item] + ) + + +def test_plugin_origin_cannot_spoof_another_plugin(tmp_path: Path) -> None: + tool = ToolSpec( + "owned", + "owned", + {"type": "object", "properties": {}}, + lambda _args: "ok", + origin=ToolOrigin(plugin="spoofed", source="remote", attributes={"key": "value"}), + ) + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[StaticPlugin("actual", PluginContribution(tools=(tool,)))], + ) + + assert harness.tools[0].origin == ToolOrigin(plugin="actual", source="remote", attributes={"key": "value"}) + + +def test_duplicate_tool_error_names_both_origins(tmp_path: Path) -> None: + with pytest.raises(ValueError, match=r"duplicate tool name: shared \(first and second\)"): + Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[ + StaticPlugin("first", PluginContribution(tools=(_tool("shared"),))), + StaticPlugin("second", PluginContribution(tools=(_tool("shared"),))), + ], + ) + + +async def test_plugins_connect_before_first_run_hook_and_close_in_reverse(tmp_path: Path) -> None: + events: list[str] = [] + connected_hook = Hook("run_start", lambda _ctx: events.append("hook:connected")) + first = ConnectedPlugin("first", events, PluginContribution(hooks=(connected_hook,))) + second = ConnectedPlugin("second", events, PluginContribution(tools=(_tool("dynamic"),))) + session = ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"})) + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([session]), + plugins=[first, second], + hooks=[Hook("run_start", lambda _ctx: events.append("hook:direct"))], + ) + + assert "dynamic" not in [tool.name for tool in harness.tools] + assert (await harness.run("go")).text == "done" + assert events == ["enter:first", "enter:second", "hook:direct", "hook:connected"] + assert "dynamic" in [tool.name for tool in harness.tools] + + await harness.aclose() + assert events[-2:] == ["exit:second", "exit:first"] + + +async def test_concurrent_connect_enters_each_binding_once(tmp_path: Path) -> None: + events: list[str] = [] + plugin = ConnectedPlugin("connected", events, PluginContribution()) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[plugin]) + + await asyncio.gather(harness.connect(), harness.connect(), harness.connect()) + + assert plugin.attempts == 1 + await harness.aclose() + + +async def test_connection_failure_rolls_back_and_retry_is_clean(tmp_path: Path) -> None: + events: list[str] = [] + first = ConnectedPlugin("first", events, PluginContribution(tools=(_tool("first_tool"),))) + failing = ConnectedPlugin("failing", events, PluginContribution(), fail_first=True) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[first, failing]) + + with pytest.raises(RuntimeError, match="failed:failing"): + await harness.connect() + assert events == ["enter:first", "enter:failing", "exit:first"] + assert harness.tools == [] + + await harness.connect() + assert [tool.name for tool in harness.tools] == ["first_tool"] + assert first.attempts == 2 + await harness.aclose() + + +async def test_connection_cancellation_rolls_back_without_run_hooks(tmp_path: Path) -> None: + events: list[str] = [] + first = ConnectedPlugin("first", events, PluginContribution(tools=(_tool("first_tool"),))) + + class CancellingPlugin: + name = "cancel" + + def bind(self, context: PluginContext) -> PluginBinding: + @asynccontextmanager + async def connect(): + events.append("enter:cancel") + raise asyncio.CancelledError + yield PluginContribution() # pragma: no cover + + return PluginBinding(connect=connect) + + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[first, CancellingPlugin()], + hooks=[Hook("run_start", lambda _ctx: events.append("hook:run_start"))], + ) + + with pytest.raises(asyncio.CancelledError): + await harness.run("go") + + assert events == ["enter:first", "enter:cancel", "exit:first"] + assert harness.tools == [] + + +async def test_invalid_dynamic_contribution_is_not_committed(tmp_path: Path) -> None: + events: list[str] = [] + plugin = ConnectedPlugin("bad", events, PluginContribution(tools=(_tool("subagent"),))) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[plugin]) + + with pytest.raises(ValueError, match="reserved tool name"): + await harness.connect() + + assert harness.tools == [] + assert events == ["enter:bad", "exit:bad"] + + +async def test_invalid_dynamic_hook_filter_is_not_committed(tmp_path: Path) -> None: + events: list[str] = [] + hook = Hook("before_subagent_run", lambda _ctx: None, agents=["missing"]) + plugin = ConnectedPlugin("bad-hook", events, PluginContribution(hooks=(hook,))) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[plugin]) + + with pytest.raises(ValueError, match="unknown subagent name"): + await harness.connect() + + assert harness.hooks.hooks == [] + assert events == ["enter:bad-hook", "exit:bad-hook"] + + +def test_caller_hook_registry_is_copied(tmp_path: Path) -> None: + direct = Hook("run_start", lambda _ctx: None) + contributed = Hook("run_start", lambda _ctx: None) + registry = HookRegistry([direct], strict_hooks=True) + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[StaticPlugin("hooks", PluginContribution(hooks=(contributed,)))], + hooks=registry, + ) + + assert registry.hooks == [direct] + assert harness.hooks.hooks == [contributed, direct] + assert harness.hooks.strict_hooks is True + + +def test_one_plugin_object_binds_independently_to_two_harnesses(tmp_path: Path) -> None: + plugin = StaticPlugin("shared", PluginContribution(tools=(_tool("shared_tool"),))) + first = Harness(HarnessConfig(root=tmp_path / "one"), model=ScriptedModel([]), plugins=[plugin]) + second = Harness(HarnessConfig(root=tmp_path / "two"), model=ScriptedModel([]), plugins=[plugin]) + + assert plugin.bindings == 2 + assert first.tools[0] is not second.tools[0] + + +def test_core_does_not_import_filesystem_implementation() -> None: + source = Path("thinharness/core.py").read_text(encoding="utf-8") + + assert "tools.filesystem" not in source + assert "plugins.filesystem" not in source diff --git a/tests/unit/test_resume.py b/tests/unit/test_resume.py index 569d1ef..a2d7b5c 100644 --- a/tests/unit/test_resume.py +++ b/tests/unit/test_resume.py @@ -11,6 +11,7 @@ from thinharness import ( AnthropicMessagesModel, + FilesystemPlugin, Harness, HarnessConfig, HarnessError, @@ -56,7 +57,11 @@ async def create_message(self, payload): async def test_openai_resume_full_replays_transcript_for_followup(tmp_path: Path) -> None: (tmp_path / "hello.txt").write_text("hello", encoding="utf-8") client = FakeClient() - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=["read"]), model=OpenAIResponsesModel("gpt-test", provider=client)) + harness = Harness( + HarnessConfig(root=tmp_path), + model=OpenAIResponsesModel("gpt-test", provider=client), + plugins=[FilesystemPlugin(tools=["read"])], + ) first = await harness.run("first") state = json.loads(json.dumps(first.resume_state)) diff --git a/tests/unit/test_subagents.py b/tests/unit/test_subagents.py index d62bdc5..9f5277d 100644 --- a/tests/unit/test_subagents.py +++ b/tests/unit/test_subagents.py @@ -18,6 +18,7 @@ DEFAULT_SUBAGENT_NAME, AfterSubagentRunContext, BeforeSubagentRunContext, + FilesystemPlugin, Harness, HarnessConfig, Hook, @@ -51,7 +52,7 @@ def test_subagent_config_validation_accepts_tool_specs() -> None: assert config.tools == [spec, sequential_tool] assert inherited.inherit_parent_tools is True with pytest.raises(ValueError, match="inherit_parent_tools"): - SubAgentConfig(name="bad", description="Bad helper.", inherit_parent_tools=True, builtin_tools=["read"]) + SubAgentConfig(name="bad", description="Bad helper.", inherit_parent_tools=True, plugins=[FilesystemPlugin(tools=["read"])]) with pytest.raises(ValueError, match="cannot be exposed"): SubAgentConfig(name="recursive", description="Recursive helper.", builtin_tools=["subagent"]) with pytest.raises(ValueError, match="cannot be exposed"): @@ -63,13 +64,13 @@ def test_subagent_config_validation_accepts_tool_specs() -> None: with pytest.raises(ValueError, match="must define"): SubAgentConfig(name="empty", description="No tools.") with pytest.raises(ValueError): - SubAgentConfig(name="bad name", description="Bad helper.", builtin_tools=["read"]) + SubAgentConfig(name="bad name", description="Bad helper.", plugins=[FilesystemPlugin(tools=["read"])]) with pytest.raises(ValueError, match="non-empty single line"): - SubAgentConfig(name="ok", description=" ", builtin_tools=["read"]) + SubAgentConfig(name="ok", description=" ", plugins=[FilesystemPlugin(tools=["read"])]) with pytest.raises(ValueError, match="non-empty single line"): - SubAgentConfig(name="ok", description="Bad\nhelper.", builtin_tools=["read"]) + SubAgentConfig(name="ok", description="Bad\nhelper.", plugins=[FilesystemPlugin(tools=["read"])]) with pytest.raises(ValueError, match="SubAgentConfig.background has been removed"): - SubAgentConfig(name="old-background", description="Old helper.", builtin_tools=["read"], background="always") + SubAgentConfig(name="old-background", description="Old helper.", plugins=[FilesystemPlugin(tools=["read"])], background="always") def test_subagent_builtin_exposure_is_selectable(tmp_path: Path) -> None: default = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([])) @@ -406,7 +407,7 @@ def on_parent_continue(outputs, _tools, _metadata): def test_unknown_named_subagent_returns_structured_error(tmp_path: Path) -> None: harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([])) - tool = create_subagent_tool(harness, [SubAgentConfig(name="research", description="Research helper.", builtin_tools=["read"])]) + tool = create_subagent_tool(harness, [SubAgentConfig(name="research", description="Research helper.", plugins=[FilesystemPlugin(tools=["read"])])]) output = tool_output(asyncio.run(tool.handler(tool.parse_args({"task": "x", "agent": "missing"}))).as_json()) @@ -552,4 +553,4 @@ def cancel(ctx): def test_default_subagent_name_is_reserved() -> None: with pytest.raises(ValueError, match="reserved"): - SubAgentConfig(name=DEFAULT_SUBAGENT_NAME, description="Reserved.", builtin_tools=["read"]) + SubAgentConfig(name=DEFAULT_SUBAGENT_NAME, description="Reserved.", plugins=[FilesystemPlugin(tools=["read"])]) diff --git a/tests/unit/test_tool_retry.py b/tests/unit/test_tool_retry.py index f613f8b..d403ebe 100644 --- a/tests/unit/test_tool_retry.py +++ b/tests/unit/test_tool_retry.py @@ -9,6 +9,7 @@ from thinharness import ( AfterToolCallContext, + FileTools, Harness, HarnessConfig, HarnessError, @@ -18,7 +19,6 @@ ToolSpec, TracingOptions, build_child_harness, - builtin_tools, call_tool, ) from thinharness.providers import ModelToolCall, ModelTurn @@ -134,7 +134,7 @@ def handler(args): def test_builtin_validation_failure_is_retryable(tmp_path: Path) -> None: - read = next(tool for tool in builtin_tools(tmp_path) if tool.name == "read") + read = next(tool for tool in FileTools(tmp_path).specs() if tool.name == "read") output = tool_output(call_tool(read, '{"path":"missing.txt","limit":0}')) diff --git a/tests/unit/test_tracing.py b/tests/unit/test_tracing.py index 281f285..d46f958 100644 --- a/tests/unit/test_tracing.py +++ b/tests/unit/test_tracing.py @@ -21,6 +21,7 @@ from pydantic import BaseModel from thinharness import ( + FilesystemPlugin, Harness, HarnessConfig, HarnessError, @@ -52,6 +53,7 @@ def test_harness_tracing_records_agent_model_and_tool_spans(tmp_path: Path) -> N harness = Harness( HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(FakeClient()), + plugins=[FilesystemPlugin(tools=["read"])], tracing=[TracingOptions( tracer=tracer, agent_name="test-agent", @@ -149,6 +151,7 @@ def test_local_tracing_writes_full_jsonl_trace(tmp_path: Path, monkeypatch: pyte local_trace_dir=trace_dir, ), model=_fake_openai(FakeClient()), + plugins=[FilesystemPlugin(tools=["read"])], ) result = harness.run_sync("read hello") @@ -209,6 +212,7 @@ def test_local_tracing_does_not_change_remote_capture_policy(tmp_path: Path, mon harness = Harness( HarnessConfig(root=tmp_path, model="openai:test-model", local_trace_dir=trace_dir), model=_fake_openai(FakeClient()), + plugins=[FilesystemPlugin(tools=["read"])], tracing=[TracingOptions(tracer=remote, capture_messages=False, capture_tool_args=False, capture_tool_results=False)], ) @@ -243,6 +247,7 @@ def test_capture_messages_false_omits_content_attributes(tmp_path: Path) -> None harness = Harness( HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(FakeClient()), + plugins=[FilesystemPlugin(tools=["read"])], tracing=[TracingOptions(tracer=tracer, capture_messages=False, capture_tool_args=True, capture_tool_results=True)], ) @@ -580,7 +585,13 @@ def on_parent_continue(outputs, _tools, _metadata): model=ScriptedModel([parent]), tracing=[TracingOptions(tracer=tracer)], ) - harness.add_tool(create_subagent_tool(harness, [SubAgentConfig(name="research", description="Research helper.", builtin_tools=["read"])])) + harness.add_tool(create_subagent_tool(harness, [ + SubAgentConfig( + name="research", + description="Research helper.", + plugins=[FilesystemPlugin(tools=["read"])], + ) + ])) assert harness.run_sync("delegate").text == "parent done" subagent_tool = next(span for span in tracer.spans if span.name == "execute_tool subagent") diff --git a/thinharness/__init__.py b/thinharness/__init__.py index 1857e58..0d86cb7 100644 --- a/thinharness/__init__.py +++ b/thinharness/__init__.py @@ -1,4 +1,4 @@ -"""Public API for the filesystem harness.""" +"""Public interface for ThinHarness.""" from importlib.metadata import PackageNotFoundError from importlib.metadata import version as _metadata_version @@ -37,6 +37,7 @@ UserPromptSubmitContext, ) from .output import NativeOutput, OutputSchema, PromptedOutput, TextOutput, ToolStructuredOutput +from .plugins import FilesystemPlugin, Plugin, PluginBinding, PluginConnector, PluginContext, PluginContribution from .providers import ( AnthropicMessagesModel, AnthropicProvider, @@ -81,9 +82,9 @@ Skill, SkillRegistry, ToolEnvelope, + ToolOrigin, ToolResult, ToolSpec, - builtin_tools, call_tool, contained_path, create_parallel_llm_tool, @@ -101,6 +102,7 @@ "BashArgs", "BashTool", "FileTools", + "FilesystemPlugin", "FilePromptSource", "InlinePromptSource", "Harness", @@ -175,8 +177,14 @@ "ToolStructuredOutput", "PathPolicy", "PathValidationError", + "Plugin", + "PluginBinding", + "PluginConnector", + "PluginContext", + "PluginContribution", "McpToolInfo", "ToolEnvelope", + "ToolOrigin", "ParallelLlmArgs", "ParallelLlmTool", "ToolResult", @@ -186,7 +194,6 @@ "OtlpTracing", "TracingOptions", "build_child_harness", - "builtin_tools", "call_tool", "contained_path", "create_parallel_llm_tool", diff --git a/thinharness/core.py b/thinharness/core.py index f66e67a..f092b9e 100644 --- a/thinharness/core.py +++ b/thinharness/core.py @@ -5,6 +5,7 @@ import asyncio import os from contextlib import AsyncExitStack +from dataclasses import replace from pathlib import Path from typing import Any, Literal, cast @@ -43,6 +44,7 @@ resolve_output_schema_for_model, structured_instructions, ) +from .plugins.base import Plugin, PluginBinding, PluginContext, PluginContribution from .providers import ( Model, ModelSession, @@ -55,8 +57,7 @@ model_capabilities, ) from .subagents import DEFAULT_SUBAGENT_NAME, SubAgentConfig, create_subagent_tool -from .tools.base import ToolSpec -from .tools.filesystem import builtin_tools as make_builtin_tools +from .tools.base import ToolOrigin, ToolSpec from .tools.mcp import MCPServer from .tools.parallel_llm import create_parallel_llm_tool from .tools.skills import SkillRegistry @@ -70,8 +71,6 @@ from .turns import TurnStart, advance_until_terminal from .types import ApprovalDecision, HarnessError, HarnessResult, Json, PendingApproval, RunUsage, UnexpectedModelBehavior -DEFAULT_BUILTIN_TOOLS = {"read", "write", "edit", "search", "list", "glob"} - def _local_tracing_enabled(configured: bool) -> bool: """Return whether local plaintext tracing should be active.""" @@ -114,19 +113,12 @@ class HarnessConfig(BaseModel): skills_dir: str | Path | list[str | Path] | None = None selected_skills: list[str] | None = None builtin_tools: list[str] | None = None - output_dir: str | Path | None = None max_model_requests: int = 64 max_tool_calls: int | None = None strict_hooks: bool = False request_timeout: int = 120 request_retries: int = Field(default=3, ge=0, le=10) request_retry_backoff: float = Field(default=1.0, ge=0, allow_inf_nan=False) - max_read_chars: int = 40_000 - max_read_bytes: int = 1_000_000 - max_tool_chars: int = 40_000 - max_search_line_chars: int = 180 - rg_timeout: int = 30 - search_exclude_globs: list[str] = Field(default_factory=list) read_paths: list[str | Path] | None = None write_paths: list[str | Path] | None = None temperature: float | None = None @@ -156,13 +148,14 @@ def validate_config(self) -> HarnessConfig: class Harness: - """A non-interactive filesystem agent harness for SDK use.""" + """A non-interactive agent harness for SDK use.""" def __init__( self, config: HarnessConfig | None = None, *, model: Model | None = None, + plugins: list[Plugin] | None = None, tools: list[ToolSpec] | None = None, tracing: list[TracingOptions] | None = None, skills: SkillRegistry | None = None, @@ -176,7 +169,6 @@ def __init__( if skills is not None and (self.config.skills_dir is not None or self.config.selected_skills is not None): raise ValueError("skills cannot be combined with skills_dir or selected_skills") self.root = Path(self.config.root).expanduser().resolve() - self.root.mkdir(parents=True, exist_ok=True) self.model_ref = os.getenv("HARNESS_MODEL", self.config.model) self.model = model or infer_model( self.model_ref, @@ -194,26 +186,35 @@ def __init__( self.model_capabilities = model_capabilities(self.model) self.skills = skills or SkillRegistry(self.config.skills_dir, selected_skills=self.config.selected_skills) output_schema = resolve_output_schema_for_model(self.model, self.config.output_type, self.config.output_mode) - filesystem_tools = make_builtin_tools( - self.root, - output_dir=self.config.output_dir, - max_read_chars=self.config.max_read_chars, - max_read_bytes=self.config.max_read_bytes, - max_tool_chars=self.config.max_tool_chars, - max_search_line_chars=self.config.max_search_line_chars, - rg_timeout=self.config.rg_timeout, - search_exclude_globs=self.config.search_exclude_globs, - read_paths=self.config.read_paths, - write_paths=self.config.write_paths, - ) + self.output_schema = output_schema + + configured_plugins = tuple(plugins or []) + plugin_names = [plugin.name for plugin in configured_plugins] + if any(not name.strip() for name in plugin_names): + raise ValueError("plugin name must be non-empty") + duplicate_plugin = next((name for index, name in enumerate(plugin_names) if name in plugin_names[:index]), None) + if duplicate_plugin is not None: + raise ValueError(f"duplicate plugin name: {duplicate_plugin}") + bindings = tuple(plugin.bind(PluginContext(root=self.root)) for plugin in configured_plugins) + for plugin, binding in zip(configured_plugins, bindings, strict=True): + if not isinstance(binding, PluginBinding): + raise TypeError(f"plugin {plugin.name!r} returned an invalid binding") + static_tools: list[ToolSpec] = [] + static_instructions: list[str] = [] + static_hooks: list[Hook] = [] + for plugin, binding in zip(configured_plugins, bindings, strict=True): + contribution = self._normalize_contribution(plugin.name, binding.static) + static_tools.extend(contribution.tools) + static_instructions.extend(contribution.instructions) + static_hooks.extend(contribution.hooks) + builtin_candidates = [ - *filesystem_tools, *self.skills.specs(), create_subagent_tool(self, self.config.subagents), create_parallel_llm_tool(self), ] builtin = self._select_builtin_tools(builtin_candidates, self.config.builtin_tools) - configured_tools = [*builtin, *(tools or [])] + configured_tools = [*static_tools, *builtin, *(tools or [])] self._validate_tool_list( configured_tools, output_schema=output_schema, @@ -221,19 +222,28 @@ def __init__( is_child_run=self._is_child_run, ) tool_map = {tool.name: tool for tool in configured_tools} - hook_registry = hooks if isinstance(hooks, HookRegistry) else HookRegistry(hooks, strict_hooks=self.config.strict_hooks) + caller_hooks = list(hooks.hooks) if isinstance(hooks, HookRegistry) else list(hooks or []) + strict_hooks = hooks.strict_hooks if isinstance(hooks, HookRegistry) else self.config.strict_hooks + hook_registry = HookRegistry([*static_hooks, *caller_hooks], strict_hooks=strict_hooks) self._validate_hook_registry(hook_registry, self.config.subagents) self._validate_skill_tool_selection_for(self.skills, configured_tools) + self.plugins = configured_plugins + self._plugin_bindings = bindings + self._base_tools = list(configured_tools) + self._base_instructions = list(static_instructions) + self._strict_hooks = strict_hooks self.tools = configured_tools self._tool_map = tool_map - self.output_schema = output_schema + self._plugin_instructions = list(static_instructions) self.hooks = hook_registry self.subagent_hooks = subagent_hooks or {} self._mcp_servers = list(self.config.mcp_servers) self._resolve_mcp_server_ids() self._mcp_stack: AsyncExitStack | None = None - self._mcp_connected = False + self._plugin_stack: AsyncExitStack | None = None + self._connected = False + self._connect_lock = asyncio.Lock() self._skills_enabled = bool(self.skills.skills) and any(tool.name in {"skill_read", "skill_run"} for tool in self.tools) self.local_tracing: LocalTracing | None = None external_tracing = list(self.config.tracing if tracing is None else tracing) @@ -391,6 +401,7 @@ async def _run_streaming( from .tool_execution import ToolBatchExecutor try: + await self._ensure_connected() run_tracer = RunTracer(self.tracing) approval_pause: ApprovalPause | None = None approval_decision_map: dict[str, ApprovalDecision] | None = None @@ -434,7 +445,7 @@ async def _run_streaming( **run_ctx.stream_base(), decisions=tuple(approval_decisions or []), )) - except Exception as exc: + except BaseException as exc: self._running = False emitter.emit(RunFailedEvent( run_id=stream_context.run_id, @@ -442,7 +453,7 @@ async def _run_streaming( parent_run_id=stream_context.parent_run_id, parent_tool_call_id=stream_context.parent_tool_call_id, agent_name=stream_context.agent_name, - stop_reason="error", + stop_reason="cancelled" if isinstance(exc, asyncio.CancelledError) else "error", error_type=type(exc).__name__, message=str(exc), )) @@ -550,7 +561,6 @@ async def _prepare_run_start( max_model_requests=self.config.max_model_requests, max_tool_calls=self.config.max_tool_calls, )) - await self._ensure_mcp_connected() effective_prompt = prompt if not skip_user_prompt: prompt_ctx = UserPromptSubmitContext(harness=self, metadata=dict(run_metadata), prompt=prompt) @@ -608,14 +618,17 @@ async def _run_and_close() -> HarnessResult: return asyncio.run(_run_and_close()) async def aclose(self) -> None: - """Close MCP servers and owned provider HTTP clients.""" + """Close connected plugins, MCP servers, and an owned model.""" if self._closed: return try: if self._mcp_stack is not None: await self._mcp_stack.aclose() self._mcp_stack = None - self._mcp_connected = False + if self._plugin_stack is not None: + await self._plugin_stack.aclose() + self._plugin_stack = None + self._connected = False if self._owns_model: aclose = getattr(self.model.provider, "aclose", None) if aclose is not None: @@ -644,6 +657,8 @@ def add_tool(self, tool: ToolSpec) -> None: raise ValueError(f"duplicate tool name: {spec.name}") self.tools.append(spec) self._tool_map[spec.name] = spec + if not self._connected: + self._base_tools.append(spec) self._validate_hook_filters() def tool_schemas(self) -> list[Json]: @@ -655,7 +670,7 @@ def tool_schemas(self) -> list[Json]: def system_instructions(self) -> str: """Return the full instruction text sent to the model.""" - parts = [self.config.system_prompt, f"Workspace root: {self.root}"] + parts = [self.config.system_prompt, *self._plugin_instructions] if self._skills_enabled: skill_summary = self.skills.prompt_summary() if skill_summary: @@ -680,11 +695,14 @@ def _tool_max_retries(self, name: str) -> int: @staticmethod def _validate_unique_tools(tools: list[ToolSpec]) -> None: """Reject duplicate tool names before sending schemas to a provider.""" - seen: set[str] = set() + seen: dict[str, ToolSpec] = {} for tool in tools: - if tool.name in seen: - raise ValueError(f"duplicate tool name: {tool.name}") - seen.add(tool.name) + previous = seen.get(tool.name) + if previous is not None: + first = previous.origin.plugin if previous.origin is not None else "direct" + second = tool.origin.plugin if tool.origin is not None else "direct" + raise ValueError(f"duplicate tool name: {tool.name} ({first} and {second})") + seen[tool.name] = tool @classmethod def _validate_tool_list( @@ -781,22 +799,85 @@ def _resolve_mcp_server_ids(self) -> None: server.resolve_id(counts) async def connect(self) -> None: - """Open MCP server connections and discover their tools.""" + """Open connected plugins and the temporary MCP bridge.""" if self._closed: raise HarnessError("harness is closed") - await self._ensure_mcp_connected() + await self._ensure_connected() - async def _ensure_mcp_connected(self) -> None: - """Connect MCP servers and append their discovered tools once.""" - if self._mcp_connected: + async def _ensure_connected(self) -> None: + """Connect every dynamic contribution once and commit it atomically.""" + if self._connected: return + async with self._connect_lock: + if self._connected: + return + plugin_stack = AsyncExitStack() + mcp_stack: AsyncExitStack | None = None + base_hooks = list(self.hooks.hooks) + try: + dynamic_tools: list[ToolSpec] = [] + dynamic_instructions: list[str] = [] + dynamic_hooks: list[Hook] = [] + for plugin, binding in zip(self.plugins, self._plugin_bindings, strict=True): + if binding.connect is None: + continue + contribution = await plugin_stack.enter_async_context(binding.connect()) + normalized = self._normalize_contribution(plugin.name, contribution) + dynamic_tools.extend(normalized.tools) + dynamic_instructions.extend(normalized.instructions) + dynamic_hooks.extend(normalized.hooks) + + candidate_tools = [*self._base_tools, *dynamic_tools] + self._validate_tool_list( + candidate_tools, + output_schema=self.output_schema, + model_supports_approval_resume=self._model_supports_approval_resume(), + is_child_run=self._is_child_run, + ) + candidate_hooks = HookRegistry([*base_hooks, *dynamic_hooks], strict_hooks=self._strict_hooks) + self._validate_hook_registry(candidate_hooks, self.config.subagents) + self._validate_skill_tool_selection_for(self.skills, candidate_tools) + + mcp_stack, mcp_tools = await self._open_mcp_tools(candidate_tools) + all_tools = [*candidate_tools, *mcp_tools] + self._validate_tool_list( + all_tools, + output_schema=self.output_schema, + model_supports_approval_resume=self._model_supports_approval_resume(), + is_child_run=self._is_child_run, + ) + + self.tools = all_tools + self._tool_map = {tool.name: tool for tool in all_tools} + self._plugin_instructions = [*self._base_instructions, *dynamic_instructions] + self.hooks = candidate_hooks + self._skills_enabled = bool(self.skills.skills) and any( + tool.name in {"skill_read", "skill_run"} for tool in self.tools + ) + self._plugin_stack = plugin_stack + self._mcp_stack = mcp_stack + self._connected = True + except BaseException: + if mcp_stack is not None: + await mcp_stack.aclose() + await plugin_stack.aclose() + self.tools = list(self._base_tools) + self._tool_map = {tool.name: tool for tool in self.tools} + self._plugin_instructions = list(self._base_instructions) + self.hooks = HookRegistry(base_hooks, strict_hooks=self._strict_hooks) + self._skills_enabled = bool(self.skills.skills) and any( + tool.name in {"skill_read", "skill_run"} for tool in self.tools + ) + raise + + async def _open_mcp_tools(self, existing_tools: list[ToolSpec]) -> tuple[AsyncExitStack | None, list[ToolSpec]]: + """Open the temporary MCP bridge and stage its discovered tools.""" if not self._mcp_servers: - self._mcp_connected = True - return + return None, [] stack = AsyncExitStack() try: mcp_tools: list[ToolSpec] = [] - seen = set(self._tool_map) + seen = {tool.name for tool in existing_tools} if self.output_schema is not None and self.output_schema.mode == "tool": seen.add(FINAL_RESULT_TOOL_NAME) for server in self._mcp_servers: @@ -809,14 +890,30 @@ async def _ensure_mcp_connected(self) -> None: self._validate_tool_approval_policy(tool) seen.add(tool.name) mcp_tools.append(tool) - self.tools.extend(mcp_tools) - self._tool_map.update({tool.name: tool for tool in mcp_tools}) - self._mcp_stack = stack - self._mcp_connected = True + return stack, mcp_tools except BaseException: await stack.aclose() raise + @staticmethod + def _normalize_contribution(plugin_name: str, contribution: PluginContribution) -> PluginContribution: + """Validate contribution values and stamp missing tool provenance.""" + if not isinstance(contribution, PluginContribution): + raise TypeError(f"plugin {plugin_name!r} returned an invalid contribution") + instructions = tuple(instruction for instruction in contribution.instructions if instruction.strip()) + tools = tuple( + replace( + tool, + origin=ToolOrigin( + plugin=plugin_name, + source=tool.origin.source if tool.origin is not None else tool.name, + attributes=dict(tool.origin.attributes) if tool.origin is not None else {}, + ), + ) + for tool in contribution.tools + ) + return PluginContribution(tools=tools, instructions=instructions, hooks=tuple(contribution.hooks)) + def _structured_output_request(self) -> StructuredOutputRequest | None: """Return native structured-output request metadata.""" if self.output_schema is None: @@ -828,13 +925,16 @@ def _select_builtin_tools(tools: list[ToolSpec], selected_names: list[str] | Non """Return all or the explicitly selected built-in tools.""" by_name = {tool.name: tool for tool in tools} if selected_names is None: - return [tool for tool in tools if tool.name in DEFAULT_BUILTIN_TOOLS] + return [] selected: list[ToolSpec] = [] seen: set[str] = set() for name in selected_names: if name in seen: raise ValueError(f"duplicate selected builtin tool: {name}") if name not in by_name: + filesystem_names = {"read", "write", "edit", "search", "list", "glob", "jsonl_search"} + if name in filesystem_names: + raise ValueError(f"unknown builtin tool: {name}; use FilesystemPlugin(tools=[{name!r}])") available = ", ".join(sorted(by_name)) or "none" raise ValueError(f"unknown builtin tool: {name}; available: {available}") selected.append(by_name[name]) diff --git a/thinharness/defaults.py b/thinharness/defaults.py index 7f81fc8..a2cbb46 100644 --- a/thinharness/defaults.py +++ b/thinharness/defaults.py @@ -1,15 +1,8 @@ """Shared defaults for thinharness.""" -DEFAULT_SYSTEM_PROMPT = """You are a filesystem automation agent working inside the workspace root. +DEFAULT_SYSTEM_PROMPT = """You are a focused automation agent. -Use search to find relevant text, filenames, and repeated patterns. -Use read to inspect files before editing. -Use edit for targeted replacements and write for creating or replacing files. -Start narrow, broaden only if needed, and use offset/limit when a file is large or you only need a known section. -Prefer batching independent tool calls in one assistant turn. When several reads, searches, listings, or other inspections do not -depend on each other's results, emit them together instead of waiting between calls. -When making edits, batch independent replacements into one edit call, order dependent replacements deliberately within the edits list, -and after any per-edit failure, retry only the failed items after reading the per-edit results. +Use the available tools when they help complete the task. Prefer batching independent tool calls in one assistant turn instead of waiting between calls. When finished, respond concisely with what changed and any verification run.""" diff --git a/thinharness/plugins/__init__.py b/thinharness/plugins/__init__.py new file mode 100644 index 0000000..93d830d --- /dev/null +++ b/thinharness/plugins/__init__.py @@ -0,0 +1,13 @@ +"""Built-in plugin contracts and adapters.""" + +from .base import Plugin, PluginBinding, PluginConnector, PluginContext, PluginContribution +from .filesystem import FilesystemPlugin + +__all__ = [ + "FilesystemPlugin", + "Plugin", + "PluginBinding", + "PluginConnector", + "PluginContext", + "PluginContribution", +] diff --git a/thinharness/plugins/base.py b/thinharness/plugins/base.py new file mode 100644 index 0000000..4a40431 --- /dev/null +++ b/thinharness/plugins/base.py @@ -0,0 +1,57 @@ +"""Plugin composition contracts.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator, Callable +from contextlib import AbstractAsyncContextManager, asynccontextmanager +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Protocol, runtime_checkable + +if TYPE_CHECKING: + from ..hooks import Hook + from ..tools.base import ToolSpec + + +@dataclass(frozen=True) +class PluginContext: + """Stable core context available while binding one plugin.""" + + root: Path + + +@dataclass(frozen=True) +class PluginContribution: + """Tools, instructions, and hooks contributed by one plugin.""" + + tools: tuple[ToolSpec, ...] = () + instructions: tuple[str, ...] = () + hooks: tuple[Hook, ...] = () + + +PluginConnector = Callable[[], AbstractAsyncContextManager[PluginContribution]] + + +@dataclass(frozen=True) +class PluginBinding: + """Static plugin state and its optional connected contribution.""" + + static: PluginContribution = field(default_factory=PluginContribution) + connect: PluginConnector | None = None + + +@runtime_checkable +class Plugin(Protocol): + """Configured plugin that can bind independently to a harness.""" + + name: str + + def bind(self, context: PluginContext) -> PluginBinding: + """Bind this plugin without file or network I/O.""" + ... + + +@asynccontextmanager +async def empty_connector() -> AsyncIterator[PluginContribution]: + """Return an empty connected contribution.""" + yield PluginContribution() diff --git a/thinharness/plugins/filesystem.py b/thinharness/plugins/filesystem.py new file mode 100644 index 0000000..352acf9 --- /dev/null +++ b/thinharness/plugins/filesystem.py @@ -0,0 +1,68 @@ +"""Filesystem plugin.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import replace +from pathlib import Path + +from ..tools.base import ToolOrigin +from ..tools.filesystem import FileTools +from .base import PluginBinding, PluginContext, PluginContribution + +_DEFAULT_TOOLS = ("read", "write", "edit", "search", "list", "glob") + + +class FilesystemPlugin: + """Provide root-scoped filesystem tools to one harness.""" + + name = "filesystem" + + def __init__( + self, + *, + tools: Sequence[str] | None = None, + output_dir: str | Path | None = None, + max_read_chars: int = 40_000, + max_read_bytes: int = 1_000_000, + max_tool_chars: int = 40_000, + max_search_line_chars: int = 180, + rg_timeout: int = 30, + search_exclude_globs: list[str] | None = None, + read_paths: Sequence[str | Path] | None = None, + write_paths: Sequence[str | Path] | None = None, + ) -> None: + if isinstance(tools, (set, frozenset)): + raise TypeError("FilesystemPlugin tools must be an ordered sequence, not a set") + selected = tuple(_DEFAULT_TOOLS if tools is None else tools) + if len(set(selected)) != len(selected): + raise ValueError("FilesystemPlugin tools contains a duplicate name") + self._selected = selected + self._options = { + "output_dir": output_dir, + "max_read_chars": max_read_chars, + "max_read_bytes": max_read_bytes, + "max_tool_chars": max_tool_chars, + "max_search_line_chars": max_search_line_chars, + "rg_timeout": rg_timeout, + "search_exclude_globs": search_exclude_globs, + "read_paths": read_paths, + "write_paths": write_paths, + } + + def bind(self, context: PluginContext) -> PluginBinding: + """Build static tool specifications without filesystem I/O.""" + collection = FileTools(context.root, **self._options) + by_name = {tool.name: tool for tool in collection.specs()} + unknown = [name for name in self._selected if name not in by_name] + if unknown: + available = ", ".join(by_name) + raise ValueError(f"unknown FilesystemPlugin tool: {unknown[0]}; available: {available}") + specs = tuple( + replace(by_name[name], origin=ToolOrigin(plugin=self.name, source=name)) + for name in self._selected + ) + return PluginBinding(static=PluginContribution( + tools=specs, + instructions=(f"Workspace root: {context.root}",), + )) diff --git a/thinharness/subagents.py b/thinharness/subagents.py index 6b93c2a..435c986 100644 --- a/thinharness/subagents.py +++ b/thinharness/subagents.py @@ -10,6 +10,7 @@ from .defaults import DEFAULT_SYSTEM_PROMPT from .events import RunCompletedEvent, current_stream_emitter from .hooks import AfterSubagentRunContext, BeforeSubagentRunContext, HookRegistry, current_tool_call_context, current_tool_runtime_context +from .plugins.base import Plugin from .providers import infer_model, same_provider_model_ref from .tools.base import Json, ToolResult, ToolSpec from .tools.mcp import MCPServer @@ -33,6 +34,7 @@ class SubAgentConfig(BaseModel): inherit_parent_tools: bool = False inherit_mcp_servers: bool = False builtin_tools: list[str] = Field(default_factory=list) + plugins: list[Plugin] = Field(default_factory=list) tools: list[ToolSpec] = Field(default_factory=list) mcp_servers: list[MCPServer] = Field(default_factory=list) model: str | None = None @@ -65,16 +67,17 @@ def validate_subagent(self) -> SubAgentConfig: raise ValueError("subagent cannot be exposed inside a child subagent") if any(tool.requires_approval for tool in self.tools): raise ValueError("approval-required tools are not supported inside subagents") - if self.inherit_parent_tools and (self.builtin_tools or self.tools): - raise ValueError("inherit_parent_tools cannot be combined with builtin_tools or tools") + if self.inherit_parent_tools and (self.builtin_tools or self.plugins or self.tools): + raise ValueError("inherit_parent_tools cannot be combined with builtin_tools, plugins, or tools") if not ( self.inherit_parent_tools or self.builtin_tools + or self.plugins or self.tools or self.inherit_mcp_servers or self.mcp_servers ): - raise ValueError("named subagents must define builtin_tools, tools, inherit_parent_tools=True, inherit_mcp_servers=True, or mcp_servers") + raise ValueError("named subagents must define builtin_tools, plugins, tools, inherit_parent_tools=True, inherit_mcp_servers=True, or mcp_servers") return self @@ -288,6 +291,7 @@ def build_child_harness(parent: Harness, config: SubAgentConfig | None) -> Harne return Harness( child_config, model=child_model, + plugins=[] if inherit_tools or config is None else config.plugins, tools=_effective_custom_tools(parent, config), tracing=_child_tracing(parent, config), skills=parent.skills if inherit_tools else None, diff --git a/thinharness/tools/__init__.py b/thinharness/tools/__init__.py index 1cce65d..34af6c8 100644 --- a/thinharness/tools/__init__.py +++ b/thinharness/tools/__init__.py @@ -7,13 +7,14 @@ PathPolicy, PathValidationError, ToolEnvelope, + ToolOrigin, ToolResult, ToolSpec, call_tool, contained_path, ) from .bash import BashArgs, BashTool -from .filesystem import FileTools, builtin_tools +from .filesystem import FileTools from .jsonl import JsonlFieldSearch, JsonlSearch, JsonlSearchArgs, JsonlWhereFilter from .mcp import MCPDependencyError, MCPError, MCPServer, MCPServerSSE, MCPServerStdio, MCPServerStreamableHTTP from .parallel_llm import FilePromptSource, InlinePromptSource, ParallelLlmArgs, ParallelLlmTool, create_parallel_llm_tool @@ -39,6 +40,7 @@ "PathPolicy", "PathValidationError", "ToolEnvelope", + "ToolOrigin", "FilePromptSource", "InlinePromptSource", "ParallelLlmArgs", @@ -47,7 +49,6 @@ "SkillRegistry", "ToolResult", "ToolSpec", - "builtin_tools", "call_tool", "contained_path", "create_parallel_llm_tool", diff --git a/thinharness/tools/base.py b/thinharness/tools/base.py index 79936b6..51aba5f 100644 --- a/thinharness/tools/base.py +++ b/thinharness/tools/base.py @@ -21,6 +21,15 @@ T = TypeVar("T", bound=BaseModel) +@dataclass(frozen=True) +class ToolOrigin: + """Plugin provenance for one model-callable tool.""" + + plugin: str + source: str | None = None + attributes: Json = field(default_factory=dict) + + @dataclass(frozen=True) class McpToolInfo: """Framework-owned identity for an MCP-backed tool.""" @@ -42,6 +51,7 @@ class ToolSpec: max_retries: int | None = None instructions: str | None = None requires_approval: bool = False + origin: ToolOrigin | None = None kind: ToolKind = "user" mcp: McpToolInfo | None = None diff --git a/thinharness/tools/filesystem.py b/thinharness/tools/filesystem.py index 1c5a334..d58a68a 100644 --- a/thinharness/tools/filesystem.py +++ b/thinharness/tools/filesystem.py @@ -133,7 +133,6 @@ def __init__( from .jsonl import JsonlSearch self.root = Path(root).expanduser().resolve() - self.root.mkdir(parents=True, exist_ok=True) self.output_dir = contained_path(self.root, output_dir or ".thinharness/outputs") self._spill_artifacts: set[Path] = set() self.read_policy = PathPolicy(self.root, read_paths, "read") @@ -581,10 +580,6 @@ def _truncate(self, text: str, *, prefix: str, max_chars: int | None = None) -> # ============================================================================= -def builtin_tools(root: str | Path = ".", **kwargs: Any) -> list[ToolSpec]: - """Create the default filesystem tool set.""" - return FileTools(root, **kwargs).specs() - def _exclude_glob(pattern: str) -> str: """Return a ripgrep exclusion glob.""" return pattern if pattern.startswith("!") else f"!{pattern}" From 5c750cc770e6d58ba924098ca7a708587d1a9f5c Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Tue, 18 Aug 2026 18:01:14 -0400 Subject: [PATCH 03/30] Fix plugin lifecycle review findings --- docs/behavior.md | 2 +- docs/docs.md | 15 +- tests/unit/test_file_tools.py | 26 +- tests/unit/test_plugins.py | 439 +++++++++++++++++++++++++++++- tests/unit/test_subagents.py | 19 ++ thinharness/core.py | 226 ++++++++++----- thinharness/plugins/__init__.py | 3 +- thinharness/plugins/base.py | 18 +- thinharness/plugins/filesystem.py | 34 ++- thinharness/subagents.py | 12 +- thinharness/tools/base.py | 37 ++- thinharness/tools/filesystem.py | 14 +- 12 files changed, 723 insertions(+), 122 deletions(-) diff --git a/docs/behavior.md b/docs/behavior.md index 87e86db..0047588 100644 --- a/docs/behavior.md +++ b/docs/behavior.md @@ -110,7 +110,7 @@ Callers opt into root-scoped workspace tools without making filesystem behavior - FILESYSTEM-PLUGIN-1: `Harness` has no implicit filesystem tools. `FilesystemPlugin` provides `read`, `write`, `edit`, `search`, `list`, and `glob` by default; callers select an ordered subset explicitly. - FILESYSTEM-PLUGIN-2: `jsonl_search` is an opt-in tool of `FilesystemPlugin` and shares its root, read policy, search process, truncation, and spill-output handling. - FILESYSTEM-PLUGIN-3: `HarnessConfig.root` is the one run root. `FilesystemPlugin` uses that root and cannot configure a different root. -- FILESYSTEM-PLUGIN-4: Harness construction and plugin binding do not create the root. A harness without `FilesystemPlugin` has a generic default prompt, adds no workspace-root instruction, and has no filesystem side effect. +- FILESYSTEM-PLUGIN-4: Harness construction and plugin binding do not create the workspace root. A harness without `FilesystemPlugin` has a generic default prompt, adds no workspace-root instruction, and has no workspace filesystem side effect. Observability sinks keep their independent configured storage behavior. - FILESYSTEM-PLUGIN-5: Filesystem limits, output location, search settings, and path policies belong to `FilesystemPlugin`. `HarnessConfig.read_paths` and `write_paths` remain temporarily as parallel-LLM policy and do not configure filesystem plugin tools. - FILESYSTEM-PLUGIN-6: Independent custom tools continue to use `tools=[ToolSpec(...)]`; callers do not need to wrap one tool in a plugin. diff --git a/docs/docs.md b/docs/docs.md index 6c2b097..997a83b 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -314,7 +314,7 @@ Set `output_type` to validate the final result with Pydantic. `result.text` rema ```python from pydantic import BaseModel -from thinharness import Harness, HarnessConfig +from thinharness import FilesystemPlugin, Harness, HarnessConfig class Summary(BaseModel): @@ -322,11 +322,14 @@ class Summary(BaseModel): bullets: list[str] -harness = Harness(HarnessConfig( - root=".", - output_type=Summary, - output_mode="auto", -)) +harness = Harness( + HarnessConfig( + root=".", + output_type=Summary, + output_mode="auto", + ), + plugins=[FilesystemPlugin(tools=["read"])], +) result = await harness.run("Summarize README.md.") summary: Summary = result.output diff --git a/tests/unit/test_file_tools.py b/tests/unit/test_file_tools.py index c4294f6..d259802 100644 --- a/tests/unit/test_file_tools.py +++ b/tests/unit/test_file_tools.py @@ -16,8 +16,8 @@ DEFAULT_SEARCH_DESCRIPTION, DEFAULT_WRITE_DESCRIPTION, ) -from thinharness.tools.base import StrictArgs, tool_parameters -from thinharness.tools.filesystem import FileTools, PathValidationError, SearchArgs +from thinharness.tools.base import PathPolicy, PathValidationError, StrictArgs, tool_parameters +from thinharness.tools.filesystem import FileTools, SearchArgs from thinharness.tools.jsonl import JsonlSearchArgs @@ -1291,6 +1291,28 @@ def test_gitignore_ignores_thinharness_outputs() -> None: assert ".thinharness/" in ignore + +def test_path_policy_defers_file_classification_until_use(tmp_path: Path) -> None: + future = tmp_path / "future" + policy = PathPolicy(tmp_path, ["future"], "read") + + future.write_text("value", encoding="utf-8") + + assert policy.resolve("future") == future + with pytest.raises(PathValidationError, match="outside allowed read paths"): + policy.resolve("future/child.txt") + + +def test_path_policy_allows_existing_directory_descendants(tmp_path: Path) -> None: + directory = tmp_path / "docs" + directory.mkdir() + child = directory / "child.txt" + child.write_text("value", encoding="utf-8") + policy = PathPolicy(tmp_path, ["docs"], "read") + + assert policy.resolve("docs/child.txt") == child + + def _contains_key(value: object, key: str) -> bool: if isinstance(value, dict): return key in value or any(_contains_key(item, key) for item in value.values()) diff --git a/tests/unit/test_plugins.py b/tests/unit/test_plugins.py index 2f27470..7e5591a 100644 --- a/tests/unit/test_plugins.py +++ b/tests/unit/test_plugins.py @@ -1,18 +1,23 @@ from __future__ import annotations import asyncio -from contextlib import asynccontextmanager +import json +from contextlib import AsyncExitStack, asynccontextmanager from pathlib import Path import pytest from fakes import ScriptedModel, ScriptedSession, echo_tool +from pydantic import BaseModel +import thinharness.core as core_module from thinharness import ( FilesystemPlugin, Harness, HarnessConfig, + HarnessError, Hook, HookRegistry, + ModelToolCall, ModelTurn, PluginBinding, PluginContext, @@ -20,6 +25,8 @@ ToolOrigin, ToolSpec, ) +from thinharness.plugins import ToolOrigin as PluginToolOrigin +from thinharness.tools.base import ToolOrigin as DefinedToolOrigin def _tool(name: str) -> ToolSpec: @@ -288,8 +295,436 @@ def test_one_plugin_object_binds_independently_to_two_harnesses(tmp_path: Path) assert first.tools[0] is not second.tools[0] +def test_filesystem_bind_performs_no_metadata_io(tmp_path: Path, monkeypatch) -> None: + root = tmp_path.resolve() + + def fail(*_args, **_kwargs): + raise AssertionError("filesystem metadata used during bind") + + monkeypatch.setattr(Path, "resolve", fail) + monkeypatch.setattr(Path, "exists", fail) + monkeypatch.setattr(Path, "is_file", fail) + + binding = FilesystemPlugin(read_paths=["future"], write_paths=["outputs"]).bind(PluginContext(root=root)) + + assert [tool.name for tool in binding.static.tools] == ["read", "write", "edit", "search", "list", "glob"] + + +def test_tool_origin_is_exported_from_plugin_contract() -> None: + assert PluginToolOrigin is ToolOrigin + assert DefinedToolOrigin is ToolOrigin + + +def test_plugin_name_must_be_string(tmp_path: Path) -> None: + plugin = StaticPlugin("valid") + plugin.name = 1 # type: ignore[assignment] + + with pytest.raises(ValueError, match="non-empty string"): + Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[plugin]) + + +async def test_sequential_runs_reuse_one_connection(tmp_path: Path) -> None: + events: list[str] = [] + plugin = ConnectedPlugin("connected", events, PluginContribution()) + model = ScriptedModel([ + ScriptedSession(start_turn=ModelTurn(text="first", raw={"id": "first"})), + ScriptedSession(start_turn=ModelTurn(text="second", raw={"id": "second"})), + ]) + harness = Harness(HarnessConfig(root=tmp_path), model=model, plugins=[plugin]) + + assert (await harness.run("one")).text == "first" + assert (await harness.run("two")).text == "second" + assert plugin.attempts == 1 + + await harness.aclose() + + +async def test_concurrent_failed_connection_is_shared_then_retryable(tmp_path: Path) -> None: + entered = asyncio.Event() + release = asyncio.Event() + + class GatedPlugin: + name = "gated" + + def __init__(self) -> None: + self.attempts = 0 + + def bind(self, context: PluginContext) -> PluginBinding: + @asynccontextmanager + async def connect(): + self.attempts += 1 + if self.attempts == 1: + entered.set() + await release.wait() + raise RuntimeError("shared failure") + yield PluginContribution() + + return PluginBinding(connect=connect) + + plugin = GatedPlugin() + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[plugin]) + calls = [asyncio.create_task(harness.connect()) for _ in range(3)] + await entered.wait() + await asyncio.sleep(0) + release.set() + results = await asyncio.gather(*calls, return_exceptions=True) + + assert plugin.attempts == 1 + assert all(isinstance(result, RuntimeError) and str(result) == "shared failure" for result in results) + + await harness.connect() + assert plugin.attempts == 2 + await harness.aclose() + + +async def test_concurrent_cancelled_connection_is_shared_then_retryable(tmp_path: Path) -> None: + entered = asyncio.Event() + release = asyncio.Event() + + class CancellingPlugin: + name = "cancel" + + def __init__(self) -> None: + self.attempts = 0 + + def bind(self, context: PluginContext) -> PluginBinding: + @asynccontextmanager + async def connect(): + self.attempts += 1 + if self.attempts == 1: + entered.set() + await release.wait() + raise asyncio.CancelledError + yield PluginContribution() + + return PluginBinding(connect=connect) + + plugin = CancellingPlugin() + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[plugin]) + calls = [asyncio.create_task(harness.connect()) for _ in range(3)] + await entered.wait() + await asyncio.sleep(0) + release.set() + results = await asyncio.gather(*calls, return_exceptions=True) + + assert plugin.attempts == 1 + assert all(isinstance(result, asyncio.CancelledError) for result in results) + + await harness.connect() + assert plugin.attempts == 2 + await harness.aclose() + + +async def test_aclose_cancels_inflight_connection_without_leak(tmp_path: Path) -> None: + entered = asyncio.Event() + exited = asyncio.Event() + + class SlowPlugin: + name = "slow" + + def bind(self, context: PluginContext) -> PluginBinding: + @asynccontextmanager + async def connect(): + entered.set() + try: + await asyncio.Event().wait() + yield PluginContribution() # pragma: no cover + finally: + exited.set() + + return PluginBinding(connect=connect) + + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[SlowPlugin()]) + connection = asyncio.create_task(harness.connect()) + await entered.wait() + + await harness.aclose() + + with pytest.raises(asyncio.CancelledError): + await connection + assert exited.is_set() + assert harness._plugin_stack is None + assert harness._mcp_stack is None + with pytest.raises(HarnessError, match="harness is closed"): + await harness.connect() + + +async def test_aclose_propagates_caller_cancellation_after_connection_cleanup(tmp_path: Path) -> None: + entered = asyncio.Event() + cleanup_started = asyncio.Event() + exited = asyncio.Event() + + class SlowCleanupPlugin: + name = "slow-cleanup" + + def bind(self, context: PluginContext) -> PluginBinding: + @asynccontextmanager + async def connect(): + entered.set() + try: + await asyncio.Event().wait() + yield PluginContribution() # pragma: no cover + finally: + cleanup_started.set() + try: + await asyncio.Event().wait() + finally: + exited.set() + + return PluginBinding(connect=connect) + + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[SlowCleanupPlugin()]) + connection = asyncio.create_task(harness.connect()) + await entered.wait() + closing = asyncio.create_task(harness.aclose()) + await cleanup_started.wait() + + closing.cancel() + with pytest.raises(asyncio.CancelledError): + await closing + with pytest.raises(asyncio.CancelledError): + await connection + assert exited.is_set() + assert harness._plugin_stack is None + + +async def test_failed_connection_attempts_every_plugin_cleanup(tmp_path: Path) -> None: + events: list[str] = [] + + class CleanupPlugin: + def __init__(self, name: str, contribution: PluginContribution, *, fail_close: bool = False) -> None: + self.name = name + self.contribution = contribution + self.fail_close = fail_close + + def bind(self, context: PluginContext) -> PluginBinding: + @asynccontextmanager + async def connect(): + try: + yield self.contribution + finally: + events.append(self.name) + if self.fail_close: + raise RuntimeError(f"{self.name} close failed") + + return PluginBinding(connect=connect) + + first = CleanupPlugin("first", PluginContribution(), fail_close=True) + second = CleanupPlugin("second", PluginContribution(tools=(_tool("subagent"),))) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[first, second]) + + with pytest.raises(ValueError, match="reserved tool name") as raised: + await harness.connect() + + assert events == ["second", "first"] + assert any("cleanup also failed" in note for note in raised.value.__notes__) + assert harness.tools == [] + + +async def test_close_attempts_all_resources_after_failures(tmp_path: Path) -> None: + events: list[str] = [] + + class FailingClosePlugin: + name = "plugin" + + def bind(self, context: PluginContext) -> PluginBinding: + @asynccontextmanager + async def connect(): + try: + yield PluginContribution() + finally: + events.append("plugin") + raise RuntimeError("plugin close failed") + + return PluginBinding(connect=connect) + + class ClosingProvider: + name = "OpenAI" + + async def aclose(self) -> None: + events.append("model") + raise RuntimeError("model close failed") + + model = ScriptedModel([]) + model.provider = ClosingProvider() + harness = Harness( + HarnessConfig(root=tmp_path), + model=model, + plugins=[FailingClosePlugin()], + _owns_model=True, + ) + await harness.connect() + mcp_stack = AsyncExitStack() + + async def close_mcp() -> None: + events.append("mcp") + raise RuntimeError("mcp close failed") + + mcp_stack.push_async_callback(close_mcp) + harness._mcp_stack = mcp_stack + + with pytest.raises(RuntimeError, match="mcp close failed"): + await harness.aclose() + + assert events == ["mcp", "plugin", "model"] + assert harness._mcp_stack is None + assert harness._plugin_stack is None + + +class _Answer(BaseModel): + value: str + + +async def test_dynamic_structured_output_collision_rolls_back(tmp_path: Path) -> None: + plugin = ConnectedPlugin("bad", [], PluginContribution(tools=(_tool("final_result"),))) + harness = Harness( + HarnessConfig(root=tmp_path, output_type=_Answer, output_mode="tool"), + model=ScriptedModel([]), + plugins=[plugin], + ) + + with pytest.raises(ValueError, match="reserved for structured output"): + await harness.connect() + assert harness.tools == [] + + +async def test_dynamic_approval_tool_requires_resumable_model(tmp_path: Path) -> None: + class NonResumableModel: + model = "non-resumable" + provider = type("Provider", (), {"name": "OpenAI"})() + + def new_session(self): + raise AssertionError("not used") + + approval = ToolSpec("approve", "approve", {"type": "object", "properties": {}}, lambda _args: "ok", requires_approval=True) + plugin = ConnectedPlugin("bad", [], PluginContribution(tools=(approval,))) + harness = Harness(HarnessConfig(root=tmp_path), model=NonResumableModel(), plugins=[plugin]) + + with pytest.raises(ValueError, match="resumable model"): + await harness.connect() + assert harness.tools == [] + + +async def test_dynamic_non_callable_handler_rolls_back(tmp_path: Path) -> None: + invalid = ToolSpec("invalid", "invalid", {"type": "object", "properties": {}}, None) # type: ignore[arg-type] + plugin = ConnectedPlugin("bad", [], PluginContribution(tools=(invalid,))) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[plugin]) + + with pytest.raises(TypeError, match="not callable"): + await harness.connect() + assert harness.tools == [] + + +async def test_connected_plugin_toolset_is_frozen_during_run(tmp_path: Path) -> None: + outputs: list[str] = [] + + class FreezeSession: + def __init__(self) -> None: + self.continues = 0 + + async def start(self, prompt, constants, **_kwargs): + assert [tool["name"] for tool in constants.tools] == ["register"] + return ModelTurn(tool_calls=[ModelToolCall(id="call_1", name="register", arguments="{}")], raw={"id": "start"}) + + async def continue_with_tools(self, tool_outputs, constants, **_kwargs): + outputs.extend(output.output for output in tool_outputs) + self.continues += 1 + if self.continues == 1: + assert [tool["name"] for tool in constants.tools] == ["register"] + return ModelTurn(tool_calls=[ModelToolCall(id="call_2", name="late", arguments="{}")], raw={"id": "late"}) + return ModelTurn(text="done", raw={"id": "done"}) + + async def continue_with_user_text(self, text, constants, **_kwargs): + raise AssertionError("not used") + + def dump_state(self): + return {"kind": "scripted", "version": 1, "model": "scripted-model"} + + harness: Harness + + def register(_args): + harness.add_tool(_tool("late")) + return "registered" + + plugin = ConnectedPlugin("dynamic", [], PluginContribution(tools=(ToolSpec( + "register", "register", {"type": "object", "properties": {}}, register, + ),))) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([FreezeSession()]), plugins=[plugin]) + + assert (await harness.run("go")).text == "done" + assert "late" in [tool.name for tool in harness.tools] + late = json.loads(outputs[-1]) + assert late["ok"] is False + assert "unknown tool late" in late["content"] + + await harness.aclose() + + +async def test_added_tool_survives_failed_connection_and_retry(tmp_path: Path) -> None: + plugin = ConnectedPlugin("failing", [], PluginContribution(tools=(_tool("dynamic"),)), fail_first=True) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[plugin]) + harness.add_tool(_tool("direct")) + + with pytest.raises(RuntimeError, match="failed:failing"): + await harness.connect() + assert [tool.name for tool in harness.tools] == ["direct"] + + await harness.connect() + assert [tool.name for tool in harness.tools] == ["direct", "dynamic"] + await harness.aclose() + + +def test_missing_root_filesystem_tools_and_write_creation(tmp_path: Path) -> None: + root = tmp_path / "missing" + harness = Harness( + HarnessConfig(root=root), + model=ScriptedModel([]), + plugins=[FilesystemPlugin(tools=["list", "glob", "jsonl_search", "write"])], + ) + by_name = {tool.name: tool for tool in harness.tools} + + listed = by_name["list"].handler(by_name["list"].parse_args({"path": "."})) + globbed = by_name["glob"].handler(by_name["glob"].parse_args({"pattern": "**/*"})) + jsonl = by_name["jsonl_search"].handler(by_name["jsonl_search"].parse_args({"path": "."})) + + assert listed.ok is False + assert globbed.ok is True + assert jsonl.ok is True + assert not root.exists() + + written = by_name["write"].handler(by_name["write"].parse_args({"path": "nested/out.txt", "content": "ok"})) + assert written.ok is True + assert (root / "nested/out.txt").read_text() == "ok" + + +def test_shared_filesystem_plugin_has_independent_binding_state(tmp_path: Path) -> None: + plugin = FilesystemPlugin(tools=["write"]) + first = Harness(HarnessConfig(root=tmp_path / "first"), model=ScriptedModel([]), plugins=[plugin]) + second = Harness(HarnessConfig(root=tmp_path / "second"), model=ScriptedModel([]), plugins=[plugin]) + + first.tools[0].handler(first.tools[0].parse_args({"path": "same.txt", "content": "first"})) + second.tools[0].handler(second.tools[0].parse_args({"path": "same.txt", "content": "second"})) + + assert (tmp_path / "first/same.txt").read_text() == "first" + assert (tmp_path / "second/same.txt").read_text() == "second" + + +def test_harness_plugins_preserve_order_and_empty_filesystem_instruction(tmp_path: Path) -> None: + first = StaticPlugin("first") + filesystem = FilesystemPlugin(tools=[]) + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[first, filesystem], + ) + + assert harness.plugins == (first, filesystem) + assert harness.tools == [] + assert harness.system_instructions().count(f"Workspace root: {tmp_path.resolve()}") == 1 + + def test_core_does_not_import_filesystem_implementation() -> None: - source = Path("thinharness/core.py").read_text(encoding="utf-8") + source = Path(core_module.__file__).read_text(encoding="utf-8") assert "tools.filesystem" not in source assert "plugins.filesystem" not in source diff --git a/tests/unit/test_subagents.py b/tests/unit/test_subagents.py index 9f5277d..abbc850 100644 --- a/tests/unit/test_subagents.py +++ b/tests/unit/test_subagents.py @@ -216,6 +216,25 @@ def test_named_inherited_subagent_gets_parent_tools_without_subagent(tmp_path: P assert child.skills is parent.skills assert child.config.subagents == [] + +def test_inherited_subagents_keep_workspace_instruction_without_duplicate_tools(tmp_path: Path) -> None: + parent = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + model=ScriptedModel([]), + plugins=[FilesystemPlugin(tools=["read", "write"])], + ) + + default_child = build_child_harness(parent, None) + named_child = build_child_harness( + parent, + SubAgentConfig(name="general", description="General helper.", inherit_parent_tools=True), + ) + + for child in (default_child, named_child): + assert [tool.name for tool in child.tools] == ["read", "write"] + assert child.system_instructions().count(f"Workspace root: {tmp_path.resolve()}") == 1 + + def test_inherited_subagent_reuses_parent_skill_registry(tmp_path: Path) -> None: skill = tmp_path / "skills" / "demo" skill.mkdir(parents=True) diff --git a/thinharness/core.py b/thinharness/core.py index f092b9e..dfe9dad 100644 --- a/thinharness/core.py +++ b/thinharness/core.py @@ -190,8 +190,8 @@ def __init__( configured_plugins = tuple(plugins or []) plugin_names = [plugin.name for plugin in configured_plugins] - if any(not name.strip() for name in plugin_names): - raise ValueError("plugin name must be non-empty") + if any(not isinstance(name, str) or not name.strip() for name in plugin_names): + raise ValueError("plugin name must be a non-empty string") duplicate_plugin = next((name for index, name in enumerate(plugin_names) if name in plugin_names[:index]), None) if duplicate_plugin is not None: raise ValueError(f"duplicate plugin name: {duplicate_plugin}") @@ -244,6 +244,8 @@ def __init__( self._plugin_stack: AsyncExitStack | None = None self._connected = False self._connect_lock = asyncio.Lock() + self._connect_task: asyncio.Task[None] | None = None + self._connect_waiters = 0 self._skills_enabled = bool(self.skills.skills) and any(tool.name in {"skill_read", "skill_run"} for tool in self.tools) self.local_tracing: LocalTracing | None = None external_tracing = list(self.config.tracing if tracing is None else tracing) @@ -619,22 +621,39 @@ async def _run_and_close() -> HarnessResult: async def aclose(self) -> None: """Close connected plugins, MCP servers, and an owned model.""" - if self._closed: - return - try: - if self._mcp_stack is not None: - await self._mcp_stack.aclose() - self._mcp_stack = None - if self._plugin_stack is not None: - await self._plugin_stack.aclose() - self._plugin_stack = None - self._connected = False - if self._owns_model: - aclose = getattr(self.model.provider, "aclose", None) - if aclose is not None: - await aclose() - finally: + async with self._connect_lock: + if self._closed: + return self._closed = True + connect_task = self._connect_task + caller_cancelled = False + if connect_task is not None and not connect_task.done(): + current_task = asyncio.current_task() + pending_cancels = current_task.cancelling() if current_task is not None else 0 + connect_task.cancel() + try: + await connect_task + except asyncio.CancelledError: + caller_cancelled = current_task is not None and current_task.cancelling() > pending_cancels + except BaseException: + pass + mcp_stack = self._mcp_stack + plugin_stack = self._plugin_stack + self._mcp_stack = None + self._plugin_stack = None + self._connected = False + close_error = await self._close_resources( + mcp_stack=mcp_stack, + plugin_stack=plugin_stack, + close_model=self._owns_model, + ) + if caller_cancelled: + cancellation = asyncio.CancelledError() + if close_error is not None: + cancellation.add_note(f"cleanup also failed: {type(close_error).__name__}: {close_error}") + raise cancellation + if close_error is not None: + raise close_error async def __aenter__(self) -> Harness: """Enter an async harness lifecycle.""" @@ -805,70 +824,127 @@ async def connect(self) -> None: await self._ensure_connected() async def _ensure_connected(self) -> None: - """Connect every dynamic contribution once and commit it atomically.""" + """Share one connection attempt and commit its contributions atomically.""" + if self._closed: + raise HarnessError("harness is closed") if self._connected: return async with self._connect_lock: + if self._closed: + raise HarnessError("harness is closed") if self._connected: return - plugin_stack = AsyncExitStack() - mcp_stack: AsyncExitStack | None = None - base_hooks = list(self.hooks.hooks) - try: - dynamic_tools: list[ToolSpec] = [] - dynamic_instructions: list[str] = [] - dynamic_hooks: list[Hook] = [] - for plugin, binding in zip(self.plugins, self._plugin_bindings, strict=True): - if binding.connect is None: - continue - contribution = await plugin_stack.enter_async_context(binding.connect()) - normalized = self._normalize_contribution(plugin.name, contribution) - dynamic_tools.extend(normalized.tools) - dynamic_instructions.extend(normalized.instructions) - dynamic_hooks.extend(normalized.hooks) - - candidate_tools = [*self._base_tools, *dynamic_tools] - self._validate_tool_list( - candidate_tools, - output_schema=self.output_schema, - model_supports_approval_resume=self._model_supports_approval_resume(), - is_child_run=self._is_child_run, - ) - candidate_hooks = HookRegistry([*base_hooks, *dynamic_hooks], strict_hooks=self._strict_hooks) - self._validate_hook_registry(candidate_hooks, self.config.subagents) - self._validate_skill_tool_selection_for(self.skills, candidate_tools) - - mcp_stack, mcp_tools = await self._open_mcp_tools(candidate_tools) - all_tools = [*candidate_tools, *mcp_tools] - self._validate_tool_list( - all_tools, - output_schema=self.output_schema, - model_supports_approval_resume=self._model_supports_approval_resume(), - is_child_run=self._is_child_run, - ) + task = self._connect_task + if task is None or (task.done() and self._connect_waiters == 0): + task = asyncio.create_task(self._connect_once()) + self._connect_task = task + self._connect_waiters += 1 + try: + await task + finally: + async with self._connect_lock: + self._connect_waiters -= 1 + if ( + self._connect_waiters == 0 + and task.done() + and not self._connected + and self._connect_task is task + ): + self._connect_task = None + + async def _connect_once(self) -> None: + """Open every dynamic contribution for one shared connection attempt.""" + plugin_stack = AsyncExitStack() + mcp_stack: AsyncExitStack | None = None + base_hooks = list(self.hooks.hooks) + try: + dynamic_tools: list[ToolSpec] = [] + dynamic_instructions: list[str] = [] + dynamic_hooks: list[Hook] = [] + for plugin, binding in zip(self.plugins, self._plugin_bindings, strict=True): + if binding.connect is None: + continue + contribution = await plugin_stack.enter_async_context(binding.connect()) + normalized = self._normalize_contribution(plugin.name, contribution) + dynamic_tools.extend(normalized.tools) + dynamic_instructions.extend(normalized.instructions) + dynamic_hooks.extend(normalized.hooks) + + candidate_tools = [*self._base_tools, *dynamic_tools] + self._validate_tool_list( + candidate_tools, + output_schema=self.output_schema, + model_supports_approval_resume=self._model_supports_approval_resume(), + is_child_run=self._is_child_run, + ) + candidate_hooks = HookRegistry([*base_hooks, *dynamic_hooks], strict_hooks=self._strict_hooks) + self._validate_hook_registry(candidate_hooks, self.config.subagents) + self._validate_skill_tool_selection_for(self.skills, candidate_tools) + + mcp_stack, mcp_tools = await self._open_mcp_tools(candidate_tools) + all_tools = [*candidate_tools, *mcp_tools] + self._validate_tool_list( + all_tools, + output_schema=self.output_schema, + model_supports_approval_resume=self._model_supports_approval_resume(), + is_child_run=self._is_child_run, + ) + if self._closed: + raise HarnessError("harness is closed") + + self.tools = all_tools + self._tool_map = {tool.name: tool for tool in all_tools} + self._plugin_instructions = [*self._base_instructions, *dynamic_instructions] + self.hooks = candidate_hooks + self._skills_enabled = bool(self.skills.skills) and any( + tool.name in {"skill_read", "skill_run"} for tool in self.tools + ) + self._plugin_stack = plugin_stack + self._mcp_stack = mcp_stack + self._connected = True + except BaseException as exc: + cleanup_error = await self._close_resources( + mcp_stack=mcp_stack, + plugin_stack=plugin_stack, + close_model=False, + ) + self.tools = list(self._base_tools) + self._tool_map = {tool.name: tool for tool in self.tools} + self._plugin_instructions = list(self._base_instructions) + self.hooks = HookRegistry(base_hooks, strict_hooks=self._strict_hooks) + self._skills_enabled = bool(self.skills.skills) and any( + tool.name in {"skill_read", "skill_run"} for tool in self.tools + ) + if cleanup_error is not None: + exc.add_note(f"cleanup also failed: {type(cleanup_error).__name__}: {cleanup_error}") + raise - self.tools = all_tools - self._tool_map = {tool.name: tool for tool in all_tools} - self._plugin_instructions = [*self._base_instructions, *dynamic_instructions] - self.hooks = candidate_hooks - self._skills_enabled = bool(self.skills.skills) and any( - tool.name in {"skill_read", "skill_run"} for tool in self.tools - ) - self._plugin_stack = plugin_stack - self._mcp_stack = mcp_stack - self._connected = True - except BaseException: - if mcp_stack is not None: - await mcp_stack.aclose() - await plugin_stack.aclose() - self.tools = list(self._base_tools) - self._tool_map = {tool.name: tool for tool in self.tools} - self._plugin_instructions = list(self._base_instructions) - self.hooks = HookRegistry(base_hooks, strict_hooks=self._strict_hooks) - self._skills_enabled = bool(self.skills.skills) and any( - tool.name in {"skill_read", "skill_run"} for tool in self.tools - ) - raise + async def _close_resources( + self, + *, + mcp_stack: AsyncExitStack | None, + plugin_stack: AsyncExitStack | None, + close_model: bool, + ) -> BaseException | None: + """Attempt every close in order and return the first failure.""" + first_error: BaseException | None = None + for stack in (mcp_stack, plugin_stack): + if stack is None: + continue + try: + await stack.aclose() + except BaseException as exc: + if first_error is None: + first_error = exc + if close_model: + aclose = getattr(self.model.provider, "aclose", None) + if aclose is not None: + try: + await aclose() + except BaseException as exc: + if first_error is None: + first_error = exc + return first_error async def _open_mcp_tools(self, existing_tools: list[ToolSpec]) -> tuple[AsyncExitStack | None, list[ToolSpec]]: """Open the temporary MCP bridge and stage its discovered tools.""" diff --git a/thinharness/plugins/__init__.py b/thinharness/plugins/__init__.py index 93d830d..6b7848f 100644 --- a/thinharness/plugins/__init__.py +++ b/thinharness/plugins/__init__.py @@ -1,6 +1,6 @@ """Built-in plugin contracts and adapters.""" -from .base import Plugin, PluginBinding, PluginConnector, PluginContext, PluginContribution +from .base import Plugin, PluginBinding, PluginConnector, PluginContext, PluginContribution, ToolOrigin from .filesystem import FilesystemPlugin __all__ = [ @@ -10,4 +10,5 @@ "PluginConnector", "PluginContext", "PluginContribution", + "ToolOrigin", ] diff --git a/thinharness/plugins/base.py b/thinharness/plugins/base.py index 4a40431..b4266b0 100644 --- a/thinharness/plugins/base.py +++ b/thinharness/plugins/base.py @@ -2,12 +2,14 @@ from __future__ import annotations -from collections.abc import AsyncIterator, Callable -from contextlib import AbstractAsyncContextManager, asynccontextmanager +from collections.abc import Callable +from contextlib import AbstractAsyncContextManager from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Protocol, runtime_checkable +from ..tools.base import ToolOrigin + if TYPE_CHECKING: from ..hooks import Hook from ..tools.base import ToolSpec @@ -51,7 +53,11 @@ def bind(self, context: PluginContext) -> PluginBinding: ... -@asynccontextmanager -async def empty_connector() -> AsyncIterator[PluginContribution]: - """Return an empty connected contribution.""" - yield PluginContribution() +__all__ = [ + "Plugin", + "PluginBinding", + "PluginConnector", + "PluginContext", + "PluginContribution", + "ToolOrigin", +] diff --git a/thinharness/plugins/filesystem.py b/thinharness/plugins/filesystem.py index 352acf9..c9fc92c 100644 --- a/thinharness/plugins/filesystem.py +++ b/thinharness/plugins/filesystem.py @@ -38,21 +38,31 @@ def __init__( if len(set(selected)) != len(selected): raise ValueError("FilesystemPlugin tools contains a duplicate name") self._selected = selected - self._options = { - "output_dir": output_dir, - "max_read_chars": max_read_chars, - "max_read_bytes": max_read_bytes, - "max_tool_chars": max_tool_chars, - "max_search_line_chars": max_search_line_chars, - "rg_timeout": rg_timeout, - "search_exclude_globs": search_exclude_globs, - "read_paths": read_paths, - "write_paths": write_paths, - } + self._output_dir = output_dir + self._max_read_chars = max_read_chars + self._max_read_bytes = max_read_bytes + self._max_tool_chars = max_tool_chars + self._max_search_line_chars = max_search_line_chars + self._rg_timeout = rg_timeout + self._search_exclude_globs = list(search_exclude_globs) if search_exclude_globs is not None else None + self._read_paths = tuple(read_paths) if read_paths is not None else None + self._write_paths = tuple(write_paths) if write_paths is not None else None def bind(self, context: PluginContext) -> PluginBinding: """Build static tool specifications without filesystem I/O.""" - collection = FileTools(context.root, **self._options) + collection = FileTools( + context.root, + output_dir=self._output_dir, + max_read_chars=self._max_read_chars, + max_read_bytes=self._max_read_bytes, + max_tool_chars=self._max_tool_chars, + max_search_line_chars=self._max_search_line_chars, + rg_timeout=self._rg_timeout, + search_exclude_globs=self._search_exclude_globs, + read_paths=self._read_paths, + write_paths=self._write_paths, + _root_is_resolved=True, + ) by_name = {tool.name: tool for tool in collection.specs()} unknown = [name for name in self._selected if name not in by_name] if unknown: diff --git a/thinharness/subagents.py b/thinharness/subagents.py index 435c986..8626975 100644 --- a/thinharness/subagents.py +++ b/thinharness/subagents.py @@ -291,7 +291,7 @@ def build_child_harness(parent: Harness, config: SubAgentConfig | None) -> Harne return Harness( child_config, model=child_model, - plugins=[] if inherit_tools or config is None else config.plugins, + plugins=_inherited_instruction_plugins(parent) if inherit_tools else (config.plugins if config is not None else []), tools=_effective_custom_tools(parent, config), tracing=_child_tracing(parent, config), skills=parent.skills if inherit_tools else None, @@ -322,6 +322,16 @@ def _effective_custom_tools(parent: Harness, config: SubAgentConfig | None) -> l return list(config.tools) +def _inherited_instruction_plugins(parent: Harness) -> list[Plugin]: + """Preserve instructions for inherited filesystem tools without duplicating them.""" + has_filesystem_tools = any(tool.origin is not None and tool.origin.plugin == "filesystem" for tool in parent.tools) + if not has_filesystem_tools: + return [] + from .plugins.filesystem import FilesystemPlugin + + return [FilesystemPlugin(tools=[])] + + def _child_tracing(parent: Harness, config: SubAgentConfig | None) -> list[TracingOptions]: """Return child tracing options that share the parent's tracer.""" name = config.name if config is not None else DEFAULT_SUBAGENT_NAME diff --git a/thinharness/tools/base.py b/thinharness/tools/base.py index 51aba5f..c0dbbb2 100644 --- a/thinharness/tools/base.py +++ b/thinharness/tools/base.py @@ -6,6 +6,7 @@ import copy import inspect import json +import os from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass, field from functools import partial @@ -142,10 +143,9 @@ def __init__(self, message: str) -> None: @dataclass(frozen=True) class AllowedPath: - """One resolved path allowed by a workspace path policy.""" + """One lexically normalized path allowed by a workspace path policy.""" path: Path - exact: bool = False class PathValidationError(ValueError): @@ -179,22 +179,22 @@ def allows(self, path: Path) -> bool: resolved = path.resolve() if not _is_relative_to(resolved, self.root): return False - for allowed in self.allowed_paths: - if allowed.exact: - if resolved == allowed.path: - return True - elif resolved == allowed.path or allowed.path in resolved.parents: + for configured in self.allowed_paths: + allowed = contained_path(self.root, configured.path) + if resolved == allowed: + return True + if allowed in resolved.parents and not allowed.is_file(): return True return False def existing_search_roots(self) -> list[Path]: """Return existing allow roots for commands that accept search paths.""" - return [allowed.path for allowed in self.allowed_paths if allowed.path.exists()] + roots = [contained_path(self.root, configured.path) for configured in self.allowed_paths] + return [root for root in roots if root.exists()] def _allowed_path(self, raw: str | Path) -> AllowedPath: - """Normalize a configured allow path under the workspace root.""" - resolved = contained_path(self.root, raw) - return AllowedPath(resolved, exact=resolved.exists() and resolved.is_file()) + """Normalize a configured allow path without filesystem metadata I/O.""" + return AllowedPath(_lexical_path_under_root(self.root, raw)) class StrictArgs(BaseModel): """Base class for tool arguments.""" @@ -331,6 +331,21 @@ def contained_path(root: Path, raw: str | Path) -> Path: return _resolve_under_root(root, raw) +def lexical_contained_path(root: Path, raw: str | Path) -> Path: + """Normalize a contained path without consulting filesystem metadata.""" + return _lexical_path_under_root(root, raw) + + +def _lexical_path_under_root(root: Path, raw: str | Path) -> Path: + """Normalize raw under root lexically and reject parent traversal.""" + path = Path(raw).expanduser() + candidate = path if path.is_absolute() else root / path + normalized = Path(os.path.abspath(candidate)) + if not _is_relative_to(normalized, root): + raise PathValidationError(f"path escapes root: {raw}") + return normalized + + def _resolve_under_root(root: Path, raw: str | Path) -> Path: """Resolve raw under root and require the result to stay inside root.""" path = Path(raw).expanduser() diff --git a/thinharness/tools/filesystem.py b/thinharness/tools/filesystem.py index d58a68a..0f418fa 100644 --- a/thinharness/tools/filesystem.py +++ b/thinharness/tools/filesystem.py @@ -40,6 +40,7 @@ _timeout_error_message, coerce_args, contained_path, + lexical_contained_path, ) from .search_support import ( SearchFile, @@ -129,11 +130,13 @@ def __init__( search_exclude_globs: list[str] | None = None, read_paths: Sequence[str | Path] | None = None, write_paths: Sequence[str | Path] | None = None, + _root_is_resolved: bool = False, ) -> None: from .jsonl import JsonlSearch - self.root = Path(root).expanduser().resolve() - self.output_dir = contained_path(self.root, output_dir or ".thinharness/outputs") + root_path = Path(root).expanduser() + self.root = root_path if _root_is_resolved else root_path.resolve() + self.output_dir = lexical_contained_path(self.root, output_dir or ".thinharness/outputs") self._spill_artifacts: set[Path] = set() self.read_policy = PathPolicy(self.root, read_paths, "read") self.write_policy = PathPolicy(self.root, write_paths, "write") @@ -497,7 +500,7 @@ def _resolve_read_path(self, raw: str | Path) -> Path: def _is_readable_spill_artifact(self, path: Path) -> bool: """Return whether path is an exact generated spill artifact.""" resolved = path.resolve() - output_dir = self.output_dir.resolve() + output_dir = contained_path(self.root, self.output_dir) return resolved in self._spill_artifacts and (resolved == output_dir or output_dir in resolved.parents) @staticmethod @@ -555,8 +558,9 @@ def _truncate(self, text: str, *, prefix: str, max_chars: int | None = None) -> limit = max_chars or self.max_tool_chars if len(text) <= limit: return ToolResult(True, text) - self.output_dir.mkdir(parents=True, exist_ok=True) - artifact = self.output_dir / f"{prefix}-{int(time.time() * 1000)}-{uuid.uuid4().hex[:8]}.txt" + output_dir = contained_path(self.root, self.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + artifact = output_dir / f"{prefix}-{int(time.time() * 1000)}-{uuid.uuid4().hex[:8]}.txt" artifact.write_text(text, encoding="utf-8") resolved_artifact = artifact.resolve() self._spill_artifacts.add(resolved_artifact) From 4ea73356a02dd818df1e8b9d59688af843eb6ade Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Tue, 18 Aug 2026 18:19:54 -0400 Subject: [PATCH 04/30] Document MCP plugin behavior --- docs/behavior.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/behavior.md b/docs/behavior.md index 0047588..1361309 100644 --- a/docs/behavior.md +++ b/docs/behavior.md @@ -213,16 +213,16 @@ Tracing and streaming expose projections of the same neutral per-request model-v ### Purpose -ThinHarness exposes tools from MCP servers through wrapper objects whose transport, session, and connection sharing come from the FastMCP client, while ThinHarness keeps tool selection, conversion, error envelopes, and trace attribution. +ThinHarness exposes tools from MCP servers through explicit `MCPPlugin` composition. Server wrappers retain FastMCP transport, session, and connection sharing, while the plugin owns harness binding, discovery, lifecycle, and tool attribution. ### Requirements -- MCP-1: `MCPServer` accepts exactly one FastMCP `ClientTransport`, including `FastMCPTransport` for an MCP server object in the same Python process; URL strings, script paths, server objects, and configuration dictionaries are rejected with `TypeError`. The stdio, SSE, and Streamable HTTP compatibility wrappers take command- or URL-based constructors and derive their ids from them; the generic class derives its default id from the transport class name, and duplicate ids get `-2`, `-3` suffixes. -- MCP-2: Connections open lazily on `Harness.connect()` or the first run; one discovered tool snapshot is reused for all runs of one harness; `Harness.aclose()` closes harness-entered servers, and a partial multi-server connection failure closes servers that were already opened. -- MCP-3: A wrapper owns the FastMCP client built on its transport: nested and concurrent entries share one connection, the final exit closes the transport (terminating a stdio child process), and the same wrapper can reconnect afterwards. One stateful transport object must not be reused across wrappers; reusing the same wrapper shares one session. +- MCP-1: `MCPServer` accepts exactly one FastMCP `ClientTransport`, including `FastMCPTransport` for an MCP server object in the same Python process; URL strings, script paths, server objects, and configuration dictionaries are rejected with `TypeError`. The stdio, SSE, and Streamable HTTP compatibility wrappers take command- or URL-based constructors and derive their public base ids from them; the generic class derives its base id from the transport class name. Each `MCPPlugin` binding resolves duplicate ids locally with deterministic `-2`, `-3` suffixes without mutating shared wrappers. +- MCP-2: MCP is enabled only by adding one fixed-name `MCPPlugin` to `Harness(plugins=...)`. Generic plugin connection opens servers lazily on `Harness.connect()` or the first run, discovers one tool snapshot per binding, and reuses that snapshot for all runs. The complete discovered contribution is validated and installed atomically. A connection, discovery, validation, or cancellation failure closes entered servers in reverse order, installs no tools, and permits retry; `Harness.aclose()` closes the plugin once. +- MCP-3: A wrapper owns the FastMCP client built on its transport: nested and concurrent entries share one connection, the final exit closes the transport (terminating a stdio child process), and the same wrapper can reconnect afterwards. One stateful transport object must not be reused across wrappers; reusing the same wrapper across parent, child, or independent harness bindings shares one reference-counted session. - MCP-4: Final close is bounded — the bound comes from FastMCP's `client_disconnect_timeout` setting (default 5 seconds) — and a caller cancellation consumed by transport cleanup is re-raised after cleanup completes. Cancelling a first connection or a final close propagates the cancellation and leaves the wrapper reusable. -- MCP-5: `include_tools` and `exclude_tools` match original MCP tool names before prefixing and normalization; `tool_prefix`, schema cleanup, sanitized-name collision errors, and cross-harness tool collision errors are ThinHarness behavior. Discovered MCP tools are ordinary `ToolSpec` objects with `kind="mcp"` and `McpToolInfo` attribution, and tracing reads that attribution from the `ToolSpec`, so an after-tool hook cannot erase it. +- MCP-5: `include_tools` and `exclude_tools` match original MCP tool names before prefixing and normalization; `tool_prefix`, schema cleanup, sanitized-name collision errors, and cross-contribution tool collision errors are ThinHarness behavior. Discovered MCP tools are ordinary `ToolSpec` objects with `ToolOrigin(plugin="mcp", source=resolved_server_id, attributes={"tool_name": original_tool_name})`. Tracing reads this origin from the `ToolSpec`, so an after-tool hook cannot erase attribution. Model-visible result metadata uses the same binding-local server id. - MCP-6: Successful `structuredContent` is returned as a JSON string; text, image, audio, embedded-resource, and resource-link blocks convert in order to model-visible text. A protocol-level tool failure (`isError`) returns a failed `ToolResult` with `error_type="MCPToolError"` and `retry=True`; known transport and protocol failures during a tool call return `error_type="MCPError"`, including when wrapped in an exception group or explicit cause chain — a group whose members are all `Exception`s is normalized when any member's cause chain holds a known failure, even alongside sibling exception noise from teardown. An exception group carrying cancellation or any other non-`Exception` failure propagates, and exceptions with no known failure in their group or cause chain propagate as programming errors. -- MCP-7: MCP tools and connection details never enter resume state, and subagents see MCP servers only through explicit `inherit_mcp_servers` and `mcp_servers` settings. -- MCP-8: The base install works without MCP packages: importing ThinHarness and constructing any wrapper needs no extra, and opening a connection without `mcp` or `fastmcp` raises `MCPDependencyError` with the `thinharness[mcp]` install hint. +- MCP-7: MCP tools and connection details never enter resume state. The temporary subagent bridge maps explicit `SubAgentConfig.mcp_servers` values to a child `MCPPlugin`; `inherit_mcp_servers=True` copies parent MCP server wrappers by identity and unions explicit child servers without duplicates. Default parent-tool inheritance excludes tools whose origin plugin is `"mcp"`, so each child connects its own plugin and lifecycle. +- MCP-8: The base install works without MCP packages: importing ThinHarness and constructing any wrapper or `MCPPlugin` needs no extra, and opening a connection without `mcp` or `fastmcp` raises `MCPDependencyError` with the `thinharness[mcp]` install hint. - MCP-9: `timeout` bounds MCP initialization and HTTP connection establishment; `read_timeout` bounds MCP requests, HTTP reads, and SSE reads. From 60bd56dbd5713dd326d8dfd1a3a2db3f03a10eb5 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Tue, 18 Aug 2026 18:26:06 -0400 Subject: [PATCH 05/30] Add MCP plugin composition --- CHANGELOG.md | 2 + README.md | 13 + docs/docs.md | 50 ++- examples/mcp_plugin.py | 29 ++ tests/e2e/README.md | 6 +- tests/e2e/mcp_journey.py | 100 +++-- tests/unit/test_architecture.py | 17 + tests/unit/test_mcp.py | 486 +++++++++++++++++---- tests/unit/test_mcp_optional_dependency.py | 5 +- tests/unit/test_plugins.py | 45 +- thinharness/__init__.py | 5 +- thinharness/core.py | 211 ++++----- thinharness/plugins/__init__.py | 2 + thinharness/plugins/mcp.py | 71 +++ thinharness/subagents.py | 125 +++--- thinharness/tool_execution.py | 93 ++-- thinharness/tools/__init__.py | 2 - thinharness/tools/base.py | 19 +- thinharness/tools/mcp.py | 51 +-- 19 files changed, 878 insertions(+), 454 deletions(-) create mode 100644 examples/mcp_plugin.py create mode 100644 tests/unit/test_architecture.py create mode 100644 thinharness/plugins/mcp.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3026192..61f7a3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ - Added explicit plugin composition with static and connected contributions, atomic connection rollback, unique plugin names, generic tool origin, and plugin-provided hooks and instructions. - Added `FilesystemPlugin` for the ordered workspace tool surface; `jsonl_search` remains opt-in through this plugin. +- Added `MCPPlugin` for lazy MCP server connection, binding-local server identity, atomic tool discovery, and generic tool origin attribution. +- **Breaking:** Removed `HarnessConfig.mcp_servers`, `McpToolInfo`, and the MCP `ToolKind`; configure one `MCPPlugin` with all harness servers. - **Breaking:** `Harness` no longer enables filesystem tools by default. Pass `plugins=[FilesystemPlugin(...)]`; independent custom tools still use `tools=`. - **Breaking:** Removed filesystem settings from `HarnessConfig` and removed the `builtin_tools()` helper. `read_paths` and `write_paths` remain temporarily for the transitional parallel-LLM built-in. - Changed connection setup to complete before `run_start` hooks. A connection failure does not fire run lifecycle hooks. diff --git a/README.md b/README.md index e79de50..1a34225 100644 --- a/README.md +++ b/README.md @@ -256,6 +256,19 @@ asyncio.run(main()) There's a synchronous wrapper too: `Harness(...).run_sync(...)`. +Optional MCP servers use the same plugin composition model: + +```python +from thinharness import MCPPlugin, MCPServerStdio + +harness = Harness( + HarnessConfig(root="."), + plugins=[MCPPlugin(servers=[MCPServerStdio("python", ["server.py"])])], +) +``` + +MCP tools connect and discover one tool snapshot lazily on `Harness.connect()` or the first run. Install support with `uv add 'thinharness[mcp]'`. + Built-in provider requests retry transient HTTP failures three times by default. Configure the shared policy with `request_retries` and `request_retry_backoff` on `HarnessConfig`. If an injected `http_client` owns retries, set `request_retries=0` on the provider. This prevents nested retry policies from multiplying attempts. diff --git a/docs/docs.md b/docs/docs.md index 997a83b..70b580e 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -98,7 +98,7 @@ Important groups: - `root` defines the run root. `FilesystemPlugin` owns filesystem paths, limits, search settings, and output location. - `model`, `api_key`, `base_url`, `temperature`, `max_tokens`, `effort`, `extra_body`, `request_timeout`, `request_retries`, and `request_retry_backoff` define provider settings. -- The `Harness` constructor's `plugins=` and `tools=` inputs, plus `builtin_tools`, `subagents`, `mcp_servers`, and `skills_dir`, define the model-callable surface. `builtin_tools` is temporary for features that have not migrated to plugins. +- The `Harness` constructor's `plugins=` and `tools=` inputs, plus `builtin_tools`, `subagents`, and `skills_dir`, define the model-callable surface. Filesystem and MCP tools use explicit plugins. `builtin_tools` is temporary for features that have not migrated to plugins. - `max_model_requests`, `max_tool_calls`, `output_retries`, and `tool_retries` bound the run. - `output_type` and `output_mode` define structured output. - `tracing`, `local_tracing`, and `local_trace_dir` define observability. @@ -503,19 +503,21 @@ MCP support is optional. Importing ThinHarness does not require the MCP packages ```python from fastmcp.client.transports import FastMCPTransport -from thinharness import Harness, HarnessConfig, MCPServer +from thinharness import Harness, HarnessConfig, MCPPlugin, MCPServer -harness = Harness(HarnessConfig( - root=".", - mcp_servers=[ - MCPServer( - FastMCPTransport(my_server), - id="inprocess", - include_tools=["step", "reset_session"], - ) +harness = Harness( + HarnessConfig(root="."), + plugins=[ + MCPPlugin(servers=[ + MCPServer( + FastMCPTransport(my_server), + id="inprocess", + include_tools=["step", "reset_session"], + ) + ]) ], -)) +) ``` `MCPServer` accepts only a transport object — not a URL, script path, server object, or configuration dictionary. Once a transport is passed to an `MCPServer`, that wrapper owns the client built on it and closes its transport on the final exit; to share one session, reuse the wrapper rather than passing one stateful transport to several wrappers. @@ -523,23 +525,25 @@ harness = Harness(HarnessConfig( The stdio, SSE, and Streamable HTTP wrappers take command- or URL-based constructors and build the matching FastMCP transport when the connection opens: ```python -from thinharness import Harness, HarnessConfig, MCPServerStdio +from thinharness import Harness, HarnessConfig, MCPPlugin, MCPServerStdio -harness = Harness(HarnessConfig( - root=".", - mcp_servers=[ - MCPServerStdio( - "uvx", - ["my-mcp-server"], - tool_prefix="external", - include_tools=["lookup"], - ) +harness = Harness( + HarnessConfig(root="."), + plugins=[ + MCPPlugin(servers=[ + MCPServerStdio( + "uvx", + ["my-mcp-server"], + tool_prefix="external", + include_tools=["lookup"], + ) + ]) ], -)) +) ``` -MCP servers connect lazily during harness startup. Discovered MCP tools become normal `ToolSpec` objects in the live harness tool map. Name collisions are rejected; use `tool_prefix`, `include_tools`, or `exclude_tools` to keep the model-facing tool surface explicit. +Use one `MCPPlugin` per harness and put all servers in caller order. Servers connect lazily on `Harness.connect()` or the first run. The binding discovers one tool snapshot and reuses it until the harness closes. Discovered MCP tools become normal `ToolSpec` objects with generic origin data in the live harness tool map. Name collisions reject the whole discovered contribution; use `tool_prefix`, `include_tools`, or `exclude_tools` to keep the model-facing tool surface explicit. Available wrappers: diff --git a/examples/mcp_plugin.py b/examples/mcp_plugin.py new file mode 100644 index 0000000..489c1b6 --- /dev/null +++ b/examples/mcp_plugin.py @@ -0,0 +1,29 @@ +"""Connect one ordered group of MCP servers through MCPPlugin.""" + +import asyncio + +from thinharness import Harness, HarnessConfig, MCPPlugin, MCPServerStdio + + +async def main() -> None: + """Run an agent with tools discovered from a local MCP server.""" + async with Harness( + HarnessConfig(root=".", model="openai:gpt-5.5", builtin_tools=[]), + plugins=[ + MCPPlugin( + servers=[ + MCPServerStdio( + "python", + ["server.py"], + tool_prefix="docs", + ) + ] + ) + ], + ) as harness: + result = await harness.run("Use the docs tools to answer the question.") + print(result.text) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/e2e/README.md b/tests/e2e/README.md index f044eb2..4222da4 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -1,6 +1,6 @@ # E2E Journeys -These scripts run real provider calls against temporary workspaces. They are intentionally not wired into pytest or CI. +Most scripts run real provider calls against temporary workspaces. The deterministic MCP journey uses a local scripted model. Journeys are intentionally not wired into pytest or CI. Run one script with environment from `.env`: @@ -8,7 +8,7 @@ Run one script with environment from `.env`: uv run --env-file .env python tests/e2e/workspace_tools_journey.py ``` -Each script skips when `CI` is set or when the required provider key is missing. Model defaults can be overridden with the per-script `E2E_*_MODEL` environment variable. +Credential-based scripts skip when `CI` is set or when the required provider key is missing. Their model defaults can be overridden with the per-script `E2E_*_MODEL` environment variable. Current journeys: @@ -16,7 +16,7 @@ Current journeys: - `skills_journey.py`: skill discovery, `skill_read`, and `skill_run`. - `control_plane_journey.py`: hooks, sequential execution, and retry-limit behavior. - `structured_output_journey.py`: Pydantic structured output after tool use. -- `mcp_journey.py`: local stdio MCP tool discovery and execution. +- `mcp_journey.py`: deterministic local stdio MCP tool discovery, execution, and cleanup without provider credentials. - `parallel_llm_tool_journey.py`: direct `ParallelLlmTool` calls across all configured providers. - `parallel_llm_agent_journey.py`: an agent run using both built-in `parallel_llm` and a renamed custom `ParallelLlmTool`. - `prompt_caching_journey.py`: Anthropic prompt caching — asserts a multi-request run reports cached input tokens. diff --git a/tests/e2e/mcp_journey.py b/tests/e2e/mcp_journey.py index 7afbc07..cd9b81f 100644 --- a/tests/e2e/mcp_journey.py +++ b/tests/e2e/mcp_journey.py @@ -1,55 +1,78 @@ from __future__ import annotations -import importlib.util -import os +import asyncio import sys from pathlib import Path from tempfile import TemporaryDirectory +from types import SimpleNamespace +from typing import Any sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from thinharness import Harness, HarnessConfig, Hook, MCPServerStdio +from thinharness import Harness, HarnessConfig, Hook, MCPPlugin, MCPServerStdio, ModelToolCall, ModelTurn -MODEL = os.getenv("E2E_MCP_MODEL", "openrouter:anthropic/claude-sonnet-4.5") -SYSTEM_PROMPT = """You are an MCP test agent. Use the discovered MCP tool for arithmetic.""" -PROMPT = """ -Use the MCP multiply tool to multiply 6 by 7. -Your final answer must include "product=42" and end with MCP_DONE. -""".strip() + +class DeterministicSession: + """Drive one MCP call without provider credentials.""" + + async def start(self, prompt: str, constants: Any, **_kwargs: Any) -> ModelTurn: + assert prompt == "multiply" + assert [tool["name"] for tool in constants.tools] == ["multiply"] + return ModelTurn( + tool_calls=[ModelToolCall(id="multiply-1", name="multiply", arguments='{"left":6,"right":7}')], + raw={"id": "start"}, + ) + + async def continue_with_tools(self, outputs: list[Any], constants: Any, **_kwargs: Any) -> ModelTurn: + del constants + assert len(outputs) == 1 + assert "product=42" in outputs[0].output + return ModelTurn(text="product=42 MCP_DONE", raw={"id": "done"}) + + async def continue_with_user_text(self, text: str, constants: Any, **_kwargs: Any) -> ModelTurn: + raise AssertionError(f"unexpected user continuation: {text!r}, {constants!r}") + + def dump_state(self) -> None: + return None + + +class DeterministicModel: + """Return the deterministic MCP journey session.""" + + model = "deterministic:mcp" + api_key = "unused" + provider = SimpleNamespace(name="Deterministic") + + def new_session(self) -> DeterministicSession: + return DeterministicSession() def main() -> None: - if _should_skip(MODEL): - return + """Run local stdio discovery, execution, and cleanup end to end.""" + asyncio.run(_run()) + +async def _run() -> None: with TemporaryDirectory(prefix="thinharness-e2e-mcp-") as raw_root: root = Path(raw_root) server_path = root / "tiny_mcp_server.py" server_path.write_text(SERVER_CODE, encoding="utf-8") tool_names: list[str] = [] + server = MCPServerStdio(sys.executable, [str(server_path)]) - # Config - harness = Harness( - HarnessConfig( - root=root, - model=MODEL, - system_prompt=SYSTEM_PROMPT, - builtin_tools=[], - mcp_servers=[MCPServerStdio(sys.executable, [str(server_path)])], - max_model_requests=6, - max_tool_calls=3, - ), + async with Harness( + HarnessConfig(root=root, builtin_tools=[], max_model_requests=2, max_tool_calls=1), + model=DeterministicModel(), + plugins=[MCPPlugin(servers=[server])], hooks=[Hook("before_tool_call", lambda ctx: tool_names.append(ctx.tool_name))], - ) + ) as harness: + assert harness.tools == [] + result = await harness.run("multiply") + assert [tool.name for tool in harness.tools] == ["multiply"] - # Run - result = harness.run_sync(PROMPT) - - # Assertions - assert tool_names == ["multiply"], f"expected MCP multiply call; saw {tool_names}" - assert "product=42" in result.text - assert "MCP_DONE" in result.text - print(f"PASS mcp_journey model={MODEL} tools={tool_names}") + assert tool_names == ["multiply"] + assert result.text == "product=42 MCP_DONE" + print(f"PASS mcp_journey tools={tool_names} server={server.id}") SERVER_CODE = """ @@ -70,20 +93,5 @@ def multiply(left: int, right: int) -> str: """.lstrip() -def _should_skip(model: str) -> bool: - if os.getenv("CI"): - print("SKIP mcp_journey: CI is set") - return True - if importlib.util.find_spec("mcp") is None or importlib.util.find_spec("fastmcp") is None: - print("SKIP mcp_journey: install MCP support with `uv sync --extra mcp`") - return True - provider = model.split(":", 1)[0] - env_name = {"openai": "OPENAI_API_KEY", "anthropic": "ANTHROPIC_API_KEY", "openrouter": "OPENROUTER_API_KEY"}[provider] - if not os.getenv(env_name): - print(f"SKIP mcp_journey: {env_name} is not set") - return True - return False - - if __name__ == "__main__": main() diff --git a/tests/unit/test_architecture.py b/tests/unit/test_architecture.py new file mode 100644 index 0000000..0ca895c --- /dev/null +++ b/tests/unit/test_architecture.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +import ast +from pathlib import Path + + +def test_core_has_no_mcp_imports_or_lifecycle_state() -> None: + """Core stays independent from MCP composition and lifecycle details.""" + core_path = Path(__file__).resolve().parents[2] / "thinharness" / "core.py" + source = core_path.read_text(encoding="utf-8") + tree = ast.parse(source) + imported_modules = {node.module for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) and node.module is not None} + imported_modules.update(alias.name for node in ast.walk(tree) if isinstance(node, ast.Import) for alias in node.names) + + assert not any(module.endswith(("tools.mcp", "plugins.mcp")) for module in imported_modules) + for forbidden in ("MCPServer", "mcp_servers", "_mcp_", "_open_mcp_tools"): + assert forbidden not in source diff --git a/tests/unit/test_mcp.py b/tests/unit/test_mcp.py index 6682d84..9b15009 100644 --- a/tests/unit/test_mcp.py +++ b/tests/unit/test_mcp.py @@ -21,12 +21,16 @@ HarnessError, Hook, MCPError, + MCPPlugin, MCPServer, MCPServerSSE, MCPServerStdio, MCPServerStreamableHTTP, ModelTurn, + PluginBinding, + PluginContribution, SubAgentConfig, + ToolOrigin, TracingOptions, build_child_harness, ) @@ -65,10 +69,10 @@ async def __aexit__(self, *exc: object) -> None: self.exited += 1 return await super().__aexit__(*exc) - async def list_tools(self) -> list[ToolSpec]: + async def list_tools(self, *, server_id: str | None = None) -> list[ToolSpec]: """Count discovery calls.""" self.list_calls += 1 - return await super().list_tools() + return await super().list_tools(server_id=server_id) def _lifespan_tracker() -> tuple[Any, dict[str, int]]: @@ -88,6 +92,7 @@ async def lifespan(server: Any): def _echo_handler(tool_name: str, records: list[tuple[str, Json]]) -> Any: """Build a recording echo tool function for an in-process backend.""" + def handler(value: str = "") -> str: records.append((tool_name, {"value": value})) return f"{tool_name}:{value}" @@ -118,10 +123,7 @@ def _script_key(backend: Any) -> int: async def _scripted_list_tools(self: Any, max_pages: int = 250) -> list[Any]: """Return the scripted tool declarations for this client's backend.""" script = _SCRIPTS[_script_key(self.transport.server)] - return [ - SimpleNamespace(name=name, description=f"{name} tool", inputSchema=schema) - for name, schema in script["schemas"].items() - ] + return [SimpleNamespace(name=name, description=f"{name} tool", inputSchema=schema) for name, schema in script["schemas"].items()] async def _scripted_call_tool_mcp(self: Any, name: str, arguments: dict[str, Any], **_kwargs: Any) -> Any: @@ -164,8 +166,9 @@ def scripted_server( class FailingListServer(ObservedMCPServer): """Server double whose discovery fails after entering the context.""" - async def list_tools(self) -> list[ToolSpec]: + async def list_tools(self, *, server_id: str | None = None) -> list[ToolSpec]: """Fail discovery after entering the context.""" + del server_id async with self: raise MCPError("list failed") @@ -516,12 +519,12 @@ def make_backend(tool_name: str) -> FastMCP: first = MCPServer(FastMCPTransport(make_backend("one"))) second = MCPServer(FastMCPTransport(make_backend("two"))) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[first, second]), model=_fake_openai(MultiCallClient([]))) + harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[first, second])], model=_fake_openai(MultiCallClient([]))) await harness.connect() await harness.aclose() - metadata = {tool.name: tool.mcp.server_id for tool in harness.tools if tool.mcp is not None} + metadata = {tool.name: tool.origin.source for tool in harness.tools if tool.origin is not None} assert metadata == {"one": "FastMCPTransport", "two": "FastMCPTransport-2"} @@ -580,7 +583,8 @@ def hidden() -> str: backend.tool(hidden) server = MCPServer(FastMCPTransport(backend), id="semley", include_tools=["step", "reset_session"]) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[server]), + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([("step", '{"action":"go"}')])), ) @@ -897,8 +901,9 @@ async def test_harness_connects_mcp_once_across_async_runs(tmp_path, monkeypatch """Harness runs reuse the discovered MCP tools until aclose.""" server = scripted_server(monkeypatch, {"remote": _schema()}) client = MultiCallClient([("remote", '{"value":"ok"}')]) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[server]), model=_fake_openai(client)) + harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[server])], model=_fake_openai(client)) + assert harness.tools == [] result = await harness.run("go") second = await harness.run("done") await harness.aclose() @@ -911,10 +916,24 @@ async def test_harness_connects_mcp_once_across_async_runs(tmp_path, monkeypatch assert server.call_records == [("remote", {"value": "ok"})] +def test_mcp_plugin_name_is_unique_and_config_path_is_removed(tmp_path, monkeypatch) -> None: + """One fixed-name MCP plugin is allowed and the old config path is rejected.""" + first = scripted_server(monkeypatch, {"first": _schema()}) + second = scripted_server(monkeypatch, {"second": _schema()}) + + with pytest.raises(ValueError, match="duplicate plugin name: mcp"): + Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[first]), MCPPlugin(servers=[second])], + model=ScriptedModel([]), + ) + assert "mcp_servers" not in HarnessConfig.model_fields + + async def test_explicit_connect_does_not_reconnect_on_run(tmp_path, monkeypatch) -> None: """Explicit connect discovers MCP tools once before run.""" server = scripted_server(monkeypatch, {"remote": _schema()}) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[server]), model=_fake_openai(MultiCallClient([]))) + harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([]))) await harness.connect() result = await harness.run("go") @@ -936,7 +955,8 @@ async def test_is_error_drives_harness_retry(tmp_path, monkeypatch) -> None: ModelTurn(tool_calls=[ModelToolCall(id="call_2", name="error", arguments="{}")], raw={"id": "retry"}), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[server], tool_retries=1), + HarnessConfig(root=tmp_path, builtin_tools=[], tool_retries=1), + plugins=[MCPPlugin(servers=[server])], model=ScriptedModel([session]), hooks=[Hook("run_end", lambda ctx: run_end.append((ctx.stop_reason, dict(ctx.usage.tool_retries))))], ) @@ -950,15 +970,101 @@ async def test_is_error_drives_harness_retry(tmp_path, monkeypatch) -> None: assert run_end == [("tool_retries_exceeded", {"error": 2})] -async def test_partial_connect_failure_cleans_up(tmp_path, monkeypatch) -> None: - """A later MCP discovery failure closes earlier entered servers.""" - first = scripted_server(monkeypatch, {"ok": _schema()}) - second = FailingListServer(FastMCPTransport(FastMCP("failing-backend")), id="failing") - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[first, second]), model=_fake_openai(MultiCallClient([]))) +async def test_failed_second_server_entry_rolls_back_and_retries(tmp_path, monkeypatch) -> None: + """A second-server entry failure closes the first and leaves no contribution.""" + first = scripted_server(monkeypatch, {"first": _schema()}, id="first") + + class FailOnceEnterServer(ObservedMCPServer): + async def __aenter__(self) -> MCPServer: + """Fail the first outer entry, then use the normal wrapper.""" + self.entered += 1 + if self.entered == 1: + raise MCPError("entry failed") + return await MCPServer.__aenter__(self) + + scripted_second = scripted_server(monkeypatch, {"second": _schema()}, id="second") + second = FailOnceEnterServer(scripted_second._transport, id="second") + harness = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[first, second])], + model=ScriptedModel([]), + ) + + with pytest.raises(MCPError, match="entry failed"): + await harness.connect() + assert harness.tools == [] + assert first.exited == 2 + + await harness.connect() + assert [tool.name for tool in harness.tools] == ["first", "second"] + await harness.aclose() + + +async def test_partial_connect_failure_cleans_up(tmp_path) -> None: + """A later discovery failure closes earlier servers and permits retry.""" + first = observed_server("ok") + + class FailOnceListServer(ObservedMCPServer): + def __init__(self, transport: Any, **kwargs: Any) -> None: + super().__init__(transport, **kwargs) + self.attempts = 0 + + async def list_tools(self, *, server_id: str | None = None) -> list[ToolSpec]: + self.attempts += 1 + if self.attempts == 1: + async with self: + raise MCPError("list failed") + return await super().list_tools(server_id=server_id) + + backend = FastMCP("failing-backend") + backend.tool(_echo_handler("recovered", []), name="recovered") + second = FailOnceListServer(FastMCPTransport(backend), id="failing") + harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[first, second])], model=_fake_openai(MultiCallClient([]))) with pytest.raises(MCPError, match="list failed"): await harness.connect() + assert harness.tools == [] + assert first.exited == 2 + assert second.exited == 2 + + await harness.connect() + assert [tool.name for tool in harness.tools] == ["ok", "recovered"] + await harness.aclose() + + +async def test_direct_tool_collision_rolls_back_mcp(tmp_path, monkeypatch) -> None: + """Discovered MCP tools collide atomically with direct tools.""" + server = scripted_server(monkeypatch, {"shared": _schema()}) + direct = ToolSpec("shared", "Direct", _schema(), lambda _args: "direct") + harness = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[server])], + tools=[direct], + model=ScriptedModel([]), + ) + + with pytest.raises(ValueError, match=r"duplicate tool name: shared \(direct and mcp\)"): + await harness.connect() + + assert harness.tools == [direct] + assert server.exited == 2 + + +async def test_mcp_server_tool_collision_rolls_back_all_servers(tmp_path, monkeypatch) -> None: + """Duplicate tools from different MCP servers install no contribution.""" + first = scripted_server(monkeypatch, {"shared": _schema()}, id="first") + second = scripted_server(monkeypatch, {"shared": _schema()}, id="second") + harness = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[first, second])], + model=ScriptedModel([]), + ) + + with pytest.raises(ValueError, match=r"duplicate tool name: shared \(mcp and mcp\)"): + await harness.connect() + + assert harness.tools == [] assert first.exited == 2 assert second.exited == 2 @@ -968,12 +1074,12 @@ async def test_mcp_collision_detected_before_model_request(tmp_path, monkeypatch server = scripted_server(monkeypatch, {"read": _schema()}) client = MultiCallClient([]) harness = Harness( - HarnessConfig(root=tmp_path, mcp_servers=[server]), + HarnessConfig(root=tmp_path), model=_fake_openai(client), - plugins=[FilesystemPlugin(tools=["read"])], + plugins=[FilesystemPlugin(tools=["read"]), MCPPlugin(servers=[server])], ) - with pytest.raises(HarnessError, match="tool name collision"): + with pytest.raises(ValueError, match="duplicate tool name: read"): await harness.run("go") assert client.payloads == [] assert server.exited == 2 @@ -981,6 +1087,7 @@ async def test_mcp_collision_detected_before_model_request(tmp_path, monkeypatch async def test_final_result_mcp_collision_detected(tmp_path, monkeypatch) -> None: """MCP tools also collide with synthetic structured-output tools.""" + class Answer(BaseModel): """Structured output type.""" @@ -988,11 +1095,12 @@ class Answer(BaseModel): server = scripted_server(monkeypatch, {"final_result": _schema()}) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Answer, output_mode="tool", mcp_servers=[server]), + HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Answer, output_mode="tool"), + plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([])), ) - with pytest.raises(HarnessError, match="tool name collision"): + with pytest.raises(ValueError, match="reserved for structured output"): await harness.connect() @@ -1000,19 +1108,171 @@ async def test_duplicate_derived_id_disambiguated(tmp_path, monkeypatch) -> None """Duplicate MCP server ids get readable suffixes.""" first = scripted_server(monkeypatch, {"one": _schema()}, id="same") second = scripted_server(monkeypatch, {"two": _schema()}, id="same") - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[first, second]), model=_fake_openai(MultiCallClient([]))) + harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[first, second])], model=_fake_openai(MultiCallClient([]))) await harness.connect() await harness.aclose() - metadata = {tool.name: tool.mcp.server_id for tool in harness.tools if tool.mcp is not None} + metadata = {tool.name: tool.origin.source for tool in harness.tools if tool.origin is not None} assert metadata == {"one": "same", "two": "same-2"} +async def test_binding_local_ids_stay_stable_for_shared_server(tmp_path, monkeypatch) -> None: + """One shared wrapper gets independent ids, handlers, and trace attribution.""" + shared = scripted_server(monkeypatch, {"shared": _schema()}, id="same") + first_neighbor = scripted_server(monkeypatch, {"first_neighbor": _schema()}, id="same") + second_neighbor = scripted_server(monkeypatch, {"second_neighbor": _schema()}, id="same") + first_tracer = FakeTracer() + second_tracer = FakeTracer() + first = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[shared, first_neighbor])], + model=_fake_openai(MultiCallClient([("shared", '{"value":"first"}')])), + tracing=[TracingOptions(tracer=first_tracer)], + ) + second = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[second_neighbor, shared])], + model=_fake_openai(MultiCallClient([("shared", '{"value":"second"}')])), + tracing=[TracingOptions(tracer=second_tracer)], + ) + + assert shared.id == "same" + await first.connect() + await second.connect() + first_tool = next(tool for tool in first.tools if tool.name == "shared") + second_tool = next(tool for tool in second.tools if tool.name == "shared") + first_result = await first_tool.handler({"value": "direct-first"}) + second_result = await second_tool.handler({"value": "direct-second"}) + + assert first_tool.origin == ToolOrigin(plugin="mcp", source="same", attributes={"tool_name": "shared"}) + assert second_tool.origin == ToolOrigin(plugin="mcp", source="same-2", attributes={"tool_name": "shared"}) + assert first_result.metadata["mcp_server_id"] == "same" + assert second_result.metadata["mcp_server_id"] == "same-2" + assert shared.id == "same" + + await first.run("first") + await second.run("second") + first_span = next(span for span in first_tracer.spans if span.name == "execute_tool shared") + second_span = next(span for span in second_tracer.spans if span.name == "execute_tool shared") + assert first_span.attributes["mcp.server.id"] == "same" + assert second_span.attributes["mcp.server.id"] == "same-2" + await second.aclose() + await first.aclose() + + +async def test_mcp_connect_cancellation_rolls_back_and_retries(tmp_path) -> None: + """Cancellation during discovery closes the server and leaves a clean retry.""" + entered_discovery = asyncio.Event() + + class CancelOnceListServer(ObservedMCPServer): + def __init__(self, transport: Any, **kwargs: Any) -> None: + super().__init__(transport, **kwargs) + self.attempts = 0 + + async def list_tools(self, *, server_id: str | None = None) -> list[ToolSpec]: + self.attempts += 1 + if self.attempts == 1: + entered_discovery.set() + await asyncio.Event().wait() + return await MCPServer.list_tools(self, server_id=server_id) + + backend = FastMCP("cancel-discovery") + backend.tool(_echo_handler("remote", []), name="remote") + server = CancelOnceListServer(FastMCPTransport(backend), id="cancel") + harness = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[server])], + model=ScriptedModel([]), + ) + connecting = asyncio.create_task(harness.connect()) + await entered_discovery.wait() + + connecting.cancel() + with pytest.raises(asyncio.CancelledError): + await connecting + assert harness.tools == [] + assert server.exited == 1 + + await harness.connect() + assert [tool.name for tool in harness.tools] == ["remote"] + await harness.aclose() + + +async def test_mcp_plugin_close_cancellation_propagates(tmp_path) -> None: + """Cancellation during harness close propagates after MCP cleanup.""" + + @asynccontextmanager + async def slow_stop(server: Any): + try: + yield {} + finally: + await asyncio.sleep(1) + + backend = FastMCP("plugin-slow-stop", lifespan=slow_stop) + backend.tool(_echo_handler("remote", []), name="remote") + server = MCPServer(FastMCPTransport(backend), id="slow-stop") + harness = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[server])], + model=ScriptedModel([]), + ) + await harness.connect() + closing = asyncio.create_task(harness.aclose()) + await asyncio.sleep(0.1) + + closing.cancel() + with pytest.raises(asyncio.CancelledError): + await closing + + assert harness._closed is True + async with server: + result = await server.call_tool("remote", {"value": "reused"}) + assert result.ok is True + + +async def test_mcp_and_other_plugins_close_in_reverse_order(tmp_path) -> None: + """Generic plugin order controls MCP cleanup with no core special case.""" + events: list[str] = [] + + class LoggingServer(ObservedMCPServer): + async def __aexit__(self, *exc: object) -> None: + events.append("mcp") + await super().__aexit__(*exc) + + class OtherPlugin: + name = "other" + + def bind(self, context) -> PluginBinding: + @asynccontextmanager + async def connect(): + try: + yield PluginContribution() + finally: + events.append("other") + + return PluginBinding(connect=connect) + + backend = FastMCP("close-order") + backend.tool(_echo_handler("remote", []), name="remote") + server = LoggingServer(FastMCPTransport(backend), id="close-order") + harness = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[server]), OtherPlugin()], + model=ScriptedModel([]), + ) + + await harness.connect() + events.clear() + await harness.aclose() + + assert events == ["other", "mcp"] + + async def test_closed_harness_rejects_run_and_connect_but_keeps_schema(tmp_path, monkeypatch) -> None: """Closed harnesses are terminal but still inspectable.""" server = scripted_server(monkeypatch, {"remote": _schema()}) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[server]), model=_fake_openai(MultiCallClient([]))) + harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([]))) await harness.connect() await harness.aclose() @@ -1041,7 +1301,7 @@ async def test_aclose_with_injected_model_closes_mcp(tmp_path, monkeypatch) -> N """Harness-owned MCP resources close even when the model is injected.""" server = scripted_server(monkeypatch, {"remote": _schema()}) model = _fake_openai(MultiCallClient([])) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[server]), model=model) + harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[server])], model=model) await harness.connect() await harness.aclose() @@ -1066,11 +1326,11 @@ async def test_unknown_tool_hook_filter_is_allowed_and_never_fires(tmp_path) -> def test_default_subagent_does_not_implicitly_inherit_mcp(tmp_path, monkeypatch) -> None: """MCP inheritance for child harnesses is explicit.""" server = scripted_server(monkeypatch, {"remote": _schema()}) - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[server]), model=ScriptedModel([])) + parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[server])], model=ScriptedModel([])) child = build_child_harness(parent, None) - assert child.config.mcp_servers == [] + assert not any(isinstance(plugin, MCPPlugin) for plugin in child.plugins) def test_subagent_empty_fails_validation() -> None: @@ -1083,25 +1343,30 @@ def test_subagent_mcp_override_and_union_config(tmp_path, monkeypatch) -> None: """Child config encodes MCP override and union semantics.""" parent_server = scripted_server(monkeypatch, {"parent": _schema()}) child_server = scripted_server(monkeypatch, {"child": _schema()}) - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[parent_server]), model=ScriptedModel([])) + parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[parent_server])], model=ScriptedModel([])) override = build_child_harness(parent, SubAgentConfig(name="override", description="Override helper.", mcp_servers=[child_server])) - union = build_child_harness(parent, SubAgentConfig( - name="union", - description="Union helper.", - inherit_mcp_servers=True, - mcp_servers=[parent_server, child_server], - )) + union = build_child_harness( + parent, + SubAgentConfig( + name="union", + description="Union helper.", + inherit_mcp_servers=True, + mcp_servers=[parent_server, child_server], + ), + ) - assert override.config.mcp_servers == [child_server] - assert union.config.mcp_servers == [parent_server, child_server] + override_plugin = next(plugin for plugin in override.plugins if isinstance(plugin, MCPPlugin)) + union_plugin = next(plugin for plugin in union.plugins if isinstance(plugin, MCPPlugin)) + assert override_plugin.servers == (child_server,) + assert union_plugin.servers == (parent_server, child_server) async def test_subagent_overrides_mcp_only_runtime(tmp_path, monkeypatch) -> None: """An override-only child sees explicit MCP servers but not parent MCP servers.""" parent_server = scripted_server(monkeypatch, {"parent": _schema()}) child_server = scripted_server(monkeypatch, {"child": _schema()}) - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[parent_server]), model=ScriptedModel([])) + parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[parent_server])], model=ScriptedModel([])) child = build_child_harness(parent, SubAgentConfig(name="override", description="Override helper.", mcp_servers=[child_server])) await child.connect() @@ -1114,14 +1379,17 @@ async def test_subagent_unions_inherit_plus_override_runtime(tmp_path, monkeypat """An inherited-plus-override child sees both MCP tool sets.""" parent_server = scripted_server(monkeypatch, {"parent": _schema()}) child_server = scripted_server(monkeypatch, {"child": _schema()}) - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[parent_server]), model=ScriptedModel([])) - - child = build_child_harness(parent, SubAgentConfig( - name="union", - description="Union helper.", - inherit_mcp_servers=True, - mcp_servers=[child_server], - )) + parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[parent_server])], model=ScriptedModel([])) + + child = build_child_harness( + parent, + SubAgentConfig( + name="union", + description="Union helper.", + inherit_mcp_servers=True, + mcp_servers=[child_server], + ), + ) await child.connect() await child.aclose() @@ -1131,49 +1399,92 @@ async def test_subagent_unions_inherit_plus_override_runtime(tmp_path, monkeypat async def test_subagent_identity_dedup_runtime(tmp_path, monkeypatch) -> None: """The same inherited and explicit MCP object is entered once in a child.""" server = scripted_server(monkeypatch, {"remote": _schema()}) - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[server]), model=ScriptedModel([])) - - child = build_child_harness(parent, SubAgentConfig( - name="dedup", - description="Dedup helper.", - inherit_mcp_servers=True, - mcp_servers=[server], - )) + parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[server])], model=ScriptedModel([])) + + child = build_child_harness( + parent, + SubAgentConfig( + name="dedup", + description="Dedup helper.", + inherit_mcp_servers=True, + mcp_servers=[server], + ), + ) await child.connect() await child.aclose() assert [tool.name for tool in child.tools] == ["remote"] - assert child.config.mcp_servers == [server] + child_plugin = next(plugin for plugin in child.plugins if isinstance(plugin, MCPPlugin)) + assert child_plugin.servers == (server,) async def test_subagent_id_equal_but_distinct_collides(tmp_path, monkeypatch) -> None: """Distinct MCP objects with identical tools collide in child connect.""" first = scripted_server(monkeypatch, {"remote": _schema()}, id="same") second = scripted_server(monkeypatch, {"remote": _schema()}, id="same") - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[first]), model=ScriptedModel([])) - child = build_child_harness(parent, SubAgentConfig( - name="child", - description="Child helper.", - inherit_mcp_servers=True, - mcp_servers=[second], - )) - - with pytest.raises(HarnessError, match="tool name collision"): + parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[first])], model=ScriptedModel([])) + child = build_child_harness( + parent, + SubAgentConfig( + name="child", + description="Child helper.", + inherit_mcp_servers=True, + mcp_servers=[second], + ), + ) + + with pytest.raises(ValueError, match="duplicate tool name: remote"): + await child.connect() + + +async def test_child_second_server_failure_keeps_parent_shared_session_live(tmp_path) -> None: + """Child rollback releases only its reference to an inherited parent server.""" + shared = observed_server("remote", id="shared") + failing = FailingListServer(FastMCPTransport(FastMCP("child-failing")), id="failing") + parent = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[shared])], + model=ScriptedModel([]), + ) + await parent.connect() + child = build_child_harness( + parent, + SubAgentConfig( + name="child", + description="Child helper.", + inherit_mcp_servers=True, + mcp_servers=[failing], + ), + ) + + with pytest.raises(MCPError, match="list failed"): await child.connect() + assert shared.backend_log == {"starts": 1, "stops": 0} + parent_tool = next(tool for tool in parent.tools if tool.name == "remote") + result = await parent_tool.handler({"value": "still-live"}) + assert result.ok is True + assert shared.backend_log == {"starts": 1, "stops": 0} + await child.aclose() + await parent.aclose() + assert shared.backend_log == {"starts": 1, "stops": 1} + async def test_subagent_inherited_parent_tools_skip_mcp_duplicates(tmp_path, monkeypatch) -> None: """Parent MCP tools are not copied as custom tools when also inherited as MCP.""" server = scripted_server(monkeypatch, {"remote": _schema()}) - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[server]), model=ScriptedModel([])) + parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[server])], model=ScriptedModel([])) await parent.connect() - child = build_child_harness(parent, SubAgentConfig( - name="child", - description="Child helper.", - inherit_parent_tools=True, - inherit_mcp_servers=True, - )) + child = build_child_harness( + parent, + SubAgentConfig( + name="child", + description="Child helper.", + inherit_parent_tools=True, + inherit_mcp_servers=True, + ), + ) await child.connect() await child.aclose() await parent.aclose() @@ -1187,7 +1498,8 @@ async def test_subagent_mcp_only_validates_and_inherits(tmp_path, monkeypatch) - config = SubAgentConfig(name="mcp", description="MCP helper.", inherit_mcp_servers=True) parent_client = MultiCallClient([("subagent", '{"task":"use remote","agent":"mcp"}')]) parent = Harness( - HarnessConfig(root=tmp_path, builtin_tools=["subagent"], mcp_servers=[server], subagents=[config]), + HarnessConfig(root=tmp_path, builtin_tools=["subagent"], subagents=[config]), + plugins=[MCPPlugin(servers=[server])], model=_fake_openai(parent_client), ) @@ -1206,7 +1518,8 @@ def boom(_args): raise RuntimeError("boom") harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[server]), + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([("boom", "{}")])), tools=[ToolSpec("boom", "Boom", {"type": "object", "properties": {}}, boom)], ) @@ -1227,7 +1540,8 @@ async def slow(_args): await asyncio.sleep(60) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[server]), + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([("slow", "{}")])), tools=[ToolSpec("slow", "Slow", {"type": "object", "properties": {}}, slow)], ) @@ -1247,7 +1561,8 @@ async def test_subagent_effective_tools_include_mcp(tmp_path, monkeypatch) -> No server = scripted_server(monkeypatch, {"remote": _schema()}) config = SubAgentConfig(name="mcp", description="MCP helper.", inherit_mcp_servers=True) parent = Harness( - HarnessConfig(root=tmp_path, builtin_tools=["subagent"], mcp_servers=[server], subagents=[config]), + HarnessConfig(root=tmp_path, builtin_tools=["subagent"], subagents=[config]), + plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([("subagent", '{"task":"use remote","agent":"mcp"}')])), hooks=[Hook("after_subagent_run", lambda ctx: seen_tools.append(ctx.tools), agents=["mcp"])], ) @@ -1263,7 +1578,9 @@ async def test_resume_with_mcp_reuses_connection_and_keeps_state_clean(tmp_path, server = scripted_server(monkeypatch, {"remote": _schema()}) first_session = SequenceSession(ModelTurn(text="first", raw={"id": "first"})) second_session = SequenceSession(ModelTurn(text="second", raw={"id": "second"})) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[server]), model=ScriptedModel([first_session, second_session])) + harness = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[server])], model=ScriptedModel([first_session, second_session]) + ) first = await harness.run("first") second = await harness.run("second", resume_from=first.resume_state) @@ -1299,7 +1616,8 @@ async def test_approval_resume_connects_mcp_before_validating_and_preserves_unkn requires_approval=True, ) first_harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[server]), + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[server])], model=model, tools=[approval_tool], ) @@ -1307,7 +1625,8 @@ async def test_approval_resume_connects_mcp_before_validating_and_preserves_unkn await first_harness.aclose() second_harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[server]), + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[server])], model=model, tools=[approval_tool], ) @@ -1334,7 +1653,8 @@ def rewrite(ctx) -> None: ctx.output = ToolResult(True, "rewritten", {}).as_json() harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[server]), + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([("remote", '{"value":"ok"}')])), hooks=[Hook("after_tool_call", rewrite, tools=["remote"])], tracing=[TracingOptions(tracer=tracer)], @@ -1354,13 +1674,15 @@ async def test_connection_failure_happens_before_run_hooks(tmp_path) -> None: tracer = FakeTracer() class FailingConnectServer(ObservedMCPServer): - async def list_tools(self) -> list[ToolSpec]: + async def list_tools(self, *, server_id: str | None = None) -> list[ToolSpec]: """Fail MCP setup.""" + del server_id raise MCPError("connect failed") failing = FailingConnectServer(FastMCPTransport(FastMCP("failing-connect")), id="failing") harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], mcp_servers=[failing]), + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[failing])], model=_fake_openai(MultiCallClient([])), hooks=[ Hook("run_start", lambda ctx: events.append("start")), diff --git a/tests/unit/test_mcp_optional_dependency.py b/tests/unit/test_mcp_optional_dependency.py index 9e4e4d7..3bf52e4 100644 --- a/tests/unit/test_mcp_optional_dependency.py +++ b/tests/unit/test_mcp_optional_dependency.py @@ -7,7 +7,7 @@ import pytest -from thinharness import MCPDependencyError, MCPServer, MCPServerSSE, MCPServerStdio, MCPServerStreamableHTTP +from thinharness import MCPDependencyError, MCPPlugin, MCPServer, MCPServerSSE, MCPServerStdio, MCPServerStreamableHTTP def _block_imports(monkeypatch: pytest.MonkeyPatch, blocked: set[str]) -> None: @@ -33,6 +33,7 @@ async def test_construction_without_extra(monkeypatch: pytest.MonkeyPatch) -> No MCPServerStreamableHTTP(url="http://localhost/mcp"), MCPServer(object()), ] + assert MCPPlugin(servers=servers).servers == tuple(servers) for server in servers: with pytest.raises(MCPDependencyError, match="thinharness\\[mcp\\]"): @@ -70,6 +71,8 @@ def find_spec(self, name, path=None, target=None): thinharness.MCPServerStreamableHTTP(url="http://localhost/mcp"), thinharness.MCPServer(object()), ] +plugin = thinharness.MCPPlugin(servers=servers) +assert plugin.servers == tuple(servers) async def main(): for server in servers: diff --git a/tests/unit/test_plugins.py b/tests/unit/test_plugins.py index 7e5591a..bafbb2e 100644 --- a/tests/unit/test_plugins.py +++ b/tests/unit/test_plugins.py @@ -2,7 +2,7 @@ import asyncio import json -from contextlib import AsyncExitStack, asynccontextmanager +from contextlib import asynccontextmanager from pathlib import Path import pytest @@ -326,10 +326,12 @@ def test_plugin_name_must_be_string(tmp_path: Path) -> None: async def test_sequential_runs_reuse_one_connection(tmp_path: Path) -> None: events: list[str] = [] plugin = ConnectedPlugin("connected", events, PluginContribution()) - model = ScriptedModel([ - ScriptedSession(start_turn=ModelTurn(text="first", raw={"id": "first"})), - ScriptedSession(start_turn=ModelTurn(text="second", raw={"id": "second"})), - ]) + model = ScriptedModel( + [ + ScriptedSession(start_turn=ModelTurn(text="first", raw={"id": "first"})), + ScriptedSession(start_turn=ModelTurn(text="second", raw={"id": "second"})), + ] + ) harness = Harness(HarnessConfig(root=tmp_path), model=model, plugins=[plugin]) assert (await harness.run("one")).text == "first" @@ -444,7 +446,6 @@ async def connect(): await connection assert exited.is_set() assert harness._plugin_stack is None - assert harness._mcp_stack is None with pytest.raises(HarnessError, match="harness is closed"): await harness.connect() @@ -521,7 +522,7 @@ async def connect(): assert harness.tools == [] -async def test_close_attempts_all_resources_after_failures(tmp_path: Path) -> None: +async def test_close_attempts_model_after_plugin_failure(tmp_path: Path) -> None: events: list[str] = [] class FailingClosePlugin: @@ -554,20 +555,11 @@ async def aclose(self) -> None: _owns_model=True, ) await harness.connect() - mcp_stack = AsyncExitStack() - - async def close_mcp() -> None: - events.append("mcp") - raise RuntimeError("mcp close failed") - mcp_stack.push_async_callback(close_mcp) - harness._mcp_stack = mcp_stack - - with pytest.raises(RuntimeError, match="mcp close failed"): + with pytest.raises(RuntimeError, match="plugin close failed"): await harness.aclose() - assert events == ["mcp", "plugin", "model"] - assert harness._mcp_stack is None + assert events == ["plugin", "model"] assert harness._plugin_stack is None @@ -646,9 +638,20 @@ def register(_args): harness.add_tool(_tool("late")) return "registered" - plugin = ConnectedPlugin("dynamic", [], PluginContribution(tools=(ToolSpec( - "register", "register", {"type": "object", "properties": {}}, register, - ),))) + plugin = ConnectedPlugin( + "dynamic", + [], + PluginContribution( + tools=( + ToolSpec( + "register", + "register", + {"type": "object", "properties": {}}, + register, + ), + ) + ), + ) harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([FreezeSession()]), plugins=[plugin]) assert (await harness.run("go")).text == "done" diff --git a/thinharness/__init__.py b/thinharness/__init__.py index 0d86cb7..67a412f 100644 --- a/thinharness/__init__.py +++ b/thinharness/__init__.py @@ -37,7 +37,7 @@ UserPromptSubmitContext, ) from .output import NativeOutput, OutputSchema, PromptedOutput, TextOutput, ToolStructuredOutput -from .plugins import FilesystemPlugin, Plugin, PluginBinding, PluginConnector, PluginContext, PluginContribution +from .plugins import FilesystemPlugin, MCPPlugin, Plugin, PluginBinding, PluginConnector, PluginContext, PluginContribution from .providers import ( AnthropicMessagesModel, AnthropicProvider, @@ -73,7 +73,6 @@ MCPServerSSE, MCPServerStdio, MCPServerStreamableHTTP, - McpToolInfo, ModelRetry, ParallelLlmArgs, ParallelLlmTool, @@ -131,6 +130,7 @@ "ModelRetry", "UnexpectedModelBehavior", "MCPDependencyError", + "MCPPlugin", "MCPError", "MCPServer", "MCPServerSSE", @@ -182,7 +182,6 @@ "PluginConnector", "PluginContext", "PluginContribution", - "McpToolInfo", "ToolEnvelope", "ToolOrigin", "ParallelLlmArgs", diff --git a/thinharness/core.py b/thinharness/core.py index dfe9dad..660f5b0 100644 --- a/thinharness/core.py +++ b/thinharness/core.py @@ -58,7 +58,6 @@ ) from .subagents import DEFAULT_SUBAGENT_NAME, SubAgentConfig, create_subagent_tool from .tools.base import ToolOrigin, ToolSpec -from .tools.mcp import MCPServer from .tools.parallel_llm import create_parallel_llm_tool from .tools.skills import SkillRegistry from .tracing import ( @@ -137,7 +136,6 @@ class HarnessConfig(BaseModel): builtin_parallel_llm_model: str | None = None builtin_parallel_llm_temperature: float | None = None parallel_llm_max_prompts: int = Field(default=100, ge=1) - mcp_servers: list[MCPServer] = Field(default_factory=list) @model_validator(mode="after") def validate_config(self) -> HarnessConfig: @@ -238,9 +236,6 @@ def __init__( self._plugin_instructions = list(static_instructions) self.hooks = hook_registry self.subagent_hooks = subagent_hooks or {} - self._mcp_servers = list(self.config.mcp_servers) - self._resolve_mcp_server_ids() - self._mcp_stack: AsyncExitStack | None = None self._plugin_stack: AsyncExitStack | None = None self._connected = False self._connect_lock = asyncio.Lock() @@ -347,15 +342,17 @@ def stream( ) emitter = StreamEmitter(stream_context) loop = asyncio.get_running_loop() - task = loop.create_task(self._run_streaming( - prompt, - resume_from=resume_from, - approval_state=None, - approval_decisions=None, - metadata=metadata, - emitter=emitter, - stream_context=stream_context, - )) + task = loop.create_task( + self._run_streaming( + prompt, + resume_from=resume_from, + approval_state=None, + approval_decisions=None, + metadata=metadata, + emitter=emitter, + stream_context=stream_context, + ) + ) self._running = True return HarnessStream(task, emitter) @@ -375,15 +372,17 @@ def stream_approvals( stream_context = create_stream_context(options=stream_options) emitter = StreamEmitter(stream_context) loop = asyncio.get_running_loop() - task = loop.create_task(self._run_streaming( - "", - resume_from=None, - approval_state=state, - approval_decisions=decisions, - metadata=metadata, - emitter=emitter, - stream_context=stream_context, - )) + task = loop.create_task( + self._run_streaming( + "", + resume_from=None, + approval_state=state, + approval_decisions=decisions, + metadata=metadata, + emitter=emitter, + stream_context=stream_context, + ) + ) self._running = True return HarnessStream(task, emitter) @@ -435,30 +434,36 @@ async def _run_streaming( run_ctx.responses = restored_responses run_ctx.tool_call_records = restored_records run_ctx.emitted_limit_warnings = restored_warnings - run_ctx.emit(RunStartedEvent( - **run_ctx.stream_base(), - prompt=None if approval_pause is not None else prompt, - root=str(self.root), - max_model_requests=self.config.max_model_requests, - max_tool_calls=self.config.max_tool_calls, - )) - if approval_pause is not None: - run_ctx.emit(ApprovalResumedEvent( + run_ctx.emit( + RunStartedEvent( **run_ctx.stream_base(), - decisions=tuple(approval_decisions or []), - )) + prompt=None if approval_pause is not None else prompt, + root=str(self.root), + max_model_requests=self.config.max_model_requests, + max_tool_calls=self.config.max_tool_calls, + ) + ) + if approval_pause is not None: + run_ctx.emit( + ApprovalResumedEvent( + **run_ctx.stream_base(), + decisions=tuple(approval_decisions or []), + ) + ) except BaseException as exc: self._running = False - emitter.emit(RunFailedEvent( - run_id=stream_context.run_id, - sequence=0, - parent_run_id=stream_context.parent_run_id, - parent_tool_call_id=stream_context.parent_tool_call_id, - agent_name=stream_context.agent_name, - stop_reason="cancelled" if isinstance(exc, asyncio.CancelledError) else "error", - error_type=type(exc).__name__, - message=str(exc), - )) + emitter.emit( + RunFailedEvent( + run_id=stream_context.run_id, + sequence=0, + parent_run_id=stream_context.parent_run_id, + parent_tool_call_id=stream_context.parent_tool_call_id, + agent_name=stream_context.agent_name, + stop_reason="cancelled" if isinstance(exc, asyncio.CancelledError) else "error", + error_type=type(exc).__name__, + message=str(exc), + ) + ) emitter.finish() raise @@ -534,12 +539,14 @@ async def _run_streaming( run_ctx.terminal_error = exc if run_ctx.stop_reason == "end_turn": run_ctx.stop_reason = "error" - run_ctx.emit(RunFailedEvent( - **run_ctx.stream_base(), - stop_reason=run_ctx.stop_reason, - error_type=type(exc).__name__, - message=str(exc), - )) + run_ctx.emit( + RunFailedEvent( + **run_ctx.stream_base(), + stop_reason=run_ctx.stop_reason, + error_type=type(exc).__name__, + message=str(exc), + ) + ) raise finally: self._running = False @@ -555,14 +562,16 @@ async def _prepare_run_start( skip_user_prompt: bool = False, ) -> tuple[str, str]: """Fire start hooks and return the effective prompt plus instructions.""" - self.hooks.fire(RunStartContext( - harness=self, - metadata=dict(run_metadata), - prompt=prompt, - root=self.root, - max_model_requests=self.config.max_model_requests, - max_tool_calls=self.config.max_tool_calls, - )) + self.hooks.fire( + RunStartContext( + harness=self, + metadata=dict(run_metadata), + prompt=prompt, + root=self.root, + max_model_requests=self.config.max_model_requests, + max_tool_calls=self.config.max_tool_calls, + ) + ) effective_prompt = prompt if not skip_user_prompt: prompt_ctx = UserPromptSubmitContext(harness=self, metadata=dict(run_metadata), prompt=prompt) @@ -592,7 +601,7 @@ def _resume_approval_session(self, provider_state: Json) -> ModelSession: except HarnessError as exc: message = str(exc) if message.startswith("resume_from"): - message = f"approval state provider_state{message[len('resume_from'):]}" + message = f"approval state provider_state{message[len('resume_from') :]}" raise HarnessError(message) from exc def _pending_approval_record(self, call: ModelToolCall) -> PendingApproval: @@ -620,7 +629,7 @@ async def _run_and_close() -> HarnessResult: return asyncio.run(_run_and_close()) async def aclose(self) -> None: - """Close connected plugins, MCP servers, and an owned model.""" + """Close connected plugins and an owned model.""" async with self._connect_lock: if self._closed: return @@ -637,13 +646,10 @@ async def aclose(self) -> None: caller_cancelled = current_task is not None and current_task.cancelling() > pending_cancels except BaseException: pass - mcp_stack = self._mcp_stack plugin_stack = self._plugin_stack - self._mcp_stack = None self._plugin_stack = None self._connected = False close_error = await self._close_resources( - mcp_stack=mcp_stack, plugin_stack=plugin_stack, close_model=self._owns_model, ) @@ -755,11 +761,7 @@ def _validate_tool_spec( raise TypeError(f"handler for tool {spec.name!r} is not callable") if spec.name == "subagent" and spec.kind != "subagent": raise ValueError("subagent is a reserved tool name") - if ( - spec.name == FINAL_RESULT_TOOL_NAME - and output_schema is not None - and output_schema.mode != "text" - ): + if spec.name == FINAL_RESULT_TOOL_NAME and output_schema is not None and output_schema.mode != "text": raise ValueError(f"{FINAL_RESULT_TOOL_NAME} is reserved for structured output") Harness._validate_tool_approval_policy_for( spec, @@ -811,14 +813,8 @@ def _model_supports_approval_resume(self) -> bool: """Return whether this harness model can resume provider sessions.""" return hasattr(self.model, "resume_kind") and hasattr(self.model, "resume_session") - def _resolve_mcp_server_ids(self) -> None: - """Assign stable suffixes to duplicate MCP server ids.""" - counts: dict[str, int] = {} - for server in self._mcp_servers: - server.resolve_id(counts) - async def connect(self) -> None: - """Open connected plugins and the temporary MCP bridge.""" + """Open connected plugins.""" if self._closed: raise HarnessError("harness is closed") await self._ensure_connected() @@ -844,18 +840,12 @@ async def _ensure_connected(self) -> None: finally: async with self._connect_lock: self._connect_waiters -= 1 - if ( - self._connect_waiters == 0 - and task.done() - and not self._connected - and self._connect_task is task - ): + if self._connect_waiters == 0 and task.done() and not self._connected and self._connect_task is task: self._connect_task = None async def _connect_once(self) -> None: """Open every dynamic contribution for one shared connection attempt.""" plugin_stack = AsyncExitStack() - mcp_stack: AsyncExitStack | None = None base_hooks = list(self.hooks.hooks) try: dynamic_tools: list[ToolSpec] = [] @@ -881,30 +871,18 @@ async def _connect_once(self) -> None: self._validate_hook_registry(candidate_hooks, self.config.subagents) self._validate_skill_tool_selection_for(self.skills, candidate_tools) - mcp_stack, mcp_tools = await self._open_mcp_tools(candidate_tools) - all_tools = [*candidate_tools, *mcp_tools] - self._validate_tool_list( - all_tools, - output_schema=self.output_schema, - model_supports_approval_resume=self._model_supports_approval_resume(), - is_child_run=self._is_child_run, - ) if self._closed: raise HarnessError("harness is closed") - self.tools = all_tools - self._tool_map = {tool.name: tool for tool in all_tools} + self.tools = candidate_tools + self._tool_map = {tool.name: tool for tool in candidate_tools} self._plugin_instructions = [*self._base_instructions, *dynamic_instructions] self.hooks = candidate_hooks - self._skills_enabled = bool(self.skills.skills) and any( - tool.name in {"skill_read", "skill_run"} for tool in self.tools - ) + self._skills_enabled = bool(self.skills.skills) and any(tool.name in {"skill_read", "skill_run"} for tool in self.tools) self._plugin_stack = plugin_stack - self._mcp_stack = mcp_stack self._connected = True except BaseException as exc: cleanup_error = await self._close_resources( - mcp_stack=mcp_stack, plugin_stack=plugin_stack, close_model=False, ) @@ -912,9 +890,7 @@ async def _connect_once(self) -> None: self._tool_map = {tool.name: tool for tool in self.tools} self._plugin_instructions = list(self._base_instructions) self.hooks = HookRegistry(base_hooks, strict_hooks=self._strict_hooks) - self._skills_enabled = bool(self.skills.skills) and any( - tool.name in {"skill_read", "skill_run"} for tool in self.tools - ) + self._skills_enabled = bool(self.skills.skills) and any(tool.name in {"skill_read", "skill_run"} for tool in self.tools) if cleanup_error is not None: exc.add_note(f"cleanup also failed: {type(cleanup_error).__name__}: {cleanup_error}") raise @@ -922,20 +898,16 @@ async def _connect_once(self) -> None: async def _close_resources( self, *, - mcp_stack: AsyncExitStack | None, plugin_stack: AsyncExitStack | None, close_model: bool, ) -> BaseException | None: """Attempt every close in order and return the first failure.""" first_error: BaseException | None = None - for stack in (mcp_stack, plugin_stack): - if stack is None: - continue + if plugin_stack is not None: try: - await stack.aclose() + await plugin_stack.aclose() except BaseException as exc: - if first_error is None: - first_error = exc + first_error = exc if close_model: aclose = getattr(self.model.provider, "aclose", None) if aclose is not None: @@ -946,31 +918,6 @@ async def _close_resources( first_error = exc return first_error - async def _open_mcp_tools(self, existing_tools: list[ToolSpec]) -> tuple[AsyncExitStack | None, list[ToolSpec]]: - """Open the temporary MCP bridge and stage its discovered tools.""" - if not self._mcp_servers: - return None, [] - stack = AsyncExitStack() - try: - mcp_tools: list[ToolSpec] = [] - seen = {tool.name for tool in existing_tools} - if self.output_schema is not None and self.output_schema.mode == "tool": - seen.add(FINAL_RESULT_TOOL_NAME) - for server in self._mcp_servers: - await stack.enter_async_context(server) - for tool in await server.list_tools(): - if tool.name in seen: - raise HarnessError( - f"MCP tool name collision for {tool.name!r}; use tool_prefix or exclude_tools to disambiguate" - ) - self._validate_tool_approval_policy(tool) - seen.add(tool.name) - mcp_tools.append(tool) - return stack, mcp_tools - except BaseException: - await stack.aclose() - raise - @staticmethod def _normalize_contribution(plugin_name: str, contribution: PluginContribution) -> PluginContribution: """Validate contribution values and stamp missing tool provenance.""" diff --git a/thinharness/plugins/__init__.py b/thinharness/plugins/__init__.py index 6b7848f..5c9a5eb 100644 --- a/thinharness/plugins/__init__.py +++ b/thinharness/plugins/__init__.py @@ -2,9 +2,11 @@ from .base import Plugin, PluginBinding, PluginConnector, PluginContext, PluginContribution, ToolOrigin from .filesystem import FilesystemPlugin +from .mcp import MCPPlugin __all__ = [ "FilesystemPlugin", + "MCPPlugin", "Plugin", "PluginBinding", "PluginConnector", diff --git a/thinharness/plugins/mcp.py b/thinharness/plugins/mcp.py new file mode 100644 index 0000000..c536fd0 --- /dev/null +++ b/thinharness/plugins/mcp.py @@ -0,0 +1,71 @@ +"""Model Context Protocol plugin composition.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator, Sequence +from contextlib import AsyncExitStack, asynccontextmanager +from dataclasses import dataclass + +from ..tools.base import ToolSpec +from ..tools.mcp import MCPServer +from .base import PluginBinding, PluginContext, PluginContribution + + +@dataclass(frozen=True) +class _BoundServer: + """One server wrapper with identity local to a plugin binding.""" + + server: MCPServer + resolved_id: str + + async def list_tools(self) -> list[ToolSpec]: + """Discover tools with handlers and attribution fixed to this binding.""" + return await self.server.list_tools(server_id=self.resolved_id) + + +class MCPPlugin: + """Expose one ordered group of MCP servers through a harness plugin.""" + + name = "mcp" + + def __init__(self, *, servers: Sequence[MCPServer]) -> None: + if isinstance(servers, (set, frozenset)): + raise TypeError("MCPPlugin servers must be an ordered sequence, not a set") + self.servers = tuple(servers) + if any(not isinstance(server, MCPServer) for server in self.servers): + raise TypeError("MCPPlugin servers must contain only MCPServer values") + + def bind(self, context: PluginContext) -> PluginBinding: + """Resolve binding-local server ids without opening a connection.""" + del context + counts: dict[str, int] = {} + bound_servers: list[_BoundServer] = [] + for server in self.servers: + base_id = server.id + counts[base_id] = counts.get(base_id, 0) + 1 + resolved_id = base_id if counts[base_id] == 1 else f"{base_id}-{counts[base_id]}" + bound_servers.append(_BoundServer(server, resolved_id)) + snapshot = tuple(bound_servers) + + @asynccontextmanager + async def connect() -> AsyncIterator[PluginContribution]: + stack = AsyncExitStack() + try: + tools: list[ToolSpec] = [] + for bound in snapshot: + await stack.enter_async_context(bound.server) + tools.extend(await bound.list_tools()) + yield PluginContribution(tools=tuple(tools)) + except BaseException as exc: + try: + await stack.aclose() + except BaseException as cleanup_error: + exc.add_note(f"cleanup also failed: {type(cleanup_error).__name__}: {cleanup_error}") + raise + else: + await stack.aclose() + + return PluginBinding(connect=connect) + + +__all__ = ["MCPPlugin"] diff --git a/thinharness/subagents.py b/thinharness/subagents.py index 8626975..5627333 100644 --- a/thinharness/subagents.py +++ b/thinharness/subagents.py @@ -11,6 +11,7 @@ from .events import RunCompletedEvent, current_stream_emitter from .hooks import AfterSubagentRunContext, BeforeSubagentRunContext, HookRegistry, current_tool_call_context, current_tool_runtime_context from .plugins.base import Plugin +from .plugins.mcp import MCPPlugin from .providers import infer_model, same_provider_model_ref from .tools.base import Json, ToolResult, ToolSpec from .tools.mcp import MCPServer @@ -60,23 +61,14 @@ def validate_subagent(self) -> SubAgentConfig: raise ValueError(f"{DEFAULT_SUBAGENT_NAME!r} is reserved for the framework default subagent") if not self.description.strip() or "\n" in self.description or "\r" in self.description: raise ValueError("subagent description must be a non-empty single line") - exposes_subagent = any(name.lower() == "subagent" for name in self.builtin_tools) or any( - _tool_name(tool).lower() == "subagent" for tool in self.tools - ) + exposes_subagent = any(name.lower() == "subagent" for name in self.builtin_tools) or any(_tool_name(tool).lower() == "subagent" for tool in self.tools) if exposes_subagent: raise ValueError("subagent cannot be exposed inside a child subagent") if any(tool.requires_approval for tool in self.tools): raise ValueError("approval-required tools are not supported inside subagents") if self.inherit_parent_tools and (self.builtin_tools or self.plugins or self.tools): raise ValueError("inherit_parent_tools cannot be combined with builtin_tools, plugins, or tools") - if not ( - self.inherit_parent_tools - or self.builtin_tools - or self.plugins - or self.tools - or self.inherit_mcp_servers - or self.mcp_servers - ): + if not (self.inherit_parent_tools or self.builtin_tools or self.plugins or self.tools or self.inherit_mcp_servers or self.mcp_servers): raise ValueError("named subagents must define builtin_tools, plugins, tools, inherit_parent_tools=True, inherit_mcp_servers=True, or mcp_servers") return self @@ -92,6 +84,7 @@ class SubAgentArgs(BaseModel): def create_subagent_tool(parent: Harness, configs: list[SubAgentConfig]) -> ToolSpec: """Create the parent-facing subagent delegation tool.""" + async def handler(args: SubAgentArgs) -> ToolResult: """Run the selected subagent.""" return await run_subagent_tool(parent, configs, args) @@ -183,15 +176,17 @@ async def run_subagent_tool(parent: Harness, configs: list[SubAgentConfig], args if run_error is not None: raise run_error.with_traceback(run_traceback) except Exception as exc: - parent.hooks.fire(AfterSubagentRunContext( - harness=parent, - metadata=_parent_run_metadata(), - agent=agent_name, - task=args.task, - error=exc, - tools=effective_tools, - parent_call_id=parent_call_id, - )) + parent.hooks.fire( + AfterSubagentRunContext( + harness=parent, + metadata=_parent_run_metadata(), + agent=agent_name, + task=args.task, + error=exc, + tools=effective_tools, + parent_call_id=parent_call_id, + ) + ) return ToolResult( False, str(exc), @@ -204,16 +199,18 @@ async def run_subagent_tool(parent: Harness, configs: list[SubAgentConfig], args }, ) assert result is not None - parent.hooks.fire(AfterSubagentRunContext( - harness=parent, - metadata=_parent_run_metadata(), - agent=agent_name, - task=args.task, - result=result, - tools=effective_tools, - usage=result.usage, - parent_call_id=parent_call_id, - )) + parent.hooks.fire( + AfterSubagentRunContext( + harness=parent, + metadata=_parent_run_metadata(), + agent=agent_name, + task=args.task, + result=result, + tools=effective_tools, + usage=result.usage, + parent_call_id=parent_call_id, + ) + ) structured_output = result.output is not None content = child.output_schema.dump(result.output) if structured_output and child.output_schema is not None else result.text return ToolResult( @@ -242,37 +239,38 @@ def build_child_harness(parent: Harness, config: SubAgentConfig | None) -> Harne else: assert config is not None child_builtin_tools = config.builtin_tools + # Remove this MCP-specific bridge when subagents migrate to plugin composition. child_mcp_servers: list[MCPServer] = [] if config is not None and config.inherit_mcp_servers: - child_mcp_servers.extend(parent._mcp_servers) + parent_mcp = next((plugin for plugin in parent.plugins if isinstance(plugin, MCPPlugin)), None) + if parent_mcp is not None: + child_mcp_servers.extend(parent_mcp.servers) if config is not None: for server in config.mcp_servers: if not any(server is existing for existing in child_mcp_servers): child_mcp_servers.append(server) - child_config = parent_config.model_copy(update={ - "model": config.model if config is not None and config.model is not None else parent_config.model, - "root": parent.root, - "system_prompt": DEFAULT_SYSTEM_PROMPT if config is None else config.system_prompt, - "builtin_tools": child_builtin_tools, - "skills_dir": parent_config.skills_dir if child_wants_skills and not inherit_tools else None, - "selected_skills": parent_config.selected_skills if child_wants_skills and not inherit_tools else None, - "max_model_requests": ( - config.max_model_requests - if config is not None and config.max_model_requests is not None - else parent_config.max_model_requests - ), - "max_tool_calls": ( - config.max_tool_calls - if config is not None and config.max_tool_calls is not None - else parent_config.max_tool_calls - ), - "output_type": config.output_type if config is not None else None, - "output_mode": config.output_mode if config is not None else "auto", - "output_retries": config.output_retries if config is not None else 1, - "tool_retries": config.tool_retries if config is not None else parent_config.tool_retries, - "subagents": [], - "mcp_servers": child_mcp_servers, - }) + child_plugins = list(_inherited_instruction_plugins(parent) if inherit_tools else (config.plugins if config is not None else [])) + if child_mcp_servers: + child_plugins.append(MCPPlugin(servers=child_mcp_servers)) + child_config = parent_config.model_copy( + update={ + "model": config.model if config is not None and config.model is not None else parent_config.model, + "root": parent.root, + "system_prompt": DEFAULT_SYSTEM_PROMPT if config is None else config.system_prompt, + "builtin_tools": child_builtin_tools, + "skills_dir": parent_config.skills_dir if child_wants_skills and not inherit_tools else None, + "selected_skills": parent_config.selected_skills if child_wants_skills and not inherit_tools else None, + "max_model_requests": ( + config.max_model_requests if config is not None and config.max_model_requests is not None else parent_config.max_model_requests + ), + "max_tool_calls": (config.max_tool_calls if config is not None and config.max_tool_calls is not None else parent_config.max_tool_calls), + "output_type": config.output_type if config is not None else None, + "output_mode": config.output_mode if config is not None else "auto", + "output_retries": config.output_retries if config is not None else 1, + "tool_retries": config.tool_retries if config is not None else parent_config.tool_retries, + "subagents": [], + } + ) child_model = parent.model if config is not None and config.model is not None: same_provider = _same_provider(parent, config.model) @@ -291,7 +289,7 @@ def build_child_harness(parent: Harness, config: SubAgentConfig | None) -> Harne return Harness( child_config, model=child_model, - plugins=_inherited_instruction_plugins(parent) if inherit_tools else (config.plugins if config is not None else []), + plugins=child_plugins, tools=_effective_custom_tools(parent, config), tracing=_child_tracing(parent, config), skills=parent.skills if inherit_tools else None, @@ -316,8 +314,9 @@ def _effective_custom_tools(parent: Harness, config: SubAgentConfig | None) -> l """Return custom tools to register on the child harness.""" if config is None or config.inherit_parent_tools: return [ - tool for tool in parent.tools - if tool.name != "subagent" and tool.kind != "mcp" and not tool.requires_approval + tool + for tool in parent.tools + if tool.name != "subagent" and not (tool.origin is not None and tool.origin.plugin == "mcp") and not tool.requires_approval ] return list(config.tools) @@ -336,10 +335,12 @@ def _child_tracing(parent: Harness, config: SubAgentConfig | None) -> list[Traci """Return child tracing options that share the parent's tracer.""" name = config.name if config is not None else DEFAULT_SUBAGENT_NAME return [ - option.model_copy(update={ - "agent_name": f"subagent.{name}", - "agent_description": config.description if config is not None else "Framework default subagent", - }) + option.model_copy( + update={ + "agent_name": f"subagent.{name}", + "agent_description": config.description if config is not None else "Framework default subagent", + } + ) for option in parent.tracing ] diff --git a/thinharness/tool_execution.py b/thinharness/tool_execution.py index 4e6e8cc..1eb10ed 100644 --- a/thinharness/tool_execution.py +++ b/thinharness/tool_execution.py @@ -163,13 +163,15 @@ async def execute_one(self, call: ModelToolCall, index: int) -> ToolCallExecutio tool_spec=spec, tool_index=index, ) - self.run_context.emit(ToolCallStartedEvent( - **self.run_context.stream_base(), - call_id=call.id, - tool_name=call.name, - tool_index=index, - arguments=call.arguments, - )) + self.run_context.emit( + ToolCallStartedEvent( + **self.run_context.stream_base(), + call_id=call.id, + tool_name=call.name, + tool_index=index, + arguments=call.arguments, + ) + ) self.harness.hooks.fire(before) if before.cancelled: cancelled = True @@ -215,18 +217,20 @@ async def execute_one(self, call: ModelToolCall, index: int) -> ToolCallExecutio return ToolCallExecution(envelope=envelope, output=output, cancelled=cancelled, retry_kind=retry_kind) except Exception as exc: if not completed_emitted: - self.run_context.emit(ToolCallCompletedEvent( - **self.run_context.stream_base(), - call_id=call.id, - tool_name=call.name, - ok=False, - cancelled=cancelled, - retry_kind=retry_kind, - error_type=type(exc).__name__, - message=str(exc), - duration_ms=(time.perf_counter() - start) * 1000, - output=output, - )) + self.run_context.emit( + ToolCallCompletedEvent( + **self.run_context.stream_base(), + call_id=call.id, + tool_name=call.name, + ok=False, + cancelled=cancelled, + retry_kind=retry_kind, + error_type=type(exc).__name__, + message=str(exc), + duration_ms=(time.perf_counter() - start) * 1000, + output=output, + ) + ) raise finally: _CURRENT_STREAM_EMITTER.reset(emitter_token) @@ -244,18 +248,20 @@ def _emit_completed( duration_ms: float, ) -> None: """Emit a public tool completion event.""" - self.run_context.emit(ToolCallCompletedEvent( - **self.run_context.stream_base(), - call_id=call.id, - tool_name=call.name, - ok=envelope.ok, - cancelled=cancelled, - retry_kind=retry_kind, - error_type=envelope.error_type(), - message=envelope.content if not envelope.ok else None, - duration_ms=duration_ms, - output=output, - )) + self.run_context.emit( + ToolCallCompletedEvent( + **self.run_context.stream_base(), + call_id=call.id, + tool_name=call.name, + ok=envelope.ok, + cancelled=cancelled, + retry_kind=retry_kind, + error_type=envelope.error_type(), + message=envelope.content if not envelope.ok else None, + duration_ms=duration_ms, + output=output, + ) + ) async def _call_output(self, name: str, arguments: str) -> ToolEnvelope: """Execute one model tool call and format its output.""" @@ -267,14 +273,19 @@ async def _call_output(self, name: str, arguments: str) -> ToolEnvelope: def _annotate_special_tool(self, span: _TraceSpan, name: str, envelope: ToolEnvelope) -> None: """Add tool-family trace attributes for framework and MCP tools.""" if name == "subagent": - span.set_attributes({ - "subagent.name": envelope.metadata.get("agent"), - "subagent.tool_mode": envelope.metadata.get("tool_mode"), - "subagent.tools": envelope.metadata.get("tools"), - }) + span.set_attributes( + { + "subagent.name": envelope.metadata.get("agent"), + "subagent.tool_mode": envelope.metadata.get("tool_mode"), + "subagent.tools": envelope.metadata.get("tools"), + } + ) spec = self.tool_map.get(str(name)) - if spec is not None and spec.mcp is not None: - span.set_attributes({ - "mcp.server.id": spec.mcp.server_id, - "mcp.tool.name": spec.mcp.tool_name, - }) + origin = spec.origin if spec is not None else None + if origin is not None and origin.plugin == "mcp": + span.set_attributes( + { + "mcp.server.id": origin.source, + "mcp.tool.name": origin.attributes.get("tool_name"), + } + ) diff --git a/thinharness/tools/__init__.py b/thinharness/tools/__init__.py index 34af6c8..113a08e 100644 --- a/thinharness/tools/__init__.py +++ b/thinharness/tools/__init__.py @@ -2,7 +2,6 @@ from .base import ( Json, - McpToolInfo, ModelRetry, PathPolicy, PathValidationError, @@ -36,7 +35,6 @@ "MCPServerStdio", "MCPServerStreamableHTTP", "ModelRetry", - "McpToolInfo", "PathPolicy", "PathValidationError", "ToolEnvelope", diff --git a/thinharness/tools/base.py b/thinharness/tools/base.py index c0dbbb2..33a1f0a 100644 --- a/thinharness/tools/base.py +++ b/thinharness/tools/base.py @@ -17,7 +17,7 @@ from ..types import Json -ToolKind = Literal["user", "subagent", "parallel_llm", "mcp"] +ToolKind = Literal["user", "subagent", "parallel_llm"] ToolHandler = Callable[[Any], Any | Awaitable[Any]] T = TypeVar("T", bound=BaseModel) @@ -31,14 +31,6 @@ class ToolOrigin: attributes: Json = field(default_factory=dict) -@dataclass(frozen=True) -class McpToolInfo: - """Framework-owned identity for an MCP-backed tool.""" - - server_id: str - tool_name: str - - @dataclass(frozen=True) class ToolSpec: """A JSON-schema-described callable exposed to the model.""" @@ -54,11 +46,10 @@ class ToolSpec: requires_approval: bool = False origin: ToolOrigin | None = None kind: ToolKind = "user" - mcp: McpToolInfo | None = None def __post_init__(self) -> None: """Validate per-tool retry configuration.""" - if self.kind not in {"user", "subagent", "parallel_llm", "mcp"}: + if self.kind not in {"user", "subagent", "parallel_llm"}: raise ValueError(f"unknown tool kind: {self.kind}") if self.max_retries is not None and self.max_retries < 0: raise ValueError(f"max_retries must be >= 0, got {self.max_retries}") @@ -141,6 +132,7 @@ def __init__(self, message: str) -> None: self.message = message super().__init__(message) + @dataclass(frozen=True) class AllowedPath: """One lexically normalized path allowed by a workspace path policy.""" @@ -196,11 +188,13 @@ def _allowed_path(self, raw: str | Path) -> AllowedPath: """Normalize a configured allow path without filesystem metadata I/O.""" return AllowedPath(_lexical_path_under_root(self.root, raw)) + class StrictArgs(BaseModel): """Base class for tool arguments.""" model_config = ConfigDict(extra="forbid") + def _prepare_args(spec: ToolSpec, raw_args: str | Json) -> ToolEnvelope | Any: """Parse and validate raw tool arguments.""" try: @@ -326,6 +320,7 @@ def _is_async_callable(handler: ToolHandler) -> bool: obj = obj.func return inspect.iscoroutinefunction(obj) or (callable(obj) and inspect.iscoroutinefunction(obj.__call__)) + def contained_path(root: Path, raw: str | Path) -> Path: """Resolve a path and require it to remain inside root.""" return _resolve_under_root(root, raw) @@ -426,10 +421,12 @@ def _clean_schema(schema: Any) -> None: continue _clean_schema(value) + def _timeout_error_message(command_name: str, timeout: int) -> str: """Return a compact timeout failure message.""" return f"{command_name} timed out after {timeout}s" + def _is_relative_to(path: Path, root: Path) -> bool: """Return whether path is inside root.""" try: diff --git a/thinharness/tools/mcp.py b/thinharness/tools/mcp.py index d63a044..37d465c 100644 --- a/thinharness/tools/mcp.py +++ b/thinharness/tools/mcp.py @@ -11,7 +11,7 @@ import httpx -from .base import Json, McpToolInfo, ToolResult, ToolSpec +from .base import Json, ToolOrigin, ToolResult, ToolSpec _INSTALL_HINT = "Install MCP support with: pip install thinharness[mcp]" _MCP_NAME_RE = re.compile(r"[^a-zA-Z0-9_-]") @@ -56,7 +56,6 @@ def __init__( self.include_tools = list(include_tools) if include_tools is not None else None self.exclude_tools = list(exclude_tools) if exclude_tools is not None else None self._id = id - self._resolved_id: str | None = None self._transport = transport self._client: Any | None = None if type(self) is MCPServer: @@ -64,14 +63,8 @@ def __init__( @property def id(self) -> str: - """Stable readable identifier; falls back to a derived default.""" - return self._resolved_id or self._id or self._default_id() - - def resolve_id(self, existing_counts: dict[str, int]) -> None: - """Resolve this server's final id using duplicate counts owned by the caller.""" - base_id = self._id or self._default_id() - existing_counts[base_id] = existing_counts.get(base_id, 0) + 1 - self._resolved_id = base_id if existing_counts[base_id] == 1 else f"{base_id}-{existing_counts[base_id]}" + """Return the public base identifier without binding-local suffixes.""" + return self._id or self._default_id() def _default_id(self) -> str: """Derive a readable default id from the transport class name.""" @@ -111,8 +104,9 @@ async def __aexit__(self, exc_type: object = None, exc_val: object = None, exc_t if task is not None and task.cancelling() > pending_cancels: raise asyncio.CancelledError - async def list_tools(self) -> list[ToolSpec]: + async def list_tools(self, *, server_id: str | None = None) -> list[ToolSpec]: """Discover and convert the MCP server's current tool snapshot.""" + resolved_id = server_id or self.id async with self: tools = await self._ensure_client().list_tools() seen: dict[str, str] = {} @@ -126,25 +120,27 @@ async def list_tools(self) -> list[ToolSpec]: public_name = _normalize_mcp_name(f"{self.tool_prefix}_{original_name}" if self.tool_prefix else original_name) if public_name in seen: raise MCPError( - f"MCP tool name collision after sanitization on server {self.id!r}: " + f"MCP tool name collision after sanitization on server {resolved_id!r}: " f"{seen[public_name]!r} and {original_name!r} both map to {public_name!r}" ) seen[public_name] = original_name - specs.append(ToolSpec( - public_name, - str(tool.description or ""), - _clean_mcp_schema(tool.inputSchema, original_name), - _make_tool_handler(self, original_name), - sequential=False, - kind="mcp", - mcp=McpToolInfo(server_id=self.id, tool_name=original_name), - max_retries=None, - )) + specs.append( + ToolSpec( + public_name, + str(tool.description or ""), + _clean_mcp_schema(tool.inputSchema, original_name), + _make_tool_handler(self, original_name, resolved_id), + sequential=False, + origin=ToolOrigin(plugin="mcp", source=resolved_id, attributes={"tool_name": original_name}), + max_retries=None, + ) + ) return specs - async def call_tool(self, name: str, arguments: Json) -> ToolResult: + async def call_tool(self, name: str, arguments: Json, *, server_id: str | None = None) -> ToolResult: """Call one MCP tool and normalize its result.""" - base_metadata = {"source": "mcp", "mcp_server_id": self.id, "mcp_tool_name": name} + resolved_id = server_id or self.id + base_metadata = {"source": "mcp", "mcp_server_id": resolved_id, "mcp_tool_name": name} failure_types = _mcp_failure_types() try: async with self: @@ -337,11 +333,12 @@ def _find_known_failure(exc: BaseException, failure_types: tuple[type[BaseExcept return None -def _make_tool_handler(server: MCPServer, tool_name: str) -> Any: - """Build an async ToolSpec handler for one MCP tool.""" +def _make_tool_handler(server: MCPServer, tool_name: str, server_id: str) -> Any: + """Build an async ToolSpec handler with binding-local attribution.""" + async def handler(args: Json) -> ToolResult: """Call the backing MCP tool.""" - return await server.call_tool(tool_name, args) + return await server.call_tool(tool_name, args, server_id=server_id) return handler From b7b4a36152aa98ec399e44b9ed2c9146cfd6c030 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Tue, 18 Aug 2026 18:42:19 -0400 Subject: [PATCH 06/30] Fix MCP plugin review findings --- CHANGELOG.md | 2 + tests/e2e/README.md | 4 +- tests/e2e/mcp_journey.py | 19 +++++ tests/unit/test_mcp.py | 92 +++++++++++++++++++++- tests/unit/test_mcp_optional_dependency.py | 16 ++++ tests/unit/test_subagents.py | 32 ++++++++ thinharness/core.py | 8 -- thinharness/plugins/mcp.py | 24 +++++- thinharness/providers.py | 4 +- thinharness/subagents.py | 3 + 10 files changed, 189 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61f7a3c..f0f8ebd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ - Added `FilesystemPlugin` for the ordered workspace tool surface; `jsonl_search` remains opt-in through this plugin. - Added `MCPPlugin` for lazy MCP server connection, binding-local server identity, atomic tool discovery, and generic tool origin attribution. - **Breaking:** Removed `HarnessConfig.mcp_servers`, `McpToolInfo`, and the MCP `ToolKind`; configure one `MCPPlugin` with all harness servers. +- **Breaking:** Generic plugin validation now reports MCP tool collisions as duplicate tool names. Use MCP `tool_prefix`, `include_tools`, or `exclude_tools` to prevent collisions. +- **Breaking:** Removed `MCPServer.resolve_id()` and post-bind mutation of `server.id`. The public `server.id` remains the base ID; binding-local IDs, including duplicate suffixes, appear in tool origin and result metadata. - **Breaking:** `Harness` no longer enables filesystem tools by default. Pass `plugins=[FilesystemPlugin(...)]`; independent custom tools still use `tools=`. - **Breaking:** Removed filesystem settings from `HarnessConfig` and removed the `builtin_tools()` helper. `read_paths` and `write_paths` remain temporarily for the transitional parallel-LLM built-in. - Changed connection setup to complete before `run_start` hooks. A connection failure does not fire run lifecycle hooks. diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 4222da4..9701c6b 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -1,6 +1,6 @@ # E2E Journeys -Most scripts run real provider calls against temporary workspaces. The deterministic MCP journey uses a local scripted model. Journeys are intentionally not wired into pytest or CI. +Most scripts run real provider calls against temporary workspaces. The deterministic MCP journey uses a local scripted model, needs the `mcp` extra, and needs no provider credentials. Journeys are intentionally not wired into pytest or CI. Run one script with environment from `.env`: @@ -16,7 +16,7 @@ Current journeys: - `skills_journey.py`: skill discovery, `skill_read`, and `skill_run`. - `control_plane_journey.py`: hooks, sequential execution, and retry-limit behavior. - `structured_output_journey.py`: Pydantic structured output after tool use. -- `mcp_journey.py`: deterministic local stdio MCP tool discovery, execution, and cleanup without provider credentials. +- `mcp_journey.py`: deterministic local stdio MCP tool discovery, execution, and cleanup. It reports a skip when the `mcp` extra is not installed and does not use provider credentials. - `parallel_llm_tool_journey.py`: direct `ParallelLlmTool` calls across all configured providers. - `parallel_llm_agent_journey.py`: an agent run using both built-in `parallel_llm` and a renamed custom `ParallelLlmTool`. - `prompt_caching_journey.py`: Anthropic prompt caching — asserts a multi-request run reports cached input tokens. diff --git a/tests/e2e/mcp_journey.py b/tests/e2e/mcp_journey.py index cd9b81f..53768a4 100644 --- a/tests/e2e/mcp_journey.py +++ b/tests/e2e/mcp_journey.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import importlib.util import sys from pathlib import Path from tempfile import TemporaryDirectory @@ -49,9 +50,27 @@ def new_session(self) -> DeterministicSession: def main() -> None: """Run local stdio discovery, execution, and cleanup end to end.""" + missing = _missing_mcp_dependencies() + if missing: + packages = ", ".join(missing) + print(f"SKIP mcp_journey missing optional dependencies: {packages}; install thinharness[mcp]") + return asyncio.run(_run()) +def _missing_mcp_dependencies() -> list[str]: + """Return MCP packages that are not installed.""" + missing: list[str] = [] + for package in ("mcp", "fastmcp"): + try: + available = importlib.util.find_spec(package) is not None + except (ImportError, ValueError): + available = False + if not available: + missing.append(package) + return missing + + async def _run() -> None: with TemporaryDirectory(prefix="thinharness-e2e-mcp-") as raw_root: root = Path(raw_root) diff --git a/tests/unit/test_mcp.py b/tests/unit/test_mcp.py index 9b15009..dfbe0c7 100644 --- a/tests/unit/test_mcp.py +++ b/tests/unit/test_mcp.py @@ -916,6 +916,29 @@ async def test_harness_connects_mcp_once_across_async_runs(tmp_path, monkeypatch assert server.call_records == [("remote", {"value": "ok"})] +def test_mcp_plugin_validates_server_collection() -> None: + """MCP server configuration is ordered, typed, and unique by identity.""" + server = MCPServerStdio("unused") + + with pytest.raises(TypeError, match="ordered sequence"): + MCPPlugin(servers={server}) # type: ignore[arg-type] + with pytest.raises(TypeError, match="only MCPServer"): + MCPPlugin(servers=[object()]) # type: ignore[list-item] + with pytest.raises(ValueError, match="same server object"): + MCPPlugin(servers=[server, server]) + + +def test_mcp_plugin_name_is_fixed() -> None: + """Instances and subclasses cannot replace the MCP plugin name.""" + plugin = MCPPlugin(servers=[]) + + assert plugin.name == "mcp" + with pytest.raises(AttributeError, match="fixed"): + plugin.name = "renamed" + with pytest.raises(TypeError, match="cannot override"): + type("RenamedMCPPlugin", (MCPPlugin,), {"name": "renamed"}) + + def test_mcp_plugin_name_is_unique_and_config_path_is_removed(tmp_path, monkeypatch) -> None: """One fixed-name MCP plugin is allowed and the old config path is rejected.""" first = scripted_server(monkeypatch, {"first": _schema()}) @@ -930,11 +953,25 @@ def test_mcp_plugin_name_is_unique_and_config_path_is_removed(tmp_path, monkeypa assert "mcp_servers" not in HarnessConfig.model_fields +async def test_empty_mcp_plugin_connects_without_tools(tmp_path) -> None: + """An empty MCP plugin is a valid connected contribution.""" + harness = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[])], + model=ScriptedModel([]), + ) + + await harness.connect() + assert harness.tools == [] + await harness.aclose() + + async def test_explicit_connect_does_not_reconnect_on_run(tmp_path, monkeypatch) -> None: """Explicit connect discovers MCP tools once before run.""" server = scripted_server(monkeypatch, {"remote": _schema()}) harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([]))) + await harness.connect() await harness.connect() result = await harness.run("go") await harness.aclose() @@ -1033,6 +1070,54 @@ async def list_tools(self, *, server_id: str | None = None) -> list[ToolSpec]: await harness.aclose() +async def test_cancellation_during_failed_discovery_cleanup_propagates_and_retries(tmp_path) -> None: + """Caller cancellation during failed discovery cleanup wins after cleanup.""" + cleanup_started = asyncio.Event() + release_cleanup = asyncio.Event() + + class FailingDiscoverySlowCleanupServer(ObservedMCPServer): + def __init__(self, transport: Any, **kwargs: Any) -> None: + super().__init__(transport, **kwargs) + self.attempts = 0 + self.block_cleanup = True + + async def list_tools(self, *, server_id: str | None = None) -> list[ToolSpec]: + self.attempts += 1 + if self.attempts == 1: + raise MCPError("discovery failed") + return await super().list_tools(server_id=server_id) + + async def __aexit__(self, *exc: object) -> None: + if self.block_cleanup: + self.block_cleanup = False + cleanup_started.set() + await release_cleanup.wait() + await super().__aexit__(*exc) + + backend = FastMCP("failed-discovery-slow-cleanup") + backend.tool(_echo_handler("recovered", []), name="recovered") + server = FailingDiscoverySlowCleanupServer(FastMCPTransport(backend), id="slow-cleanup") + harness = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + plugins=[MCPPlugin(servers=[server])], + model=ScriptedModel([]), + ) + connecting = asyncio.create_task(harness.connect()) + await cleanup_started.wait() + + connecting.cancel() + await asyncio.sleep(0) + assert not connecting.done() + release_cleanup.set() + with pytest.raises(asyncio.CancelledError): + await connecting + assert harness.tools == [] + + await harness.connect() + assert [tool.name for tool in harness.tools] == ["recovered"] + await harness.aclose() + + async def test_direct_tool_collision_rolls_back_mcp(tmp_path, monkeypatch) -> None: """Discovered MCP tools collide atomically with direct tools.""" server = scripted_server(monkeypatch, {"shared": _schema()}) @@ -1113,8 +1198,11 @@ async def test_duplicate_derived_id_disambiguated(tmp_path, monkeypatch) -> None await harness.connect() await harness.aclose() - metadata = {tool.name: tool.origin.source for tool in harness.tools if tool.origin is not None} - assert metadata == {"one": "same", "two": "same-2"} + metadata = {tool.name: tool.origin for tool in harness.tools if tool.origin is not None} + assert metadata == { + "one": ToolOrigin(plugin="mcp", source="same", attributes={"tool_name": "one"}), + "two": ToolOrigin(plugin="mcp", source="same-2", attributes={"tool_name": "two"}), + } async def test_binding_local_ids_stay_stable_for_shared_server(tmp_path, monkeypatch) -> None: diff --git a/tests/unit/test_mcp_optional_dependency.py b/tests/unit/test_mcp_optional_dependency.py index 3bf52e4..ec5fd31 100644 --- a/tests/unit/test_mcp_optional_dependency.py +++ b/tests/unit/test_mcp_optional_dependency.py @@ -1,6 +1,8 @@ from __future__ import annotations import builtins +import importlib.util +import runpy import subprocess import sys from pathlib import Path @@ -40,6 +42,20 @@ async def test_construction_without_extra(monkeypatch: pytest.MonkeyPatch) -> No await server.__aenter__() +def test_mcp_journey_skips_when_extra_is_missing(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + """The deterministic journey reports a skip instead of a dependency traceback.""" + journey_path = Path(__file__).resolve().parents[1] / "e2e" / "mcp_journey.py" + journey = runpy.run_path(str(journey_path)) + real_find_spec = importlib.util.find_spec + monkeypatch.setattr(importlib.util, "find_spec", lambda name: None if name == "mcp" else real_find_spec(name)) + + main = journey["main"] + assert callable(main) + main() + + assert capsys.readouterr().out == "SKIP mcp_journey missing optional dependencies: mcp; install thinharness[mcp]\n" + + async def test_missing_fastmcp_alone_gives_install_hint(monkeypatch: pytest.MonkeyPatch) -> None: """A stale environment with mcp but no fastmcp still gets the install hint.""" _block_imports(monkeypatch, {"fastmcp"}) diff --git a/tests/unit/test_subagents.py b/tests/unit/test_subagents.py index abbc850..321b040 100644 --- a/tests/unit/test_subagents.py +++ b/tests/unit/test_subagents.py @@ -23,6 +23,8 @@ HarnessConfig, Hook, HookRegistry, + MCPPlugin, + MCPServerStdio, SubAgentConfig, ToolSpec, TracingOptions, @@ -72,6 +74,36 @@ def test_subagent_config_validation_accepts_tool_specs() -> None: with pytest.raises(ValueError, match="SubAgentConfig.background has been removed"): SubAgentConfig(name="old-background", description="Old helper.", plugins=[FilesystemPlugin(tools=["read"])], background="always") +def test_subagent_rejects_duplicate_mcp_configuration_early() -> None: + server = MCPServerStdio("unused") + plugin = MCPPlugin(servers=[server]) + + with pytest.raises(ValueError, match="explicit MCPPlugin"): + SubAgentConfig( + name="explicit-and-servers", + description="Invalid MCP helper.", + plugins=[plugin], + mcp_servers=[server], + ) + with pytest.raises(ValueError, match="explicit MCPPlugin"): + SubAgentConfig( + name="explicit-and-inherit", + description="Invalid MCP helper.", + plugins=[plugin], + inherit_mcp_servers=True, + ) + assert SubAgentConfig(name="explicit", description="Explicit MCP helper.", plugins=[plugin]).plugins == [plugin] + + +def test_inherited_mcp_without_parent_plugin_adds_no_child_plugin(tmp_path: Path) -> None: + parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([])) + config = SubAgentConfig(name="mcp", description="MCP helper.", inherit_mcp_servers=True) + + child = build_child_harness(parent, config) + + assert not any(isinstance(plugin, MCPPlugin) for plugin in child.plugins) + + def test_subagent_builtin_exposure_is_selectable(tmp_path: Path) -> None: default = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([])) disabled = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([])) diff --git a/thinharness/core.py b/thinharness/core.py index 660f5b0..f8667a0 100644 --- a/thinharness/core.py +++ b/thinharness/core.py @@ -769,14 +769,6 @@ def _validate_tool_spec( is_child_run=is_child_run, ) - def _validate_tool_approval_policy(self, tool: ToolSpec) -> None: - """Reject approval policies incompatible with this harness configuration.""" - self._validate_tool_approval_policy_for( - tool, - model_supports_approval_resume=self._model_supports_approval_resume(), - is_child_run=self._is_child_run, - ) - @staticmethod def _validate_tool_approval_policy_for( tool: ToolSpec, diff --git a/thinharness/plugins/mcp.py b/thinharness/plugins/mcp.py index c536fd0..dd31302 100644 --- a/thinharness/plugins/mcp.py +++ b/thinharness/plugins/mcp.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio from collections.abc import AsyncIterator, Sequence from contextlib import AsyncExitStack, asynccontextmanager from dataclasses import dataclass @@ -28,12 +29,26 @@ class MCPPlugin: name = "mcp" + def __init_subclass__(cls) -> None: + """Reject subclasses that replace the fixed plugin name.""" + super().__init_subclass__() + if "name" in cls.__dict__: + raise TypeError("MCPPlugin subclasses cannot override the fixed name 'mcp'") + + def __setattr__(self, attribute: str, value: object) -> None: + """Reject instance changes to the fixed plugin name.""" + if attribute == "name": + raise AttributeError("MCPPlugin.name is fixed to 'mcp'") + super().__setattr__(attribute, value) + def __init__(self, *, servers: Sequence[MCPServer]) -> None: if isinstance(servers, (set, frozenset)): raise TypeError("MCPPlugin servers must be an ordered sequence, not a set") self.servers = tuple(servers) if any(not isinstance(server, MCPServer) for server in self.servers): raise TypeError("MCPPlugin servers must contain only MCPServer values") + if len({id(server) for server in self.servers}) != len(self.servers): + raise ValueError("MCPPlugin servers must not contain the same server object more than once") def bind(self, context: PluginContext) -> PluginBinding: """Resolve binding-local server ids without opening a connection.""" @@ -57,8 +72,15 @@ async def connect() -> AsyncIterator[PluginContribution]: tools.extend(await bound.list_tools()) yield PluginContribution(tools=tuple(tools)) except BaseException as exc: + cleanup = asyncio.create_task(stack.aclose()) try: - await stack.aclose() + await asyncio.shield(cleanup) + except asyncio.CancelledError as cancellation: + try: + await cleanup + except BaseException as cleanup_error: + cancellation.add_note(f"cleanup also failed: {type(cleanup_error).__name__}: {cleanup_error}") + raise cancellation except BaseException as cleanup_error: exc.add_note(f"cleanup also failed: {type(cleanup_error).__name__}: {cleanup_error}") raise diff --git a/thinharness/providers.py b/thinharness/providers.py index d482717..fc5dfe0 100644 --- a/thinharness/providers.py +++ b/thinharness/providers.py @@ -138,8 +138,8 @@ class StructuredOutputRequest: class RequestConstants: """Per-run request constants passed to every ModelSession request. - Built once per run after run-start hooks and MCP connection, so the - toolset and instructions are frozen for the run. + Built once per run after generic plugin connection and run-start hooks, + so the toolset and instructions are frozen for the run. """ instructions: str diff --git a/thinharness/subagents.py b/thinharness/subagents.py index 5627333..f51e4f5 100644 --- a/thinharness/subagents.py +++ b/thinharness/subagents.py @@ -66,6 +66,9 @@ def validate_subagent(self) -> SubAgentConfig: raise ValueError("subagent cannot be exposed inside a child subagent") if any(tool.requires_approval for tool in self.tools): raise ValueError("approval-required tools are not supported inside subagents") + has_explicit_mcp_plugin = any(isinstance(plugin, MCPPlugin) for plugin in self.plugins) + if has_explicit_mcp_plugin and (self.mcp_servers or self.inherit_mcp_servers): + raise ValueError("an explicit MCPPlugin cannot be combined with mcp_servers or inherit_mcp_servers=True") if self.inherit_parent_tools and (self.builtin_tools or self.plugins or self.tools): raise ValueError("inherit_parent_tools cannot be combined with builtin_tools, plugins, or tools") if not (self.inherit_parent_tools or self.builtin_tools or self.plugins or self.tools or self.inherit_mcp_servers or self.mcp_servers): From 0c239cbe6270ea1f4bb69790f4ad115f007c93ad Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Tue, 18 Aug 2026 18:44:42 -0400 Subject: [PATCH 07/30] Keep MCP plugin name fixed at runtime --- tests/unit/test_mcp.py | 25 ++++++++++++++++++++++++- thinharness/plugins/mcp.py | 16 +++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_mcp.py b/tests/unit/test_mcp.py index dfbe0c7..6f4e291 100644 --- a/tests/unit/test_mcp.py +++ b/tests/unit/test_mcp.py @@ -929,14 +929,37 @@ def test_mcp_plugin_validates_server_collection() -> None: def test_mcp_plugin_name_is_fixed() -> None: - """Instances and subclasses cannot replace the MCP plugin name.""" + """Instances and classes cannot replace or remove the MCP plugin name.""" plugin = MCPPlugin(servers=[]) assert plugin.name == "mcp" with pytest.raises(AttributeError, match="fixed"): plugin.name = "renamed" + assert plugin.name == "mcp" + + with pytest.raises(AttributeError, match="fixed"): + MCPPlugin.name = "renamed" + assert plugin.name == "mcp" + + with pytest.raises(AttributeError, match="fixed"): + del MCPPlugin.name + assert plugin.name == "mcp" + with pytest.raises(TypeError, match="cannot override"): type("RenamedMCPPlugin", (MCPPlugin,), {"name": "renamed"}) + assert plugin.name == "mcp" + + class CustomMCPPlugin(MCPPlugin): + pass + + custom_plugin = CustomMCPPlugin(servers=[]) + with pytest.raises(AttributeError, match="fixed"): + CustomMCPPlugin.name = "renamed" + assert custom_plugin.name == "mcp" + + with pytest.raises(AttributeError, match="fixed"): + del CustomMCPPlugin.name + assert custom_plugin.name == "mcp" def test_mcp_plugin_name_is_unique_and_config_path_is_removed(tmp_path, monkeypatch) -> None: diff --git a/thinharness/plugins/mcp.py b/thinharness/plugins/mcp.py index dd31302..7e009a8 100644 --- a/thinharness/plugins/mcp.py +++ b/thinharness/plugins/mcp.py @@ -24,7 +24,21 @@ async def list_tools(self) -> list[ToolSpec]: return await self.server.list_tools(server_id=self.resolved_id) -class MCPPlugin: +class _MCPPluginMeta(type): + """Keep the MCP plugin name fixed on the class hierarchy.""" + + def __setattr__(cls, attribute: str, value: object) -> None: + if attribute == "name": + raise AttributeError("MCPPlugin.name is fixed to 'mcp'") + super().__setattr__(attribute, value) + + def __delattr__(cls, attribute: str) -> None: + if attribute == "name": + raise AttributeError("MCPPlugin.name is fixed to 'mcp'") + super().__delattr__(attribute) + + +class MCPPlugin(metaclass=_MCPPluginMeta): """Expose one ordered group of MCP servers through a harness plugin.""" name = "mcp" From a963f873eb0f32008546b6e8c8340d6fa6a78c26 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Tue, 18 Aug 2026 22:00:01 -0400 Subject: [PATCH 08/30] Plan skills and parallel LLM plugin migration --- .plans/39-skills-and-parallel-llm-plugins.md | 331 +++++++++++++++++++ 1 file changed, 331 insertions(+) create mode 100644 .plans/39-skills-and-parallel-llm-plugins.md diff --git a/.plans/39-skills-and-parallel-llm-plugins.md b/.plans/39-skills-and-parallel-llm-plugins.md new file mode 100644 index 0000000..7fcd117 --- /dev/null +++ b/.plans/39-skills-and-parallel-llm-plugins.md @@ -0,0 +1,331 @@ +# Skills and parallel LLM plugins — plan v2 + +Migrate the remaining non-subagent built-in tools onto the plugin seam created by plans 37 and 38. This slice adds `SkillsPlugin` and `ParallelLlmPlugin`, removes their configuration and composition logic from core, and leaves subagent composition for a later plan. + +## Resolved decisions + +1. **Keep both low-level modules.** `SkillRegistry` remains the deep module for skill discovery and execution. `ParallelLlmTool` remains the deep, renameable module for one-shot batches. The plugins own harness composition, defaults, and origin attribution rather than copying either implementation. +2. **Skills stay static.** `SkillsPlugin` discovers one catalog when the plugin object is constructed. Its `bind()` method performs no I/O, and its selected tools and summary are visible immediately after harness construction. Reusing one plugin object across harnesses reuses that catalog; callers construct a new plugin to rediscover added or removed skills. +3. **The skill catalog is frozen, not skill contents.** Discovery metadata, selected names, and the summary are fixed at plugin construction. Existing `SKILL.md` content, trees, and scripts remain live and are read when `skill_read` or `skill_run` executes. +4. **Skill paths keep their current meaning.** Relative `skills_dir` values resolve from the process working directory through `SkillRegistry`, not from `HarnessConfig.root`. Documentation must state this difference from root-scoped filesystem and parallel tools. +5. **Skill tool selection stays explicit.** `SkillsPlugin` requires a non-empty ordered selection of `skill_read`, `skill_run`, or both. Enabling skill discovery does not silently enable script execution. +6. **The parallel plugin uses the harness model by default.** Add the configured harness `Model` to `PluginContext`. `ParallelLlmPlugin(model=None)` borrows that model; an explicit model object is caller-owned; an explicit model string uses plugin-owned provider settings and the existing per-call create/close behavior. +7. **Alternate parallel models do not inherit hidden parent settings.** Provider and request settings are valid only when `model` is a string. Reject them for `model=None` or a model object instead of storing ignored values. +8. **Plugin names are runtime-fixed.** `SkillsPlugin.name == "skills"` and `ParallelLlmPlugin.name == "parallel_llm"` cannot be changed on an instance or class, including a subclass. One plugin of each type is allowed per harness; one SkillsPlugin accepts several skill directories. +9. **Subagents are not redesigned here.** Named child agents use their existing `plugins` field for explicit skills and parallel LLM. Remove the now-empty child `builtin_tools` path. A small SkillsPlugin-specific inheritance bridge preserves default and `inherit_parent_tools=True` behavior until `SubagentsPlugin` replaces child composition. +10. **The main built-in selector stays temporarily.** `HarnessConfig.builtin_tools` remains only to select the deferred `subagent` tool. Skill and parallel names fail with errors that point to their plugins. + +These are breaking pre-1.0 changes. Do not add aliases, fallback configuration reads, or compatibility constructors. + +## Goal + +After this plan, callers compose both features explicitly: + +```python +from thinharness import Harness, HarnessConfig, ParallelLlmPlugin, SkillsPlugin + +harness = Harness( + HarnessConfig(root="."), + plugins=[ + # Relative skill paths use the process working directory, not root. + SkillsPlugin(".agents/skills", tools=["skill_read"]), + ParallelLlmPlugin(), + ], +) +``` + +`thinharness/core.py` does not import skill or parallel-LLM implementation modules, construct either tool family, discover skills, render skill summaries, own parallel path policy, or copy parallel provider settings. + +## Plugin context + +Extend the existing context by one stable core dependency: + +```python +@dataclass(frozen=True) +class PluginContext: + root: Path + model: Model +``` + +Core constructs the model before binding plugins, as it does now. Do not add `Harness`, `HarnessConfig`, credentials, provider settings, plugin lookup, child factories, tracing, or mutable services to the context. + +A plugin must not close `context.model`; model ownership remains with the harness or caller. Add contract coverage that one plugin object bound to two harnesses sees each harness's own root and model. + +## SkillsPlugin + +Add `thinharness/plugins/skills.py`: + +```python +SkillsPlugin( + skills_dir: str | Path | Sequence[str | Path], + *, + selected_skills: Sequence[str] | None = None, + tools: Sequence[Literal["skill_read", "skill_run"]], +) +``` + +Behavior: + +- The fixed name is `"skills"`. +- Reject a set for `skills_dir` or `tools`. Reject an empty skill-directory sequence, an empty tool selection, duplicate tools, and unknown tools when the plugin is constructed. +- Construct one `SkillRegistry` in the plugin constructor. Discovery and selected-skill validation therefore occur before harness binding. +- A valid tool selection with no discovered skills contributes no tools and no summary. `selected_skills` naming a missing skill still fails during construction. +- `bind()` selects the requested `ToolSpec` values in caller order and returns one static `PluginContribution` without calling filesystem metadata functions. +- When at least one selected tool exists, contribute the compact skill summary once as plugin instructions. +- Make summary wording conditional. When `skill_read` is absent, do not tell the model to call it. +- Generic normalization assigns `ToolOrigin(plugin="skills", source=)`. +- `skill_run` remains sequential. `skill_read` remains parallel-safe. +- Keep current frontmatter, containment, tree rendering, truncation, runner selection, working directory, merged output, timeout, and result metadata behavior inside `SkillRegistry`. +- Keep `Skill`, `SkillRegistry`, argument types, and parser helpers public at their current module level. + +The frozen catalog contains skill names, paths, metadata, selection, and summary text. `skill_read` still reads current file content and builds the current tree at invocation time. `skill_run` still executes the current script file. Adding or removing skill entries after plugin construction does not change the catalog; editing a discovered skill's files remains visible. + +The skill summary becomes a normal plugin instruction. Its position follows caller plugin order, before all per-tool instructions under PLUGIN-8. This intentionally replaces the old fixed transitional-summary position. + +## ParallelLlmPlugin + +Add `thinharness/plugins/parallel_llm.py` with fixed name `"parallel_llm"`. It contributes one static tool named `parallel_llm` by constructing the existing `ParallelLlmTool` at bind time with the canonical context root and resolved model. + +The plugin accepts: + +- `model: Model | str | None = None`; +- `description` and `instructions`, defaulting to the current parallel defaults; +- `read_paths`, `write_paths`, and `max_prompts`; +- for a string model only: `api_key`, `base_url`, request timeout, retry count, retry backoff, temperature, max tokens, effort, and extra body. + +Use `None` sentinels for optional provider/request arguments on the plugin. For a string model, map omitted values to the current `ParallelLlmTool` defaults. For `model=None` or a model object, reject any supplied provider or request argument because the borrowed model already owns those settings. + +Reject `max_prompts < 1` during plugin construction. Do not add `root`, output schema settings, or a caller-settable tool name to the plugin. Callers that need a renamed or structured-output batch tool continue to use `ParallelLlmTool(...).spec()` through direct `tools=` composition. + +### I/O-free binding + +`ParallelLlmTool.__init__` currently resolves its root. Add a private `_root_is_resolved: bool = False` argument, matching `FileTools`. The plugin passes the already-resolved `PluginContext.root` with `_root_is_resolved=True`. Do not call `Path.resolve()`, `Path.exists()`, `Path.stat()`, provider inference, or network code during `ParallelLlmPlugin.bind()`. + +### Model ownership + +- `model=None`: bind to `PluginContext.model`; the plugin does not close it. +- `model=`: bind to that object; the caller owns it and the plugin does not close it. +- `model="provider:model"`: preserve `ParallelLlmTool` behavior — infer a model for each batch invocation and close the created provider after the batch, including schema-resolution, request, and cancellation failures. + +Preserve all current parallel behavior: + +- independent stateless prompts and fresh sessions; +- ordered sparse results with bounded concurrency; +- no parent system prompt, tools, memory, or continuation; +- file input and output policies under `PluginContext.root`; +- prompt caps, atomic JSON files, cancellation, and batch-local request counts; +- provider transport retries inside one logical request; +- no model override in model-visible arguments; +- default description and per-tool instructions; +- text-only plugin batches. + +Batch model requests and tokens remain outside parent `RunUsage` and `max_model_requests`. The parent counts one `parallel_llm` tool call toward `max_tool_calls`; batch metadata reports its own `model_requests`, `total`, `succeeded`, and `failed` values. + +Generic normalization assigns `ToolOrigin(plugin="parallel_llm", source="parallel_llm")`. + +Remove `"parallel_llm"` from `ToolKind` because no core control flow uses it. Remove `kind="parallel_llm"` from `ParallelLlmTool.spec()`, so direct and plugin-created specs use the valid default kind `"user"`. Update the `ToolKind` literal and its runtime validation set, tests, and changelog together. The low-level tool's execution behavior does not change. + +## Core removals and fail-loud guards + +Remove from `HarnessConfig`: + +- `skills_dir`; +- `selected_skills`; +- `read_paths`; +- `write_paths`; +- `builtin_parallel_llm_model`; +- `builtin_parallel_llm_temperature`; +- `parallel_llm_max_prompts`. + +Add a `model_validator(mode="before")` that rejects each removed field by name with a plugin migration message. Do not rely on Pydantic's default extra-field behavior. Replace the old `selected_skills requires skills_dir` test with explicit removed-field tests. Do not change all unknown extras to `extra="forbid"` in this slice. + +Remove from `Harness`: + +- the `skills=` constructor argument; +- `self.skills` and `_skills_enabled`; +- skill discovery, selected-skill validation, summary rendering, and skill tool validation; +- `create_parallel_llm_tool(self)` and parallel provider-setting projection; +- direct imports of `SkillRegistry` and the parallel-LLM module. + +Delete `create_parallel_llm_tool(parent)` and its exports. Keep `ParallelLlmTool` as the direct low-level interface. + +Reduce the main built-in candidate list to `subagent`. Keep `HarnessConfig.builtin_tools` until the subagent plugin plan. Requests for `skill_read` or `skill_run` must point to `SkillsPlugin`; requests for `parallel_llm` must point to `ParallelLlmPlugin`. Other unknown values keep the normal unknown-built-in error. + +Remove `SubAgentConfig.builtin_tools`. Extend its existing before-validator to reject the removed field by name, as it already does for `background`, so a second valid tool source cannot hide the mistake. Update validation to: + +- detect recursive `subagent` exposure only through direct `tools`; +- reject `inherit_parent_tools` combined with `plugins` or direct `tools`; +- require named subagents to define `plugins`, `tools`, `inherit_parent_tools=True`, `inherit_mcp_servers=True`, or `mcp_servers`; +- use that exact source list in the validation error. + +Migrate every caller, including the rejected Bash built-in test, instead of leaving obsolete child configuration examples. + +## Temporary subagent behavior + +Preserve these behaviors while deferring `SubagentsPlugin`: + +- `SubAgentConfig.plugins` accepts `SkillsPlugin` and `ParallelLlmPlugin` for explicit child composition. +- A non-inheriting named child that wants skills must configure its own `SkillsPlugin`; it does not implicitly reuse the parent's catalog. +- A default child and a child with `inherit_parent_tools=True` continue to receive the parent's resolved, non-approval tools. +- Resolved MCP tools remain excluded unless MCP inheritance is explicit. +- Find the exact parent `SkillsPlugin` with an `isinstance` scan over the read-only `parent.plugins` tuple, matching the MCP bridge pattern. +- Only when that plugin exists, exclude inherited resolved tools whose origin plugin is `"skills"` **and** whose names are in that exact plugin's configured tool selection. Generic direct tools or another custom plugin with a caller-supplied `ToolOrigin(plugin="skills")` remain inherited. +- Rebind the same parent SkillsPlugin object in the child instead of copying those selected specs. The child receives the same registry object, catalog, tool order, and one summary without a second discovery pass. +- Insert the rebound SkillsPlugin after the current filesystem instruction-only plugin in the temporary inherited-plugin list. This fixes child instruction order: filesystem root instruction, skill summary, then per-tool instructions. +- Other inherited static tools, including `parallel_llm`, keep the current resolved-handler inheritance behavior. A child with a model override therefore keeps the parent batch handler and parent batch model, matching current `inherit_parent_tools` behavior. +- Keep the existing filesystem instruction-only bridge and MCP server bridge. +- Child model, tracing, hooks, limits, output, lifecycle, and cleanup behavior do not change. + +Mark the SkillsPlugin-specific bridge, filesystem instruction bridge, and MCP bridge for deletion in the later subagent plugin plan. Do not add generic plugin inheritance, clone methods, or child factories now. + +## Behavior contract changes before implementation + +After plan review and before code changes, update only affected sections of `docs/behavior.md`: + +- extend PLUGIN-3 to state that `PluginContext` contains the canonical root and configured model while binding remains synchronous and I/O-free; +- update PLUGIN-8 so the skill summary is a plugin instruction ordered by plugin position; +- remove the temporary parallel-LLM path-policy wording from FILESYSTEM-PLUGIN-5; +- add a Skills Plugin section covering one fixed-name plugin per harness, explicit tool selection, constructor-time catalog discovery, static visibility, cwd-relative paths, frozen catalog/live content, summary conditions, tool behavior, and temporary child inheritance; +- add a Parallel LLM Plugin section covering one fixed-name plugin per harness, explicit composition, model ownership, provider-setting validation, root/path policy, text-only behavior, parent-run accounting exclusion, results, retries, cancellation, and the direct `ParallelLlmTool` escape hatch; +- update PROVIDER-RETRY-6 so plugin-owned string models use their own settings while a plugin borrowing the harness model uses that model's configured transport retries. + +Do not update unrelated behavior sections. + +## Architecture guard + +Extend `tests/unit/test_architecture.py` with a core-only check. Reject these tokens in `thinharness/core.py`: + +- `tools.skills`, `SkillRegistry`, `skills_dir`, `selected_skills`, `_skills_enabled`, and `prompt_summary`; +- `tools.parallel_llm`, `create_parallel_llm_tool`, `builtin_parallel_llm`, and `parallel_llm_max_prompts`. + +Scope the test to direct core source. Temporary imports and bridges in `thinharness/subagents.py` remain allowed until its plugin migration. + +Also test that `ParallelLlmPlugin.bind()` does not call root-resolution or filesystem metadata methods and does not infer or contact a provider. + +## Implementation steps + +1. Update the affected behavior contracts. +2. Add `model` to `PluginContext` and migrate every binding call and contract test. +3. Add `SkillsPlugin`, focused tests, and public exports. +4. Add the private resolved-root path to `ParallelLlmTool`; add `ParallelLlmPlugin`, focused tests, and public exports. +5. Migrate main harness and explicit child callers from skill and parallel built-ins to plugins. +6. Implement the exact temporary inherited-skills bridge. +7. Add fail-loud removed-field guards, then remove the listed core fields, constructor arguments, state, helpers, imports, `ToolKind` value, and child built-in path. +8. Add the concrete architecture and I/O-free binding tests. +9. Update README, `docs/docs.md`, all checked-in site pages, examples, exports, changelog, and end-to-end journeys. +10. Regenerate the README-derived site with `uv run scripts/build_site.py` and verify `uv run scripts/build_site.py --check`. + +## Tests + +Retain the existing `SkillRegistry` and `ParallelLlmTool` behavior suites. Add focused coverage for: + +### Plugin context + +- root and model identity passed to static and connected test plugins; +- one plugin object bound to two harnesses with independent context values; +- plugins do not close caller-owned models. + +### Skills + +- constructor rejects unordered or empty skill directories, and unordered, empty, duplicate, or unknown tool selections; +- constructor discovers recursively, filters selected skills, and rejects duplicate or missing selected names; +- relative skill paths preserve process-working-directory resolution across harnesses with different roots; +- selected tools and summary are static and visible immediately after harness construction; +- caller tool order and plugin-order-dependent summary order are preserved; +- no discovered skills contributes no tools or summary, while a missing selected skill fails; +- adding a skill after construction is ignored; editing a discovered `SKILL.md` or script remains visible to execution; a new plugin sees the added skill; +- plugin reuse across harnesses shares one catalog and registry object; +- runtime name mutation and subclass overrides are rejected; +- origin stamping and collisions with direct or other plugin tools; +- custom direct tools and custom plugins with `origin.plugin == "skills"` remain inherited unless they match the exact parent SkillsPlugin selection; +- summary appears once, and its wording does not mention unavailable `skill_read`; +- existing read, run, timeout, containment, truncation, runner, and sequential behavior; +- explicit child plugin composition and inherited parent skill tools plus exactly one summary. + +### Parallel LLM + +- static visibility, runtime-fixed plugin/tool name, origin, description, and instructions; +- I/O-free bind with the resolved context root; +- `model=None` binds each harness model independently when one plugin object is reused; +- explicit model objects are borrowed and never closed; +- provider/request options are rejected unless model is a string; +- explicit model strings use plugin request settings and close created providers on success, failure, schema failure, and cancellation; +- `max_prompts` fails during plugin construction; +- canonical root, read/write policies, prompt cap, concurrency, ordering, batch-local accounting, output files, retries, cancellation, and text-only output; +- one parent tool call is counted while nested batch requests and tokens stay outside parent run usage and limits; +- no parent system prompt or nested tool execution; +- direct `ParallelLlmTool` remains renameable, supports structured output, and now emits kind `"user"`; +- explicit child plugin composition and inherited resolved-tool behavior, including a child model override retaining the parent's batch model. + +### Removal and integration + +- each removed `HarnessConfig` field raises its own migration error; +- `Harness(skills=...)` raises rather than being ignored; +- `builtin_tools=["skill_read"]`, `builtin_tools=["skill_run"]`, and `builtin_tools=["parallel_llm"]` give plugin migration errors; +- `builtin_tools=["subagent"]` still works and other unknown names keep the normal error; +- `SubAgentConfig.builtin_tools` raises even when another valid source is present; +- updated named-subagent validation uses the exact remaining source list; +- plugin/direct/structured-output tool collisions remain atomic; +- resume and approval state do not serialize plugin configuration or `PluginContext.model`; +- system instructions preserve plugin order and place all per-tool instructions last. + +Run: + +```bash +uv run pytest tests/unit/test_skills.py tests/unit/test_parallel_llm.py tests/unit/test_plugins.py tests/unit/test_architecture.py tests/unit/test_file_tools.py +uv run pytest tests/unit/test_harness.py tests/unit/test_subagents.py tests/unit/test_tracing.py tests/unit/test_resume.py tests/unit/test_approvals.py +uv run pytest +uv run ruff check . +uv run pyright +uv run scripts/build_site.py --check +uv run --env-file .env python tests/e2e/skills_journey.py +uv run --env-file .env python tests/e2e/parallel_llm_agent_journey.py +uv run --env-file .env python tests/e2e/parallel_llm_tool_journey.py +``` + +Report credential-based skips separately and do not count them as passes. + +## Documentation and caller migration + +Update: + +- README feature text and examples, including the current `builtin_parallel_llm_model` text; +- `docs/docs.md` configuration, plugin, skills, parallel batch, and child-agent examples; +- `docs/site/explainer/index.html` architecture tree, composition text, and feature tables; +- hand-written `docs/site/index.html` parallel-LLM card; +- the README-derived `docs/site/about/index.html` through the site builder; +- `tests/e2e/skills_journey.py` and `parallel_llm_agent_journey.py`; +- examples and tests using core skill or parallel settings; +- `examples/web_research_report/agent.py` to remove obsolete core `read_paths` and `write_paths` while keeping its direct structured `ParallelLlmTool`; +- the rejected Bash child built-in test; +- public exports and changelog breaking entries, including direct `ParallelLlmTool` specs changing from kind `"parallel_llm"` to `"user"`. + +Do not replace direct `ParallelLlmTool` values that use a custom name or structured output. Do not wrap independent custom tools in plugins. + +## Success criteria + +- Skills and the default parallel batch tool are enabled only through explicit plugins. +- `PluginContext` contains only the canonical root and configured model. +- Core contains no direct skill or parallel-LLM imports, configuration, discovery, summary, path policy, model projection, or tool construction. +- Skill tools and summary are static, ordered, and based on one constructor-time catalog while discovered files remain live. +- Parallel plugin binding is I/O-free, and model ownership and provider-setting rules are explicit and tested. +- Low-level `SkillRegistry` and `ParallelLlmTool` behavior remains available without plugin composition. +- Main `builtin_tools` selects only `subagent`; the child built-in selector is gone and rejected loudly. +- Existing child tool inheritance remains safe through the exact temporary bridge. +- Focused suites, full suite, Ruff, Pyright, site drift check, and the three relevant end-to-end journeys pass. + +## Out of scope + +- `SubagentsPlugin`, child-harness factories, or generic plugin inheritance. +- Removing `HarnessConfig.builtin_tools`, `HarnessConfig.subagents`, or the main `subagent` built-in. +- Replacing `SkillRegistry` discovery or frontmatter parsing. +- Sandboxing skill scripts or changing supported runners. +- Dynamic skill catalog refresh or filesystem watchers. +- Nested tools, memory, or multi-turn sessions inside parallel LLM batches. +- Changing the low-level `ParallelLlmTool` structured-output interface. +- Provider, tracing, structured output, approval, resume, limit, hook, Bash, or direct custom-tool migration. +- Plugin discovery, package manifests, hot reload, dependency ordering, or plugin-to-plugin lookup. + +## Review record + +One Codex, Claude, and GLM panel round reviewed plan v1. Plan v2 applies the verified findings on I/O-free parallel binding, exact skill inheritance, frozen catalog versus live contents, removed-field rejection, model-setting validation, prompt order, fixed names, batch accounting, subagent validation, `ToolKind`, documentation coverage, summary wording, and focused tests. No second plan-review round is scheduled. From 1506fe6d7c0cd398fd56cde5fd571baf8d621df2 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Tue, 18 Aug 2026 22:18:39 -0400 Subject: [PATCH 09/30] Add skills and parallel LLM plugins --- CHANGELOG.md | 8 +- README.md | 22 ++- docs/behavior.md | 40 ++++- docs/docs.md | 61 ++++--- docs/site/about/index.html | 6 +- docs/site/explainer/index.html | 50 +++--- docs/site/index.html | 2 +- examples/web_research_report/agent.py | 2 - tests/e2e/parallel_llm_agent_journey.py | 4 +- tests/e2e/skills_journey.py | 12 +- tests/unit/test_architecture.py | 21 +++ tests/unit/test_bash_tool.py | 10 +- tests/unit/test_harness.py | 71 ++++++--- tests/unit/test_parallel_llm.py | 202 +++++++++++++++++++++--- tests/unit/test_plugins.py | 22 ++- tests/unit/test_skills.py | 172 +++++++++++++++++++- tests/unit/test_subagents.py | 100 ++++++++++-- thinharness/__init__.py | 16 +- thinharness/core.py | 71 ++++----- thinharness/plugins/__init__.py | 4 + thinharness/plugins/base.py | 2 + thinharness/plugins/parallel_llm.py | 129 +++++++++++++++ thinharness/plugins/skills.py | 91 +++++++++++ thinharness/subagents.py | 63 +++++--- thinharness/tools/__init__.py | 3 +- thinharness/tools/base.py | 4 +- thinharness/tools/parallel_llm.py | 40 +---- thinharness/tools/skills.py | 5 +- 28 files changed, 986 insertions(+), 247 deletions(-) create mode 100644 thinharness/plugins/parallel_llm.py create mode 100644 thinharness/plugins/skills.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f0f8ebd..7b0853b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,17 @@ - Added explicit plugin composition with static and connected contributions, atomic connection rollback, unique plugin names, generic tool origin, and plugin-provided hooks and instructions. - Added `FilesystemPlugin` for the ordered workspace tool surface; `jsonl_search` remains opt-in through this plugin. - Added `MCPPlugin` for lazy MCP server connection, binding-local server identity, atomic tool discovery, and generic tool origin attribution. +- Added `SkillsPlugin` for constructor-time skill discovery, explicit ordered skill-tool selection, static summaries, and shared inherited-child catalogs. +- Added `ParallelLlmPlugin` for explicit text-only batch composition with borrowed harness or caller models and plugin-owned string-model provider settings. +- Added the configured harness model to the I/O-free `PluginContext` alongside the canonical root. - **Breaking:** Removed `HarnessConfig.mcp_servers`, `McpToolInfo`, and the MCP `ToolKind`; configure one `MCPPlugin` with all harness servers. - **Breaking:** Generic plugin validation now reports MCP tool collisions as duplicate tool names. Use MCP `tool_prefix`, `include_tools`, or `exclude_tools` to prevent collisions. - **Breaking:** Removed `MCPServer.resolve_id()` and post-bind mutation of `server.id`. The public `server.id` remains the base ID; binding-local IDs, including duplicate suffixes, appear in tool origin and result metadata. - **Breaking:** `Harness` no longer enables filesystem tools by default. Pass `plugins=[FilesystemPlugin(...)]`; independent custom tools still use `tools=`. -- **Breaking:** Removed filesystem settings from `HarnessConfig` and removed the `builtin_tools()` helper. `read_paths` and `write_paths` remain temporarily for the transitional parallel-LLM built-in. +- **Breaking:** Removed filesystem settings from `HarnessConfig` and removed the `builtin_tools()` helper. +- **Breaking:** Removed `HarnessConfig.skills_dir`, `selected_skills`, `read_paths`, `write_paths`, `builtin_parallel_llm_model`, `builtin_parallel_llm_temperature`, and `parallel_llm_max_prompts`; use `SkillsPlugin` and `ParallelLlmPlugin`. +- **Breaking:** Removed the `Harness(skills=...)` composition path, `SubAgentConfig.builtin_tools`, and `create_parallel_llm_tool`; named children now use explicit plugins or tools. +- **Breaking:** Removed the `parallel_llm` `ToolKind`; direct and plugin-created `ParallelLlmTool` specifications now use kind `"user"`. - Changed connection setup to complete before `run_start` hooks. A connection failure does not fire run lifecycle hooks. ## 0.6.0 - 2026-08-07 diff --git a/README.md b/README.md index 1a34225..be1b383 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,7 @@ ThinHarness has opinions. They are the reason it stays small. **Search is a top priority.** The `search` tool exposes ripgrep as compact grouped path/line results, tuned for document and business-workflow agents rather than code navigation. There's also a `jsonl_search` variant, because JSONL is the right shape when you're replacing RAG with agent-driven search over structured data: ripgrep row prefiltering, jq-style field projection, `where` filters, range filters, and snippets from large multiline fields. -**Parallel LLM calls, built in.** Fan out from inside the harness when a workflow needs efficient parallel processing or majority vote for reliability. Set `builtin_parallel_llm_model` to enable the default `parallel_llm` tool for plain-text batches; for validated structured output per call, instantiate `ParallelLlmTool` yourself with `output_type` (a Pydantic model). Each call is stateless, and large batches can write JSON to `output_file`. +**Parallel LLM calls, built in.** Fan out from inside the harness when a workflow needs efficient parallel processing or majority vote for reliability. Add `ParallelLlmPlugin()` for a plain-text batch tool that borrows the harness model, or give the plugin a model string and its own provider settings. For validated structured output per call, instantiate `ParallelLlmTool` with `output_type` (a Pydantic model). Each call is stateless, and large batches can write JSON to `output_file`. **No token streaming.** Streaming is for workflow progress, not live chatbot text. ThinHarness emits run, model-turn, tool, retry, limit, and subagent events, but it does not stream provider token deltas. Token streaming would add provider-specific plumbing, event merging, cancellation edge cases, and more surface area to keep stable. For workflow-style agents, step-level updates are usually the useful signal. @@ -269,6 +269,22 @@ harness = Harness( MCP tools connect and discover one tool snapshot lazily on `Harness.connect()` or the first run. Install support with `uv add 'thinharness[mcp]'`. +Skills and plain-text parallel batches are explicit plugins too: + +```python +from thinharness import ParallelLlmPlugin, SkillsPlugin + +harness = Harness( + HarnessConfig(root="."), + plugins=[ + # Relative skill directories use the process working directory. + SkillsPlugin(".agents/skills", tools=["skill_read"]), + # Parallel paths use HarnessConfig.root; no model means borrow the harness model. + ParallelLlmPlugin(read_paths=["inputs"], write_paths=["outputs"]), + ], +) +``` + Built-in provider requests retry transient HTTP failures three times by default. Configure the shared policy with `request_retries` and `request_retry_backoff` on `HarnessConfig`. If an injected `http_client` owns retries, set `request_retries=0` on the provider. This prevents nested retry policies from multiplying attempts. @@ -297,8 +313,8 @@ Streaming emits coarse run, model, tool, retry, limit, and subagent events, then - **Structured output:** Pydantic-validated results with native, tool, prompted, and text modes. - **Hooks:** lifecycle and tool-call interception for prompt submission, tool calls, subagents, limits, and run boundaries. - **Subagents:** opt-in delegation through a built-in `subagent` tool and explicit `SubAgentConfig`. -- **Parallel LLM:** opt-in `parallel_llm` fan-out for batches of independent one-shot prompts, plus `ParallelLlmTool(...).spec()` for renameable tools with explicit model, path, prompt, and provider request settings. -- **Skills:** explicit `skill_read` and `skill_run` tools for selected skill directories, with Python, shell, JavaScript, and Go script runners. +- **Parallel LLM:** explicit `ParallelLlmPlugin` fan-out for batches of independent one-shot prompts, plus `ParallelLlmTool(...).spec()` for renameable or structured tools with explicit model, path, prompt, and provider request settings. +- **Skills:** explicit `SkillsPlugin` composition with an ordered `skill_read` and/or `skill_run` selection, plus Python, shell, JavaScript, and Go script runners. - **Resume:** clean new-turn continuation through self-contained transcript state that can replay across built-in providers and models, preserving native reasoning on same-provider resume and degrading it to text across providers. - **MCP:** optional MCP support built on the FastMCP client, including in-process servers via `FastMCPTransport`, with lazy tool discovery and collision checks. - **Parallel tool calls:** same-turn tool batches run concurrently when every called tool is parallel-safe. diff --git a/docs/behavior.md b/docs/behavior.md index 1361309..ec058e3 100644 --- a/docs/behavior.md +++ b/docs/behavior.md @@ -90,12 +90,12 @@ Callers compose optional harness behavior explicitly while independent custom to - PLUGIN-1: `Harness` accepts plugins in caller order through `plugins=`; no plugin is loaded through entry points, directories, manifests, or implicit defaults. - PLUGIN-2: Plugin names are non-empty and unique within one harness. A duplicate name fails before either plugin binds. -- PLUGIN-3: Plugin binding is synchronous and performs no file or network I/O. Static tools, instructions, and hooks are validated and visible immediately after harness construction. +- PLUGIN-3: `PluginContext` contains the canonical harness root and configured model. Plugin binding is synchronous and performs no file or network I/O. Static tools, instructions, and hooks are validated and visible immediately after harness construction. - PLUGIN-4: `Harness.connect()` or the first run opens connected plugin bindings once in caller order. Concurrent connection calls share that attempt, and connection completes before `run_start` hooks fire. - PLUGIN-5: Dynamic tools, instructions, and hooks are staged and receive the same complete validation as static contributions. ThinHarness commits the full dynamic set only after every binding opens successfully. - PLUGIN-6: A connection failure, including cancellation, closes entered bindings in reverse order, installs no dynamic contribution, and leaves connection retryable. `run_start` and `run_end` do not fire for an attempt that fails during connection. - PLUGIN-7: Closing a harness closes plugin bindings in reverse order before closing a model owned by the harness. Repeated close calls have no effect. -- PLUGIN-8: Contribution order is plugin static contributions, direct `tools=` and `hooks=`, then plugin dynamic contributions. System instructions are the configured system prompt, plugin instructions, the transitional skill summary, and per-tool instructions; structured-output instructions are added through the existing output path. +- PLUGIN-8: Contribution order is plugin static contributions, direct `tools=` and `hooks=`, then plugin dynamic contributions. System instructions are the configured system prompt, plugin instructions in caller plugin order, and all per-tool instructions; structured-output instructions are added through the existing output path. A skill summary is an ordinary plugin instruction at the `SkillsPlugin` position. - PLUGIN-9: ThinHarness copies caller-supplied hook registries before adding plugin hooks. Plugin composition never mutates a caller-owned registry. - PLUGIN-10: Plugins are trusted in-process code. ThinHarness does not isolate them or resolve dependencies between them. @@ -111,9 +111,41 @@ Callers opt into root-scoped workspace tools without making filesystem behavior - FILESYSTEM-PLUGIN-2: `jsonl_search` is an opt-in tool of `FilesystemPlugin` and shares its root, read policy, search process, truncation, and spill-output handling. - FILESYSTEM-PLUGIN-3: `HarnessConfig.root` is the one run root. `FilesystemPlugin` uses that root and cannot configure a different root. - FILESYSTEM-PLUGIN-4: Harness construction and plugin binding do not create the workspace root. A harness without `FilesystemPlugin` has a generic default prompt, adds no workspace-root instruction, and has no workspace filesystem side effect. Observability sinks keep their independent configured storage behavior. -- FILESYSTEM-PLUGIN-5: Filesystem limits, output location, search settings, and path policies belong to `FilesystemPlugin`. `HarnessConfig.read_paths` and `write_paths` remain temporarily as parallel-LLM policy and do not configure filesystem plugin tools. +- FILESYSTEM-PLUGIN-5: Filesystem limits, output location, search settings, and path policies belong to `FilesystemPlugin`. - FILESYSTEM-PLUGIN-6: Independent custom tools continue to use `tools=[ToolSpec(...)]`; callers do not need to wrap one tool in a plugin. +## Skills Plugin + +### Purpose + +Callers explicitly compose a fixed skill catalog and select which skill operations a harness can use. + +### Requirements + +- SKILLS-PLUGIN-1: A harness accepts at most one runtime-fixed `SkillsPlugin` named `"skills"`. The plugin requires one or more ordered skill directories and an explicit non-empty ordered selection of `skill_read`, `skill_run`, or both. +- SKILLS-PLUGIN-2: The plugin discovers and validates its catalog during construction. Its selected tools and summary are static and visible immediately after harness construction, and binding performs no I/O. +- SKILLS-PLUGIN-3: Relative skill directories resolve from the process working directory, not from `HarnessConfig.root`. Reusing one plugin object across harnesses reuses the same registry and catalog. +- SKILLS-PLUGIN-4: Catalog names, paths, metadata, selection, and summary are frozen at plugin construction. Existing skill content, file trees, and scripts remain live and are read or executed when a tool is invoked. A new plugin is required to discover added or removed skills. +- SKILLS-PLUGIN-5: A non-empty catalog contributes selected tools in caller order and one compact summary. The summary mentions `skill_read` only when that tool is selected. A catalog with no skills contributes no tools or summary. +- SKILLS-PLUGIN-6: `skill_read` preserves live content, tree, containment, and truncation behavior and is parallel-safe. `skill_run` preserves runner, working-directory, merged-output, timeout, metadata, and containment behavior and runs sequentially. +- SKILLS-PLUGIN-7: An explicitly configured named child uses its own `SkillsPlugin`. A default child or a child that inherits parent tools rebinds the exact parent plugin, sharing its registry, catalog, tool order, and one summary without another discovery pass. + +## Parallel LLM Plugin + +### Purpose + +Callers explicitly compose a stateless parallel batch tool and choose whether it borrows a model or owns per-call provider construction. + +### Requirements + +- PARALLEL-LLM-PLUGIN-1: A harness accepts at most one runtime-fixed `ParallelLlmPlugin` named `"parallel_llm"`. It contributes one text-only `parallel_llm` tool with no model-visible model override. +- PARALLEL-LLM-PLUGIN-2: With no model argument, the plugin borrows the configured harness model. With a model object, it borrows that caller-owned object. The plugin does not close either borrowed model. +- PARALLEL-LLM-PLUGIN-3: A string model uses plugin-owned provider and request settings, creates a provider for each batch invocation, and closes it after success, schema-resolution failure, request failure, or cancellation. Provider and request settings are rejected for borrowed models. +- PARALLEL-LLM-PLUGIN-4: The plugin uses the canonical harness root. Read and write policies are root-scoped, outputs are atomic JSON files, prompt count and concurrency are bounded, and ordered sparse results report batch-local request, total, success, and failure counts. +- PARALLEL-LLM-PLUGIN-5: Batch prompts use independent fresh sessions and receive no parent system prompt, tools, memory, or continuation. Batch requests and tokens are outside parent `RunUsage` and `max_model_requests`; the parent counts one batch invocation toward `max_tool_calls`. +- PARALLEL-LLM-PLUGIN-6: Provider transport retries remain inside one logical batch request. Cancellation propagates and closes plugin-owned string-model providers. +- PARALLEL-LLM-PLUGIN-7: Callers that need a renamed tool or structured batch output use `ParallelLlmTool(...).spec()` directly. Direct specifications have the ordinary `"user"` tool kind. + ## Run Toolset Freeze ### Purpose @@ -176,7 +208,7 @@ Built-in provider requests recover from transient HTTP failures without repeatin - PROVIDER-RETRY-3: Retry delay uses `request_retry_backoff * 2**retry_index` plus up to 25 percent positive jitter. A valid numeric or HTTP-date `Retry-After` can increase that delay, and every delay is capped at 60 seconds. - PROVIDER-RETRY-4: Cancellation during a request or delay propagates immediately. Exhaustion raises the final attempt's `ProviderError`, preserving provider-error run classification. - PROVIDER-RETRY-5: Transport attempts stay inside one logical model request. They do not increase model request limits, usage counts, stream event counts, trace span counts, parallel completion request counts, or provider session history. -- PROVIDER-RETRY-6: Named subagent override models and inferred parallel completion models inherit the parent request retry settings. The parallel LLM tool has no separate provider retry loop or attempt budget. +- PROVIDER-RETRY-6: Named subagent override models inherit the parent request retry settings. A `ParallelLlmPlugin` with a string model uses its own request retry settings, while a plugin that borrows the harness model uses that model's configured transport retries. The parallel LLM tool has no separate provider retry loop or attempt budget. - PROVIDER-RETRY-7: Retries use at-least-once HTTP delivery. A transport failure after provider acceptance can cause duplicate provider work or charges because built-in providers do not share a portable idempotency-key contract. - PROVIDER-RETRY-8: A custom `http_client` can apply its own retry policy below the provider retry loop. Callers set `request_retries=0` when the custom client owns retries to avoid multiplying attempt budgets. diff --git a/docs/docs.md b/docs/docs.md index 70b580e..d881039 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -98,7 +98,7 @@ Important groups: - `root` defines the run root. `FilesystemPlugin` owns filesystem paths, limits, search settings, and output location. - `model`, `api_key`, `base_url`, `temperature`, `max_tokens`, `effort`, `extra_body`, `request_timeout`, `request_retries`, and `request_retry_backoff` define provider settings. -- The `Harness` constructor's `plugins=` and `tools=` inputs, plus `builtin_tools`, `subagents`, and `skills_dir`, define the model-callable surface. Filesystem and MCP tools use explicit plugins. `builtin_tools` is temporary for features that have not migrated to plugins. +- The `Harness` constructor's `plugins=` and `tools=` inputs, plus `builtin_tools` and `subagents`, define the model-callable surface. Filesystem, MCP, skills, and parallel LLM tools use explicit plugins. `builtin_tools` temporarily selects only `subagent`. - `max_model_requests`, `max_tool_calls`, `output_retries`, and `tool_retries` bound the run. - `output_type` and `output_mode` define structured output. - `tracing`, `local_tracing`, and `local_trace_dir` define observability. @@ -408,7 +408,7 @@ Calling `subagent` without an `agent` argument uses the framework default subage Named subagents can: - inherit parent tools with `inherit_parent_tools=True` -- choose explicit `plugins` or transitional `builtin_tools` +- choose explicit `plugins` - receive explicit custom `tools` - opt into MCP with `inherit_mcp_servers=True` or `mcp_servers=[...]` - use their own model, limits, and structured output @@ -417,21 +417,28 @@ Named subagents can: ## Parallel LLM Batches -`parallel_llm` is an opt-in built-in tool for batches of independent one-shot prompts: +Add `ParallelLlmPlugin` for batches of independent one-shot text prompts: ```python -harness = Harness(HarnessConfig( - root=".", - builtin_tools=["parallel_llm"], - builtin_parallel_llm_model="openai:gpt-5.5-mini", - builtin_parallel_llm_temperature=0, - parallel_llm_max_prompts=100, - request_retries=3, - request_retry_backoff=1.0, -)) +from thinharness import Harness, HarnessConfig, ParallelLlmPlugin + +harness = Harness( + HarnessConfig(root="."), + plugins=[ParallelLlmPlugin( + model="openai:gpt-5.5-mini", + temperature=0, + max_prompts=100, + request_retries=3, + request_retry_backoff=1.0, + read_paths=["inputs"], + write_paths=["outputs"], + )], +) ``` -Each batch call is stateless. Per-prompt calls receive no tools, no memory, no continuation, and no inherited parent harness system prompt. Pass `system` when the batch needs shared instructions. +Omit `model` to borrow the configured harness model, or pass a model object to borrow a caller-owned model. The plugin never closes a borrowed model. Provider and request settings are accepted only with a model string; that form creates and closes a configured provider for each batch invocation. Alternate string models do not inherit hidden harness provider settings. + +Each batch call is stateless and text-only. Per-prompt calls receive no tools, no memory, no continuation, and no inherited parent harness system prompt. Pass `system` when the batch needs shared instructions. The model-facing prompt source is structurally discriminated: @@ -440,7 +447,7 @@ The model-facing prompt source is structurally discriminated: Use `output_file` when combined results may be large. Inline output returns compact JSON in `ToolResult.content`; file output writes pretty JSON under the write path policy and returns a summary. -`max_concurrency` is model-controlled per tool call and limits in-flight requests. `parallel_llm_max_prompts` is a host-controlled `HarnessConfig` field. Built-in provider retries use the same `request_retries` and `request_retry_backoff` policy as normal agent requests. Transport attempts do not increase the tool's `model_requests` count or consume `max_model_requests`; the `parallel_llm` invocation still counts as one tool call. +`max_concurrency` is model-controlled per tool call and limits in-flight requests. `max_prompts` is host-controlled on `ParallelLlmPlugin`. Read and write paths resolve under `HarnessConfig.root`. A plugin that borrows the harness model uses that model's transport retry settings; a string-model plugin uses its own. Transport attempts do not increase the tool's batch-local `model_requests` count. Batch requests and tokens do not enter parent `RunUsage` or consume parent `max_model_requests`; the `parallel_llm` invocation counts as one parent tool call. Cancellation propagates and still closes a string-model provider. For a custom, renameable version, construct `ParallelLlmTool` directly: @@ -477,23 +484,29 @@ When `output_type` is set on a custom `ParallelLlmTool`, successful entries cont ## Skills -Skills are explicit tools, not auto-discovery. Configure `skills_dir`, then expose `skill_read` and/or `skill_run` through `builtin_tools`. +Skills are explicit plugins, not auto-discovery. `SkillsPlugin` requires an ordered, non-empty tool selection, so discovery never silently enables script execution. ```python +from thinharness import FilesystemPlugin, Harness, HarnessConfig, SkillsPlugin + harness = Harness( - HarnessConfig( - root=".", - skills_dir="skills", - selected_skills=["invoice-review"], - builtin_tools=["skill_read", "skill_run"], - ), - plugins=[FilesystemPlugin(tools=["read", "search"])], + HarnessConfig(root="."), + plugins=[ + FilesystemPlugin(tools=["read", "search"]), + SkillsPlugin( + "skills", + selected_skills=["invoice-review"], + tools=["skill_read", "skill_run"], + ), + ], ) ``` -If skills are configured and skill tools are exposed, the system prompt includes a compact skill summary. The model still has to call `skill_read` to inspect details. +Relative skill directories resolve from the process working directory, not from `HarnessConfig.root`. The plugin discovers one catalog when it is constructed. Names, paths, selected skills, and summary text stay fixed; added or removed skills require a new plugin. Existing `SKILL.md` content, file trees, and scripts stay live when tools run. Reusing one plugin object across harnesses reuses the same catalog and registry. + +A non-empty catalog contributes the selected tools in caller order and one compact summary. The summary tells the model to call `skill_read` only when that tool is selected. `skill_read` is parallel-safe. `skill_run` is sequential and runs scripts from trusted skill directories: Python through `uv run`, shell through `bash`, JavaScript through `node`, and Go through `go run`. -`skill_run` runs scripts from trusted skill directories. Python scripts run through `uv run`; shell scripts run through `bash`; JavaScript and Go files use `node` and `go run`. +An explicitly configured named child uses its own `SkillsPlugin`. Default children and named children with `inherit_parent_tools=True` temporarily rebind the exact parent plugin, so they share one catalog and one summary. ## MCP diff --git a/docs/site/about/index.html b/docs/site/about/index.html index 9b4ffd3..ba9517c 100644 --- a/docs/site/about/index.html +++ b/docs/site/about/index.html @@ -145,7 +145,7 @@

Opinions

purpose_built

Purpose-built agents, not universal agents

ThinHarness is for bounded agent loops, not open-ended interactive assistants like Claude Code or OpenClaw. For business use cases, focused agent loops orchestrated by deterministic code are usually a better fit than sprawling multi-agent systems with broad authority.

no_bash

No bash by default

Purpose-built business agents usually don't need a shell. Bash is a broad security and reliability surface: it gives the model open-ended authority instead of typed, bounded actions. ThinHarness keeps bash out of the default and built-in tool sets, but exposes an opt-in BashTool for exploratory runs before the workflow is hardened with typed tools.

search

Search is a top priority

The search tool exposes ripgrep as compact grouped path/line results, tuned for document and business-workflow agents rather than code navigation. There's also a jsonl_search variant, because JSONL is the right shape when you're replacing RAG with agent-driven search over structured data: ripgrep row prefiltering, jq-style field projection, where filters, range filters, and snippets from large multiline fields.

-
parallel_llm

Parallel LLM calls, built in

Fan out from inside the harness when a workflow needs efficient parallel processing or majority vote for reliability. Set builtin_parallel_llm_model to enable the default parallel_llm tool for plain-text batches; for validated structured output per call, instantiate ParallelLlmTool yourself with output_type (a Pydantic model). Each call is stateless, and large batches can write JSON to output_file.

+
parallel_llm

Parallel LLM calls, built in

Fan out from inside the harness when a workflow needs efficient parallel processing or majority vote for reliability. Add ParallelLlmPlugin() for a plain-text batch tool that borrows the harness model, or give the plugin a model string and its own provider settings. For validated structured output per call, instantiate ParallelLlmTool with output_type (a Pydantic model). Each call is stateless, and large batches can write JSON to output_file.

no_token_streaming

No token streaming

Streaming is for workflow progress, not live chatbot text. ThinHarness emits run, model-turn, tool, retry, limit, and subagent events, but it does not stream provider token deltas. Token streaming would add provider-specific plumbing, event merging, cancellation edge cases, and more surface area to keep stable. For workflow-style agents, step-level updates are usually the useful signal.

providers

Three providers, no matrix

ThinHarness ships small provider classes for OpenAI, Anthropic, and OpenRouter. If your gateway speaks one of those protocols, you swap a base URL and move on. If not, the provider classes are small enough to fork or replace, and ignoring the bundled ones costs you nothing.

no_compaction

No compaction

Compaction is a workaround for context windows filling up across long, accumulating runs — useful for interactive coding sessions that sprawl over hours. For SDK-based business agents, the right answer to "context is getting big" is almost always better task decomposition: shorter runs, separate harness instances, narrower subagents.

@@ -190,8 +190,8 @@

Features

Structured output

Pydantic-validated results with native, tool, prompted, and text modes.

Hooks

Lifecycle and tool-call interception for prompt submission, tool calls, subagents, limits, and run boundaries.

Subagents

Opt-in delegation through a built-in subagent tool and explicit SubAgentConfig.

-
Parallel LLM

Opt-in parallel_llm fan-out for batches of independent one-shot prompts, plus ParallelLlmTool(...).spec() for renameable tools with explicit model, path, prompt, and provider request settings.

-
Skills

Explicit skill_read and skill_run tools for selected skill directories, with Python, shell, JavaScript, and Go script runners.

+
Parallel LLM

Explicit ParallelLlmPlugin fan-out for batches of independent one-shot prompts, plus ParallelLlmTool(...).spec() for renameable or structured tools with explicit model, path, prompt, and provider request settings.

+
Skills

Explicit SkillsPlugin composition with an ordered skill_read and/or skill_run selection, plus Python, shell, JavaScript, and Go script runners.

Resume

Clean new-turn continuation through self-contained transcript state that can replay across built-in providers and models, preserving native reasoning on same-provider resume and degrading it to text across providers.

MCP

Optional MCP support built on the FastMCP client, including in-process servers via FastMCPTransport, with lazy tool discovery and collision checks.

Parallel tool calls

Same-turn tool batches run concurrently when every called tool is parallel-safe.

diff --git a/docs/site/explainer/index.html b/docs/site/explainer/index.html index 206f905..9102d82 100644 --- a/docs/site/explainer/index.html +++ b/docs/site/explainer/index.html @@ -164,7 +164,13 @@

Repository File Map

| |-- hooks.py hook dataclasses, registry, context variables | |-- subagents.py subagent tool and child harness construction | |-- tracing.py OTel-compatible spans and local JSONL tracing -| |-- defaults.py default filesystem-agent system prompt +| |-- defaults.py default system and tool instructions +| |-- plugins/ +| | |-- base.py plugin context, binding, contribution contracts +| | |-- filesystem.py FilesystemPlugin composition and root policy +| | |-- skills.py SkillsPlugin catalog and tool selection +| | |-- parallel_llm.py ParallelLlmPlugin model and path composition +| | `-- mcp.py MCPPlugin connection and discovery lifecycle | `-- tools/ | |-- __init__.py tool package exports | |-- base.py ToolSpec, ToolResult, path policy, invocation @@ -190,9 +196,9 @@

Harness-facing objects

- - - + + + @@ -259,9 +265,9 @@

Sequential flag

Plugin composition

- ThinHarness has no implicit filesystem tools. plugins=[FilesystemPlugin(...)] contributes an ordered - filesystem tool set, workspace instructions, and plugin origin data. builtin_tools remains temporarily - for skill_read, skill_run, subagent, and parallel_llm. + ThinHarness composes filesystem, skills, MCP, and parallel LLM behavior through explicit plugins. Plugins receive + the canonical root and configured model, and contribute ordered tools, instructions, hooks, or connected state. + builtin_tools remains temporarily only for subagent.

@@ -280,10 +286,16 @@

Tool surfaces

- - - - + + + + + + + + + + @@ -441,8 +453,8 @@

Extras

- - + + @@ -456,8 +468,8 @@

Extras

- - + +
NameMeaningRelationship
HarnessConfigPydantic setup model: root, model ref, tool selection, limits, output mode, tracing, MCP, subagents, path policies.Configures a Harness.
HarnessLong-lived configured runner. It owns tool maps, model object, hooks, skill registry, MCP server list, and tracing configuration.Creates a fresh RunContext for each run.
RequestConstantsFrozen per-run request constants: instructions, tool schemas, metadata, and the structured-output request. Built once after run-start hooks and MCP connection, so the toolset is frozen for the run.Passed to every ModelSession request by the turn machine in turns.py.
HarnessConfigPydantic setup model: root, model ref, deferred subagent selection, limits, output mode, tracing, and subagents.Configures a Harness.
HarnessLong-lived configured runner. It owns the model object, resolved tool map, plugin bindings, hooks, and tracing configuration.Creates a fresh RunContext for each run.
RequestConstantsFrozen per-run request constants: instructions, tool schemas, metadata, and the structured-output request. Built once after run-start hooks and plugin connection, so the toolset is frozen for the run.Passed to every ModelSession request by the turn machine in turns.py.
RunContextInternal state for one Harness.run(...): responses, tool records, usage, retry/notice state, terminal error, stop reason, tracing span, and final result.References the reusable Harness, but is not stored on it after the run.
HarnessResultFinal run result: final text, parsed structured output, raw provider responses, tool call records, usage, stop reason, and resume state.Receives finalized state from RunContext.
RunUsageCounts model requests, tool calls, cancelled tool calls, output retries, per-tool retry counts, and run token totals (input_tokens/output_tokens).Per-run counter owned by RunContext and returned in HarnessResult.
The default set is read, write, edit, search, list, glob; jsonl_search is opt-in. Mutating tools are sequential.
Framework-providedVaries: skills, subagents, MCP adapters, JSONL search, and ParallelLlmToolBuilt-in candidates, configured extras, or discovered MCP tools are converted into ToolSpec objects and added to the same runtime tool map.Optional surfaces still use normal tool execution. The Extras section covers feature-specific ownership and constraints.Plugin-providedSkillsPlugin, ParallelLlmPlugin, and MCPPluginExplicit plugin contributions are normalized into the same ToolSpec map with plugin origin attribution.Skills and parallel LLM contribute static tools; MCP connects lazily and contributes one discovered snapshot.
Deferred built-inSubagent compositionbuiltin_tools=["subagent"] selects the remaining core candidate.Child composition stays in subagents.py until its plugin migration.
Custom
Skillsskills_dir discovers available skills, but the model only gets skill tools when builtin_tools includes skill_read or skill_run.Skills add prompt summaries plus skill_read and skill_run; they do not create one tool per skill.SkillsPlugin(..., tools=[...]) discovers one catalog at construction and explicitly selects skill_read, skill_run, or both.Relative directories use the process cwd. Catalog metadata and summary stay fixed, while reads and scripts use live discovered files.
Subagents
Parallel LLMAvailable built-in tool candidate via create_parallel_llm_tool(parent), or custom renameable ParallelLlmTool(...).spec().Runs independent one-shot prompts concurrently. The built-in is configured by application code and text-only; custom ParallelLlmTool can opt into structured output.ParallelLlmPlugin(...) contributes the standard text-only tool; custom renameable or structured batches use ParallelLlmTool(...).spec().The plugin borrows the harness model by default or uses explicit model ownership rules. Paths stay under the canonical harness root.
@@ -914,13 +926,13 @@

Implementation Deep Dive

Skills - SkillRegistry exposes skill_read and skill_run when skills are configured. - Skills add prompt summaries plus skill_read and skill_run; they do not create one tool per skill. Script runners are extension-based. + SkillsPlugin owns constructor-time discovery and ordered tool selection over SkillRegistry. + The catalog and summary are static, discovered file content stays live, and inherited children rebind the exact parent plugin temporarily. Parallel LLM - A normal ToolSpec wrapper for independent one-shot model calls. - The built-in is configured by application code and text-only; custom ParallelLlmTool can opt into structured output. + ParallelLlmPlugin composes a normal text-only ToolSpec over independent one-shot model calls. + It borrows a harness or caller model, or owns per-call providers for model strings. Custom ParallelLlmTool can rename the tool or opt into structured output. diff --git a/docs/site/index.html b/docs/site/index.html index c32528b..236ec82 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -47,7 +47,7 @@

A minimal, opinionated agent harness.
purpose_built

Purpose-built agents

ThinHarness is for bounded agent loops inside software you control, not open-ended interactive assistants.

no_bash

No bash by default

Bash stays out of the default tools, with an opt-in BashTool only for prototyping before typed tools.

search

Search is a top priority

Ripgrep exposed as compact grouped results, tuned for documents and business workflows — plus a custom JSONL search tool for structured corpuses.

-
parallel_llm

Parallel LLM calls, built in

Fan out from inside the harness when a workflow needs efficient parallel processing or majority vote for reliability.

+
parallel_llm

Parallel LLM calls, built in

Add ParallelLlmPlugin to fan out independent prompts with the harness model, or configure a separate batch model.

no_compaction

No compaction

Compaction makes sense for sprawling coding sessions. For business agents the fix is smarter task decomposition and context management

no_deployment

No deployment layer

Serving, auth, durable jobs, and session storage stay yours. ThinHarness owns the agent loop, not the production stack around it.

diff --git a/examples/web_research_report/agent.py b/examples/web_research_report/agent.py index 9b7cc6d..4a6422a 100644 --- a/examples/web_research_report/agent.py +++ b/examples/web_research_report/agent.py @@ -494,8 +494,6 @@ def build_harness(root: Path, *, model: str = DEFAULT_MODEL) -> Harness: tool_retries=2, max_model_requests=64, max_tool_calls=96, - read_paths=["outputs"], - write_paths=["outputs"], tool_execution="sequential", request_timeout=240, temperature=0, diff --git a/tests/e2e/parallel_llm_agent_journey.py b/tests/e2e/parallel_llm_agent_journey.py index 5fd9819..7d723ac 100644 --- a/tests/e2e/parallel_llm_agent_journey.py +++ b/tests/e2e/parallel_llm_agent_journey.py @@ -10,7 +10,7 @@ from pydantic import BaseModel -from thinharness import Harness, HarnessConfig, Hook, ParallelLlmTool +from thinharness import Harness, HarnessConfig, Hook, ParallelLlmPlugin, ParallelLlmTool AGENT_MODEL = os.getenv("E2E_PARALLEL_AGENT_MODEL", "openai:gpt-5-mini") CUSTOM_TOOL_MODEL = os.getenv("E2E_PARALLEL_AGENT_TOOL_MODEL", os.getenv("E2E_PARALLEL_OPENROUTER_MODEL", "openrouter:google/gemini-2.5-flash")) @@ -76,11 +76,11 @@ def main() -> None: root=root, model=AGENT_MODEL, system_prompt=SYSTEM_PROMPT, - builtin_tools=["parallel_llm"], max_model_requests=8, max_tool_calls=4, tool_retries=2, ), + plugins=[ParallelLlmPlugin()], tools=[custom_tool], hooks=[Hook("before_tool_call", lambda ctx: tool_names.append(ctx.tool_name))], ) diff --git a/tests/e2e/skills_journey.py b/tests/e2e/skills_journey.py index 3fb3752..985ada1 100644 --- a/tests/e2e/skills_journey.py +++ b/tests/e2e/skills_journey.py @@ -7,7 +7,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from thinharness import Harness, HarnessConfig, Hook +from thinharness import Harness, HarnessConfig, Hook, SkillsPlugin MODEL = os.getenv("E2E_SKILLS_MODEL", "anthropic:claude-sonnet-4-5-20250929") SYSTEM_PROMPT = """You are an exacting skill-using agent. Read a skill before running any script from it.""" @@ -37,12 +37,16 @@ def main() -> None: root=root, model=MODEL, system_prompt=SYSTEM_PROMPT, - skills_dir=skills_dir, - selected_skills=["arithmetic-auditor"], - builtin_tools=["skill_read", "skill_run"], max_model_requests=6, max_tool_calls=4, ), + plugins=[ + SkillsPlugin( + skills_dir, + selected_skills=["arithmetic-auditor"], + tools=["skill_read", "skill_run"], + ) + ], hooks=[Hook("before_tool_call", lambda ctx: tool_names.append(ctx.tool_name))], ) diff --git a/tests/unit/test_architecture.py b/tests/unit/test_architecture.py index 0ca895c..9d500ab 100644 --- a/tests/unit/test_architecture.py +++ b/tests/unit/test_architecture.py @@ -15,3 +15,24 @@ def test_core_has_no_mcp_imports_or_lifecycle_state() -> None: assert not any(module.endswith(("tools.mcp", "plugins.mcp")) for module in imported_modules) for forbidden in ("MCPServer", "mcp_servers", "_mcp_", "_open_mcp_tools"): assert forbidden not in source + + +def test_core_has_no_skills_or_parallel_llm_implementation_details() -> None: + """Core stays independent from skills and parallel LLM composition.""" + core_path = Path(__file__).resolve().parents[2] / "thinharness" / "core.py" + source = core_path.read_text(encoding="utf-8") + + forbidden = ( + "tools.skills", + "SkillRegistry", + "skills_dir", + "selected_skills", + "_skills_enabled", + "prompt_summary", + "tools.parallel_llm", + "create_parallel_llm_tool", + "builtin_parallel_llm", + "parallel_llm_max_prompts", + ) + for token in forbidden: + assert token not in source diff --git a/tests/unit/test_bash_tool.py b/tests/unit/test_bash_tool.py index 3012a73..f763876 100644 --- a/tests/unit/test_bash_tool.py +++ b/tests/unit/test_bash_tool.py @@ -9,7 +9,6 @@ from thinharness import BashArgs, BashTool, Harness, HarnessConfig, SubAgentConfig, call_tool from thinharness.providers import ModelToolCall, ModelTurn -from thinharness.subagents import build_child_harness def test_bash_spec_exposes_expected_schema() -> None: @@ -151,12 +150,9 @@ def test_bash_is_not_a_builtin_tool(tmp_path: Path) -> None: Harness(HarnessConfig(root=tmp_path, builtin_tools=["bash"]), model=ScriptedModel([])) -def test_named_subagent_cannot_opt_into_bash_as_builtin(tmp_path: Path) -> None: - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([])) - config = SubAgentConfig(name="shell", description="Shell helper.", builtin_tools=["bash"]) - - with pytest.raises(ValueError, match="unknown builtin tool: bash"): - build_child_harness(parent, config) +def test_named_subagent_builtin_selector_is_removed() -> None: + with pytest.raises(ValueError, match="SubAgentConfig.builtin_tools has been removed"): + SubAgentConfig(name="shell", description="Shell helper.", builtin_tools=["bash"], tools=[BashTool().spec()]) def test_mixed_batch_containing_bash_runs_sequentially(tmp_path: Path) -> None: diff --git a/tests/unit/test_harness.py b/tests/unit/test_harness.py index acc3ae0..d40e069 100644 --- a/tests/unit/test_harness.py +++ b/tests/unit/test_harness.py @@ -32,7 +32,9 @@ OpenAIProvider, OpenAIResponsesModel, OpenRouterModel, + ParallelLlmPlugin, RequestConstants, + SkillsPlugin, SubAgentConfig, TokenUsage, ToolSpec, @@ -317,9 +319,9 @@ def test_specialized_filesystem_tools_are_explicit_opt_ins(tmp_path: Path) -> No def test_enabled_tool_instructions_are_appended_after_base_instructions(tmp_path: Path) -> None: harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=["parallel_llm"], system_prompt="Caller instructions."), + HarnessConfig(root=tmp_path, system_prompt="Caller instructions."), model=ScriptedModel([]), - plugins=[FilesystemPlugin(tools=[])], + plugins=[FilesystemPlugin(tools=[]), ParallelLlmPlugin()], ) instructions = harness.system_instructions() @@ -342,12 +344,9 @@ def test_tool_instructions_follow_skill_summary(tmp_path: Path) -> None: demo.mkdir(parents=True) (demo / "SKILL.md").write_text("---\nname: demo\ndescription: Demo skill\n---\nDemo", encoding="utf-8") harness = Harness( - HarnessConfig( - root=tmp_path, - skills_dir=tmp_path / "skills", - builtin_tools=["skill_read", "parallel_llm"], - ), + HarnessConfig(root=tmp_path), model=ScriptedModel([]), + plugins=[SkillsPlugin(tmp_path / "skills", tools=["skill_read"]), ParallelLlmPlugin()], ) instructions = harness.system_instructions() @@ -395,18 +394,19 @@ def test_tool_instructions_do_not_change_tool_schema(tmp_path: Path) -> None: } assert "Use echo_json only when echoing JSON." in harness.system_instructions() -def test_skill_dirs_require_selected_skill_tools(tmp_path: Path) -> None: +def test_skills_plugin_requires_explicit_selected_tools(tmp_path: Path) -> None: skill = tmp_path / "skills" / "demo" skill.mkdir(parents=True) (skill / "SKILL.md").write_text("---\nname: demo\n---\nDemo", encoding="utf-8") harness = Harness( - HarnessConfig(root=tmp_path, skills_dir=tmp_path / "skills", builtin_tools=["skill_read"]), + HarnessConfig(root=tmp_path), model=_fake_openai(FakeClient()), + plugins=[SkillsPlugin(tmp_path / "skills", tools=["skill_read"])], ) assert "skill_read" in [tool["name"] for tool in harness.tool_schemas()] - with pytest.raises(ValueError, match="skill_read or skill_run"): - Harness(HarnessConfig(root=tmp_path, skills_dir=tmp_path / "skills", builtin_tools=[]), model=_fake_openai(FakeClient())) + with pytest.raises(ValueError, match="must not be empty"): + SkillsPlugin(tmp_path / "skills", tools=[]) def test_skills_are_not_discovered_without_explicit_skills_dir(tmp_path: Path) -> None: skill = tmp_path / ".agents" / "skills" / "demo" @@ -427,23 +427,52 @@ def test_selected_skills_are_exposed_when_skill_tool_is_selected(tmp_path: Path) (other / "SKILL.md").write_text("---\nname: other\ndescription: Other skill\n---\nOther", encoding="utf-8") harness = Harness( - HarnessConfig( - root=tmp_path, - skills_dir=tmp_path / "skills", - selected_skills=["demo"], - builtin_tools=["skill_read"], - ), + HarnessConfig(root=tmp_path), model=_fake_openai(FakeClient()), - plugins=[FilesystemPlugin(tools=["read"])], + plugins=[ + FilesystemPlugin(tools=["read"]), + SkillsPlugin(tmp_path / "skills", selected_skills=["demo"], tools=["skill_read"]), + ], ) assert [tool["name"] for tool in harness.tool_schemas()] == ["read", "skill_read"] assert "demo - Demo skill" in harness.system_instructions() assert "other - Other skill" not in harness.system_instructions() -def test_selected_skills_without_skills_dir_fails() -> None: - with pytest.raises(ValueError, match="selected_skills requires skills_dir"): - HarnessConfig(selected_skills=["demo"]) +@pytest.mark.parametrize( + ("field", "plugin"), + [ + ("skills_dir", "SkillsPlugin"), + ("selected_skills", "SkillsPlugin"), + ("read_paths", "ParallelLlmPlugin"), + ("write_paths", "ParallelLlmPlugin"), + ("builtin_parallel_llm_model", "ParallelLlmPlugin"), + ("builtin_parallel_llm_temperature", "ParallelLlmPlugin"), + ("parallel_llm_max_prompts", "ParallelLlmPlugin"), + ], +) +def test_removed_harness_config_fields_fail_loudly(field: str, plugin: str) -> None: + with pytest.raises(ValueError, match=rf"HarnessConfig\.{field}.*{plugin}"): + HarnessConfig(**{field: "removed"}) + + +def test_removed_harness_skills_argument_fails_loudly() -> None: + with pytest.raises(TypeError, match="skills"): + Harness(HarnessConfig(), skills=object()) + + +@pytest.mark.parametrize( + ("name", "plugin"), + [("skill_read", "SkillsPlugin"), ("skill_run", "SkillsPlugin"), ("parallel_llm", "ParallelLlmPlugin")], +) +def test_removed_builtin_tool_names_point_to_plugins(name: str, plugin: str) -> None: + with pytest.raises(ValueError, match=plugin): + Harness(HarnessConfig(builtin_tools=[name]), model=ScriptedModel([])) + + +def test_unknown_builtin_tool_keeps_normal_error() -> None: + with pytest.raises(ValueError, match=r"unknown builtin tool: unknown; available: subagent"): + Harness(HarnessConfig(builtin_tools=["unknown"]), model=ScriptedModel([])) def test_child_harness_tool_surfaces_follow_subagent_policy(tmp_path: Path) -> None: parent_echo = echo_tool() diff --git a/tests/unit/test_parallel_llm.py b/tests/unit/test_parallel_llm.py index 1ae2d27..541f8b0 100644 --- a/tests/unit/test_parallel_llm.py +++ b/tests/unit/test_parallel_llm.py @@ -9,7 +9,7 @@ import pytest from pydantic import BaseModel, ValidationError -from thinharness import Harness, HarnessConfig, ModelCapabilities, ModelToolCall, ModelTurn, ToolOutput +from thinharness import Harness, HarnessConfig, ModelCapabilities, ModelToolCall, ModelTurn, ParallelLlmPlugin, PluginContext, ToolOutput from thinharness.providers import ModelSettings, OpenAIProvider, OpenAIResponsesModel, ProviderError from thinharness.tools.base import _invoke_tool from thinharness.tools.parallel_llm import ( @@ -18,7 +18,6 @@ ParallelLlmArgs, ParallelLlmTool, _atomic_write_json, - create_parallel_llm_tool, ) @@ -149,14 +148,24 @@ def dump_state(self): return None -def _parent(tmp_path: Path, batch_model: BatchModel | None = None, **config: Any) -> Harness: - """Build a harness parent for direct tool tests.""" - return Harness(HarnessConfig(root=tmp_path, **config), model=batch_model or BatchModel()) +def _parent( + tmp_path: Path, + batch_model: BatchModel | None = None, + *, + plugin: ParallelLlmPlugin | None = None, + **config: Any, +) -> Harness: + """Build a harness parent with the parallel LLM plugin.""" + return Harness( + HarnessConfig(root=tmp_path, **config), + model=batch_model or BatchModel(), + plugins=[plugin or ParallelLlmPlugin()], + ) async def _call_parallel(parent: Harness, args: dict[str, Any]) -> dict[str, Any]: """Invoke parallel_llm through the normal tool envelope.""" - spec = create_parallel_llm_tool(parent) + spec = next(tool for tool in parent.tools if tool.name == "parallel_llm") output = await _invoke_tool(spec, args) parsed = json.loads(output.to_json()) if parsed["ok"]: @@ -266,9 +275,7 @@ async def test_parallel_llm_enforces_path_policies_and_prompt_cap(tmp_path: Path parent = _parent( tmp_path, batch_model=model, - read_paths=["allowed"], - write_paths=["allowed"], - parallel_llm_max_prompts=1, + plugin=ParallelLlmPlugin(read_paths=["allowed"], write_paths=["allowed"], max_prompts=1), ) read_result = await _call_parallel(parent, _file("prompts.json")) @@ -543,7 +550,7 @@ async def test_builtin_parallel_llm_stays_text_only_with_json_output(tmp_path: P model = BatchModel(outcomes=['{"name":"Ada","age":37}']) model.capabilities = ModelCapabilities(supports_json_schema_output=True, default_structured_output_mode="native") parent = _parent(tmp_path, model) - spec = create_parallel_llm_tool(parent) + spec = next(tool for tool in parent.tools if tool.name == "parallel_llm") result = await _call_parallel(parent, {**_inline(["extract"]), "max_concurrency": 1}) @@ -656,6 +663,7 @@ def test_parallel_llm_tool_custom_spec_and_model_resolution(tmp_path: Path) -> N assert spec.name == "parallel_extract" assert spec.description == "Extract fields." + assert spec.kind == "user" assert isinstance(model, OpenAIResponsesModel) assert should_close is True assert model.provider.api_key == "key" @@ -666,7 +674,7 @@ def test_parallel_llm_tool_custom_spec_and_model_resolution(tmp_path: Path) -> N assert model.settings == ModelSettings(temperature=0.3, max_tokens=2048, effort="medium", extra_body={"seed": 1}) -async def test_builtin_parallel_llm_model_and_temperature_are_host_configured(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: +async def test_parallel_llm_plugin_string_model_uses_its_provider_settings(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: captured: dict[str, Any] = {} inferred = BatchModel() @@ -676,46 +684,46 @@ def fake_infer_model(model_ref: str, **kwargs: Any) -> BatchModel: return inferred monkeypatch.setattr("thinharness.providers.infer_model", fake_infer_model) - parent = _parent( - tmp_path, - BatchModel(), - api_key="parent-key", + plugin = ParallelLlmPlugin( + "openai:gpt-cheap", + api_key="plugin-key", base_url="https://example.test", max_tokens=4096, effort="low", request_retries=2, request_retry_backoff=0.25, - builtin_parallel_llm_model="openai:gpt-cheap", - builtin_parallel_llm_temperature=0.2, + temperature=0.2, ) + parent = _parent(tmp_path, BatchModel(), plugin=plugin) result = await _call_parallel(parent, _inline(["x"])) assert result["payload"]["succeeded"] == 1 assert captured["model_ref"] == "openai:gpt-cheap" - assert captured["kwargs"]["api_key"] == "parent-key" + assert captured["kwargs"]["api_key"] == "plugin-key" assert captured["kwargs"]["base_url"] == "https://example.test" assert captured["kwargs"]["temperature"] == 0.2 assert captured["kwargs"]["max_tokens"] == 4096 assert captured["kwargs"]["effort"] == "low" assert captured["kwargs"]["request_retries"] == 2 assert captured["kwargs"]["request_retry_backoff"] == 0.25 + assert inferred.provider.closed is True -def test_parallel_llm_builtin_selection(tmp_path: Path) -> None: +def test_parallel_llm_plugin_composition_and_builtin_migration(tmp_path: Path) -> None: default_harness = Harness(HarnessConfig(root=tmp_path / "default")) - selected_harness = Harness(HarnessConfig(root=tmp_path / "selected", builtin_tools=["parallel_llm"])) + selected_harness = Harness(HarnessConfig(root=tmp_path / "selected"), plugins=[ParallelLlmPlugin()]) assert "parallel_llm" not in {tool.name for tool in default_harness.tools} assert "parallel_llm" in {tool.name for tool in selected_harness.tools} assert next(tool for tool in selected_harness.tools if tool.name == "parallel_llm").instructions == DEFAULT_PARALLEL_LLM_INSTRUCTIONS - with pytest.raises(ValueError, match="parallel_llm"): - Harness(HarnessConfig(root=tmp_path / "bad", builtin_tools=["not_a_tool"])) + with pytest.raises(ValueError, match="ParallelLlmPlugin"): + Harness(HarnessConfig(root=tmp_path / "bad", builtin_tools=["parallel_llm"])) async def test_parallel_llm_usage_accounting_in_harness_run(tmp_path: Path) -> None: model = HybridModel() - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=["parallel_llm"], max_model_requests=3), model=model) + harness = Harness(HarnessConfig(root=tmp_path, max_model_requests=3), model=model, plugins=[ParallelLlmPlugin()]) result = await harness.run("go") @@ -725,3 +733,151 @@ async def test_parallel_llm_usage_accounting_in_harness_run(tmp_path: Path) -> N tool_record = json.loads(result.tool_call_records[0]["output"]) assert tool_record["metadata"]["model_requests"] == 2 assert [call["prompt"] for call in model.calls] == ["a", "b"] + + +def test_parallel_llm_plugin_static_contract_and_fixed_names(tmp_path: Path) -> None: + plugin = ParallelLlmPlugin(description="Batch now.", instructions="Use carefully.") + harness = Harness(HarnessConfig(root=tmp_path), model=BatchModel(), plugins=[plugin]) + spec = harness.tools[0] + + assert spec.name == "parallel_llm" + assert spec.description == "Batch now." + assert spec.instructions == "Use carefully." + assert spec.kind == "user" + assert spec.origin is not None + assert spec.origin.plugin == "parallel_llm" + assert spec.origin.source == "parallel_llm" + with pytest.raises(AttributeError, match="fixed"): + plugin.name = "other" + with pytest.raises(AttributeError, match="fixed"): + ParallelLlmPlugin.name = "other" + with pytest.raises(TypeError, match="cannot override"): + class RenamedParallelLlmPlugin(ParallelLlmPlugin): + name = "other" + + +@pytest.mark.parametrize( + "option", + [ + {"api_key": "key"}, + {"base_url": "https://example.test"}, + {"request_timeout": 1}, + {"request_retries": 1}, + {"request_retry_backoff": 0.1}, + {"temperature": 0.1}, + {"max_tokens": 1}, + {"effort": "low"}, + {"extra_body": {}}, + ], +) +def test_parallel_llm_plugin_rejects_provider_settings_for_borrowed_models(option: dict[str, Any]) -> None: + with pytest.raises(ValueError, match="valid only when"): + ParallelLlmPlugin(**option) + with pytest.raises(ValueError, match="valid only when"): + ParallelLlmPlugin(BatchModel(), **option) + + +def test_parallel_llm_plugin_rejects_invalid_prompt_cap() -> None: + with pytest.raises(ValueError, match="max_prompts"): + ParallelLlmPlugin(max_prompts=0) + + +async def test_parallel_llm_plugin_reuse_borrows_each_harness_model(tmp_path: Path) -> None: + plugin = ParallelLlmPlugin() + first_model = BatchModel(outcomes=["first"]) + second_model = BatchModel(outcomes=["second"]) + first = _parent(tmp_path / "first", first_model, plugin=plugin) + second = _parent(tmp_path / "second", second_model, plugin=plugin) + + first_result = await _call_parallel(first, _inline(["one"])) + second_result = await _call_parallel(second, _inline(["two"])) + + assert first_result["payload"]["results"][0]["result"] == "first" + assert second_result["payload"]["results"][0]["result"] == "second" + assert [call["prompt"] for call in first_model.calls] == ["one"] + assert [call["prompt"] for call in second_model.calls] == ["two"] + + +async def test_parallel_llm_plugin_borrows_explicit_model_without_closing_it(tmp_path: Path) -> None: + explicit_model = BatchModel(outcomes=["explicit"]) + harness_model = BatchModel() + harness = _parent(tmp_path, harness_model, plugin=ParallelLlmPlugin(explicit_model)) + + result = await _call_parallel(harness, _inline(["x"])) + await harness.aclose() + + assert result["payload"]["results"][0]["result"] == "explicit" + assert explicit_model.provider.closed is False + assert harness_model.provider.closed is False + + +def test_parallel_llm_plugin_bind_is_io_free_and_does_not_infer_provider(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + plugin = ParallelLlmPlugin("openai:gpt-child", read_paths=["future"], write_paths=["outputs"]) + + def fail(*_args, **_kwargs): + raise AssertionError("I/O or provider inference used during bind") + + monkeypatch.setattr(Path, "resolve", fail) + monkeypatch.setattr(Path, "exists", fail) + monkeypatch.setattr(Path, "stat", fail) + monkeypatch.setattr("thinharness.providers.infer_model", fail) + + binding = plugin.bind(PluginContext(root=tmp_path, model=BatchModel())) + assert binding.static.tools[0].name == "parallel_llm" + + +async def test_parallel_llm_plugin_string_model_closes_provider_after_request_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + inferred = BatchModel(outcomes=[RuntimeError("failed")]) + monkeypatch.setattr("thinharness.providers.infer_model", lambda *_args, **_kwargs: inferred) + harness = _parent(tmp_path, BatchModel(), plugin=ParallelLlmPlugin("openai:gpt-child")) + + result = await _call_parallel(harness, _inline(["x"])) + + assert result["payload"]["failed"] == 1 + assert inferred.provider.closed is True + + +async def test_parallel_llm_plugin_string_model_closes_provider_after_schema_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + inferred = BatchModel() + monkeypatch.setattr("thinharness.providers.infer_model", lambda *_args, **_kwargs: inferred) + monkeypatch.setattr( + "thinharness.tools.parallel_llm.resolve_output_schema_for_model", + lambda *_args, **_kwargs: (_ for _ in ()).throw(ValueError("schema failed")), + ) + harness = _parent(tmp_path, BatchModel(), plugin=ParallelLlmPlugin("openai:gpt-child")) + + result = await _call_parallel(harness, _inline(["x"])) + + assert result["ok"] is False + assert result["content"] == "schema failed" + assert inferred.provider.closed is True + + +async def test_parallel_llm_plugin_string_model_closes_provider_after_cancellation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + started = asyncio.Event() + + class BlockingModel(BatchModel): + async def complete(self, prompt: str, instructions: str, tools: list[dict[str, Any]], structured_output: Any = None) -> ModelTurn: + started.set() + await asyncio.Event().wait() + raise AssertionError("unreachable") + + inferred = BlockingModel() + monkeypatch.setattr("thinharness.providers.infer_model", lambda *_args, **_kwargs: inferred) + harness = _parent(tmp_path, BatchModel(), plugin=ParallelLlmPlugin("openai:gpt-child")) + task = asyncio.create_task(_call_parallel(harness, _inline(["x"]))) + await started.wait() + + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert inferred.provider.closed is True diff --git a/tests/unit/test_plugins.py b/tests/unit/test_plugins.py index bafbb2e..ebdfcd3 100644 --- a/tests/unit/test_plugins.py +++ b/tests/unit/test_plugins.py @@ -38,9 +38,11 @@ def __init__(self, name: str, contribution: PluginContribution | None = None) -> self.name = name self.contribution = contribution or PluginContribution() self.bindings = 0 + self.contexts: list[PluginContext] = [] def bind(self, context: PluginContext) -> PluginBinding: self.bindings += 1 + self.contexts.append(context) return PluginBinding(static=self.contribution) @@ -58,8 +60,11 @@ def __init__(self, name: str, events: list[str], contribution: PluginContributio self.contribution = contribution self.fail_first = fail_first self.attempts = 0 + self.contexts: list[PluginContext] = [] def bind(self, context: PluginContext) -> PluginBinding: + self.contexts.append(context) + @asynccontextmanager async def connect(): self.attempts += 1 @@ -179,6 +184,9 @@ async def test_plugins_connect_before_first_run_hook_and_close_in_reverse(tmp_pa ) assert "dynamic" not in [tool.name for tool in harness.tools] + assert first.contexts[0].root == tmp_path.resolve() + assert first.contexts[0].model is harness.model + assert second.contexts[0].model is harness.model assert (await harness.run("go")).text == "done" assert events == ["enter:first", "enter:second", "hook:direct", "hook:connected"] assert "dynamic" in [tool.name for tool in harness.tools] @@ -286,12 +294,16 @@ def test_caller_hook_registry_is_copied(tmp_path: Path) -> None: assert harness.hooks.strict_hooks is True -def test_one_plugin_object_binds_independently_to_two_harnesses(tmp_path: Path) -> None: +def test_one_plugin_object_receives_each_harness_root_and_model(tmp_path: Path) -> None: plugin = StaticPlugin("shared", PluginContribution(tools=(_tool("shared_tool"),))) - first = Harness(HarnessConfig(root=tmp_path / "one"), model=ScriptedModel([]), plugins=[plugin]) - second = Harness(HarnessConfig(root=tmp_path / "two"), model=ScriptedModel([]), plugins=[plugin]) + first_model = ScriptedModel([]) + second_model = ScriptedModel([]) + first = Harness(HarnessConfig(root=tmp_path / "one"), model=first_model, plugins=[plugin]) + second = Harness(HarnessConfig(root=tmp_path / "two"), model=second_model, plugins=[plugin]) assert plugin.bindings == 2 + assert [context.root for context in plugin.contexts] == [(tmp_path / "one").resolve(), (tmp_path / "two").resolve()] + assert [context.model for context in plugin.contexts] == [first_model, second_model] assert first.tools[0] is not second.tools[0] @@ -305,7 +317,9 @@ def fail(*_args, **_kwargs): monkeypatch.setattr(Path, "exists", fail) monkeypatch.setattr(Path, "is_file", fail) - binding = FilesystemPlugin(read_paths=["future"], write_paths=["outputs"]).bind(PluginContext(root=root)) + binding = FilesystemPlugin(read_paths=["future"], write_paths=["outputs"]).bind( + PluginContext(root=root, model=ScriptedModel([])) + ) assert [tool.name for tool in binding.static.tools] == ["read", "write", "edit", "search", "list", "glob"] diff --git a/tests/unit/test_skills.py b/tests/unit/test_skills.py index 2729ab6..11c6ca5 100644 --- a/tests/unit/test_skills.py +++ b/tests/unit/test_skills.py @@ -4,10 +4,9 @@ from pathlib import Path import pytest +from fakes import ScriptedModel -from thinharness import ( - SkillRegistry, -) +from thinharness import Harness, HarnessConfig, PluginBinding, PluginContext, PluginContribution, SkillRegistry, SkillsPlugin, ToolSpec def test_skill_registry_reads_and_runs_skill(tmp_path: Path) -> None: @@ -113,3 +112,170 @@ def test_skill_registry_rejects_duplicate_skill_names(tmp_path: Path) -> None: with pytest.raises(ValueError, match="duplicate skill name: demo"): SkillRegistry([tmp_path / "first", tmp_path / "second"]) + + +def _write_skill(root: Path, name: str, body: str = "Body", *, description: str = "Demo skill") -> Path: + skill = root / name + skill.mkdir(parents=True) + (skill / "SKILL.md").write_text( + f"---\nname: {name}\ndescription: {description}\n---\n{body}", + encoding="utf-8", + ) + return skill + + +@pytest.mark.parametrize( + "kwargs", + [ + {"skills_dir": set()}, + {"skills_dir": []}, + {"skills_dir": ["skills"], "tools": set()}, + {"skills_dir": ["skills"], "tools": []}, + {"skills_dir": ["skills"], "tools": ["skill_read", "skill_read"]}, + {"skills_dir": ["skills"], "tools": ["missing"]}, + ], +) +def test_skills_plugin_rejects_invalid_ordered_inputs(kwargs) -> None: + kwargs.setdefault("tools", ["skill_read"]) + with pytest.raises((TypeError, ValueError)): + SkillsPlugin(**kwargs) + + +def test_skills_plugin_discovers_and_selects_at_construction(tmp_path: Path) -> None: + _write_skill(tmp_path / "skills", "alpha", description="Alpha") + _write_skill(tmp_path / "skills" / "nested", "beta", description="Beta") + + plugin = SkillsPlugin(tmp_path / "skills", selected_skills=["beta"], tools=["skill_run", "skill_read"]) + harness = Harness(HarnessConfig(root=tmp_path / "workspace"), model=ScriptedModel([]), plugins=[plugin]) + + assert list(plugin.registry.skills) == ["beta"] + assert [tool.name for tool in harness.tools] == ["skill_run", "skill_read"] + assert [tool.origin.plugin for tool in harness.tools if tool.origin] == ["skills", "skills"] + assert "beta - Beta" in harness.system_instructions() + assert "alpha - Alpha" not in harness.system_instructions() + assert harness.tools[0].sequential is True + assert harness.tools[1].sequential is False + + + +def test_skills_plugin_empty_catalog_has_no_contribution_and_missing_selection_fails(tmp_path: Path) -> None: + plugin = SkillsPlugin(tmp_path / "missing", tools=["skill_read"]) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[plugin]) + + assert harness.tools == [] + assert "Available skills" not in harness.system_instructions() + with pytest.raises(ValueError, match="unknown selected skill"): + SkillsPlugin(tmp_path / "missing", selected_skills=["absent"], tools=["skill_read"]) + + + +def test_skills_plugin_catalog_is_frozen_but_discovered_content_and_scripts_are_live(tmp_path: Path) -> None: + skill = _write_skill(tmp_path / "skills", "demo", "Old body") + scripts = skill / "scripts" + scripts.mkdir() + script = scripts / "live.sh" + script.write_text("printf old\n", encoding="utf-8") + plugin = SkillsPlugin(tmp_path / "skills", tools=["skill_read", "skill_run"]) + + (skill / "SKILL.md").write_text("---\nname: demo\ndescription: Changed metadata\n---\nNew body", encoding="utf-8") + script.write_text("printf new\n", encoding="utf-8") + _write_skill(tmp_path / "skills", "added") + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[plugin]) + by_name = {tool.name: tool for tool in harness.tools} + + read_result = by_name["skill_read"].handler(by_name["skill_read"].parse_args({"skill_name": "demo"})) + run_result = by_name["skill_run"].handler( + by_name["skill_run"].parse_args({"skill_name": "demo", "script": "scripts/live.sh"}) + ) + assert "New body" in read_result.content + assert "new" in run_result.content + assert "added" not in plugin.registry.skills + assert "Changed metadata" not in harness.system_instructions() + assert "added" in SkillsPlugin(tmp_path / "skills", tools=["skill_read"]).registry.skills + + + +def test_skills_plugin_relative_paths_use_cwd_and_reuse_one_catalog(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + process_dir = tmp_path / "process" + _write_skill(process_dir / "skills", "demo") + process_dir.mkdir(exist_ok=True) + monkeypatch.chdir(process_dir) + plugin = SkillsPlugin("skills", tools=["skill_read"]) + + first = Harness(HarnessConfig(root=tmp_path / "one"), model=ScriptedModel([]), plugins=[plugin]) + second = Harness(HarnessConfig(root=tmp_path / "two"), model=ScriptedModel([]), plugins=[plugin]) + + assert plugin.registry.skills["demo"].root == process_dir / "skills" / "demo" + assert first.plugins[0] is second.plugins[0] is plugin + assert first.tools[0].handler.__self__ is second.tools[0].handler.__self__ is plugin.registry + + + +def test_skills_plugin_summary_wording_and_plugin_order(tmp_path: Path) -> None: + _write_skill(tmp_path / "skills", "demo") + + class InstructionPlugin: + def __init__(self, name: str, instruction: str) -> None: + self.name = name + self.instruction = instruction + + def bind(self, context: PluginContext) -> PluginBinding: + return PluginBinding(static=PluginContribution(instructions=(self.instruction,))) + + harness = Harness( + HarnessConfig(root=tmp_path, system_prompt="base"), + model=ScriptedModel([]), + plugins=[ + InstructionPlugin("before", "before marker"), + SkillsPlugin(tmp_path / "skills", tools=["skill_run"]), + InstructionPlugin("after", "after marker"), + ], + tools=[ToolSpec("direct", "direct", {"type": "object", "properties": {}}, lambda _args: "ok", instructions="tool marker")], + ) + instructions = harness.system_instructions() + + assert "call skill_read" not in instructions + assert instructions.index("before marker") < instructions.index("Available skills:") < instructions.index("after marker") + assert instructions.index("after marker") < instructions.index("tool marker") + assert instructions.count("Available skills:") == 1 + + + +def test_skills_plugin_name_is_fixed_and_collisions_are_atomic(tmp_path: Path) -> None: + _write_skill(tmp_path / "skills", "demo") + plugin = SkillsPlugin(tmp_path / "skills", tools=["skill_read"]) + + with pytest.raises(AttributeError, match="fixed"): + plugin.name = "other" + with pytest.raises(AttributeError, match="fixed"): + SkillsPlugin.name = "other" + with pytest.raises(TypeError, match="cannot override"): + class RenamedSkillsPlugin(SkillsPlugin): + name = "other" + + duplicate = ToolSpec("skill_read", "duplicate", {"type": "object", "properties": {}}, lambda _args: "ok") + with pytest.raises(ValueError, match="duplicate tool name"): + Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[plugin], tools=[duplicate]) + with pytest.raises(ValueError, match="duplicate plugin name: skills"): + Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[plugin, SkillsPlugin(tmp_path / "skills", tools=["skill_run"])], + ) + + + +def test_skills_plugin_bind_is_io_free_after_construction(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _write_skill(tmp_path / "skills", "demo") + plugin = SkillsPlugin(tmp_path / "skills", tools=["skill_read"]) + model = ScriptedModel([]) + + def fail(*_args, **_kwargs): + raise AssertionError("filesystem metadata used during bind") + + monkeypatch.setattr(Path, "resolve", fail) + monkeypatch.setattr(Path, "exists", fail) + monkeypatch.setattr(Path, "stat", fail) + + binding = plugin.bind(PluginContext(root=tmp_path, model=model)) + assert binding.static.tools[0].name == "skill_read" diff --git a/tests/unit/test_subagents.py b/tests/unit/test_subagents.py index 321b040..6f20e83 100644 --- a/tests/unit/test_subagents.py +++ b/tests/unit/test_subagents.py @@ -25,7 +25,10 @@ HookRegistry, MCPPlugin, MCPServerStdio, + ParallelLlmPlugin, + SkillsPlugin, SubAgentConfig, + ToolOrigin, ToolSpec, TracingOptions, build_child_harness, @@ -55,15 +58,18 @@ def test_subagent_config_validation_accepts_tool_specs() -> None: assert inherited.inherit_parent_tools is True with pytest.raises(ValueError, match="inherit_parent_tools"): SubAgentConfig(name="bad", description="Bad helper.", inherit_parent_tools=True, plugins=[FilesystemPlugin(tools=["read"])]) - with pytest.raises(ValueError, match="cannot be exposed"): - SubAgentConfig(name="recursive", description="Recursive helper.", builtin_tools=["subagent"]) + with pytest.raises(ValueError, match="SubAgentConfig.builtin_tools has been removed"): + SubAgentConfig(name="removed", description="Removed helper.", builtin_tools=["subagent"], tools=[spec]) with pytest.raises(ValueError, match="cannot be exposed"): SubAgentConfig( name="recursive-custom", description="Recursive helper.", tools=[ToolSpec("subagent", "Recursive", {"type": "object", "properties": {}}, lambda args: "bad")], ) - with pytest.raises(ValueError, match="must define"): + with pytest.raises( + ValueError, + match="named subagents must define plugins, tools, inherit_parent_tools=True, inherit_mcp_servers=True, or mcp_servers", + ): SubAgentConfig(name="empty", description="No tools.") with pytest.raises(ValueError): SubAgentConfig(name="bad name", description="Bad helper.", plugins=[FilesystemPlugin(tools=["read"])]) @@ -245,7 +251,6 @@ def test_named_inherited_subagent_gets_parent_tools_without_subagent(tmp_path: P child = build_child_harness(parent, SubAgentConfig(name="general", description="General helper.", inherit_parent_tools=True)) assert child.tools == [parent_echo] - assert child.skills is parent.skills assert child.config.subagents == [] @@ -271,30 +276,40 @@ def test_inherited_subagent_reuses_parent_skill_registry(tmp_path: Path) -> None skill = tmp_path / "skills" / "demo" skill.mkdir(parents=True) (skill / "SKILL.md").write_text("---\nname: demo\ndescription: Demo skill\n---\nDemo body", encoding="utf-8") + skills_plugin = SkillsPlugin(tmp_path / "skills", tools=["skill_read"]) parent = Harness( - HarnessConfig(root=tmp_path, skills_dir=tmp_path / "skills", builtin_tools=["skill_read"]), + HarnessConfig(root=tmp_path), model=ScriptedModel([]), + plugins=[skills_plugin], ) child = build_child_harness(parent, SubAgentConfig(name="general", description="General helper.", inherit_parent_tools=True)) - assert child.skills is parent.skills - assert "demo - Demo skill" in child.system_instructions() + child_skills_plugin = next(plugin for plugin in child.plugins if isinstance(plugin, SkillsPlugin)) + assert child_skills_plugin is skills_plugin + assert child_skills_plugin.registry is skills_plugin.registry + assert child.system_instructions().count("demo - Demo skill") == 1 skill_read = next(tool for tool in child.tools if tool.name == "skill_read") - assert skill_read.handler.__self__ is parent.skills + assert skill_read.handler.__self__ is skills_plugin.registry -def test_explicit_subagent_skill_tools_use_parent_skill_config(tmp_path: Path) -> None: +def test_explicit_subagent_skill_tools_use_its_own_plugin(tmp_path: Path) -> None: skill = tmp_path / "skills" / "demo" skill.mkdir(parents=True) (skill / "SKILL.md").write_text("---\nname: demo\ndescription: Demo skill\n---\nDemo body", encoding="utf-8") + parent_plugin = SkillsPlugin(tmp_path / "skills", tools=["skill_read"]) + child_plugin = SkillsPlugin(tmp_path / "skills", tools=["skill_read"]) parent = Harness( - HarnessConfig(root=tmp_path, skills_dir=tmp_path / "skills", builtin_tools=["skill_read"]), + HarnessConfig(root=tmp_path), model=ScriptedModel([]), + plugins=[parent_plugin], ) - child = build_child_harness(parent, SubAgentConfig(name="skilled", description="Skill helper.", builtin_tools=["skill_read"])) + child = build_child_harness( + parent, + SubAgentConfig(name="skilled", description="Skill helper.", plugins=[child_plugin]), + ) - assert child.skills is not parent.skills + assert child_plugin.registry is not parent_plugin.registry assert [tool.name for tool in child.tools] == ["skill_read"] assert "demo - Demo skill" in child.system_instructions() @@ -605,3 +620,64 @@ def cancel(ctx): def test_default_subagent_name_is_reserved() -> None: with pytest.raises(ValueError, match="reserved"): SubAgentConfig(name=DEFAULT_SUBAGENT_NAME, description="Reserved.", plugins=[FilesystemPlugin(tools=["read"])]) + + +def test_inherited_skills_bridge_orders_instructions_and_keeps_unrelated_origin_tools(tmp_path: Path) -> None: + skill = tmp_path / "skills" / "demo" + skill.mkdir(parents=True) + (skill / "SKILL.md").write_text("---\nname: demo\ndescription: Demo skill\n---\nBody", encoding="utf-8") + unrelated = ToolSpec( + "skill_helper", + "Unrelated direct tool", + {"type": "object", "properties": {}}, + lambda _args: "ok", + origin=ToolOrigin(plugin="skills", source="caller"), + ) + skills_plugin = SkillsPlugin(tmp_path / "skills", tools=["skill_read"]) + parent = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[skills_plugin, FilesystemPlugin(tools=["read"])], + tools=[unrelated], + ) + + child = build_child_harness(parent, None) + instructions = child.system_instructions() + + assert [plugin.name for plugin in child.plugins] == ["filesystem", "skills"] + assert [tool.name for tool in child.tools] == ["skill_read", "read", "skill_helper"] + assert instructions.index("Workspace root:") < instructions.index("demo - Demo skill") + assert instructions.count("demo - Demo skill") == 1 + assert next(tool for tool in child.tools if tool.name == "skill_helper") is unrelated + + + +def test_parallel_llm_explicit_child_plugin_and_inherited_handler_behavior(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + parent_batch_model = ScriptedModel([]) + parent = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[ParallelLlmPlugin(parent_batch_model)], + ) + parent_spec = next(tool for tool in parent.tools if tool.name == "parallel_llm") + + explicit_plugin = ParallelLlmPlugin(ScriptedModel([])) + explicit_child = build_child_harness( + parent, + SubAgentConfig(name="explicit", description="Explicit helper.", plugins=[explicit_plugin]), + ) + monkeypatch.setattr("thinharness.subagents.infer_model", lambda *_args, **_kwargs: ScriptedModel([])) + inherited_child = build_child_harness( + parent, + SubAgentConfig( + name="inherited", + description="Inherited helper.", + inherit_parent_tools=True, + model="openai:child", + ), + ) + + assert [tool.name for tool in explicit_child.tools] == ["parallel_llm"] + inherited_spec = next(tool for tool in inherited_child.tools if tool.name == "parallel_llm") + assert inherited_spec is parent_spec + assert not any(isinstance(plugin, ParallelLlmPlugin) for plugin in inherited_child.plugins) diff --git a/thinharness/__init__.py b/thinharness/__init__.py index 67a412f..c7a7aab 100644 --- a/thinharness/__init__.py +++ b/thinharness/__init__.py @@ -37,7 +37,17 @@ UserPromptSubmitContext, ) from .output import NativeOutput, OutputSchema, PromptedOutput, TextOutput, ToolStructuredOutput -from .plugins import FilesystemPlugin, MCPPlugin, Plugin, PluginBinding, PluginConnector, PluginContext, PluginContribution +from .plugins import ( + FilesystemPlugin, + MCPPlugin, + ParallelLlmPlugin, + Plugin, + PluginBinding, + PluginConnector, + PluginContext, + PluginContribution, + SkillsPlugin, +) from .providers import ( AnthropicMessagesModel, AnthropicProvider, @@ -86,7 +96,6 @@ ToolSpec, call_tool, contained_path, - create_parallel_llm_tool, ) from .tracing import LocalTracing, OtlpTracing, TracingOptions, create_local_tracing, create_local_tracing_options, create_otlp_tracing from .types import ApprovalDecision, HarnessError, HarnessResult, PendingApproval, RunUsage, UnexpectedModelBehavior @@ -131,6 +140,8 @@ "UnexpectedModelBehavior", "MCPDependencyError", "MCPPlugin", + "ParallelLlmPlugin", + "SkillsPlugin", "MCPError", "MCPServer", "MCPServerSSE", @@ -195,7 +206,6 @@ "build_child_harness", "call_tool", "contained_path", - "create_parallel_llm_tool", "create_local_tracing", "create_local_tracing_options", "create_subagent_tool", diff --git a/thinharness/core.py b/thinharness/core.py index f8667a0..b65165a 100644 --- a/thinharness/core.py +++ b/thinharness/core.py @@ -58,8 +58,6 @@ ) from .subagents import DEFAULT_SUBAGENT_NAME, SubAgentConfig, create_subagent_tool from .tools.base import ToolOrigin, ToolSpec -from .tools.parallel_llm import create_parallel_llm_tool -from .tools.skills import SkillRegistry from .tracing import ( LocalTracing, RunTracer, @@ -109,8 +107,6 @@ class HarnessConfig(BaseModel): api_key: str | None = None base_url: str | None = None system_prompt: str = DEFAULT_SYSTEM_PROMPT - skills_dir: str | Path | list[str | Path] | None = None - selected_skills: list[str] | None = None builtin_tools: list[str] | None = None max_model_requests: int = 64 max_tool_calls: int | None = None @@ -118,8 +114,6 @@ class HarnessConfig(BaseModel): request_timeout: int = 120 request_retries: int = Field(default=3, ge=0, le=10) request_retry_backoff: float = Field(default=1.0, ge=0, allow_inf_nan=False) - read_paths: list[str | Path] | None = None - write_paths: list[str | Path] | None = None temperature: float | None = None max_tokens: int | None = Field(default=None, ge=1) effort: str | None = None @@ -133,16 +127,27 @@ class HarnessConfig(BaseModel): output_mode: OutputMode = "auto" output_retries: int = Field(default=1, ge=0) tool_retries: int = Field(default=1, ge=0) - builtin_parallel_llm_model: str | None = None - builtin_parallel_llm_temperature: float | None = None - parallel_llm_max_prompts: int = Field(default=100, ge=1) - - @model_validator(mode="after") - def validate_config(self) -> HarnessConfig: - """Validate cross-field configuration settings.""" - if self.selected_skills is not None and self.skills_dir is None: - raise ValueError("selected_skills requires skills_dir") - return self + + @model_validator(mode="before") + @classmethod + def reject_removed_fields(cls, data: object) -> object: + """Fail loudly when callers use configuration moved to plugins.""" + if not isinstance(data, dict): + return data + migrations = ( + (("skills", "_dir"), "SkillsPlugin"), + (("selected", "_skills"), "SkillsPlugin"), + (("read", "_paths"), "ParallelLlmPlugin"), + (("write", "_paths"), "ParallelLlmPlugin"), + (("builtin", "_parallel", "_llm", "_model"), "ParallelLlmPlugin"), + (("builtin", "_parallel", "_llm", "_temperature"), "ParallelLlmPlugin"), + (("parallel", "_llm", "_max", "_prompts"), "ParallelLlmPlugin"), + ) + for parts, plugin_name in migrations: + field_name = "".join(parts) + if field_name in data: + raise ValueError(f"HarnessConfig.{field_name} has been removed; use {plugin_name}") + return data class Harness: @@ -156,7 +161,6 @@ def __init__( plugins: list[Plugin] | None = None, tools: list[ToolSpec] | None = None, tracing: list[TracingOptions] | None = None, - skills: SkillRegistry | None = None, hooks: list[Hook] | HookRegistry | None = None, subagent_hooks: dict[str, list[Hook] | HookRegistry] | None = None, _owns_model: bool | None = None, @@ -164,8 +168,6 @@ def __init__( ) -> None: self.config = config or HarnessConfig() self._is_child_run = _is_child_run - if skills is not None and (self.config.skills_dir is not None or self.config.selected_skills is not None): - raise ValueError("skills cannot be combined with skills_dir or selected_skills") self.root = Path(self.config.root).expanduser().resolve() self.model_ref = os.getenv("HARNESS_MODEL", self.config.model) self.model = model or infer_model( @@ -182,7 +184,6 @@ def __init__( ) self._owns_model = _owns_model if _owns_model is not None else model is None self.model_capabilities = model_capabilities(self.model) - self.skills = skills or SkillRegistry(self.config.skills_dir, selected_skills=self.config.selected_skills) output_schema = resolve_output_schema_for_model(self.model, self.config.output_type, self.config.output_mode) self.output_schema = output_schema @@ -193,7 +194,7 @@ def __init__( duplicate_plugin = next((name for index, name in enumerate(plugin_names) if name in plugin_names[:index]), None) if duplicate_plugin is not None: raise ValueError(f"duplicate plugin name: {duplicate_plugin}") - bindings = tuple(plugin.bind(PluginContext(root=self.root)) for plugin in configured_plugins) + bindings = tuple(plugin.bind(PluginContext(root=self.root, model=self.model)) for plugin in configured_plugins) for plugin, binding in zip(configured_plugins, bindings, strict=True): if not isinstance(binding, PluginBinding): raise TypeError(f"plugin {plugin.name!r} returned an invalid binding") @@ -206,11 +207,7 @@ def __init__( static_instructions.extend(contribution.instructions) static_hooks.extend(contribution.hooks) - builtin_candidates = [ - *self.skills.specs(), - create_subagent_tool(self, self.config.subagents), - create_parallel_llm_tool(self), - ] + builtin_candidates = [create_subagent_tool(self, self.config.subagents)] builtin = self._select_builtin_tools(builtin_candidates, self.config.builtin_tools) configured_tools = [*static_tools, *builtin, *(tools or [])] self._validate_tool_list( @@ -224,7 +221,6 @@ def __init__( strict_hooks = hooks.strict_hooks if isinstance(hooks, HookRegistry) else self.config.strict_hooks hook_registry = HookRegistry([*static_hooks, *caller_hooks], strict_hooks=strict_hooks) self._validate_hook_registry(hook_registry, self.config.subagents) - self._validate_skill_tool_selection_for(self.skills, configured_tools) self.plugins = configured_plugins self._plugin_bindings = bindings @@ -241,7 +237,6 @@ def __init__( self._connect_lock = asyncio.Lock() self._connect_task: asyncio.Task[None] | None = None self._connect_waiters = 0 - self._skills_enabled = bool(self.skills.skills) and any(tool.name in {"skill_read", "skill_run"} for tool in self.tools) self.local_tracing: LocalTracing | None = None external_tracing = list(self.config.tracing if tracing is None else tracing) if _local_tracing_enabled(self.config.local_tracing) and not _is_child_run: @@ -696,10 +691,6 @@ def tool_schemas(self) -> list[Json]: def system_instructions(self) -> str: """Return the full instruction text sent to the model.""" parts = [self.config.system_prompt, *self._plugin_instructions] - if self._skills_enabled: - skill_summary = self.skills.prompt_summary() - if skill_summary: - parts.append(skill_summary) tool_instructions = [] for tool in self.tools: if tool.instructions is None: @@ -782,15 +773,6 @@ def _validate_tool_approval_policy_for( if tool.requires_approval and is_child_run: raise ValueError("approval-required tools are not supported inside subagents") - @staticmethod - def _validate_skill_tool_selection_for(skills: SkillRegistry, tools: list[ToolSpec]) -> None: - """Require explicit skill tool selection for explicit skills and tool state.""" - if not skills.skills: - return - tool_names = {tool.name for tool in tools} - if not tool_names.intersection({"skill_read", "skill_run"}): - raise ValueError("configured skills require exposing skill_read or skill_run") - def _validate_hook_filters(self) -> None: """Validate hook filters against registered subagents.""" self._validate_hook_registry(self.hooks, self.config.subagents) @@ -861,7 +843,6 @@ async def _connect_once(self) -> None: ) candidate_hooks = HookRegistry([*base_hooks, *dynamic_hooks], strict_hooks=self._strict_hooks) self._validate_hook_registry(candidate_hooks, self.config.subagents) - self._validate_skill_tool_selection_for(self.skills, candidate_tools) if self._closed: raise HarnessError("harness is closed") @@ -870,7 +851,6 @@ async def _connect_once(self) -> None: self._tool_map = {tool.name: tool for tool in candidate_tools} self._plugin_instructions = [*self._base_instructions, *dynamic_instructions] self.hooks = candidate_hooks - self._skills_enabled = bool(self.skills.skills) and any(tool.name in {"skill_read", "skill_run"} for tool in self.tools) self._plugin_stack = plugin_stack self._connected = True except BaseException as exc: @@ -882,7 +862,6 @@ async def _connect_once(self) -> None: self._tool_map = {tool.name: tool for tool in self.tools} self._plugin_instructions = list(self._base_instructions) self.hooks = HookRegistry(base_hooks, strict_hooks=self._strict_hooks) - self._skills_enabled = bool(self.skills.skills) and any(tool.name in {"skill_read", "skill_run"} for tool in self.tools) if cleanup_error is not None: exc.add_note(f"cleanup also failed: {type(cleanup_error).__name__}: {cleanup_error}") raise @@ -950,6 +929,10 @@ def _select_builtin_tools(tools: list[ToolSpec], selected_names: list[str] | Non filesystem_names = {"read", "write", "edit", "search", "list", "glob", "jsonl_search"} if name in filesystem_names: raise ValueError(f"unknown builtin tool: {name}; use FilesystemPlugin(tools=[{name!r}])") + if name in {"skill_read", "skill_run"}: + raise ValueError(f"unknown builtin tool: {name}; use SkillsPlugin(tools=[{name!r}])") + if name == "parallel_llm": + raise ValueError("unknown builtin tool: parallel_llm; use ParallelLlmPlugin()") available = ", ".join(sorted(by_name)) or "none" raise ValueError(f"unknown builtin tool: {name}; available: {available}") selected.append(by_name[name]) diff --git a/thinharness/plugins/__init__.py b/thinharness/plugins/__init__.py index 5c9a5eb..67cad48 100644 --- a/thinharness/plugins/__init__.py +++ b/thinharness/plugins/__init__.py @@ -3,10 +3,14 @@ from .base import Plugin, PluginBinding, PluginConnector, PluginContext, PluginContribution, ToolOrigin from .filesystem import FilesystemPlugin from .mcp import MCPPlugin +from .parallel_llm import ParallelLlmPlugin +from .skills import SkillsPlugin __all__ = [ "FilesystemPlugin", "MCPPlugin", + "ParallelLlmPlugin", + "SkillsPlugin", "Plugin", "PluginBinding", "PluginConnector", diff --git a/thinharness/plugins/base.py b/thinharness/plugins/base.py index b4266b0..d81f2dc 100644 --- a/thinharness/plugins/base.py +++ b/thinharness/plugins/base.py @@ -12,6 +12,7 @@ if TYPE_CHECKING: from ..hooks import Hook + from ..providers import Model from ..tools.base import ToolSpec @@ -20,6 +21,7 @@ class PluginContext: """Stable core context available while binding one plugin.""" root: Path + model: Model @dataclass(frozen=True) diff --git a/thinharness/plugins/parallel_llm.py b/thinharness/plugins/parallel_llm.py new file mode 100644 index 0000000..3af222d --- /dev/null +++ b/thinharness/plugins/parallel_llm.py @@ -0,0 +1,129 @@ +"""Parallel LLM plugin.""" + +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from ..tools.parallel_llm import ( + DEFAULT_PARALLEL_LLM_DESCRIPTION, + DEFAULT_PARALLEL_LLM_INSTRUCTIONS, + ParallelLlmTool, +) +from .base import PluginBinding, PluginContext, PluginContribution + +if TYPE_CHECKING: + from ..providers import Model + + +class _ParallelLlmPluginMeta(type): + """Keep the parallel LLM plugin name fixed on the class hierarchy.""" + + def __setattr__(cls, attribute: str, value: object) -> None: + if attribute == "name": + raise AttributeError("ParallelLlmPlugin.name is fixed to 'parallel_llm'") + super().__setattr__(attribute, value) + + def __delattr__(cls, attribute: str) -> None: + if attribute == "name": + raise AttributeError("ParallelLlmPlugin.name is fixed to 'parallel_llm'") + super().__delattr__(attribute) + + +class ParallelLlmPlugin(metaclass=_ParallelLlmPluginMeta): + """Expose one root-scoped text-only parallel completion tool.""" + + name = "parallel_llm" + + def __init_subclass__(cls) -> None: + """Reject subclasses that replace the fixed plugin name.""" + super().__init_subclass__() + if "name" in cls.__dict__: + raise TypeError("ParallelLlmPlugin subclasses cannot override the fixed name 'parallel_llm'") + + def __setattr__(self, attribute: str, value: object) -> None: + """Reject instance changes to the fixed plugin name.""" + if attribute == "name": + raise AttributeError("ParallelLlmPlugin.name is fixed to 'parallel_llm'") + super().__setattr__(attribute, value) + + def __init__( + self, + model: Model | str | None = None, + *, + description: str = DEFAULT_PARALLEL_LLM_DESCRIPTION, + instructions: str | None = DEFAULT_PARALLEL_LLM_INSTRUCTIONS, + read_paths: Sequence[str | Path] | None = None, + write_paths: Sequence[str | Path] | None = None, + max_prompts: int = 100, + api_key: str | None = None, + base_url: str | None = None, + request_timeout: int | None = None, + request_retries: int | None = None, + request_retry_backoff: float | None = None, + temperature: float | None = None, + max_tokens: int | None = None, + effort: str | None = None, + extra_body: dict[str, Any] | None = None, + ) -> None: + if max_prompts < 1: + raise ValueError("max_prompts must be >= 1") + provider_options = { + "api_key": api_key, + "base_url": base_url, + "request_timeout": request_timeout, + "request_retries": request_retries, + "request_retry_backoff": request_retry_backoff, + "temperature": temperature, + "max_tokens": max_tokens, + "effort": effort, + "extra_body": extra_body, + } + if not isinstance(model, str): + supplied = next((name for name, value in provider_options.items() if value is not None), None) + if supplied is not None: + raise ValueError(f"{supplied} is valid only when ParallelLlmPlugin model is a string") + + self.model = model + self.description = description + self.instructions = instructions + self.read_paths = tuple(read_paths) if read_paths is not None else None + self.write_paths = tuple(write_paths) if write_paths is not None else None + self.max_prompts = max_prompts + self.api_key = api_key + self.base_url = base_url + self.request_timeout = request_timeout + self.request_retries = request_retries + self.request_retry_backoff = request_retry_backoff + self.temperature = temperature + self.max_tokens = max_tokens + self.effort = effort + self.extra_body = dict(extra_body) if extra_body is not None else None + + def bind(self, context: PluginContext) -> PluginBinding: + """Build the static tool with the canonical root and resolved model.""" + model = context.model if self.model is None else self.model + tool = ParallelLlmTool( + model=model, + root=context.root, + description=self.description, + instructions=self.instructions, + read_paths=list(self.read_paths) if self.read_paths is not None else None, + write_paths=list(self.write_paths) if self.write_paths is not None else None, + max_prompts=self.max_prompts, + api_key=self.api_key, + base_url=self.base_url, + request_timeout=120 if self.request_timeout is None else self.request_timeout, + request_retries=3 if self.request_retries is None else self.request_retries, + request_retry_backoff=1.0 if self.request_retry_backoff is None else self.request_retry_backoff, + temperature=self.temperature, + max_tokens=self.max_tokens, + effort=self.effort, + extra_body=self.extra_body, + _root_is_resolved=True, + ) + return PluginBinding(static=PluginContribution(tools=(tool.spec(),))) + + +__all__ = ["ParallelLlmPlugin"] diff --git a/thinharness/plugins/skills.py b/thinharness/plugins/skills.py new file mode 100644 index 0000000..4c773bb --- /dev/null +++ b/thinharness/plugins/skills.py @@ -0,0 +1,91 @@ +"""Skills plugin.""" + +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path +from typing import Literal + +from ..tools.skills import SkillRegistry +from .base import PluginBinding, PluginContext, PluginContribution + +SkillToolName = Literal["skill_read", "skill_run"] +_VALID_TOOLS = ("skill_read", "skill_run") + + +class _SkillsPluginMeta(type): + """Keep the skills plugin name fixed on the class hierarchy.""" + + def __setattr__(cls, attribute: str, value: object) -> None: + if attribute == "name": + raise AttributeError("SkillsPlugin.name is fixed to 'skills'") + super().__setattr__(attribute, value) + + def __delattr__(cls, attribute: str) -> None: + if attribute == "name": + raise AttributeError("SkillsPlugin.name is fixed to 'skills'") + super().__delattr__(attribute) + + +class SkillsPlugin(metaclass=_SkillsPluginMeta): + """Expose one constructor-time skill catalog through selected tools.""" + + name = "skills" + + def __init_subclass__(cls) -> None: + """Reject subclasses that replace the fixed plugin name.""" + super().__init_subclass__() + if "name" in cls.__dict__: + raise TypeError("SkillsPlugin subclasses cannot override the fixed name 'skills'") + + def __setattr__(self, attribute: str, value: object) -> None: + """Reject instance changes to the fixed plugin name.""" + if attribute == "name": + raise AttributeError("SkillsPlugin.name is fixed to 'skills'") + super().__setattr__(attribute, value) + + def __init__( + self, + skills_dir: str | Path | Sequence[str | Path], + *, + selected_skills: Sequence[str] | None = None, + tools: Sequence[SkillToolName], + ) -> None: + if isinstance(skills_dir, (set, frozenset)): + raise TypeError("SkillsPlugin skills_dir must be an ordered sequence, not a set") + if isinstance(skills_dir, str | Path): + directories: str | Path | tuple[str | Path, ...] = skills_dir + else: + directories = tuple(skills_dir) + if not directories: + raise ValueError("SkillsPlugin skills_dir must not be empty") + if isinstance(tools, (set, frozenset)): + raise TypeError("SkillsPlugin tools must be an ordered sequence, not a set") + selected_tools = tuple(tools) + if not selected_tools: + raise ValueError("SkillsPlugin tools must not be empty") + if len(set(selected_tools)) != len(selected_tools): + raise ValueError("SkillsPlugin tools contains a duplicate name") + unknown = next((name for name in selected_tools if name not in _VALID_TOOLS), None) + if unknown is not None: + available = ", ".join(_VALID_TOOLS) + raise ValueError(f"unknown SkillsPlugin tool: {unknown}; available: {available}") + + self.tools = selected_tools + self.registry = SkillRegistry(directories, selected_skills=selected_skills) + by_name = {spec.name: spec for spec in self.registry.specs()} + specs = tuple(by_name[name] for name in selected_tools if name in by_name) + instructions: tuple[str, ...] = () + if specs: + summary = self.registry.prompt_summary(include_read_hint="skill_read" in selected_tools) + if summary: + instructions = (summary,) + self._contribution = PluginContribution(tools=specs, instructions=instructions) + + def bind(self, context: PluginContext) -> PluginBinding: + """Return the constructor-time contribution without I/O.""" + del context + return PluginBinding(static=self._contribution) + + +__all__ = ["SkillsPlugin"] diff --git a/thinharness/subagents.py b/thinharness/subagents.py index f51e4f5..12a1d43 100644 --- a/thinharness/subagents.py +++ b/thinharness/subagents.py @@ -12,6 +12,7 @@ from .hooks import AfterSubagentRunContext, BeforeSubagentRunContext, HookRegistry, current_tool_call_context, current_tool_runtime_context from .plugins.base import Plugin from .plugins.mcp import MCPPlugin +from .plugins.skills import SkillsPlugin from .providers import infer_model, same_provider_model_ref from .tools.base import Json, ToolResult, ToolSpec from .tools.mcp import MCPServer @@ -34,7 +35,6 @@ class SubAgentConfig(BaseModel): system_prompt: str = DEFAULT_SYSTEM_PROMPT inherit_parent_tools: bool = False inherit_mcp_servers: bool = False - builtin_tools: list[str] = Field(default_factory=list) plugins: list[Plugin] = Field(default_factory=list) tools: list[ToolSpec] = Field(default_factory=list) mcp_servers: list[MCPServer] = Field(default_factory=list) @@ -50,8 +50,12 @@ class SubAgentConfig(BaseModel): @classmethod def reject_removed_fields(cls, data: object) -> object: """Fail loudly when callers pass fields removed from the public API.""" - if isinstance(data, dict) and "background" in data: - raise ValueError("SubAgentConfig.background has been removed") + if isinstance(data, dict): + if "background" in data: + raise ValueError("SubAgentConfig.background has been removed") + removed_builtin_field = "builtin" + "_tools" + if removed_builtin_field in data: + raise ValueError("SubAgentConfig.builtin_tools has been removed; use plugins or tools") return data @model_validator(mode="after") @@ -61,7 +65,7 @@ def validate_subagent(self) -> SubAgentConfig: raise ValueError(f"{DEFAULT_SUBAGENT_NAME!r} is reserved for the framework default subagent") if not self.description.strip() or "\n" in self.description or "\r" in self.description: raise ValueError("subagent description must be a non-empty single line") - exposes_subagent = any(name.lower() == "subagent" for name in self.builtin_tools) or any(_tool_name(tool).lower() == "subagent" for tool in self.tools) + exposes_subagent = any(_tool_name(tool).lower() == "subagent" for tool in self.tools) if exposes_subagent: raise ValueError("subagent cannot be exposed inside a child subagent") if any(tool.requires_approval for tool in self.tools): @@ -69,10 +73,10 @@ def validate_subagent(self) -> SubAgentConfig: has_explicit_mcp_plugin = any(isinstance(plugin, MCPPlugin) for plugin in self.plugins) if has_explicit_mcp_plugin and (self.mcp_servers or self.inherit_mcp_servers): raise ValueError("an explicit MCPPlugin cannot be combined with mcp_servers or inherit_mcp_servers=True") - if self.inherit_parent_tools and (self.builtin_tools or self.plugins or self.tools): - raise ValueError("inherit_parent_tools cannot be combined with builtin_tools, plugins, or tools") - if not (self.inherit_parent_tools or self.builtin_tools or self.plugins or self.tools or self.inherit_mcp_servers or self.mcp_servers): - raise ValueError("named subagents must define builtin_tools, plugins, tools, inherit_parent_tools=True, inherit_mcp_servers=True, or mcp_servers") + if self.inherit_parent_tools and (self.plugins or self.tools): + raise ValueError("inherit_parent_tools cannot be combined with plugins or tools") + if not (self.inherit_parent_tools or self.plugins or self.tools or self.inherit_mcp_servers or self.mcp_servers): + raise ValueError("named subagents must define plugins, tools, inherit_parent_tools=True, inherit_mcp_servers=True, or mcp_servers") return self @@ -236,12 +240,6 @@ def build_child_harness(parent: Harness, config: SubAgentConfig | None) -> Harne parent_config = parent.config inherit_tools = config is None or config.inherit_parent_tools - child_wants_skills = bool(config and any(name.lower() in {"skill_read", "skill_run"} for name in config.builtin_tools)) - if inherit_tools: - child_builtin_tools: list[str] = [] - else: - assert config is not None - child_builtin_tools = config.builtin_tools # Remove this MCP-specific bridge when subagents migrate to plugin composition. child_mcp_servers: list[MCPServer] = [] if config is not None and config.inherit_mcp_servers: @@ -260,9 +258,7 @@ def build_child_harness(parent: Harness, config: SubAgentConfig | None) -> Harne "model": config.model if config is not None and config.model is not None else parent_config.model, "root": parent.root, "system_prompt": DEFAULT_SYSTEM_PROMPT if config is None else config.system_prompt, - "builtin_tools": child_builtin_tools, - "skills_dir": parent_config.skills_dir if child_wants_skills and not inherit_tools else None, - "selected_skills": parent_config.selected_skills if child_wants_skills and not inherit_tools else None, + "builtin_tools": [], "max_model_requests": ( config.max_model_requests if config is not None and config.max_model_requests is not None else parent_config.max_model_requests ), @@ -295,7 +291,6 @@ def build_child_harness(parent: Harness, config: SubAgentConfig | None) -> Harne plugins=child_plugins, tools=_effective_custom_tools(parent, config), tracing=_child_tracing(parent, config), - skills=parent.skills if inherit_tools else None, hooks=_child_hooks(parent, config), subagent_hooks={}, _owns_model=config is not None and config.model is not None, @@ -316,22 +311,42 @@ def _select_config(configs: list[SubAgentConfig], agent: str | None) -> SubAgent def _effective_custom_tools(parent: Harness, config: SubAgentConfig | None) -> list[ToolSpec]: """Return custom tools to register on the child harness.""" if config is None or config.inherit_parent_tools: + skills_plugin = _parent_skills_plugin(parent) + selected_skill_tools = set(skills_plugin.tools) if skills_plugin is not None else set() return [ tool for tool in parent.tools - if tool.name != "subagent" and not (tool.origin is not None and tool.origin.plugin == "mcp") and not tool.requires_approval + if tool.name != "subagent" + and not (tool.origin is not None and tool.origin.plugin == "mcp") + and not ( + skills_plugin is not None + and tool.origin is not None + and tool.origin.plugin == "skills" + and tool.name in selected_skill_tools + ) + and not tool.requires_approval ] return list(config.tools) def _inherited_instruction_plugins(parent: Harness) -> list[Plugin]: - """Preserve instructions for inherited filesystem tools without duplicating them.""" + """Preserve filesystem and skill plugin instructions for inherited tools.""" + plugins: list[Plugin] = [] has_filesystem_tools = any(tool.origin is not None and tool.origin.plugin == "filesystem" for tool in parent.tools) - if not has_filesystem_tools: - return [] - from .plugins.filesystem import FilesystemPlugin + if has_filesystem_tools: + from .plugins.filesystem import FilesystemPlugin + + # Remove this filesystem instruction bridge when subagents migrate to plugin composition. + plugins.append(FilesystemPlugin(tools=[])) + # Remove this skills bridge when subagents migrate to plugin composition. + if skills_plugin := _parent_skills_plugin(parent): + plugins.append(skills_plugin) + return plugins + - return [FilesystemPlugin(tools=[])] +def _parent_skills_plugin(parent: Harness) -> SkillsPlugin | None: + """Return the exact parent skills plugin used by the temporary child bridge.""" + return next((plugin for plugin in parent.plugins if isinstance(plugin, SkillsPlugin)), None) def _child_tracing(parent: Harness, config: SubAgentConfig | None) -> list[TracingOptions]: diff --git a/thinharness/tools/__init__.py b/thinharness/tools/__init__.py index 113a08e..b1907d4 100644 --- a/thinharness/tools/__init__.py +++ b/thinharness/tools/__init__.py @@ -16,7 +16,7 @@ from .filesystem import FileTools from .jsonl import JsonlFieldSearch, JsonlSearch, JsonlSearchArgs, JsonlWhereFilter from .mcp import MCPDependencyError, MCPError, MCPServer, MCPServerSSE, MCPServerStdio, MCPServerStreamableHTTP -from .parallel_llm import FilePromptSource, InlinePromptSource, ParallelLlmArgs, ParallelLlmTool, create_parallel_llm_tool +from .parallel_llm import FilePromptSource, InlinePromptSource, ParallelLlmArgs, ParallelLlmTool from .skills import Skill, SkillRegistry __all__ = [ @@ -49,5 +49,4 @@ "ToolSpec", "call_tool", "contained_path", - "create_parallel_llm_tool", ] diff --git a/thinharness/tools/base.py b/thinharness/tools/base.py index 33a1f0a..09fe1b7 100644 --- a/thinharness/tools/base.py +++ b/thinharness/tools/base.py @@ -17,7 +17,7 @@ from ..types import Json -ToolKind = Literal["user", "subagent", "parallel_llm"] +ToolKind = Literal["user", "subagent"] ToolHandler = Callable[[Any], Any | Awaitable[Any]] T = TypeVar("T", bound=BaseModel) @@ -49,7 +49,7 @@ class ToolSpec: def __post_init__(self) -> None: """Validate per-tool retry configuration.""" - if self.kind not in {"user", "subagent", "parallel_llm"}: + if self.kind not in {"user", "subagent"}: raise ValueError(f"unknown tool kind: {self.kind}") if self.max_retries is not None and self.max_retries < 0: raise ValueError(f"max_retries must be >= 0, got {self.max_retries}") diff --git a/thinharness/tools/parallel_llm.py b/thinharness/tools/parallel_llm.py index d7c40c5..061a167 100644 --- a/thinharness/tools/parallel_llm.py +++ b/thinharness/tools/parallel_llm.py @@ -25,7 +25,6 @@ from .base import Json, PathPolicy, PathValidationError, StrictArgs, ToolResult, ToolSpec, coerce_args if TYPE_CHECKING: - from ..core import Harness from ..providers import Model @@ -99,6 +98,7 @@ def __init__( output_type: OutputSpec | None = None, output_mode: OutputMode = "auto", output_retries: int = 1, + _root_is_resolved: bool = False, ) -> None: from ..providers import _validate_retry_settings @@ -106,7 +106,8 @@ def __init__( self.name = name self.description = description self.instructions = instructions - self.root = Path(root).expanduser().resolve() + root_path = Path(root).expanduser() + self.root = root_path if _root_is_resolved else root_path.resolve() self.read_policy = PathPolicy(self.root, read_paths, "read") self.write_policy = PathPolicy(self.root, write_paths, "write") if max_prompts < 1: @@ -149,7 +150,6 @@ async def handler(raw_args: ParallelLlmArgs | Json) -> ToolResult: ParallelLlmArgs, handler, instructions=self.instructions, - kind="parallel_llm", ) async def run(self, args: ParallelLlmArgs) -> ToolResult: @@ -260,40 +260,6 @@ def _resolve_model(self) -> tuple[Model, bool]: ), True -def create_parallel_llm_tool(parent: Harness) -> ToolSpec: - """Create the built-in parallel LLM tool from a parent harness.""" - from ..providers import same_provider_model_ref - - model: Model | str = parent.config.builtin_parallel_llm_model or parent.model - model_ref = parent.config.builtin_parallel_llm_model or parent.model_ref - api_key = parent.config.api_key - base_url = parent.config.base_url - if parent.config.builtin_parallel_llm_model is not None: - if not same_provider_model_ref(parent.model, parent.config.builtin_parallel_llm_model): - api_key = None - base_url = None - return ParallelLlmTool( - model=model, - model_ref=model_ref, - root=parent.root, - description=_defaults.DEFAULT_PARALLEL_LLM_DESCRIPTION, - read_paths=parent.config.read_paths, - write_paths=parent.config.write_paths, - max_prompts=parent.config.parallel_llm_max_prompts, - instructions=_defaults.DEFAULT_PARALLEL_LLM_INSTRUCTIONS, - api_key=api_key, - base_url=base_url, - request_timeout=parent.config.request_timeout, - request_retries=parent.config.request_retries, - request_retry_backoff=parent.config.request_retry_backoff, - temperature=parent.config.builtin_parallel_llm_temperature - if parent.config.builtin_parallel_llm_temperature is not None - else parent.config.temperature, - max_tokens=parent.config.max_tokens, - effort=parent.config.effort, - extra_body=parent.config.extra_body, - ).spec() - def _load_prompts(args: ParallelLlmArgs, read_policy: PathPolicy) -> list[str]: """Load inline prompts or parse a prompt file under the read policy.""" diff --git a/thinharness/tools/skills.py b/thinharness/tools/skills.py index 796a5a2..e07e837 100644 --- a/thinharness/tools/skills.py +++ b/thinharness/tools/skills.py @@ -67,11 +67,12 @@ def skills(self) -> dict[str, Skill]: """Return a copy of the discovered skills map.""" return dict(self._skills) - def prompt_summary(self) -> str: + def prompt_summary(self, *, include_read_hint: bool = True) -> str: """Return a compact skill list for the system prompt.""" if not self._skills: return "" - lines = ["Available skills (call skill_read before using details):"] + heading = "Available skills (call skill_read before using details):" if include_read_hint else "Available skills:" + lines = [heading] for skill in self._skills.values(): desc = f" - {skill.description}" if skill.description else "" lines.append(f"- {skill.name}{desc}") From 2d63cdd156489b4113b6f2d4cd314e4b0b223813 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Tue, 18 Aug 2026 22:32:49 -0400 Subject: [PATCH 10/30] Apply plan 39 review fixes --- tests/unit/test_approvals.py | 39 +++++++++++++++ tests/unit/test_architecture.py | 12 +++++ tests/unit/test_parallel_llm.py | 73 +++++++++++++++++++++++++++++ tests/unit/test_resume.py | 34 +++++++++++++- thinharness/_migration.py | 28 +++++++++++ thinharness/core.py | 20 ++------ thinharness/plugins/parallel_llm.py | 25 ++++++---- thinharness/subagents.py | 9 ++-- 8 files changed, 210 insertions(+), 30 deletions(-) create mode 100644 thinharness/_migration.py diff --git a/tests/unit/test_approvals.py b/tests/unit/test_approvals.py index 307620c..6ff61f7 100644 --- a/tests/unit/test_approvals.py +++ b/tests/unit/test_approvals.py @@ -19,10 +19,12 @@ ModelToolCall, ModelTurn, OpenRouterModel, + ParallelLlmPlugin, PendingApproval, RunCompletedEvent, RunStartedEvent, RunUsage, + SkillsPlugin, SubAgentConfig, TokenUsage, ToolCallCompletedEvent, @@ -70,6 +72,43 @@ def approval_echo_tool(called: list[dict] | None = None) -> ToolSpec: ) +async def test_approval_state_excludes_plugin_configuration_and_context_model(tmp_path: Path) -> None: + skill = tmp_path / "skills" / "approval-state-skill-sentinel" + skill.mkdir(parents=True) + (skill / "SKILL.md").write_text( + "---\nname: approval-state-skill-sentinel\ndescription: approval-state-description-sentinel\n---\nBody", + encoding="utf-8", + ) + session = ScriptedSession( + start_turn=ModelTurn( + tool_calls=[ModelToolCall(id="call_1", name="deploy", arguments='{"env":"prod"}')], + raw={"id": "start"}, + ) + ) + model = ScriptedModel([session]) + model.context_marker = object() + harness = Harness( + HarnessConfig(root=tmp_path), + model=model, + plugins=[ + SkillsPlugin(tmp_path / "skills", tools=["skill_read"]), + ParallelLlmPlugin(description="approval-parallel-description-sentinel"), + ], + tools=[approval_tool()], + ) + + result = await harness.run("request deployment") + state = json.loads(json.dumps(result.resume_state)) + serialized = json.dumps(state, sort_keys=True) + + assert state == result.resume_state + assert "approval-state-skill-sentinel" not in serialized + assert "approval-state-description-sentinel" not in serialized + assert "approval-parallel-description-sentinel" not in serialized + assert "PluginContext" not in serialized + assert "context_marker" not in serialized + + async def test_approval_required_tool_pauses_without_executing_or_hooks(tmp_path: Path) -> None: called: list[dict] = [] hook_calls: list[str] = [] diff --git a/tests/unit/test_architecture.py b/tests/unit/test_architecture.py index 9d500ab..0031a38 100644 --- a/tests/unit/test_architecture.py +++ b/tests/unit/test_architecture.py @@ -36,3 +36,15 @@ def test_core_has_no_skills_or_parallel_llm_implementation_details() -> None: ) for token in forbidden: assert token not in source + + migration_source = (core_path.parent / "_migration.py").read_text(encoding="utf-8") + for field_name in ( + "skills_dir", + "selected_skills", + "read_paths", + "write_paths", + "builtin_parallel_llm_model", + "builtin_parallel_llm_temperature", + "parallel_llm_max_prompts", + ): + assert f'"{field_name}"' in migration_source diff --git a/tests/unit/test_parallel_llm.py b/tests/unit/test_parallel_llm.py index 541f8b0..b1b36e5 100644 --- a/tests/unit/test_parallel_llm.py +++ b/tests/unit/test_parallel_llm.py @@ -9,6 +9,7 @@ import pytest from pydantic import BaseModel, ValidationError +import thinharness.plugins.parallel_llm as parallel_plugin_module from thinharness import Harness, HarnessConfig, ModelCapabilities, ModelToolCall, ModelTurn, ParallelLlmPlugin, PluginContext, ToolOutput from thinharness.providers import ModelSettings, OpenAIProvider, OpenAIResponsesModel, ProviderError from thinharness.tools.base import _invoke_tool @@ -782,6 +783,59 @@ def test_parallel_llm_plugin_rejects_invalid_prompt_cap() -> None: ParallelLlmPlugin(max_prompts=0) +def test_parallel_llm_plugin_omits_default_sentinels_and_preserves_explicit_falsey_settings( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: list[dict[str, Any]] = [] + real_tool = parallel_plugin_module.ParallelLlmTool + + def capture_tool(**kwargs: Any) -> ParallelLlmTool: + captured.append(kwargs) + return real_tool(**kwargs) + + monkeypatch.setattr(parallel_plugin_module, "ParallelLlmTool", capture_tool) + context = PluginContext(root=tmp_path, model=BatchModel()) + + ParallelLlmPlugin("openai:default").bind(context) + ParallelLlmPlugin( + "openai:explicit", + api_key="", + base_url="", + request_timeout=0, + request_retries=0, + request_retry_backoff=0, + temperature=0, + max_tokens=1, + effort="", + extra_body={}, + ).bind(context) + + provider_names = { + "api_key", + "base_url", + "request_timeout", + "request_retries", + "request_retry_backoff", + "temperature", + "max_tokens", + "effort", + "extra_body", + } + assert provider_names.isdisjoint(captured[0]) + assert {name: captured[1][name] for name in provider_names} == { + "api_key": "", + "base_url": "", + "request_timeout": 0, + "request_retries": 0, + "request_retry_backoff": 0, + "temperature": 0, + "max_tokens": 1, + "effort": "", + "extra_body": {}, + } + + async def test_parallel_llm_plugin_reuse_borrows_each_harness_model(tmp_path: Path) -> None: plugin = ParallelLlmPlugin() first_model = BatchModel(outcomes=["first"]) @@ -811,6 +865,25 @@ async def test_parallel_llm_plugin_borrows_explicit_model_without_closing_it(tmp assert harness_model.provider.closed is False +async def test_parallel_llm_plugin_does_not_close_borrowed_harness_owned_model_during_batch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + inferred = BatchModel(outcomes=["borrowed"]) + monkeypatch.setattr("thinharness.core.infer_model", lambda *_args, **_kwargs: inferred) + harness = Harness( + HarnessConfig(root=tmp_path, model="openai:owned"), + plugins=[ParallelLlmPlugin()], + ) + + result = await _call_parallel(harness, _inline(["x"])) + + assert result["payload"]["results"][0]["result"] == "borrowed" + assert inferred.provider.closed is False + await harness.aclose() + assert inferred.provider.closed is True + + def test_parallel_llm_plugin_bind_is_io_free_and_does_not_infer_provider(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: plugin = ParallelLlmPlugin("openai:gpt-child", read_paths=["future"], write_paths=["outputs"]) diff --git a/tests/unit/test_resume.py b/tests/unit/test_resume.py index a2d7b5c..0ba7551 100644 --- a/tests/unit/test_resume.py +++ b/tests/unit/test_resume.py @@ -6,7 +6,7 @@ from pathlib import Path import pytest -from fakes import FakeAnthropicProvider, FakeClient, FakeOpenRouterProvider, ScriptedProvider, ScriptedSession, echo_tool +from fakes import FakeAnthropicProvider, FakeClient, FakeOpenRouterProvider, ScriptedModel, ScriptedProvider, ScriptedSession, echo_tool from pydantic import BaseModel from thinharness import ( @@ -22,6 +22,8 @@ OpenAIProvider, OpenAIResponsesModel, OpenRouterModel, + ParallelLlmPlugin, + SkillsPlugin, ToolSpec, ) from thinharness.hooks import RunEndContext @@ -54,6 +56,36 @@ async def create_message(self, payload): return {"content": [{"type": "text", "text": "done"}], "stop_reason": "end_turn"} +async def test_resume_state_excludes_plugin_configuration_and_context_model(tmp_path: Path) -> None: + skill = tmp_path / "skills" / "resume-state-skill-sentinel" + skill.mkdir(parents=True) + (skill / "SKILL.md").write_text( + "---\nname: resume-state-skill-sentinel\ndescription: resume-state-description-sentinel\n---\nBody", + encoding="utf-8", + ) + model = ScriptedModel([ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"}))]) + model.context_marker = object() + harness = Harness( + HarnessConfig(root=tmp_path), + model=model, + plugins=[ + SkillsPlugin(tmp_path / "skills", tools=["skill_read"]), + ParallelLlmPlugin(description="resume-parallel-description-sentinel"), + ], + ) + + result = await harness.run("finish without tools") + state = json.loads(json.dumps(result.resume_state)) + serialized = json.dumps(state, sort_keys=True) + + assert state == result.resume_state + assert "resume-state-skill-sentinel" not in serialized + assert "resume-state-description-sentinel" not in serialized + assert "resume-parallel-description-sentinel" not in serialized + assert "PluginContext" not in serialized + assert "context_marker" not in serialized + + async def test_openai_resume_full_replays_transcript_for_followup(tmp_path: Path) -> None: (tmp_path / "hello.txt").write_text("hello", encoding="utf-8") client = FakeClient() diff --git a/thinharness/_migration.py b/thinharness/_migration.py new file mode 100644 index 0000000..8df7fbc --- /dev/null +++ b/thinharness/_migration.py @@ -0,0 +1,28 @@ +"""Helpers for fail-loud removal guards.""" + +from __future__ import annotations + +from collections.abc import Mapping + +REMOVED_HARNESS_CONFIG_FIELDS = { + "skills_dir": "SkillsPlugin", + "selected_skills": "SkillsPlugin", + "read_paths": "ParallelLlmPlugin", + "write_paths": "ParallelLlmPlugin", + "builtin_parallel_llm_model": "ParallelLlmPlugin", + "builtin_parallel_llm_temperature": "ParallelLlmPlugin", + "parallel_llm_max_prompts": "ParallelLlmPlugin", +} + + +def reject_removed_fields(data: object, *, owner: str, migrations: Mapping[str, str]) -> object: + """Reject the first removed field with its migration target.""" + if not isinstance(data, dict): + return data + for field_name, migration in migrations.items(): + if field_name in data: + raise ValueError(f"{owner}.{field_name} has been removed; use {migration}") + return data + + +__all__ = ["REMOVED_HARNESS_CONFIG_FIELDS", "reject_removed_fields"] diff --git a/thinharness/core.py b/thinharness/core.py index b65165a..b55e3cf 100644 --- a/thinharness/core.py +++ b/thinharness/core.py @@ -11,6 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator +from ._migration import REMOVED_HARNESS_CONFIG_FIELDS, reject_removed_fields from .approvals import ( ApprovalPause, copy_restored_run_state, @@ -132,22 +133,11 @@ class HarnessConfig(BaseModel): @classmethod def reject_removed_fields(cls, data: object) -> object: """Fail loudly when callers use configuration moved to plugins.""" - if not isinstance(data, dict): - return data - migrations = ( - (("skills", "_dir"), "SkillsPlugin"), - (("selected", "_skills"), "SkillsPlugin"), - (("read", "_paths"), "ParallelLlmPlugin"), - (("write", "_paths"), "ParallelLlmPlugin"), - (("builtin", "_parallel", "_llm", "_model"), "ParallelLlmPlugin"), - (("builtin", "_parallel", "_llm", "_temperature"), "ParallelLlmPlugin"), - (("parallel", "_llm", "_max", "_prompts"), "ParallelLlmPlugin"), + return reject_removed_fields( + data, + owner="HarnessConfig", + migrations=REMOVED_HARNESS_CONFIG_FIELDS, ) - for parts, plugin_name in migrations: - field_name = "".join(parts) - if field_name in data: - raise ValueError(f"HarnessConfig.{field_name} has been removed; use {plugin_name}") - return data class Harness: diff --git a/thinharness/plugins/parallel_llm.py b/thinharness/plugins/parallel_llm.py index 3af222d..6b3d632 100644 --- a/thinharness/plugins/parallel_llm.py +++ b/thinharness/plugins/parallel_llm.py @@ -104,6 +104,21 @@ def __init__( def bind(self, context: PluginContext) -> PluginBinding: """Build the static tool with the canonical root and resolved model.""" model = context.model if self.model is None else self.model + provider_options: dict[str, Any] = {} + for name in ( + "api_key", + "base_url", + "request_timeout", + "request_retries", + "request_retry_backoff", + "temperature", + "max_tokens", + "effort", + "extra_body", + ): + value = getattr(self, name) + if value is not None: + provider_options[name] = value tool = ParallelLlmTool( model=model, root=context.root, @@ -112,16 +127,8 @@ def bind(self, context: PluginContext) -> PluginBinding: read_paths=list(self.read_paths) if self.read_paths is not None else None, write_paths=list(self.write_paths) if self.write_paths is not None else None, max_prompts=self.max_prompts, - api_key=self.api_key, - base_url=self.base_url, - request_timeout=120 if self.request_timeout is None else self.request_timeout, - request_retries=3 if self.request_retries is None else self.request_retries, - request_retry_backoff=1.0 if self.request_retry_backoff is None else self.request_retry_backoff, - temperature=self.temperature, - max_tokens=self.max_tokens, - effort=self.effort, - extra_body=self.extra_body, _root_is_resolved=True, + **provider_options, ) return PluginBinding(static=PluginContribution(tools=(tool.spec(),))) diff --git a/thinharness/subagents.py b/thinharness/subagents.py index 12a1d43..1ef4169 100644 --- a/thinharness/subagents.py +++ b/thinharness/subagents.py @@ -53,8 +53,7 @@ def reject_removed_fields(cls, data: object) -> object: if isinstance(data, dict): if "background" in data: raise ValueError("SubAgentConfig.background has been removed") - removed_builtin_field = "builtin" + "_tools" - if removed_builtin_field in data: + if "builtin_tools" in data: raise ValueError("SubAgentConfig.builtin_tools has been removed; use plugins or tools") return data @@ -250,7 +249,7 @@ def build_child_harness(parent: Harness, config: SubAgentConfig | None) -> Harne for server in config.mcp_servers: if not any(server is existing for existing in child_mcp_servers): child_mcp_servers.append(server) - child_plugins = list(_inherited_instruction_plugins(parent) if inherit_tools else (config.plugins if config is not None else [])) + child_plugins = list(_inherited_bridge_plugins(parent) if inherit_tools else (config.plugins if config is not None else [])) if child_mcp_servers: child_plugins.append(MCPPlugin(servers=child_mcp_servers)) child_config = parent_config.model_copy( @@ -329,8 +328,8 @@ def _effective_custom_tools(parent: Harness, config: SubAgentConfig | None) -> l return list(config.tools) -def _inherited_instruction_plugins(parent: Harness) -> list[Plugin]: - """Preserve filesystem and skill plugin instructions for inherited tools.""" +def _inherited_bridge_plugins(parent: Harness) -> list[Plugin]: + """Return temporary plugins needed to preserve inherited child behavior.""" plugins: list[Plugin] = [] has_filesystem_tools = any(tool.origin is not None and tool.origin.plugin == "filesystem" for tool in parent.tools) if has_filesystem_tools: From 94d6984779651935f9066b996fe785853a724df2 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Wed, 19 Aug 2026 00:03:08 -0400 Subject: [PATCH 11/30] Plan subagents plugin migration --- .plans/40-subagents-plugin.md | 490 ++++++++++++++++++++++++++++++++++ 1 file changed, 490 insertions(+) create mode 100644 .plans/40-subagents-plugin.md diff --git a/.plans/40-subagents-plugin.md b/.plans/40-subagents-plugin.md new file mode 100644 index 0000000..876bcb7 --- /dev/null +++ b/.plans/40-subagents-plugin.md @@ -0,0 +1,490 @@ +# Subagents plugin — plan v2 + +Move delegation from the last built-in tool path to an explicit `SubagentsPlugin`. The plugin owns the `subagent` tool, child definitions, inheritance policy, child hooks, and delegation results. Core keeps only a narrow child-harness execution module that any trusted plugin can call without receiving the parent `Harness` object. + +This is a clean pre-1.0 break. Do not add aliases, fallback reads, deprecated paths, dual configuration, or migration constructors. + +## Resolved decisions + +1. **Delegation is explicit.** A harness gets delegation only from one `SubagentsPlugin`. Plain `Harness(...)` has no model-callable delegation tool. +2. **The unnamed child remains.** `SubagentsPlugin()` contributes `subagent`; omitting the model-visible `agent` argument selects the framework default child. Named children are optional. +3. **Children cannot delegate.** `SubagentsPlugin` never inherits into a child and is rejected in explicit child plugin configuration. Every child receives a disabled child-harness host that rejects before creating resources, so an inherited custom plugin cannot create a grandchild through `PluginContext`. A custom tool named `subagent` is not delegation and remains an ordinary tool. +4. **Inheritance rebinds plugins.** Replace `inherit_parent_tools` with additive `inherit_parent=True`. Safe parent plugins bind again against the child context rather than copying their parent-bound handlers. +5. **Safe plugin inheritance is explicit.** A plugin opts in through a structural `for_child() -> Plugin` method. `FilesystemPlugin`, `SkillsPlugin`, and `ParallelLlmPlugin` opt in and return themselves from a frozen constructor configuration. `MCPPlugin` and `SubagentsPlugin` do not opt in. A custom plugin without `for_child()` does not inherit. +6. **Parallel batches follow the child model.** Rebinding `ParallelLlmPlugin(model=None)` makes it borrow the child model. An explicit model object or model string keeps its configured model behavior. +7. **MCP is explicit in children.** Remove `SubAgentConfig.mcp_servers` and `inherit_mcp_servers`. A child that needs MCP includes `MCPPlugin(...)` in its own `plugins` list. Parent MCP connections never inherit implicitly. +8. **Inheritance is additive.** A named child may combine `inherit_parent=True` with explicit child `plugins` and `tools`. Inherited values come first. Duplicate plugin or tool names fail through normal atomic composition; explicit values do not silently replace inherited values. +9. **Child hooks stay with child configuration.** Named-child lifecycle hooks live in `SubAgentConfig.hooks`. Hooks for the unnamed child live in `SubagentsPlugin(default_hooks=...)`. Parent `before_subagent_run` and `after_subagent_run` hooks remain ordinary `Harness(..., hooks=...)` hooks. +10. **No special tool kind or reserved name remains.** Remove `ToolKind`, `ToolSpec.kind`, the `"subagent"` kind, and the core name reservation. A direct custom tool may use the name `subagent` when no plugin tool has that name. Normal duplicate-tool validation rejects a collision with `SubagentsPlugin`. +11. **Core does not receive subagent configuration.** Remove `HarnessConfig.builtin_tools`, `HarnessConfig.subagents`, and `Harness(subagent_hooks=...)`. Core does not import `SubAgentConfig`, `SubagentsPlugin`, or subagent tool construction helpers. +12. **Named children may have no tools.** A system-prompt-only or model-only named child is valid. Hooks alone do not expose tools, and no artificial tool-source requirement remains. +13. **Direct-tool inheritance follows run freezing.** The child sees the direct tools frozen for the active parent run. A direct tool added during that run is first available to a child in the next run. This is an intentional change from the current live-list behavior. +14. **Other execution behavior stays stable.** Child runs remain fresh, one level deep, independently budgeted, streamed and traced under the parent, and always closed. Model ownership, provider-setting projection, structured results, cancellation, errors, metadata, and before/after hooks keep their current meaning. + +## Target interface + +```python +from thinharness import ( + FilesystemPlugin, + Harness, + HarnessConfig, + Hook, + MCPPlugin, + MCPServerStdio, + SubAgentConfig, + SubagentsPlugin, +) + +research_mcp = MCPPlugin( + servers=[MCPServerStdio("python", ["research_server.py"])], +) + +harness = Harness( + HarnessConfig(model="openai:gpt-5.5"), + plugins=[ + FilesystemPlugin(tools=["read", "search"]), + SubagentsPlugin( + default_hooks=[ + Hook("run_start", prepare_default_child), + ], + agents=[ + SubAgentConfig( + name="researcher", + description="Research one focused question.", + system_prompt="Return concise findings with sources.", + model="anthropic:claude-opus-4-6", + inherit_parent=True, + plugins=[research_mcp], + tools=[custom_research_tool], + hooks=[ + Hook("run_start", prepare_researcher), + Hook("run_end", inspect_researcher_result), + ], + ), + ], + ), + ], + hooks=[ + Hook("before_subagent_run", approve_delegation), + Hook("after_subagent_run", record_delegation), + ], +) +``` + +`SubagentsPlugin` has the fixed runtime name `"subagents"` and contributes one tool named `subagent`. Its constructor is keyword-only: + +```python +SubagentsPlugin( + *, + agents: Sequence[SubAgentConfig] = (), + default_hooks: Sequence[Hook] | HookRegistry | None = None, +) +``` + +Reject unordered agent collections, duplicate names, the reserved config name `"default"`, invalid plugin values, and any child configuration that contains `SubagentsPlugin`. Perform the recursion check in `thinharness/plugins/subagents.py` by object type, not by plugin name in core. Reusing one plugin object across harnesses binds it to each harness's own child host, root, and model. + +The unnamed child intentionally keeps today's fixed parent-derived model, system prompt, limits, output behavior, and inheritance policy. `default_hooks` is its only plugin-level override. Do not add a second default-child configuration type in this slice. + +`SubAgentConfig` keeps: + +```python +SubAgentConfig( + *, + name: str, + description: str, + system_prompt: str = DEFAULT_SYSTEM_PROMPT, + inherit_parent: bool = False, + plugins: Sequence[Plugin] = (), + tools: Sequence[ToolSpec] = (), + hooks: Sequence[Hook] | HookRegistry | None = None, + model: str | None = None, + max_model_requests: int | None = None, + max_tool_calls: int | None = None, + output_type: Any | None = None, + output_mode: Literal["auto", "native", "tool", "prompted"] = "auto", + output_retries: int = 1, + tool_retries: int = 1, +) +``` + +Keep the current name and one-line description validation. Reject approval-required explicit tools because approval pauses remain unsupported inside children. Reject `SubagentsPlugin` in explicit child plugins by object type. Child hook collections must not contain `before_subagent_run` or `after_subagent_run` hooks and must not use `Hook.agents`; children cannot delegate, so these settings are invalid. Validate named and default child hooks when `SubagentsPlugin` is constructed, not when a child first runs. Remove `inherit_parent_tools`, `inherit_mcp_servers`, `mcp_servers`, and `builtin_tools`; reject these removed names with direct migration errors rather than accepting them. + +## Narrow child-harness host + +Do not put `Harness`, `HarnessConfig`, plugin lookup, mutable tool maps, tracing internals, or provider credentials into `PluginContext`. + +Add one narrow host capability to `PluginContext`: + +```python +@dataclass(frozen=True) +class PluginContext: + root: Path + model: Model + child_harnesses: ChildHarnessHost +``` + +Define the public `ChildHarnessHost` protocol and immutable request/outcome types in `thinharness/children.py`; export them from `thinharness/__init__.py` so third-party plugins can type and fake `PluginContext.child_harnesses`. Keep the parent-holding implementation private in the same neutral module. `thinharness/plugins/base.py` imports the host type only under `TYPE_CHECKING`, which avoids a core↔plugins runtime cycle. + +The request carries the plugin-owned delegation vocabulary as opaque data: agent name, agent description, trace agent name, task, `inherited`, `tool_mode`, child system prompt, model override, explicit plugins/tools/hooks, limits, output settings, and retry settings. Core does not construct `"subagent."` names or interpret the tool-mode values. The outcome carries the child result, effective tool names, structured-output serialization data, and any failure needed for the plugin's `ToolResult`. + +`tool_mode` has three stable values: + +- `"inherited"` for the default child or a named child with `inherit_parent=True` and no explicit plugins/tools; +- `"inherited+explicit"` for `inherit_parent=True` plus explicit plugins or tools; +- `"explicit"` for `inherit_parent=False`, including a model-only child. + +The host owns parent-dependent mechanics: + +- register a tool returned by its delegation interface in core-owned composition provenance; this internal role is not a `ToolSpec` field and survives generic plugin normalization; +- build one child harness from parent defaults plus explicit child overrides; +- obtain the active parent run's frozen direct-tool snapshot from core-owned tool runtime state, never from the live `Harness.tools` list; +- rebind child-inheritable plugins in parent plugin order; +- append explicit child plugins and direct tools in caller order; +- infer and own an override model, or borrow the parent model; +- project same-provider credentials and the existing request settings exactly as today; +- derive child tracing options and child hooks; +- connect the child, forward nested stream events, and preserve parent run and tool-call correlation; +- fire existing parent `before_subagent_run` and `after_subagent_run` hooks with the actual parent harness in their contexts; +- close the child after success, failure, or cancellation without hiding the original run error. + +A top-level host rejects a request made outside an active parent tool call before model inference, plugin connection, filesystem access, or child construction. Every child `PluginContext` receives a disabled host that always rejects before creating resources. This blocks grandchild creation even when an inherited custom plugin captures and calls its child context host. + +`SubagentsPlugin.bind()` captures only `context.child_harnesses` in its static tool handler and registers that returned tool through the host's delegation interface. Binding stays synchronous and I/O-free. Child model inference, plugin connection, filesystem access, and child execution happen only when the tool runs. + +Add contract coverage for all in-repo direct `PluginContext(...)` constructions in `tests/unit/test_plugins.py`, `tests/unit/test_parallel_llm.py`, and `tests/unit/test_skills.py`. Also prove that one plugin object bound to two parent harnesses receives independent child hosts and cannot cross parent roots, models, tools, hooks, metadata, or streams. + +## Child plugin inheritance + +Add and publicly export a runtime-checkable structural protocol without changing the base `Plugin.bind()` interface: + +```python +@runtime_checkable +class ChildInheritablePlugin(Protocol): + def for_child(self) -> Plugin: + """Return the plugin object to bind to one inherited child.""" +``` + +Rules: + +- `for_child()` is synchronous and performs no file, provider, or network I/O. +- `FilesystemPlugin.for_child()`, `SkillsPlugin.for_child()`, and `ParallelLlmPlugin.for_child()` return `self`. +- Make these three plugins use one private immutable constructor snapshot for every bind. Copy mutable inputs on construction, expose no mutable configuration container used by binding, and reject configuration assignment/deletion after construction. Mutation of an original input or a value obtained from a public property cannot change later parent or child bindings. Make `FilesystemPlugin.name` runtime-fixed like the other built-in plugin names so child rebinding cannot change identity. +- Returning `self` therefore reuses the same frozen configuration while normal child binding creates independent `ToolSpec` values and instructions. +- `SkillsPlugin` shares its constructor-time registry and frozen catalog while live skill files remain live. +- `ParallelLlmPlugin(model=None)` sees the child `PluginContext.model`; explicit plugin models keep their existing ownership rules. +- `MCPPlugin` has no `for_child()`. A child may still list the same MCP plugin object explicitly; normal binding and reference-counted server lifecycle apply. +- `SubagentsPlugin` has no `for_child()` and is also rejected if listed explicitly in a child's plugins. +- A custom plugin opts in only by implementing `for_child()`. It may return itself or a fresh configured plugin. Validate the returned object and fixed name before child construction. The custom plugin owns its snapshot and thread-safety contract. +- A child-inheritable plugin must not contribute approval-required tools. Reject known static violations while the parent harness is constructed; reject connected violations atomically when the child connects. Do not silently filter plugin tools or leave their instructions/hooks behind. +- Preserve parent plugin order among inherited plugins. Append explicit child plugins after inherited plugins. +- Duplicate names across inherited and explicit child plugins fail. Do not add implicit replacement, exclusion lists, clone fallbacks, or plugin-name special cases. `inherit_parent=True` cannot mean “inherit some and override one”; callers use `inherit_parent=False` and list the wanted plugins/tools explicitly. +- `SubagentsPlugin.bind()` registers each static child recipe with the child host. After all parent plugins bind, the host validates named-child plugin/tool collisions and known approval violations before `Harness(...)` returns. Revalidate direct-tool collisions after `Harness.add_tool()` and connected composition where the relevant names become known. Only genuinely dynamic child connection collisions may surface when the child connects. + +## Direct tool inheritance and run freezing + +`ToolOrigin` is caller-visible attribution metadata, not authoritative ownership. A direct caller may forge any origin value. Track tool ownership in a separate core-owned composition record that never comes from `ToolSpec.origin`. + +Track direct tools and plugin tools as explicit internal composition sources: + +- tools passed through `Harness(tools=...)` are direct; +- `Harness.add_tool()` adds a direct tool; +- tools contributed by plugins are not direct; +- freeze the eligible direct-tool list and authoritative plugin ownership with the run toolset; +- place that frozen composition snapshot in the existing core-owned tool runtime context before any tool handler starts, so the child host reads the active snapshot without inspecting live harness state; +- a direct tool added during a parent run does not enter a child delegated during that run, but it is available to children in the next run; +- a valid child-host request outside an active tool runtime fails clearly instead of falling back to current direct tools; +- inherited direct tools preserve parent order and object identity; +- exclude approval-required direct tools; +- append explicit child tools after inherited direct tools; +- reject duplicate tool names through normal child composition; +- do not filter by the string name `subagent`; the actual delegation tool comes from a non-inheritable plugin and is never in the direct source. + +The default child always uses `inherit_parent=True`. A named child uses its `inherit_parent` value. A named child with `inherit_parent=False`, no plugins, and no tools receives no model-callable tools and remains valid. + +## Plugin contribution and hook-filter names + +Core currently imports subagent configuration only to validate `Hook.agents` filters. Replace that feature-specific path with static plugin binding metadata. + +Extend `PluginBinding` with an immutable tuple of valid agent names, empty by default. `SubagentsPlugin` contributes `("default", *named_agent_names)` when it binds. + +Core requirements: + +- combine static agent names from every plugin binding before validating caller and plugin hooks; +- reject blank names and duplicate names within or across bindings; +- validate `Hook.agents` at harness construction, after the existing `Harness.add_tool()` revalidation point, and after connected contributions; +- do not add a public `Harness.add_hook()` method in this slice; +- connected plugins cannot change agent names; +- a harness without `SubagentsPlugin` rejects any agent-filtered hook because no agent names exist; +- dynamic agent catalogs are out of scope. + +Child lifecycle hooks from `SubAgentConfig.hooks` or `default_hooks` belong to the child harness. A supplied `HookRegistry` keeps its own `strict_hooks` value; a plain hook sequence uses the parent harness `strict_hooks` setting, matching current child behavior. Copy caller-owned hook registries before child composition. Parent subagent hooks remain in the parent registry and continue to support `agents=[...]` filters. + +## SubagentsPlugin behavior + +Preserve these model-visible and runtime behaviors: + +- tool arguments remain `task: str` and optional non-empty `agent: str`; +- omitting `agent` selects `"default"`; +- an unknown name returns a failed result with the requested name, sorted available names, and `UnknownSubAgent`; +- the description lists named agents in caller order and tells the model how to select the default; +- each child starts a fresh provider session with its own system prompt and no parent transcript; +- parent and child limits remain independent; +- the parent counts one `subagent` tool call; child requests and tokens stay in child usage and result metadata; +- override models inherit current timeout, retry, backoff, temperature, token, effort, and extra-body settings; only same-provider overrides receive parent API key and base URL; +- parent-model children borrow the model and never close it; override models are child-owned and close once; +- child structured output becomes the tool content and reports `structured_output=True`; +- successful metadata keeps agent, inheritance mode, effective tools, model requests, and structured-output status; +- cancellation and strict-hook failures do not hang sibling tool calls; +- child close runs after success, provider failure, hook failure, stream cancellation, or parent cancellation; +- child events remain flattened only when `StreamOptions.include_subagents=True`; +- conversation id and parent call id propagation remain unchanged; +- before-hook metadata mutation does not alter child metadata; +- plugin state, child configuration, child host, and models never enter resume or approval state. + +The generic plugin normalizer assigns `ToolOrigin(plugin="subagents", source="subagent")`. + +## Remove core built-ins and special tool state + +Remove from `HarnessConfig`: + +- `builtin_tools`; +- `subagents`. + +Add explicit removed-field guards in `thinharness/_migration.py` that point both names to plugin composition and name `SubagentsPlugin` for delegation. Delete `_select_builtin_tools()` and every built-in migration branch from core. A plain harness now has no implicit or selected built-in tool path. The changelog must state that old filesystem, skills, and parallel values inside `builtin_tools` now move to their respective plugins rather than treating the generic removed-field message as feature-specific guidance. + +Remove from `Harness`: + +- `subagent_hooks=`; +- `self.subagent_hooks`; +- direct imports of subagent configuration or tool builders; +- subagent-specific hook-filter lookup; +- the `subagent` reserved-name check. + +Keep an internal child-harness marker only where core needs it to enforce no approval-required tools inside children, install the disabled child host, and suppress top-level-only local tracing. Rename the marker and the approval error to say `child harnesses` rather than `subagents`. + +Remove from tool contracts: + +- `ToolKind`; +- `ToolSpec.kind`; +- runtime kind validation. + +All `ToolSpec` values then use one ordinary contract. Update direct construction, tests, exports, documentation, and changelog together. + +Delete the public low-level composition helpers `create_subagent_tool()` and `build_child_harness()` and remove their top-level exports. Do not leave wrappers or aliases. Move retained public types (`SubAgentConfig`, `SubAgentArgs`, and `DEFAULT_SUBAGENT_NAME`) behind the plugin module. Export them and `SubagentsPlugin` from both `thinharness/plugins/__init__.py` and `thinharness/__init__.py`. Export `ChildInheritablePlugin`, `ChildHarnessHost`, and the child request/outcome types through the same public plugin-contract surface. + +Delete `thinharness/subagents.py` when its retained implementation has moved behind `thinharness/plugins/subagents.py` and the narrow child host module. Do not keep a pass-through module. + +## Tracing and transcript classification + +Keep current public event names and result trace attributes for real delegation. Detect delegation from the core-owned frozen composition source, not from the tool name or caller-forgeable `ToolOrigin`: + +- when a tool span starts, look up the frozen authoritative composition role for that exact tool and set `subagent.delegation=true` before any hook or handler can cancel or fail; +- only a tool registered through the child host's delegation interface receives that marker; ordinary plugin origin metadata cannot create the role; +- `ToolOrigin` remains attribution metadata and never decides delegation control flow or transcript kind; +- a custom direct tool named `subagent`, including one with `ToolOrigin(plugin="subagents")`, receives ordinary tool tracing; +- keep `subagent.name`, `subagent.tool_mode`, and `subagent.tools` when a real delegation outcome supplies them; +- a cancelled or early-failing real delegation still has the marker even when result metadata is absent; +- keep child `invoke_agent subagent.` spans and agent descriptions; +- update `scripts/build_transcripts.py` to classify a span as a subagent only when `subagent.delegation` is true; +- do not add a legacy tool-name fallback for stored traces; update the checked-in web-research trace's known real delegation spans with the truthful marker, or regenerate that trace with the new implementation, before rebuilding transcript HTML; +- retain rendered transcript output for the checked-in real subagent trace; +- add regressions for a normal delegation, a before-tool-cancelled delegation, an early failure, and a forged-origin direct tool. + +`subagent.delegation` is the only new trace attribute. Do not rename existing public streaming events, hook events, trace attributes, stop reasons, or result metadata. + +## Behavior contract changes before implementation + +After plan review and before code changes, update only affected sections of `docs/behavior.md`: + +- extend PLUGIN-3 with the narrow child-harness host while keeping binding synchronous and I/O-free; +- extend the plugin binding contract with static agent names, authoritative core composition provenance, and explicit `for_child()` inheritance; +- update PLUGIN-8 for the `SubagentsPlugin` tool and child plugin ordering; +- update FILESYSTEM-PLUGIN requirements for a fixed plugin name and frozen constructor configuration; +- replace SKILLS-PLUGIN-7 with generic safe-plugin rebinding and shared registry behavior, and state that inheritable plugin configuration is frozen; +- update PARALLEL-LLM-PLUGIN-2 so an inherited borrowed-model plugin uses the child model and frozen plugin configuration; +- update PARALLEL-LLM-PLUGIN-7 to remove the obsolete ordinary `"user"` tool-kind statement; +- update PROVIDER-RETRY-6 for child model override and rebound parallel model settings; +- replace MCP-7's temporary bridge with explicit child `MCPPlugin` composition; +- update TOOLSET-FREEZE-1 through TOOLSET-FREEZE-3 so the same run snapshot carries direct-tool ownership used by delegated children; +- add a Subagents Plugin section covering explicit composition, the default child, named configs, additive inheritance, safe plugin rebinding, direct-tool freezing, child hooks, disabled child hosts, no recursion, model ownership, limits, streaming, tracing, results, cancellation, and state exclusion; +- remove every statement that calls subagent a built-in tool or documents `builtin_tools`. + +Do not change unrelated behavior sections. + +## Architecture guards + +Extend `tests/unit/test_architecture.py` with AST import and named-identifier checks rather than a blanket `"subagents" not in source` substring assertion. Direct `thinharness/core.py` source rejects: + +- imports of `subagents` or `plugins.subagents`; +- `SubAgentConfig`, `SubagentsPlugin`, `SubAgentArgs`, and `DEFAULT_SUBAGENT_NAME`; +- `create_subagent_tool`, `build_child_harness`, and `_select_builtin_tools`; +- `builtin_tools`, `subagent_hooks`, and direct subagent name reservation; +- filesystem, skills, parallel-LLM, or MCP inheritance logic. + +Add structural checks that: + +- `thinharness/subagents.py` no longer exists; +- core constructs only the narrow child host and does not expose the parent harness through `PluginContext`; +- `SubagentsPlugin` owns the delegation tool and configuration; +- only plugins with `for_child()` enter automatic child plugin composition; +- no name-based delegation detection remains in core or tool execution. + +## Tests + +Retain and migrate the existing subagent behavior suite. Add focused coverage for: + +### Plugin construction and composition + +- `SubagentsPlugin()` contributes one static `subagent` tool and the default agent name; +- no plugin means no delegation tool; +- fixed plugin name, origin, description, schema, order, and normal collisions; +- ordered agent validation, duplicate names, reserved `default`, invalid descriptions, and plugin reuse across harnesses with different roots and models; +- duplicate agent names within or across binding metadata fail; +- a named child with no tools is valid; +- a child cannot list `SubagentsPlugin` explicitly, and the error comes from plugin-module validation rather than core; +- plugin bind is I/O-free and does not infer a child model; +- all direct third-party-style `PluginContext(...)` constructions receive a fake child host cleanly; +- a top-level child host rejects outside an active tool runtime, and a child disabled host rejects an inherited custom plugin's grandchild attempt before resources open; +- plugin state and host references do not serialize into resume or approval state. + +### Inheritance + +- default and named `inherit_parent=True` children rebind filesystem, skills, and parallel plugins in parent order; +- skills reuse the exact registry and summary once; +- `ParallelLlmPlugin(model=None)` uses a named child's cross-provider override model without inheriting the parent API key or base URL; +- parallel plugins with explicit model objects or strings keep those models and ownership rules; +- MCP does not inherit, while an explicit child MCP plugin connects and closes through its own binding; +- a custom plugin with `for_child()` inherits; one without it does not; +- an invalid `for_child()` return fails clearly; +- mutation attempts and mutable constructor inputs cannot change later filesystem, skills, or parallel child bindings; +- inherited and explicit plugins/tools are additive and preserve their stated order; +- a known inherited/explicit filesystem or direct-tool collision fails while the parent is constructed, and later `add_tool()` collisions revalidate before a run; +- dynamic child connection collisions fail atomically; +- `inherit_parent=True` plus explicit sources reports `tool_mode="inherited+explicit"` in hooks, results, and traces; +- plugin tools are not copied again as direct tools; +- direct tools with caller-supplied or forged origins still inherit and trace as direct tools; +- approval-required direct and explicit tools never enter children; +- an inheritable plugin with a known approval-required tool fails parent construction rather than being silently filtered; +- a direct tool added during a run appears only in children of the next run. + +### Hooks and execution + +- named hooks come from `SubAgentConfig.hooks`; default hooks come from `default_hooks`; +- child hooks with subagent events or any `agents=` filter fail at plugin construction; +- caller-owned `HookRegistry` values are copied; registry strictness is preserved, while plain sequences use parent strictness; +- parent before/after hooks, cancellation, metadata isolation, and agent filters remain stable; +- a default child works with `default_hooks` and parent hooks filtered to `agents=["default"]`; +- agent-filtered parent hooks fail without the plugin and for unknown names; +- default, unknown, blank, model-only, structured-output, provider-failure, and child-close paths; +- shared parent model versus owned override model lifecycle; +- two concurrent delegations to one override-model config create two owned child providers and close each once; +- same-provider credential forwarding and cross-provider credential isolation; +- fresh budgets, retries, notices, metadata, and usage accounting; +- concurrent delegation and strict sibling abort do not hang; +- nested streaming correlation and `include_subagents` behavior; +- tracing attributes and child spans remain stable; +- `subagent.delegation=true` exists from span start for normal, cancelled, and early-failing real delegation; +- a direct tool named `subagent` with a forged `ToolOrigin(plugin="subagents")` remains an ordinary trace and transcript event. + +### Removal + +- each removed `HarnessConfig` field raises its own `SubagentsPlugin` migration error; +- removed `SubAgentConfig` fields fail by name; +- `Harness(subagent_hooks=...)`, `create_subagent_tool`, and `build_child_harness` are gone; +- `ToolSpec` has no `kind` field and no `ToolKind` remains; +- a direct custom tool named `subagent`, even with forged subagents origin metadata, works without the plugin and has ordinary tracing; +- the same custom tool collides normally when the plugin is present; +- core has no feature-specific subagent composition. + +## Caller, documentation, and site migration + +Update every checked-in caller. This includes: + +- `README.md` feature text and examples; +- `docs/docs.md` configuration, hooks, subagents, MCP, tracing, and plugin examples, including the tool-surface sentence, rejected Bash example, and SkillsPlugin child wording; +- `docs/site/explainer/index.html` architecture and composition text; +- `docs/site/about/index.html` through `scripts/build_site.py`; +- the checked-in web-research trace marker and `docs/site/examples/index.html` through transcript regeneration; +- `CHANGELOG.md` with all breaking removals and inheritance changes; +- `examples/web_research_report/agent.py`, including required-tool and trace-name assertions; +- `examples/mcp_plugin.py`; +- `tests/e2e/langfuse_tracing_journey.py`; +- `tests/unit/test_harness.py` built-in migration tests and the removed add-after-construction delegation capability; +- every test and journey that passes `builtin_tools=[]` or `builtin_tools=["subagent"]`; +- every import of `create_subagent_tool()` or `build_child_harness()`; +- every direct `PluginContext(...)` constructor affected by the required host field. + +Delete `builtin_tools=[]` rather than replacing it: no built-in tools remain. Replace enabled delegation with `plugins=[..., SubagentsPlugin(...)]` in caller order. + +Add `tests/e2e/subagents_journey.py` with real provider calls. It must prove that the plugin delegates to the default child and to a named override-model child, returns child output to the parent, prevents child delegation, and emits the expected metadata without relying only on mocks. + +## Implementation steps + +1. Update the affected behavior contracts. +2. Add the narrow child-harness host and its request/outcome types without moving feature configuration into core. +3. Extend `PluginContext` and `PluginBinding` with the child host and static agent names. +4. Add `ChildInheritablePlugin` and implement `for_child()` on filesystem, skills, and parallel plugins. +5. Add `SubagentsPlugin`, move the retained public configuration types behind it, and implement static tool binding. +6. Move child construction, execution, hooks, streaming, tracing, and cleanup behind the narrow host. +7. Implement explicit direct-tool source tracking and run-frozen child inheritance. +8. Implement additive inherited and explicit child composition with normal collision errors. +9. Remove MCP and skills/filesystem temporary bridges and the parent-bound parallel handler path. +10. Migrate child hooks to `SubAgentConfig.hooks` and `default_hooks`; replace core agent-filter lookup with binding metadata. +11. Remove built-in selection, old config fields, constructor arguments, public helpers, `ToolKind`, `ToolSpec.kind`, and reserved-name logic. +12. Change tracing and transcript classification to use core-owned composition provenance and the start-of-span delegation marker. +13. Migrate unit tests, examples, documentation, site pages, and all end-to-end journeys. +14. Delete the obsolete `thinharness/subagents.py` module and add architecture guards. +15. Regenerate checked-in generated pages and run all validation. + +## Validation + +Run: + +```bash +uv run pytest tests/unit/test_subagents.py tests/unit/test_plugins.py tests/unit/test_harness.py tests/unit/test_hooks.py tests/unit/test_architecture.py +uv run pytest tests/unit/test_streaming.py tests/unit/test_tracing.py tests/unit/test_approvals.py tests/unit/test_structured_output.py tests/unit/test_tool_retry.py tests/unit/test_mcp.py tests/unit/test_parallel_llm.py +uv run pytest tests/unit/test_web_research_report_example.py +uv run pytest +uv run ruff check . +uv run pyright +uv run scripts/build_site.py +uv run scripts/build_site.py --check +uv run scripts/build_transcripts.py +for journey in tests/e2e/*_journey.py; do uv run --env-file .env python "$journey"; done +git diff --check +``` + +Keep the migrated suite at `tests/unit/test_subagents.py`; do not rename it. Report every journey separately. Credential- or service-based skips are not passes. + +## Success criteria + +- ThinHarness has no built-in tool selector or implicit feature tool path. +- Delegation exists only through explicit `SubagentsPlugin` composition. +- The plugin owns child definitions, the default child, tool description, inheritance choices, child hooks, and result shaping. +- Core does not import subagent configuration or plugin implementation and exposes only a narrow child-harness host. +- Children receive a disabled child host and cannot create grandchildren through built-in or custom inherited plugins, but a custom ordinary tool may use the same model-facing name. +- Safe parent plugins rebind against the child; unknown custom plugins do not inherit without `for_child()`. +- Inherited parallel LLM batches borrow the child model when configured with `model=None`. +- Parent MCP never inherits implicitly; explicit child MCP keeps independent binding and cleanup. +- Additive inherited and explicit composition is ordered and collision-safe. +- Child inheritance uses the active parent run's frozen direct tools. +- Child hooks are local to each named config or the plugin default. +- Streaming, tracing, hooks, usage, limits, model ownership, structured output, errors, and cancellation retain their current behavior except for the approved child-model parallel rebinding and run-frozen direct-tool inheritance changes. +- Core, docs, examples, tests, and site pages contain no obsolete subagent built-in configuration. +- Focused suites, the full suite, Ruff, Pyright, generated-page checks, and every live end-to-end journey pass. + +## Out of scope + +- Nested or recursive delegation. +- Forking the parent conversation into a child. +- Background children, detached jobs, scheduling, or persistent child sessions. +- Child approval pauses. +- Dynamic agent catalogs or plugin discovery. +- Automatic inheritance of MCP or unknown custom plugins. +- Selective inherited-plugin exclusion, implicit override by name, or dependency resolution. +- Changing child result shape, stream event names, trace attribute names, hook event names, provider retry policy, or structured-output semantics. +- Compatibility aliases, deprecation periods, config migrations, or fallback reads. + +## Review record + +One Codex, Claude, and GLM panel round reviewed plan v1. Plan v2 applies the verified findings on disabled child hosts, authoritative tool provenance, start-of-span delegation marking, immutable inheritable plugin configuration, approval-tool policy, eager collision validation, child-hook validation, child request fields, tool-mode values, public host types, direct `PluginContext` callers, run-frozen tool plumbing, behavior-contract coverage, AST-scoped architecture guards, transcript regressions, and caller migration. No second plan-review round is scheduled. From c05a7e4feb4645e68179aae8c4995b53715015b0 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Wed, 19 Aug 2026 00:48:42 -0400 Subject: [PATCH 12/30] Add explicit subagents plugin --- CHANGELOG.md | 12 +- README.md | 29 +- docs/behavior.md | 44 +- docs/docs.md | 56 +- docs/site/about/index.html | 8 +- docs/site/explainer/index.html | 40 +- examples/mcp_plugin.py | 2 +- examples/web_research_report/agent.py | 37 +- scripts/build_transcripts.py | 4 +- tests/e2e/README.md | 4 +- tests/e2e/anthropic_modernization_journey.py | 6 +- tests/e2e/control_plane_journey.py | 1 - tests/e2e/langfuse_tracing_journey.py | 13 +- tests/e2e/mcp_journey.py | 2 +- tests/e2e/prompt_caching_journey.py | 1 - tests/e2e/structured_output_journey.py | 1 - tests/e2e/subagents_journey.py | 149 +++ tests/e2e/workspace_tools_journey.py | 1 - tests/unit/fakes.py | 16 + tests/unit/test_approvals.py | 91 +- tests/unit/test_architecture.py | 70 ++ tests/unit/test_bash_tool.py | 8 +- tests/unit/test_harness.py | 90 +- tests/unit/test_hooks.py | 52 +- tests/unit/test_mcp.py | 291 ++--- tests/unit/test_parallel_llm.py | 15 +- tests/unit/test_parallel_tools.py | 18 +- tests/unit/test_plugins.py | 30 +- tests/unit/test_providers.py | 16 +- tests/unit/test_reasoning_fidelity.py | 6 +- tests/unit/test_resume.py | 82 +- tests/unit/test_skills.py | 4 +- tests/unit/test_streaming.py | 50 +- tests/unit/test_structured_output.py | 167 +-- tests/unit/test_subagents.py | 1091 +++++++++--------- tests/unit/test_tool_retry.py | 59 +- tests/unit/test_tracing.py | 92 +- tests/unit/test_turns.py | 6 +- thinharness/__init__.py | 16 +- thinharness/_migration.py | 2 + thinharness/children.py | 459 ++++++++ thinharness/core.py | 153 ++- thinharness/hooks.py | 6 +- thinharness/plugins/__init__.py | 12 +- thinharness/plugins/base.py | 13 + thinharness/plugins/filesystem.py | 109 +- thinharness/plugins/parallel_llm.py | 100 +- thinharness/plugins/skills.py | 57 +- thinharness/plugins/subagents.py | 340 ++++++ thinharness/providers.py | 6 +- thinharness/runtime.py | 4 +- thinharness/subagents.py | 415 ------- thinharness/tool_execution.py | 29 +- thinharness/tools/base.py | 6 +- 54 files changed, 2632 insertions(+), 1759 deletions(-) create mode 100644 tests/e2e/subagents_journey.py create mode 100644 thinharness/children.py create mode 100644 thinharness/plugins/subagents.py delete mode 100644 thinharness/subagents.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b0853b..80de3db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,16 +7,22 @@ - Added `MCPPlugin` for lazy MCP server connection, binding-local server identity, atomic tool discovery, and generic tool origin attribution. - Added `SkillsPlugin` for constructor-time skill discovery, explicit ordered skill-tool selection, static summaries, and shared inherited-child catalogs. - Added `ParallelLlmPlugin` for explicit text-only batch composition with borrowed harness or caller models and plugin-owned string-model provider settings. -- Added the configured harness model to the I/O-free `PluginContext` alongside the canonical root. +- Added explicit `SubagentsPlugin` composition for the default child, named child recipes, child-local hooks, additive inheritance, and delegation result shaping. +- Added a narrow `ChildHarnessHost` to `PluginContext`, public child request/outcome contracts, and structural `ChildInheritablePlugin.for_child()` rebinding. +- Added run-frozen authoritative tool composition provenance for direct-tool inheritance and `subagent.delegation` tracing. - **Breaking:** Removed `HarnessConfig.mcp_servers`, `McpToolInfo`, and the MCP `ToolKind`; configure one `MCPPlugin` with all harness servers. - **Breaking:** Generic plugin validation now reports MCP tool collisions as duplicate tool names. Use MCP `tool_prefix`, `include_tools`, or `exclude_tools` to prevent collisions. - **Breaking:** Removed `MCPServer.resolve_id()` and post-bind mutation of `server.id`. The public `server.id` remains the base ID; binding-local IDs, including duplicate suffixes, appear in tool origin and result metadata. - **Breaking:** `Harness` no longer enables filesystem tools by default. Pass `plugins=[FilesystemPlugin(...)]`; independent custom tools still use `tools=`. - **Breaking:** Removed filesystem settings from `HarnessConfig` and removed the `builtin_tools()` helper. - **Breaking:** Removed `HarnessConfig.skills_dir`, `selected_skills`, `read_paths`, `write_paths`, `builtin_parallel_llm_model`, `builtin_parallel_llm_temperature`, and `parallel_llm_max_prompts`; use `SkillsPlugin` and `ParallelLlmPlugin`. -- **Breaking:** Removed the `Harness(skills=...)` composition path, `SubAgentConfig.builtin_tools`, and `create_parallel_llm_tool`; named children now use explicit plugins or tools. -- **Breaking:** Removed the `parallel_llm` `ToolKind`; direct and plugin-created `ParallelLlmTool` specifications now use kind `"user"`. +- **Breaking:** Removed the `Harness(skills=...)` composition path and `create_parallel_llm_tool`; named children now use explicit plugins or tools. +- **Breaking:** Removed `HarnessConfig.builtin_tools`, `HarnessConfig.subagents`, and `Harness(subagent_hooks=...)`. ThinHarness has no implicit tool selector; use `FilesystemPlugin`, `SkillsPlugin`, `ParallelLlmPlugin`, and `SubagentsPlugin` explicitly. +- **Breaking:** Removed `SubAgentConfig.builtin_tools`, `inherit_parent_tools`, `inherit_mcp_servers`, and `mcp_servers`. Use additive `inherit_parent=True`, explicit child plugins, and an explicit child `MCPPlugin`. +- **Breaking:** Removed `create_subagent_tool`, `build_child_harness`, the `thinharness.subagents` module, `ToolKind`, and `ToolSpec.kind`. A custom direct tool can now use the name `subagent` when `SubagentsPlugin` is absent. +- **Breaking:** Parent plugins inherit only when they implement `for_child()`. Filesystem, skills, and parallel LLM plugins opt in with frozen constructor configuration; MCP and subagents do not. An inherited `ParallelLlmPlugin(model=None)` now borrows the child model. - Changed connection setup to complete before `run_start` hooks. A connection failure does not fire run lifecycle hooks. +- Fixed Anthropic metadata projection to send only the Messages API's supported `user_id`, while keeping child correlation metadata inside the harness. ## 0.6.0 - 2026-08-07 diff --git a/README.md b/README.md index be1b383..073a2bf 100644 --- a/README.md +++ b/README.md @@ -215,11 +215,11 @@ ThinHarness has opinions. They are the reason it stays small. **Purpose-built agents, not universal agents.** ThinHarness is for bounded agent loops, not open-ended interactive assistants like Claude Code or OpenClaw. For business use cases, focused agent loops orchestrated by deterministic code are usually a better fit than sprawling multi-agent systems with broad authority. -**No bash by default.** Purpose-built business agents usually don't need a shell. Bash is a broad security and reliability surface: it gives the model open-ended authority instead of typed, bounded actions. ThinHarness keeps bash out of the default and built-in tool sets, but exposes an opt-in `BashTool` for exploratory runs before the workflow is hardened with typed tools. +**No bash by default.** Purpose-built business agents usually don't need a shell. Bash is a broad security and reliability surface: it gives the model open-ended authority instead of typed, bounded actions. ThinHarness has no implicit tools and exposes Bash only through an opt-in `BashTool` for exploratory runs before the workflow is hardened with typed tools. **Search is a top priority.** The `search` tool exposes ripgrep as compact grouped path/line results, tuned for document and business-workflow agents rather than code navigation. There's also a `jsonl_search` variant, because JSONL is the right shape when you're replacing RAG with agent-driven search over structured data: ripgrep row prefiltering, jq-style field projection, `where` filters, range filters, and snippets from large multiline fields. -**Parallel LLM calls, built in.** Fan out from inside the harness when a workflow needs efficient parallel processing or majority vote for reliability. Add `ParallelLlmPlugin()` for a plain-text batch tool that borrows the harness model, or give the plugin a model string and its own provider settings. For validated structured output per call, instantiate `ParallelLlmTool` with `output_type` (a Pydantic model). Each call is stateless, and large batches can write JSON to `output_file`. +**Parallel LLM calls, explicitly composed.** Fan out from inside the harness when a workflow needs efficient parallel processing or majority vote for reliability. Add `ParallelLlmPlugin()` for a plain-text batch tool that borrows the harness model, or give the plugin a model string and its own provider settings. For validated structured output per call, instantiate `ParallelLlmTool` with `output_type` (a Pydantic model). Each call is stateless, and large batches can write JSON to `output_file`. **No token streaming.** Streaming is for workflow progress, not live chatbot text. ThinHarness emits run, model-turn, tool, retry, limit, and subagent events, but it does not stream provider token deltas. Token streaming would add provider-specific plumbing, event merging, cancellation edge cases, and more surface area to keep stable. For workflow-style agents, step-level updates are usually the useful signal. @@ -269,6 +269,25 @@ harness = Harness( MCP tools connect and discover one tool snapshot lazily on `Harness.connect()` or the first run. Install support with `uv add 'thinharness[mcp]'`. +Delegation is also an explicit plugin: + +```python +from thinharness import SubAgentConfig, SubagentsPlugin + +harness = Harness( + HarnessConfig(root="."), + plugins=[SubagentsPlugin(agents=[ + SubAgentConfig( + name="reviewer", + description="Reviews one draft.", + system_prompt="Return concise issues.", + ) + ])], +) +``` + +Omit `agent` in a `subagent` call to use the default child. Named children can add tools and plugins or set `inherit_parent=True` to rebind safe parent plugins and inherit the active run's frozen direct tools. + Skills and plain-text parallel batches are explicit plugins too: ```python @@ -307,12 +326,12 @@ Streaming emits coarse run, model, tool, retry, limit, and subagent events, then - **Filesystem plugin:** explicit `FilesystemPlugin` composition for `read`, `write`, batched exact-replacement `edit`, `search`, `list`, and `glob` with root-scoped path policies. - **JSONL search:** opt-in `jsonl_search` for structured line-delimited data, with ripgrep prefiltering, field projection, equality/contains/regex/range `where` filters, and field-level snippets from large multiline string values. -- **Bash prototype tool:** opt-in `BashTool` for exploratory shell commands. It is lightweight, custom-registration only, and is not included in the default or built-in tool set. +- **Bash prototype tool:** opt-in `BashTool` for exploratory shell commands. It is lightweight and available only through direct custom registration. - **Provider adapters:** built-in OpenAI, Anthropic, and OpenRouter adapters, plus public model/session protocols for implementing another provider. - **Custom typed tools:** define sync or async `ToolSpec` handlers with Pydantic argument models, normalized `ToolResult` envelopes, sequential/approval flags, and per-tool retry settings. - **Structured output:** Pydantic-validated results with native, tool, prompted, and text modes. - **Hooks:** lifecycle and tool-call interception for prompt submission, tool calls, subagents, limits, and run boundaries. -- **Subagents:** opt-in delegation through a built-in `subagent` tool and explicit `SubAgentConfig`. +- **Subagents:** explicit `SubagentsPlugin` composition with a default child, ordered named `SubAgentConfig` recipes, additive safe-plugin inheritance, local child hooks, and no recursive delegation. - **Parallel LLM:** explicit `ParallelLlmPlugin` fan-out for batches of independent one-shot prompts, plus `ParallelLlmTool(...).spec()` for renameable or structured tools with explicit model, path, prompt, and provider request settings. - **Skills:** explicit `SkillsPlugin` composition with an ordered `skill_read` and/or `skill_run` selection, plus Python, shell, JavaScript, and Go script runners. - **Resume:** clean new-turn continuation through self-contained transcript state that can replay across built-in providers and models, preserving native reasoning on same-provider resume and degrading it to text across providers. @@ -339,7 +358,7 @@ It isn't meant to be a state-of-the-art research agent; it's a worked example sh I ran ThinHarness on a retrieval-heavy 127 question subset of a benchmark for long-term agent memory, and did a local reproduction of the benchmark's optimized harness on the same subset. - **Performance:** Matched-or-better accuracy (74.0% vs 72.4% on the 127 dynamic questions) with ~46% less token usage (62M vs. 116M). See [my fork](https://github.com/ryanbbrown/LongMemEval-V2) for more details. -- **Simpler Setup:** ThinHarness only had its built-in filesystem tools (with `jsonl_search` doing the heavy lifting), while the benchmark harness was a full Codex instance with shell and a custom Python tool designed for the task. +- **Simpler Setup:** ThinHarness only used its filesystem tools (with `jsonl_search` doing the heavy lifting), while the benchmark harness was a full Codex instance with shell and a custom Python tool designed for the task. ### 3. Personal Opinions Agent diff --git a/docs/behavior.md b/docs/behavior.md index ec058e3..e2ff8a9 100644 --- a/docs/behavior.md +++ b/docs/behavior.md @@ -90,14 +90,36 @@ Callers compose optional harness behavior explicitly while independent custom to - PLUGIN-1: `Harness` accepts plugins in caller order through `plugins=`; no plugin is loaded through entry points, directories, manifests, or implicit defaults. - PLUGIN-2: Plugin names are non-empty and unique within one harness. A duplicate name fails before either plugin binds. -- PLUGIN-3: `PluginContext` contains the canonical harness root and configured model. Plugin binding is synchronous and performs no file or network I/O. Static tools, instructions, and hooks are validated and visible immediately after harness construction. +- PLUGIN-3: `PluginContext` contains the canonical harness root, configured model, and a narrow `ChildHarnessHost`; it does not expose the parent harness. Plugin binding is synchronous and performs no file, provider, or network I/O. Static tools, instructions, hooks, and agent names are validated and visible immediately after harness construction. - PLUGIN-4: `Harness.connect()` or the first run opens connected plugin bindings once in caller order. Concurrent connection calls share that attempt, and connection completes before `run_start` hooks fire. - PLUGIN-5: Dynamic tools, instructions, and hooks are staged and receive the same complete validation as static contributions. ThinHarness commits the full dynamic set only after every binding opens successfully. - PLUGIN-6: A connection failure, including cancellation, closes entered bindings in reverse order, installs no dynamic contribution, and leaves connection retryable. `run_start` and `run_end` do not fire for an attempt that fails during connection. - PLUGIN-7: Closing a harness closes plugin bindings in reverse order before closing a model owned by the harness. Repeated close calls have no effect. -- PLUGIN-8: Contribution order is plugin static contributions, direct `tools=` and `hooks=`, then plugin dynamic contributions. System instructions are the configured system prompt, plugin instructions in caller plugin order, and all per-tool instructions; structured-output instructions are added through the existing output path. A skill summary is an ordinary plugin instruction at the `SkillsPlugin` position. +- PLUGIN-8: Contribution order is plugin static contributions, direct `tools=` and `hooks=`, then plugin dynamic contributions. System instructions are the configured system prompt, plugin instructions in caller plugin order, and all per-tool instructions; structured-output instructions are added through the existing output path. A skill summary is an ordinary plugin instruction at the `SkillsPlugin` position. `SubagentsPlugin` contributes its ordinary `subagent` tool at its plugin position. In a child, automatically inherited plugins keep parent plugin order, explicit child plugins follow them, inherited direct tools follow inherited plugins, and explicit child tools are last. - PLUGIN-9: ThinHarness copies caller-supplied hook registries before adding plugin hooks. Plugin composition never mutates a caller-owned registry. - PLUGIN-10: Plugins are trusted in-process code. ThinHarness does not isolate them or resolve dependencies between them. +- PLUGIN-11: Each static plugin binding supplies an immutable agent-name tuple. Core combines these names before validating agent-filtered hooks and rejects blank or duplicate names. Connected contributions cannot change the agent catalog. +- PLUGIN-12: Core records authoritative direct, plugin, and delegation composition roles independently of caller-visible `ToolOrigin`; these records control inheritance and delegation tracing and cannot be forged through tool metadata. +- PLUGIN-13: Automatic child inheritance is explicit and structural. Only a plugin with synchronous `for_child()` is rebound against the child context; its returned plugin must be valid and keep the expected fixed name. + +## Subagents Plugin + +### Purpose + +Callers add delegation explicitly through `SubagentsPlugin`, which owns the model-facing tool, child recipes, inheritance policy, child hooks, and delegation result shaping. + +### Requirements + +- SUBAGENTS-PLUGIN-1: A plain harness has no delegation tool. `SubagentsPlugin` has fixed name `"subagents"`, contributes one ordinary tool named `subagent`, and provides an unnamed `"default"` child plus ordered optional named children. Normal tool and plugin collision rules apply; no tool name is reserved. +- SUBAGENTS-PLUGIN-2: The unnamed child uses parent-derived model, prompt, limits, output, and additive inheritance defaults. Named configurations can override the model, prompt, limits, output, hooks, plugins, and tools; named children with no tools are valid. +- SUBAGENTS-PLUGIN-3: `inherit_parent=True` rebinds only plugins that explicitly implement `for_child()` and inherits eligible direct tools from the active run snapshot. Inherited sources keep parent order, explicit child plugins and tools follow them, and duplicates fail rather than replace inherited values. Approval-required tools never enter a child. +- SUBAGENTS-PLUGIN-4: Child hooks belong to the selected child configuration. Default-child hooks come from `default_hooks`; named hooks come from `SubAgentConfig.hooks`. Parent `before_subagent_run` and `after_subagent_run` hooks remain on the parent and can filter against the plugin's static agent catalog. +- SUBAGENTS-PLUGIN-5: Every top-level plugin context receives a narrow child host that accepts delegation only during an active parent tool call. Every child context receives a disabled host, and `SubagentsPlugin` is invalid in explicit child plugins, so children cannot create grandchildren through built-in or custom plugin paths. +- SUBAGENTS-PLUGIN-6: Each child is a fresh, independently budgeted run with no parent transcript. A parent-model child borrows the model; an override creates and closes its own model while projecting parent request settings and only same-provider credentials. +- SUBAGENTS-PLUGIN-7: Child events remain nested unless subagent streaming is enabled, then preserve parent run and tool-call correlation. Real delegation is traced from authoritative composition with `subagent.delegation=true` from tool-span start; a same-named direct tool or forged `ToolOrigin` remains an ordinary tool event. +- SUBAGENTS-PLUGIN-8: Successful results preserve agent, inheritance mode, effective tools, child request usage, and structured-output metadata. Unknown agents, provider failures, hook failures, strict sibling cancellation, and parent cancellation preserve existing error and cleanup behavior; every created child closes without hiding the original error. +- SUBAGENTS-PLUGIN-9: Child configuration, child hosts, plugin state, and model objects never enter resume or approval state. Parent usage counts one delegation tool call while child requests and tokens remain child usage. +- SUBAGENTS-PLUGIN-10: Child inheritance has three result modes: `"inherited"` for default or inherited-only children, `"inherited+explicit"` for additive inherited and explicit sources, and `"explicit"` when parent inheritance is disabled. ## Filesystem Plugin @@ -113,6 +135,7 @@ Callers opt into root-scoped workspace tools without making filesystem behavior - FILESYSTEM-PLUGIN-4: Harness construction and plugin binding do not create the workspace root. A harness without `FilesystemPlugin` has a generic default prompt, adds no workspace-root instruction, and has no workspace filesystem side effect. Observability sinks keep their independent configured storage behavior. - FILESYSTEM-PLUGIN-5: Filesystem limits, output location, search settings, and path policies belong to `FilesystemPlugin`. - FILESYSTEM-PLUGIN-6: Independent custom tools continue to use `tools=[ToolSpec(...)]`; callers do not need to wrap one tool in a plugin. +- FILESYSTEM-PLUGIN-7: `FilesystemPlugin` has the runtime-fixed name `"filesystem"`. Its constructor configuration is frozen: mutation of constructor inputs, returned property values, or plugin attributes cannot change later parent or child bindings. ## Skills Plugin @@ -128,7 +151,7 @@ Callers explicitly compose a fixed skill catalog and select which skill operatio - SKILLS-PLUGIN-4: Catalog names, paths, metadata, selection, and summary are frozen at plugin construction. Existing skill content, file trees, and scripts remain live and are read or executed when a tool is invoked. A new plugin is required to discover added or removed skills. - SKILLS-PLUGIN-5: A non-empty catalog contributes selected tools in caller order and one compact summary. The summary mentions `skill_read` only when that tool is selected. A catalog with no skills contributes no tools or summary. - SKILLS-PLUGIN-6: `skill_read` preserves live content, tree, containment, and truncation behavior and is parallel-safe. `skill_run` preserves runner, working-directory, merged-output, timeout, metadata, and containment behavior and runs sequentially. -- SKILLS-PLUGIN-7: An explicitly configured named child uses its own `SkillsPlugin`. A default child or a child that inherits parent tools rebinds the exact parent plugin, sharing its registry, catalog, tool order, and one summary without another discovery pass. +- SKILLS-PLUGIN-7: `SkillsPlugin` explicitly opts into safe child inheritance and rebinds through the generic plugin contract. Inherited bindings share the exact constructor-time registry, frozen catalog, tool order, and one summary without another discovery pass. Its constructor configuration is frozen so later input, property-value, or attribute mutation cannot change a binding. ## Parallel LLM Plugin @@ -139,12 +162,12 @@ Callers explicitly compose a stateless parallel batch tool and choose whether it ### Requirements - PARALLEL-LLM-PLUGIN-1: A harness accepts at most one runtime-fixed `ParallelLlmPlugin` named `"parallel_llm"`. It contributes one text-only `parallel_llm` tool with no model-visible model override. -- PARALLEL-LLM-PLUGIN-2: With no model argument, the plugin borrows the configured harness model. With a model object, it borrows that caller-owned object. The plugin does not close either borrowed model. +- PARALLEL-LLM-PLUGIN-2: With no model argument, the plugin borrows the model from the context where it is bound, so an inherited binding uses the child model. With a model object, it borrows that caller-owned object. The plugin does not close either borrowed model. Constructor configuration is frozen for parent and inherited bindings. - PARALLEL-LLM-PLUGIN-3: A string model uses plugin-owned provider and request settings, creates a provider for each batch invocation, and closes it after success, schema-resolution failure, request failure, or cancellation. Provider and request settings are rejected for borrowed models. - PARALLEL-LLM-PLUGIN-4: The plugin uses the canonical harness root. Read and write policies are root-scoped, outputs are atomic JSON files, prompt count and concurrency are bounded, and ordered sparse results report batch-local request, total, success, and failure counts. - PARALLEL-LLM-PLUGIN-5: Batch prompts use independent fresh sessions and receive no parent system prompt, tools, memory, or continuation. Batch requests and tokens are outside parent `RunUsage` and `max_model_requests`; the parent counts one batch invocation toward `max_tool_calls`. - PARALLEL-LLM-PLUGIN-6: Provider transport retries remain inside one logical batch request. Cancellation propagates and closes plugin-owned string-model providers. -- PARALLEL-LLM-PLUGIN-7: Callers that need a renamed tool or structured batch output use `ParallelLlmTool(...).spec()` directly. Direct specifications have the ordinary `"user"` tool kind. +- PARALLEL-LLM-PLUGIN-7: Callers that need a renamed tool or structured batch output use `ParallelLlmTool(...).spec()` directly. Direct specifications use the same ordinary `ToolSpec` contract as all other tools. ## Run Toolset Freeze @@ -154,9 +177,9 @@ The set of tools a model can call is fixed when a run starts, so every provider ### Requirements -- TOOLSET-FREEZE-1: The run's tool schemas, system instructions, request metadata, and structured-output request are captured once per run after harness connection and run-start hooks, and every provider request in that run uses that captured set. -- TOOLSET-FREEZE-2: A tool added with `add_tool` during an in-flight run does not appear in that run's later provider requests; it takes effect on the next run. -- TOOLSET-FREEZE-3: The executable tool map is frozen with the schemas: a model call naming a tool added mid-run resolves as an unknown tool for the current run, and approval-required detection uses the same frozen map. +- TOOLSET-FREEZE-1: The run's tool schemas, executable map, authoritative direct/plugin/delegation composition roles, system instructions, request metadata, and structured-output request are captured once per run after harness connection and run-start hooks. Every provider request and delegated child in that run uses that snapshot. +- TOOLSET-FREEZE-2: A direct tool added with `add_tool` during an in-flight run does not appear in that run's later provider requests or delegated children; it takes effect on the next run. +- TOOLSET-FREEZE-3: A model call naming a tool added mid-run resolves as an unknown tool for the current run. Approval detection, direct-tool inheritance, and delegation tracing use the same frozen authoritative map rather than live tools or caller-supplied origins. ## Run Token Accounting @@ -194,6 +217,7 @@ Provider-neutral request settings let callers tune output length and reasoning d - PROVIDER-SETTINGS-3: `HarnessConfig.effort` and `ModelSettings.effort` pass through as provider-neutral strings. OpenAI Responses and OpenRouter send `reasoning: {"effort": ...}`; Anthropic sends `output_config.effort` and injects adaptive thinking unless `extra_body` supplies its own top-level `thinking` key. - PROVIDER-SETTINGS-4: The harness does not client-validate provider/model-specific `effort`, `temperature`, or thinking combinations. Invalid combinations surface as provider API errors. - PROVIDER-SETTINGS-5: `HarnessConfig.request_timeout` applies independently to every built-in provider transport attempt, including retry attempts. +- PROVIDER-SETTINGS-6: Anthropic request metadata includes only a string `user_id`, the sole metadata field accepted by the Messages API. Other run metadata remains available to harness hooks, tracing, and child correlation but is not sent in the Anthropic payload. ## Provider Request Retries @@ -208,7 +232,7 @@ Built-in provider requests recover from transient HTTP failures without repeatin - PROVIDER-RETRY-3: Retry delay uses `request_retry_backoff * 2**retry_index` plus up to 25 percent positive jitter. A valid numeric or HTTP-date `Retry-After` can increase that delay, and every delay is capped at 60 seconds. - PROVIDER-RETRY-4: Cancellation during a request or delay propagates immediately. Exhaustion raises the final attempt's `ProviderError`, preserving provider-error run classification. - PROVIDER-RETRY-5: Transport attempts stay inside one logical model request. They do not increase model request limits, usage counts, stream event counts, trace span counts, parallel completion request counts, or provider session history. -- PROVIDER-RETRY-6: Named subagent override models inherit the parent request retry settings. A `ParallelLlmPlugin` with a string model uses its own request retry settings, while a plugin that borrows the harness model uses that model's configured transport retries. The parallel LLM tool has no separate provider retry loop or attempt budget. +- PROVIDER-RETRY-6: Named child override models inherit the parent request retry settings. A `ParallelLlmPlugin` with a string model uses its own request retry settings; one rebound with no model uses the child model settings; one configured with a model object uses that model's settings. The parallel LLM tool has no separate provider retry loop or attempt budget. - PROVIDER-RETRY-7: Retries use at-least-once HTTP delivery. A transport failure after provider acceptance can cause duplicate provider work or charges because built-in providers do not share a portable idempotency-key contract. - PROVIDER-RETRY-8: A custom `http_client` can apply its own retry policy below the provider retry loop. Callers set `request_retries=0` when the custom client owns retries to avoid multiplying attempt budgets. @@ -255,6 +279,6 @@ ThinHarness exposes tools from MCP servers through explicit `MCPPlugin` composit - MCP-4: Final close is bounded — the bound comes from FastMCP's `client_disconnect_timeout` setting (default 5 seconds) — and a caller cancellation consumed by transport cleanup is re-raised after cleanup completes. Cancelling a first connection or a final close propagates the cancellation and leaves the wrapper reusable. - MCP-5: `include_tools` and `exclude_tools` match original MCP tool names before prefixing and normalization; `tool_prefix`, schema cleanup, sanitized-name collision errors, and cross-contribution tool collision errors are ThinHarness behavior. Discovered MCP tools are ordinary `ToolSpec` objects with `ToolOrigin(plugin="mcp", source=resolved_server_id, attributes={"tool_name": original_tool_name})`. Tracing reads this origin from the `ToolSpec`, so an after-tool hook cannot erase attribution. Model-visible result metadata uses the same binding-local server id. - MCP-6: Successful `structuredContent` is returned as a JSON string; text, image, audio, embedded-resource, and resource-link blocks convert in order to model-visible text. A protocol-level tool failure (`isError`) returns a failed `ToolResult` with `error_type="MCPToolError"` and `retry=True`; known transport and protocol failures during a tool call return `error_type="MCPError"`, including when wrapped in an exception group or explicit cause chain — a group whose members are all `Exception`s is normalized when any member's cause chain holds a known failure, even alongside sibling exception noise from teardown. An exception group carrying cancellation or any other non-`Exception` failure propagates, and exceptions with no known failure in their group or cause chain propagate as programming errors. -- MCP-7: MCP tools and connection details never enter resume state. The temporary subagent bridge maps explicit `SubAgentConfig.mcp_servers` values to a child `MCPPlugin`; `inherit_mcp_servers=True` copies parent MCP server wrappers by identity and unions explicit child servers without duplicates. Default parent-tool inheritance excludes tools whose origin plugin is `"mcp"`, so each child connects its own plugin and lifecycle. +- MCP-7: MCP tools and connection details never enter resume state. `MCPPlugin` does not inherit automatically into children. A child that needs MCP lists an explicit `MCPPlugin` in its plugin configuration; that child binding owns its connection lifecycle, while reuse of the same server wrapper keeps the wrapper's reference-counted session behavior. - MCP-8: The base install works without MCP packages: importing ThinHarness and constructing any wrapper or `MCPPlugin` needs no extra, and opening a connection without `mcp` or `fastmcp` raises `MCPDependencyError` with the `thinharness[mcp]` install hint. - MCP-9: `timeout` bounds MCP initialization and HTTP connection establishment; `read_timeout` bounds MCP requests, HTTP reads, and SSE reads. diff --git a/docs/docs.md b/docs/docs.md index d881039..0d367b2 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -98,7 +98,7 @@ Important groups: - `root` defines the run root. `FilesystemPlugin` owns filesystem paths, limits, search settings, and output location. - `model`, `api_key`, `base_url`, `temperature`, `max_tokens`, `effort`, `extra_body`, `request_timeout`, `request_retries`, and `request_retry_backoff` define provider settings. -- The `Harness` constructor's `plugins=` and `tools=` inputs, plus `builtin_tools` and `subagents`, define the model-callable surface. Filesystem, MCP, skills, and parallel LLM tools use explicit plugins. `builtin_tools` temporarily selects only `subagent`. +- The `Harness` constructor's ordered `plugins=` and direct `tools=` inputs define the complete model-callable surface. ThinHarness has no implicit or selected built-in tool path. Filesystem, MCP, skills, parallel LLM, and subagent delegation use explicit plugins. - `max_model_requests`, `max_tool_calls`, `output_retries`, and `tool_retries` bound the run. - `output_type` and `output_mode` define structured output. - `tracing`, `local_tracing`, and `local_trace_dir` define observability. @@ -116,7 +116,7 @@ harness = Harness( ) ``` -Independent custom tools and hooks stay direct constructor inputs. Plugin names must be unique within one harness. ThinHarness binds static contributions during construction, then opens connected plugins on `Harness.connect()` or before the first run. Connection is atomic: a failure installs no dynamic contribution, closes opened plugins in reverse order, and allows retry. Closing the harness also closes plugins in reverse order. +Independent custom tools and hooks stay direct constructor inputs. Plugin names must be unique within one harness. `PluginContext` exposes only the canonical root, configured model, and a narrow child-harness host. ThinHarness binds static contributions during construction, then opens connected plugins on `Harness.connect()` or before the first run. Connection is atomic: a failure installs no dynamic contribution, closes opened plugins in reverse order, and allows retry. Closing the harness also closes plugins in reverse order. A plugin opts into automatic child rebinding only by implementing `for_child()`. Plugins are trusted in-process code. ThinHarness does not discover them from entry points or directories, isolate them, resolve dependencies between them, or hot reload them. @@ -267,11 +267,11 @@ The paused result includes: Resume with `resume_approvals(...)`, `stream_approvals(...)`, or `resume_approvals_sync(...)` and one `ApprovalDecision` per pending approval. Approved calls execute through the normal tool machinery, including hooks, tracing, retry accounting, and stream events. Rejected calls do not execute or fire tool hooks; the model receives a failed tool result with `error_type="ApprovalRejected"` and can explain, recover, or request another tool. -Approval-required tools need a resumable model because the harness must continue after the paused assistant tool-call turn. They are not supported inside child subagent harnesses. Built-in tools remain non-approval tools in this version; wrap built-in behavior in a custom `ToolSpec` when host review is required. +Approval-required tools need a resumable model because the harness must continue after the paused assistant tool-call turn. They are not supported inside child harnesses. Configure approval only on direct top-level `ToolSpec` values. ### Bash Prototype Tool -`BashTool` is an opt-in custom tool for exploratory agent runs. It is not part of the default built-ins, and `builtin_tools=["bash"]` is intentionally rejected. +`BashTool` is an opt-in custom tool for exploratory agent runs. ThinHarness has no implicit tools; add `BashTool(...).spec()` through direct `tools=` composition. ```python from thinharness import BashTool, Harness, HarnessConfig @@ -382,38 +382,36 @@ By default, hook exceptions are logged and the run continues. Set `strict_hooks= ## Subagents -The `subagent` tool is opt-in. It lets the parent delegate a bounded task to a child harness. Child runs start fresh; they do not inherit the parent provider transcript. +Add `SubagentsPlugin` to expose one `subagent` tool. Each call delegates one bounded task to a fresh child harness with no parent provider transcript. ```python -from thinharness import FilesystemPlugin, Harness, HarnessConfig, SubAgentConfig +from thinharness import FilesystemPlugin, Harness, HarnessConfig, SubAgentConfig, SubagentsPlugin -harness = Harness(HarnessConfig( - root=".", - builtin_tools=["subagent"], - subagents=[ - SubAgentConfig( - name="reviewer", - description="Review a draft for factual and citation issues.", - system_prompt="You are a careful review agent.", - inherit_parent_tools=True, - max_model_requests=12, - ) +harness = Harness( + HarnessConfig(root="."), + plugins=[ + FilesystemPlugin(tools=["read", "search"]), + SubagentsPlugin(agents=[ + SubAgentConfig( + name="reviewer", + description="Review a draft for factual and citation issues.", + system_prompt="You are a careful review agent.", + inherit_parent=True, + max_model_requests=12, + ) + ]), ], -), plugins=[FilesystemPlugin(tools=["read", "search"])]) +) ``` -Calling `subagent` without an `agent` argument uses the framework default subagent, which inherits parent tools except for recursive `subagent` access and MCP-discovered tools. Named subagents use their own `SubAgentConfig`. +Omitting `agent` selects the framework default child. It borrows the parent model and inherits safe parent plugins plus direct tools from the active run's frozen tool snapshot. Named children can set their own model, limits, output settings, hooks, plugins, and tools. `inherit_parent=True` is additive: inherited plugins and direct tools come first, then explicit child plugins and tools. Duplicate names fail; explicit values do not replace inherited values. -Named subagents can: +`FilesystemPlugin`, `SkillsPlugin`, and `ParallelLlmPlugin` opt into child rebinding. A borrowed `ParallelLlmPlugin(model=None)` uses the child model. `MCPPlugin` never inherits automatically; put an explicit `MCPPlugin(...)` in the child's `plugins` list. Approval-required tools are excluded or rejected. -- inherit parent tools with `inherit_parent_tools=True` -- choose explicit `plugins` -- receive explicit custom `tools` -- opt into MCP with `inherit_mcp_servers=True` or `mcp_servers=[...]` -- use their own model, limits, and structured output +Default-child hooks use `SubagentsPlugin(default_hooks=...)`. Named child hooks use `SubAgentConfig(hooks=...)`. Parent `before_subagent_run` and `after_subagent_run` hooks stay on the parent and can use `agents=[...]` filters. -`default` is reserved for the framework default subagent name. +Children receive a disabled child host and cannot delegate again. `SubagentsPlugin` is also invalid in explicit child plugins. A custom ordinary tool can still use the name `subagent` when the delegation plugin is absent. `default` is reserved only as the framework default child configuration name. ## Parallel LLM Batches @@ -506,7 +504,7 @@ Relative skill directories resolve from the process working directory, not from A non-empty catalog contributes the selected tools in caller order and one compact summary. The summary tells the model to call `skill_read` only when that tool is selected. `skill_read` is parallel-safe. `skill_run` is sequential and runs scripts from trusted skill directories: Python through `uv run`, shell through `bash`, JavaScript through `node`, and Go through `go run`. -An explicitly configured named child uses its own `SkillsPlugin`. Default children and named children with `inherit_parent_tools=True` temporarily rebind the exact parent plugin, so they share one catalog and one summary. +`SkillsPlugin` has frozen constructor configuration and opts into generic child rebinding. Default children and named children with `inherit_parent=True` reuse the exact registry and catalog, so they share one catalog and one summary. A non-inheriting child lists its own plugin explicitly. ## MCP @@ -565,7 +563,7 @@ Available wrappers: - `MCPServerSSE` - `MCPServerStreamableHTTP` -ThinHarness only turns MCP tools into harness tools; transport execution and session lifecycle come from the FastMCP client. MCP prompts, resources, sampling, OAuth flows, provider-native MCP, and `.mcp.json` discovery are outside the current scope. +ThinHarness only turns MCP tools into harness tools; transport execution and session lifecycle come from the FastMCP client. MCP never inherits automatically into a child. A child that needs MCP lists an explicit `MCPPlugin` in `SubAgentConfig.plugins`, and that child binding owns its connection lifecycle. MCP prompts, resources, sampling, OAuth flows, provider-native MCP, and `.mcp.json` discovery are outside the current scope. ## Resume @@ -677,7 +675,7 @@ harness = Harness( ) ``` -Each tracing sink owns its capture policy. External spans can exist without recording raw prompts or tool payloads unless capture flags are enabled. +Each tracing sink owns its capture policy. External spans can exist without recording raw prompts or tool payloads unless capture flags are enabled. Real delegation tool spans have `subagent.delegation=true` from span start and child `invoke_agent subagent.` spans remain nested below them. Classification uses core-owned composition provenance, so a same-named direct tool or forged `ToolOrigin` remains an ordinary tool span. ## Result Object diff --git a/docs/site/about/index.html b/docs/site/about/index.html index ba9517c..bcab630 100644 --- a/docs/site/about/index.html +++ b/docs/site/about/index.html @@ -143,9 +143,9 @@

Opinions

ThinHarness has opinions. They are the reason it stays small.

purpose_built

Purpose-built agents, not universal agents

ThinHarness is for bounded agent loops, not open-ended interactive assistants like Claude Code or OpenClaw. For business use cases, focused agent loops orchestrated by deterministic code are usually a better fit than sprawling multi-agent systems with broad authority.

-
no_bash

No bash by default

Purpose-built business agents usually don't need a shell. Bash is a broad security and reliability surface: it gives the model open-ended authority instead of typed, bounded actions. ThinHarness keeps bash out of the default and built-in tool sets, but exposes an opt-in BashTool for exploratory runs before the workflow is hardened with typed tools.

+
no_bash

No bash by default

Purpose-built business agents usually don't need a shell. Bash is a broad security and reliability surface: it gives the model open-ended authority instead of typed, bounded actions. ThinHarness has no implicit tools and exposes Bash only through an opt-in BashTool for exploratory runs before the workflow is hardened with typed tools.

search

Search is a top priority

The search tool exposes ripgrep as compact grouped path/line results, tuned for document and business-workflow agents rather than code navigation. There's also a jsonl_search variant, because JSONL is the right shape when you're replacing RAG with agent-driven search over structured data: ripgrep row prefiltering, jq-style field projection, where filters, range filters, and snippets from large multiline fields.

-
parallel_llm

Parallel LLM calls, built in

Fan out from inside the harness when a workflow needs efficient parallel processing or majority vote for reliability. Add ParallelLlmPlugin() for a plain-text batch tool that borrows the harness model, or give the plugin a model string and its own provider settings. For validated structured output per call, instantiate ParallelLlmTool with output_type (a Pydantic model). Each call is stateless, and large batches can write JSON to output_file.

+
parallel_llm_calls_explicitly_composed

Parallel LLM calls, explicitly composed

Fan out from inside the harness when a workflow needs efficient parallel processing or majority vote for reliability. Add ParallelLlmPlugin() for a plain-text batch tool that borrows the harness model, or give the plugin a model string and its own provider settings. For validated structured output per call, instantiate ParallelLlmTool with output_type (a Pydantic model). Each call is stateless, and large batches can write JSON to output_file.

no_token_streaming

No token streaming

Streaming is for workflow progress, not live chatbot text. ThinHarness emits run, model-turn, tool, retry, limit, and subagent events, but it does not stream provider token deltas. Token streaming would add provider-specific plumbing, event merging, cancellation edge cases, and more surface area to keep stable. For workflow-style agents, step-level updates are usually the useful signal.

providers

Three providers, no matrix

ThinHarness ships small provider classes for OpenAI, Anthropic, and OpenRouter. If your gateway speaks one of those protocols, you swap a base URL and move on. If not, the provider classes are small enough to fork or replace, and ignoring the bundled ones costs you nothing.

no_compaction

No compaction

Compaction is a workaround for context windows filling up across long, accumulating runs — useful for interactive coding sessions that sprawl over hours. For SDK-based business agents, the right answer to "context is getting big" is almost always better task decomposition: shorter runs, separate harness instances, narrower subagents.

@@ -184,12 +184,12 @@

Features

Filesystem plugin

Explicit FilesystemPlugin composition for read, write, batched exact-replacement edit, search, list, and glob with root-scoped path policies.

JSONL search

Opt-in jsonl_search for structured line-delimited data, with ripgrep prefiltering, field projection, equality/contains/regex/range where filters, and field-level snippets from large multiline string values.

-
Bash prototype tool

Opt-in BashTool for exploratory shell commands. It is lightweight, custom-registration only, and is not included in the default or built-in tool set.

+
Bash prototype tool

Opt-in BashTool for exploratory shell commands. It is lightweight and available only through direct custom registration.

Provider adapters

Built-in OpenAI, Anthropic, and OpenRouter adapters, plus public model/session protocols for implementing another provider.

Custom typed tools

Define sync or async ToolSpec handlers with Pydantic argument models, normalized ToolResult envelopes, sequential/approval flags, and per-tool retry settings.

Structured output

Pydantic-validated results with native, tool, prompted, and text modes.

Hooks

Lifecycle and tool-call interception for prompt submission, tool calls, subagents, limits, and run boundaries.

-
Subagents

Opt-in delegation through a built-in subagent tool and explicit SubAgentConfig.

+
Subagents

Explicit SubagentsPlugin composition with a default child, ordered named SubAgentConfig recipes, additive safe-plugin inheritance, local child hooks, and no recursive delegation.

Parallel LLM

Explicit ParallelLlmPlugin fan-out for batches of independent one-shot prompts, plus ParallelLlmTool(...).spec() for renameable or structured tools with explicit model, path, prompt, and provider request settings.

Skills

Explicit SkillsPlugin composition with an ordered skill_read and/or skill_run selection, plus Python, shell, JavaScript, and Go script runners.

Resume

Clean new-turn continuation through self-contained transcript state that can replay across built-in providers and models, preserving native reasoning on same-provider resume and degrading it to text across providers.

diff --git a/docs/site/explainer/index.html b/docs/site/explainer/index.html index 9102d82..6146eac 100644 --- a/docs/site/explainer/index.html +++ b/docs/site/explainer/index.html @@ -162,7 +162,7 @@

Repository File Map

| |-- providers.py provider transports, model adapters, session state | |-- output.py structured-output schemas and validation | |-- hooks.py hook dataclasses, registry, context variables -| |-- subagents.py subagent tool and child harness construction +| |-- children.py narrow child-harness host and execution | |-- tracing.py OTel-compatible spans and local JSONL tracing | |-- defaults.py default system and tool instructions | |-- plugins/ @@ -196,7 +196,7 @@

Harness-facing objects

- + @@ -265,9 +265,9 @@

Sequential flag

Plugin composition

- ThinHarness composes filesystem, skills, MCP, and parallel LLM behavior through explicit plugins. Plugins receive - the canonical root and configured model, and contribute ordered tools, instructions, hooks, or connected state. - builtin_tools remains temporarily only for subagent. + ThinHarness composes filesystem, skills, MCP, parallel LLM, and subagent behavior through explicit plugins. Plugins receive + the canonical root, configured model, and narrow child host, then contribute ordered tools, instructions, hooks, agent names, or connected state. + The core has no implicit or selected built-in tool path.

@@ -287,15 +287,15 @@

Tool surfaces

- + - - - - + + + + @@ -458,13 +458,13 @@

Extras

- - + + - + @@ -484,9 +484,9 @@

Recommended Reading Path

  • thinharness/output.py so final-answer decisions make sense before reading the loop.
  • thinharness/core.py, thinharness/turns.py, and thinharness/runtime.py together. Read Harness.__init__, then Harness.run, then advance_until_terminal and RunContext.advance_model.
  • thinharness/tool_execution.py to see how a model-emitted batch becomes ordered provider outputs.
  • -
  • thinharness/tools/filesystem.py to understand the default workspace tools the model can call.
  • +
  • thinharness/tools/filesystem.py to understand workspace tools that FilesystemPlugin can expose.
  • thinharness/hooks.py and thinharness/tracing.py to understand lifecycle callbacks and run observability.
  • -
  • Pick extras as needed: thinharness/tools/jsonl.py, thinharness/subagents.py, thinharness/tools/mcp.py, thinharness/tools/skills.py, and thinharness/tools/parallel_llm.py.
  • +
  • Pick extras as needed: thinharness/tools/jsonl.py, thinharness/plugins/subagents.py, thinharness/children.py, thinharness/tools/mcp.py, thinharness/tools/skills.py, and thinharness/tools/parallel_llm.py.
  • Use tests as executable documentation. Start with tests/test_harness.py, then the feature-specific test file for whatever you are changing.
  • @@ -545,8 +545,8 @@

    Implementation Deep Dive

    NameMeaningRelationship
    HarnessConfigPydantic setup model: root, model ref, deferred subagent selection, limits, output mode, tracing, and subagents.Configures a Harness.
    HarnessConfigPydantic setup model: root, model ref, limits, output mode, tracing, and provider settings.Configures a Harness.
    HarnessLong-lived configured runner. It owns the model object, resolved tool map, plugin bindings, hooks, and tracing configuration.Creates a fresh RunContext for each run.
    RequestConstantsFrozen per-run request constants: instructions, tool schemas, metadata, and the structured-output request. Built once after run-start hooks and plugin connection, so the toolset is frozen for the run.Passed to every ModelSession request by the turn machine in turns.py.
    RunContextInternal state for one Harness.run(...): responses, tool records, usage, retry/notice state, terminal error, stop reason, tracing span, and final result.References the reusable Harness, but is not stored on it after the run.
    Plugin-providedSkillsPlugin, ParallelLlmPlugin, and MCPPluginSkillsPlugin, ParallelLlmPlugin, MCPPlugin, and SubagentsPlugin Explicit plugin contributions are normalized into the same ToolSpec map with plugin origin attribution. Skills and parallel LLM contribute static tools; MCP connects lazily and contributes one discovered snapshot.
    Deferred built-inSubagent compositionbuiltin_tools=["subagent"] selects the remaining core candidate.Child composition stays in subagents.py until its plugin migration.DelegationSubagentsPluginExplicit plugin composition contributes one ordinary subagent tool and static agent names.The plugin owns child recipes; the neutral child host owns isolated execution and authoritative delegation provenance.
    Custom
    SubagentsThe subagent framework tool builds a child Harness and returns the child result as a tool result.The parent tool call awaits the child run. Child harnesses always receive subagents=[], structurally disabling recursion.SubagentsPlugin contributes the subagent tool and passes an opaque recipe to a narrow core child host.The parent tool call awaits a fresh child. Safe plugins rebind through for_child(), direct tools come from the frozen run snapshot, and the child's disabled host blocks recursion.
    MCP Configured server objects connect lazily and discover tools into live ToolSpec objects.Subagents do not inherit parent-discovered MCP tools as ordinary custom ToolSpec objects. A named subagent can opt into MCP with inherit_mcp_servers=True or its own mcp_servers=[...]; then the child harness discovers MCP tools through its own connection lifecycle against those configured server objects.MCP does not inherit into children. A named child lists an explicit MCPPlugin in its plugin configuration, then discovers tools through that child binding's connection lifecycle.
    Parallel LLM

    Inheritance is intentionally light. Provider session classes implement the same session protocol, but the core - loop mostly uses composition: Harness owns a model, tool specs, hooks, tracing options, skill registry, - and MCP server list. Built-in tools often use classes as state holders, then hand bound methods to ToolSpec. + loop mostly uses composition: Harness owns a model, tool specs, plugin bindings, hooks, and tracing options. + Plugin and direct tool objects hand callable handlers to ordinary ToolSpec values.

    @@ -916,13 +916,13 @@

    Implementation Deep Dive

    Subagents - The subagent framework tool builds a child Harness and returns the child result as a tool result. - Child runs start fresh, recursion is structurally disabled, and inherited custom tools are live ToolSpec objects. + SubagentsPlugin owns the delegation tool and child recipes; children.py implements the narrow host. + Child runs start fresh, safe plugins rebind, direct tools use the frozen parent-run snapshot, and a disabled child host blocks recursion. MCP Configured server objects connect lazily and discover tools into live ToolSpec objects. - Subagents do not inherit parent-discovered MCP tools as ordinary custom ToolSpec objects. They opt into MCP through inherited or explicit server config, then discover tools in the child harness lifecycle. + MCP does not inherit automatically. A child lists an explicit MCPPlugin and discovers tools in its own harness lifecycle. Skills diff --git a/examples/mcp_plugin.py b/examples/mcp_plugin.py index 489c1b6..80969d3 100644 --- a/examples/mcp_plugin.py +++ b/examples/mcp_plugin.py @@ -8,7 +8,7 @@ async def main() -> None: """Run an agent with tools discovered from a local MCP server.""" async with Harness( - HarnessConfig(root=".", model="openai:gpt-5.5", builtin_tools=[]), + HarnessConfig(root=".", model="openai:gpt-5.5"), plugins=[ MCPPlugin( servers=[ diff --git a/examples/web_research_report/agent.py b/examples/web_research_report/agent.py index 4a6422a..da20502 100644 --- a/examples/web_research_report/agent.py +++ b/examples/web_research_report/agent.py @@ -16,7 +16,19 @@ import httpx from pydantic import BaseModel, ConfigDict, Field -from thinharness import FilesystemPlugin, Harness, HarnessConfig, Hook, ParallelLlmTool, PathPolicy, PathValidationError, SubAgentConfig, ToolResult, ToolSpec +from thinharness import ( + FilesystemPlugin, + Harness, + HarnessConfig, + Hook, + ParallelLlmTool, + PathPolicy, + PathValidationError, + SubAgentConfig, + SubagentsPlugin, + ToolResult, + ToolSpec, +) REPO_ROOT = Path(__file__).resolve().parents[2] EXAMPLE_ROOT = Path(__file__).resolve().parent @@ -487,7 +499,6 @@ def build_harness(root: Path, *, model: str = DEFAULT_MODEL) -> Harness: root=root, model=model, system_prompt=SYSTEM_PROMPT, - builtin_tools=["subagent"], output_type=ReportReceipt, output_mode=output_mode, output_retries=2, @@ -498,7 +509,16 @@ def build_harness(root: Path, *, model: str = DEFAULT_MODEL) -> Harness: request_timeout=240, temperature=0, extra_body=_model_extra_body(model), - subagents=[ + ), + plugins=[ + FilesystemPlugin( + tools=["read", "write", "edit", "search", "list", "glob", "jsonl_search"], + read_paths=["outputs"], + write_paths=["outputs"], + max_read_chars=80_000, + max_tool_chars=80_000, + ), + SubagentsPlugin(agents=[ SubAgentConfig( name="citation_critic", description="Citation and evidence critic for saved draft reports.", @@ -509,15 +529,8 @@ def build_harness(root: Path, *, model: str = DEFAULT_MODEL) -> Harness: output_retries=1, tool_retries=1, ) - ], - ), - plugins=[FilesystemPlugin( - tools=["read", "write", "edit", "search", "list", "glob", "jsonl_search"], - read_paths=["outputs"], - write_paths=["outputs"], - max_read_chars=80_000, - max_tool_chars=80_000, - )], + ]), + ], tools=[*exa_tools.specs(), parallel_tool], hooks=[Hook("after_tool_call", _source_audit_hook)], ) diff --git a/scripts/build_transcripts.py b/scripts/build_transcripts.py index 547dbdf..9c84cd5 100644 --- a/scripts/build_transcripts.py +++ b/scripts/build_transcripts.py @@ -10,7 +10,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1] EXAMPLES_ROOT = REPO_ROOT / "examples" LONGMEMEVAL_MD = EXAMPLES_ROOT / "longmemeval.md" -DEFAULT_OUTPUT = REPO_ROOT / "docs" / "site" / "examples.html" +DEFAULT_OUTPUT = REPO_ROOT / "docs" / "site" / "examples" / "index.html" LONG_PREVIEW_CHARS = 1200 WEB_RESEARCH_REPORT_META = { "slug": "web_research_report", @@ -375,7 +375,7 @@ def event_from_span(span: dict[str, Any], trace_rel: str, index: int, call_label tool_name = str(attrs.get("gen_ai.tool.name") or name.removeprefix("execute_tool ")) args = parse_jsonish(attrs.get("gen_ai.tool.call.arguments")) result = tool_result_parts(attrs.get("gen_ai.tool.call.result")) - is_subagent = tool_name == "subagent" + is_subagent = attrs.get("subagent.delegation") is True event = { **base, "kind": "subagent" if is_subagent else "tool", diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 9701c6b..451c364 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -18,6 +18,8 @@ Current journeys: - `structured_output_journey.py`: Pydantic structured output after tool use. - `mcp_journey.py`: deterministic local stdio MCP tool discovery, execution, and cleanup. It reports a skip when the `mcp` extra is not installed and does not use provider credentials. - `parallel_llm_tool_journey.py`: direct `ParallelLlmTool` calls across all configured providers. -- `parallel_llm_agent_journey.py`: an agent run using both built-in `parallel_llm` and a renamed custom `ParallelLlmTool`. +- `parallel_llm_agent_journey.py`: an agent run using `ParallelLlmPlugin` and a renamed custom `ParallelLlmTool`. - `prompt_caching_journey.py`: Anthropic prompt caching — asserts a multi-request run reports cached input tokens. - `anthropic_modernization_journey.py`: Anthropic native structured output, max-token/effort payloads, and adaptive/default-on thinking resume. +- `subagents_journey.py`: explicit default and cross-provider named delegation, child output metadata, and disabled nested child creation. +- `langfuse_tracing_journey.py`: nested subagent and tool spans through a live Langfuse OTLP sink. diff --git a/tests/e2e/anthropic_modernization_journey.py b/tests/e2e/anthropic_modernization_journey.py index e93f857..a99d78e 100644 --- a/tests/e2e/anthropic_modernization_journey.py +++ b/tests/e2e/anthropic_modernization_journey.py @@ -69,7 +69,6 @@ async def _assert_native_structured_output_and_defaults(root: Path, model_name: harness = Harness( HarnessConfig( root=root, - builtin_tools=[], output_type=InventoryAnswer, max_model_requests=4, max_tool_calls=2, @@ -100,7 +99,6 @@ async def _assert_effort_merges_with_native_structured_output(root: Path, model_ harness = Harness( HarnessConfig( root=root, - builtin_tools=[], output_type=InventoryAnswer, max_model_requests=2, ), @@ -124,7 +122,7 @@ async def _assert_default_on_thinking_resume(root: Path, model_name: str) -> Non first_provider = RecordingAnthropicProvider() try: first = await Harness( - HarnessConfig(root=root, builtin_tools=[], max_model_requests=4, max_tool_calls=2), + HarnessConfig(root=root, max_model_requests=4, max_tool_calls=2), model=AnthropicMessagesModel(model_name, provider=first_provider), tools=[multiply_tool()], ).run("Use the multiply tool to compute 37 times 29, then state the product.") @@ -137,7 +135,7 @@ async def _assert_default_on_thinking_resume(root: Path, model_name: str) -> Non second_provider = RecordingAnthropicProvider() try: second = await Harness( - HarnessConfig(root=root, builtin_tools=[], max_model_requests=3, max_tool_calls=1), + HarnessConfig(root=root, max_model_requests=3, max_tool_calls=1), model=AnthropicMessagesModel(model_name, provider=second_provider), tools=[multiply_tool()], ).run("Add 11 to that product and answer with the new number.", resume_from=state) diff --git a/tests/e2e/control_plane_journey.py b/tests/e2e/control_plane_journey.py index c7a52b6..d4efdba 100644 --- a/tests/e2e/control_plane_journey.py +++ b/tests/e2e/control_plane_journey.py @@ -41,7 +41,6 @@ def handler(ctx): root=root, model=MODEL, system_prompt=SYSTEM_PROMPT, - builtin_tools=[], max_model_requests=4, max_tool_calls=2, tool_retries=0, diff --git a/tests/e2e/langfuse_tracing_journey.py b/tests/e2e/langfuse_tracing_journey.py index a83074a..202a71b 100644 --- a/tests/e2e/langfuse_tracing_journey.py +++ b/tests/e2e/langfuse_tracing_journey.py @@ -9,7 +9,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from thinharness import FilesystemPlugin, Harness, HarnessConfig, SubAgentConfig, TracingOptions, create_otlp_tracing +from thinharness import FilesystemPlugin, Harness, HarnessConfig, SubAgentConfig, SubagentsPlugin, TracingOptions, create_otlp_tracing MODEL = os.getenv("E2E_LANGFUSE_TRACING_MODEL", "openrouter:anthropic/claude-haiku-4.5") SYSTEM_PROMPT = "You are a tracing validation parent. Do your own parent checks, then delegate child file work to the named subagent." @@ -52,11 +52,13 @@ def main() -> None: root=root, model=MODEL, system_prompt=SYSTEM_PROMPT, - builtin_tools=["subagent"], max_model_requests=40, max_tool_calls=12, local_trace_dir=trace_dir, - subagents=[ + ), + plugins=[ + FilesystemPlugin(tools=["list", "read", "write"]), + SubagentsPlugin(agents=[ SubAgentConfig( name="writer", description="Creates and revises files for tracing validation.", @@ -64,9 +66,8 @@ def main() -> None: max_model_requests=20, max_tool_calls=8, ) - ], - ), - plugins=[FilesystemPlugin(tools=["list", "read", "write"])], + ]), + ], tracing=[TracingOptions( tracer=tracing.tracer, agent_name="langfuse-parent", diff --git a/tests/e2e/mcp_journey.py b/tests/e2e/mcp_journey.py index 53768a4..c174035 100644 --- a/tests/e2e/mcp_journey.py +++ b/tests/e2e/mcp_journey.py @@ -80,7 +80,7 @@ async def _run() -> None: server = MCPServerStdio(sys.executable, [str(server_path)]) async with Harness( - HarnessConfig(root=root, builtin_tools=[], max_model_requests=2, max_tool_calls=1), + HarnessConfig(root=root, max_model_requests=2, max_tool_calls=1), model=DeterministicModel(), plugins=[MCPPlugin(servers=[server])], hooks=[Hook("before_tool_call", lambda ctx: tool_names.append(ctx.tool_name))], diff --git a/tests/e2e/prompt_caching_journey.py b/tests/e2e/prompt_caching_journey.py index c7312fe..01150b7 100644 --- a/tests/e2e/prompt_caching_journey.py +++ b/tests/e2e/prompt_caching_journey.py @@ -34,7 +34,6 @@ def main() -> None: root=root, model=MODEL, system_prompt=SYSTEM_PROMPT, - builtin_tools=[], max_model_requests=4, max_tool_calls=2, ), diff --git a/tests/e2e/structured_output_journey.py b/tests/e2e/structured_output_journey.py index aedebb7..a4842ed 100644 --- a/tests/e2e/structured_output_journey.py +++ b/tests/e2e/structured_output_journey.py @@ -41,7 +41,6 @@ def main() -> None: root=root, model=MODEL, system_prompt=SYSTEM_PROMPT, - builtin_tools=[], output_type=InventoryAnswer, output_mode="native", max_model_requests=4, diff --git a/tests/e2e/subagents_journey.py b/tests/e2e/subagents_journey.py new file mode 100644 index 0000000..983f908 --- /dev/null +++ b/tests/e2e/subagents_journey.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import os +import sys +from pathlib import Path +from tempfile import TemporaryDirectory + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from thinharness import ( + AfterSubagentRunContext, + ChildHarnessRequest, + Harness, + HarnessConfig, + HarnessError, + Hook, + PluginBinding, + PluginContribution, + SubAgentConfig, + SubagentsPlugin, + ToolResult, + ToolSpec, +) + +PARENT_MODEL = os.getenv("E2E_SUBAGENTS_MODEL", "anthropic:claude-sonnet-4-5-20250929") +CHILD_MODEL = os.getenv("E2E_SUBAGENTS_CHILD_MODEL", "openai:gpt-5-mini") +SYSTEM_PROMPT = """You are a delegation test parent. Follow the requested delegation steps exactly.""" +PROMPT = """ +Use the subagent tool exactly twice, in this order: +1. Call subagent with exactly one argument field: {"task":"Reply with exactly DEFAULT_CHILD_OK."}. The agent field must be absent; do not set it to "default". +2. Call subagent with agent="guard". Ask it to call attempt_nested once, then reply with exactly NAMED_CHILD_OK if that tool reports NESTED_BLOCKED. +Do not call attempt_nested yourself. After both calls, state both child markers and end with SUBAGENTS_JOURNEY_DONE. +""".strip() + + +class NestedAttemptPlugin: + """Expose a child-inherited probe of the disabled child host.""" + + name = "nested_attempt" + + def __init__(self, blocked: list[str]) -> None: + self.blocked = blocked + + def for_child(self) -> NestedAttemptPlugin: + return self + + def bind(self, context) -> PluginBinding: + async def attempt_nested(_args) -> ToolResult: + request = ChildHarnessRequest( + agent_name="forbidden", + agent_description="Forbidden nested child.", + trace_agent_name="subagent.forbidden", + task="This must not run.", + inherited=False, + tool_mode="explicit", + system_prompt="This must not run.", + ) + try: + await context.child_harnesses.run(request) + except HarnessError as exc: + self.blocked.append(str(exc)) + return ToolResult(True, "NESTED_BLOCKED") + raise AssertionError("child host allowed nested delegation") + + return PluginBinding(static=PluginContribution(tools=( + ToolSpec("attempt_nested", "Verify that nested child creation is blocked.", {"type": "object", "properties": {}}, attempt_nested), + ))) + + +def main() -> None: + """Run real default and named delegation, including the disabled child-host probe.""" + if _should_skip(PARENT_MODEL, CHILD_MODEL): + return + + with TemporaryDirectory(prefix="thinharness-e2e-subagents-") as raw_root: + blocked: list[str] = [] + child_events: list[tuple[str, list[str], str]] = [] + + def after_child(ctx) -> None: + assert isinstance(ctx, AfterSubagentRunContext) + assert ctx.result is not None + child_events.append((ctx.agent, list(ctx.tools), ctx.result.text)) + + probe = NestedAttemptPlugin(blocked) + harness = Harness( + HarnessConfig( + root=Path(raw_root), + model=PARENT_MODEL, + system_prompt=SYSTEM_PROMPT, + max_model_requests=12, + max_tool_calls=4, + ), + plugins=[ + probe, + SubagentsPlugin(agents=[ + SubAgentConfig( + name="guard", + description="Tests that child delegation is disabled.", + system_prompt="Call attempt_nested once. If it returns NESTED_BLOCKED, reply with exactly NAMED_CHILD_OK.", + inherit_parent=True, + model=CHILD_MODEL, + max_model_requests=4, + max_tool_calls=1, + ) + ]), + ], + hooks=[Hook("after_subagent_run", after_child)], + ) + + result = harness.run_sync(PROMPT) + delegation_records = [record for record in result.tool_call_records if record.get("call", {}).get("name") == "subagent"] + metadata = [ToolResult.from_json(record["output"]).metadata for record in delegation_records] + + assert len(delegation_records) == 2, f"expected two delegations, got {len(delegation_records)}" + assert [item["agent"] for item in metadata] == ["default", "guard"] + assert [item["tool_mode"] for item in metadata] == ["inherited", "inherited"] + assert all(item["tools"] == ["attempt_nested"] for item in metadata) + assert [item["structured_output"] for item in metadata] == [False, False] + assert blocked and "cannot create nested child harnesses" in blocked[0] + assert [event[0] for event in child_events] == ["default", "guard"] + assert "DEFAULT_CHILD_OK" in child_events[0][2] + assert "NAMED_CHILD_OK" in child_events[1][2] + assert "DEFAULT_CHILD_OK" in result.text + assert "NAMED_CHILD_OK" in result.text + assert "SUBAGENTS_JOURNEY_DONE" in result.text + print( + f"PASS subagents_journey parent={PARENT_MODEL} child={CHILD_MODEL} " + f"agents={[item['agent'] for item in metadata]}" + ) + + +def _should_skip(*models: str) -> bool: + if os.getenv("CI"): + print("SKIP subagents_journey: CI is set") + return True + provider_env = { + "openai": "OPENAI_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", + "openrouter": "OPENROUTER_API_KEY", + } + missing = sorted({provider_env[model.split(":", 1)[0]] for model in models if not os.getenv(provider_env[model.split(":", 1)[0]])}) + if missing: + print(f"SKIP subagents_journey: missing {', '.join(missing)}") + return True + return False + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/workspace_tools_journey.py b/tests/e2e/workspace_tools_journey.py index 4c0dc21..fe4eeae 100644 --- a/tests/e2e/workspace_tools_journey.py +++ b/tests/e2e/workspace_tools_journey.py @@ -47,7 +47,6 @@ def main() -> None: root=root, model=MODEL, system_prompt=SYSTEM_PROMPT, - builtin_tools=[], max_model_requests=30, max_tool_calls=12, local_trace_dir=trace_dir, diff --git a/tests/unit/fakes.py b/tests/unit/fakes.py index 6c64007..fc37082 100644 --- a/tests/unit/fakes.py +++ b/tests/unit/fakes.py @@ -9,6 +9,8 @@ from thinharness import ( AnthropicProvider, + ChildHarnessOutcome, + ChildHarnessRequest, OpenAIProvider, OpenAIResponsesModel, OpenRouterProvider, @@ -19,6 +21,18 @@ SCRIPTED_MODEL_NAME = "scripted-model" +class FakeChildHarnessHost: + """No-op child host for direct third-party-style plugin bindings.""" + + def register_delegation_tool(self, tool: ToolSpec, recipes) -> ToolSpec: + del recipes + return tool + + async def run(self, request: ChildHarnessRequest) -> ChildHarnessOutcome: + del request + raise AssertionError("unexpected child harness request") + + class FakeClient(OpenAIProvider): def __init__(self) -> None: super().__init__(api_key="fake") @@ -207,6 +221,7 @@ def __init__( self.on_start = on_start self.on_continue = on_continue self.notice_calls: list[tuple[str, list[ModelNotice]]] = [] + self.continue_calls: list[tuple[Any, Any, Any]] = [] self._dump_state = dump_state if dump_state is not None else {"kind": "scripted", "version": 1, "model": SCRIPTED_MODEL_NAME} async def start(self, prompt, constants, *, previous_response_id=None, notices=None): @@ -219,6 +234,7 @@ async def start(self, prompt, constants, *, previous_response_id=None, notices=N async def continue_with_tools(self, outputs, constants, *, notices=None): """Return the scripted continuation turn.""" self.notice_calls.append(("continue_with_tools", list(notices or []))) + self.continue_calls.append((outputs, constants.tools, constants.metadata)) if self.on_continue: self.on_continue(outputs, constants.tools, constants.metadata) return self.continue_turn diff --git a/tests/unit/test_approvals.py b/tests/unit/test_approvals.py index 6ff61f7..21d45f7 100644 --- a/tests/unit/test_approvals.py +++ b/tests/unit/test_approvals.py @@ -26,6 +26,7 @@ RunUsage, SkillsPlugin, SubAgentConfig, + SubagentsPlugin, TokenUsage, ToolCallCompletedEvent, ToolCallStartedEvent, @@ -121,7 +122,7 @@ async def test_approval_required_tool_pauses_without_executing_or_hooks(tmp_path ), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([session]), tools=[approval_tool(called)], hooks=[ @@ -159,7 +160,7 @@ async def test_approval_resume_approve_executes_original_call_and_finishes(tmp_p ) resumed = ScriptedSession(start_turn=ModelTurn(raw={"unused": True}), continue_turn=ModelTurn(text="done", raw={"id": "done"})) model = ScriptedModel([first, resumed]) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model, tools=[approval_tool(called)]) + harness = Harness(HarnessConfig(root=tmp_path), model=model, tools=[approval_tool(called)]) paused = await harness.run("deploy") result = await harness.resume_approvals( @@ -191,10 +192,10 @@ def test_resume_approvals_sync_success_on_fresh_harness(tmp_path: Path) -> None: ) resumed = ScriptedSession(start_turn=ModelTurn(raw={"unused": True}), continue_turn=ModelTurn(text="done", raw={"id": "done"})) model = ScriptedModel([first, resumed]) - paused = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model, tools=[approval_tool(called)]).run_sync("deploy") + paused = Harness(HarnessConfig(root=tmp_path), model=model, tools=[approval_tool(called)]).run_sync("deploy") result = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=model, tools=[approval_tool(called)], ).resume_approvals_sync( @@ -219,7 +220,7 @@ async def test_approval_resume_reject_sends_model_visible_rejection(tmp_path: Pa continue_turn=ModelTurn(text="not deployed", raw={"id": "done"}), on_continue=lambda outputs, _tools, _metadata: seen_outputs.extend(outputs), ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([first, resumed]), tools=[approval_tool([])]) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([first, resumed]), tools=[approval_tool([])]) paused = await harness.run("deploy") result = await harness.resume_approvals( @@ -252,7 +253,7 @@ async def test_approval_resume_reject_without_reason_omits_reason_line(tmp_path: continue_turn=ModelTurn(text="not deployed", raw={"id": "done"}), on_continue=lambda outputs, _tools, _metadata: seen_outputs.extend(outputs), ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([first, resumed]), tools=[approval_tool([])]) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([first, resumed]), tools=[approval_tool([])]) paused = await harness.run("deploy") result = await harness.resume_approvals(paused.resume_state, [ApprovalDecision(call_id="call_1", approved=False)]) @@ -280,7 +281,7 @@ async def test_mixed_batch_pauses_everything_and_resumes_in_model_order(tmp_path on_continue=lambda outputs, _tools, _metadata: seen_call_ids.extend(output.call_id for output in outputs), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([first, resumed]), tools=[approval_tool([]), echo_tool(normal_called)], hooks=[Hook("before_tool_call", lambda ctx: hook_indices.append(ctx.tool_index))], @@ -312,7 +313,7 @@ async def test_approval_resume_tracing_uses_restored_conversation_id(tmp_path: P ) resumed = ScriptedSession(start_turn=ModelTurn(raw={"unused": True}), continue_turn=ModelTurn(text="done", raw={"id": "done"})) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([first, resumed]), tools=[approval_tool([])], tracing=[TracingOptions(tracer=tracer)], @@ -333,7 +334,7 @@ async def test_approval_decision_validation_happens_before_execution(tmp_path: P raw={"id": "start"}, ), ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([first]), tools=[approval_tool(called)]) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([first]), tools=[approval_tool(called)]) paused = await harness.run("deploy") with pytest.raises(HarnessError, match="missing approval decision"): @@ -373,7 +374,7 @@ async def test_tampered_approval_required_ids_fail_closed(tmp_path: Path) -> Non ) resumed = ScriptedSession(start_turn=ModelTurn(raw={"unused": True}), continue_turn=ModelTurn(text="done", raw={"id": "done"})) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([first, resumed]), tools=[approval_tool(approval_called), echo_tool(normal_called)], ) @@ -403,7 +404,7 @@ def cancel(ctx) -> None: ctx.cancel_reason = "blocked by policy" harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([first, resumed]), tools=[approval_tool(called)], hooks=[Hook("before_tool_call", cancel)], @@ -439,7 +440,7 @@ async def test_approved_retry_output_uses_retry_accounting_and_can_repause(tmp_p lambda args: ToolResult(False, "try again", {"error_type": "RetryMe", "retry": True}), requires_approval=True, ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], tool_retries=2), model=ScriptedModel([first, resumed]), tools=[tool]) + harness = Harness(HarnessConfig(root=tmp_path, tool_retries=2), model=ScriptedModel([first, resumed]), tools=[tool]) paused = await harness.run("deploy") result = await harness.resume_approvals(paused.resume_state, [ApprovalDecision(call_id="call_1", approved=True)]) @@ -460,7 +461,7 @@ async def test_resume_budget_spans_approval_pause(tmp_path: Path) -> None: ) resumed = ScriptedSession(start_turn=ModelTurn(raw={"unused": True}), continue_turn=ModelTurn(text="done", raw={"id": "done"})) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], max_model_requests=1), + HarnessConfig(root=tmp_path, max_model_requests=1), model=ScriptedModel([first, resumed]), tools=[approval_tool(called)], ) @@ -481,7 +482,7 @@ async def test_over_budget_approval_batch_fails_before_pause(tmp_path: Path) -> ), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], max_tool_calls=0), + HarnessConfig(root=tmp_path, max_tool_calls=0), model=ScriptedModel([session]), tools=[approval_tool(called)], ) @@ -497,7 +498,7 @@ async def test_openai_approval_pause_round_trips_provider_state(tmp_path: Path) client = FakeClient() model = _fake_openai(client) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=model, tools=[ ToolSpec( @@ -512,7 +513,7 @@ async def test_openai_approval_pause_round_trips_provider_state(tmp_path: Path) paused = await harness.run("read") result = await Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=model, tools=[ ToolSpec( @@ -543,9 +544,9 @@ async def test_anthropic_approval_pause_round_trips_provider_state(tmp_path: Pat called: list[dict] = [] provider = FakeAnthropicProvider() model = AnthropicMessagesModel("claude-test", provider=provider) - paused = await Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model, tools=[approval_echo_tool(called)]).run("first") + paused = await Harness(HarnessConfig(root=tmp_path), model=model, tools=[approval_echo_tool(called)]).run("first") - result = await Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model, tools=[approval_echo_tool(called)]).resume_approvals( + result = await Harness(HarnessConfig(root=tmp_path), model=model, tools=[approval_echo_tool(called)]).resume_approvals( json.loads(json.dumps(paused.resume_state)), [ApprovalDecision(call_id="toolu_1", approved=True)], ) @@ -563,9 +564,9 @@ async def test_openrouter_approval_pause_round_trips_provider_state(tmp_path: Pa called: list[dict] = [] provider = FakeOpenRouterProvider() model = OpenRouterModel("openai/test", provider=provider) - paused = await Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model, tools=[approval_echo_tool(called)]).run("first") + paused = await Harness(HarnessConfig(root=tmp_path), model=model, tools=[approval_echo_tool(called)]).run("first") - result = await Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model, tools=[approval_echo_tool(called)]).resume_approvals( + result = await Harness(HarnessConfig(root=tmp_path), model=model, tools=[approval_echo_tool(called)]).resume_approvals( json.loads(json.dumps(paused.resume_state)), [ApprovalDecision(call_id="call_1", approved=True)], ) @@ -597,7 +598,7 @@ class Answer(BaseModel): ), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Answer, output_mode="tool"), + HarnessConfig(root=tmp_path, output_type=Answer, output_mode="tool"), model=ScriptedModel([first, resumed]), tools=[approval_tool([])], ) @@ -622,7 +623,7 @@ async def test_approval_resume_metadata_override_replaces_envelope_metadata(tmp_ continue_turn=ModelTurn(text="done", raw={"id": "done"}), on_continue=lambda _outputs, _tools, metadata: seen_metadata.update(metadata), ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([first, resumed]), tools=[approval_tool([])]) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([first, resumed]), tools=[approval_tool([])]) paused = await harness.run("deploy", metadata={"conversation_id": "original", "keep": "old"}) await harness.resume_approvals( @@ -641,7 +642,7 @@ async def test_approval_resume_rejects_duplicate_batch_call_ids(tmp_path: Path) raw={"id": "start"}, ), ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([first]), tools=[approval_tool([])]) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([first]), tools=[approval_tool([])]) paused = await harness.run("deploy") state = json.loads(json.dumps(paused.resume_state)) state["batch"].append({"id": "call_1", "name": "echo", "arguments": '{"value":"ok"}'}) @@ -658,7 +659,7 @@ async def test_approval_resume_labels_inner_provider_state_errors(tmp_path: Path ), ) resumed = ScriptedSession(start_turn=ModelTurn(raw={"unused": True}), continue_turn=ModelTurn(text="done", raw={"id": "done"})) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([first, resumed]), tools=[approval_tool([])]) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([first, resumed]), tools=[approval_tool([])]) paused = await harness.run("deploy") state = json.loads(json.dumps(paused.resume_state)) state["provider_state"]["kind"] = "wrong" @@ -671,7 +672,7 @@ async def test_approval_resume_labels_builtin_provider_state_errors(tmp_path: Pa client = FakeClient() model = _fake_openai(client) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=model, tools=[ ToolSpec( @@ -700,7 +701,7 @@ async def test_limit_warning_dedup_keys_survive_approval_round_trip(tmp_path: Pa ) resumed = ScriptedSession(start_turn=ModelTurn(raw={"unused": True}), continue_turn=ModelTurn(text="done", raw={"id": "done"})) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], max_tool_calls=1), + HarnessConfig(root=tmp_path, max_tool_calls=1), model=ScriptedModel([first, resumed]), tools=[approval_tool([])], ) @@ -723,23 +724,16 @@ def new_session(self): raise AssertionError("unused") with pytest.raises(ValueError, match="approval-required tools require a resumable model"): - Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=NonResumableModel(), tools=[approval_tool([])]) + Harness(HarnessConfig(root=tmp_path), model=NonResumableModel(), tools=[approval_tool([])]) -def test_subagents_reject_explicit_approval_tools_and_filter_inherited(tmp_path: Path) -> None: - with pytest.raises(ValueError, match="approval-required tools are not supported inside subagents"): +def test_subagents_reject_explicit_approval_tools(tmp_path: Path) -> None: + del tmp_path + with pytest.raises(ValueError, match="approval-required tools are not supported inside child harnesses"): SubAgentConfig(name="helper", description="Helper.", tools=[approval_tool([])]) - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([]), tools=[approval_tool([]), echo_tool([])]) - from thinharness.subagents import build_child_harness - - child = build_child_harness(parent, None) - - assert "deploy" not in {tool.name for tool in child.tools} - assert "echo" in {tool.name for tool in child.tools} - -async def test_inherit_parent_tools_subagent_runs_without_parent_approval_tool(tmp_path: Path) -> None: +async def test_inherited_child_runs_without_parent_approval_tool(tmp_path: Path) -> None: subagent_outputs = [] echo_called: list[dict] = [] child_session = ScriptedSession( @@ -759,12 +753,11 @@ async def test_inherit_parent_tools_subagent_runs_without_parent_approval_tool(t ) model = ScriptedModel([parent_session, child_session]) harness = Harness( - HarnessConfig( - root=tmp_path, - builtin_tools=["subagent"], - subagents=[SubAgentConfig(name="helper", description="Helper.", inherit_parent_tools=True)], - ), + HarnessConfig(root=tmp_path), model=model, + plugins=[SubagentsPlugin(agents=[ + SubAgentConfig(name="helper", description="Helper.", inherit_parent=True) + ])], tools=[approval_tool([]), echo_tool(echo_called)], ) @@ -778,7 +771,7 @@ async def test_inherit_parent_tools_subagent_runs_without_parent_approval_tool(t async def test_resume_approvals_closed_harness_guard(tmp_path: Path) -> None: session = ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"})) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([session])) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session])) await harness.aclose() with pytest.raises(HarnessError, match="harness is closed"): @@ -795,7 +788,7 @@ async def test_streaming_approval_resume_marker_and_request_kind(tmp_path: Path) ), ) resumed = ScriptedSession(start_turn=ModelTurn(raw={"unused": True}), continue_turn=ModelTurn(text="done", raw={"id": "done"})) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([first, resumed]), tools=[approval_tool([])]) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([first, resumed]), tools=[approval_tool([])]) paused = await harness.run("deploy") events = [] @@ -814,10 +807,10 @@ async def test_streaming_approval_resume_marker_and_request_kind(tmp_path: Path) def test_directed_resume_api_errors(tmp_path: Path) -> None: session = ScriptedSession(start_turn=ModelTurn(text="ready", raw={"id": "first"})) - provider_state = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([session])).run_sync("first").resume_state + provider_state = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session])).run_sync("first").resume_state with pytest.raises(HarnessError, match="approval state kind"): - Harness(HarnessConfig(root=tmp_path / "resume", builtin_tools=[]), model=ScriptedModel([])).resume_approvals_sync(provider_state, []) + Harness(HarnessConfig(root=tmp_path / "resume"), model=ScriptedModel([])).resume_approvals_sync(provider_state, []) approval_state = { "kind": "approval_pause", @@ -832,7 +825,7 @@ def test_directed_resume_api_errors(tmp_path: Path) -> None: "metadata": {}, } with pytest.raises(HarnessError, match="approval pause state must be resumed with resume_approvals"): - Harness(HarnessConfig(root=tmp_path / "other", builtin_tools=[]), model=ScriptedModel([])).run_sync("next", resume_from=approval_state) + Harness(HarnessConfig(root=tmp_path / "other"), model=ScriptedModel([])).run_sync("next", resume_from=approval_state) def _codec_envelope(usage: dict | None = None, emitted_limit_warnings: list | None = None) -> dict: @@ -1032,7 +1025,7 @@ class Person(BaseModel): ) model = ScriptedModel([_QueueSession(pause_turn), _QueueSession(bad_final, good_final)]) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool"), + HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool"), model=model, tools=[approval_tool()], ) diff --git a/tests/unit/test_architecture.py b/tests/unit/test_architecture.py index 0031a38..004fc50 100644 --- a/tests/unit/test_architecture.py +++ b/tests/unit/test_architecture.py @@ -17,6 +17,76 @@ def test_core_has_no_mcp_imports_or_lifecycle_state() -> None: assert forbidden not in source +def test_core_has_no_subagent_feature_imports_or_identifiers() -> None: + """Core depends only on the neutral child host and authoritative provenance.""" + root = Path(__file__).resolve().parents[2] + core_path = root / "thinharness" / "core.py" + source = core_path.read_text(encoding="utf-8") + tree = ast.parse(source) + imported_modules = { + node.module + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.module is not None + } + imported_modules.update( + alias.name + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for alias in node.names + ) + names = {node.id for node in ast.walk(tree) if isinstance(node, ast.Name)} + attributes = {node.attr for node in ast.walk(tree) if isinstance(node, ast.Attribute)} + + assert not any(module.endswith(("subagents", "plugins.subagents")) for module in imported_modules) + forbidden_names = { + "SubAgentConfig", + "SubagentsPlugin", + "SubAgentArgs", + "DEFAULT_SUBAGENT_NAME", + "create_subagent_tool", + "build_child_harness", + "_select_builtin_tools", + "builtin_tools", + "subagent_hooks", + } + assert forbidden_names.isdisjoint(names | attributes) + assert not (root / "thinharness" / "subagents.py").exists() + assert "PluginContext(root=self.root, model=self.model, child_harnesses=child_harnesses)" in source + assert "harness=self" not in source.split("PluginContext(", 1)[1].split(")", 1)[0] + + +def test_subagents_plugin_owns_configuration_and_delegation_tool() -> None: + """Feature vocabulary and tool construction stay in the plugin module.""" + root = Path(__file__).resolve().parents[2] + source = (root / "thinharness" / "plugins" / "subagents.py").read_text(encoding="utf-8") + tree = ast.parse(source) + classes = {node.name for node in ast.walk(tree) if isinstance(node, ast.ClassDef)} + string_values = {node.value for node in ast.walk(tree) if isinstance(node, ast.Constant) and isinstance(node.value, str)} + + assert {"SubagentsPlugin", "SubAgentConfig", "SubAgentArgs"} <= classes + assert "subagent" in string_values + assert "register_delegation_tool" in source + + +def test_child_inheritance_and_delegation_detection_are_structural() -> None: + """Automatic inheritance and tracing do not use plugin or tool name guesses.""" + root = Path(__file__).resolve().parents[2] + children_source = (root / "thinharness" / "children.py").read_text(encoding="utf-8") + execution_source = (root / "thinharness" / "tool_execution.py").read_text(encoding="utf-8") + execution_tree = ast.parse(execution_source) + + assert "isinstance(plugin, ChildInheritablePlugin)" in children_source + comparisons = [node for node in ast.walk(execution_tree) if isinstance(node, ast.Compare)] + compared_strings = { + comparator.value + for node in comparisons + for comparator in node.comparators + if isinstance(comparator, ast.Constant) and isinstance(comparator.value, str) + } + assert "subagent" not in compared_strings + assert "composition.delegation" in execution_source + + def test_core_has_no_skills_or_parallel_llm_implementation_details() -> None: """Core stays independent from skills and parallel LLM composition.""" core_path = Path(__file__).resolve().parents[2] / "thinharness" / "core.py" diff --git a/tests/unit/test_bash_tool.py b/tests/unit/test_bash_tool.py index f763876..0d31df2 100644 --- a/tests/unit/test_bash_tool.py +++ b/tests/unit/test_bash_tool.py @@ -131,7 +131,7 @@ def on_continue(outputs, _tools, _metadata) -> None: session = ScriptedSession(start_turn=call, continue_turn=ModelTurn(text="done", raw={"id": "done"}), on_continue=on_continue) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([session]), tools=[BashTool(tmp_path).spec()], ) @@ -146,8 +146,8 @@ def test_bash_is_not_a_builtin_tool(tmp_path: Path) -> None: default = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([])) assert "bash" not in [tool["name"] for tool in default.tool_schemas()] - with pytest.raises(ValueError, match="unknown builtin tool: bash"): - Harness(HarnessConfig(root=tmp_path, builtin_tools=["bash"]), model=ScriptedModel([])) + with pytest.raises(ValueError, match="SubagentsPlugin"): + HarnessConfig(root=tmp_path, builtin_tools=["bash"]) def test_named_subagent_builtin_selector_is_removed() -> None: @@ -158,7 +158,7 @@ def test_named_subagent_builtin_selector_is_removed() -> None: def test_mixed_batch_containing_bash_runs_sequentially(tmp_path: Path) -> None: client = MultiCallClient([("bash", '{"command":"sleep 0.2; printf bash"}'), ("slow", "{}")]) harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[BashTool(tmp_path).spec(), slow_tool("slow", 0.2)], ) diff --git a/tests/unit/test_harness.py b/tests/unit/test_harness.py index d40e069..4ebe78b 100644 --- a/tests/unit/test_harness.py +++ b/tests/unit/test_harness.py @@ -35,13 +35,11 @@ ParallelLlmPlugin, RequestConstants, SkillsPlugin, - SubAgentConfig, + SubagentsPlugin, TokenUsage, ToolSpec, UnexpectedModelBehavior, - build_child_harness, call_tool, - create_subagent_tool, ) from thinharness.core import _classify_run_failure from thinharness.defaults import DEFAULT_PARALLEL_LLM_INSTRUCTIONS, DEFAULT_SEARCH_INSTRUCTIONS @@ -72,7 +70,7 @@ def test_session_receives_falsy_metadata_when_run_has_no_metadata(tmp_path: Path start_turn=ModelTurn(text="done", raw={"id": "done"}), on_start=lambda _prompt, _instructions, _tools, metadata, _previous: captured.setdefault("metadata", metadata), ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([session])) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session])) assert harness.run_sync("go").text == "done" @@ -132,7 +130,7 @@ def test_classify_run_failure_preserves_existing_harness_stop_reason() -> None: def test_max_model_requests_zero_blocks_before_provider_request(tmp_path: Path) -> None: session = ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"})) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], max_model_requests=0), model=ScriptedModel([session])) + harness = Harness(HarnessConfig(root=tmp_path, max_model_requests=0), model=ScriptedModel([session])) with pytest.raises(HarnessError, match="max_model_requests=0"): harness.run_sync("go") @@ -141,7 +139,7 @@ def test_max_model_requests_zero_blocks_before_provider_request(tmp_path: Path) def test_final_model_request_notice_is_sent_on_initial_request(tmp_path: Path) -> None: session = ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"})) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], max_model_requests=1), model=ScriptedModel([session])) + harness = Harness(HarnessConfig(root=tmp_path, max_model_requests=1), model=ScriptedModel([session])) assert harness.run_sync("go").text == "done" @@ -153,7 +151,7 @@ def test_warning_only_run_does_not_fire_limit_reached(tmp_path: Path) -> None: events = [] session = ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"})) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], max_model_requests=1, max_tool_calls=0), + HarnessConfig(root=tmp_path, max_model_requests=1, max_tool_calls=0), model=ScriptedModel([session]), hooks=[Hook("limit_reached", lambda ctx: events.append(ctx.limit_kind))], ) @@ -172,7 +170,7 @@ def test_limit_notices_are_sent_on_tool_continuation(tmp_path: Path) -> None: continue_turn=ModelTurn(text="done", raw={"id": "done"}), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], max_model_requests=2, max_tool_calls=1), + HarnessConfig(root=tmp_path, max_model_requests=2, max_tool_calls=1), model=ScriptedModel([session]), tools=[echo_tool()], ) @@ -186,7 +184,7 @@ def test_limit_notices_are_sent_on_tool_continuation(tmp_path: Path) -> None: def test_exhausted_tool_budget_notice_is_sent_on_initial_request(tmp_path: Path) -> None: session = ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"})) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], max_tool_calls=0), model=ScriptedModel([session])) + harness = Harness(HarnessConfig(root=tmp_path, max_tool_calls=0), model=ScriptedModel([session])) assert harness.run_sync("go").text == "done" @@ -197,7 +195,7 @@ def test_exhausted_tool_budget_notice_is_sent_on_initial_request(tmp_path: Path) def test_combined_limit_notices_use_stable_order(tmp_path: Path, max_tool_calls: int, expected_remaining: int) -> None: session = ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"})) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], max_model_requests=1, max_tool_calls=max_tool_calls), + HarnessConfig(root=tmp_path, max_model_requests=1, max_tool_calls=max_tool_calls), model=ScriptedModel([session]), ) @@ -216,7 +214,7 @@ def test_same_turn_tool_overage_does_not_send_continuation_notice(tmp_path: Path ], raw={"id": "start"}) ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], max_tool_calls=1), + HarnessConfig(root=tmp_path, max_tool_calls=1), model=ScriptedModel([session]), tools=[echo_tool()], ) @@ -229,7 +227,7 @@ def test_same_turn_tool_overage_does_not_send_continuation_notice(tmp_path: Path async def test_anthropic_harness_reuses_model_without_message_leak(tmp_path: Path) -> None: provider = FakeAnthropicProvider() model = AnthropicMessagesModel("claude-test", provider=provider) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model, tools=[echo_tool()]) + harness = Harness(HarnessConfig(root=tmp_path), model=model, tools=[echo_tool()]) assert (await harness.run("first")).text == "done" assert (await harness.run("second")).text == "done" @@ -240,7 +238,7 @@ async def test_anthropic_harness_reuses_model_without_message_leak(tmp_path: Pat async def test_openrouter_harness_reuses_model_without_message_leak(tmp_path: Path) -> None: provider = FakeOpenRouterProvider() model = OpenRouterModel("openai/test", provider=provider) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model, tools=[echo_tool()]) + harness = Harness(HarnessConfig(root=tmp_path), model=model, tools=[echo_tool()]) assert (await harness.run("first")).text == "done" assert (await harness.run("second")).text == "done" @@ -311,9 +309,9 @@ def test_harness_has_no_implicit_filesystem_tools(tmp_path: Path) -> None: def test_specialized_filesystem_tools_are_explicit_opt_ins(tmp_path: Path) -> None: harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=["subagent"]), + HarnessConfig(root=tmp_path), model=_fake_openai(FakeClient()), - plugins=[FilesystemPlugin(tools=["jsonl_search"])], + plugins=[FilesystemPlugin(tools=["jsonl_search"]), SubagentsPlugin()], ) assert [tool["name"] for tool in harness.tool_schemas()] == ["jsonl_search", "subagent"] @@ -370,7 +368,7 @@ def test_blank_tool_instructions_are_omitted(tmp_path: Path) -> None: lambda args: "ok", instructions=" ", ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([]), tools=[custom]) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), tools=[custom]) assert harness.system_instructions() == harness.config.system_prompt @@ -382,7 +380,7 @@ def test_tool_instructions_do_not_change_tool_schema(tmp_path: Path) -> None: lambda args: {"echo": args["value"]}, instructions="Use echo_json only when echoing JSON.", ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([]), tools=[custom]) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), tools=[custom]) schema = harness.tool_schemas()[0] @@ -461,34 +459,10 @@ def test_removed_harness_skills_argument_fails_loudly() -> None: Harness(HarnessConfig(), skills=object()) -@pytest.mark.parametrize( - ("name", "plugin"), - [("skill_read", "SkillsPlugin"), ("skill_run", "SkillsPlugin"), ("parallel_llm", "ParallelLlmPlugin")], -) -def test_removed_builtin_tool_names_point_to_plugins(name: str, plugin: str) -> None: - with pytest.raises(ValueError, match=plugin): - Harness(HarnessConfig(builtin_tools=[name]), model=ScriptedModel([])) - - -def test_unknown_builtin_tool_keeps_normal_error() -> None: - with pytest.raises(ValueError, match=r"unknown builtin tool: unknown; available: subagent"): - Harness(HarnessConfig(builtin_tools=["unknown"]), model=ScriptedModel([])) - -def test_child_harness_tool_surfaces_follow_subagent_policy(tmp_path: Path) -> None: - parent_echo = echo_tool() - explicit_tool = ToolSpec("explicit", "Explicit sequential tool", {"type": "object", "properties": {}}, lambda args: "ok", sequential=True) - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([]), tools=[parent_echo]) - parent.add_tool(create_subagent_tool(parent, [])) - - default_child = build_child_harness(parent, None) - explicit_child = build_child_harness(parent, SubAgentConfig(name="special", description="Special helper.", tools=[explicit_tool])) - - assert default_child.tools == [parent_echo] - assert default_child.config.subagents == [] - assert [tool.name for tool in explicit_child.tools] == ["explicit"] - assert explicit_child.tools[0].sequential is True - assert explicit_child.config.subagents == [] - assert build_child_harness(parent, None).model is parent.model +@pytest.mark.parametrize("field", ["builtin_tools", "subagents"]) +def test_removed_delegation_config_fields_point_to_plugin(field: str) -> None: + with pytest.raises(ValueError, match=rf"HarnessConfig\.{field}.*SubagentsPlugin"): + HarnessConfig(**{field: []}) def test_duplicate_tool_names_are_rejected(tmp_path: Path) -> None: duplicate = ToolSpec("read", "Duplicate read", {"type": "object", "properties": {}}, lambda args: "ok") @@ -512,7 +486,7 @@ def after(ctx): seen.append(ctx.envelope.metadata["error_type"]) harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ToolSpec("boom", "boom", {"type": "object", "properties": {}}, boom)], hooks=[Hook("after_tool_call", after)], @@ -529,7 +503,7 @@ async def async_echo(args): return args["value"] harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ ToolSpec( @@ -561,7 +535,7 @@ async def async_partial(args, *, value): loop_thread = threading.get_ident() harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ToolSpec("async_partial", "Async partial", {"type": "object", "properties": {}}, partial(async_partial, value="ok"))], ) @@ -584,7 +558,7 @@ async def __call__(self, args): return "ok" harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ToolSpec("callable_async", "Callable async", {"type": "object", "properties": {}}, CallableAsync())], ) @@ -637,7 +611,7 @@ def sync_ctx(args): return "sync" harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ ToolSpec("async_ctx", "Async ctx", {"type": "object", "properties": {}}, async_ctx), @@ -651,7 +625,7 @@ def sync_ctx(args): async def test_run_sync_inside_running_loop_raises(tmp_path: Path) -> None: harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"}))]), ) @@ -672,7 +646,7 @@ async def aclose(self) -> None: model = ScriptedModel([ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"}))]) model.provider = provider - async with Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model, _owns_model=True) as harness: + async with Harness(HarnessConfig(root=tmp_path), model=model, _owns_model=True) as harness: assert (await harness.run("go")).text == "done" await harness.aclose() @@ -682,7 +656,7 @@ async def test_injected_http_client_is_not_closed_by_harness(tmp_path: Path) -> client = httpx.AsyncClient(transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"id": "resp", "output_text": "done"}))) provider = OpenAIProvider(api_key="key", http_client=client) model = OpenAIResponsesModel("gpt-test", provider=provider) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model, _owns_model=True) + harness = Harness(HarnessConfig(root=tmp_path), model=model, _owns_model=True) assert (await harness.run("go")).text == "done" await harness.aclose() @@ -710,7 +684,7 @@ async def continue_with_user_text(self, text, constants, *, notices=None): ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"})), ]) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=model, hooks=[Hook("run_end", lambda ctx: events.append((ctx.stop_reason, type(ctx.error).__name__)))], ) @@ -734,7 +708,7 @@ async def wait(_args): await asyncio.Event().wait() harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ToolSpec("wait", "Wait", {"type": "object", "properties": {}}, wait)], hooks=[Hook("after_tool_call", lambda ctx: after_calls.append(ctx.tool_name))], @@ -772,7 +746,7 @@ def register(_args): return "registered" harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([session]), tools=[ToolSpec("register", "Register a tool mid-run", {"type": "object", "properties": {}}, register)], ) @@ -816,7 +790,7 @@ def register(_args): ModelTurn(text="done again", raw={"id": "five"}), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([first_run, second_run]), tools=[ToolSpec("register", "Register a tool mid-run", {"type": "object", "properties": {}}, register)], ) @@ -843,7 +817,7 @@ def test_run_usage_token_totals_accumulate_partial_usage(tmp_path: Path) -> None ), continue_turn=ModelTurn(text="done", raw={"id": "done"}, usage=TokenUsage(output_tokens=7, cached_tokens=2)), ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([session]), tools=[echo_tool()]) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session]), tools=[echo_tool()]) result = harness.run_sync("go") diff --git a/tests/unit/test_hooks.py b/tests/unit/test_hooks.py index 508a16c..e3b3dce 100644 --- a/tests/unit/test_hooks.py +++ b/tests/unit/test_hooks.py @@ -27,10 +27,10 @@ LimitReachedContext, RunEndContext, RunStartContext, + SubagentsPlugin, ToolResult, ToolSpec, UserPromptSubmitContext, - create_subagent_tool, ) from thinharness.hooks import current_tool_runtime_context from thinharness.providers import ModelToolCall, ModelTurn, ProviderError @@ -53,7 +53,7 @@ def test_hook_filter_warnings_wait_for_constructor_tools(tmp_path: Path, caplog) hook = Hook("before_tool_call", lambda ctx: None, tools=["second"]) Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([]), tools=[ ToolSpec("first", "first", {"type": "object", "properties": {}}, lambda args: "first"), @@ -76,7 +76,7 @@ def on_end(ctx): events.append((ctx.stop_reason, type(ctx.error).__name__, ctx.usage.model_requests)) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=BrokenModel([]), hooks=[Hook("run_end", on_end)], ) @@ -96,7 +96,7 @@ def new_session(self): raise provider_error harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=BrokenProviderModel([]), hooks=[Hook("run_end", lambda ctx: events.append((ctx.stop_reason, type(ctx.error).__name__)))], ) @@ -111,7 +111,7 @@ def new_session(self): def test_run_end_fires_for_provider_and_unexpected_errors(tmp_path: Path) -> None: events = [] provider_harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([FailingSession()]), hooks=[Hook("run_end", lambda ctx: events.append((ctx.stop_reason, type(ctx.error).__name__)))], ) @@ -121,7 +121,7 @@ def test_run_end_fires_for_provider_and_unexpected_errors(tmp_path: Path) -> Non unexpected = ScriptedSession(start_turn=ModelTurn(), on_start=lambda *_args: (_ for _ in ()).throw(ValueError("boom"))) error_harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([unexpected]), hooks=[Hook("run_end", lambda ctx: events.append((ctx.stop_reason, type(ctx.error).__name__)))], ) @@ -141,7 +141,7 @@ def fail_once(ctx): raise RuntimeError("end hook failed") harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], strict_hooks=True), + HarnessConfig(root=tmp_path, strict_hooks=True), model=ScriptedModel([ ScriptedSession(start_turn=ModelTurn(text="first", raw={"id": "first"})), ScriptedSession(start_turn=ModelTurn(text="second", raw={"id": "second"})), @@ -171,7 +171,7 @@ def add_context(ctx): Hook("run_end", lambda ctx: events.append((ctx.event, isinstance(ctx, RunEndContext), ctx.result.usage.model_requests))), ] session = ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"}), on_start=on_start) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([session]), hooks=hooks) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session]), hooks=hooks) result = harness.run_sync("summarize") @@ -199,7 +199,7 @@ def on_run_end(ctx): on_start=lambda *_args: pytest.fail("model should not be called"), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([session]), hooks=[Hook("user_prompt_submit", cancel), Hook("run_end", on_run_end)], ) @@ -212,7 +212,7 @@ def on_run_end(ctx): def test_same_harness_reentrant_run_is_rejected(tmp_path: Path) -> None: captured = [] harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"}))]), ) @@ -243,7 +243,7 @@ def after(ctx): ctx.output = json.dumps({"ok": True, "content": "rewritten", "metadata": {}}) harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ ToolSpec("block", "blocked", {"type": "object", "properties": {}}, lambda args: "bad"), @@ -281,7 +281,7 @@ def after(ctx): seen.append(("after", dict(ctx.metadata))) harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ToolSpec("ok", "ok", {"type": "object", "properties": {}}, lambda args: "ok")], hooks=[Hook("before_tool_call", before), Hook("after_tool_call", after)], @@ -306,7 +306,7 @@ def observe(ctx): seen.append(ctx.envelope) harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ToolSpec("ok", "ok", {"type": "object", "properties": {}}, lambda args: "original")], hooks=[Hook("after_tool_call", rewrite), Hook("after_tool_call", observe)], @@ -326,7 +326,7 @@ def rewrite(ctx): ctx.envelope.metadata["stage"] = 1 harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ToolSpec("ok", "ok", {"type": "object", "properties": {}}, lambda args: "original")], hooks=[Hook("after_tool_call", rewrite)], @@ -348,7 +348,7 @@ def fail(ctx): raise RuntimeError("after failed") harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[], strict_hooks=True), + HarnessConfig(root=tmp_path, model="openai:test-model", strict_hooks=True), model=_fake_openai(client), tools=[ToolSpec("ok", "ok", {"type": "object", "properties": {}}, lambda args: "ok")], hooks=[Hook("after_tool_call", fail)], @@ -385,7 +385,7 @@ def fail_for_b(ctx): raise RuntimeError("strict hook failed") harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[], strict_hooks=True), + HarnessConfig(root=tmp_path, model="openai:test-model", strict_hooks=True), model=_fake_openai(client), tools=[slow_tool("a", 0.01), slow_tool("b", 0.01)], hooks=[Hook("before_tool_call", fail_for_b)], @@ -408,7 +408,7 @@ def on_run_end(ctx): run_end_usage.append((ctx.stop_reason, ctx.usage.tool_calls, ctx.usage.cancelled_tool_calls)) harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[], strict_hooks=True), + HarnessConfig(root=tmp_path, model="openai:test-model", strict_hooks=True), model=_fake_openai(client), tools=[slow_tool("a", 0.01), slow_tool("b", 0.01)], hooks=[Hook("before_tool_call", fail_for_a), Hook("run_end", on_run_end)], @@ -435,7 +435,7 @@ def fail_for_fail(ctx): raise RuntimeError("strict hook failed") harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[], strict_hooks=True), + HarnessConfig(root=tmp_path, model="openai:test-model", strict_hooks=True), model=_fake_openai(client), tools=[ ToolSpec("wait", "wait", {"type": "object", "properties": {}}, wait), @@ -453,7 +453,7 @@ def fail_for_fail(ctx): def test_explicit_hook_registry_strict_mode_is_preserved(tmp_path: Path) -> None: registry = HookRegistry([Hook("user_prompt_submit", lambda ctx: (_ for _ in ()).throw(RuntimeError("strict registry")))], strict_hooks=True) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], strict_hooks=False), + HarnessConfig(root=tmp_path, strict_hooks=False), model=ScriptedModel([ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"}))]), hooks=registry, ) @@ -475,7 +475,7 @@ async def continue_with_user_text(self, text, constants, *, notices=None): raise HarnessError("bare harness error") harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=ScriptedModel([BareHarnessErrorSession()]), tools=[ToolSpec("ok", "ok", {"type": "object", "properties": {}}, lambda args: "ok")], hooks=[Hook("run_end", lambda ctx: events.append((ctx.stop_reason, type(ctx.error).__name__)))], @@ -499,7 +499,7 @@ def on_end(ctx): events.append((ctx.event, ctx.stop_reason, ctx.usage.tool_calls)) harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[], max_tool_calls=2), + HarnessConfig(root=tmp_path, model="openai:test-model", max_tool_calls=2), model=_fake_openai(client), tools=[slow_tool("a", 0), slow_tool("b", 0), slow_tool("c", 0)], hooks=[Hook("limit_reached", on_limit), Hook("run_end", on_end)], @@ -513,14 +513,14 @@ def on_end(ctx): def test_max_model_requests_limits_provider_continuations(tmp_path: Path) -> None: immediate = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], max_model_requests=1), + HarnessConfig(root=tmp_path, max_model_requests=1), model=ScriptedModel([ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"}))]), ) assert immediate.run_sync("go").usage.model_requests == 1 client = MultiCallClient([("ok", "{}")]) limited = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[], max_model_requests=1), + HarnessConfig(root=tmp_path, model="openai:test-model", max_model_requests=1), model=_fake_openai(client), tools=[ToolSpec("ok", "ok", {"type": "object", "properties": {}}, lambda args: "ok")], ) @@ -529,7 +529,7 @@ def test_max_model_requests_limits_provider_continuations(tmp_path: Path) -> Non allowed_client = MultiCallClient([("ok", "{}")]) allowed = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[], max_model_requests=2), + HarnessConfig(root=tmp_path, model="openai:test-model", max_model_requests=2), model=_fake_openai(allowed_client), tools=[ToolSpec("ok", "ok", {"type": "object", "properties": {}}, lambda args: "ok")], ) @@ -546,11 +546,11 @@ def fail(ctx): raise RuntimeError("strict subagent hook failed") harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], strict_hooks=True), + HarnessConfig(root=tmp_path, strict_hooks=True), model=ScriptedModel([parent]), + plugins=[SubagentsPlugin()], hooks=[Hook("before_subagent_run", fail)], ) - harness.add_tool(create_subagent_tool(harness, [])) with pytest.raises(RuntimeError, match="strict subagent hook failed"): harness.run_sync("delegate") diff --git a/tests/unit/test_mcp.py b/tests/unit/test_mcp.py index 6f4e291..3174293 100644 --- a/tests/unit/test_mcp.py +++ b/tests/unit/test_mcp.py @@ -30,9 +30,9 @@ PluginBinding, PluginContribution, SubAgentConfig, + SubagentsPlugin, ToolOrigin, TracingOptions, - build_child_harness, ) from thinharness.providers import ModelToolCall, ToolOutput from thinharness.tools.base import Json, ToolResult, ToolSpec @@ -519,7 +519,7 @@ def make_backend(tool_name: str) -> FastMCP: first = MCPServer(FastMCPTransport(make_backend("one"))) second = MCPServer(FastMCPTransport(make_backend("two"))) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[first, second])], model=_fake_openai(MultiCallClient([]))) + harness = Harness(HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[first, second])], model=_fake_openai(MultiCallClient([]))) await harness.connect() await harness.aclose() @@ -583,7 +583,7 @@ def hidden() -> str: backend.tool(hidden) server = MCPServer(FastMCPTransport(backend), id="semley", include_tools=["step", "reset_session"]) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([("step", '{"action":"go"}')])), ) @@ -901,7 +901,7 @@ async def test_harness_connects_mcp_once_across_async_runs(tmp_path, monkeypatch """Harness runs reuse the discovered MCP tools until aclose.""" server = scripted_server(monkeypatch, {"remote": _schema()}) client = MultiCallClient([("remote", '{"value":"ok"}')]) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[server])], model=_fake_openai(client)) + harness = Harness(HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[server])], model=_fake_openai(client)) assert harness.tools == [] result = await harness.run("go") @@ -969,7 +969,7 @@ def test_mcp_plugin_name_is_unique_and_config_path_is_removed(tmp_path, monkeypa with pytest.raises(ValueError, match="duplicate plugin name: mcp"): Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[first]), MCPPlugin(servers=[second])], model=ScriptedModel([]), ) @@ -979,7 +979,7 @@ def test_mcp_plugin_name_is_unique_and_config_path_is_removed(tmp_path, monkeypa async def test_empty_mcp_plugin_connects_without_tools(tmp_path) -> None: """An empty MCP plugin is a valid connected contribution.""" harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[])], model=ScriptedModel([]), ) @@ -992,7 +992,7 @@ async def test_empty_mcp_plugin_connects_without_tools(tmp_path) -> None: async def test_explicit_connect_does_not_reconnect_on_run(tmp_path, monkeypatch) -> None: """Explicit connect discovers MCP tools once before run.""" server = scripted_server(monkeypatch, {"remote": _schema()}) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([]))) + harness = Harness(HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([]))) await harness.connect() await harness.connect() @@ -1015,7 +1015,7 @@ async def test_is_error_drives_harness_retry(tmp_path, monkeypatch) -> None: ModelTurn(tool_calls=[ModelToolCall(id="call_2", name="error", arguments="{}")], raw={"id": "retry"}), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], tool_retries=1), + HarnessConfig(root=tmp_path, tool_retries=1), plugins=[MCPPlugin(servers=[server])], model=ScriptedModel([session]), hooks=[Hook("run_end", lambda ctx: run_end.append((ctx.stop_reason, dict(ctx.usage.tool_retries))))], @@ -1045,7 +1045,7 @@ async def __aenter__(self) -> MCPServer: scripted_second = scripted_server(monkeypatch, {"second": _schema()}, id="second") second = FailOnceEnterServer(scripted_second._transport, id="second") harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[first, second])], model=ScriptedModel([]), ) @@ -1079,7 +1079,7 @@ async def list_tools(self, *, server_id: str | None = None) -> list[ToolSpec]: backend = FastMCP("failing-backend") backend.tool(_echo_handler("recovered", []), name="recovered") second = FailOnceListServer(FastMCPTransport(backend), id="failing") - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[first, second])], model=_fake_openai(MultiCallClient([]))) + harness = Harness(HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[first, second])], model=_fake_openai(MultiCallClient([]))) with pytest.raises(MCPError, match="list failed"): await harness.connect() @@ -1121,7 +1121,7 @@ async def __aexit__(self, *exc: object) -> None: backend.tool(_echo_handler("recovered", []), name="recovered") server = FailingDiscoverySlowCleanupServer(FastMCPTransport(backend), id="slow-cleanup") harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[server])], model=ScriptedModel([]), ) @@ -1146,7 +1146,7 @@ async def test_direct_tool_collision_rolls_back_mcp(tmp_path, monkeypatch) -> No server = scripted_server(monkeypatch, {"shared": _schema()}) direct = ToolSpec("shared", "Direct", _schema(), lambda _args: "direct") harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[server])], tools=[direct], model=ScriptedModel([]), @@ -1164,7 +1164,7 @@ async def test_mcp_server_tool_collision_rolls_back_all_servers(tmp_path, monkey first = scripted_server(monkeypatch, {"shared": _schema()}, id="first") second = scripted_server(monkeypatch, {"shared": _schema()}, id="second") harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[first, second])], model=ScriptedModel([]), ) @@ -1203,7 +1203,7 @@ class Answer(BaseModel): server = scripted_server(monkeypatch, {"final_result": _schema()}) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Answer, output_mode="tool"), + HarnessConfig(root=tmp_path, output_type=Answer, output_mode="tool"), plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([])), ) @@ -1216,7 +1216,7 @@ async def test_duplicate_derived_id_disambiguated(tmp_path, monkeypatch) -> None """Duplicate MCP server ids get readable suffixes.""" first = scripted_server(monkeypatch, {"one": _schema()}, id="same") second = scripted_server(monkeypatch, {"two": _schema()}, id="same") - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[first, second])], model=_fake_openai(MultiCallClient([]))) + harness = Harness(HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[first, second])], model=_fake_openai(MultiCallClient([]))) await harness.connect() await harness.aclose() @@ -1236,13 +1236,13 @@ async def test_binding_local_ids_stay_stable_for_shared_server(tmp_path, monkeyp first_tracer = FakeTracer() second_tracer = FakeTracer() first = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[shared, first_neighbor])], model=_fake_openai(MultiCallClient([("shared", '{"value":"first"}')])), tracing=[TracingOptions(tracer=first_tracer)], ) second = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[second_neighbor, shared])], model=_fake_openai(MultiCallClient([("shared", '{"value":"second"}')])), tracing=[TracingOptions(tracer=second_tracer)], @@ -1292,7 +1292,7 @@ async def list_tools(self, *, server_id: str | None = None) -> list[ToolSpec]: backend.tool(_echo_handler("remote", []), name="remote") server = CancelOnceListServer(FastMCPTransport(backend), id="cancel") harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[server])], model=ScriptedModel([]), ) @@ -1324,7 +1324,7 @@ async def slow_stop(server: Any): backend.tool(_echo_handler("remote", []), name="remote") server = MCPServer(FastMCPTransport(backend), id="slow-stop") harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[server])], model=ScriptedModel([]), ) @@ -1368,7 +1368,7 @@ async def connect(): backend.tool(_echo_handler("remote", []), name="remote") server = LoggingServer(FastMCPTransport(backend), id="close-order") harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[server]), OtherPlugin()], model=ScriptedModel([]), ) @@ -1383,7 +1383,7 @@ async def connect(): async def test_closed_harness_rejects_run_and_connect_but_keeps_schema(tmp_path, monkeypatch) -> None: """Closed harnesses are terminal but still inspectable.""" server = scripted_server(monkeypatch, {"remote": _schema()}) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([]))) + harness = Harness(HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([]))) await harness.connect() await harness.aclose() @@ -1399,7 +1399,7 @@ async def test_closed_harness_rejects_run_and_connect_but_keeps_schema(tmp_path, def test_run_sync_is_one_shot(tmp_path) -> None: """run_sync closes the harness after one call.""" harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"}))]), ) @@ -1412,7 +1412,7 @@ async def test_aclose_with_injected_model_closes_mcp(tmp_path, monkeypatch) -> N """Harness-owned MCP resources close even when the model is injected.""" server = scripted_server(monkeypatch, {"remote": _schema()}) model = _fake_openai(MultiCallClient([])) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[server])], model=model) + harness = Harness(HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[server])], model=model) await harness.connect() await harness.aclose() @@ -1424,7 +1424,7 @@ async def test_unknown_tool_hook_filter_is_allowed_and_never_fires(tmp_path) -> """Tool hook filters are passive when a tool name is never registered.""" seen = [] harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=_fake_openai(MultiCallClient([])), hooks=[Hook("before_tool_call", lambda ctx: seen.append(ctx.tool_name), tools=["missing"])], ) @@ -1434,191 +1434,68 @@ async def test_unknown_tool_hook_filter_is_allowed_and_never_fires(tmp_path) -> assert seen == [] -def test_default_subagent_does_not_implicitly_inherit_mcp(tmp_path, monkeypatch) -> None: - """MCP inheritance for child harnesses is explicit.""" +async def test_default_child_does_not_inherit_parent_mcp(tmp_path, monkeypatch) -> None: + """A default child receives no parent MCP binding.""" server = scripted_server(monkeypatch, {"remote": _schema()}) - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[server])], model=ScriptedModel([])) - - child = build_child_harness(parent, None) - - assert not any(isinstance(plugin, MCPPlugin) for plugin in child.plugins) - + seen = {} -def test_subagent_empty_fails_validation() -> None: - """Named subagents must still expose some tool source.""" - with pytest.raises(ValueError, match="named subagents"): - SubAgentConfig(name="empty", description="Empty helper.") + def child_start(_prompt, _instructions, tools, _metadata, _previous_response_id): + seen["tools"] = [tool["name"] for tool in tools] - -def test_subagent_mcp_override_and_union_config(tmp_path, monkeypatch) -> None: - """Child config encodes MCP override and union semantics.""" - parent_server = scripted_server(monkeypatch, {"parent": _schema()}) - child_server = scripted_server(monkeypatch, {"child": _schema()}) - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[parent_server])], model=ScriptedModel([])) - - override = build_child_harness(parent, SubAgentConfig(name="override", description="Override helper.", mcp_servers=[child_server])) - union = build_child_harness( - parent, - SubAgentConfig( - name="union", - description="Union helper.", - inherit_mcp_servers=True, - mcp_servers=[parent_server, child_server], + parent_session = ScriptedSession( + start_turn=ModelTurn( + tool_calls=[ModelToolCall(id="delegate", name="subagent", arguments='{"task":"help"}')], + raw={}, ), + continue_turn=ModelTurn(text="done", raw={}), ) - - override_plugin = next(plugin for plugin in override.plugins if isinstance(plugin, MCPPlugin)) - union_plugin = next(plugin for plugin in union.plugins if isinstance(plugin, MCPPlugin)) - assert override_plugin.servers == (child_server,) - assert union_plugin.servers == (parent_server, child_server) - - -async def test_subagent_overrides_mcp_only_runtime(tmp_path, monkeypatch) -> None: - """An override-only child sees explicit MCP servers but not parent MCP servers.""" - parent_server = scripted_server(monkeypatch, {"parent": _schema()}) - child_server = scripted_server(monkeypatch, {"child": _schema()}) - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[parent_server])], model=ScriptedModel([])) - - child = build_child_harness(parent, SubAgentConfig(name="override", description="Override helper.", mcp_servers=[child_server])) - await child.connect() - await child.aclose() - - assert [tool.name for tool in child.tools] == ["child"] - - -async def test_subagent_unions_inherit_plus_override_runtime(tmp_path, monkeypatch) -> None: - """An inherited-plus-override child sees both MCP tool sets.""" - parent_server = scripted_server(monkeypatch, {"parent": _schema()}) - child_server = scripted_server(monkeypatch, {"child": _schema()}) - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[parent_server])], model=ScriptedModel([])) - - child = build_child_harness( - parent, - SubAgentConfig( - name="union", - description="Union helper.", - inherit_mcp_servers=True, - mcp_servers=[child_server], - ), + child_session = ScriptedSession( + start_turn=ModelTurn(text="child", raw={}), + on_start=child_start, ) - await child.connect() - await child.aclose() - - assert [tool.name for tool in child.tools] == ["parent", "child"] - - -async def test_subagent_identity_dedup_runtime(tmp_path, monkeypatch) -> None: - """The same inherited and explicit MCP object is entered once in a child.""" - server = scripted_server(monkeypatch, {"remote": _schema()}) - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[server])], model=ScriptedModel([])) - - child = build_child_harness( - parent, - SubAgentConfig( - name="dedup", - description="Dedup helper.", - inherit_mcp_servers=True, - mcp_servers=[server], - ), - ) - await child.connect() - await child.aclose() - - assert [tool.name for tool in child.tools] == ["remote"] - child_plugin = next(plugin for plugin in child.plugins if isinstance(plugin, MCPPlugin)) - assert child_plugin.servers == (server,) - - -async def test_subagent_id_equal_but_distinct_collides(tmp_path, monkeypatch) -> None: - """Distinct MCP objects with identical tools collide in child connect.""" - first = scripted_server(monkeypatch, {"remote": _schema()}, id="same") - second = scripted_server(monkeypatch, {"remote": _schema()}, id="same") - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[first])], model=ScriptedModel([])) - child = build_child_harness( - parent, - SubAgentConfig( - name="child", - description="Child helper.", - inherit_mcp_servers=True, - mcp_servers=[second], - ), - ) - - with pytest.raises(ValueError, match="duplicate tool name: remote"): - await child.connect() - - -async def test_child_second_server_failure_keeps_parent_shared_session_live(tmp_path) -> None: - """Child rollback releases only its reference to an inherited parent server.""" - shared = observed_server("remote", id="shared") - failing = FailingListServer(FastMCPTransport(FastMCP("child-failing")), id="failing") parent = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), - plugins=[MCPPlugin(servers=[shared])], - model=ScriptedModel([]), - ) - await parent.connect() - child = build_child_harness( - parent, - SubAgentConfig( - name="child", - description="Child helper.", - inherit_mcp_servers=True, - mcp_servers=[failing], - ), + HarnessConfig(root=tmp_path), + plugins=[MCPPlugin(servers=[server]), SubagentsPlugin()], + model=ScriptedModel([parent_session, child_session]), ) - with pytest.raises(MCPError, match="list failed"): - await child.connect() - - assert shared.backend_log == {"starts": 1, "stops": 0} - parent_tool = next(tool for tool in parent.tools if tool.name == "remote") - result = await parent_tool.handler({"value": "still-live"}) - assert result.ok is True - assert shared.backend_log == {"starts": 1, "stops": 0} - await child.aclose() + assert (await parent.run("delegate")).text == "done" await parent.aclose() - assert shared.backend_log == {"starts": 1, "stops": 1} + assert seen["tools"] == [] -async def test_subagent_inherited_parent_tools_skip_mcp_duplicates(tmp_path, monkeypatch) -> None: - """Parent MCP tools are not copied as custom tools when also inherited as MCP.""" - server = scripted_server(monkeypatch, {"remote": _schema()}) - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[server])], model=ScriptedModel([])) - await parent.connect() - - child = build_child_harness( - parent, - SubAgentConfig( - name="child", - description="Child helper.", - inherit_parent_tools=True, - inherit_mcp_servers=True, +async def test_named_child_uses_explicit_mcp_plugin(tmp_path, monkeypatch) -> None: + """A named child connects and closes its explicit MCP binding.""" + parent_server = scripted_server(monkeypatch, {"parent": _schema()}) + child_server = scripted_server(monkeypatch, {"child": _schema()}) + seen_tools: list[list[str]] = [] + config = SubAgentConfig( + name="mcp", + description="MCP helper.", + plugins=[MCPPlugin(servers=[child_server])], + ) + parent_session = ScriptedSession( + start_turn=ModelTurn( + tool_calls=[ModelToolCall(id="delegate", name="subagent", arguments='{"task":"help","agent":"mcp"}')], + raw={}, ), + continue_turn=ModelTurn(text="done", raw={}), ) - await child.connect() - await child.aclose() - await parent.aclose() - - assert [tool.name for tool in child.tools] == ["remote"] - - -async def test_subagent_mcp_only_validates_and_inherits(tmp_path, monkeypatch) -> None: - """A named subagent can get its only tools from inherited MCP servers.""" - server = scripted_server(monkeypatch, {"remote": _schema()}) - config = SubAgentConfig(name="mcp", description="MCP helper.", inherit_mcp_servers=True) - parent_client = MultiCallClient([("subagent", '{"task":"use remote","agent":"mcp"}')]) + child_session = ScriptedSession(start_turn=ModelTurn(text="child", raw={})) parent = Harness( - HarnessConfig(root=tmp_path, builtin_tools=["subagent"], subagents=[config]), - plugins=[MCPPlugin(servers=[server])], - model=_fake_openai(parent_client), + HarnessConfig(root=tmp_path), + plugins=[ + MCPPlugin(servers=[parent_server]), + SubagentsPlugin(agents=[config]), + ], + model=ScriptedModel([parent_session, child_session]), + hooks=[Hook("after_subagent_run", lambda ctx: seen_tools.append(ctx.tools), agents=["mcp"])], ) - result = await parent.run("delegate") + assert (await parent.run("delegate")).text == "done" await parent.aclose() - - assert result.text == "done" - assert server.list_calls == 2 + assert seen_tools == [["child"]] + assert child_server.exited == 2 async def test_run_teardown_after_tool_exception_closes_mcp(tmp_path, monkeypatch) -> None: @@ -1629,7 +1506,7 @@ def boom(_args): raise RuntimeError("boom") harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([("boom", "{}")])), tools=[ToolSpec("boom", "Boom", {"type": "object", "properties": {}}, boom)], @@ -1651,7 +1528,7 @@ async def slow(_args): await asyncio.sleep(60) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([("slow", "{}")])), tools=[ToolSpec("slow", "Slow", {"type": "object", "properties": {}}, slow)], @@ -1666,31 +1543,13 @@ async def slow(_args): assert server.exited == 2 -async def test_subagent_effective_tools_include_mcp(tmp_path, monkeypatch) -> None: - """After-subagent hooks observe MCP-discovered child tools.""" - seen_tools: list[list[str]] = [] - server = scripted_server(monkeypatch, {"remote": _schema()}) - config = SubAgentConfig(name="mcp", description="MCP helper.", inherit_mcp_servers=True) - parent = Harness( - HarnessConfig(root=tmp_path, builtin_tools=["subagent"], subagents=[config]), - plugins=[MCPPlugin(servers=[server])], - model=_fake_openai(MultiCallClient([("subagent", '{"task":"use remote","agent":"mcp"}')])), - hooks=[Hook("after_subagent_run", lambda ctx: seen_tools.append(ctx.tools), agents=["mcp"])], - ) - - await parent.run("delegate") - await parent.aclose() - - assert seen_tools == [["remote"]] - - async def test_resume_with_mcp_reuses_connection_and_keeps_state_clean(tmp_path, monkeypatch) -> None: """MCP tools are harness-local and not serialized into resume state.""" server = scripted_server(monkeypatch, {"remote": _schema()}) first_session = SequenceSession(ModelTurn(text="first", raw={"id": "first"})) second_session = SequenceSession(ModelTurn(text="second", raw={"id": "second"})) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), plugins=[MCPPlugin(servers=[server])], model=ScriptedModel([first_session, second_session]) + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[server])], model=ScriptedModel([first_session, second_session]) ) first = await harness.run("first") @@ -1727,7 +1586,7 @@ async def test_approval_resume_connects_mcp_before_validating_and_preserves_unkn requires_approval=True, ) first_harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[server])], model=model, tools=[approval_tool], @@ -1736,7 +1595,7 @@ async def test_approval_resume_connects_mcp_before_validating_and_preserves_unkn await first_harness.aclose() second_harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[server])], model=model, tools=[approval_tool], @@ -1764,7 +1623,7 @@ def rewrite(ctx) -> None: ctx.output = ToolResult(True, "rewritten", {}).as_json() harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[server])], model=_fake_openai(MultiCallClient([("remote", '{"value":"ok"}')])), hooks=[Hook("after_tool_call", rewrite, tools=["remote"])], @@ -1792,7 +1651,7 @@ async def list_tools(self, *, server_id: str | None = None) -> list[ToolSpec]: failing = FailingConnectServer(FastMCPTransport(FastMCP("failing-connect")), id="failing") harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), plugins=[MCPPlugin(servers=[failing])], model=_fake_openai(MultiCallClient([])), hooks=[ diff --git a/tests/unit/test_parallel_llm.py b/tests/unit/test_parallel_llm.py index b1b36e5..d93c464 100644 --- a/tests/unit/test_parallel_llm.py +++ b/tests/unit/test_parallel_llm.py @@ -7,10 +7,11 @@ import httpx import pytest +from fakes import FakeChildHarnessHost from pydantic import BaseModel, ValidationError import thinharness.plugins.parallel_llm as parallel_plugin_module -from thinharness import Harness, HarnessConfig, ModelCapabilities, ModelToolCall, ModelTurn, ParallelLlmPlugin, PluginContext, ToolOutput +from thinharness import Harness, HarnessConfig, ModelCapabilities, ModelToolCall, ModelTurn, ParallelLlmPlugin, PluginContext, ToolOutput, ToolSpec from thinharness.providers import ModelSettings, OpenAIProvider, OpenAIResponsesModel, ProviderError from thinharness.tools.base import _invoke_tool from thinharness.tools.parallel_llm import ( @@ -664,7 +665,7 @@ def test_parallel_llm_tool_custom_spec_and_model_resolution(tmp_path: Path) -> N assert spec.name == "parallel_extract" assert spec.description == "Extract fields." - assert spec.kind == "user" + assert "kind" not in ToolSpec.__dataclass_fields__ assert isinstance(model, OpenAIResponsesModel) assert should_close is True assert model.provider.api_key == "key" @@ -718,8 +719,8 @@ def test_parallel_llm_plugin_composition_and_builtin_migration(tmp_path: Path) - assert "parallel_llm" not in {tool.name for tool in default_harness.tools} assert "parallel_llm" in {tool.name for tool in selected_harness.tools} assert next(tool for tool in selected_harness.tools if tool.name == "parallel_llm").instructions == DEFAULT_PARALLEL_LLM_INSTRUCTIONS - with pytest.raises(ValueError, match="ParallelLlmPlugin"): - Harness(HarnessConfig(root=tmp_path / "bad", builtin_tools=["parallel_llm"])) + with pytest.raises(ValueError, match="SubagentsPlugin"): + HarnessConfig(root=tmp_path / "bad", builtin_tools=["parallel_llm"]) async def test_parallel_llm_usage_accounting_in_harness_run(tmp_path: Path) -> None: @@ -744,7 +745,7 @@ def test_parallel_llm_plugin_static_contract_and_fixed_names(tmp_path: Path) -> assert spec.name == "parallel_llm" assert spec.description == "Batch now." assert spec.instructions == "Use carefully." - assert spec.kind == "user" + assert "kind" not in ToolSpec.__dataclass_fields__ assert spec.origin is not None assert spec.origin.plugin == "parallel_llm" assert spec.origin.source == "parallel_llm" @@ -795,7 +796,7 @@ def capture_tool(**kwargs: Any) -> ParallelLlmTool: return real_tool(**kwargs) monkeypatch.setattr(parallel_plugin_module, "ParallelLlmTool", capture_tool) - context = PluginContext(root=tmp_path, model=BatchModel()) + context = PluginContext(root=tmp_path, model=BatchModel(), child_harnesses=FakeChildHarnessHost()) ParallelLlmPlugin("openai:default").bind(context) ParallelLlmPlugin( @@ -895,7 +896,7 @@ def fail(*_args, **_kwargs): monkeypatch.setattr(Path, "stat", fail) monkeypatch.setattr("thinharness.providers.infer_model", fail) - binding = plugin.bind(PluginContext(root=tmp_path, model=BatchModel())) + binding = plugin.bind(PluginContext(root=tmp_path, model=BatchModel(), child_harnesses=FakeChildHarnessHost())) assert binding.static.tools[0].name == "parallel_llm" diff --git a/tests/unit/test_parallel_tools.py b/tests/unit/test_parallel_tools.py index 478ceb5..f434e0e 100644 --- a/tests/unit/test_parallel_tools.py +++ b/tests/unit/test_parallel_tools.py @@ -43,7 +43,7 @@ def test_parallel_safe_batch_runs_concurrently(tmp_path: Path) -> None: delay = 0.2 client = MultiCallClient([("slow_a", "{}"), ("slow_b", "{}")]) harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[slow_tool("slow_a", delay), slow_tool("slow_b", delay)], ) @@ -63,7 +63,7 @@ def test_sequential_tool_forces_serial_batch(tmp_path: Path) -> None: delay = 0.2 client = MultiCallClient([("slow_a", "{}"), ("slow_b", "{}")]) harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[slow_tool("slow_a", delay), slow_tool("slow_b", delay, sequential=True)], ) @@ -81,7 +81,7 @@ def test_tool_execution_sequential_forces_serial_even_for_safe_tools(tmp_path: P delay = 0.15 client = MultiCallClient([("slow_a", "{}"), ("slow_b", "{}")]) harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[], tool_execution="sequential"), + HarnessConfig(root=tmp_path, model="openai:test-model", tool_execution="sequential"), model=_fake_openai(client), tools=[slow_tool("slow_a", delay), slow_tool("slow_b", delay)], ) @@ -95,7 +95,7 @@ def test_tool_execution_sequential_forces_serial_even_for_safe_tools(tmp_path: P def test_parallel_batch_preserves_model_call_order(tmp_path: Path) -> None: client = MultiCallClient([("slow_first", "{}"), ("fast_second", "{}")]) harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[slow_tool("slow_first", 0.2), slow_tool("fast_second", 0.01)], ) @@ -115,7 +115,7 @@ def boom(_args): boom_spec = ToolSpec("boom", "Always raises.", {"type": "object", "properties": {}}, boom) ok_spec = ToolSpec("ok", "Returns ok.", {"type": "object", "properties": {}}, lambda args: "ok") harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[boom_spec, ok_spec], ) @@ -132,7 +132,7 @@ def boom(_args): def test_parallel_batch_makes_one_provider_continuation(tmp_path: Path) -> None: client = MultiCallClient([("a", "{}"), ("b", "{}"), ("c", "{}")]) harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[slow_tool("a", 0.01), slow_tool("b", 0.01), slow_tool("c", 0.01)], ) @@ -174,7 +174,7 @@ def test_tool_spec_can_opt_into_sequential(tmp_path: Path) -> None: ) client = MultiCallClient([("slow_a", "{}"), ("slow_b", "{}")]) harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[slow_tool("slow_a", delay), sequential_tool], ) @@ -190,7 +190,7 @@ def test_parallel_batch_with_more_calls_than_worker_cap(tmp_path: Path) -> None: client = MultiCallClient(batch) tools = [slow_tool(name, 0.01) for name, _ in batch] harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=tools, ) @@ -217,7 +217,7 @@ def run_b(_args): return "b" harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ ToolSpec("track_a", "a", {"type": "object", "properties": {}}, run_a), diff --git a/tests/unit/test_plugins.py b/tests/unit/test_plugins.py index ebdfcd3..963d184 100644 --- a/tests/unit/test_plugins.py +++ b/tests/unit/test_plugins.py @@ -6,7 +6,7 @@ from pathlib import Path import pytest -from fakes import ScriptedModel, ScriptedSession, echo_tool +from fakes import FakeChildHarnessHost, ScriptedModel, ScriptedSession, echo_tool from pydantic import BaseModel import thinharness.core as core_module @@ -255,13 +255,18 @@ async def connect(): async def test_invalid_dynamic_contribution_is_not_committed(tmp_path: Path) -> None: events: list[str] = [] - plugin = ConnectedPlugin("bad", events, PluginContribution(tools=(_tool("subagent"),))) - harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[plugin]) + plugin = ConnectedPlugin("bad", events, PluginContribution(tools=(_tool("collision"),))) + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[plugin], + tools=[_tool("collision")], + ) - with pytest.raises(ValueError, match="reserved tool name"): + with pytest.raises(ValueError, match="duplicate tool name"): await harness.connect() - assert harness.tools == [] + assert [tool.name for tool in harness.tools] == ["collision"] assert events == ["enter:bad", "exit:bad"] @@ -318,7 +323,7 @@ def fail(*_args, **_kwargs): monkeypatch.setattr(Path, "is_file", fail) binding = FilesystemPlugin(read_paths=["future"], write_paths=["outputs"]).bind( - PluginContext(root=root, model=ScriptedModel([])) + PluginContext(root=root, model=ScriptedModel([]), child_harnesses=FakeChildHarnessHost()) ) assert [tool.name for tool in binding.static.tools] == ["read", "write", "edit", "search", "list", "glob"] @@ -525,15 +530,20 @@ async def connect(): return PluginBinding(connect=connect) first = CleanupPlugin("first", PluginContribution(), fail_close=True) - second = CleanupPlugin("second", PluginContribution(tools=(_tool("subagent"),))) - harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[first, second]) + second = CleanupPlugin("second", PluginContribution(tools=(_tool("collision"),))) + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[first, second], + tools=[_tool("collision")], + ) - with pytest.raises(ValueError, match="reserved tool name") as raised: + with pytest.raises(ValueError, match="duplicate tool name") as raised: await harness.connect() assert events == ["second", "first"] assert any("cleanup also failed" in note for note in raised.value.__notes__) - assert harness.tools == [] + assert [tool.name for tool in harness.tools] == ["collision"] async def test_close_attempts_model_after_plugin_failure(tmp_path: Path) -> None: diff --git a/tests/unit/test_providers.py b/tests/unit/test_providers.py index 5c32aba..f9361d9 100644 --- a/tests/unit/test_providers.py +++ b/tests/unit/test_providers.py @@ -404,6 +404,20 @@ def handler(request: httpx.Request) -> httpx.Response: assert second.text == "done" assert calls[0][1]["tools"][0]["input_schema"]["type"] == "object" +async def test_anthropic_projects_only_supported_request_metadata() -> None: + provider = FakeAnthropicProvider() + session = AnthropicMessagesModel("claude-test", provider=provider).new_session() + constants = RequestConstants( + instructions="system", + tools=[], + metadata={"user_id": "user-1", "conversation_id": "conv-1", "parent_call_id": "call-1"}, + ) + + await session.start("hi", constants) + + assert provider.payloads[0]["metadata"] == {"user_id": "user-1"} + + async def test_anthropic_requests_opt_into_prompt_caching() -> None: provider = FakeAnthropicProvider() session = AnthropicMessagesModel("claude-test", provider=provider).new_session() @@ -879,7 +893,7 @@ async def no_sleep(_delay: float) -> None: "test-model", provider=OpenAIProvider(api_key="key", request_retries=1, request_retry_backoff=0, http_client=client), ) - result = await Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model, tools=[echo_tool()]).run("go") + result = await Harness(HarnessConfig(root=tmp_path), model=model, tools=[echo_tool()]).run("go") assert result.text == "done" assert payloads[1] == payloads[2] diff --git a/tests/unit/test_reasoning_fidelity.py b/tests/unit/test_reasoning_fidelity.py index 311ef86..344db0b 100644 --- a/tests/unit/test_reasoning_fidelity.py +++ b/tests/unit/test_reasoning_fidelity.py @@ -145,7 +145,7 @@ async def create_chat_completion(self, payload): def _harness(tmp_path: Path, model, **config) -> Harness: - return Harness(HarnessConfig(root=tmp_path, builtin_tools=[], **config), model=model, tools=[echo_tool()]) + return Harness(HarnessConfig(root=tmp_path, **config), model=model, tools=[echo_tool()]) async def _capture_state(tmp_path: Path, model) -> dict: @@ -552,13 +552,13 @@ def _has_signed_reasoning(state: dict) -> bool: async def _run_reasoning_resume_live(tmp_path: Path, make_model) -> None: """Capture native reasoning, then resume on the same provider/model and assert acceptance.""" - first = await Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=make_model(), tools=[_multiply_tool()]).run( + first = await Harness(HarnessConfig(root=tmp_path), model=make_model(), tools=[_multiply_tool()]).run( "Use the multiply tool to compute 21 times 19, then state the product." ) state = json.loads(json.dumps(first.resume_state)) assert _has_signed_reasoning(state), "no signed native reasoning captured in resume_state" - second = await Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=make_model(), tools=[_multiply_tool()]).run( + second = await Harness(HarnessConfig(root=tmp_path), model=make_model(), tools=[_multiply_tool()]).run( "Add 100 to that product.", resume_from=state ) assert second.text diff --git a/tests/unit/test_resume.py b/tests/unit/test_resume.py index 0ba7551..e636d71 100644 --- a/tests/unit/test_resume.py +++ b/tests/unit/test_resume.py @@ -124,7 +124,7 @@ async def test_openai_resume_full_replays_transcript_for_followup(tmp_path: Path async def test_anthropic_resume_replays_transcript_and_appends_new_user_turn(tmp_path: Path) -> None: provider = FakeAnthropicProvider() model = AnthropicMessagesModel("claude-test", provider=provider) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model, tools=[echo_tool()]) + harness = Harness(HarnessConfig(root=tmp_path), model=model, tools=[echo_tool()]) first = await harness.run("first") state = json.loads(json.dumps(first.resume_state)) @@ -143,7 +143,7 @@ async def test_anthropic_resume_replays_transcript_and_appends_new_user_turn(tmp async def test_openrouter_resume_replays_transcript_and_appends_new_user_turn(tmp_path: Path) -> None: provider = FakeOpenRouterProvider() model = OpenRouterModel("openai/test", provider=provider) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model, tools=[echo_tool()]) + harness = Harness(HarnessConfig(root=tmp_path), model=model, tools=[echo_tool()]) first = await harness.run("first") state = json.loads(json.dumps(first.resume_state)) @@ -163,12 +163,12 @@ async def test_openrouter_resume_replays_transcript_and_appends_new_user_turn(tm async def test_cross_provider_resume_after_tool_round_trip(tmp_path: Path) -> None: source_provider = FakeAnthropicProvider() - source = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=AnthropicMessagesModel("claude-test", provider=source_provider), tools=[echo_tool()]) + source = Harness(HarnessConfig(root=tmp_path), model=AnthropicMessagesModel("claude-test", provider=source_provider), tools=[echo_tool()]) state = json.loads(json.dumps((await source.run("first")).resume_state)) openai_provider = _TerminalOpenAIProvider() openai_result = await Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=OpenAIResponsesModel("gpt-test", provider=openai_provider), tools=[echo_tool()], ).run("follow-up", resume_from=state) @@ -180,7 +180,7 @@ async def test_cross_provider_resume_after_tool_round_trip(tmp_path: Path) -> No openrouter_provider = FakeOpenRouterProvider() openrouter_result = await Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=OpenRouterModel("openai/test", provider=openrouter_provider), tools=[echo_tool()], ).run("follow-up", resume_from=state) @@ -192,13 +192,13 @@ async def test_cross_provider_resume_after_tool_round_trip(tmp_path: Path) -> No async def test_multi_tool_batch_replay_shapes(tmp_path: Path) -> None: source_provider = _MultiToolAnthropicProvider() - source = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=AnthropicMessagesModel("claude-test", provider=source_provider), tools=[echo_tool()]) + source = Harness(HarnessConfig(root=tmp_path), model=AnthropicMessagesModel("claude-test", provider=source_provider), tools=[echo_tool()]) state = json.loads(json.dumps((await source.run("first")).resume_state)) assert json.loads(json.dumps(state)) == state anthropic_provider = FakeAnthropicProvider() await Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=AnthropicMessagesModel("claude-test", provider=anthropic_provider), tools=[echo_tool()], ).run("follow-up", resume_from=state) @@ -208,7 +208,7 @@ async def test_multi_tool_batch_replay_shapes(tmp_path: Path) -> None: openrouter_provider = FakeOpenRouterProvider() await Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=OpenRouterModel("openai/test", provider=openrouter_provider), tools=[echo_tool()], ).run("follow-up", resume_from=state) @@ -218,14 +218,14 @@ async def test_multi_tool_batch_replay_shapes(tmp_path: Path) -> None: async def test_resume_rederives_live_system_prompt(tmp_path: Path) -> None: anthropic_source = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], system_prompt="old system"), + HarnessConfig(root=tmp_path, system_prompt="old system"), model=AnthropicMessagesModel("claude-test", provider=FakeAnthropicProvider()), tools=[echo_tool()], ) anthropic_state = json.loads(json.dumps((await anthropic_source.run("first")).resume_state)) anthropic_provider = FakeAnthropicProvider() await Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], system_prompt="new system"), + HarnessConfig(root=tmp_path, system_prompt="new system"), model=AnthropicMessagesModel("claude-test", provider=anthropic_provider), tools=[echo_tool()], ).run("follow-up", resume_from=anthropic_state) @@ -233,14 +233,14 @@ async def test_resume_rederives_live_system_prompt(tmp_path: Path) -> None: assert "old system" not in anthropic_provider.payloads[0]["system"] openrouter_source = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], system_prompt="old system"), + HarnessConfig(root=tmp_path, system_prompt="old system"), model=OpenRouterModel("openai/test", provider=FakeOpenRouterProvider()), tools=[echo_tool()], ) openrouter_state = json.loads(json.dumps((await openrouter_source.run("first")).resume_state)) openrouter_provider = FakeOpenRouterProvider() await Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], system_prompt="new system"), + HarnessConfig(root=tmp_path, system_prompt="new system"), model=OpenRouterModel("openai/test", provider=openrouter_provider), tools=[echo_tool()], ).run("follow-up", resume_from=openrouter_state) @@ -248,13 +248,13 @@ async def test_resume_rederives_live_system_prompt(tmp_path: Path) -> None: assert "old system" not in openrouter_provider.payloads[0]["messages"][0]["content"] openai_source = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], system_prompt="old system"), + HarnessConfig(root=tmp_path, system_prompt="old system"), model=OpenAIResponsesModel("gpt-test", provider=_TerminalOpenAIProvider()), ) openai_state = json.loads(json.dumps((await openai_source.run("first")).resume_state)) openai_provider = _TerminalOpenAIProvider() await Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], system_prompt="new system"), + HarnessConfig(root=tmp_path, system_prompt="new system"), model=OpenAIResponsesModel("gpt-test", provider=openai_provider), ).run("follow-up", resume_from=openai_state) assert openai_provider.payloads[0]["instructions"].startswith("new system") @@ -271,10 +271,10 @@ async def create_response(self, payload): def openai_harness(model_name: str = "gpt-test") -> Harness: """Create a fresh OpenAI resume harness.""" - return Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=OpenAIResponsesModel(model_name, provider=NoToolOpenAIProvider())) + return Harness(HarnessConfig(root=tmp_path), model=OpenAIResponsesModel(model_name, provider=NoToolOpenAIProvider())) state = openai_harness().run_sync("first").resume_state - anthropic = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=AnthropicMessagesModel("claude-test", provider=FakeAnthropicProvider())) + anthropic = Harness(HarnessConfig(root=tmp_path), model=AnthropicMessagesModel("claude-test", provider=FakeAnthropicProvider())) assert anthropic.run_sync("follow-up", resume_from=state).text == "done" assert openai_harness("other").run_sync("follow-up", resume_from=state).text == "done" @@ -301,7 +301,7 @@ def test_resume_rejects_malformed_shapes_before_hooks_fire(tmp_path: Path) -> No def harness() -> Harness: """Create a fresh harness for one run_sync validation case.""" return Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=AnthropicMessagesModel("claude-test", provider=FakeAnthropicProvider()), hooks=[ Hook("run_start", lambda ctx: events.append(type(ctx).__name__)), @@ -365,7 +365,7 @@ class Answer(BaseModel): ) ) model = _ScriptedResumeModel([session]) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Answer, output_mode="tool"), model=model) + harness = Harness(HarnessConfig(root=tmp_path, output_type=Answer, output_mode="tool"), model=model) result = harness.run_sync("make output") @@ -390,8 +390,8 @@ class Answer(BaseModel): ) ) model = _ScriptedResumeModel([first_session, resumed_session]) - first = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model).run_sync("first") - resumed = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Answer, output_mode="tool"), model=model).run_sync( + first = Harness(HarnessConfig(root=tmp_path), model=model).run_sync("first") + resumed = Harness(HarnessConfig(root=tmp_path, output_type=Answer, output_mode="tool"), model=model).run_sync( "structured follow-up", resume_from=first.resume_state, ) @@ -406,9 +406,9 @@ def test_resumed_user_prompt_receives_limit_notice(tmp_path: Path) -> None: ) resumed_session = ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "second"})) model = _ScriptedResumeModel([first_session, resumed_session]) - first = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], max_model_requests=1), model=model).run_sync("first") + first = Harness(HarnessConfig(root=tmp_path, max_model_requests=1), model=model).run_sync("first") - resumed = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], max_model_requests=1), model=model).run_sync( + resumed = Harness(HarnessConfig(root=tmp_path, max_model_requests=1), model=model).run_sync( "follow-up", resume_from=first.resume_state, ) @@ -431,14 +431,14 @@ def test_resumed_user_prompt_runs_prompt_submit_hooks_before_notices(tmp_path: P on_start=lambda prompt, _instructions, _tools, _metadata, _previous: captured.setdefault("prompt", prompt), ) model = _ScriptedResumeModel([first_session, resumed_session]) - first = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model).run_sync("first") + first = Harness(HarnessConfig(root=tmp_path), model=model).run_sync("first") def add_context(ctx) -> None: events.append("user_prompt_submit") ctx.additional_context.append("resume policy") resumed = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], max_model_requests=1), + HarnessConfig(root=tmp_path, max_model_requests=1), model=model, hooks=[Hook("user_prompt_submit", add_context)], ).run_sync("follow-up", resume_from=first.resume_state) @@ -460,12 +460,12 @@ async def create_response(self, payload): return {"output_text": "done"} provider = NoIdProvider() - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=OpenAIResponsesModel("gpt-test", provider=provider)) + harness = Harness(HarnessConfig(root=tmp_path), model=OpenAIResponsesModel("gpt-test", provider=provider)) first = harness.run_sync("first") assert first.resume_state is not None assert first.resume_state["kind"] == "transcript" - resumed = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=OpenAIResponsesModel("gpt-test", provider=provider)).run_sync( + resumed = Harness(HarnessConfig(root=tmp_path), model=OpenAIResponsesModel("gpt-test", provider=provider)).run_sync( "follow-up", resume_from=json.loads(json.dumps(first.resume_state)), ) @@ -479,14 +479,14 @@ def test_non_clean_exits_omit_resume_state(tmp_path: Path) -> None: start_turn=ModelTurn(tool_calls=[ModelToolCall(id="call_1", name="missing", arguments="{}")], raw={"id": "start"}), dump_state={"kind": "scripted", "version": 1, "model": "scripted"}, ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], max_tool_calls=0), model=_ScriptedResumeModel([session])) + harness = Harness(HarnessConfig(root=tmp_path, max_tool_calls=0), model=_ScriptedResumeModel([session])) with pytest.raises(HarnessError, match="max_tool_calls"): harness.run_sync("go") captured: list[RunEndContext] = [] harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], max_tool_calls=0), + HarnessConfig(root=tmp_path, max_tool_calls=0), model=_ScriptedResumeModel([session]), hooks=[Hook("run_end", lambda ctx: captured.append(ctx))], ) @@ -504,7 +504,7 @@ async def start(self, prompt, constants, *, previous_response_id=None, notices=N captured: list[RunEndContext] = [] harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=_ScriptedResumeModel([FailingProviderSession()]), hooks=[Hook("run_end", lambda ctx: captured.append(ctx))], ) @@ -522,7 +522,7 @@ def test_tool_retries_exceeded_omits_resume_state(tmp_path: Path) -> None: ) captured: list[RunEndContext] = [] harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], tool_retries=0), + HarnessConfig(root=tmp_path, tool_retries=0), model=_ScriptedResumeModel([session]), tools=[ToolSpec("flaky", "Flaky", {"type": "object", "properties": {}}, lambda args: (_ for _ in ()).throw(ModelRetry("again")))], hooks=[Hook("run_end", lambda ctx: captured.append(ctx))], @@ -542,7 +542,7 @@ def cancel(ctx) -> None: ctx.cancel_reason = "blocked" harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=_ScriptedResumeModel([_NoResumeSession()]), hooks=[Hook("user_prompt_submit", cancel), Hook("run_end", lambda ctx: captured.append(ctx))], ) @@ -563,7 +563,7 @@ class Answer(BaseModel): ) captured: list[RunEndContext] = [] harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Answer, output_mode="tool", output_retries=0), + HarnessConfig(root=tmp_path, output_type=Answer, output_mode="tool", output_retries=0), model=_ScriptedResumeModel([session]), hooks=[Hook("run_end", lambda ctx: captured.append(ctx))], ) @@ -577,7 +577,7 @@ class Answer(BaseModel): async def test_resume_state_is_detached_outbound_and_inbound(tmp_path: Path) -> None: provider = FakeOpenRouterProvider() model = OpenRouterModel("openai/test", provider=provider) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model, tools=[echo_tool()]) + harness = Harness(HarnessConfig(root=tmp_path), model=model, tools=[echo_tool()]) first = await harness.run("first") stashed = json.loads(json.dumps(first.resume_state)) @@ -592,10 +592,10 @@ async def test_resume_state_is_detached_outbound_and_inbound(tmp_path: Path) -> async def test_fresh_harness_persistence_and_sequential_branching(tmp_path: Path) -> None: provider = FakeOpenRouterProvider() - first_harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=OpenRouterModel("openai/test", provider=provider), tools=[echo_tool()]) + first_harness = Harness(HarnessConfig(root=tmp_path), model=OpenRouterModel("openai/test", provider=provider), tools=[echo_tool()]) state = json.loads(json.dumps((await first_harness.run("first")).resume_state)) - second_harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=OpenRouterModel("openai/test", provider=provider), tools=[echo_tool()]) + second_harness = Harness(HarnessConfig(root=tmp_path), model=OpenRouterModel("openai/test", provider=provider), tools=[echo_tool()]) first_branch = await second_harness.run("branch one", resume_from=state) second_branch = await second_harness.run("branch two", resume_from=state) @@ -626,7 +626,7 @@ def test_adapter_non_json_dump_propagates_type_error(tmp_path: Path) -> None: start_turn=ModelTurn(text="done", raw={"id": "done"}), dump_state={"kind": "scripted", "version": 1, "model": "scripted", "bad": object()}, ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=_ScriptedResumeModel([session])) + harness = Harness(HarnessConfig(root=tmp_path), model=_ScriptedResumeModel([session])) with pytest.raises(TypeError): harness.run_sync("go") @@ -636,7 +636,7 @@ def test_custom_model_without_resume_support_can_run_but_cannot_resume(tmp_path: events: list[str] = [] model = _NoResumeModel([_NoResumeSession(), _NoResumeSession()]) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=model, hooks=[ Hook("run_start", lambda ctx: events.append(type(ctx).__name__)), @@ -648,7 +648,7 @@ def test_custom_model_without_resume_support_can_run_but_cannot_resume(tmp_path: events.clear() with pytest.raises(HarnessError, match="does not support resume"): Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=model, hooks=[ Hook("run_start", lambda ctx: events.append(type(ctx).__name__)), @@ -664,7 +664,7 @@ class MissingDumpStateSession(_NoResumeSession): dump_state = None - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=_ScriptedResumeModel([MissingDumpStateSession()])) + harness = Harness(HarnessConfig(root=tmp_path), model=_ScriptedResumeModel([MissingDumpStateSession()])) with pytest.raises(HarnessError, match="resumable model session is missing dump_state"): harness.run_sync("go") @@ -673,7 +673,7 @@ class MissingDumpStateSession(_NoResumeSession): def test_run_end_context_sees_resume_state(tmp_path: Path) -> None: captured: list[dict | None] = [] harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=OpenAIResponsesModel("gpt-test", provider=FakeClient()), hooks=[Hook("run_end", lambda ctx: captured.append(ctx.result.resume_state if ctx.result else None))], ) @@ -694,7 +694,7 @@ async def start(self, prompt, constants, *, previous_response_id=None, notices=N await release.wait() return ModelTurn(text="done", raw={"id": "done"}) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=_NoResumeModel([SlowSession()])) + harness = Harness(HarnessConfig(root=tmp_path), model=_NoResumeModel([SlowSession()])) task = asyncio.create_task(harness.run("go")) await started.wait() with pytest.raises(HarnessError, match="not re-entrant"): diff --git a/tests/unit/test_skills.py b/tests/unit/test_skills.py index 11c6ca5..04d7f93 100644 --- a/tests/unit/test_skills.py +++ b/tests/unit/test_skills.py @@ -4,7 +4,7 @@ from pathlib import Path import pytest -from fakes import ScriptedModel +from fakes import FakeChildHarnessHost, ScriptedModel from thinharness import Harness, HarnessConfig, PluginBinding, PluginContext, PluginContribution, SkillRegistry, SkillsPlugin, ToolSpec @@ -277,5 +277,5 @@ def fail(*_args, **_kwargs): monkeypatch.setattr(Path, "exists", fail) monkeypatch.setattr(Path, "stat", fail) - binding = plugin.bind(PluginContext(root=tmp_path, model=model)) + binding = plugin.bind(PluginContext(root=tmp_path, model=model, child_harnesses=FakeChildHarnessHost())) assert binding.static.tools[0].name == "skill_read" diff --git a/tests/unit/test_streaming.py b/tests/unit/test_streaming.py index e7cd2cd..1640a25 100644 --- a/tests/unit/test_streaming.py +++ b/tests/unit/test_streaming.py @@ -23,6 +23,7 @@ RunStartedEvent, StreamOptions, SubAgentConfig, + SubagentsPlugin, ToolCallCompletedEvent, ToolCallStartedEvent, ToolSpec, @@ -80,7 +81,7 @@ async def _collect_events(harness: Harness, prompt: str, **kwargs): async def test_stream_returns_final_harness_result(tmp_path: Path) -> None: session = ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"})) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([session])) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session])) events = await _collect_events(harness, "go") @@ -94,8 +95,8 @@ async def test_stream_returns_final_harness_result(tmp_path: Path) -> None: async def test_run_consumes_stream_and_returns_result(tmp_path: Path) -> None: run_session = ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"})) stream_session = ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"})) - run_harness = Harness(HarnessConfig(root=tmp_path / "run", builtin_tools=[]), model=ScriptedModel([run_session])) - stream_harness = Harness(HarnessConfig(root=tmp_path / "stream", builtin_tools=[]), model=ScriptedModel([stream_session])) + run_harness = Harness(HarnessConfig(root=tmp_path / "run"), model=ScriptedModel([run_session])) + stream_harness = Harness(HarnessConfig(root=tmp_path / "stream"), model=ScriptedModel([stream_session])) run_result = await run_harness.run("go") events = await _collect_events(stream_harness, "go") @@ -114,7 +115,7 @@ async def test_stream_emits_model_and_tool_lifecycle(tmp_path: Path) -> None: ), continue_turn=ModelTurn(text="done", raw={"id": "done"}), ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([session]), tools=[echo_tool()]) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session]), tools=[echo_tool()]) events = await _collect_events(harness, "go") @@ -145,7 +146,7 @@ async def test_stream_payloads_are_high_level_without_raw_provider_payloads(tmp_ ), continue_turn=ModelTurn(text="done", raw={"id": "done"}), ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([session]), tools=[echo_tool()]) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session]), tools=[echo_tool()]) events = await _collect_events(harness, "go") @@ -163,7 +164,7 @@ async def test_stream_payloads_are_high_level_without_raw_provider_payloads(tmp_ async def test_stream_options_keep_model_text_visible(tmp_path: Path) -> None: session = ScriptedSession(start_turn=ModelTurn(text="visible", raw={"id": "done"})) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([session])) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session])) events = await _collect_events(harness, "go", stream_options=StreamOptions()) @@ -190,7 +191,7 @@ async def no_sleep(_delay: float) -> None: "test-model", provider=OpenAIProvider(api_key="key", request_retries=1, request_retry_backoff=0, http_client=client), ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model) + harness = Harness(HarnessConfig(root=tmp_path), model=model) events = await _collect_events(harness, "go") assert calls == 2 @@ -201,7 +202,7 @@ async def no_sleep(_delay: float) -> None: async def test_stream_limit_warning_events(tmp_path: Path) -> None: session = ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"})) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], max_model_requests=1), model=ScriptedModel([session])) + harness = Harness(HarnessConfig(root=tmp_path, max_model_requests=1), model=ScriptedModel([session])) events = await _collect_events(harness, "go") @@ -217,7 +218,7 @@ async def test_stream_failed_tool_content_is_included_by_default(tmp_path: Path) continue_turn=ModelTurn(text="done", raw={"id": "done"}), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([session]), tools=[ToolSpec("fail", "Fail", {"type": "object", "properties": {}}, lambda args: ToolResult(False, error_message).as_json())], ) @@ -230,7 +231,7 @@ async def test_stream_failed_tool_content_is_included_by_default(tmp_path: Path) async def test_stream_failure_yields_failed_event_then_raises(tmp_path: Path) -> None: - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([FailingSession()])) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([FailingSession()])) events = [] stream = harness.stream("go") @@ -251,12 +252,11 @@ async def test_stream_subagent_events_include_parent_ids(tmp_path: Path) -> None continue_turn=ModelTurn(text="parent done", raw={"id": "parent-done"}), ) harness = Harness( - HarnessConfig( - root=tmp_path, - builtin_tools=["subagent"], - subagents=[SubAgentConfig(name="helper", description="Helper.", tools=[ToolSpec("x", "X", {"type": "object"}, lambda args: "x")])], - ), + HarnessConfig(root=tmp_path), model=ScriptedModel([parent, child]), + plugins=[SubagentsPlugin(agents=[ + SubAgentConfig(name="helper", description="Helper.", tools=[ToolSpec("x", "X", {"type": "object"}, lambda args: "x")]) + ])], ) events = await _collect_events(harness, "delegate") @@ -276,7 +276,11 @@ async def test_stream_options_can_hide_subagent_events(tmp_path: Path) -> None: start_turn=ModelTurn(tool_calls=[ModelToolCall(id="call_1", name="subagent", arguments='{"task":"help"}')], raw={"id": "parent"}), continue_turn=ModelTurn(text="parent done", raw={"id": "parent-done"}), ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=["subagent"]), model=ScriptedModel([parent, child])) + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([parent, child]), + plugins=[SubagentsPlugin()], + ) events = await _collect_events(harness, "delegate", stream_options=StreamOptions(include_subagents=False)) @@ -294,7 +298,7 @@ async def slow_start(*_args, **_kwargs): session = ScriptedSession(start_turn=ModelTurn(text="unused", raw={"id": "unused"})) session.start = slow_start harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([session, ScriptedSession(start_turn=ModelTurn(text="again", raw={}))]), ) @@ -321,7 +325,7 @@ class Person(BaseModel): ), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool", output_retries=1), + HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool", output_retries=1), model=ScriptedModel([session]), ) @@ -354,7 +358,7 @@ def flaky(_args): return "ok" harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([session]), tools=[ToolSpec("flaky", "Flaky", {"type": "object", "properties": {}}, flaky)], ) @@ -368,7 +372,7 @@ def flaky(_args): async def test_stream_resume_from_emits_resume_kind(tmp_path: Path) -> None: session = SequenceSession(ModelTurn(text="done", raw={"id": "done"})) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([session])) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session])) events = await _collect_events(harness, "go", resume_from={"kind": "scripted", "version": 1, "model": "scripted-model"}) @@ -378,7 +382,7 @@ async def test_stream_resume_from_emits_resume_kind(tmp_path: Path) -> None: def test_stream_without_running_loop_does_not_brick_harness(tmp_path: Path) -> None: harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"}))]), ) @@ -390,7 +394,7 @@ def test_stream_without_running_loop_does_not_brick_harness(tmp_path: Path) -> N def test_stream_run_sync_still_works(tmp_path: Path) -> None: harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"}))]), ) @@ -403,7 +407,7 @@ async def test_stream_strict_after_tool_hook_failure_completes_started_tool(tmp_ continue_turn=ModelTurn(text="done", raw={"id": "done"}), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], strict_hooks=True), + HarnessConfig(root=tmp_path, strict_hooks=True), model=ScriptedModel([session]), tools=[echo_tool()], hooks=[Hook("after_tool_call", lambda ctx: (_ for _ in ()).throw(RuntimeError("after failed")))], diff --git a/tests/unit/test_structured_output.py b/tests/unit/test_structured_output.py index 53f9b99..1e9d082 100644 --- a/tests/unit/test_structured_output.py +++ b/tests/unit/test_structured_output.py @@ -4,7 +4,7 @@ from dataclasses import dataclass import pytest -from fakes import FakeAnthropicProvider, FakeTracer, ScriptedModel, ScriptedProvider, ScriptedSession +from fakes import FakeAnthropicProvider, FakeTracer, ScriptedModel, ScriptedProvider, ScriptedSession, tool_output from pydantic import BaseModel from typing_extensions import TypedDict @@ -21,17 +21,15 @@ OpenRouterModel, RunUsage, SubAgentConfig, + SubagentsPlugin, TextOutput, ToolSpec, TracingOptions, UnexpectedModelBehavior, - build_child_harness, - create_subagent_tool, ) from thinharness.output import OutputSchema from thinharness.providers import ProviderError from thinharness.runtime import _compute_limit_notices -from thinharness.subagents import SubAgentArgs, run_subagent_tool from thinharness.turns import resolve_turn_output @@ -169,7 +167,7 @@ def test_base_model_output_via_tool_mode(tmp_path) -> None: raw={"id": "resp_1"}, ) ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool"), model=ScriptedModel([session])) + harness = Harness(HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool"), model=ScriptedModel([session])) result = harness.run_sync("make a person") @@ -187,7 +185,7 @@ def test_base_model_output_via_native_mode(tmp_path) -> None: ) model = ScriptedModel([session]) model.capabilities = ModelCapabilities(supports_json_schema_output=True, default_structured_output_mode="native") - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person), model=model) + harness = Harness(HarnessConfig(root=tmp_path, output_type=Person), model=model) result = harness.run_sync("make a person") @@ -197,7 +195,7 @@ def test_base_model_output_via_native_mode(tmp_path) -> None: def test_prompted_mode_strips_json_fence(tmp_path) -> None: session = ScriptedSession(start_turn=ModelTurn(text='```json\n{"name":"Ada","age":37}\n```', raw={"id": "resp_1"})) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="prompted"), model=ScriptedModel([session])) + harness = Harness(HarnessConfig(root=tmp_path, output_type=Person, output_mode="prompted"), model=ScriptedModel([session])) result = harness.run_sync("make a person") @@ -206,17 +204,17 @@ def test_prompted_mode_strips_json_fence(tmp_path) -> None: def test_typed_dict_dataclass_and_list_outputs(tmp_path) -> None: typed = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Item, output_mode="tool"), + HarnessConfig(root=tmp_path, output_type=Item, output_mode="tool"), model=ScriptedModel([ScriptedSession(start_turn=ModelTurn(tool_calls=[ ModelToolCall(id="call_final", name="final_result", arguments='{"name":"bolt","count":2}') ], raw={"id": "typed"}))]), ) data_class = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=City, output_mode="prompted"), + HarnessConfig(root=tmp_path, output_type=City, output_mode="prompted"), model=ScriptedModel([ScriptedSession(start_turn=ModelTurn(text='{"name":"Paris","country":"FR"}', raw={"id": "dataclass"}))]), ) people = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=list[Person], output_mode="tool"), + HarnessConfig(root=tmp_path, output_type=list[Person], output_mode="tool"), model=ScriptedModel([ScriptedSession(start_turn=ModelTurn(tool_calls=[ ModelToolCall(id="call_final", name="final_result", arguments='{"value":[{"name":"Ada","age":37}]}') ], raw={"id": "list"}))]), @@ -229,7 +227,7 @@ def test_typed_dict_dataclass_and_list_outputs(tmp_path) -> None: def test_union_output_uses_wrapped_value_schema(tmp_path) -> None: harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person | City, output_mode="tool"), + HarnessConfig(root=tmp_path, output_type=Person | City, output_mode="tool"), model=ScriptedModel([ScriptedSession(start_turn=ModelTurn(tool_calls=[ ModelToolCall(id="call_final", name="final_result", arguments='{"value":{"name":"Paris","country":"FR"}}') ], raw={"id": "union"}))]), @@ -251,7 +249,7 @@ def test_validation_failure_retries_and_succeeds(tmp_path) -> None: ], raw={"id": "good"}), on_continue=lambda outputs, _tools, _metadata: seen_outputs.extend(outputs), ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool", output_retries=1), model=ScriptedModel([session])) + harness = Harness(HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool", output_retries=1), model=ScriptedModel([session])) result = harness.run_sync("make a person") @@ -278,7 +276,7 @@ async def continue_with_user_text(self, text, constants, *, notices=None): ], raw={"id": "good"}), on_continue=lambda outputs, _tools, _metadata: seen_outputs.extend(outputs), ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool"), model=ScriptedModel([session])) + harness = Harness(HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool"), model=ScriptedModel([session])) result = harness.run_sync("make a person") @@ -297,7 +295,7 @@ def test_tool_mode_text_only_end_turn_retries_and_succeeds(tmp_path) -> None: ], raw={"id": "good"}), on_continue=lambda message, _tools, _metadata: messages.append(message), ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool"), model=ScriptedModel([session])) + harness = Harness(HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool"), model=ScriptedModel([session])) result = harness.run_sync("make a person") @@ -313,7 +311,7 @@ def test_validation_failure_exhausts_retries_and_reports_run_end(tmp_path) -> No start_turn=ModelTurn(tool_calls=[ModelToolCall(id="call_final", name="final_result", arguments='{"name":"Ada"}')], raw={"id": "bad"}) ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool", output_retries=0), + HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool", output_retries=0), model=ScriptedModel([session]), hooks=[Hook("run_end", lambda ctx: events.append((ctx.stop_reason, ctx.usage.output_retries)))], ) @@ -330,7 +328,7 @@ def test_retry_not_counted_when_model_limit_blocks_corrective_request(tmp_path) start_turn=ModelTurn(tool_calls=[ModelToolCall(id="call_final", name="final_result", arguments='{"name":"Ada"}')], raw={"id": "bad"}) ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool", output_retries=1, max_model_requests=1), + HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool", output_retries=1, max_model_requests=1), model=ScriptedModel([session]), hooks=[Hook("run_end", lambda ctx: events.append((ctx.stop_reason, ctx.usage.output_retries)))], ) @@ -346,7 +344,7 @@ def test_limit_notice_dedupes_across_structured_output_retries(tmp_path) -> None continue_turn=ModelTurn(text="still not structured", raw={"id": "bad-again"}), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool", output_retries=2, max_tool_calls=0), + HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool", output_retries=2, max_tool_calls=0), model=ScriptedModel([session]), ) @@ -368,7 +366,7 @@ def test_invalid_final_result_correction_receives_near_limit_notice(tmp_path) -> ], raw={"id": "good"}), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool", max_model_requests=2), + HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool", max_model_requests=2), model=ScriptedModel([session]), ) @@ -414,7 +412,7 @@ def test_limit_notices_only_mention_final_result_for_tool_output_mode(tmp_path, model = ScriptedModel([session]) model.capabilities = ModelCapabilities(supports_json_schema_output=True, default_structured_output_mode="native") harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=output_type, output_mode=output_mode, max_model_requests=1, max_tool_calls=0), + HarnessConfig(root=tmp_path, output_type=output_type, output_mode=output_mode, max_model_requests=1, max_tool_calls=0), model=model, ) @@ -433,7 +431,7 @@ def test_final_result_mixed_with_tool_calls_is_unexpected_and_dispatches_no_tool ], raw={"id": "bad"}) ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool"), + HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool"), model=ScriptedModel([session]), tools=[ToolSpec("boom", "boom", {"type": "object", "properties": {}}, lambda args: called.append(True))], hooks=[Hook("before_tool_call", lambda ctx: called.append(True))], @@ -458,7 +456,7 @@ def test_ordinary_tool_executes_before_tool_mode_final_result(tmp_path) -> None: ), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool"), + HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool"), model=ScriptedModel([session]), tools=[ToolSpec("lookup", "lookup", {"type": "object", "properties": {"query": {"type": "string"}}}, lambda args: called.append(args) or "found")], ) @@ -479,7 +477,7 @@ def test_repeated_final_result_calls_are_unexpected(tmp_path) -> None: ModelToolCall(id="call_final_2", name="final_result", arguments='{"name":"Grace","age":85}'), ], raw={"id": "bad"}) ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool"), model=ScriptedModel([session])) + harness = Harness(HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool"), model=ScriptedModel([session])) with pytest.raises(UnexpectedModelBehavior): harness.run_sync("make a person") @@ -492,7 +490,7 @@ def on_start(_prompt, _instructions, tools, _metadata, _previous): captured["tools"] = tools session = ScriptedSession(start_turn=ModelTurn(text="plain", raw={"id": "resp_1"}), on_start=on_start) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], output_type=TextOutput()), model=ScriptedModel([session])) + harness = Harness(HarnessConfig(root=tmp_path, output_type=TextOutput()), model=ScriptedModel([session])) result = harness.run_sync("say hi") @@ -504,7 +502,7 @@ def on_start(_prompt, _instructions, tools, _metadata, _previous): def test_final_result_tool_name_collision_is_rejected(tmp_path) -> None: with pytest.raises(ValueError, match="reserved"): Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool"), + HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool"), model=ScriptedModel([]), tools=[ToolSpec("final_result", "reserved", {"type": "object", "properties": {}}, lambda args: "bad")], ) @@ -512,7 +510,7 @@ def test_final_result_tool_name_collision_is_rejected(tmp_path) -> None: def test_late_final_result_tool_name_collision_is_rejected(tmp_path) -> None: harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool"), + HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool"), model=ScriptedModel([]), ) @@ -530,7 +528,7 @@ def test_final_result_hook_filter_is_allowed_but_never_fires(tmp_path) -> None: ) ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool"), + HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool"), model=ScriptedModel([session]), hooks=[Hook("before_tool_call", lambda ctx: seen.append(ctx.tool_name), tools=["final_result"])], ) @@ -556,8 +554,8 @@ async def create_message(self, payload): def test_anthropic_native_mode_constructs_and_defaults_to_native(tmp_path) -> None: model = AnthropicMessagesModel("claude-test", provider=FakeAnthropicProvider()) - explicit = Harness(HarnessConfig(root=tmp_path / "explicit", builtin_tools=[], output_type=Person, output_mode="native"), model=model) - auto = Harness(HarnessConfig(root=tmp_path / "auto", builtin_tools=[], output_type=Person), model=model) + explicit = Harness(HarnessConfig(root=tmp_path / "explicit", output_type=Person, output_mode="native"), model=model) + auto = Harness(HarnessConfig(root=tmp_path / "auto", output_type=Person), model=model) assert explicit.output_schema is not None assert explicit.output_schema.mode == "native" @@ -568,7 +566,7 @@ def test_anthropic_native_mode_constructs_and_defaults_to_native(tmp_path) -> No def test_anthropic_native_output_marker_constructs(tmp_path) -> None: model = AnthropicMessagesModel("claude-test", provider=FakeAnthropicProvider()) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], output_type=NativeOutput(Person)), model=model) + harness = Harness(HarnessConfig(root=tmp_path, output_type=NativeOutput(Person)), model=model) assert harness.output_schema is not None assert harness.output_schema.mode == "native" @@ -577,7 +575,7 @@ def test_anthropic_native_output_marker_constructs(tmp_path) -> None: def test_anthropic_native_mode_parses_json_text_result(tmp_path) -> None: provider = JsonAnthropicProvider() model = AnthropicMessagesModel("claude-test", provider=provider) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person), model=model) + harness = Harness(HarnessConfig(root=tmp_path, output_type=Person), model=model) result = harness.run_sync("make a person") @@ -589,7 +587,7 @@ def test_anthropic_native_mode_parses_json_text_result(tmp_path) -> None: def test_anthropic_native_truncated_json_retries_and_succeeds(tmp_path) -> None: provider = TruncatingJsonAnthropicProvider() model = AnthropicMessagesModel("claude-test", provider=provider) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_retries=1), model=model) + harness = Harness(HarnessConfig(root=tmp_path, output_type=Person, output_retries=1), model=model) result = harness.run_sync("make a person") @@ -604,7 +602,7 @@ def test_anthropic_native_truncated_json_retries_and_succeeds(tmp_path) -> None: def test_native_schema_is_strict_normalized_for_openai(tmp_path) -> None: model = ScriptedModel([ScriptedSession(start_turn=ModelTurn(text='{"name":"Ada","age":37}', raw={"id": "native"}))]) model.capabilities = ModelCapabilities(supports_json_schema_output=True, default_structured_output_mode="native") - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person), model=model) + harness = Harness(HarnessConfig(root=tmp_path, output_type=Person), model=model) request = harness.output_schema.structured_output_request() @@ -615,7 +613,7 @@ def test_native_schema_is_strict_normalized_for_openai(tmp_path) -> None: def test_native_nested_schema_is_strict_normalized_for_openai(tmp_path) -> None: model = ScriptedModel([ScriptedSession(start_turn=ModelTurn(text='{"name":"Ada","address":{"city":"London","zip_code":"NW1"}}', raw={"id": "native"}))]) model.capabilities = ModelCapabilities(supports_json_schema_output=True, default_structured_output_mode="native") - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], output_type=PersonWithAddress), model=model) + harness = Harness(HarnessConfig(root=tmp_path, output_type=PersonWithAddress), model=model) request = harness.output_schema.structured_output_request() address_schema = request.schema["properties"]["address"] @@ -633,7 +631,7 @@ async def create_chat_completion(self, payload): raise ProviderError("native rejected") model = OpenRouterModel("openai/test", provider=RejectingProvider()) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="native"), model=model) + harness = Harness(HarnessConfig(root=tmp_path, output_type=Person, output_mode="native"), model=model) with pytest.raises(HarnessError, match="native rejected"): harness.run_sync("make a person") @@ -653,7 +651,7 @@ def test_structured_finalization_marks_model_span(tmp_path, mode: str, turn: Mod if mode == "native": model.capabilities = ModelCapabilities(supports_json_schema_output=True, default_structured_output_mode="native") harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode=mode, tracing=[TracingOptions(tracer=tracer)]), + HarnessConfig(root=tmp_path, output_type=Person, output_mode=mode, tracing=[TracingOptions(tracer=tracer)]), model=model, ) @@ -673,7 +671,7 @@ def test_structured_retry_span_is_not_marked_finalized(tmp_path) -> None: continue_turn=ModelTurn(text='{"name":"Ada","age":37}', raw={"id": "good"}), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="prompted", tracing=[TracingOptions(tracer=tracer)]), + HarnessConfig(root=tmp_path, output_type=Person, output_mode="prompted", tracing=[TracingOptions(tracer=tracer)]), model=ScriptedModel([session]), ) @@ -686,69 +684,92 @@ def test_structured_retry_span_is_not_marked_finalized(tmp_path) -> None: async def test_named_subagent_structured_output_is_serialized_for_parent(tmp_path) -> None: - session = ScriptedSession( - start_turn=ModelTurn(tool_calls=[ModelToolCall(id="call_final", name="final_result", arguments='{"name":"Ada","age":37}')], raw={"id": "child"}) + config = SubAgentConfig(name="typed", description="Typed helper.", output_type=Person, output_mode="tool") + parent_session = ScriptedSession( + start_turn=ModelTurn( + tool_calls=[ModelToolCall(id="delegate", name="subagent", arguments='{"task":"make a person","agent":"typed"}')], + raw={"id": "parent"}, + ), + continue_turn=ModelTurn(text="done", raw={"id": "done"}), + ) + child_session = ScriptedSession( + start_turn=ModelTurn( + tool_calls=[ModelToolCall(id="call_final", name="final_result", arguments='{"name":"Ada","age":37}')], + raw={"id": "child"}, + ) ) - child_model = ScriptedModel([session]) - config = SubAgentConfig(name="typed", description="Typed helper.", inherit_parent_tools=True, output_type=Person, output_mode="tool") parent = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], subagents=[config]), - model=child_model, + HarnessConfig(root=tmp_path), + model=ScriptedModel([parent_session, child_session]), + plugins=[SubagentsPlugin(agents=[config])], ) - parent.add_tool(create_subagent_tool(parent, [config])) - - result = await run_subagent_tool(parent, [config], SubAgentArgs(task="make a person", agent="typed")) - assert result.ok is True - assert json.loads(result.content) == {"name": "Ada", "age": 37} - assert result.metadata["structured_output"] is True + assert (await parent.run("delegate")).text == "done" + output = tool_output(parent_session.continue_calls[0][0][0].output) + assert json.loads(output["content"]) == {"name": "Ada", "age": 37} + assert output["metadata"]["structured_output"] is True async def test_named_subagent_without_output_type_returns_text(tmp_path) -> None: - config = SubAgentConfig(name="plain", description="Plain helper.", inherit_parent_tools=True) + config = SubAgentConfig(name="plain", description="Plain helper.") + parent_session = ScriptedSession( + start_turn=ModelTurn( + tool_calls=[ModelToolCall(id="delegate", name="subagent", arguments='{"task":"plain work","agent":"plain"}')], + raw={}, + ), + continue_turn=ModelTurn(text="done", raw={}), + ) parent = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], subagents=[config]), - model=ScriptedModel([ScriptedSession(start_turn=ModelTurn(text="child text", raw={"id": "child"}))]), + HarnessConfig(root=tmp_path), + model=ScriptedModel([parent_session, ScriptedSession(start_turn=ModelTurn(text="child text", raw={}))]), + plugins=[SubagentsPlugin(agents=[config])], ) - parent.add_tool(create_subagent_tool(parent, [config])) - - result = await run_subagent_tool(parent, [config], SubAgentArgs(task="plain work", agent="plain")) - assert result.ok is True - assert result.content == "child text" - assert result.metadata["structured_output"] is False + await parent.run("delegate") + output = tool_output(parent_session.continue_calls[0][0][0].output) + assert output["content"] == "child text" + assert output["metadata"]["structured_output"] is False def test_parent_output_type_is_not_inherited_by_subagents(tmp_path) -> None: - config = SubAgentConfig(name="plain", description="Plain helper.", inherit_parent_tools=True) + config = SubAgentConfig(name="plain", description="Plain helper.") parent_model = ScriptedModel([]) parent_model.capabilities = ModelCapabilities(supports_json_schema_output=True, default_structured_output_mode="native") parent = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, subagents=[config]), + HarnessConfig(root=tmp_path, output_type=Person), model=parent_model, + plugins=[SubagentsPlugin(agents=[config])], ) assert parent.output_schema.mode == "native" - assert build_child_harness(parent, None).output_schema is None - assert build_child_harness(parent, config).output_schema is None + assert config.output_type is None async def test_parent_native_mode_and_child_tool_mode_do_not_bleed_config(tmp_path) -> None: - config = SubAgentConfig(name="typed", description="Typed helper.", inherit_parent_tools=True, output_type=Person, output_mode="tool") - model = ScriptedModel([ - ScriptedSession(start_turn=ModelTurn(tool_calls=[ - ModelToolCall(id="call_final", name="final_result", arguments='{"name":"Ada","age":37}') - ], raw={"id": "child"})) - ]) + config = SubAgentConfig(name="typed", description="Typed helper.", output_type=Person, output_mode="tool") + parent_session = ScriptedSession( + start_turn=ModelTurn( + tool_calls=[ModelToolCall(id="delegate", name="subagent", arguments='{"task":"make a person","agent":"typed"}')], + raw={}, + ), + continue_turn=ModelTurn(text='{"name":"Grace","age":45}', raw={}), + ) + child_session = ScriptedSession( + start_turn=ModelTurn( + tool_calls=[ModelToolCall(id="child_final", name="final_result", arguments='{"name":"Ada","age":37}')], + raw={}, + ) + ) + model = ScriptedModel([parent_session, child_session]) model.capabilities = ModelCapabilities(supports_json_schema_output=True, default_structured_output_mode="native") parent = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, subagents=[config]), + HarnessConfig(root=tmp_path, output_type=Person), model=model, + plugins=[SubagentsPlugin(agents=[config])], ) - parent.add_tool(create_subagent_tool(parent, [config])) - - result = await run_subagent_tool(parent, [config], SubAgentArgs(task="make a person", agent="typed")) + result = await parent.run("delegate") + output = tool_output(parent_session.continue_calls[0][0][0].output) assert parent.output_schema.mode == "native" - assert json.loads(result.content) == {"name": "Ada", "age": 37} - assert result.metadata["structured_output"] is True + assert result.output == Person(name="Grace", age=45) + assert json.loads(output["content"]) == {"name": "Ada", "age": 37} diff --git a/tests/unit/test_subagents.py b/tests/unit/test_subagents.py index 6f20e83..29b3f65 100644 --- a/tests/unit/test_subagents.py +++ b/tests/unit/test_subagents.py @@ -4,680 +4,725 @@ from pathlib import Path import pytest -from fakes import ( - FailingSession, - FakeTracer, - RecordingModel, - ScriptedModel, - ScriptedSession, - echo_tool, - tool_output, -) +from fakes import FailingSession, FakeTracer, ScriptedModel, ScriptedSession, echo_tool, tool_output from thinharness import ( DEFAULT_SUBAGENT_NAME, AfterSubagentRunContext, BeforeSubagentRunContext, + ChildHarnessOutcome, + ChildHarnessRequest, FilesystemPlugin, Harness, HarnessConfig, Hook, HookRegistry, - MCPPlugin, - MCPServerStdio, - ParallelLlmPlugin, + PluginBinding, + PluginContext, + PluginContribution, SkillsPlugin, SubAgentConfig, + SubagentsPlugin, ToolOrigin, + ToolResult, ToolSpec, TracingOptions, - build_child_harness, call_tool, - create_subagent_tool, ) -from thinharness.hooks import current_tool_call_context, current_tool_runtime_context from thinharness.providers import ModelToolCall, ModelTurn +class FakeChildHost: + def __init__(self, outcome: ChildHarnessOutcome | None = None) -> None: + self.outcome = outcome + self.tools: list[ToolSpec] = [] + self.recipes: list[ChildHarnessRequest] = [] + self.requests: list[ChildHarnessRequest] = [] + + def register_delegation_tool(self, tool: ToolSpec, recipes) -> ToolSpec: + self.tools.append(tool) + self.recipes.extend(recipes) + return tool + + async def run(self, request: ChildHarnessRequest) -> ChildHarnessOutcome: + self.requests.append(request) + if self.outcome is None: + raise AssertionError("unexpected child run") + return self.outcome + + class ClosingProvider: + name = "OpenAI" + def __init__(self) -> None: - self.name = "OpenAI" self.closed = 0 async def aclose(self) -> None: self.closed += 1 -def test_subagent_config_validation_accepts_tool_specs() -> None: - spec = echo_tool() - sequential_tool = ToolSpec("sequential_echo", "Sequential echo", {"type": "object", "properties": {}}, lambda args: "ok", sequential=True) - config = SubAgentConfig(name="research.1", description="Research helper.", tools=[spec, sequential_tool]) - inherited = SubAgentConfig(name="general", description="General helper.", inherit_parent_tools=True) - - assert config.tools == [spec, sequential_tool] - assert inherited.inherit_parent_tools is True - with pytest.raises(ValueError, match="inherit_parent_tools"): - SubAgentConfig(name="bad", description="Bad helper.", inherit_parent_tools=True, plugins=[FilesystemPlugin(tools=["read"])]) - with pytest.raises(ValueError, match="SubAgentConfig.builtin_tools has been removed"): - SubAgentConfig(name="removed", description="Removed helper.", builtin_tools=["subagent"], tools=[spec]) - with pytest.raises(ValueError, match="cannot be exposed"): - SubAgentConfig( - name="recursive-custom", - description="Recursive helper.", - tools=[ToolSpec("subagent", "Recursive", {"type": "object", "properties": {}}, lambda args: "bad")], - ) - with pytest.raises( - ValueError, - match="named subagents must define plugins, tools, inherit_parent_tools=True, inherit_mcp_servers=True, or mcp_servers", - ): - SubAgentConfig(name="empty", description="No tools.") - with pytest.raises(ValueError): - SubAgentConfig(name="bad name", description="Bad helper.", plugins=[FilesystemPlugin(tools=["read"])]) - with pytest.raises(ValueError, match="non-empty single line"): - SubAgentConfig(name="ok", description=" ", plugins=[FilesystemPlugin(tools=["read"])]) - with pytest.raises(ValueError, match="non-empty single line"): - SubAgentConfig(name="ok", description="Bad\nhelper.", plugins=[FilesystemPlugin(tools=["read"])]) - with pytest.raises(ValueError, match="SubAgentConfig.background has been removed"): - SubAgentConfig(name="old-background", description="Old helper.", plugins=[FilesystemPlugin(tools=["read"])], background="always") - -def test_subagent_rejects_duplicate_mcp_configuration_early() -> None: - server = MCPServerStdio("unused") - plugin = MCPPlugin(servers=[server]) - - with pytest.raises(ValueError, match="explicit MCPPlugin"): - SubAgentConfig( - name="explicit-and-servers", - description="Invalid MCP helper.", - plugins=[plugin], - mcp_servers=[server], - ) - with pytest.raises(ValueError, match="explicit MCPPlugin"): - SubAgentConfig( - name="explicit-and-inherit", - description="Invalid MCP helper.", - plugins=[plugin], - inherit_mcp_servers=True, - ) - assert SubAgentConfig(name="explicit", description="Explicit MCP helper.", plugins=[plugin]).plugins == [plugin] +def _parent_call(*, agent: str | None = None, task: str = "help", call_id: str = "call_1") -> ModelTurn: + agent_arg = f',"agent":"{agent}"' if agent is not None else "" + return ModelTurn( + tool_calls=[ModelToolCall(id=call_id, name="subagent", arguments=f'{{"task":"{task}"{agent_arg}}}')], + raw={"id": "parent-start"}, + ) -def test_inherited_mcp_without_parent_plugin_adds_no_child_plugin(tmp_path: Path) -> None: - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([])) - config = SubAgentConfig(name="mcp", description="MCP helper.", inherit_mcp_servers=True) +def _plugin_tool(plugin: SubagentsPlugin, tmp_path: Path) -> tuple[FakeChildHost, ToolSpec, PluginBinding]: + host = FakeChildHost() + binding = plugin.bind(PluginContext(root=tmp_path, model=ScriptedModel([]), child_harnesses=host)) + return host, binding.static.tools[0], binding - child = build_child_harness(parent, config) - assert not any(isinstance(plugin, MCPPlugin) for plugin in child.plugins) +def test_plugin_contributes_static_tool_and_default_agent(tmp_path: Path) -> None: + host, tool, binding = _plugin_tool(SubagentsPlugin(), tmp_path) + assert binding.agent_names == (DEFAULT_SUBAGENT_NAME,) + assert host.tools == [tool] + assert tool.name == "subagent" + assert tool.origin is None + assert set(tool.response_tool()["parameters"]["properties"]) == {"task", "agent"} + assert "Omit `agent`" in tool.description + assert host.recipes[0].tool_mode == "inherited" -def test_subagent_builtin_exposure_is_selectable(tmp_path: Path) -> None: - default = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([])) - disabled = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([])) - only_subagent = Harness(HarnessConfig(root=tmp_path, builtin_tools=["subagent"]), model=ScriptedModel([])) - assert "subagent" not in [tool["name"] for tool in default.tool_schemas()] - assert "subagent" not in [tool["name"] for tool in disabled.tool_schemas()] - assert [tool["name"] for tool in only_subagent.tool_schemas()] == ["subagent"] - schema = only_subagent.tool_schemas()[0]["parameters"] - assert set(schema["properties"]) == {"task", "agent"} - assert "tools" not in schema["properties"] +def test_plain_harness_has_no_delegation_and_same_named_direct_tool_is_valid(tmp_path: Path) -> None: + direct = ToolSpec( + "subagent", + "An ordinary custom tool.", + {"type": "object", "properties": {}}, + lambda _args: "ordinary", + origin=ToolOrigin(plugin="subagents"), + ) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), tools=[direct]) + + assert harness.tools == [direct] + with pytest.raises(ValueError, match="duplicate tool name"): + Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[SubagentsPlugin()], + tools=[direct], + ) -def test_default_subagent_runs_child_with_inherited_tools_and_structured_result(tmp_path: Path) -> None: - parent_call = ModelTurn( - tool_calls=[ModelToolCall(id="call_1", name="subagent", arguments='{"task":"help"}')], - raw={"id": "parent-start"}, + +def test_subagent_config_accepts_additive_and_model_only_children() -> None: + spec = echo_tool() + config = SubAgentConfig( + name="research.1", + description="Research helper.", + inherit_parent=True, + plugins=[FilesystemPlugin(tools=["read"])], + tools=[spec], ) - child_start_metadata = {} + model_only = SubAgentConfig(name="model-only", description="No tools.", model="openai:child") - def on_child_start(prompt, _instructions, tools, metadata, _previous_response_id): - child_start_metadata.update({"prompt": prompt, "tools": [tool["name"] for tool in tools], "metadata": metadata}) + assert config.inherit_parent is True + assert config.tools == (spec,) + assert model_only.plugins == () + assert model_only.tools == () - def on_parent_continue(outputs, _tools, _metadata): - envelope = tool_output(outputs[0].output) - assert envelope["ok"] is True - assert envelope["content"] == "child done" - assert envelope["metadata"]["agent"] == "default" - assert envelope["metadata"]["inherited"] is True - assert envelope["metadata"]["tools"] == ["echo"] - child = ScriptedSession(start_turn=ModelTurn(text="child done", raw={"id": "child"}), on_start=on_child_start) - parent = ScriptedSession(start_turn=parent_call, continue_turn=ModelTurn(text="parent done", raw={"id": "parent-done"}), on_continue=on_parent_continue) - model = ScriptedModel([parent, child]) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model, tools=[echo_tool()]) - harness.add_tool(create_subagent_tool(harness, [])) +@pytest.mark.parametrize("field", ["inherit_parent_tools", "inherit_mcp_servers", "mcp_servers", "builtin_tools"]) +def test_removed_subagent_config_fields_fail_by_name(field: str) -> None: + with pytest.raises(ValueError, match=rf"SubAgentConfig\.{field} has been removed"): + SubAgentConfig(name="old", description="Old helper.", **{field: True}) - result = harness.run_sync("delegate", metadata={"conversation_id": "conv-1", "extra": "ignored"}) - assert result.text == "parent done" - assert child_start_metadata == { - "prompt": "help", - "tools": ["echo"], - "metadata": {"conversation_id": "conv-1", "parent_call_id": "call_1"}, - } +def test_subagent_config_and_plugin_validate_order_names_plugins_and_tools() -> None: + with pytest.raises(ValueError, match="reserved"): + SubAgentConfig(name="default", description="Reserved.") + with pytest.raises(ValueError, match="single line"): + SubAgentConfig(name="bad", description="Bad\nline") + with pytest.raises(ValueError): + SubAgentConfig(name="bad name", description="Bad name.") + with pytest.raises(TypeError, match="ordered sequence"): + SubagentsPlugin(agents={SubAgentConfig(name="a", description="A.")}) # type: ignore[arg-type] + with pytest.raises(ValueError, match="duplicate subagent name"): + SubagentsPlugin(agents=[SubAgentConfig(name="a", description="A."), SubAgentConfig(name="a", description="Again.")]) + with pytest.raises(TypeError, match="Plugin values"): + SubAgentConfig(name="bad-plugin", description="Bad.", plugins=[object()]) + with pytest.raises(ValueError, match="SubagentsPlugin cannot"): + SubAgentConfig(name="nested", description="Nested.", plugins=[SubagentsPlugin()]) + approval = ToolSpec("approve", "Approve", {"type": "object"}, lambda _args: "ok", requires_approval=True) + with pytest.raises(ValueError, match="child harnesses"): + SubAgentConfig(name="approval", description="Approval.", tools=[approval]) + + +def test_child_hook_validation_and_registry_copy(tmp_path: Path) -> None: + registry = HookRegistry([Hook("run_start", lambda _ctx: None)], strict_hooks=True) + config = SubAgentConfig(name="safe", description="Safe.", hooks=registry) + plugin = SubagentsPlugin(default_hooks=registry, agents=[config]) + registry.hooks.clear() + host, _tool, _binding = _plugin_tool(plugin, tmp_path) + + assert isinstance(plugin.default_hooks, HookRegistry) + assert len(plugin.default_hooks.hooks) == 1 + assert all(isinstance(recipe.hooks, HookRegistry) for recipe in host.recipes) + assert all(recipe.hooks.strict_hooks for recipe in host.recipes if isinstance(recipe.hooks, HookRegistry)) + assert all(len(recipe.hooks.hooks) == 1 for recipe in host.recipes if isinstance(recipe.hooks, HookRegistry)) + with pytest.raises(ValueError, match="lifecycle"): + SubagentsPlugin(default_hooks=[Hook("before_subagent_run", lambda _ctx: None)]) + invalid = SubAgentConfig( + name="bad", + description="Bad.", + hooks=[Hook("after_subagent_run", lambda _ctx: None, agents=["bad"])], + ) + with pytest.raises(ValueError, match="lifecycle"): + SubagentsPlugin(agents=[invalid]) -def test_subagent_metadata_uses_runtime_context_without_leaking_hook_mutation(tmp_path: Path) -> None: - parent_call = ModelTurn( - tool_calls=[ModelToolCall(id="call_1", name="subagent", arguments='{"task":"help"}')], - raw={"id": "parent-start"}, + +def test_plugin_name_and_configuration_are_frozen() -> None: + plugin = SubagentsPlugin(agents=[SubAgentConfig(name="a", description="A.")]) + + with pytest.raises(AttributeError, match="fixed"): + plugin.name = "other" # type: ignore[misc] + with pytest.raises(AttributeError, match="frozen"): + plugin._agents = () + + +def test_unknown_and_blank_agent_are_normal_tool_results(tmp_path: Path) -> None: + host, tool, _ = _plugin_tool( + SubagentsPlugin(agents=[SubAgentConfig(name="research", description="Research.")]), + tmp_path, ) - child_start_metadata = {} - hook_metadata = [] - tool_contexts = [] - def on_child_start(prompt, _instructions, _tools, metadata, _previous_response_id): - child_start_metadata.update({"prompt": prompt, "metadata": metadata}) + unknown = tool_output(asyncio.run(tool.handler(tool.parse_args({"task": "x", "agent": "missing"}))).to_json()) + blank = tool_output(call_tool(tool, '{"task":"x","agent":""}')) - def before_subagent(ctx): - assert isinstance(ctx, BeforeSubagentRunContext) - hook_metadata.append(("before", dict(ctx.metadata))) - tool_contexts.append((current_tool_call_context(), current_tool_runtime_context())) - ctx.metadata["conversation_id"] = "mutated" - ctx.metadata["new"] = "ignored" + assert unknown["metadata"] == { + "agent": "missing", + "available": ["research"], + "error_type": "UnknownSubAgent", + } + assert blank["metadata"]["error_type"] == "ValidationError" + assert host.requests == [] - def after_subagent(ctx): - assert isinstance(ctx, AfterSubagentRunContext) - hook_metadata.append(("after", dict(ctx.metadata))) +def test_default_child_runs_with_frozen_direct_tools_and_returns_metadata(tmp_path: Path) -> None: + child_seen = {} + + def on_child_start(prompt, _instructions, tools, metadata, _previous_response_id): + child_seen.update(prompt=prompt, tools=[tool["name"] for tool in tools], metadata=metadata) + + parent = ScriptedSession( + start_turn=_parent_call(), + continue_turn=ModelTurn(text="parent done", raw={"id": "parent-done"}), + ) child = ScriptedSession(start_turn=ModelTurn(text="child done", raw={"id": "child"}), on_start=on_child_start) - parent = ScriptedSession(start_turn=parent_call, continue_turn=ModelTurn(text="parent done", raw={"id": "parent-done"})) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([parent, child]), + plugins=[SubagentsPlugin()], tools=[echo_tool()], - hooks=[Hook("before_subagent_run", before_subagent), Hook("after_subagent_run", after_subagent)], ) - harness.add_tool(create_subagent_tool(harness, [])) - assert harness.run_sync("delegate", metadata={"conversation_id": "conv-1", "extra": "hook-only"}).text == "parent done" + result = harness.run_sync("delegate", metadata={"conversation_id": "conv", "private": "parent"}) + envelope = tool_output(parent.continue_calls[0][0][0].output) - assert hook_metadata == [ - ("before", {"conversation_id": "conv-1", "extra": "hook-only"}), - ("after", {"conversation_id": "conv-1", "extra": "hook-only"}), - ] - assert child_start_metadata == {"prompt": "help", "metadata": {"conversation_id": "conv-1", "parent_call_id": "call_1"}} - assert tool_contexts == [ - ( - {"call_id": "call_1", "name": "subagent"}, - {"run_metadata": {"conversation_id": "conv-1", "extra": "hook-only"}}, - ) - ] + assert result.text == "parent done" + assert child_seen == { + "prompt": "help", + "tools": ["echo"], + "metadata": {"conversation_id": "conv", "parent_call_id": "call_1"}, + } + assert envelope["metadata"] == { + "agent": "default", + "inherited": True, + "tool_mode": "inherited", + "tools": ["echo"], + "model_requests": 1, + "structured_output": False, + } -def test_subagent_receives_fresh_child_budget_notices(tmp_path: Path) -> None: - parent_call = ModelTurn( - tool_calls=[ - ModelToolCall(id="call_1", name="echo", arguments='{"value":"parent"}'), - ModelToolCall(id="call_2", name="subagent", arguments='{"task":"help"}'), - ], - raw={"id": "parent-start"}, - ) - parent = ScriptedSession(start_turn=parent_call, continue_turn=ModelTurn(text="parent done", raw={"id": "parent-done"})) - child = ScriptedSession(start_turn=ModelTurn(text="child done", raw={"id": "child"})) - model = ScriptedModel([parent, child]) - harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], max_model_requests=2, max_tool_calls=2), - model=model, - tools=[echo_tool()], - ) - harness.add_tool(create_subagent_tool(harness, [])) - assert harness.run_sync("delegate").text == "parent done" +def test_named_model_only_child_has_no_tools(tmp_path: Path) -> None: + child_seen = {} - assert parent.notice_calls[0][1] == [] - assert [(notice.limit_kind, notice.remaining) for notice in parent.notice_calls[1][1]] == [("model_requests", 1), ("tool_calls", 0)] - assert child.notice_calls[0][1] == [] + def on_child_start(_prompt, _instructions, tools, _metadata, _previous_response_id): + child_seen["tools"] = tools -def test_default_subagent_does_not_close_shared_parent_provider(tmp_path: Path) -> None: - parent_call = ModelTurn( - tool_calls=[ModelToolCall(id="call_1", name="subagent", arguments='{"task":"help"}')], - raw={"id": "parent-start"}, + parent = ScriptedSession( + start_turn=_parent_call(agent="solo"), + continue_turn=ModelTurn(text="done", raw={}), + ) + child_model = ScriptedModel([ + ScriptedSession(start_turn=ModelTurn(text="solo answer", raw={}), on_start=on_child_start) + ]) + child_model.provider = ClosingProvider() + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([parent]), + plugins=[SubagentsPlugin(agents=[SubAgentConfig(name="solo", description="Solo.", model="openai:child")])], ) - parent = ScriptedSession(start_turn=parent_call, continue_turn=ModelTurn(text="parent done", raw={"id": "parent-done"})) - child = ScriptedSession(start_turn=ModelTurn(text="child done", raw={"id": "child"})) - model = ScriptedModel([parent, child]) - provider = ClosingProvider() - model.provider = provider - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model, tools=[echo_tool()]) - harness.add_tool(create_subagent_tool(harness, [])) - assert harness.run_sync("delegate").text == "parent done" + import thinharness.children as children_module - assert provider.closed == 0 + original = children_module.infer_model + children_module.infer_model = lambda *_args, **_kwargs: child_model + try: + assert harness.run_sync("delegate").text == "done" + finally: + children_module.infer_model = original -def test_named_inherited_subagent_gets_parent_tools_without_subagent(tmp_path: Path) -> None: - parent_echo = echo_tool() - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([]), tools=[parent_echo]) - parent.add_tool(create_subagent_tool(parent, [])) + envelope = tool_output(parent.continue_calls[0][0][0].output) + assert child_seen["tools"] == [] + assert envelope["metadata"]["tool_mode"] == "explicit" + assert child_model.provider.closed == 1 - child = build_child_harness(parent, SubAgentConfig(name="general", description="General helper.", inherit_parent_tools=True)) - assert child.tools == [parent_echo] - assert child.config.subagents == [] +def test_override_model_closes_when_child_construction_fails(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + parent_model = ScriptedModel([]) + child_model = ScriptedModel([]) + child_provider = ClosingProvider() + child_model.provider = child_provider + class ModelSensitivePlugin: + name = "model-sensitive" -def test_inherited_subagents_keep_workspace_instruction_without_duplicate_tools(tmp_path: Path) -> None: - parent = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), - model=ScriptedModel([]), - plugins=[FilesystemPlugin(tools=["read", "write"])], - ) + def bind(self, context): + tools = () if context.model is parent_model else ( + ToolSpec("duplicate", "Duplicate", {"type": "object"}, lambda _args: "plugin"), + ) + return PluginBinding(static=PluginContribution(tools=tools)) - default_child = build_child_harness(parent, None) - named_child = build_child_harness( - parent, - SubAgentConfig(name="general", description="General helper.", inherit_parent_tools=True), + parent = ScriptedSession( + start_turn=_parent_call(agent="broken"), + continue_turn=ModelTurn(text="done", raw={}), + ) + monkeypatch.setattr("thinharness.children.infer_model", lambda *_args, **_kwargs: child_model) + harness = Harness( + HarnessConfig(root=tmp_path), + model=parent_model, + plugins=[SubagentsPlugin(agents=[ + SubAgentConfig( + name="broken", + description="Fails during child construction.", + model="openai:child", + plugins=[ModelSensitivePlugin()], + tools=[ToolSpec("duplicate", "Duplicate", {"type": "object"}, lambda _args: "direct")], + ) + ])], ) + parent_model.sessions.append(parent) - for child in (default_child, named_child): - assert [tool.name for tool in child.tools] == ["read", "write"] - assert child.system_instructions().count(f"Workspace root: {tmp_path.resolve()}") == 1 + assert harness.run_sync("delegate").text == "done" + envelope = tool_output(parent.continue_calls[0][0][0].output) + assert envelope["metadata"]["error_type"] == "ValueError" + assert child_provider.closed == 1 -def test_inherited_subagent_reuses_parent_skill_registry(tmp_path: Path) -> None: - skill = tmp_path / "skills" / "demo" - skill.mkdir(parents=True) - (skill / "SKILL.md").write_text("---\nname: demo\ndescription: Demo skill\n---\nDemo body", encoding="utf-8") - skills_plugin = SkillsPlugin(tmp_path / "skills", tools=["skill_read"]) - parent = Harness( +def test_additive_inheritance_rebinds_plugins_and_appends_direct_tools(tmp_path: Path) -> None: + child_seen = {} + + def on_child_start(_prompt, instructions, tools, _metadata, _previous_response_id): + child_seen.update(instructions=instructions, tools=[tool["name"] for tool in tools]) + + parent = ScriptedSession(start_turn=_parent_call(agent="mixed"), continue_turn=ModelTurn(text="done", raw={})) + child = ScriptedSession(start_turn=ModelTurn(text="child", raw={}), on_start=on_child_start) + explicit = ToolSpec("explicit", "Explicit", {"type": "object"}, lambda _args: "ok") + harness = Harness( HarnessConfig(root=tmp_path), - model=ScriptedModel([]), - plugins=[skills_plugin], + model=ScriptedModel([parent, child]), + plugins=[ + FilesystemPlugin(tools=["read"]), + SubagentsPlugin(agents=[ + SubAgentConfig( + name="mixed", + description="Mixed.", + inherit_parent=True, + tools=[explicit], + ) + ]), + ], + tools=[echo_tool()], ) - child = build_child_harness(parent, SubAgentConfig(name="general", description="General helper.", inherit_parent_tools=True)) + harness.run_sync("delegate") + envelope = tool_output(parent.continue_calls[0][0][0].output) + + assert child_seen["tools"] == ["read", "echo", "explicit"] + assert child_seen["instructions"].count(f"Workspace root: {tmp_path.resolve()}") == 1 + assert envelope["metadata"]["tool_mode"] == "inherited+explicit" - child_skills_plugin = next(plugin for plugin in child.plugins if isinstance(plugin, SkillsPlugin)) - assert child_skills_plugin is skills_plugin - assert child_skills_plugin.registry is skills_plugin.registry - assert child.system_instructions().count("demo - Demo skill") == 1 - skill_read = next(tool for tool in child.tools if tool.name == "skill_read") - assert skill_read.handler.__self__ is skills_plugin.registry -def test_explicit_subagent_skill_tools_use_its_own_plugin(tmp_path: Path) -> None: +def test_skills_plugin_reuses_registry_and_summary_in_child(tmp_path: Path) -> None: skill = tmp_path / "skills" / "demo" skill.mkdir(parents=True) - (skill / "SKILL.md").write_text("---\nname: demo\ndescription: Demo skill\n---\nDemo body", encoding="utf-8") - parent_plugin = SkillsPlugin(tmp_path / "skills", tools=["skill_read"]) - child_plugin = SkillsPlugin(tmp_path / "skills", tools=["skill_read"]) - parent = Harness( - HarnessConfig(root=tmp_path), - model=ScriptedModel([]), - plugins=[parent_plugin], - ) - - child = build_child_harness( - parent, - SubAgentConfig(name="skilled", description="Skill helper.", plugins=[child_plugin]), - ) - - assert child_plugin.registry is not parent_plugin.registry - assert [tool.name for tool in child.tools] == ["skill_read"] - assert "demo - Demo skill" in child.system_instructions() - -def test_subagent_model_override_credential_forwarding(tmp_path: Path, monkeypatch) -> None: - calls = [] - - def fake_infer_model(model_ref, **kwargs): - calls.append((model_ref, kwargs)) - return ScriptedModel([]) - - monkeypatch.setattr("thinharness.subagents.infer_model", fake_infer_model) - parent = Harness( - HarnessConfig( - root=tmp_path, - builtin_tools=[], - model="openai:parent", - api_key="parent-key", - base_url="https://parent.example", - temperature=0.2, - max_tokens=4096, - effort="low", - extra_body={"seed": 1}, - request_retries=2, - request_retry_backoff=0.25, - ), - model=ScriptedModel([]), - ) - same_provider = SubAgentConfig(name="same", description="Same provider.", model="openai:child", tools=[echo_tool()]) - other_provider = SubAgentConfig(name="other", description="Other provider.", model="anthropic:child", tools=[echo_tool()]) - - same_child = build_child_harness(parent, same_provider) - other_child = build_child_harness(parent, other_provider) - - assert same_child.config.model == "openai:child" - assert other_child.config.model == "anthropic:child" - assert calls[0][1]["api_key"] == "parent-key" - assert calls[0][1]["base_url"] == "https://parent.example" - assert calls[0][1]["temperature"] == 0.2 - assert calls[0][1]["max_tokens"] == 4096 - assert calls[0][1]["effort"] == "low" - assert calls[0][1]["extra_body"] == {"seed": 1} - assert calls[0][1]["request_retries"] == 2 - assert calls[0][1]["request_retry_backoff"] == 0.25 - assert calls[1][1]["api_key"] is None - assert calls[1][1]["base_url"] is None - assert calls[1][1]["max_tokens"] == 4096 - assert calls[1][1]["effort"] == "low" - assert calls[1][1]["request_retries"] == 2 - assert calls[1][1]["request_retry_backoff"] == 0.25 - -def test_subagent_model_override_is_used_for_child_run(tmp_path: Path, monkeypatch) -> None: - child_model = RecordingModel([ScriptedSession(start_turn=ModelTurn(text="child done", raw={"id": "child"}))], model="child-model") - parent_model = RecordingModel([], model="parent-model") - - def fake_infer_model(_model_ref, **_kwargs): - return child_model - - monkeypatch.setattr("thinharness.subagents.infer_model", fake_infer_model) - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=parent_model) - child = build_child_harness(parent, SubAgentConfig(name="special", description="Special helper.", model="openai:child", tools=[echo_tool()])) - - assert child.model is child_model - assert child.run_sync("delegate").text == "child done" - assert child_model.session_requests == 1 - assert parent_model.session_requests == 0 - -def test_subagent_model_override_closes_child_provider(tmp_path: Path, monkeypatch) -> None: - parent_call = ModelTurn( - tool_calls=[ModelToolCall(id="call_1", name="subagent", arguments='{"task":"help","agent":"special"}')], - raw={"id": "parent-start"}, - ) - parent = ScriptedSession(start_turn=parent_call, continue_turn=ModelTurn(text="parent done", raw={"id": "parent-done"})) - child = ScriptedSession(start_turn=ModelTurn(text="child done", raw={"id": "child"})) - child_model = ScriptedModel([child], model="child-model") - child_provider = ClosingProvider() - child_model.provider = child_provider + (skill / "SKILL.md").write_text("---\nname: demo\ndescription: Demo skill\n---\nBody", encoding="utf-8") + plugin = SkillsPlugin(tmp_path / "skills", tools=["skill_read"]) + child_seen = {} - def fake_infer_model(_model_ref, **_kwargs): - return child_model + def on_child_start(_prompt, instructions, tools, _metadata, _previous_response_id): + child_seen.update(instructions=instructions, tools=[tool["name"] for tool in tools]) - monkeypatch.setattr("thinharness.subagents.infer_model", fake_infer_model) harness = Harness( - HarnessConfig( - root=tmp_path, - builtin_tools=[], - subagents=[SubAgentConfig(name="special", description="Special helper.", model="openai:child", tools=[echo_tool()])], - ), - model=ScriptedModel([parent]), + HarnessConfig(root=tmp_path), + model=ScriptedModel([ + ScriptedSession(start_turn=_parent_call(), continue_turn=ModelTurn(text="done", raw={})), + ScriptedSession(start_turn=ModelTurn(text="child", raw={}), on_start=on_child_start), + ]), + plugins=[plugin, SubagentsPlugin()], ) - harness.add_tool(create_subagent_tool(harness, harness.config.subagents)) + harness.run_sync("go") - assert harness.run_sync("delegate").text == "parent done" + assert plugin.for_child() is plugin + assert child_seen["tools"] == ["skill_read"] + assert child_seen["instructions"].count("demo - Demo skill") == 1 - assert child_provider.closed == 1 -async def test_concurrent_subagent_strict_abort_does_not_hang(tmp_path: Path) -> None: - parent_call = ModelTurn( - tool_calls=[ - ModelToolCall(id="call_1", name="subagent", arguments='{"task":"one"}'), - ModelToolCall(id="call_2", name="subagent", arguments='{"task":"two"}'), - ], - raw={"id": "parent-start"}, - ) +def test_mcp_and_non_inheritable_custom_plugins_do_not_inherit(tmp_path: Path) -> None: + class OrdinaryPlugin: + name = "ordinary" - async def wait_forever(_args): - await asyncio.Event().wait() + def bind(self, _context): + return PluginBinding(static=PluginContribution(tools=(ToolSpec("ordinary", "Ordinary", {"type": "object"}, lambda _args: "ok"),))) - child_tool = ToolSpec("wait", "Wait", {"type": "object", "properties": {}}, wait_forever) + seen = {} - def fail_second_subagent(ctx): - if ctx.call_id == "call_2": - raise RuntimeError("strict abort") + def on_child_start(_prompt, _instructions, tools, _metadata, _previous_response_id): + seen["tools"] = [tool["name"] for tool in tools] harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], strict_hooks=True), + HarnessConfig(root=tmp_path), model=ScriptedModel([ - ScriptedSession(start_turn=parent_call), - ScriptedSession(start_turn=ModelTurn(tool_calls=[ModelToolCall(id="child_call", name="wait", arguments="{}")], raw={"id": "child"})), + ScriptedSession(start_turn=_parent_call(), continue_turn=ModelTurn(text="done", raw={})), + ScriptedSession(start_turn=ModelTurn(text="child", raw={}), on_start=on_child_start), ]), - tools=[child_tool], - hooks=[Hook("before_tool_call", fail_second_subagent)], + plugins=[OrdinaryPlugin(), SubagentsPlugin()], ) - harness.add_tool(create_subagent_tool(harness, [])) + harness.run_sync("go") - task = asyncio.create_task(harness.run("delegate")) + assert seen["tools"] == [] - with pytest.raises(RuntimeError, match="strict abort"): - await asyncio.wait_for(task, timeout=1) -def test_subagent_child_provider_failure_returns_tool_error(tmp_path: Path) -> None: - parent_call = ModelTurn( - tool_calls=[ModelToolCall(id="call_1", name="subagent", arguments='{"task":"help"}')], - raw={"id": "parent-start"}, - ) +def test_custom_for_child_plugin_rebinds_and_invalid_return_fails(tmp_path: Path) -> None: + roots = [] - def on_parent_continue(outputs, _tools, _metadata): - envelope = tool_output(outputs[0].output) - assert envelope["ok"] is False - assert envelope["metadata"]["agent"] == "default" - assert envelope["metadata"]["inherited"] is True - assert envelope["metadata"]["tool_mode"] == "inherited" - assert envelope["metadata"]["tools"] == ["echo"] - assert envelope["metadata"]["error_type"] == "HarnessError" + class Inheritable: + name = "custom" - parent = ScriptedSession(start_turn=parent_call, continue_turn=ModelTurn(text="parent done", raw={"id": "parent-done"}), on_continue=on_parent_continue) - tracer = FakeTracer() - harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), - model=ScriptedModel([parent, FailingSession()]), - tools=[echo_tool()], - tracing=[TracingOptions(tracer=tracer)], - ) - harness.add_tool(create_subagent_tool(harness, [])) - - assert harness.run_sync("delegate").text == "parent done" - subagent_tool = next(span for span in tracer.spans if span.name == "execute_tool subagent") - assert subagent_tool.attributes["subagent.name"] == "default" - assert subagent_tool.attributes["subagent.tool_mode"] == "inherited" - assert subagent_tool.attributes["subagent.tools"] == ["echo"] - assert subagent_tool.status is not None + def for_child(self): + return self -def test_unknown_named_subagent_returns_structured_error(tmp_path: Path) -> None: - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([])) - tool = create_subagent_tool(harness, [SubAgentConfig(name="research", description="Research helper.", plugins=[FilesystemPlugin(tools=["read"])])]) + def bind(self, context): + roots.append(context.root) + return PluginBinding(static=PluginContribution(tools=(ToolSpec("custom", "Custom", {"type": "object"}, lambda _args: "ok"),))) - output = tool_output(asyncio.run(tool.handler(tool.parse_args({"task": "x", "agent": "missing"}))).as_json()) - - assert output["ok"] is False - assert output["metadata"]["available"] == ["research"] - assert output["metadata"]["error_type"] == "UnknownSubAgent" + plugin = Inheritable() + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([ + ScriptedSession(start_turn=_parent_call(), continue_turn=ModelTurn(text="done", raw={})), + ScriptedSession(start_turn=ModelTurn(text="child", raw={})), + ]), + plugins=[plugin, SubagentsPlugin()], + ) + harness.run_sync("go") + assert roots == [tmp_path.resolve(), tmp_path.resolve()] + + class Invalid(Inheritable): + name = "invalid" + + def for_child(self): + return object() + + with pytest.raises(TypeError, match="for_child"): + Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[Invalid(), SubagentsPlugin()]) + + +def test_known_child_plugin_and_tool_collisions_fail_parent_construction(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="duplicate plugin name"): + Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[ + FilesystemPlugin(tools=["read"]), + SubagentsPlugin(agents=[ + SubAgentConfig( + name="collision", + description="Collision.", + inherit_parent=True, + plugins=[FilesystemPlugin(tools=["write"])], + ) + ]), + ], + ) -def test_blank_subagent_name_is_normal_argument_validation_error(tmp_path: Path) -> None: - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([])) - tool = create_subagent_tool(harness, []) - harness.add_tool(tool) + direct = echo_tool() + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[SubagentsPlugin(agents=[ + SubAgentConfig(name="collision", description="Collision.", inherit_parent=True, tools=[direct]) + ])], + ) + with pytest.raises(ValueError, match="duplicate tool name"): + harness.add_tool(direct) - output = tool_output(call_tool(tool, '{"task":"x","agent":""}')) - assert output["ok"] is False - assert output["metadata"]["error_type"] == "ValidationError" - assert "Invalid arguments" in output["content"] +def test_inheritable_plugin_approval_tool_fails_parent_construction(tmp_path: Path) -> None: + class Unsafe: + name = "unsafe" -def test_subagent_tool_name_is_reserved_for_custom_tools(tmp_path: Path) -> None: - custom = ToolSpec("subagent", "Not the framework tool.", {"type": "object", "properties": {}}, lambda args: "bad") - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([])) + def for_child(self): + return self - with pytest.raises(ValueError, match="reserved tool name"): - Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([]), tools=[custom]) - with pytest.raises(ValueError, match="reserved tool name"): - harness.add_tool(custom) + def bind(self, _context): + tool = ToolSpec("unsafe", "Unsafe", {"type": "object"}, lambda _args: "ok", requires_approval=True) + return PluginBinding(static=PluginContribution(tools=(tool,))) -def test_framework_subagent_tool_can_be_added_after_construction(tmp_path: Path) -> None: - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([])) + with pytest.raises(ValueError, match="child harnesses"): + Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[Unsafe(), SubagentsPlugin()]) - harness.add_tool(create_subagent_tool(harness, [])) - assert [tool["name"] for tool in harness.tool_schemas()] == ["subagent"] +def test_direct_tool_added_during_run_is_visible_only_next_run(tmp_path: Path) -> None: + late = ToolSpec("late", "Late", {"type": "object"}, lambda _args: "ok") + seen: list[list[str]] = [] -def test_explicit_subagent_hook_registry_strict_mode_is_preserved(tmp_path: Path) -> None: - parent_call = ModelTurn( - tool_calls=[ModelToolCall(id="call_1", name="subagent", arguments='{"task":"help"}')], - raw={"id": "parent-start"}, - ) - outputs_seen = [] + def add_late(ctx): + if ctx.tool_name == "subagent" and "late" not in [tool.name for tool in ctx.harness.tools]: + ctx.harness.add_tool(late) - def on_continue(outputs, _tools, _metadata): - outputs_seen.append(tool_output(outputs[0].output)) + def child_start(_prompt, _instructions, tools, _metadata, _previous_response_id): + seen.append([tool["name"] for tool in tools]) - parent = ScriptedSession( - start_turn=parent_call, - continue_turn=ModelTurn(text="parent done", raw={"id": "parent-done"}), - on_continue=on_continue, - ) - child_registry = HookRegistry([Hook("run_start", lambda ctx: (_ for _ in ()).throw(RuntimeError("strict child")))], strict_hooks=True) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], strict_hooks=False), - model=ScriptedModel([parent, ScriptedSession(start_turn=ModelTurn(text="child", raw={"id": "child"}))]), - subagent_hooks={DEFAULT_SUBAGENT_NAME: child_registry}, + HarnessConfig(root=tmp_path), + model=ScriptedModel([ + ScriptedSession(start_turn=_parent_call(), continue_turn=ModelTurn(text="one", raw={})), + ScriptedSession(start_turn=ModelTurn(text="child one", raw={}), on_start=child_start), + ScriptedSession(start_turn=_parent_call(call_id="call_2"), continue_turn=ModelTurn(text="two", raw={})), + ScriptedSession(start_turn=ModelTurn(text="child two", raw={}), on_start=child_start), + ]), + plugins=[SubagentsPlugin()], + hooks=[Hook("before_tool_call", add_late, tools=["subagent"])], ) - harness.add_tool(create_subagent_tool(harness, [])) - assert harness.run_sync("go").text == "parent done" - assert child_registry.strict_hooks is True - assert outputs_seen[0]["ok"] is False - assert outputs_seen[0]["metadata"]["error_type"] == "RuntimeError" + assert asyncio.run(harness.run("first")).text == "one" + assert asyncio.run(harness.run("second")).text == "two" + assert seen == [[], ["late"]] -def test_subagent_hooks_and_child_hooks_are_explicit(tmp_path: Path) -> None: - parent_call = ModelTurn( - tool_calls=[ModelToolCall(id="call_1", name="subagent", arguments='{"task":"help","agent":"research"}')], - raw={"id": "parent-start"}, - ) - child = ScriptedSession(start_turn=ModelTurn(text="child done", raw={"id": "child"})) - parent = ScriptedSession(start_turn=parent_call, continue_turn=ModelTurn(text="parent done", raw={"id": "parent-done"})) + +def test_parent_hooks_child_hooks_metadata_and_cancellation(tmp_path: Path) -> None: events = [] - def before_subagent(ctx): + def before(ctx): assert isinstance(ctx, BeforeSubagentRunContext) - events.append((ctx.event, ctx.agent, ctx.parent_call_id)) + events.append((ctx.event, ctx.agent, dict(ctx.metadata))) + ctx.metadata["changed"] = True def child_start(ctx): - events.append((ctx.event, DEFAULT_SUBAGENT_NAME if ctx.harness.config.subagents else "child")) + events.append((ctx.event, "child", dict(ctx.metadata))) - def after_subagent(ctx): + def after(ctx): assert isinstance(ctx, AfterSubagentRunContext) - events.append((ctx.event, ctx.agent, ctx.usage.model_requests)) + events.append((ctx.event, ctx.agent, dict(ctx.metadata))) harness = Harness( - HarnessConfig( - root=tmp_path, - builtin_tools=[], - subagents=[SubAgentConfig(name="research", description="Research helper.", tools=[echo_tool()])], - ), - model=ScriptedModel([parent, child]), + HarnessConfig(root=tmp_path), + model=ScriptedModel([ + ScriptedSession(start_turn=_parent_call(), continue_turn=ModelTurn(text="done", raw={})), + ScriptedSession(start_turn=ModelTurn(text="child", raw={})), + ]), + plugins=[SubagentsPlugin(default_hooks=[Hook("run_start", child_start)])], hooks=[ - Hook("before_subagent_run", before_subagent, agents=["research"]), - Hook("after_subagent_run", after_subagent, agents=["research"]), + Hook("before_subagent_run", before, agents=["default"]), + Hook("after_subagent_run", after, agents=["default"]), ], - subagent_hooks={"research": [Hook("run_start", child_start)]}, ) - harness.add_tool(create_subagent_tool(harness, harness.config.subagents)) + harness.run_sync("go", metadata={"conversation_id": "c"}) - assert build_child_harness(harness, harness.config.subagents[0]).hooks.hooks - assert build_child_harness(harness, None).hooks.hooks == [] - assert harness.run_sync("delegate").text == "parent done" assert events == [ - ("before_subagent_run", "research", "call_1"), - ("run_start", "child"), - ("after_subagent_run", "research", 1), + ("before_subagent_run", "default", {"conversation_id": "c"}), + ("run_start", "child", {"conversation_id": "c", "parent_call_id": "call_1"}), + ("after_subagent_run", "default", {"conversation_id": "c"}), ] -def test_subagent_hook_can_cancel_default_agent_without_child_run(tmp_path: Path) -> None: - parent_call = ModelTurn( - tool_calls=[ModelToolCall(id="call_1", name="subagent", arguments='{"task":"help"}')], - raw={"id": "parent-start"}, - ) - events = [] - outputs_seen = [] - - def on_continue(outputs, _tools, _metadata): - outputs_seen.append(tool_output(outputs[0].output)) - - parent = ScriptedSession( - start_turn=parent_call, - continue_turn=ModelTurn(text="parent done", raw={"id": "parent-done"}), - on_continue=on_continue, - ) - def cancel(ctx): - assert isinstance(ctx, BeforeSubagentRunContext) - events.append((ctx.agent, ctx.parent_call_id)) ctx.cancelled = True ctx.cancel_reason = "blocked" - harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + parent = ScriptedSession(start_turn=_parent_call(), continue_turn=ModelTurn(text="blocked", raw={})) + blocked = Harness( + HarnessConfig(root=tmp_path), model=ScriptedModel([parent]), - hooks=[Hook("before_subagent_run", cancel, agents=[DEFAULT_SUBAGENT_NAME])], - subagent_hooks={DEFAULT_SUBAGENT_NAME: [Hook("run_start", lambda ctx: pytest.fail("child should not run"))]}, + plugins=[SubagentsPlugin()], + hooks=[Hook("before_subagent_run", cancel, agents=["default"])], ) - harness.add_tool(create_subagent_tool(harness, [])) + blocked.run_sync("go") + envelope = tool_output(parent.continue_calls[0][0][0].output) + assert envelope["metadata"]["error_type"] == "SubAgentCancelled" + + +def test_agent_filtered_hook_requires_plugin_and_known_name(tmp_path: Path) -> None: + hook = Hook("before_subagent_run", lambda _ctx: None, agents=["default"]) + with pytest.raises(ValueError, match="unknown subagent name"): + Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), hooks=[hook]) + with pytest.raises(ValueError, match="missing"): + Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[SubagentsPlugin()], + hooks=[Hook("before_subagent_run", lambda _ctx: None, agents=["missing"])], + ) - assert harness.run_sync("delegate").text == "parent done" - assert events == [(DEFAULT_SUBAGENT_NAME, "call_1")] - assert outputs_seen[0]["ok"] is False - assert outputs_seen[0]["content"] == "Subagent execution blocked by hook: blocked" - assert outputs_seen[0]["metadata"]["error_type"] == "SubAgentCancelled" -def test_default_subagent_name_is_reserved() -> None: - with pytest.raises(ValueError, match="reserved"): - SubAgentConfig(name=DEFAULT_SUBAGENT_NAME, description="Reserved.", plugins=[FilesystemPlugin(tools=["read"])]) +def test_child_provider_failure_and_close_are_reported(tmp_path: Path) -> None: + parent = ScriptedSession(start_turn=_parent_call(), continue_turn=ModelTurn(text="done", raw={})) + tracer = FakeTracer() + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([parent, FailingSession()]), + plugins=[SubagentsPlugin()], + tools=[echo_tool()], + tracing=[TracingOptions(tracer=tracer)], + ) + assert harness.run_sync("delegate").text == "done" + envelope = tool_output(parent.continue_calls[0][0][0].output) + assert envelope["ok"] is False + assert envelope["metadata"]["error_type"] == "HarnessError" + assert envelope["metadata"]["tools"] == ["echo"] + span = next(span for span in tracer.spans if span.name == "execute_tool subagent") + assert span.attributes["subagent.delegation"] is True -def test_inherited_skills_bridge_orders_instructions_and_keeps_unrelated_origin_tools(tmp_path: Path) -> None: - skill = tmp_path / "skills" / "demo" - skill.mkdir(parents=True) - (skill / "SKILL.md").write_text("---\nname: demo\ndescription: Demo skill\n---\nBody", encoding="utf-8") - unrelated = ToolSpec( - "skill_helper", - "Unrelated direct tool", - {"type": "object", "properties": {}}, - lambda _args: "ok", - origin=ToolOrigin(plugin="skills", source="caller"), + +def test_real_delegation_trace_marker_and_forged_direct_tool_classification(tmp_path: Path) -> None: + tracer = FakeTracer() + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([ + ScriptedSession(start_turn=_parent_call(), continue_turn=ModelTurn(text="done", raw={})), + ScriptedSession(start_turn=ModelTurn(text="child", raw={})), + ]), + plugins=[SubagentsPlugin()], + tracing=[TracingOptions(tracer=tracer)], ) - skills_plugin = SkillsPlugin(tmp_path / "skills", tools=["skill_read"]) - parent = Harness( + harness.run_sync("go") + span = next(span for span in tracer.spans if span.name == "execute_tool subagent") + assert span.attributes["subagent.delegation"] is True + assert span.attributes["subagent.name"] == "default" + + direct_tracer = FakeTracer() + ordinary = ToolSpec( + "subagent", + "Ordinary", + {"type": "object"}, + lambda _args: ToolResult(True, "ordinary", {"agent": "forged"}), + origin=ToolOrigin(plugin="subagents"), + ) + direct = Harness( HarnessConfig(root=tmp_path), - model=ScriptedModel([]), - plugins=[skills_plugin, FilesystemPlugin(tools=["read"])], - tools=[unrelated], + model=ScriptedModel([ + ScriptedSession( + start_turn=ModelTurn(tool_calls=[ModelToolCall(id="c", name="subagent", arguments="{}")], raw={}), + continue_turn=ModelTurn(text="done", raw={}), + ) + ]), + tools=[ordinary], + tracing=[TracingOptions(tracer=direct_tracer)], ) + direct.run_sync("go") + direct_span = next(span for span in direct_tracer.spans if span.name == "execute_tool subagent") + assert "subagent.delegation" not in direct_span.attributes + assert "subagent.name" not in direct_span.attributes - child = build_child_harness(parent, None) - instructions = child.system_instructions() - - assert [plugin.name for plugin in child.plugins] == ["filesystem", "skills"] - assert [tool.name for tool in child.tools] == ["skill_read", "read", "skill_helper"] - assert instructions.index("Workspace root:") < instructions.index("demo - Demo skill") - assert instructions.count("demo - Demo skill") == 1 - assert next(tool for tool in child.tools if tool.name == "skill_helper") is unrelated +def test_cancelled_real_delegation_is_marked_before_hook(tmp_path: Path) -> None: + tracer = FakeTracer() + def cancel(ctx): + ctx.cancelled = True -def test_parallel_llm_explicit_child_plugin_and_inherited_handler_behavior(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - parent_batch_model = ScriptedModel([]) - parent = Harness( + parent = ScriptedSession(start_turn=_parent_call(), continue_turn=ModelTurn(text="done", raw={})) + harness = Harness( HarnessConfig(root=tmp_path), - model=ScriptedModel([]), - plugins=[ParallelLlmPlugin(parent_batch_model)], + model=ScriptedModel([parent]), + plugins=[SubagentsPlugin()], + hooks=[Hook("before_tool_call", cancel, tools=["subagent"])], + tracing=[TracingOptions(tracer=tracer)], ) - parent_spec = next(tool for tool in parent.tools if tool.name == "parallel_llm") - - explicit_plugin = ParallelLlmPlugin(ScriptedModel([])) - explicit_child = build_child_harness( - parent, - SubAgentConfig(name="explicit", description="Explicit helper.", plugins=[explicit_plugin]), + harness.run_sync("go") + span = next(span for span in tracer.spans if span.name == "execute_tool subagent") + assert span.attributes["subagent.delegation"] is True + + +def test_child_disabled_host_blocks_custom_plugin_grandchild(tmp_path: Path) -> None: + class GrandchildPlugin: + name = "grandchild" + + def for_child(self): + return self + + def bind(self, context): + async def attempt(_args): + request = ChildHarnessRequest( + agent_name="nested", + agent_description="Nested", + trace_agent_name="nested", + task="x", + inherited=False, + tool_mode="explicit", + system_prompt="nested", + ) + await context.child_harnesses.run(request) + return "unexpected" + + return PluginBinding(static=PluginContribution(tools=(ToolSpec("attempt", "Attempt", {"type": "object"}, attempt),))) + + parent = ScriptedSession(start_turn=_parent_call(), continue_turn=ModelTurn(text="done", raw={})) + child = ScriptedSession( + start_turn=ModelTurn(tool_calls=[ModelToolCall(id="nested", name="attempt", arguments="{}")], raw={}), + continue_turn=ModelTurn(text="child done", raw={}), + ) + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([parent, child]), + plugins=[GrandchildPlugin(), SubagentsPlugin()], ) - monkeypatch.setattr("thinharness.subagents.infer_model", lambda *_args, **_kwargs: ScriptedModel([])) - inherited_child = build_child_harness( - parent, - SubAgentConfig( - name="inherited", - description="Inherited helper.", - inherit_parent_tools=True, - model="openai:child", - ), + harness.run_sync("go") + nested = tool_output(child.continue_calls[0][0][0].output) + assert nested["ok"] is False + assert nested["metadata"]["error_type"] == "HarnessError" + assert "cannot create nested" in nested["content"] + + +async def test_top_level_host_rejects_outside_active_tool_runtime(tmp_path: Path) -> None: + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[SubagentsPlugin()]) + request = ChildHarnessRequest( + agent_name="x", + agent_description="X", + trace_agent_name="x", + task="x", + inherited=False, + tool_mode="explicit", + system_prompt="x", ) - assert [tool.name for tool in explicit_child.tools] == ["parallel_llm"] - inherited_spec = next(tool for tool in inherited_child.tools if tool.name == "parallel_llm") - assert inherited_spec is parent_spec - assert not any(isinstance(plugin, ParallelLlmPlugin) for plugin in inherited_child.plugins) + with pytest.raises(Exception, match="active parent tool call"): + await harness._child_harnesses.run(request) + + +def test_plugin_object_reuse_binds_independent_hosts(tmp_path: Path) -> None: + plugin = SubagentsPlugin() + first = Harness(HarnessConfig(root=tmp_path / "one"), model=ScriptedModel([]), plugins=[plugin]) + second = Harness(HarnessConfig(root=tmp_path / "two"), model=ScriptedModel([]), plugins=[plugin]) + + assert first.tools[0].handler is not second.tools[0].handler + assert first._child_harnesses is not second._child_harnesses + + +def test_harness_removed_fields_and_constructor_helpers_are_gone() -> None: + for field in ("builtin_tools", "subagents"): + with pytest.raises(ValueError, match=rf"HarnessConfig\.{field} has been removed.*SubagentsPlugin"): + HarnessConfig(**{field: []}) + with pytest.raises(TypeError, match="subagent_hooks"): + Harness(subagent_hooks={}) # type: ignore[call-arg] + + import thinharness + + assert not hasattr(thinharness, "create_subagent_tool") + assert not hasattr(thinharness, "build_child_harness") + assert "kind" not in ToolSpec.__dataclass_fields__ diff --git a/tests/unit/test_tool_retry.py b/tests/unit/test_tool_retry.py index d403ebe..4c25c51 100644 --- a/tests/unit/test_tool_retry.py +++ b/tests/unit/test_tool_retry.py @@ -4,7 +4,7 @@ from pathlib import Path import pytest -from fakes import FakeTracer, MultiCallClient, ScriptedModel, _fake_openai, echo_tool, tool_output +from fakes import FakeChildHarnessHost, FakeTracer, MultiCallClient, ScriptedModel, _fake_openai, echo_tool, tool_output from pydantic import BaseModel, model_validator from thinharness import ( @@ -15,10 +15,11 @@ HarnessError, Hook, ModelRetry, + PluginContext, SubAgentConfig, + SubagentsPlugin, ToolSpec, TracingOptions, - build_child_harness, call_tool, ) from thinharness.providers import ModelToolCall, ModelTurn @@ -72,7 +73,7 @@ def flaky(_args): ModelTurn(tool_calls=[_call("flaky", "{}", "call_2")], raw={"id": "retry"}), ModelTurn(text="done", raw={"id": "done"}), ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([session]), tools=[ + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session]), tools=[ ToolSpec("flaky", "Flaky", {"type": "object", "properties": {}}, flaky), ]) @@ -95,7 +96,7 @@ class AgeArgs(BaseModel): ModelTurn(tool_calls=[_call("age", '{"age":5}', "call_2")], raw={"id": "retry"}), ModelTurn(text="done", raw={"id": "done"}), ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([session]), tools=[ + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session]), tools=[ ToolSpec("age", "Age", AgeArgs, lambda args: seen.append(args.age) or "ok"), ]) @@ -121,7 +122,7 @@ def handler(args): return "never" client = MultiCallClient([("inner", '{"value":"bad"}')]) - harness = Harness(HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), model=_fake_openai(client), tools=[ + harness = Harness(HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ ToolSpec("inner", "Inner", OuterArgs, handler), ]) @@ -149,7 +150,7 @@ def test_malformed_json_retries_then_succeeds(tmp_path: Path) -> None: ModelTurn(tool_calls=[_call("echo", '{"value":"ok"}', "call_2")], raw={"id": "retry"}), ModelTurn(text="done", raw={"id": "done"}), ) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([session]), tools=[echo_tool()]) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session]), tools=[echo_tool()]) result = harness.run_sync("go") retry = tool_output(session.tool_outputs[0][0].output) @@ -191,7 +192,7 @@ def test_tool_retries_exceeded_counts_over_budget_failure(tmp_path: Path) -> Non ModelTurn(tool_calls=[_call("flaky", "{}", "call_2")], raw={"id": "retry"}), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], tool_retries=1), + HarnessConfig(root=tmp_path, tool_retries=1), model=ScriptedModel([session]), tools=[ToolSpec("flaky", "Flaky", {"type": "object", "properties": {}}, lambda args: (_ for _ in ()).throw(ModelRetry("again")))], hooks=[ @@ -212,7 +213,7 @@ def test_tool_retries_exceeded_counts_over_budget_failure(tmp_path: Path) -> Non def test_tool_max_retries_zero_blocks_first_retry_continuation(tmp_path: Path) -> None: session = SequenceSession(ModelTurn(tool_calls=[_call("flaky", "{}")], raw={"id": "start"})) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], tool_retries=3), model=ScriptedModel([session]), tools=[ + harness = Harness(HarnessConfig(root=tmp_path, tool_retries=3), model=ScriptedModel([session]), tools=[ ToolSpec("flaky", "Flaky", {"type": "object", "properties": {}}, lambda args: (_ for _ in ()).throw(ModelRetry("no")), max_retries=0), ]) @@ -229,7 +230,7 @@ def test_tool_max_retries_override_wins_over_config_default(tmp_path: Path) -> N ) events = [] harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], tool_retries=3), + HarnessConfig(root=tmp_path, tool_retries=3), model=ScriptedModel([session]), tools=[ ToolSpec( @@ -257,7 +258,7 @@ def test_two_calls_same_tool_share_budget_and_skip_batch_continuation(tmp_path: ], raw={"id": "start"})) events = [] harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], tool_retries=1), + HarnessConfig(root=tmp_path, tool_retries=1), model=ScriptedModel([session]), tools=[ToolSpec("flaky", "Flaky", {"type": "object", "properties": {}}, lambda args: (_ for _ in ()).throw(ModelRetry("again")))], hooks=[Hook("run_end", lambda ctx: events.append(dict(ctx.usage.tool_retries)))], @@ -279,7 +280,7 @@ def cancel(ctx): ctx.cancel_reason = "blocked" harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ToolSpec("block", "Block", {"type": "object", "properties": {}}, lambda args: "bad")], hooks=[Hook("before_tool_call", cancel)], @@ -294,7 +295,7 @@ def cancel(ctx): def test_parallel_retry_and_success_outputs_preserve_model_order(tmp_path: Path) -> None: client = MultiCallClient([("retry", "{}"), ("ok", "{}")]) harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ ToolSpec("retry", "Retry", {"type": "object", "properties": {}}, lambda args: (_ for _ in ()).throw(ModelRetry("try again"))), @@ -317,7 +318,7 @@ async def retry(_args): raise ModelRetry("async retry") client = MultiCallClient([("async_retry", "{}")]) - harness = Harness(HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), model=_fake_openai(client), tools=[ + harness = Harness(HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ ToolSpec("async_retry", "Async retry", {"type": "object", "properties": {}}, retry), ]) @@ -338,7 +339,7 @@ async def handler(args): return "never" client = MultiCallClient([("inner", '{"value":"bad"}')]) - harness = Harness(HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), model=_fake_openai(client), tools=[ + harness = Harness(HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ ToolSpec("inner", "Inner", {"type": "object", "properties": {}}, handler), ]) @@ -369,7 +370,7 @@ def after(ctx): session = SequenceSession(ModelTurn(tool_calls=[_call("flaky", "{}")], raw={"id": "start"})) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], tool_retries=0), + HarnessConfig(root=tmp_path, tool_retries=0), model=ScriptedModel([session]), tools=[ToolSpec("flaky", "Flaky", {"type": "object", "properties": {}}, lambda args: (_ for _ in ()).throw(ModelRetry("again")))], hooks=[Hook("after_tool_call", after)], @@ -389,7 +390,7 @@ class AgeArgs(BaseModel): seen = [] client = MultiCallClient([("age", '{"age":"five"}')]) harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ToolSpec("age", "Age", AgeArgs, lambda args: "ok")], hooks=[Hook("after_tool_call", lambda ctx: seen.append(ctx.envelope.metadata))], @@ -410,7 +411,7 @@ def rewrite(ctx): client = MultiCallClient([("flaky", "{}")]) harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[ToolSpec("flaky", "Flaky", {"type": "object", "properties": {}}, lambda args: (_ for _ in ()).throw(ModelRetry("again")))], hooks=[Hook("after_tool_call", rewrite)], @@ -423,13 +424,21 @@ def rewrite(ctx): assert span.attributes["error.type"] == "ModelRetry" -def test_subagent_tool_retry_budget_inheritance(tmp_path: Path) -> None: - parent = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], tool_retries=4), model=ScriptedModel([]), tools=[echo_tool()]) +def test_subagent_tool_retry_budget_recipes(tmp_path: Path) -> None: + host = FakeChildHarnessHost() + captured = [] - default_child = build_child_harness(parent, None) - named_default = build_child_harness(parent, SubAgentConfig(name="named", description="Named helper.", tools=[echo_tool()])) - named_custom = build_child_harness(parent, SubAgentConfig(name="custom", description="Custom helper.", tools=[echo_tool()], tool_retries=2)) + def register(tool, recipes): + captured.extend(recipes) + return tool - assert default_child.config.tool_retries == 4 - assert named_default.config.tool_retries == 1 - assert named_custom.config.tool_retries == 2 + host.register_delegation_tool = register # type: ignore[method-assign] + plugin = SubagentsPlugin(agents=[ + SubAgentConfig(name="named", description="Named helper.", tools=[echo_tool()]), + SubAgentConfig(name="custom", description="Custom helper.", tools=[echo_tool()], tool_retries=2), + ]) + plugin.bind(PluginContext(root=tmp_path, model=ScriptedModel([]), child_harnesses=host)) + + assert captured[0].tool_retries is None + assert captured[1].tool_retries == 1 + assert captured[2].tool_retries == 2 diff --git a/tests/unit/test_tracing.py b/tests/unit/test_tracing.py index d46f958..00d9b79 100644 --- a/tests/unit/test_tracing.py +++ b/tests/unit/test_tracing.py @@ -2,6 +2,7 @@ import json from pathlib import Path +from runpy import run_path import httpx import pytest @@ -29,16 +30,17 @@ OpenAIProvider, OpenAIResponsesModel, SubAgentConfig, + SubagentsPlugin, ToolResult, ToolSpec, TracingOptions, - build_child_harness, - create_subagent_tool, ) from thinharness.projections import model_request_delta_from_prompt, model_request_delta_from_tool_outputs from thinharness.providers import ModelNotice, ModelToolCall, ModelTurn, TokenUsage, ToolOutput from thinharness.tracing import _SpanAdapter, annotate_model_request, create_local_tracing_options, serialize_attribute_value +event_from_span = run_path(str(Path(__file__).resolve().parents[2] / "scripts" / "build_transcripts.py"))["event_from_span"] + class Person(BaseModel): """Test structured-output type.""" @@ -185,11 +187,11 @@ def test_local_tracing_nests_subagent_spans(tmp_path: Path, monkeypatch: pytest. child = ScriptedSession(start_turn=ModelTurn(text="child done", raw={"id": "child"})) parent = ScriptedSession(start_turn=parent_call, continue_turn=ModelTurn(text="parent done", raw={"id": "parent-done"})) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], local_tracing=True, local_trace_dir=trace_dir), + HarnessConfig(root=tmp_path, local_tracing=True, local_trace_dir=trace_dir), model=ScriptedModel([parent, child]), + plugins=[SubagentsPlugin()], tools=[echo_tool()], ) - harness.add_tool(create_subagent_tool(harness, [])) harness.run_sync("delegate") @@ -287,7 +289,7 @@ async def no_sleep(_delay: float) -> None: provider=OpenAIProvider(api_key="key", request_retries=1, request_retry_backoff=0, http_client=client), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=model, tracing=[TracingOptions(tracer=tracer)], ) @@ -306,7 +308,7 @@ def test_tool_tracing_marks_normalized_failures(tmp_path: Path) -> None: client = MultiCallClient([("fail", "{}")]) tracer = FakeTracer() harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model", builtin_tools=[]), + HarnessConfig(root=tmp_path, model="openai:test-model"), model=_fake_openai(client), tools=[failing], tracing=[TracingOptions(tracer=tracer, capture_messages=True)], @@ -328,7 +330,7 @@ async def test_assistant_text_and_tool_calls_project_to_trace_and_stream(tmp_pat session = ScriptedSession(start_turn=turn, continue_turn=ModelTurn(text="done", raw={"id": "done"})) tracer = FakeTracer() harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([session]), tools=[echo_tool()], tracing=[TracingOptions(tracer=tracer, capture_messages=True)], @@ -357,12 +359,12 @@ def test_subagent_tracing_nests_child_under_parent_tool_span(tmp_path: Path) -> parent = ScriptedSession(start_turn=parent_call, continue_turn=ModelTurn(text="parent done", raw={"id": "parent-done"})) tracer = FakeTracer() harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([parent, child]), + plugins=[SubagentsPlugin()], tools=[echo_tool()], tracing=[TracingOptions(tracer=tracer, capture_messages=True)], ) - harness.add_tool(create_subagent_tool(harness, [])) harness.run_sync("delegate") @@ -394,10 +396,13 @@ def test_subagent_runs_with_tracing_disabled(tmp_path: Path) -> None: ) child = ScriptedSession(start_turn=ModelTurn(text="child done", raw={"id": "child"})) parent = ScriptedSession(start_turn=parent_call, continue_turn=ModelTurn(text="parent done", raw={"id": "parent-done"})) - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([parent, child]), tools=[echo_tool()]) - harness.add_tool(create_subagent_tool(harness, [])) + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([parent, child]), + plugins=[SubagentsPlugin()], + tools=[echo_tool()], + ) - assert build_child_harness(harness, None).tracing == [] assert harness.run_sync("delegate").text == "parent done" def test_concurrent_subagent_fanout_keeps_each_child_under_own_tool_span(tmp_path: Path) -> None: @@ -413,12 +418,12 @@ def test_concurrent_subagent_fanout_keeps_each_child_under_own_tool_span(tmp_pat parent = ScriptedSession(start_turn=parent_call, continue_turn=ModelTurn(text="parent done", raw={"id": "parent-done"})) tracer = ContextFakeTracer() harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([parent, child_a, child_b]), + plugins=[SubagentsPlugin()], tools=[echo_tool()], tracing=[TracingOptions(tracer=tracer, capture_messages=True)], ) - harness.add_tool(create_subagent_tool(harness, [])) assert harness.run_sync("delegate").text == "parent done" @@ -453,18 +458,18 @@ def test_trace_request_kinds_for_resume_and_output_retries(tmp_path: Path) -> No tracer = FakeTracer() Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool", max_model_requests=2), + HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool", max_model_requests=2), model=model, tracing=[TracingOptions(tracer=tracer, capture_messages=True)], ).run_sync("make a person") Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool"), + HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool"), model=model, tracing=[TracingOptions(tracer=tracer, capture_messages=True)], ).run_sync("make another") - first = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model).run_sync("first") + first = Harness(HarnessConfig(root=tmp_path), model=model).run_sync("first") Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=model, tracing=[TracingOptions(tracer=tracer, capture_messages=True)], ).run_sync("follow-up", resume_from=first.resume_state) @@ -487,7 +492,7 @@ def _chat_span_for_turn(tmp_path: Path, turn: ModelTurn) -> FakeSpan: """Run one scripted turn and return its chat span.""" tracer = FakeTracer() Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([ScriptedSession(start_turn=turn)]), tracing=[TracingOptions(tracer=tracer)], ).run_sync("go") @@ -550,7 +555,7 @@ def test_custom_model_turn_without_normalized_fields_falls_back_to_raw(tmp_path: def test_provider_error_keeps_trace_input_without_output(tmp_path: Path) -> None: tracer = FakeTracer() harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([FailingSession()]), tracing=[TracingOptions(tracer=tracer, capture_messages=True)], ) @@ -564,6 +569,37 @@ def test_provider_error_keeps_trace_input_without_output(tmp_path: Path) -> None assert "gen_ai.completion" not in root.attributes assert root.status is not None +def test_transcript_classification_requires_authoritative_delegation_marker() -> None: + base = { + "name": "execute_tool subagent", + "attributes": { + "gen_ai.tool.name": "subagent", + "gen_ai.tool.call.id": "call_1", + "gen_ai.tool.call.result": ToolResult(True, "done", {"agent": "forged"}).to_json(), + }, + } + + ordinary = event_from_span(base, "trace.jsonl", 1, {}) + delegated = event_from_span( + { + **base, + "attributes": { + **base["attributes"], + "subagent.delegation": True, + "subagent.name": "default", + }, + }, + "trace.jsonl", + 2, + {}, + ) + + assert ordinary["kind"] == "tool" + assert ordinary["subagent_name"] == "" + assert delegated["kind"] == "subagent" + assert delegated["subagent_name"] == "default" + + def test_unknown_named_subagent_trace_marks_failed_without_child_tool_mode(tmp_path: Path) -> None: parent_call = ModelTurn( tool_calls=[ModelToolCall(id="call_1", name="subagent", arguments='{"task":"help","agent":"missing"}')], @@ -581,17 +617,17 @@ def on_parent_continue(outputs, _tools, _metadata): tracer = FakeTracer() parent = ScriptedSession(start_turn=parent_call, continue_turn=ModelTurn(text="parent done", raw={"id": "parent-done"}), on_continue=on_parent_continue) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[]), + HarnessConfig(root=tmp_path), model=ScriptedModel([parent]), + plugins=[SubagentsPlugin(agents=[ + SubAgentConfig( + name="research", + description="Research helper.", + plugins=[FilesystemPlugin(tools=["read"])], + ) + ])], tracing=[TracingOptions(tracer=tracer)], ) - harness.add_tool(create_subagent_tool(harness, [ - SubAgentConfig( - name="research", - description="Research helper.", - plugins=[FilesystemPlugin(tools=["read"])], - ) - ])) assert harness.run_sync("delegate").text == "parent done" subagent_tool = next(span for span in tracer.spans if span.name == "execute_tool subagent") diff --git a/tests/unit/test_turns.py b/tests/unit/test_turns.py index 54e28de..3d22ac6 100644 --- a/tests/unit/test_turns.py +++ b/tests/unit/test_turns.py @@ -331,10 +331,10 @@ def test_correction_following_resume_uses_same_session(tmp_path: Path) -> None: continue_turn=ModelTurn(text='{"name":"Ada","age":37}', raw={"id": "corrected"}), ) model = _ScriptedResumeModel([first_session, resumed_session]) - first = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=model).run_sync("first") + first = Harness(HarnessConfig(root=tmp_path), model=model).run_sync("first") resumed = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="prompted"), + HarnessConfig(root=tmp_path, output_type=Person, output_mode="prompted"), model=model, ).run_sync("follow-up", resume_from=first.resume_state) @@ -359,7 +359,7 @@ async def test_model_message_event_finalized_output_mode_populated(tmp_path: Pat ), ) harness = Harness( - HarnessConfig(root=tmp_path, builtin_tools=[], output_type=Person, output_mode="tool"), + HarnessConfig(root=tmp_path, output_type=Person, output_mode="tool"), model=ScriptedModel([session]), ) diff --git a/thinharness/__init__.py b/thinharness/__init__.py index c7a7aab..6e5089f 100644 --- a/thinharness/__init__.py +++ b/thinharness/__init__.py @@ -38,6 +38,11 @@ ) from .output import NativeOutput, OutputSchema, PromptedOutput, TextOutput, ToolStructuredOutput from .plugins import ( + DEFAULT_SUBAGENT_NAME, + ChildHarnessHost, + ChildHarnessOutcome, + ChildHarnessRequest, + ChildInheritablePlugin, FilesystemPlugin, MCPPlugin, ParallelLlmPlugin, @@ -47,6 +52,9 @@ PluginContext, PluginContribution, SkillsPlugin, + SubAgentArgs, + SubAgentConfig, + SubagentsPlugin, ) from .providers import ( AnthropicMessagesModel, @@ -70,7 +78,6 @@ infer_model, parse_model_ref, ) -from .subagents import DEFAULT_SUBAGENT_NAME, SubAgentArgs, SubAgentConfig, build_child_harness, create_subagent_tool from .tools import ( BashArgs, BashTool, @@ -108,6 +115,10 @@ __all__ = [ "__version__", "BashArgs", + "ChildHarnessHost", + "ChildHarnessOutcome", + "ChildHarnessRequest", + "ChildInheritablePlugin", "BashTool", "FileTools", "FilesystemPlugin", @@ -165,6 +176,7 @@ "SkillRegistry", "SubAgentArgs", "SubAgentConfig", + "SubagentsPlugin", "DEFAULT_SUBAGENT_NAME", "Model", "ModelCapabilities", @@ -203,12 +215,10 @@ "LocalTracing", "OtlpTracing", "TracingOptions", - "build_child_harness", "call_tool", "contained_path", "create_local_tracing", "create_local_tracing_options", - "create_subagent_tool", "create_otlp_tracing", "infer_model", "parse_model_ref", diff --git a/thinharness/_migration.py b/thinharness/_migration.py index 8df7fbc..3f7dcad 100644 --- a/thinharness/_migration.py +++ b/thinharness/_migration.py @@ -5,6 +5,8 @@ from collections.abc import Mapping REMOVED_HARNESS_CONFIG_FIELDS = { + "builtin_tools": "plugin composition; use SubagentsPlugin for delegation", + "subagents": "SubagentsPlugin(agents=[...])", "skills_dir": "SkillsPlugin", "selected_skills": "SkillsPlugin", "read_paths": "ParallelLlmPlugin", diff --git a/thinharness/children.py b/thinharness/children.py new file mode 100644 index 0000000..26d1036 --- /dev/null +++ b/thinharness/children.py @@ -0,0 +1,459 @@ +"""Narrow child-harness execution contracts and core host implementation.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from types import TracebackType +from typing import TYPE_CHECKING, Any, Literal, Protocol + +from .events import RunCompletedEvent, current_stream_emitter +from .hooks import ( + AfterSubagentRunContext, + BeforeSubagentRunContext, + Hook, + HookRegistry, + current_tool_call_context, + current_tool_runtime_context, +) +from .providers import Model, infer_model, same_provider_model_ref +from .tools.base import Json, ToolSpec +from .types import HarnessError, HarnessResult + +if TYPE_CHECKING: + from .core import Harness + from .plugins.base import Plugin + + +@dataclass(frozen=True) +class ChildHarnessRequest: + """Opaque plugin-owned recipe for one fresh child harness run.""" + + agent_name: str + agent_description: str + trace_agent_name: str + task: str + inherited: bool + tool_mode: Literal["inherited", "inherited+explicit", "explicit"] + system_prompt: str + model: str | None = None + plugins: tuple[Plugin, ...] = () + tools: tuple[ToolSpec, ...] = () + hooks: HookRegistry | tuple[Hook, ...] | None = None + max_model_requests: int | None = None + max_tool_calls: int | None = None + output_type: Any | None = None + output_mode: Literal["auto", "native", "tool", "prompted"] = "auto" + output_retries: int = 1 + tool_retries: int | None = 1 + + +@dataclass(frozen=True) +class ChildHarnessOutcome: + """Result data returned by a child host to its delegation plugin.""" + + result: HarnessResult | None + tools: tuple[str, ...] + content: str = "" + structured_output: bool = False + error: BaseException | None = None + error_type: str | None = None + error_message: str | None = None + + +class ChildHarnessHost(Protocol): + """Narrow host capability available to trusted plugins.""" + + def register_delegation_tool( + self, + tool: ToolSpec, + recipes: Sequence[ChildHarnessRequest], + ) -> ToolSpec: + """Register authoritative delegation provenance and static child recipes.""" + ... + + async def run(self, request: ChildHarnessRequest) -> ChildHarnessOutcome: + """Build, run, and close one child harness.""" + ... + + +@dataclass(frozen=True) +class _ToolComposition: + """Core-owned provenance for one composed tool.""" + + source: Literal["direct", "plugin"] + plugin_index: int | None = None + delegation: bool = False + + +class _DisabledChildHarnessHost: + """Reject every child request before any resource can be created.""" + + def register_delegation_tool(self, tool: ToolSpec, recipes: Sequence[ChildHarnessRequest]) -> ToolSpec: + del tool, recipes + raise HarnessError("child harnesses cannot create nested child harnesses") + + async def run(self, request: ChildHarnessRequest) -> ChildHarnessOutcome: + del request + raise HarnessError("child harnesses cannot create nested child harnesses") + + +_DISABLED_CHILD_HOST = _DisabledChildHarnessHost() + + +class _ParentChildHarnessHost: + """Parent-holding core implementation of the narrow child host.""" + + def __init__(self, parent: Harness) -> None: + self._parent = parent + self._delegation_tools: set[int] = set() + self._recipes: list[ChildHarnessRequest] = [] + + def register_delegation_tool(self, tool: ToolSpec, recipes: Sequence[ChildHarnessRequest]) -> ToolSpec: + if not isinstance(tool, ToolSpec): + raise TypeError("delegation tool must be a ToolSpec") + if isinstance(recipes, (set, frozenset)): + raise TypeError("child recipes must be an ordered sequence") + registered = tuple(recipes) + if any(not isinstance(recipe, ChildHarnessRequest) for recipe in registered): + raise TypeError("child recipe must be a ChildHarnessRequest") + self._delegation_tools.add(id(tool)) + self._recipes.extend(registered) + return tool + + def is_delegation_tool(self, tool: ToolSpec) -> bool: + """Return whether a plugin registered this exact static tool.""" + return id(tool) in self._delegation_tools + + def validate_recipes( + self, + tools: Sequence[ToolSpec], + compositions: Sequence[_ToolComposition], + ) -> None: + """Validate statically knowable child composition before a parent is usable.""" + for recipe in self._recipes: + self._validate_recipe(recipe, tools, compositions) + + def _validate_recipe( + self, + recipe: ChildHarnessRequest, + tools: Sequence[ToolSpec], + compositions: Sequence[_ToolComposition], + ) -> None: + from .plugins.base import ChildInheritablePlugin, PluginBinding, PluginContext + + child_plugins: list[Plugin] = [] + inherited_indices: list[int] = [] + if recipe.inherited: + for index, plugin in enumerate(self._parent.plugins): + if not isinstance(plugin, ChildInheritablePlugin): + continue + rebound = plugin.for_child() + _validate_rebound_plugin(plugin, rebound) + child_plugins.append(rebound) + inherited_indices.append(index) + child_plugins.extend(recipe.plugins) + _validate_plugin_names(child_plugins) + + child_tool_names: list[str] = [] + inherited_index_set = set(inherited_indices) + for tool, composition in zip(tools, compositions, strict=True): + if composition.source == "plugin" and composition.plugin_index in inherited_index_set: + if tool.requires_approval: + raise ValueError("approval-required tools are not supported inside child harnesses") + child_tool_names.append(tool.name) + + inherited_count = len(inherited_indices) + context = PluginContext(root=self._parent.root, model=self._parent.model, child_harnesses=_DISABLED_CHILD_HOST) + for plugin in child_plugins[inherited_count:]: + binding = plugin.bind(context) + if not isinstance(binding, PluginBinding): + raise TypeError(f"plugin {plugin.name!r} returned an invalid binding") + for tool in binding.static.tools: + if tool.requires_approval: + raise ValueError("approval-required tools are not supported inside child harnesses") + child_tool_names.append(tool.name) + + if recipe.inherited: + for tool, composition in zip(tools, compositions, strict=True): + if composition.source == "direct" and not tool.requires_approval: + child_tool_names.append(tool.name) + for tool in recipe.tools: + if tool.requires_approval: + raise ValueError("approval-required tools are not supported inside child harnesses") + child_tool_names.append(tool.name) + duplicate = next((name for index, name in enumerate(child_tool_names) if name in child_tool_names[:index]), None) + if duplicate is not None: + raise ValueError(f"duplicate tool name: {duplicate}") + + async def run(self, request: ChildHarnessRequest) -> ChildHarnessOutcome: + """Run one child using the active parent's frozen tool composition.""" + runtime = current_tool_runtime_context() + tool_call = current_tool_call_context() + if runtime is None or tool_call is None: + raise HarnessError("child harness request requires an active parent tool call") + tool_map = runtime.get("tool_map") + composition_map = runtime.get("tool_composition") + if not isinstance(tool_map, dict) or not isinstance(composition_map, dict): + raise HarnessError("child harness request requires an active frozen tool runtime") + active_name = str(tool_call.get("name", "")) + active_composition = composition_map.get(active_name) + if not isinstance(active_composition, _ToolComposition) or not active_composition.delegation: + raise HarnessError("child harness request requires a registered delegation tool") + + parent_metadata = _parent_run_metadata(runtime) + parent_call_id = str(tool_call["call_id"]) if tool_call.get("call_id") else None + before = BeforeSubagentRunContext( + harness=self._parent, + metadata=dict(parent_metadata), + agent=request.agent_name, + task=request.task, + inherited=request.inherited, + tool_mode=request.tool_mode, + parent_harness=self._parent, + parent_call_id=parent_call_id, + ) + self._parent.hooks.fire(before) + if before.cancelled: + reason = before.cancel_reason or "unspecified" + return ChildHarnessOutcome( + result=None, + tools=(), + error_type="SubAgentCancelled", + error_message=f"Subagent execution blocked by hook: {reason}", + ) + + effective_tools: tuple[str, ...] = () + try: + child_model, owns_model = self._resolve_child_model(request) + try: + child = self._build_child( + request, + tool_map, + composition_map, + child_model=child_model, + owns_model=owns_model, + ) + except BaseException as build_error: + if owns_model: + await _close_model_after_failure(child_model, build_error) + raise + result: HarnessResult | None = None + run_error: BaseException | None = None + run_traceback: TracebackType | None = None + try: + await child.connect() + effective_tools = tuple(tool.name for tool in child.tools) + emitter = current_stream_emitter() + child_metadata = _child_metadata(parent_metadata, parent_call_id) + if emitter is not None and emitter.ctx.options.include_subagents: + child_stream = child.stream( + request.task, + metadata=child_metadata, + stream_options=emitter.ctx.options, + _parent_run_id=emitter.ctx.run_id, + _parent_tool_call_id=parent_call_id, + _agent_name=request.agent_name, + ) + try: + async for event in child_stream: + emitter.emit_forwarded(event) + if isinstance(event, RunCompletedEvent) and event.run_id == child_stream.run_id: + result = event.result + finally: + await child_stream.aclose() + else: + result = await child.run(request.task, metadata=child_metadata) + except BaseException as exc: + run_error = exc + run_traceback = exc.__traceback__ + try: + await child.aclose() + except BaseException as close_error: + if run_error is None: + raise + run_error.add_note(f"cleanup also failed: {type(close_error).__name__}: {close_error}") + if run_error is not None: + raise run_error.with_traceback(run_traceback) + except Exception as exc: + self._parent.hooks.fire( + AfterSubagentRunContext( + harness=self._parent, + metadata=dict(parent_metadata), + agent=request.agent_name, + task=request.task, + error=exc, + tools=list(effective_tools), + parent_call_id=parent_call_id, + ) + ) + return ChildHarnessOutcome( + result=None, + tools=effective_tools, + error=exc, + error_type=type(exc).__name__, + error_message=str(exc), + ) + + assert result is not None + self._parent.hooks.fire( + AfterSubagentRunContext( + harness=self._parent, + metadata=dict(parent_metadata), + agent=request.agent_name, + task=request.task, + result=result, + tools=list(effective_tools), + usage=result.usage, + parent_call_id=parent_call_id, + ) + ) + structured_output = result.output is not None + content = child.output_schema.dump(result.output) if structured_output and child.output_schema is not None else result.text + return ChildHarnessOutcome( + result=result, + tools=effective_tools, + content=content, + structured_output=structured_output, + ) + + def _build_child( + self, + request: ChildHarnessRequest, + frozen_tool_map: dict[str, ToolSpec], + frozen_composition: dict[str, _ToolComposition], + *, + child_model: Model, + owns_model: bool, + ) -> Harness: + from .core import Harness + from .plugins.base import ChildInheritablePlugin + + parent_config = self._parent.config + child_plugins: list[Plugin] = [] + if request.inherited: + for plugin in self._parent.plugins: + if isinstance(plugin, ChildInheritablePlugin): + rebound = plugin.for_child() + _validate_rebound_plugin(plugin, rebound) + child_plugins.append(rebound) + child_plugins.extend(request.plugins) + _validate_plugin_names(child_plugins) + + child_tools: list[ToolSpec] = [] + if request.inherited: + for name, tool in frozen_tool_map.items(): + composition = frozen_composition.get(name) + if composition is not None and composition.source == "direct" and not tool.requires_approval: + child_tools.append(tool) + child_tools.extend(request.tools) + + child_config = parent_config.model_copy( + update={ + "model": request.model if request.model is not None else parent_config.model, + "root": self._parent.root, + "system_prompt": request.system_prompt, + "max_model_requests": request.max_model_requests if request.max_model_requests is not None else parent_config.max_model_requests, + "max_tool_calls": request.max_tool_calls if request.max_tool_calls is not None else parent_config.max_tool_calls, + "output_type": request.output_type, + "output_mode": request.output_mode, + "output_retries": request.output_retries, + "tool_retries": request.tool_retries if request.tool_retries is not None else parent_config.tool_retries, + } + ) + hooks: HookRegistry | list[Hook] | None + if isinstance(request.hooks, HookRegistry): + hooks = HookRegistry(list(request.hooks.hooks), strict_hooks=request.hooks.strict_hooks) + else: + hooks = list(request.hooks) if request.hooks is not None else None + tracing = [ + option.model_copy( + update={ + "agent_name": request.trace_agent_name, + "agent_description": request.agent_description, + } + ) + for option in self._parent.tracing + ] + return Harness( + child_config, + model=child_model, + plugins=child_plugins, + tools=child_tools, + tracing=tracing, + hooks=hooks, + _owns_model=owns_model, + _is_child_harness=True, + _child_harnesses=_DISABLED_CHILD_HOST, + ) + + def _resolve_child_model(self, request: ChildHarnessRequest) -> tuple[Model, bool]: + """Borrow the parent model or infer one owned override model.""" + if request.model is None: + return self._parent.model, False + parent_config = self._parent.config + same_provider = same_provider_model_ref(self._parent.model, request.model) + return infer_model( + request.model, + api_key=parent_config.api_key if same_provider else None, + base_url=parent_config.base_url if same_provider else None, + timeout=parent_config.request_timeout, + request_retries=parent_config.request_retries, + request_retry_backoff=parent_config.request_retry_backoff, + temperature=parent_config.temperature, + max_tokens=parent_config.max_tokens, + effort=parent_config.effort, + extra_body=parent_config.extra_body, + ), True + + +async def _close_model_after_failure(model: Model, original_error: BaseException) -> None: + """Close an inferred model after child construction fails without hiding that failure.""" + aclose = getattr(model.provider, "aclose", None) + if aclose is None: + return + try: + await aclose() + except BaseException as close_error: + original_error.add_note(f"cleanup also failed: {type(close_error).__name__}: {close_error}") + + +def _validate_rebound_plugin(parent_plugin: Plugin, rebound: object) -> None: + """Validate one structural child-inheritance result.""" + from .plugins.base import Plugin + + if not isinstance(rebound, Plugin): + raise TypeError(f"plugin {parent_plugin.name!r} for_child() returned an invalid plugin") + if rebound.name != parent_plugin.name: + raise ValueError( + f"plugin {parent_plugin.name!r} for_child() changed its fixed name to {rebound.name!r}" + ) + + +def _validate_plugin_names(plugins: Sequence[Plugin]) -> None: + """Validate one child plugin list with normal fixed-name rules.""" + names = [plugin.name for plugin in plugins] + if any(not isinstance(name, str) or not name.strip() for name in names): + raise ValueError("plugin name must be a non-empty string") + duplicate = next((name for index, name in enumerate(names) if name in names[:index]), None) + if duplicate is not None: + raise ValueError(f"duplicate plugin name: {duplicate}") + + +def _parent_run_metadata(runtime: dict[str, Any]) -> Json: + """Copy parent metadata from the active runtime.""" + metadata = runtime.get("run_metadata") + return dict(metadata) if isinstance(metadata, dict) else {} + + +def _child_metadata(parent_metadata: Json, parent_call_id: str | None) -> Json: + """Project correlation metadata into one child run.""" + metadata: Json = {} + if conversation_id := parent_metadata.get("conversation_id"): + metadata["conversation_id"] = conversation_id + if parent_call_id is not None: + metadata["parent_call_id"] = parent_call_id + return metadata + + +__all__ = ["ChildHarnessHost", "ChildHarnessOutcome", "ChildHarnessRequest"] diff --git a/thinharness/core.py b/thinharness/core.py index b55e3cf..5853f84 100644 --- a/thinharness/core.py +++ b/thinharness/core.py @@ -19,6 +19,7 @@ validate_approval_decisions, validate_approval_pause_state, ) +from .children import ChildHarnessHost, _ParentChildHarnessHost, _ToolComposition from .defaults import DEFAULT_SYSTEM_PROMPT from .events import ( ApprovalResumedEvent, @@ -57,7 +58,6 @@ infer_model, model_capabilities, ) -from .subagents import DEFAULT_SUBAGENT_NAME, SubAgentConfig, create_subagent_tool from .tools.base import ToolOrigin, ToolSpec from .tracing import ( LocalTracing, @@ -108,7 +108,6 @@ class HarnessConfig(BaseModel): api_key: str | None = None base_url: str | None = None system_prompt: str = DEFAULT_SYSTEM_PROMPT - builtin_tools: list[str] | None = None max_model_requests: int = 64 max_tool_calls: int | None = None strict_hooks: bool = False @@ -123,7 +122,6 @@ class HarnessConfig(BaseModel): local_tracing: bool = True local_trace_dir: str | Path = "~/.thinharness/traces" tool_execution: Literal["auto", "sequential"] = "auto" - subagents: list[SubAgentConfig] = Field(default_factory=list) output_type: OutputSpec | None = None output_mode: OutputMode = "auto" output_retries: int = Field(default=1, ge=0) @@ -152,12 +150,12 @@ def __init__( tools: list[ToolSpec] | None = None, tracing: list[TracingOptions] | None = None, hooks: list[Hook] | HookRegistry | None = None, - subagent_hooks: dict[str, list[Hook] | HookRegistry] | None = None, _owns_model: bool | None = None, - _is_child_run: bool = False, + _is_child_harness: bool = False, + _child_harnesses: ChildHarnessHost | None = None, ) -> None: self.config = config or HarnessConfig() - self._is_child_run = _is_child_run + self._is_child_harness = _is_child_harness self.root = Path(self.config.root).expanduser().resolve() self.model_ref = os.getenv("HARNESS_MODEL", self.config.model) self.model = model or infer_model( @@ -177,6 +175,7 @@ def __init__( output_schema = resolve_output_schema_for_model(self.model, self.config.output_type, self.config.output_mode) self.output_schema = output_schema + child_harnesses = _child_harnesses or _ParentChildHarnessHost(self) configured_plugins = tuple(plugins or []) plugin_names = [plugin.name for plugin in configured_plugins] if any(not isinstance(name, str) or not name.strip() for name in plugin_names): @@ -184,44 +183,68 @@ def __init__( duplicate_plugin = next((name for index, name in enumerate(plugin_names) if name in plugin_names[:index]), None) if duplicate_plugin is not None: raise ValueError(f"duplicate plugin name: {duplicate_plugin}") - bindings = tuple(plugin.bind(PluginContext(root=self.root, model=self.model)) for plugin in configured_plugins) + plugin_context = PluginContext(root=self.root, model=self.model, child_harnesses=child_harnesses) + bindings = tuple(plugin.bind(plugin_context) for plugin in configured_plugins) for plugin, binding in zip(configured_plugins, bindings, strict=True): if not isinstance(binding, PluginBinding): raise TypeError(f"plugin {plugin.name!r} returned an invalid binding") static_tools: list[ToolSpec] = [] + static_compositions: list[_ToolComposition] = [] static_instructions: list[str] = [] static_hooks: list[Hook] = [] - for plugin, binding in zip(configured_plugins, bindings, strict=True): + agent_names: list[str] = [] + for plugin_index, (plugin, binding) in enumerate(zip(configured_plugins, bindings, strict=True)): contribution = self._normalize_contribution(plugin.name, binding.static) static_tools.extend(contribution.tools) + static_compositions.extend( + _ToolComposition( + source="plugin", + plugin_index=plugin_index, + delegation=isinstance(child_harnesses, _ParentChildHarnessHost) + and child_harnesses.is_delegation_tool(raw_tool), + ) + for raw_tool in binding.static.tools + ) static_instructions.extend(contribution.instructions) static_hooks.extend(contribution.hooks) - - builtin_candidates = [create_subagent_tool(self, self.config.subagents)] - builtin = self._select_builtin_tools(builtin_candidates, self.config.builtin_tools) - configured_tools = [*static_tools, *builtin, *(tools or [])] + agent_names.extend(self._validate_binding_agent_names(binding.agent_names, agent_names)) + + direct_tools = list(tools or []) + configured_tools = [*static_tools, *direct_tools] + configured_compositions = [ + *static_compositions, + *(_ToolComposition(source="direct") for _ in direct_tools), + ] self._validate_tool_list( configured_tools, output_schema=output_schema, model_supports_approval_resume=self._model_supports_approval_resume(), - is_child_run=self._is_child_run, + is_child_harness=self._is_child_harness, ) tool_map = {tool.name: tool for tool in configured_tools} caller_hooks = list(hooks.hooks) if isinstance(hooks, HookRegistry) else list(hooks or []) strict_hooks = hooks.strict_hooks if isinstance(hooks, HookRegistry) else self.config.strict_hooks hook_registry = HookRegistry([*static_hooks, *caller_hooks], strict_hooks=strict_hooks) - self._validate_hook_registry(hook_registry, self.config.subagents) + self._validate_hook_registry(hook_registry, set(agent_names)) self.plugins = configured_plugins self._plugin_bindings = bindings self._base_tools = list(configured_tools) + self._base_compositions = list(configured_compositions) self._base_instructions = list(static_instructions) self._strict_hooks = strict_hooks self.tools = configured_tools self._tool_map = tool_map + self._tool_composition = { + tool.name: composition + for tool, composition in zip(configured_tools, configured_compositions, strict=True) + } self._plugin_instructions = list(static_instructions) self.hooks = hook_registry - self.subagent_hooks = subagent_hooks or {} + self._agent_names = set(agent_names) + self._child_harnesses = child_harnesses + if isinstance(child_harnesses, _ParentChildHarnessHost): + child_harnesses.validate_recipes(configured_tools, configured_compositions) self._plugin_stack: AsyncExitStack | None = None self._connected = False self._connect_lock = asyncio.Lock() @@ -229,7 +252,7 @@ def __init__( self._connect_waiters = 0 self.local_tracing: LocalTracing | None = None external_tracing = list(self.config.tracing if tracing is None else tracing) - if _local_tracing_enabled(self.config.local_tracing) and not _is_child_run: + if _local_tracing_enabled(self.config.local_tracing) and not _is_child_harness: self.local_tracing = create_local_tracing(self.config.local_trace_dir, project_root=self.root) self.tracing = [ TracingOptions( @@ -495,6 +518,7 @@ async def _run_streaming( harness=self, run_context=run_ctx, tool_map=dict(self._tool_map), + tool_composition=dict(self._tool_composition), run_tracer=run_tracer, tool_execution=self.config.tool_execution, ) @@ -574,7 +598,7 @@ async def _prepare_run_start( prompt=prompt, instructions=instructions, capture_messages=option.capture_messages, - top_level=not self._is_child_run, + top_level=not self._is_child_harness, ) ) return effective_prompt, instructions @@ -661,14 +685,21 @@ def add_tool(self, tool: ToolSpec) -> None: spec, output_schema=self.output_schema, model_supports_approval_resume=self._model_supports_approval_resume(), - is_child_run=self._is_child_run, + is_child_harness=self._is_child_harness, ) if spec.name in self._tool_map: raise ValueError(f"duplicate tool name: {spec.name}") + composition = _ToolComposition(source="direct") + candidate_tools = [*self.tools, spec] + candidate_compositions = [*self._tool_composition.values(), composition] + if isinstance(self._child_harnesses, _ParentChildHarnessHost): + self._child_harnesses.validate_recipes(candidate_tools, candidate_compositions) self.tools.append(spec) self._tool_map[spec.name] = spec + self._tool_composition[spec.name] = composition if not self._connected: self._base_tools.append(spec) + self._base_compositions.append(composition) self._validate_hook_filters() def tool_schemas(self) -> list[Json]: @@ -717,7 +748,7 @@ def _validate_tool_list( *, output_schema: OutputSchema | None, model_supports_approval_resume: bool, - is_child_run: bool, + is_child_harness: bool, ) -> None: """Validate a complete tool list before assigning it to a harness.""" cls._validate_unique_tools(tools) @@ -726,7 +757,7 @@ def _validate_tool_list( tool, output_schema=output_schema, model_supports_approval_resume=model_supports_approval_resume, - is_child_run=is_child_run, + is_child_harness=is_child_harness, ) @staticmethod @@ -735,19 +766,17 @@ def _validate_tool_spec( *, output_schema: OutputSchema | None, model_supports_approval_resume: bool, - is_child_run: bool, + is_child_harness: bool, ) -> None: """Validate one tool against explicit harness state.""" if not callable(spec.handler): raise TypeError(f"handler for tool {spec.name!r} is not callable") - if spec.name == "subagent" and spec.kind != "subagent": - raise ValueError("subagent is a reserved tool name") if spec.name == FINAL_RESULT_TOOL_NAME and output_schema is not None and output_schema.mode != "text": raise ValueError(f"{FINAL_RESULT_TOOL_NAME} is reserved for structured output") Harness._validate_tool_approval_policy_for( spec, model_supports_approval_resume=model_supports_approval_resume, - is_child_run=is_child_run, + is_child_harness=is_child_harness, ) @staticmethod @@ -755,24 +784,37 @@ def _validate_tool_approval_policy_for( tool: ToolSpec, *, model_supports_approval_resume: bool, - is_child_run: bool, + is_child_harness: bool, ) -> None: """Reject approval policies incompatible with explicit harness state.""" if tool.requires_approval and not model_supports_approval_resume: raise ValueError("approval-required tools require a resumable model") - if tool.requires_approval and is_child_run: - raise ValueError("approval-required tools are not supported inside subagents") + if tool.requires_approval and is_child_harness: + raise ValueError("approval-required tools are not supported inside child harnesses") def _validate_hook_filters(self) -> None: - """Validate hook filters against registered subagents.""" - self._validate_hook_registry(self.hooks, self.config.subagents) + """Validate hook filters against statically contributed agent names.""" + self._validate_hook_registry(self.hooks, self._agent_names) @staticmethod - def _validate_hook_registry(hooks: HookRegistry, subagents: list[SubAgentConfig]) -> None: - """Validate hook filters against explicit subagent configuration.""" - agent_names = {DEFAULT_SUBAGENT_NAME, *(config.name for config in subagents)} + def _validate_hook_registry(hooks: HookRegistry, agent_names: set[str]) -> None: + """Validate hook filters against statically contributed agent names.""" hooks.validate_filters(agent_names=agent_names) + @staticmethod + def _validate_binding_agent_names(names: tuple[str, ...], previous: list[str]) -> list[str]: + """Validate immutable agent names within and across plugin bindings.""" + if not isinstance(names, tuple): + raise TypeError("PluginBinding.agent_names must be a tuple") + accepted: list[str] = [] + for name in names: + if not isinstance(name, str) or not name.strip(): + raise ValueError("plugin agent name must be a non-empty string") + if name in previous or name in accepted: + raise ValueError(f"duplicate plugin agent name: {name}") + accepted.append(name) + return accepted + def _model_supports_approval_resume(self) -> bool: """Return whether this harness model can resume provider sessions.""" return hasattr(self.model, "resume_kind") and hasattr(self.model, "resume_session") @@ -813,32 +855,44 @@ async def _connect_once(self) -> None: base_hooks = list(self.hooks.hooks) try: dynamic_tools: list[ToolSpec] = [] + dynamic_compositions: list[_ToolComposition] = [] dynamic_instructions: list[str] = [] dynamic_hooks: list[Hook] = [] - for plugin, binding in zip(self.plugins, self._plugin_bindings, strict=True): + for plugin_index, (plugin, binding) in enumerate(zip(self.plugins, self._plugin_bindings, strict=True)): if binding.connect is None: continue contribution = await plugin_stack.enter_async_context(binding.connect()) normalized = self._normalize_contribution(plugin.name, contribution) dynamic_tools.extend(normalized.tools) + dynamic_compositions.extend( + _ToolComposition(source="plugin", plugin_index=plugin_index) + for _ in normalized.tools + ) dynamic_instructions.extend(normalized.instructions) dynamic_hooks.extend(normalized.hooks) candidate_tools = [*self._base_tools, *dynamic_tools] + candidate_compositions = [*self._base_compositions, *dynamic_compositions] self._validate_tool_list( candidate_tools, output_schema=self.output_schema, model_supports_approval_resume=self._model_supports_approval_resume(), - is_child_run=self._is_child_run, + is_child_harness=self._is_child_harness, ) + if isinstance(self._child_harnesses, _ParentChildHarnessHost): + self._child_harnesses.validate_recipes(candidate_tools, candidate_compositions) candidate_hooks = HookRegistry([*base_hooks, *dynamic_hooks], strict_hooks=self._strict_hooks) - self._validate_hook_registry(candidate_hooks, self.config.subagents) + self._validate_hook_registry(candidate_hooks, self._agent_names) if self._closed: raise HarnessError("harness is closed") self.tools = candidate_tools self._tool_map = {tool.name: tool for tool in candidate_tools} + self._tool_composition = { + tool.name: composition + for tool, composition in zip(candidate_tools, candidate_compositions, strict=True) + } self._plugin_instructions = [*self._base_instructions, *dynamic_instructions] self.hooks = candidate_hooks self._plugin_stack = plugin_stack @@ -850,6 +904,10 @@ async def _connect_once(self) -> None: ) self.tools = list(self._base_tools) self._tool_map = {tool.name: tool for tool in self.tools} + self._tool_composition = { + tool.name: composition + for tool, composition in zip(self.tools, self._base_compositions, strict=True) + } self._plugin_instructions = list(self._base_instructions) self.hooks = HookRegistry(base_hooks, strict_hooks=self._strict_hooks) if cleanup_error is not None: @@ -903,28 +961,3 @@ def _structured_output_request(self) -> StructuredOutputRequest | None: if self.output_schema is None: return None return self.output_schema.structured_output_request() - - @staticmethod - def _select_builtin_tools(tools: list[ToolSpec], selected_names: list[str] | None) -> list[ToolSpec]: - """Return all or the explicitly selected built-in tools.""" - by_name = {tool.name: tool for tool in tools} - if selected_names is None: - return [] - selected: list[ToolSpec] = [] - seen: set[str] = set() - for name in selected_names: - if name in seen: - raise ValueError(f"duplicate selected builtin tool: {name}") - if name not in by_name: - filesystem_names = {"read", "write", "edit", "search", "list", "glob", "jsonl_search"} - if name in filesystem_names: - raise ValueError(f"unknown builtin tool: {name}; use FilesystemPlugin(tools=[{name!r}])") - if name in {"skill_read", "skill_run"}: - raise ValueError(f"unknown builtin tool: {name}; use SkillsPlugin(tools=[{name!r}])") - if name == "parallel_llm": - raise ValueError("unknown builtin tool: parallel_llm; use ParallelLlmPlugin()") - available = ", ".join(sorted(by_name)) or "none" - raise ValueError(f"unknown builtin tool: {name}; available: {available}") - selected.append(by_name[name]) - seen.add(name) - return selected diff --git a/thinharness/hooks.py b/thinharness/hooks.py index 61c6b90..474cc39 100644 --- a/thinharness/hooks.py +++ b/thinharness/hooks.py @@ -7,13 +7,13 @@ from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING, ClassVar, Literal +from typing import TYPE_CHECKING, Any, ClassVar, Literal from .tools.base import Json, ToolEnvelope, ToolResult, ToolSpec from .types import HarnessResult, RunUsage, StopReason _CURRENT_TOOL_CALL: contextvars.ContextVar[Json | None] = contextvars.ContextVar("thinharness_current_tool_call", default=None) -_CURRENT_TOOL_RUNTIME: contextvars.ContextVar[Json | None] = contextvars.ContextVar("thinharness_current_tool_runtime", default=None) +_CURRENT_TOOL_RUNTIME: contextvars.ContextVar[dict[str, Any] | None] = contextvars.ContextVar("thinharness_current_tool_runtime", default=None) def current_tool_call_context() -> Json | None: @@ -21,7 +21,7 @@ def current_tool_call_context() -> Json | None: return _CURRENT_TOOL_CALL.get() -def current_tool_runtime_context() -> Json | None: +def current_tool_runtime_context() -> dict[str, Any] | None: """Return internal runtime context for nested framework tool handlers.""" return _CURRENT_TOOL_RUNTIME.get() diff --git a/thinharness/plugins/__init__.py b/thinharness/plugins/__init__.py index 67cad48..afe1d75 100644 --- a/thinharness/plugins/__init__.py +++ b/thinharness/plugins/__init__.py @@ -1,16 +1,26 @@ """Built-in plugin contracts and adapters.""" -from .base import Plugin, PluginBinding, PluginConnector, PluginContext, PluginContribution, ToolOrigin +from ..children import ChildHarnessHost, ChildHarnessOutcome, ChildHarnessRequest +from .base import ChildInheritablePlugin, Plugin, PluginBinding, PluginConnector, PluginContext, PluginContribution, ToolOrigin from .filesystem import FilesystemPlugin from .mcp import MCPPlugin from .parallel_llm import ParallelLlmPlugin from .skills import SkillsPlugin +from .subagents import DEFAULT_SUBAGENT_NAME, SubAgentArgs, SubAgentConfig, SubagentsPlugin __all__ = [ + "ChildHarnessHost", + "ChildHarnessOutcome", + "ChildHarnessRequest", + "ChildInheritablePlugin", + "DEFAULT_SUBAGENT_NAME", "FilesystemPlugin", "MCPPlugin", "ParallelLlmPlugin", "SkillsPlugin", + "SubAgentArgs", + "SubAgentConfig", + "SubagentsPlugin", "Plugin", "PluginBinding", "PluginConnector", diff --git a/thinharness/plugins/base.py b/thinharness/plugins/base.py index d81f2dc..dfe08e6 100644 --- a/thinharness/plugins/base.py +++ b/thinharness/plugins/base.py @@ -11,6 +11,7 @@ from ..tools.base import ToolOrigin if TYPE_CHECKING: + from ..children import ChildHarnessHost from ..hooks import Hook from ..providers import Model from ..tools.base import ToolSpec @@ -22,6 +23,7 @@ class PluginContext: root: Path model: Model + child_harnesses: ChildHarnessHost @dataclass(frozen=True) @@ -42,6 +44,7 @@ class PluginBinding: static: PluginContribution = field(default_factory=PluginContribution) connect: PluginConnector | None = None + agent_names: tuple[str, ...] = () @runtime_checkable @@ -55,7 +58,17 @@ def bind(self, context: PluginContext) -> PluginBinding: ... +@runtime_checkable +class ChildInheritablePlugin(Protocol): + """Plugin that explicitly supports rebinding against a child context.""" + + def for_child(self) -> Plugin: + """Return the configured plugin object to bind to one child.""" + ... + + __all__ = [ + "ChildInheritablePlugin", "Plugin", "PluginBinding", "PluginConnector", diff --git a/thinharness/plugins/filesystem.py b/thinharness/plugins/filesystem.py index c9fc92c..48e0e30 100644 --- a/thinharness/plugins/filesystem.py +++ b/thinharness/plugins/filesystem.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Sequence -from dataclasses import replace +from dataclasses import dataclass, replace from pathlib import Path from ..tools.base import ToolOrigin @@ -13,10 +13,62 @@ _DEFAULT_TOOLS = ("read", "write", "edit", "search", "list", "glob") -class FilesystemPlugin: +@dataclass(frozen=True) +class _FilesystemConfig: + selected: tuple[str, ...] + output_dir: str | Path | None + max_read_chars: int + max_read_bytes: int + max_tool_chars: int + max_search_line_chars: int + rg_timeout: int + search_exclude_globs: tuple[str, ...] | None + read_paths: tuple[str | Path, ...] | None + write_paths: tuple[str | Path, ...] | None + + +class _FilesystemPluginMeta(type): + """Keep the filesystem plugin name fixed on the class hierarchy.""" + + def __setattr__(cls, attribute: str, value: object) -> None: + if attribute == "name": + raise AttributeError("FilesystemPlugin.name is fixed to 'filesystem'") + super().__setattr__(attribute, value) + + def __delattr__(cls, attribute: str) -> None: + if attribute == "name": + raise AttributeError("FilesystemPlugin.name is fixed to 'filesystem'") + super().__delattr__(attribute) + + +class FilesystemPlugin(metaclass=_FilesystemPluginMeta): """Provide root-scoped filesystem tools to one harness.""" name = "filesystem" + _config: _FilesystemConfig + _frozen: bool + + def __init_subclass__(cls) -> None: + """Reject subclasses that replace the fixed plugin name.""" + super().__init_subclass__() + if "name" in cls.__dict__: + raise TypeError("FilesystemPlugin subclasses cannot override the fixed name 'filesystem'") + + def __setattr__(self, attribute: str, value: object) -> None: + """Reject configuration changes after construction.""" + if attribute == "name": + raise AttributeError("FilesystemPlugin.name is fixed to 'filesystem'") + if getattr(self, "_frozen", False): + raise AttributeError("FilesystemPlugin configuration is frozen") + object.__setattr__(self, attribute, value) + + def __delattr__(self, attribute: str) -> None: + """Reject configuration deletion after construction.""" + if attribute == "name": + raise AttributeError("FilesystemPlugin.name is fixed to 'filesystem'") + if getattr(self, "_frozen", False): + raise AttributeError("FilesystemPlugin configuration is frozen") + object.__delattr__(self, attribute) def __init__( self, @@ -37,41 +89,46 @@ def __init__( selected = tuple(_DEFAULT_TOOLS if tools is None else tools) if len(set(selected)) != len(selected): raise ValueError("FilesystemPlugin tools contains a duplicate name") - self._selected = selected - self._output_dir = output_dir - self._max_read_chars = max_read_chars - self._max_read_bytes = max_read_bytes - self._max_tool_chars = max_tool_chars - self._max_search_line_chars = max_search_line_chars - self._rg_timeout = rg_timeout - self._search_exclude_globs = list(search_exclude_globs) if search_exclude_globs is not None else None - self._read_paths = tuple(read_paths) if read_paths is not None else None - self._write_paths = tuple(write_paths) if write_paths is not None else None + object.__setattr__(self, "_config", _FilesystemConfig( + selected=selected, + output_dir=output_dir, + max_read_chars=max_read_chars, + max_read_bytes=max_read_bytes, + max_tool_chars=max_tool_chars, + max_search_line_chars=max_search_line_chars, + rg_timeout=rg_timeout, + search_exclude_globs=tuple(search_exclude_globs) if search_exclude_globs is not None else None, + read_paths=tuple(read_paths) if read_paths is not None else None, + write_paths=tuple(write_paths) if write_paths is not None else None, + )) + object.__setattr__(self, "_frozen", True) + + def for_child(self) -> FilesystemPlugin: + """Reuse the frozen constructor configuration for a child binding.""" + return self def bind(self, context: PluginContext) -> PluginBinding: """Build static tool specifications without filesystem I/O.""" + config = self._config collection = FileTools( context.root, - output_dir=self._output_dir, - max_read_chars=self._max_read_chars, - max_read_bytes=self._max_read_bytes, - max_tool_chars=self._max_tool_chars, - max_search_line_chars=self._max_search_line_chars, - rg_timeout=self._rg_timeout, - search_exclude_globs=self._search_exclude_globs, - read_paths=self._read_paths, - write_paths=self._write_paths, + output_dir=config.output_dir, + max_read_chars=config.max_read_chars, + max_read_bytes=config.max_read_bytes, + max_tool_chars=config.max_tool_chars, + max_search_line_chars=config.max_search_line_chars, + rg_timeout=config.rg_timeout, + search_exclude_globs=list(config.search_exclude_globs) if config.search_exclude_globs is not None else None, + read_paths=config.read_paths, + write_paths=config.write_paths, _root_is_resolved=True, ) by_name = {tool.name: tool for tool in collection.specs()} - unknown = [name for name in self._selected if name not in by_name] + unknown = [name for name in config.selected if name not in by_name] if unknown: available = ", ".join(by_name) raise ValueError(f"unknown FilesystemPlugin tool: {unknown[0]}; available: {available}") - specs = tuple( - replace(by_name[name], origin=ToolOrigin(plugin=self.name, source=name)) - for name in self._selected - ) + specs = tuple(replace(by_name[name], origin=ToolOrigin(plugin=self.name, source=name)) for name in config.selected) return PluginBinding(static=PluginContribution( tools=specs, instructions=(f"Workspace root: {context.root}",), diff --git a/thinharness/plugins/parallel_llm.py b/thinharness/plugins/parallel_llm.py index 6b3d632..b89658b 100644 --- a/thinharness/plugins/parallel_llm.py +++ b/thinharness/plugins/parallel_llm.py @@ -3,6 +3,8 @@ from __future__ import annotations from collections.abc import Sequence +from copy import deepcopy +from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any @@ -17,6 +19,25 @@ from ..providers import Model +@dataclass(frozen=True) +class _ParallelLlmConfig: + model: Model | str | None + description: str + instructions: str | None + read_paths: tuple[str | Path, ...] | None + write_paths: tuple[str | Path, ...] | None + max_prompts: int + api_key: str | None + base_url: str | None + request_timeout: int | None + request_retries: int | None + request_retry_backoff: float | None + temperature: float | None + max_tokens: int | None + effort: str | None + extra_body: dict[str, Any] | None + + class _ParallelLlmPluginMeta(type): """Keep the parallel LLM plugin name fixed on the class hierarchy.""" @@ -35,6 +56,8 @@ class ParallelLlmPlugin(metaclass=_ParallelLlmPluginMeta): """Expose one root-scoped text-only parallel completion tool.""" name = "parallel_llm" + _config: _ParallelLlmConfig + _frozen: bool def __init_subclass__(cls) -> None: """Reject subclasses that replace the fixed plugin name.""" @@ -43,10 +66,28 @@ def __init_subclass__(cls) -> None: raise TypeError("ParallelLlmPlugin subclasses cannot override the fixed name 'parallel_llm'") def __setattr__(self, attribute: str, value: object) -> None: - """Reject instance changes to the fixed plugin name.""" + """Reject configuration changes after construction.""" if attribute == "name": raise AttributeError("ParallelLlmPlugin.name is fixed to 'parallel_llm'") - super().__setattr__(attribute, value) + if getattr(self, "_frozen", False): + raise AttributeError("ParallelLlmPlugin configuration is frozen") + object.__setattr__(self, attribute, value) + + def __delattr__(self, attribute: str) -> None: + """Reject configuration deletion after construction.""" + if attribute == "name": + raise AttributeError("ParallelLlmPlugin.name is fixed to 'parallel_llm'") + if getattr(self, "_frozen", False): + raise AttributeError("ParallelLlmPlugin configuration is frozen") + object.__delattr__(self, attribute) + + def __getattr__(self, attribute: str) -> Any: + """Expose immutable values or copies from the constructor snapshot.""" + config = object.__getattribute__(self, "_config") + if not hasattr(config, attribute): + raise AttributeError(attribute) + value = getattr(config, attribute) + return deepcopy(value) if attribute == "extra_body" else value def __init__( self, @@ -84,26 +125,33 @@ def __init__( supplied = next((name for name, value in provider_options.items() if value is not None), None) if supplied is not None: raise ValueError(f"{supplied} is valid only when ParallelLlmPlugin model is a string") - - self.model = model - self.description = description - self.instructions = instructions - self.read_paths = tuple(read_paths) if read_paths is not None else None - self.write_paths = tuple(write_paths) if write_paths is not None else None - self.max_prompts = max_prompts - self.api_key = api_key - self.base_url = base_url - self.request_timeout = request_timeout - self.request_retries = request_retries - self.request_retry_backoff = request_retry_backoff - self.temperature = temperature - self.max_tokens = max_tokens - self.effort = effort - self.extra_body = dict(extra_body) if extra_body is not None else None + object.__setattr__(self, "_config", _ParallelLlmConfig( + model=model, + description=description, + instructions=instructions, + read_paths=tuple(read_paths) if read_paths is not None else None, + write_paths=tuple(write_paths) if write_paths is not None else None, + max_prompts=max_prompts, + api_key=api_key, + base_url=base_url, + request_timeout=request_timeout, + request_retries=request_retries, + request_retry_backoff=request_retry_backoff, + temperature=temperature, + max_tokens=max_tokens, + effort=effort, + extra_body=deepcopy(extra_body) if extra_body is not None else None, + )) + object.__setattr__(self, "_frozen", True) + + def for_child(self) -> ParallelLlmPlugin: + """Reuse the frozen settings and resolve a borrowed model at child bind time.""" + return self def bind(self, context: PluginContext) -> PluginBinding: """Build the static tool with the canonical root and resolved model.""" - model = context.model if self.model is None else self.model + config = self._config + model = context.model if config.model is None else config.model provider_options: dict[str, Any] = {} for name in ( "api_key", @@ -116,17 +164,17 @@ def bind(self, context: PluginContext) -> PluginBinding: "effort", "extra_body", ): - value = getattr(self, name) + value = getattr(config, name) if value is not None: - provider_options[name] = value + provider_options[name] = deepcopy(value) tool = ParallelLlmTool( model=model, root=context.root, - description=self.description, - instructions=self.instructions, - read_paths=list(self.read_paths) if self.read_paths is not None else None, - write_paths=list(self.write_paths) if self.write_paths is not None else None, - max_prompts=self.max_prompts, + description=config.description, + instructions=config.instructions, + read_paths=list(config.read_paths) if config.read_paths is not None else None, + write_paths=list(config.write_paths) if config.write_paths is not None else None, + max_prompts=config.max_prompts, _root_is_resolved=True, **provider_options, ) diff --git a/thinharness/plugins/skills.py b/thinharness/plugins/skills.py index 4c773bb..83b5b5e 100644 --- a/thinharness/plugins/skills.py +++ b/thinharness/plugins/skills.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import Sequence +from dataclasses import dataclass from pathlib import Path from typing import Literal @@ -13,6 +14,13 @@ _VALID_TOOLS = ("skill_read", "skill_run") +@dataclass(frozen=True) +class _SkillsConfig: + tools: tuple[SkillToolName, ...] + registry: SkillRegistry + contribution: PluginContribution + + class _SkillsPluginMeta(type): """Keep the skills plugin name fixed on the class hierarchy.""" @@ -31,6 +39,8 @@ class SkillsPlugin(metaclass=_SkillsPluginMeta): """Expose one constructor-time skill catalog through selected tools.""" name = "skills" + _config: _SkillsConfig + _frozen: bool def __init_subclass__(cls) -> None: """Reject subclasses that replace the fixed plugin name.""" @@ -39,10 +49,20 @@ def __init_subclass__(cls) -> None: raise TypeError("SkillsPlugin subclasses cannot override the fixed name 'skills'") def __setattr__(self, attribute: str, value: object) -> None: - """Reject instance changes to the fixed plugin name.""" + """Reject configuration changes after construction.""" if attribute == "name": raise AttributeError("SkillsPlugin.name is fixed to 'skills'") - super().__setattr__(attribute, value) + if getattr(self, "_frozen", False): + raise AttributeError("SkillsPlugin configuration is frozen") + object.__setattr__(self, attribute, value) + + def __delattr__(self, attribute: str) -> None: + """Reject configuration deletion after construction.""" + if attribute == "name": + raise AttributeError("SkillsPlugin.name is fixed to 'skills'") + if getattr(self, "_frozen", False): + raise AttributeError("SkillsPlugin configuration is frozen") + object.__delattr__(self, attribute) def __init__( self, @@ -71,21 +91,42 @@ def __init__( available = ", ".join(_VALID_TOOLS) raise ValueError(f"unknown SkillsPlugin tool: {unknown}; available: {available}") - self.tools = selected_tools - self.registry = SkillRegistry(directories, selected_skills=selected_skills) - by_name = {spec.name: spec for spec in self.registry.specs()} + registry = SkillRegistry( + directories, + selected_skills=tuple(selected_skills) if selected_skills is not None else None, + ) + by_name = {spec.name: spec for spec in registry.specs()} specs = tuple(by_name[name] for name in selected_tools if name in by_name) instructions: tuple[str, ...] = () if specs: - summary = self.registry.prompt_summary(include_read_hint="skill_read" in selected_tools) + summary = registry.prompt_summary(include_read_hint="skill_read" in selected_tools) if summary: instructions = (summary,) - self._contribution = PluginContribution(tools=specs, instructions=instructions) + object.__setattr__(self, "_config", _SkillsConfig( + tools=selected_tools, + registry=registry, + contribution=PluginContribution(tools=specs, instructions=instructions), + )) + object.__setattr__(self, "_frozen", True) + + @property + def tools(self) -> tuple[SkillToolName, ...]: + """Return the frozen selected tool names.""" + return self._config.tools + + @property + def registry(self) -> SkillRegistry: + """Return the shared constructor-time registry.""" + return self._config.registry + + def for_child(self) -> SkillsPlugin: + """Reuse the shared frozen registry and catalog for a child binding.""" + return self def bind(self, context: PluginContext) -> PluginBinding: """Return the constructor-time contribution without I/O.""" del context - return PluginBinding(static=self._contribution) + return PluginBinding(static=self._config.contribution) __all__ = ["SkillsPlugin"] diff --git a/thinharness/plugins/subagents.py b/thinharness/plugins/subagents.py new file mode 100644 index 0000000..2fe76b5 --- /dev/null +++ b/thinharness/plugins/subagents.py @@ -0,0 +1,340 @@ +"""Explicit subagent delegation plugin and child configuration.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import replace +from typing import Any, Final, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from ..children import ChildHarnessOutcome, ChildHarnessRequest +from ..defaults import DEFAULT_SYSTEM_PROMPT +from ..hooks import AGENT_EVENTS, Hook, HookRegistry +from ..tools.base import ToolResult, ToolSpec +from .base import Plugin, PluginBinding, PluginContext, PluginContribution + +DEFAULT_SUBAGENT_NAME: Final[str] = "default" +_REMOVED_CONFIG_FIELDS = ( + "inherit_parent_tools", + "inherit_mcp_servers", + "mcp_servers", + "builtin_tools", +) + + +class SubAgentConfig(BaseModel): + """Configuration for one named delegated child harness.""" + + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid", frozen=True) + + name: str = Field(min_length=1, pattern=r"^[A-Za-z0-9_.-]+$") + description: str = Field(min_length=1) + system_prompt: str = DEFAULT_SYSTEM_PROMPT + inherit_parent: bool = False + plugins: tuple[Any, ...] = () + tools: tuple[ToolSpec, ...] = () + hooks: Any = None + model: str | None = None + max_model_requests: int | None = None + max_tool_calls: int | None = None + output_type: Any | None = None + output_mode: Literal["auto", "native", "tool", "prompted"] = "auto" + output_retries: int = Field(default=1, ge=0) + tool_retries: int = Field(default=1, ge=0) + + @model_validator(mode="before") + @classmethod + def reject_removed_fields(cls, data: object) -> object: + """Fail loudly when callers pass fields removed from this configuration.""" + if isinstance(data, dict): + for field_name in _REMOVED_CONFIG_FIELDS: + if field_name in data: + raise ValueError( + f"SubAgentConfig.{field_name} has been removed; use inherit_parent, plugins, or tools" + ) + if "background" in data: + raise ValueError("SubAgentConfig.background has been removed") + for field_name in ("plugins", "tools"): + if isinstance(data.get(field_name), (set, frozenset)): + raise TypeError(f"SubAgentConfig {field_name} must be an ordered sequence, not a set") + return data + + @model_validator(mode="after") + def validate_child(self) -> SubAgentConfig: + """Validate display, plugin, tool, and child-hook policy.""" + if self.name == DEFAULT_SUBAGENT_NAME: + raise ValueError(f"{DEFAULT_SUBAGENT_NAME!r} is reserved for the framework default subagent") + if not self.description.strip() or "\n" in self.description or "\r" in self.description: + raise ValueError("subagent description must be a non-empty single line") + _validate_plugins(self.plugins) + if any(isinstance(plugin, SubagentsPlugin) for plugin in self.plugins): + raise ValueError("SubagentsPlugin cannot be configured inside a child harness") + if any(tool.requires_approval for tool in self.tools): + raise ValueError("approval-required tools are not supported inside child harnesses") + return self + + +class SubAgentArgs(BaseModel): + """Arguments for subagent delegation.""" + + model_config = ConfigDict(extra="forbid") + + task: str + agent: str | None = Field( + default=None, + min_length=1, + description="Optional subagent name; omit to use the framework default subagent.", + ) + + +class _SubagentsPluginMeta(type): + """Keep the subagents plugin name fixed on the class hierarchy.""" + + def __setattr__(cls, attribute: str, value: object) -> None: + if attribute == "name": + raise AttributeError("SubagentsPlugin.name is fixed to 'subagents'") + super().__setattr__(attribute, value) + + def __delattr__(cls, attribute: str) -> None: + if attribute == "name": + raise AttributeError("SubagentsPlugin.name is fixed to 'subagents'") + super().__delattr__(attribute) + + +class SubagentsPlugin(metaclass=_SubagentsPluginMeta): + """Contribute one delegation tool backed by isolated child harnesses.""" + + name = "subagents" + _agents: tuple[SubAgentConfig, ...] + _default_hooks: tuple[Hook, ...] | HookRegistry | None + _agent_hooks: tuple[tuple[Hook, ...] | HookRegistry | None, ...] + _frozen: bool + + def __init_subclass__(cls) -> None: + """Reject subclasses that replace the fixed plugin name.""" + super().__init_subclass__() + if "name" in cls.__dict__: + raise TypeError("SubagentsPlugin subclasses cannot override the fixed name 'subagents'") + + def __setattr__(self, attribute: str, value: object) -> None: + """Reject configuration changes after construction.""" + if attribute == "name": + raise AttributeError("SubagentsPlugin.name is fixed to 'subagents'") + if getattr(self, "_frozen", False): + raise AttributeError("SubagentsPlugin configuration is frozen") + object.__setattr__(self, attribute, value) + + def __delattr__(self, attribute: str) -> None: + """Reject configuration deletion after construction.""" + if attribute == "name": + raise AttributeError("SubagentsPlugin.name is fixed to 'subagents'") + if getattr(self, "_frozen", False): + raise AttributeError("SubagentsPlugin configuration is frozen") + object.__delattr__(self, attribute) + + def __init__( + self, + *, + agents: Sequence[SubAgentConfig] = (), + default_hooks: Sequence[Hook] | HookRegistry | None = None, + ) -> None: + if isinstance(agents, (set, frozenset)): + raise TypeError("SubagentsPlugin agents must be an ordered sequence, not a set") + configured = tuple(agents) + if any(not isinstance(agent, SubAgentConfig) for agent in configured): + raise TypeError("SubagentsPlugin agents must contain SubAgentConfig values") + names = [agent.name for agent in configured] + duplicate = next((name for index, name in enumerate(names) if name in names[:index]), None) + if duplicate is not None: + raise ValueError(f"duplicate subagent name: {duplicate}") + normalized_default_hooks = _normalize_hooks(default_hooks, label="SubagentsPlugin.default_hooks") + normalized_agent_hooks = tuple( + _normalize_hooks(agent.hooks, label=f"SubAgentConfig({agent.name!r}).hooks") + for agent in configured + ) + object.__setattr__(self, "_agents", configured) + object.__setattr__(self, "_default_hooks", normalized_default_hooks) + object.__setattr__(self, "_agent_hooks", normalized_agent_hooks) + object.__setattr__(self, "_frozen", True) + + @property + def agents(self) -> tuple[SubAgentConfig, ...]: + """Return the ordered frozen named-child catalog.""" + return self._agents + + @property + def default_hooks(self) -> tuple[Hook, ...] | HookRegistry | None: + """Return a copy of default-child hook configuration.""" + hooks = self._default_hooks + if isinstance(hooks, HookRegistry): + return HookRegistry(list(hooks.hooks), strict_hooks=hooks.strict_hooks) + return hooks + + def bind(self, context: PluginContext) -> PluginBinding: + """Bind one static delegation tool without creating child resources.""" + host = context.child_harnesses + agents = self._agents + recipes = ( + self._default_recipe(), + *(self._recipe(config, hooks) for config, hooks in zip(agents, self._agent_hooks, strict=True)), + ) + recipes_by_name = { + agent.name: recipe + for agent, recipe in zip(agents, recipes[1:], strict=True) + } + available_names = tuple(sorted(recipes_by_name)) + + async def handler(args: SubAgentArgs) -> ToolResult: + """Run one selected child and shape its model-visible result.""" + if args.agent is None: + recipe = recipes[0] + else: + recipe = recipes_by_name.get(args.agent) + if recipe is None: + return ToolResult( + False, + f"unknown subagent: {args.agent}", + { + "agent": args.agent, + "available": list(available_names), + "error_type": "UnknownSubAgent", + }, + ) + outcome = await host.run(replace(recipe, task=args.task)) + return _tool_result(recipe, outcome) + + tool = ToolSpec( + "subagent", + _tool_description(agents), + SubAgentArgs, + handler, + ) + registered = host.register_delegation_tool(tool, recipes) + return PluginBinding( + static=PluginContribution(tools=(registered,)), + agent_names=(DEFAULT_SUBAGENT_NAME, *(agent.name for agent in self._agents)), + ) + + def _default_recipe(self) -> ChildHarnessRequest: + """Return the fixed parent-derived unnamed-child recipe.""" + return ChildHarnessRequest( + agent_name=DEFAULT_SUBAGENT_NAME, + agent_description="Framework default subagent", + trace_agent_name=f"subagent.{DEFAULT_SUBAGENT_NAME}", + task="", + inherited=True, + tool_mode="inherited", + system_prompt=DEFAULT_SYSTEM_PROMPT, + hooks=self._default_hooks, + tool_retries=None, + ) + + @staticmethod + def _recipe( + config: SubAgentConfig, + hooks: tuple[Hook, ...] | HookRegistry | None, + ) -> ChildHarnessRequest: + """Translate one plugin-owned named configuration into a host request.""" + explicit = bool(config.plugins or config.tools) + tool_mode: Literal["inherited", "inherited+explicit", "explicit"] + if config.inherit_parent: + tool_mode = "inherited+explicit" if explicit else "inherited" + else: + tool_mode = "explicit" + return ChildHarnessRequest( + agent_name=config.name, + agent_description=config.description, + trace_agent_name=f"subagent.{config.name}", + task="", + inherited=config.inherit_parent, + tool_mode=tool_mode, + system_prompt=config.system_prompt, + model=config.model, + plugins=tuple(config.plugins), + tools=tuple(config.tools), + hooks=hooks, + max_model_requests=config.max_model_requests, + max_tool_calls=config.max_tool_calls, + output_type=config.output_type, + output_mode=config.output_mode, + output_retries=config.output_retries, + tool_retries=config.tool_retries, + ) + + +def _tool_result(recipe: ChildHarnessRequest, outcome: ChildHarnessOutcome) -> ToolResult: + """Shape a child-host outcome as the delegation tool contract.""" + metadata = { + "agent": recipe.agent_name, + "inherited": recipe.inherited, + "tool_mode": recipe.tool_mode, + "tools": list(outcome.tools), + } + if outcome.error_type is not None: + metadata["error_type"] = outcome.error_type + return ToolResult(False, outcome.error_message or outcome.error_type, metadata) + assert outcome.result is not None + metadata.update({ + "model_requests": outcome.result.usage.model_requests, + "structured_output": outcome.structured_output, + }) + return ToolResult(True, outcome.content, metadata) + + +def _validate_plugins(plugins: Sequence[object]) -> None: + """Reject invalid plugin values through the structural public contract.""" + for plugin in plugins: + if not isinstance(plugin, Plugin): + raise TypeError("SubAgentConfig plugins must contain Plugin values") + + +def _normalize_hooks( + hooks: Sequence[Hook] | HookRegistry | None, + *, + label: str, +) -> tuple[Hook, ...] | HookRegistry | None: + """Copy and validate hooks that will run inside a non-delegating child.""" + if hooks is None: + return None + if isinstance(hooks, HookRegistry): + normalized: tuple[Hook, ...] | HookRegistry = HookRegistry( + list(hooks.hooks), + strict_hooks=hooks.strict_hooks, + ) + values = normalized.hooks + else: + if isinstance(hooks, (set, frozenset)): + raise TypeError(f"{label} must be an ordered sequence, not a set") + values = list(hooks) + if any(not isinstance(hook, Hook) for hook in values): + raise TypeError(f"{label} must contain Hook values") + normalized = tuple(values) + for hook in values: + if hook.event in AGENT_EVENTS: + raise ValueError(f"{label} cannot contain subagent lifecycle hooks") + if hook.agents is not None: + raise ValueError(f"{label} cannot use Hook.agents") + return normalized + + +def _tool_description(agents: Sequence[SubAgentConfig]) -> str: + """Render the model-facing delegation tool description.""" + lines = [ + "Delegate one self-contained task to a sub-helper. Each subagent runs in isolated context.", + "", + ] + if agents: + lines.append("Available agents:") + lines.extend(f"- {agent.name}: {agent.description}" for agent in agents) + lines.append("") + lines.append("Omit `agent` to use the framework default subagent.") + return "\n".join(lines) + + +__all__ = [ + "DEFAULT_SUBAGENT_NAME", + "SubAgentArgs", + "SubAgentConfig", + "SubagentsPlugin", +] diff --git a/thinharness/providers.py b/thinharness/providers.py index fc5dfe0..3121799 100644 --- a/thinharness/providers.py +++ b/thinharness/providers.py @@ -977,8 +977,10 @@ async def _complete( # on the last cacheable block, so the growing prefix is reused. "cache_control": {"type": "ephemeral"}, } - if metadata: - payload["metadata"] = metadata + # Anthropic accepts only user_id in request metadata. + anthropic_user_id = metadata.get("user_id") if metadata is not None else None + if isinstance(anthropic_user_id, str): + payload["metadata"] = {"user_id": anthropic_user_id} if self.model.settings.temperature is not None: payload["temperature"] = self.model.settings.temperature if self.model.settings.effort is not None: diff --git a/thinharness/runtime.py b/thinharness/runtime.py index cb2a2f9..4954642 100644 --- a/thinharness/runtime.py +++ b/thinharness/runtime.py @@ -401,7 +401,7 @@ def pause_for_approval( result=self.result, output_schema=self.harness.output_schema, capture_messages=option.capture_messages, - top_level=not self.harness._is_child_run, + top_level=not self.harness._is_child_harness, ) ) self.fire_run_end_once() @@ -438,7 +438,7 @@ def finalize( result=self.result, output_schema=self.harness.output_schema, capture_messages=option.capture_messages, - top_level=not self.harness._is_child_run, + top_level=not self.harness._is_child_harness, ) ) self.attach_resume_state(active_session, require_dump_state=require_dump_state) diff --git a/thinharness/subagents.py b/thinharness/subagents.py deleted file mode 100644 index 1ef4169..0000000 --- a/thinharness/subagents.py +++ /dev/null @@ -1,415 +0,0 @@ -"""Subagent configuration and delegation tool support.""" - -from __future__ import annotations - -from types import TracebackType -from typing import TYPE_CHECKING, Any, Final, Literal - -from pydantic import BaseModel, ConfigDict, Field, model_validator - -from .defaults import DEFAULT_SYSTEM_PROMPT -from .events import RunCompletedEvent, current_stream_emitter -from .hooks import AfterSubagentRunContext, BeforeSubagentRunContext, HookRegistry, current_tool_call_context, current_tool_runtime_context -from .plugins.base import Plugin -from .plugins.mcp import MCPPlugin -from .plugins.skills import SkillsPlugin -from .providers import infer_model, same_provider_model_ref -from .tools.base import Json, ToolResult, ToolSpec -from .tools.mcp import MCPServer -from .tracing import TracingOptions - -if TYPE_CHECKING: - from .core import Harness - - -DEFAULT_SUBAGENT_NAME: Final[str] = "default" - - -class SubAgentConfig(BaseModel): - """Configuration for one delegated child harness.""" - - model_config = ConfigDict(arbitrary_types_allowed=True) - - name: str = Field(min_length=1, pattern=r"^[A-Za-z0-9_.-]+$") - description: str = Field(min_length=1) - system_prompt: str = DEFAULT_SYSTEM_PROMPT - inherit_parent_tools: bool = False - inherit_mcp_servers: bool = False - plugins: list[Plugin] = Field(default_factory=list) - tools: list[ToolSpec] = Field(default_factory=list) - mcp_servers: list[MCPServer] = Field(default_factory=list) - model: str | None = None - max_model_requests: int | None = None - max_tool_calls: int | None = None - output_type: Any | None = None - output_mode: Literal["auto", "native", "tool", "prompted"] = "auto" - output_retries: int = Field(default=1, ge=0) - tool_retries: int = Field(default=1, ge=0) - - @model_validator(mode="before") - @classmethod - def reject_removed_fields(cls, data: object) -> object: - """Fail loudly when callers pass fields removed from the public API.""" - if isinstance(data, dict): - if "background" in data: - raise ValueError("SubAgentConfig.background has been removed") - if "builtin_tools" in data: - raise ValueError("SubAgentConfig.builtin_tools has been removed; use plugins or tools") - return data - - @model_validator(mode="after") - def validate_subagent(self) -> SubAgentConfig: - """Validate subagent tool policy and display fields.""" - if self.name == DEFAULT_SUBAGENT_NAME: - raise ValueError(f"{DEFAULT_SUBAGENT_NAME!r} is reserved for the framework default subagent") - if not self.description.strip() or "\n" in self.description or "\r" in self.description: - raise ValueError("subagent description must be a non-empty single line") - exposes_subagent = any(_tool_name(tool).lower() == "subagent" for tool in self.tools) - if exposes_subagent: - raise ValueError("subagent cannot be exposed inside a child subagent") - if any(tool.requires_approval for tool in self.tools): - raise ValueError("approval-required tools are not supported inside subagents") - has_explicit_mcp_plugin = any(isinstance(plugin, MCPPlugin) for plugin in self.plugins) - if has_explicit_mcp_plugin and (self.mcp_servers or self.inherit_mcp_servers): - raise ValueError("an explicit MCPPlugin cannot be combined with mcp_servers or inherit_mcp_servers=True") - if self.inherit_parent_tools and (self.plugins or self.tools): - raise ValueError("inherit_parent_tools cannot be combined with plugins or tools") - if not (self.inherit_parent_tools or self.plugins or self.tools or self.inherit_mcp_servers or self.mcp_servers): - raise ValueError("named subagents must define plugins, tools, inherit_parent_tools=True, inherit_mcp_servers=True, or mcp_servers") - return self - - -class SubAgentArgs(BaseModel): - """Arguments for subagent delegation.""" - - model_config = ConfigDict(extra="forbid") - - task: str - agent: str | None = Field(default=None, min_length=1, description="Optional subagent name; omit to use the framework default subagent.") - - -def create_subagent_tool(parent: Harness, configs: list[SubAgentConfig]) -> ToolSpec: - """Create the parent-facing subagent delegation tool.""" - - async def handler(args: SubAgentArgs) -> ToolResult: - """Run the selected subagent.""" - return await run_subagent_tool(parent, configs, args) - - return ToolSpec( - "subagent", - _subagent_tool_description(configs), - SubAgentArgs, - handler, - kind="subagent", - ) - - -async def run_subagent_tool(parent: Harness, configs: list[SubAgentConfig], args: SubAgentArgs) -> ToolResult: - """Run a child harness and return its final text as a tool result.""" - config = _select_config(configs, args.agent) - if args.agent and config is None: - available = sorted(cfg.name for cfg in configs) - return ToolResult( - False, - f"unknown subagent: {args.agent}", - {"agent": args.agent, "available": available, "error_type": "UnknownSubAgent"}, - ) - agent_name = config.name if config is not None else DEFAULT_SUBAGENT_NAME - inherited = config is None or config.inherit_parent_tools - tool_mode = "inherited" if inherited else "explicit" - effective_tools: list[str] = [] - parent_call_id = _parent_call_id() - before = BeforeSubagentRunContext( - harness=parent, - metadata=_parent_run_metadata(), - agent=agent_name, - task=args.task, - inherited=inherited, - tool_mode=tool_mode, - parent_harness=parent, - parent_call_id=parent_call_id, - ) - parent.hooks.fire(before) - if before.cancelled: - reason = before.cancel_reason or "unspecified" - return ToolResult( - False, - f"Subagent execution blocked by hook: {reason}", - { - "agent": agent_name, - "inherited": inherited, - "tool_mode": tool_mode, - "tools": effective_tools, - "error_type": "SubAgentCancelled", - }, - ) - try: - child = build_child_harness(parent, config) - await child.connect() - effective_tools = [tool.name for tool in child.tools] - result = None - run_error: BaseException | None = None - run_traceback: TracebackType | None = None - try: - emitter = current_stream_emitter() - if emitter is not None and emitter.ctx.options.include_subagents: - child_stream = child.stream( - args.task, - metadata=_child_metadata(), - stream_options=emitter.ctx.options, - _parent_run_id=emitter.ctx.run_id, - _parent_tool_call_id=parent_call_id, - _agent_name=agent_name, - ) - try: - async for event in child_stream: - emitter.emit_forwarded(event) - if isinstance(event, RunCompletedEvent) and event.run_id == child_stream.run_id: - result = event.result - finally: - await child_stream.aclose() - else: - result = await child.run(args.task, metadata=_child_metadata()) - except BaseException as exc: - # Preserve cancellation and other BaseException exits while still closing the child harness below. - run_error = exc - run_traceback = exc.__traceback__ - try: - await child.aclose() - except Exception: - if run_error is None: - raise - if run_error is not None: - raise run_error.with_traceback(run_traceback) - except Exception as exc: - parent.hooks.fire( - AfterSubagentRunContext( - harness=parent, - metadata=_parent_run_metadata(), - agent=agent_name, - task=args.task, - error=exc, - tools=effective_tools, - parent_call_id=parent_call_id, - ) - ) - return ToolResult( - False, - str(exc), - { - "agent": agent_name, - "inherited": inherited, - "tool_mode": tool_mode, - "tools": effective_tools, - "error_type": type(exc).__name__, - }, - ) - assert result is not None - parent.hooks.fire( - AfterSubagentRunContext( - harness=parent, - metadata=_parent_run_metadata(), - agent=agent_name, - task=args.task, - result=result, - tools=effective_tools, - usage=result.usage, - parent_call_id=parent_call_id, - ) - ) - structured_output = result.output is not None - content = child.output_schema.dump(result.output) if structured_output and child.output_schema is not None else result.text - return ToolResult( - True, - content, - { - "agent": agent_name, - "inherited": inherited, - "tool_mode": tool_mode, - "tools": effective_tools, - "model_requests": result.usage.model_requests, - "structured_output": structured_output, - }, - ) - - -def build_child_harness(parent: Harness, config: SubAgentConfig | None) -> Harness: - """Create an isolated child harness for one subagent invocation.""" - from .core import Harness - - parent_config = parent.config - inherit_tools = config is None or config.inherit_parent_tools - # Remove this MCP-specific bridge when subagents migrate to plugin composition. - child_mcp_servers: list[MCPServer] = [] - if config is not None and config.inherit_mcp_servers: - parent_mcp = next((plugin for plugin in parent.plugins if isinstance(plugin, MCPPlugin)), None) - if parent_mcp is not None: - child_mcp_servers.extend(parent_mcp.servers) - if config is not None: - for server in config.mcp_servers: - if not any(server is existing for existing in child_mcp_servers): - child_mcp_servers.append(server) - child_plugins = list(_inherited_bridge_plugins(parent) if inherit_tools else (config.plugins if config is not None else [])) - if child_mcp_servers: - child_plugins.append(MCPPlugin(servers=child_mcp_servers)) - child_config = parent_config.model_copy( - update={ - "model": config.model if config is not None and config.model is not None else parent_config.model, - "root": parent.root, - "system_prompt": DEFAULT_SYSTEM_PROMPT if config is None else config.system_prompt, - "builtin_tools": [], - "max_model_requests": ( - config.max_model_requests if config is not None and config.max_model_requests is not None else parent_config.max_model_requests - ), - "max_tool_calls": (config.max_tool_calls if config is not None and config.max_tool_calls is not None else parent_config.max_tool_calls), - "output_type": config.output_type if config is not None else None, - "output_mode": config.output_mode if config is not None else "auto", - "output_retries": config.output_retries if config is not None else 1, - "tool_retries": config.tool_retries if config is not None else parent_config.tool_retries, - "subagents": [], - } - ) - child_model = parent.model - if config is not None and config.model is not None: - same_provider = _same_provider(parent, config.model) - child_model = infer_model( - config.model, - api_key=parent_config.api_key if same_provider else None, - base_url=parent_config.base_url if same_provider else None, - timeout=parent_config.request_timeout, - request_retries=parent_config.request_retries, - request_retry_backoff=parent_config.request_retry_backoff, - temperature=parent_config.temperature, - max_tokens=parent_config.max_tokens, - effort=parent_config.effort, - extra_body=parent_config.extra_body, - ) - return Harness( - child_config, - model=child_model, - plugins=child_plugins, - tools=_effective_custom_tools(parent, config), - tracing=_child_tracing(parent, config), - hooks=_child_hooks(parent, config), - subagent_hooks={}, - _owns_model=config is not None and config.model is not None, - _is_child_run=True, - ) - - -def _select_config(configs: list[SubAgentConfig], agent: str | None) -> SubAgentConfig | None: - """Return the named subagent config, or None for the default route.""" - if agent is None: - return None - for config in configs: - if config.name == agent: - return config - return None - - -def _effective_custom_tools(parent: Harness, config: SubAgentConfig | None) -> list[ToolSpec]: - """Return custom tools to register on the child harness.""" - if config is None or config.inherit_parent_tools: - skills_plugin = _parent_skills_plugin(parent) - selected_skill_tools = set(skills_plugin.tools) if skills_plugin is not None else set() - return [ - tool - for tool in parent.tools - if tool.name != "subagent" - and not (tool.origin is not None and tool.origin.plugin == "mcp") - and not ( - skills_plugin is not None - and tool.origin is not None - and tool.origin.plugin == "skills" - and tool.name in selected_skill_tools - ) - and not tool.requires_approval - ] - return list(config.tools) - - -def _inherited_bridge_plugins(parent: Harness) -> list[Plugin]: - """Return temporary plugins needed to preserve inherited child behavior.""" - plugins: list[Plugin] = [] - has_filesystem_tools = any(tool.origin is not None and tool.origin.plugin == "filesystem" for tool in parent.tools) - if has_filesystem_tools: - from .plugins.filesystem import FilesystemPlugin - - # Remove this filesystem instruction bridge when subagents migrate to plugin composition. - plugins.append(FilesystemPlugin(tools=[])) - # Remove this skills bridge when subagents migrate to plugin composition. - if skills_plugin := _parent_skills_plugin(parent): - plugins.append(skills_plugin) - return plugins - - -def _parent_skills_plugin(parent: Harness) -> SkillsPlugin | None: - """Return the exact parent skills plugin used by the temporary child bridge.""" - return next((plugin for plugin in parent.plugins if isinstance(plugin, SkillsPlugin)), None) - - -def _child_tracing(parent: Harness, config: SubAgentConfig | None) -> list[TracingOptions]: - """Return child tracing options that share the parent's tracer.""" - name = config.name if config is not None else DEFAULT_SUBAGENT_NAME - return [ - option.model_copy( - update={ - "agent_name": f"subagent.{name}", - "agent_description": config.description if config is not None else "Framework default subagent", - } - ) - for option in parent.tracing - ] - - -def _child_metadata() -> Json: - """Build minimal metadata for a child run.""" - metadata: Json = {} - parent_metadata = _parent_run_metadata() - if conversation_id := parent_metadata.get("conversation_id"): - metadata["conversation_id"] = conversation_id - if tool_call := current_tool_call_context(): - metadata["parent_call_id"] = tool_call["call_id"] - return metadata - - -def _parent_run_metadata() -> Json: - """Return copied parent run metadata from the active tool runtime context.""" - runtime = current_tool_runtime_context() or {} - parent_metadata = runtime.get("run_metadata") - return dict(parent_metadata) if isinstance(parent_metadata, dict) else {} - - -def _parent_call_id() -> str | None: - """Return the current parent tool call id when running as a tool.""" - tool_call = current_tool_call_context() - return str(tool_call["call_id"]) if tool_call else None - - -def _child_hooks(parent: Harness, config: SubAgentConfig | None) -> HookRegistry | list | None: - """Return the explicitly configured child hook registry.""" - return parent.subagent_hooks.get(config.name if config is not None else DEFAULT_SUBAGENT_NAME) - - -def _same_provider(parent: Harness, child_model_ref: str) -> bool: - """Return whether a child model ref uses the same provider as the parent model.""" - return same_provider_model_ref(parent.model, child_model_ref) - - -def _subagent_tool_description(configs: list[SubAgentConfig]) -> str: - """Render the parent-facing subagent tool description.""" - lines = [ - "Delegate one self-contained task to a sub-helper. Each subagent runs in isolated context.", - "", - ] - if configs: - lines.append("Available agents:") - lines.extend(f"- {config.name}: {config.description}" for config in configs) - lines.append("") - lines.append("Omit `agent` to use the framework default subagent.") - return "\n".join(lines) - - -def _tool_name(tool: ToolSpec) -> str: - """Return a tool name from a ToolSpec.""" - return tool.name diff --git a/thinharness/tool_execution.py b/thinharness/tool_execution.py index 1eb10ed..4951dda 100644 --- a/thinharness/tool_execution.py +++ b/thinharness/tool_execution.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING +from .children import _ToolComposition from .events import ( _CURRENT_STREAM_EMITTER, ToolCallCompletedEvent, @@ -45,18 +46,21 @@ def __init__( harness: Harness, run_context: RunContext, tool_map: dict[str, ToolSpec], + tool_composition: dict[str, _ToolComposition], run_tracer: RunTracer, tool_execution: str, ) -> None: self.harness = harness self.run_context = run_context self.tool_map = tool_map + self.tool_composition = tool_composition self.run_tracer = run_tracer self.tool_execution = tool_execution self.call_executor = ToolCallExecutor( harness=harness, run_context=run_context, tool_map=tool_map, + tool_composition=tool_composition, run_tracer=run_tracer, ) @@ -133,18 +137,27 @@ def __init__( harness: Harness, run_context: RunContext, tool_map: dict[str, ToolSpec], + tool_composition: dict[str, _ToolComposition], run_tracer: RunTracer, ) -> None: self.harness = harness self.run_context = run_context self.tool_map = tool_map + self.tool_composition = tool_composition self.run_tracer = run_tracer async def execute_one(self, call: ModelToolCall, index: int) -> ToolCallExecution: """Execute one model tool call with tracing.""" with self.run_tracer.tool(tool_name=call.name, call_id=call.id, arguments=call.arguments) as span: + composition = self.tool_composition.get(str(call.name)) + if composition is not None and composition.delegation: + span.set_attribute("subagent.delegation", True) call_token = _CURRENT_TOOL_CALL.set({"call_id": call.id, "name": call.name}) - runtime_token = _CURRENT_TOOL_RUNTIME.set({"run_metadata": dict(self.run_context.metadata)}) + runtime_token = _CURRENT_TOOL_RUNTIME.set({ + "run_metadata": dict(self.run_context.metadata), + "tool_map": self.tool_map, + "tool_composition": self.tool_composition, + }) emitter_token = _CURRENT_STREAM_EMITTER.set(self.run_context.emitter) cancelled = False start = time.perf_counter() @@ -195,7 +208,7 @@ async def execute_one(self, call: ModelToolCall, index: int) -> ToolCallExecutio self.harness.hooks.fire_after_tool_call(after) output = after.output envelope = after.envelope - self._annotate_special_tool(span, call.name, envelope) + self._annotate_special_tool(span, call.name, envelope, composition) span.set_attribute_where( lambda option: option.capture_tool_results, "gen_ai.tool.call.result", @@ -270,9 +283,15 @@ async def _call_output(self, name: str, arguments: str) -> ToolEnvelope: return ToolResult(False, f"unknown tool {name}", {"tool": name}) return await _invoke_tool(spec, arguments) - def _annotate_special_tool(self, span: _TraceSpan, name: str, envelope: ToolEnvelope) -> None: - """Add tool-family trace attributes for framework and MCP tools.""" - if name == "subagent": + def _annotate_special_tool( + self, + span: _TraceSpan, + name: str, + envelope: ToolEnvelope, + composition: _ToolComposition | None, + ) -> None: + """Add tool-family trace attributes from authoritative and attribution provenance.""" + if composition is not None and composition.delegation: span.set_attributes( { "subagent.name": envelope.metadata.get("agent"), diff --git a/thinharness/tools/base.py b/thinharness/tools/base.py index 09fe1b7..726789e 100644 --- a/thinharness/tools/base.py +++ b/thinharness/tools/base.py @@ -11,13 +11,12 @@ from dataclasses import dataclass, field from functools import partial from pathlib import Path -from typing import Any, Literal, TypeGuard, TypeVar, cast +from typing import Any, TypeGuard, TypeVar, cast from pydantic import BaseModel, ConfigDict, ValidationError from ..types import Json -ToolKind = Literal["user", "subagent"] ToolHandler = Callable[[Any], Any | Awaitable[Any]] T = TypeVar("T", bound=BaseModel) @@ -45,12 +44,9 @@ class ToolSpec: instructions: str | None = None requires_approval: bool = False origin: ToolOrigin | None = None - kind: ToolKind = "user" def __post_init__(self) -> None: """Validate per-tool retry configuration.""" - if self.kind not in {"user", "subagent"}: - raise ValueError(f"unknown tool kind: {self.kind}") if self.max_retries is not None and self.max_retries < 0: raise ValueError(f"max_retries must be >= 0, got {self.max_retries}") From abfb3183f6ce315bfeaa663819a6f350a6980db6 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Wed, 19 Aug 2026 01:23:49 -0400 Subject: [PATCH 13/30] Fix subagents plugin review findings --- docs/behavior.md | 8 +- scripts/build_transcripts.py | 19 +- tests/unit/test_approvals.py | 11 + tests/unit/test_parallel_llm.py | 42 +++ tests/unit/test_parallel_tools.py | 2 +- tests/unit/test_plugins.py | 20 ++ tests/unit/test_resume.py | 13 + tests/unit/test_skills.py | 26 ++ tests/unit/test_subagents.py | 440 ++++++++++++++++++++++++++++++ tests/unit/test_tracing.py | 15 +- thinharness/children.py | 9 + thinharness/core.py | 9 +- thinharness/hooks.py | 8 + thinharness/tool_execution.py | 11 +- thinharness/tools/skills.py | 91 ++++-- 15 files changed, 692 insertions(+), 32 deletions(-) diff --git a/docs/behavior.md b/docs/behavior.md index e2ff8a9..5b32a5d 100644 --- a/docs/behavior.md +++ b/docs/behavior.md @@ -95,12 +95,12 @@ Callers compose optional harness behavior explicitly while independent custom to - PLUGIN-5: Dynamic tools, instructions, and hooks are staged and receive the same complete validation as static contributions. ThinHarness commits the full dynamic set only after every binding opens successfully. - PLUGIN-6: A connection failure, including cancellation, closes entered bindings in reverse order, installs no dynamic contribution, and leaves connection retryable. `run_start` and `run_end` do not fire for an attempt that fails during connection. - PLUGIN-7: Closing a harness closes plugin bindings in reverse order before closing a model owned by the harness. Repeated close calls have no effect. -- PLUGIN-8: Contribution order is plugin static contributions, direct `tools=` and `hooks=`, then plugin dynamic contributions. System instructions are the configured system prompt, plugin instructions in caller plugin order, and all per-tool instructions; structured-output instructions are added through the existing output path. A skill summary is an ordinary plugin instruction at the `SkillsPlugin` position. `SubagentsPlugin` contributes its ordinary `subagent` tool at its plugin position. In a child, automatically inherited plugins keep parent plugin order, explicit child plugins follow them, inherited direct tools follow inherited plugins, and explicit child tools are last. +- PLUGIN-8: Contribution order is plugin static contributions, direct `tools=` and `hooks=`, then plugin dynamic contributions. System instructions are the configured system prompt, plugin instructions in caller plugin order, and all per-tool instructions; structured-output instructions are added through the existing output path. A skill summary is an ordinary plugin instruction at the `SkillsPlugin` position. `SubagentsPlugin` contributes its ordinary `subagent` tool at its plugin position. In a child, all plugin tools precede all direct tools: automatically inherited plugins keep parent plugin order, explicit child plugins follow them, inherited direct tools follow all plugins, and explicit child tools are last. - PLUGIN-9: ThinHarness copies caller-supplied hook registries before adding plugin hooks. Plugin composition never mutates a caller-owned registry. - PLUGIN-10: Plugins are trusted in-process code. ThinHarness does not isolate them or resolve dependencies between them. - PLUGIN-11: Each static plugin binding supplies an immutable agent-name tuple. Core combines these names before validating agent-filtered hooks and rejects blank or duplicate names. Connected contributions cannot change the agent catalog. - PLUGIN-12: Core records authoritative direct, plugin, and delegation composition roles independently of caller-visible `ToolOrigin`; these records control inheritance and delegation tracing and cannot be forged through tool metadata. -- PLUGIN-13: Automatic child inheritance is explicit and structural. Only a plugin with synchronous `for_child()` is rebound against the child context; its returned plugin must be valid and keep the expected fixed name. +- PLUGIN-13: Automatic child inheritance is explicit and structural. Only a plugin with synchronous `for_child()` is rebound against the child context; its returned plugin must be valid and keep the expected fixed name. `for_child()` and I/O-free `bind()` can run during parent construction and later child-recipe revalidation, so both operations must be repeatable and side-effect-free. ## Subagents Plugin @@ -114,7 +114,7 @@ Callers add delegation explicitly through `SubagentsPlugin`, which owns the mode - SUBAGENTS-PLUGIN-2: The unnamed child uses parent-derived model, prompt, limits, output, and additive inheritance defaults. Named configurations can override the model, prompt, limits, output, hooks, plugins, and tools; named children with no tools are valid. - SUBAGENTS-PLUGIN-3: `inherit_parent=True` rebinds only plugins that explicitly implement `for_child()` and inherits eligible direct tools from the active run snapshot. Inherited sources keep parent order, explicit child plugins and tools follow them, and duplicates fail rather than replace inherited values. Approval-required tools never enter a child. - SUBAGENTS-PLUGIN-4: Child hooks belong to the selected child configuration. Default-child hooks come from `default_hooks`; named hooks come from `SubAgentConfig.hooks`. Parent `before_subagent_run` and `after_subagent_run` hooks remain on the parent and can filter against the plugin's static agent catalog. -- SUBAGENTS-PLUGIN-5: Every top-level plugin context receives a narrow child host that accepts delegation only during an active parent tool call. Every child context receives a disabled host, and `SubagentsPlugin` is invalid in explicit child plugins, so children cannot create grandchildren through built-in or custom plugin paths. +- SUBAGENTS-PLUGIN-5: Every top-level plugin context receives a narrow child host that accepts delegation only during an active parent tool call. The active-call lease is shared across copied async contexts and becomes invalid when that tool call ends, so detached tasks cannot start children later. Every child context receives a disabled host, and `SubagentsPlugin` is invalid in explicit child plugins, so children cannot create grandchildren through built-in or custom plugin paths. - SUBAGENTS-PLUGIN-6: Each child is a fresh, independently budgeted run with no parent transcript. A parent-model child borrows the model; an override creates and closes its own model while projecting parent request settings and only same-provider credentials. - SUBAGENTS-PLUGIN-7: Child events remain nested unless subagent streaming is enabled, then preserve parent run and tool-call correlation. Real delegation is traced from authoritative composition with `subagent.delegation=true` from tool-span start; a same-named direct tool or forged `ToolOrigin` remains an ordinary tool event. - SUBAGENTS-PLUGIN-8: Successful results preserve agent, inheritance mode, effective tools, child request usage, and structured-output metadata. Unknown agents, provider failures, hook failures, strict sibling cancellation, and parent cancellation preserve existing error and cleanup behavior; every created child closes without hiding the original error. @@ -148,7 +148,7 @@ Callers explicitly compose a fixed skill catalog and select which skill operatio - SKILLS-PLUGIN-1: A harness accepts at most one runtime-fixed `SkillsPlugin` named `"skills"`. The plugin requires one or more ordered skill directories and an explicit non-empty ordered selection of `skill_read`, `skill_run`, or both. - SKILLS-PLUGIN-2: The plugin discovers and validates its catalog during construction. Its selected tools and summary are static and visible immediately after harness construction, and binding performs no I/O. - SKILLS-PLUGIN-3: Relative skill directories resolve from the process working directory, not from `HarnessConfig.root`. Reusing one plugin object across harnesses reuses the same registry and catalog. -- SKILLS-PLUGIN-4: Catalog names, paths, metadata, selection, and summary are frozen at plugin construction. Existing skill content, file trees, and scripts remain live and are read or executed when a tool is invoked. A new plugin is required to discover added or removed skills. +- SKILLS-PLUGIN-4: Catalog names, paths, metadata, selection, and summary are frozen at plugin construction. The public registry returns deep detached catalog values, so mutations cannot change later parent or child behavior. Existing skill content, file trees, and scripts remain live and are read or executed when a tool is invoked. A new plugin is required to discover added or removed skills. - SKILLS-PLUGIN-5: A non-empty catalog contributes selected tools in caller order and one compact summary. The summary mentions `skill_read` only when that tool is selected. A catalog with no skills contributes no tools or summary. - SKILLS-PLUGIN-6: `skill_read` preserves live content, tree, containment, and truncation behavior and is parallel-safe. `skill_run` preserves runner, working-directory, merged-output, timeout, metadata, and containment behavior and runs sequentially. - SKILLS-PLUGIN-7: `SkillsPlugin` explicitly opts into safe child inheritance and rebinds through the generic plugin contract. Inherited bindings share the exact constructor-time registry, frozen catalog, tool order, and one summary without another discovery pass. Its constructor configuration is frozen so later input, property-value, or attribute mutation cannot change a binding. diff --git a/scripts/build_transcripts.py b/scripts/build_transcripts.py index 9c84cd5..49f65f7 100644 --- a/scripts/build_transcripts.py +++ b/scripts/build_transcripts.py @@ -418,10 +418,10 @@ def event_from_span(span: dict[str, Any], trace_rel: str, index: int, call_label return event -def load_agents() -> list[dict[str, Any]]: +def load_agents(*, examples_root: Path = EXAMPLES_ROOT) -> list[dict[str, Any]]: agents: list[dict[str, Any]] = [] audit_metadata = spec_audit_metadata() - for summary_path in sorted(EXAMPLES_ROOT.glob("*/outputs/run_summary.json")): + for summary_path in sorted(examples_root.glob("*/outputs/run_summary.json")): summary = json.loads(summary_path.read_text(encoding="utf-8")) root = summary_path.parents[1] slug = str(summary.get("slug") or root.name) @@ -527,13 +527,22 @@ def render_html(agents: list[dict[str, Any]], *, template_path: Path | None = No return template +def write_transcripts(output: Path, *, examples_root: Path = EXAMPLES_ROOT) -> list[dict[str, Any]]: + """Rebuild one tracked transcript page without allowing an empty source set to blank it.""" + agents = load_agents(examples_root=examples_root) + if not agents: + raise ValueError("no example transcript sources found; existing output was not changed") + rendered = render_html(agents, template_path=output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(rendered, encoding="utf-8") + return agents + + def main() -> None: parser = argparse.ArgumentParser(description="Render example agent traces as readable example HTML.") parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) args = parser.parse_args() - agents = load_agents() - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(render_html(agents, template_path=args.output), encoding="utf-8") + agents = write_transcripts(args.output) print(args.output) print(json.dumps({"agents": [agent["slug"] for agent in agents], "count": len(agents)}, indent=2)) diff --git a/tests/unit/test_approvals.py b/tests/unit/test_approvals.py index 21d45f7..ed2063c 100644 --- a/tests/unit/test_approvals.py +++ b/tests/unit/test_approvals.py @@ -94,6 +94,12 @@ async def test_approval_state_excludes_plugin_configuration_and_context_model(tm plugins=[ SkillsPlugin(tmp_path / "skills", tools=["skill_read"]), ParallelLlmPlugin(description="approval-parallel-description-sentinel"), + SubagentsPlugin(agents=[SubAgentConfig( + name="approval-child-sentinel", + description="approval-child-description-sentinel", + system_prompt="approval-child-prompt-sentinel", + model="openai:approval-child-model-sentinel", + )]), ], tools=[approval_tool()], ) @@ -106,6 +112,11 @@ async def test_approval_state_excludes_plugin_configuration_and_context_model(tm assert "approval-state-skill-sentinel" not in serialized assert "approval-state-description-sentinel" not in serialized assert "approval-parallel-description-sentinel" not in serialized + assert "approval-child-sentinel" not in serialized + assert "approval-child-description-sentinel" not in serialized + assert "approval-child-prompt-sentinel" not in serialized + assert "approval-child-model-sentinel" not in serialized + assert "ChildHarnessHost" not in serialized assert "PluginContext" not in serialized assert "context_marker" not in serialized diff --git a/tests/unit/test_parallel_llm.py b/tests/unit/test_parallel_llm.py index d93c464..fb6d144 100644 --- a/tests/unit/test_parallel_llm.py +++ b/tests/unit/test_parallel_llm.py @@ -837,6 +837,48 @@ def capture_tool(**kwargs: Any) -> ParallelLlmTool: } +def test_parallel_llm_constructor_and_public_values_stay_frozen_across_bindings( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: list[dict[str, Any]] = [] + real_tool = parallel_plugin_module.ParallelLlmTool + + def capture_tool(**kwargs: Any) -> ParallelLlmTool: + captured.append(kwargs) + return real_tool(**kwargs) + + monkeypatch.setattr(parallel_plugin_module, "ParallelLlmTool", capture_tool) + read_paths = ["inputs"] + write_paths = ["outputs"] + extra_body = {"nested": {"stable": True}} + plugin = ParallelLlmPlugin( + "openai:fixed", + read_paths=read_paths, + write_paths=write_paths, + extra_body=extra_body, + ) + read_paths[0] = "mutated" + write_paths[0] = "mutated" + extra_body["nested"]["stable"] = False + returned = plugin.extra_body + returned["nested"]["stable"] = False + + first_model = BatchModel() + second_model = BatchModel() + plugin.bind(PluginContext(root=tmp_path / "first", model=first_model, child_harnesses=FakeChildHarnessHost())) + plugin.for_child().bind( + PluginContext(root=tmp_path / "second", model=second_model, child_harnesses=FakeChildHarnessHost()) + ) + + assert [entry["read_paths"] for entry in captured] == [["inputs"], ["inputs"]] + assert [entry["write_paths"] for entry in captured] == [["outputs"], ["outputs"]] + assert [entry["extra_body"] for entry in captured] == [ + {"nested": {"stable": True}}, + {"nested": {"stable": True}}, + ] + + async def test_parallel_llm_plugin_reuse_borrows_each_harness_model(tmp_path: Path) -> None: plugin = ParallelLlmPlugin() first_model = BatchModel(outcomes=["first"]) diff --git a/tests/unit/test_parallel_tools.py b/tests/unit/test_parallel_tools.py index f434e0e..c3ed286 100644 --- a/tests/unit/test_parallel_tools.py +++ b/tests/unit/test_parallel_tools.py @@ -29,7 +29,7 @@ def test_tool_spec_sequential_default_and_not_in_schema() -> None: assert flagged.sequential is True assert "sequential" not in flagged.response_tool() -def test_builtin_tools_mark_mutating_specs_sequential(tmp_path: Path) -> None: +def test_mutating_tool_specs_are_sequential(tmp_path: Path) -> None: by_name = {spec.name: spec for spec in FileTools(tmp_path).specs()} assert by_name["read"].sequential is False assert by_name["search"].sequential is False diff --git a/tests/unit/test_plugins.py b/tests/unit/test_plugins.py index 963d184..0d37dbd 100644 --- a/tests/unit/test_plugins.py +++ b/tests/unit/test_plugins.py @@ -312,6 +312,26 @@ def test_one_plugin_object_receives_each_harness_root_and_model(tmp_path: Path) assert first.tools[0] is not second.tools[0] +def test_filesystem_constructor_inputs_stay_frozen_across_later_child_bindings(tmp_path: Path) -> None: + selected = ["read"] + read_paths = ["allowed"] + excludes = ["*.tmp"] + plugin = FilesystemPlugin(tools=selected, read_paths=read_paths, search_exclude_globs=excludes) + selected[0] = "write" + read_paths[0] = "blocked" + excludes[0] = "*.txt" + + for root in (tmp_path / "parent", tmp_path / "child"): + (root / "allowed").mkdir(parents=True) + (root / "allowed" / "value.txt").write_text(root.name, encoding="utf-8") + harness = Harness(HarnessConfig(root=root), model=ScriptedModel([]), plugins=[plugin.for_child()]) + assert [tool.name for tool in harness.tools] == ["read"] + spec = harness.tools[0] + result = spec.handler(spec.parse_args({"path": "allowed/value.txt"})) + assert result.ok is True + assert root.name in result.content + + def test_filesystem_bind_performs_no_metadata_io(tmp_path: Path, monkeypatch) -> None: root = tmp_path.resolve() diff --git a/tests/unit/test_resume.py b/tests/unit/test_resume.py index e636d71..f73fbc1 100644 --- a/tests/unit/test_resume.py +++ b/tests/unit/test_resume.py @@ -24,6 +24,8 @@ OpenRouterModel, ParallelLlmPlugin, SkillsPlugin, + SubAgentConfig, + SubagentsPlugin, ToolSpec, ) from thinharness.hooks import RunEndContext @@ -71,6 +73,12 @@ async def test_resume_state_excludes_plugin_configuration_and_context_model(tmp_ plugins=[ SkillsPlugin(tmp_path / "skills", tools=["skill_read"]), ParallelLlmPlugin(description="resume-parallel-description-sentinel"), + SubagentsPlugin(agents=[SubAgentConfig( + name="resume-child-sentinel", + description="resume-child-description-sentinel", + system_prompt="resume-child-prompt-sentinel", + model="openai:resume-child-model-sentinel", + )]), ], ) @@ -82,6 +90,11 @@ async def test_resume_state_excludes_plugin_configuration_and_context_model(tmp_ assert "resume-state-skill-sentinel" not in serialized assert "resume-state-description-sentinel" not in serialized assert "resume-parallel-description-sentinel" not in serialized + assert "resume-child-sentinel" not in serialized + assert "resume-child-description-sentinel" not in serialized + assert "resume-child-prompt-sentinel" not in serialized + assert "resume-child-model-sentinel" not in serialized + assert "ChildHarnessHost" not in serialized assert "PluginContext" not in serialized assert "context_marker" not in serialized diff --git a/tests/unit/test_skills.py b/tests/unit/test_skills.py index 04d7f93..2c73684 100644 --- a/tests/unit/test_skills.py +++ b/tests/unit/test_skills.py @@ -195,6 +195,32 @@ def test_skills_plugin_catalog_is_frozen_but_discovered_content_and_scripts_are_ +def test_skills_plugin_public_catalog_and_constructor_inputs_are_deep_detached(tmp_path: Path) -> None: + skill = tmp_path / "skills" / "demo" + skill.mkdir(parents=True) + (skill / "SKILL.md").write_text( + '---\nname: demo\ndescription: Stable summary\nlabels: {"nested":{"value":"stable"}}\n---\nLive body', + encoding="utf-8", + ) + selected = ["demo"] + tools = ["skill_read"] + plugin = SkillsPlugin(tmp_path / "skills", selected_skills=selected, tools=tools) + selected[0] = "missing" + tools[0] = "skill_run" + public = plugin.registry.skills + public["demo"].metadata["labels"]["nested"]["value"] = "mutated" + public.clear() + + first = Harness(HarnessConfig(root=tmp_path / "parent"), model=ScriptedModel([]), plugins=[plugin]) + second = Harness(HarnessConfig(root=tmp_path / "child"), model=ScriptedModel([]), plugins=[plugin.for_child()]) + + assert [tool.name for tool in first.tools] == ["skill_read"] + assert [tool.name for tool in second.tools] == ["skill_read"] + assert "demo - Stable summary" in first.system_instructions() + assert "demo - Stable summary" in second.system_instructions() + assert plugin.registry.skills["demo"].metadata["labels"]["nested"]["value"] == "stable" + + def test_skills_plugin_relative_paths_use_cwd_and_reuse_one_catalog(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: process_dir = tmp_path / "process" _write_skill(process_dir / "skills", "demo") diff --git a/tests/unit/test_subagents.py b/tests/unit/test_subagents.py index 29b3f65..329a02f 100644 --- a/tests/unit/test_subagents.py +++ b/tests/unit/test_subagents.py @@ -1,7 +1,10 @@ from __future__ import annotations import asyncio +from contextlib import asynccontextmanager +from dataclasses import replace from pathlib import Path +from typing import Any import pytest from fakes import FailingSession, FakeTracer, ScriptedModel, ScriptedSession, echo_tool, tool_output @@ -17,10 +20,12 @@ HarnessConfig, Hook, HookRegistry, + ParallelLlmPlugin, PluginBinding, PluginContext, PluginContribution, SkillsPlugin, + StreamOptions, SubAgentConfig, SubagentsPlugin, ToolOrigin, @@ -714,6 +719,441 @@ def test_plugin_object_reuse_binds_independent_hosts(tmp_path: Path) -> None: assert first._child_harnesses is not second._child_harnesses +async def test_detached_delegation_task_loses_active_call_lease( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + release = asyncio.Event() + detached: list[asyncio.Task[ChildHarnessOutcome]] = [] + inferred: list[str] = [] + request = ChildHarnessRequest( + agent_name="late", + agent_description="Late child", + trace_agent_name="subagent.late", + task="late", + inherited=False, + tool_mode="explicit", + system_prompt="late", + model="openai:late", + ) + + class DetachedPlugin: + name = "detached" + + def bind(self, context: PluginContext) -> PluginBinding: + async def invoke(_args: Any) -> str: + async def delayed() -> ChildHarnessOutcome: + await release.wait() + return await context.child_harnesses.run(request) + + detached.append(asyncio.create_task(delayed())) + return "scheduled" + + tool = ToolSpec("delegate_later", "Delegate later", {"type": "object"}, invoke) + registered = context.child_harnesses.register_delegation_tool(tool, [request]) + return PluginBinding(static=PluginContribution(tools=(registered,))) + + def infer(model_ref: str, **_kwargs: Any) -> ScriptedModel: + inferred.append(model_ref) + return ScriptedModel([]) + + monkeypatch.setattr("thinharness.children.infer_model", infer) + parent = ScriptedSession( + start_turn=ModelTurn( + tool_calls=[ModelToolCall(id="late_call", name="delegate_later", arguments="{}")], + raw={}, + ), + continue_turn=ModelTurn(text="done", raw={}), + ) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([parent]), plugins=[DetachedPlugin()]) + + assert (await harness.run("go")).text == "done" + release.set() + with pytest.raises(Exception, match="active parent tool call"): + await detached[0] + assert inferred == [] + + +async def test_connected_registered_delegation_keeps_provenance_and_host_access(tmp_path: Path) -> None: + tracer = FakeTracer() + marker_at_hook: list[bool] = [] + + class ConnectedDelegationPlugin: + name = "connected-delegation" + + def bind(self, context: PluginContext) -> PluginBinding: + request = ChildHarnessRequest( + agent_name="connected", + agent_description="Connected child", + trace_agent_name="subagent.connected", + task="", + inherited=False, + tool_mode="explicit", + system_prompt="connected", + ) + + async def invoke(args: dict[str, Any]) -> str: + outcome = await context.child_harnesses.run(replace(request, task=str(args["task"]))) + return outcome.content + + raw = ToolSpec( + "connected_delegate", + "Connected delegate", + {"type": "object", "properties": {"task": {"type": "string"}}, "required": ["task"]}, + invoke, + ) + registered = context.child_harnesses.register_delegation_tool(raw, [request]) + + @asynccontextmanager + async def connect(): + yield PluginContribution(tools=(registered,)) + + return PluginBinding(connect=connect) + + parent = ScriptedSession( + start_turn=ModelTurn( + tool_calls=[ModelToolCall(id="connected_call", name="connected_delegate", arguments='{"task":"help"}')], + raw={}, + ), + continue_turn=ModelTurn(text="parent done", raw={}), + ) + child = ScriptedSession(start_turn=ModelTurn(text="connected child done", raw={})) + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([parent, child]), + plugins=[ConnectedDelegationPlugin()], + hooks=[Hook( + "before_tool_call", + lambda _ctx: marker_at_hook.append(tracer.stack[-1].attributes.get("subagent.delegation") is True), + tools=["connected_delegate"], + )], + tracing=[TracingOptions(tracer=tracer)], + ) + + assert (await harness.run("go")).text == "parent done" + output = tool_output(parent.continue_calls[0][0][0].output) + span = next(span for span in tracer.spans if span.name == "execute_tool connected_delegate") + assert output["content"] == "connected child done" + assert marker_at_hook == [True] + assert span.attributes["subagent.delegation"] is True + + +@pytest.mark.parametrize( + ("child_ref", "expected_key", "expected_base"), + [ + ("openai:child", "parent-key", "https://parent.test"), + ("anthropic:child", None, None), + ], +) +async def test_child_override_projects_only_same_provider_credentials( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + child_ref: str, + expected_key: str | None, + expected_base: str | None, +) -> None: + captured: dict[str, Any] = {} + child_model = ScriptedModel([ScriptedSession(start_turn=ModelTurn(text="child", raw={}))]) + + def infer(model_ref: str, **kwargs: Any) -> ScriptedModel: + captured.update(model_ref=model_ref, **kwargs) + return child_model + + monkeypatch.setattr("thinharness.children.infer_model", infer) + parent = ScriptedSession(start_turn=_parent_call(agent="override"), continue_turn=ModelTurn(text="done", raw={})) + harness = Harness( + HarnessConfig( + root=tmp_path, + api_key="parent-key", + base_url="https://parent.test", + request_timeout=17, + request_retries=2, + temperature=0.4, + ), + model=ScriptedModel([parent]), + plugins=[SubagentsPlugin(agents=[ + SubAgentConfig(name="override", description="Override.", model=child_ref) + ])], + ) + + assert (await harness.run("go")).text == "done" + assert captured["model_ref"] == child_ref + assert captured["api_key"] == expected_key + assert captured["base_url"] == expected_base + assert captured["timeout"] == 17 + assert captured["request_retries"] == 2 + assert captured["temperature"] == 0.4 + + +async def test_default_child_borrows_parent_model_without_closing_provider(tmp_path: Path) -> None: + parent = ScriptedSession(start_turn=_parent_call(), continue_turn=ModelTurn(text="done", raw={})) + model = ScriptedModel([parent, ScriptedSession(start_turn=ModelTurn(text="child", raw={}))]) + provider = ClosingProvider() + model.provider = provider + harness = Harness(HarnessConfig(root=tmp_path), model=model, plugins=[SubagentsPlugin()]) + + assert (await harness.run("go")).text == "done" + assert provider.closed == 0 + + +async def test_concurrent_override_delegations_own_and_close_distinct_models( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + models = [ + ScriptedModel([ScriptedSession(start_turn=ModelTurn(text="one", raw={}))]), + ScriptedModel([ScriptedSession(start_turn=ModelTurn(text="two", raw={}))]), + ] + providers = [ClosingProvider(), ClosingProvider()] + for model, provider in zip(models, providers, strict=True): + model.provider = provider + created: list[ScriptedModel] = [] + + def infer(*_args: Any, **_kwargs: Any) -> ScriptedModel: + model = models[len(created)] + created.append(model) + return model + + monkeypatch.setattr("thinharness.children.infer_model", infer) + parent = ScriptedSession( + start_turn=ModelTurn( + tool_calls=[ + ModelToolCall(id="one", name="subagent", arguments='{"task":"one","agent":"worker"}'), + ModelToolCall(id="two", name="subagent", arguments='{"task":"two","agent":"worker"}'), + ], + raw={}, + ), + continue_turn=ModelTurn(text="done", raw={}), + ) + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([parent]), + plugins=[SubagentsPlugin(agents=[ + SubAgentConfig(name="worker", description="Worker.", model="openai:child") + ])], + ) + + assert (await harness.run("go")).text == "done" + assert len({id(model) for model in created}) == 2 + assert [provider.closed for provider in providers] == [1, 1] + + +async def test_strict_sibling_abort_does_not_hang_concurrent_delegation(tmp_path: Path) -> None: + class BlockingSession: + async def start(self, *_args: Any, **_kwargs: Any) -> ModelTurn: + await asyncio.Event().wait() + raise AssertionError("unreachable") + + async def continue_with_tools(self, *_args: Any, **_kwargs: Any) -> ModelTurn: + raise AssertionError("unreachable") + + async def continue_with_user_text(self, *_args: Any, **_kwargs: Any) -> ModelTurn: + raise AssertionError("unreachable") + + def dump_state(self) -> None: + return None + + def fail_sibling(ctx: Any) -> None: + if ctx.tool_name == "fail": + raise RuntimeError("strict sibling abort") + + parent = ScriptedSession( + start_turn=ModelTurn( + tool_calls=[ + ModelToolCall(id="delegate", name="subagent", arguments='{"task":"wait"}'), + ModelToolCall(id="fail", name="fail", arguments="{}"), + ], + raw={}, + ) + ) + harness = Harness( + HarnessConfig(root=tmp_path, strict_hooks=True), + model=ScriptedModel([parent, BlockingSession()]), + plugins=[SubagentsPlugin()], + tools=[ToolSpec("fail", "Fail", {"type": "object"}, lambda _args: "unused")], + hooks=[Hook("before_tool_call", fail_sibling)], + ) + + with pytest.raises(RuntimeError, match="strict sibling abort"): + await asyncio.wait_for(harness.run("go"), timeout=1) + + +async def test_child_budgets_notices_and_tool_retry_fallback_are_fresh(tmp_path: Path) -> None: + observed: list[tuple[str, int, int | None, int]] = [] + + def record(label: str): + def hook(ctx: Any) -> None: + observed.append((label, ctx.max_model_requests, ctx.max_tool_calls, ctx.harness.config.tool_retries)) + + return hook + + first_parent = ScriptedSession(start_turn=_parent_call(), continue_turn=ModelTurn(text="first", raw={})) + first_child = ScriptedSession( + start_turn=ModelTurn( + tool_calls=[ModelToolCall(id="echo", name="echo", arguments='{"value":"ok"}')], + raw={}, + ), + continue_turn=ModelTurn(text="child first", raw={}), + ) + second_parent = ScriptedSession( + start_turn=_parent_call(agent="named", call_id="named_call"), + continue_turn=ModelTurn(text="second", raw={}), + ) + second_child = ScriptedSession(start_turn=ModelTurn(text="child second", raw={})) + harness = Harness( + HarnessConfig(root=tmp_path, max_model_requests=2, max_tool_calls=8, tool_retries=4), + model=ScriptedModel([first_parent, first_child, second_parent, second_child]), + plugins=[SubagentsPlugin( + default_hooks=[Hook("run_start", record("default"))], + agents=[SubAgentConfig( + name="named", + description="Named.", + max_model_requests=3, + max_tool_calls=2, + hooks=[Hook("run_start", record("named"))], + )], + )], + tools=[echo_tool()], + ) + + assert (await harness.run("first")).text == "first" + assert (await harness.run("second")).text == "second" + first_envelope = tool_output(first_parent.continue_calls[0][0][0].output) + assert first_envelope["metadata"]["model_requests"] == 2 + assert [(notice.limit_kind, notice.remaining) for notice in first_child.notice_calls[1][1]] == [ + ("model_requests", 1) + ] + assert observed == [ + ("default", 2, 8, 4), + ("named", 3, 2, 1), + ] + + +async def test_inherited_parallel_model_resolution_follows_frozen_configuration( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import thinharness.plugins.parallel_llm as parallel_module + + real_tool = parallel_module.ParallelLlmTool + captured: list[Any] = [] + + def capture(**kwargs: Any): + captured.append(kwargs["model"]) + return real_tool(**kwargs) + + monkeypatch.setattr(parallel_module, "ParallelLlmTool", capture) + + async def run_case(plugin: ParallelLlmPlugin) -> tuple[Any, Any, Any]: + parent_model = ScriptedModel([ + ScriptedSession(start_turn=_parent_call(agent="worker"), continue_turn=ModelTurn(text="done", raw={})) + ]) + child_model = ScriptedModel([ScriptedSession(start_turn=ModelTurn(text="child", raw={}))]) + monkeypatch.setattr("thinharness.children.infer_model", lambda *_args, **_kwargs: child_model) + before = len(captured) + harness = Harness( + HarnessConfig(root=tmp_path), + model=parent_model, + plugins=[plugin, SubagentsPlugin(agents=[ + SubAgentConfig(name="worker", description="Worker.", inherit_parent=True, model="openai:child") + ])], + ) + await harness.run("go") + return parent_model, child_model, tuple(captured[before:]) + + borrowed_parent, borrowed_child, borrowed = await run_case(ParallelLlmPlugin()) + explicit_model = ScriptedModel([]) + _object_parent, _object_child, object_models = await run_case(ParallelLlmPlugin(explicit_model)) + _string_parent, _string_child, string_models = await run_case(ParallelLlmPlugin("openai:fixed")) + + assert borrowed == (borrowed_parent, borrowed_child) + assert object_models == (explicit_model, explicit_model) + assert string_models == ("openai:fixed", "openai:fixed") + + +async def test_reused_subagents_plugin_keeps_parent_runs_fully_isolated(tmp_path: Path) -> None: + plugin = SubagentsPlugin() + hook_metadata: list[tuple[str, dict[str, Any]]] = [] + child_inputs: list[tuple[str, list[str], dict[str, Any]]] = [] + + def parent(label: str, child_text: str, tool_name: str) -> Harness: + parent_session = ScriptedSession(start_turn=_parent_call(), continue_turn=ModelTurn(text=f"{label} parent", raw={})) + + def child_start(_prompt: str, instructions: str, tools: list[dict[str, Any]], metadata: dict[str, Any], _previous: Any) -> None: + child_inputs.append((instructions, [tool["name"] for tool in tools], dict(metadata))) + + child_session = ScriptedSession( + start_turn=ModelTurn(text=child_text, raw={}), + on_start=child_start, + ) + return Harness( + HarnessConfig(root=tmp_path / label), + model=ScriptedModel([parent_session, child_session], model=f"{label}-model"), + plugins=[FilesystemPlugin(tools=["read"]), plugin], + tools=[ToolSpec(tool_name, tool_name, {"type": "object"}, lambda _args: label)], + hooks=[Hook( + "before_subagent_run", + lambda ctx: hook_metadata.append((label, dict(ctx.metadata))), + agents=["default"], + )], + ) + + first = parent("first", "first child", "first_tool") + second = parent("second", "second child", "second_tool") + + async def collect(harness: Harness, metadata: dict[str, Any]) -> tuple[str, list[Any]]: + events: list[Any] = [] + stream = harness.stream("go", metadata=metadata, stream_options=StreamOptions(include_subagents=True)) + async with stream as values: + async for event in values: + events.append(event) + result = next(event.result for event in events if event.kind == "run_completed" and event.parent_run_id is None) + return result.text, events + + first_text, first_events = await collect(first, {"conversation_id": "first-conversation"}) + second_text, second_events = await collect(second, {"conversation_id": "second-conversation"}) + + assert (first_text, second_text) == ("first parent", "second parent") + assert [entry[1] for entry in child_inputs] == [["read", "first_tool"], ["read", "second_tool"]] + assert str((tmp_path / "first").resolve()) in child_inputs[0][0] + assert str((tmp_path / "second").resolve()) in child_inputs[1][0] + assert child_inputs[0][2]["conversation_id"] == "first-conversation" + assert child_inputs[1][2]["conversation_id"] == "second-conversation" + assert hook_metadata == [ + ("first", {"conversation_id": "first-conversation"}), + ("second", {"conversation_id": "second-conversation"}), + ] + assert [event.text for event in first_events if event.kind == "model_message" and event.agent_name == "default"] == ["first child"] + assert [event.text for event in second_events if event.kind == "model_message" and event.agent_name == "default"] == ["second child"] + + +async def test_failed_child_run_with_cancelled_cleanup_propagates_cancellation(tmp_path: Path) -> None: + class CancelOnClosePlugin: + name = "cancel-on-close" + + def bind(self, _context: PluginContext) -> PluginBinding: + @asynccontextmanager + async def connect(): + try: + yield PluginContribution() + finally: + raise asyncio.CancelledError + + return PluginBinding(connect=connect) + + parent = ScriptedSession(start_turn=_parent_call(agent="cancel"), continue_turn=ModelTurn(text="unused", raw={})) + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([parent, FailingSession()]), + plugins=[SubagentsPlugin(agents=[ + SubAgentConfig(name="cancel", description="Cancel cleanup.", plugins=[CancelOnClosePlugin()]) + ])], + ) + + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(harness.run("go"), timeout=1) + + def test_harness_removed_fields_and_constructor_helpers_are_gone() -> None: for field in ("builtin_tools", "subagents"): with pytest.raises(ValueError, match=rf"HarnessConfig\.{field} has been removed.*SubagentsPlugin"): diff --git a/tests/unit/test_tracing.py b/tests/unit/test_tracing.py index 00d9b79..6619cec 100644 --- a/tests/unit/test_tracing.py +++ b/tests/unit/test_tracing.py @@ -39,7 +39,9 @@ from thinharness.providers import ModelNotice, ModelToolCall, ModelTurn, TokenUsage, ToolOutput from thinharness.tracing import _SpanAdapter, annotate_model_request, create_local_tracing_options, serialize_attribute_value -event_from_span = run_path(str(Path(__file__).resolve().parents[2] / "scripts" / "build_transcripts.py"))["event_from_span"] +_transcript_script = run_path(str(Path(__file__).resolve().parents[2] / "scripts" / "build_transcripts.py")) +event_from_span = _transcript_script["event_from_span"] +write_transcripts = _transcript_script["write_transcripts"] class Person(BaseModel): @@ -569,6 +571,17 @@ def test_provider_error_keeps_trace_input_without_output(tmp_path: Path) -> None assert "gen_ai.completion" not in root.attributes assert root.status is not None +def test_transcript_generation_refuses_to_blank_existing_output_without_sources(tmp_path: Path) -> None: + output = tmp_path / "index.html" + original = '' + output.write_text(original, encoding="utf-8") + + with pytest.raises(ValueError, match="existing output was not changed"): + write_transcripts(output, examples_root=tmp_path / "empty-examples") + + assert output.read_text(encoding="utf-8") == original + + def test_transcript_classification_requires_authoritative_delegation_marker() -> None: base = { "name": "execute_tool subagent", diff --git a/thinharness/children.py b/thinharness/children.py index 26d1036..1e90c8d 100644 --- a/thinharness/children.py +++ b/thinharness/children.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio from collections.abc import Sequence from dataclasses import dataclass from types import TracebackType @@ -13,6 +14,7 @@ BeforeSubagentRunContext, Hook, HookRegistry, + _ToolRuntimeLease, current_tool_call_context, current_tool_runtime_context, ) @@ -192,6 +194,9 @@ async def run(self, request: ChildHarnessRequest) -> ChildHarnessOutcome: tool_call = current_tool_call_context() if runtime is None or tool_call is None: raise HarnessError("child harness request requires an active parent tool call") + lease = runtime.get("lease") + if not isinstance(lease, _ToolRuntimeLease) or not lease.active: + raise HarnessError("child harness request requires an active parent tool call") tool_map = runtime.get("tool_map") composition_map = runtime.get("tool_composition") if not isinstance(tool_map, dict) or not isinstance(composition_map, dict): @@ -269,6 +274,8 @@ async def run(self, request: ChildHarnessRequest) -> ChildHarnessOutcome: run_traceback = exc.__traceback__ try: await child.aclose() + except asyncio.CancelledError: + raise except BaseException as close_error: if run_error is None: raise @@ -414,6 +421,8 @@ async def _close_model_after_failure(model: Model, original_error: BaseException return try: await aclose() + except asyncio.CancelledError: + raise except BaseException as close_error: original_error.add_note(f"cleanup also failed: {type(close_error).__name__}: {close_error}") diff --git a/thinharness/core.py b/thinharness/core.py index 5853f84..c7db997 100644 --- a/thinharness/core.py +++ b/thinharness/core.py @@ -865,8 +865,13 @@ async def _connect_once(self) -> None: normalized = self._normalize_contribution(plugin.name, contribution) dynamic_tools.extend(normalized.tools) dynamic_compositions.extend( - _ToolComposition(source="plugin", plugin_index=plugin_index) - for _ in normalized.tools + _ToolComposition( + source="plugin", + plugin_index=plugin_index, + delegation=isinstance(self._child_harnesses, _ParentChildHarnessHost) + and self._child_harnesses.is_delegation_tool(raw_tool), + ) + for raw_tool in contribution.tools ) dynamic_instructions.extend(normalized.instructions) dynamic_hooks.extend(normalized.hooks) diff --git a/thinharness/hooks.py b/thinharness/hooks.py index 474cc39..7b6041f 100644 --- a/thinharness/hooks.py +++ b/thinharness/hooks.py @@ -16,6 +16,14 @@ _CURRENT_TOOL_RUNTIME: contextvars.ContextVar[dict[str, Any] | None] = contextvars.ContextVar("thinharness_current_tool_runtime", default=None) +@dataclass +class _ToolRuntimeLease: + """Shared mutable validity for one tool call's copied runtime context.""" + + active: bool = True + + + def current_tool_call_context() -> Json | None: """Return the current tool call context for nested tool handlers.""" return _CURRENT_TOOL_CALL.get() diff --git a/thinharness/tool_execution.py b/thinharness/tool_execution.py index 4951dda..1453ad8 100644 --- a/thinharness/tool_execution.py +++ b/thinharness/tool_execution.py @@ -13,7 +13,13 @@ ToolCallCompletedEvent, ToolCallStartedEvent, ) -from .hooks import _CURRENT_TOOL_CALL, _CURRENT_TOOL_RUNTIME, AfterToolCallContext, BeforeToolCallContext +from .hooks import ( + _CURRENT_TOOL_CALL, + _CURRENT_TOOL_RUNTIME, + AfterToolCallContext, + BeforeToolCallContext, + _ToolRuntimeLease, +) from .providers import ModelToolCall, ToolOutput from .tools.base import Json, ToolEnvelope, ToolResult, ToolSpec, _invoke_tool from .tracing import RunTracer, serialize_attribute_value @@ -152,8 +158,10 @@ async def execute_one(self, call: ModelToolCall, index: int) -> ToolCallExecutio composition = self.tool_composition.get(str(call.name)) if composition is not None and composition.delegation: span.set_attribute("subagent.delegation", True) + lease = _ToolRuntimeLease() call_token = _CURRENT_TOOL_CALL.set({"call_id": call.id, "name": call.name}) runtime_token = _CURRENT_TOOL_RUNTIME.set({ + "lease": lease, "run_metadata": dict(self.run_context.metadata), "tool_map": self.tool_map, "tool_composition": self.tool_composition, @@ -246,6 +254,7 @@ async def execute_one(self, call: ModelToolCall, index: int) -> ToolCallExecutio ) raise finally: + lease.active = False _CURRENT_STREAM_EMITTER.reset(emitter_token) _CURRENT_TOOL_RUNTIME.reset(runtime_token) _CURRENT_TOOL_CALL.reset(call_token) diff --git a/thinharness/tools/skills.py b/thinharness/tools/skills.py index e07e837..1b1e57c 100644 --- a/thinharness/tools/skills.py +++ b/thinharness/tools/skills.py @@ -4,9 +4,10 @@ import json import subprocess -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass from pathlib import Path +from types import MappingProxyType from typing import Any from pydantic import BaseModel, ConfigDict, Field @@ -16,7 +17,7 @@ @dataclass(frozen=True) class Skill: - """A discovered skill directory and its metadata.""" + """A discovered skill directory and its detached metadata.""" name: str description: str @@ -25,6 +26,17 @@ class Skill: metadata: Json +@dataclass(frozen=True) +class _CatalogSkill: + """One deeply immutable constructor-time catalog entry.""" + + name: str + description: str + root: Path + skill_file: Path + metadata: object + + class SkillArgs(BaseModel): """Base class for skill tool arguments.""" @@ -58,14 +70,33 @@ def __init__( *, selected_skills: Sequence[str] | None = None, ) -> None: - self.skills_dirs = _normalize_skill_dirs(skills_dir) - self.selected_skills = list(selected_skills) if selected_skills is not None else None - self._skills = self._select_skills(self._discover()) + self._skills_dirs = tuple(_normalize_skill_dirs(skills_dir)) + self._selected_skills = tuple(selected_skills) if selected_skills is not None else None + self._skills = MappingProxyType(self._select_skills(self._discover())) + + @property + def skills_dirs(self) -> tuple[Path, ...]: + """Return the frozen discovery directories.""" + return self._skills_dirs + + @property + def selected_skills(self) -> tuple[str, ...] | None: + """Return the frozen selected skill names.""" + return self._selected_skills @property def skills(self) -> dict[str, Skill]: - """Return a copy of the discovered skills map.""" - return dict(self._skills) + """Return deep detached public catalog values.""" + return { + name: Skill( + name=skill.name, + description=skill.description, + root=skill.root, + skill_file=skill.skill_file, + metadata=_thaw_metadata(skill.metadata), + ) + for name, skill in self._skills.items() + } def prompt_summary(self, *, include_read_hint: bool = True) -> str: """Return a compact skill list for the system prompt.""" @@ -137,10 +168,10 @@ def skill_run(self, args: SkillRunArgs | Json) -> ToolResult: result.metadata.update({"returncode": proc.returncode, "cmd": command}) return result - def _discover(self) -> dict[str, Skill]: + def _discover(self) -> dict[str, _CatalogSkill]: """Discover skill files from the configured directories.""" - found: dict[str, Skill] = {} - for skills_dir in self.skills_dirs: + found: dict[str, _CatalogSkill] = {} + for skills_dir in self._skills_dirs: if not skills_dir.exists(): continue files = [path for path in skills_dir.rglob("SKILL.md") if path.is_file()] @@ -151,24 +182,24 @@ def _discover(self) -> dict[str, Skill]: name = str(metadata.get("name") or default_name).strip() if not name: continue - skill = Skill( + skill = _CatalogSkill( name=name, description=str(metadata.get("description") or "").strip(), root=path.parent.resolve(), skill_file=path.resolve(), - metadata=metadata, + metadata=_freeze_metadata(metadata), ) if name in found: raise ValueError(f"duplicate skill name: {name} in {found[name].skill_file} and {skill.skill_file}") found[name] = skill return found - def _select_skills(self, discovered: dict[str, Skill]) -> dict[str, Skill]: - """Return discovered skills filtered by selected_skills.""" - if self.selected_skills is None: + def _select_skills(self, discovered: dict[str, _CatalogSkill]) -> dict[str, _CatalogSkill]: + """Return discovered skills filtered by the frozen selection.""" + if self._selected_skills is None: return discovered - selected: dict[str, Skill] = {} - for name in self.selected_skills: + selected: dict[str, _CatalogSkill] = {} + for name in self._selected_skills: if name in selected: raise ValueError(f"duplicate selected skill: {name}") if name not in discovered: @@ -177,7 +208,7 @@ def _select_skills(self, discovered: dict[str, Skill]) -> dict[str, Skill]: selected[name] = discovered[name] return selected - def _get(self, name: str) -> Skill: + def _get(self, name: str) -> _CatalogSkill: """Look up a skill by name.""" try: return self._skills[name] @@ -209,6 +240,30 @@ def _truncate(text: str, max_chars: int) -> ToolResult: return ToolResult(True, f"[truncated {len(text)} chars to {max_chars}]\n{text[:head]}\n...\n{text[-tail:]}", {"truncated": True, "chars": len(text)}) +def _freeze_metadata(value: Any) -> object: + """Convert JSON-like metadata to recursively immutable storage.""" + if isinstance(value, dict): + return MappingProxyType({str(key): _freeze_metadata(item) for key, item in value.items()}) + if isinstance(value, list): + return tuple(_freeze_metadata(item) for item in value) + return value + + +def _thaw_metadata(value: object) -> Json: + """Return a deep mutable JSON copy of frozen metadata.""" + def thaw(item: object) -> Any: + if isinstance(item, Mapping): + return {str(key): thaw(child) for key, child in item.items()} + if isinstance(item, tuple): + return [thaw(child) for child in item] + return item + + detached = thaw(value) + if not isinstance(detached, dict): + raise TypeError("skill metadata must be an object") + return detached + + def parse_frontmatter(text: str) -> tuple[Json, str]: """Parse simple YAML-like frontmatter.""" if not text.startswith("---\n"): From 3346c1f614a04eeb5ee98e9d3e5a74cb050eee31 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Wed, 19 Aug 2026 21:06:35 -0400 Subject: [PATCH 14/30] Plan Bash plugin migration --- .plans/41-bash-plugin.md | 339 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 339 insertions(+) create mode 100644 .plans/41-bash-plugin.md diff --git a/.plans/41-bash-plugin.md b/.plans/41-bash-plugin.md new file mode 100644 index 0000000..e9fb311 --- /dev/null +++ b/.plans/41-bash-plugin.md @@ -0,0 +1,339 @@ +# Bash plugin — plan v2 + +Replace the directly registered `BashTool` with an explicit `BashPlugin`. The plugin provides one bounded, non-interactive Bash command tool. It uses the canonical harness root and does not inherit into child harnesses. + +This is a clean pre-1.0 break. Do not keep compatibility exports, aliases, fallback registration, or two public Bash interfaces. + +## Resolved decisions + +1. **Bash is an explicit plugin.** A plain harness has no Bash tool. Callers enable it with `plugins=[BashPlugin()]`. +2. **Bash stays one-shot.** Every call starts a fresh `bash -c` process. Shell variables, functions, aliases, and `cd` do not persist between calls. +3. **The tool is non-interactive.** It has no stdin, PTY, background mode, job tools, or live output stream. +4. **The plugin uses the canonical root.** `BashPlugin` receives `PluginContext.root`; callers cannot configure a second root. A model-selected cwd must resolve under that root. +5. **Cancellation stops the command.** Cancelling the task that awaits `Harness.run()` signals the command's process group, completes bounded cleanup after process handoff despite repeated cancellation requests, and then propagates cancellation. Cancellation during process creation waits for handoff to settle so no process can be orphaned. +6. **Timeouts stop the process group.** The effective timeout covers shell execution after process creation. Process creation is outside this timeout. Timeout sends `SIGTERM`, waits one second, then sends `SIGKILL` if needed. Fixed cleanup periods can make wall-clock time exceed `timeout_seconds`. +7. **Background cleanup is best effort.** After the direct shell exits, the plugin terminates remaining processes in the captured process group before it returns. A process that creates a new session can escape this cleanup, and the operating system can reuse a process-group ID after the shell exits. +8. **Output is bounded while the command runs.** The implementation reads stdout and stderr concurrently in fixed-size chunks into separate bounded head-and-tail buffers. It does not use line reads or write full output to temporary spill files. +9. **The environment is minimal by default.** The process inherits a small host allowlist plus non-interactive defaults. Full environment inheritance is an explicit host choice. Model arguments cannot set environment variables. +10. **The host controls limits.** Plugin configuration owns default timeout, maximum timeout, and per-stream output bytes. The model may request a timeout but cannot exceed the plugin maximum. +11. **Bash is sequential.** A tool batch containing Bash runs in model order without overlapping sibling calls. +12. **Child access is explicit.** `BashPlugin` does not implement `for_child()`. A child that needs Bash must list its own `BashPlugin`, and approval-required Bash remains invalid in children under existing child rules. +13. **Approval is optional.** `BashPlugin(requires_approval=True)` uses the existing top-level approval pause and resume flow. Approval is not a shell-specific prompt or UI. +14. **Local Bash is not a sandbox.** Cwd containment and environment filtering do not restrict absolute paths, network access, host credentials stored in files, or other host authority. +15. **No executor interface yet.** Keep local process execution private until ThinHarness has a real second executor such as a container or remote sandbox. +16. **POSIX only.** The plugin runs `bash` from `PATH` and depends on POSIX process groups. It does not discover Windows Bash installations or fall back to another shell. + +## Public interface + +Add `thinharness/plugins/bash.py` and export `BashPlugin` from `thinharness.plugins` and `thinharness`: + +```python +from thinharness import BashPlugin, Harness, HarnessConfig + +harness = Harness( + HarnessConfig(root="."), + plugins=[BashPlugin()], +) +``` + +The constructor is keyword-only: + +```python +BashPlugin( + *, + default_timeout: float = 30, + max_timeout: float = 120, + max_output_bytes: int = 40_000, + inherit_env: bool = False, + env: Mapping[str, str] | None = None, + requires_approval: bool = False, +) +``` + +Validate configuration when the plugin is constructed: + +- `default_timeout` and `max_timeout` must be real `int` or `float` values, not booleans, and must be finite and greater than zero; +- `default_timeout` must not exceed `max_timeout`; +- `max_output_bytes` must be an `int`, not a boolean, and must be greater than zero; +- `inherit_env` and `requires_approval` must be real booleans; +- every environment name and value must be a string; +- environment names must be non-empty and contain neither `=` nor NUL, and values must not contain NUL; +- copy `env` so later caller mutation cannot change a binding. + +The plugin name is fixed to `"bash"`. Its configuration is frozen after construction, matching the other first-party plugins. `bind()` performs no filesystem or process I/O and returns one static `ToolSpec` with `ToolOrigin(plugin="bash", source="bash")` after normal plugin composition. The contribution adds no plugin instructions; the tool description contains the model-facing execution rules. + +Remove `BashTool` and `BashArgs` from `thinharness.tools` and top-level exports. The replacement argument model is private to `thinharness.plugins.bash`; `BashPlugin` is the only public Bash interface. Remove the public direct-registration form. Keep all process helpers inside `thinharness/plugins/bash.py`; the existing synchronous process users do not justify a shared executor seam. + +## Model-facing tool + +The tool name is `bash`. Use a private Pydantic argument model based on `StrictArgs`. Here, strict means both `extra="forbid"` and no value coercion: `command` and `cwd` accept only strings, while `timeout` accepts only real integers or floats, rejects booleans and non-finite values, and must be greater than zero. + +```python +class _BashArgs(StrictArgs): + command: str = Field(min_length=1, strict=True) + cwd: str = Field(default=".", strict=True) + timeout: float | None = None +``` + +Use field validators for the numeric rules and for `command`. Reject a command that contains only whitespace during Pydantic validation so the model receives the existing retryable validation result. Do not strip or otherwise change a valid command. + +The tool description must state: + +- it runs one non-interactive Bash command; +- cwd must be inside the workspace; +- calls do not share shell state; +- same-process-group background processes are terminated on a best-effort basis; +- `timeout` is in seconds and is capped by the host. + +Do not expose output limits, environment variables, stdin, shell selection, login mode, background mode, sandbox permissions, or approval in the model schema. + +If the model omits `timeout`, use `default_timeout`. If it requests more than `max_timeout`, clamp the effective timeout to `max_timeout` and report the effective value in result metadata. + +## Process execution + +Use a native async handler and `asyncio.create_subprocess_exec()` rather than a synchronous handler in `asyncio.to_thread()`. The runner owns the stdout and stderr pipe read ends and their asyncio read transports instead of relying on `Process.stdout` and `Process.stderr`. Pass the corresponding write descriptors to `create_subprocess_exec()`. This ownership lets cleanup close the parent read transports when an escaped descendant keeps a write end open. + +Execution rules: + +- reject unsupported platforms before creating pipes or spawning, through a private capability check that requires `os.name == "posix"` and `os.killpg`; return `UnsupportedPlatform` otherwise; +- resolve cwd under `PluginContext.root` and require an existing directory; +- call `bash -c` with the resolved cwd, filtered environment, `stdin=DEVNULL`, owned stdout and stderr pipe write descriptors, and `start_new_session=True`; +- return a structured `ProcessStartError` if Bash or the process cannot start; +- close every pipe descriptor if pipe setup or process creation fails; +- make process creation a cancellation-safe handoff: keep the spawn task referenced, and if cancellation arrives during `create_subprocess_exec()`, tolerate repeated cancellation while waiting for the spawn task to settle; if it returns a process, use `process.pid` as the process-group ID created by `start_new_session=True`, signal it, complete bounded cleanup, and then propagate cancellation; never leave a spawn or cleanup task detached; +- after handoff, capture `process.pid` as the process-group ID and start one reader task for stdout and one for stderr before waiting for process completion; +- readers use fixed-size `read(n)` chunks, never line-oriented reads; +- the effective timeout starts after process handoff and covers waiting for the direct shell; +- on normal shell exit, attempt to terminate remaining members of the captured process group, then run bounded final drain; +- on timeout, terminate the captured process group, then run bounded final drain and return a timeout result; +- on `asyncio.CancelledError`, synchronously send the first group signal, then run one referenced cleanup task through a re-cancellation-tolerant await loop with a fixed deadline; after cleanup, re-raise cancellation; +- group termination sends `SIGTERM`, waits one second, then sends `SIGKILL` if the group still exists; +- final drain waits up to one second for both readers, then closes both parent read transports, cancels and joins both readers, and returns the partial output; +- close and join every owned transport, descriptor, reader task, spawn task, and cleanup task on every exit path; +- do not expose spawn, termination, or drain cleanup periods as public configuration in this slice. + +Process-group cleanup is best effort. It cannot stop a descendant that deliberately creates a new session or otherwise leaves the process group. Signalling a captured group after the direct shell has been reaped also has an unavoidable process-group-ID reuse race. State both limits without calling the tool isolated or promising that all descendants are stopped. + +## Environment policy + +When `inherit_env=False`, copy only these values when present: + +- `PATH`; +- `HOME`; +- `TMPDIR`, `TMP`, and `TEMP`; +- `LANG`, `LC_ALL`, and `LC_CTYPE`; +- `TZ`. + +When `inherit_env=True`, start from a copy of `os.environ`. + +For both modes: + +1. remove inherited `BASH_ENV` and `ENV`; +2. set `NO_COLOR=1`, `TERM=dumb`, `PAGER=cat`, and `GIT_PAGER=cat`; +3. apply the explicit host `env` mapping last, so the host can deliberately replace any value. + +This policy reduces accidental environment-secret exposure. It is not a security boundary because commands can still read host files and use other credential sources. + +## Output and results + +Keep stdout and stderr separate. Each stream gets its own `max_output_bytes` budget. + +The private buffer must: + +- retain `ceil(max_output_bytes / 2)` bytes from the start and `floor(max_output_bytes / 2)` bytes from the end; +- when `max_output_bytes == 1`, retain the first byte and no tail; +- discard the middle after the cap is reached; +- keep retained stream bytes at or below `max_output_bytes`, not total command output; +- continue draining discarded bytes so the child cannot block on a full pipe; +- decode UTF-8 with replacement after collection; +- for truncated output, insert `\n... {omitted_bytes} bytes omitted ...\n` between the decoded head and tail; the rendered marker does not count against the retained-byte budget; +- return whether truncation occurred and the total bytes seen. + +Return model content with labelled `stdout` and `stderr` sections. Preserve partial output for non-zero exits and timeouts. If both streams are empty, return only `(no output)`. If one stream is empty, keep both labelled sections and put `(no output)` in the empty section. + +Result metadata contains: + +- `exit_code`; +- `signal` when the return code represents a signal; +- `timed_out`; +- `duration_seconds`, measured from the start of spawn handoff through final cleanup; +- resolved `cwd`; +- effective `timeout_seconds`, which covers direct-shell execution after spawn handoff and does not include fixed cleanup periods; +- `stdout_bytes` and `stderr_bytes`; +- `stdout_truncated` and `stderr_truncated`. + +Result rules: + +- exit code zero returns `ok=True`; +- a non-zero exit returns `ok=False` with `error_type="NonZeroExit"`; +- timeout returns `ok=False` with `error_type="Timeout"`; +- cwd, platform, and startup failures return their specific error type; +- command outcomes preserve their output and do not request a model retry; +- argument validation keeps the existing retryable validation behavior; +- run cancellation propagates and does not become a tool result. + +## Behavior contract changes before implementation + +After plan review and before implementation, add a Bash Plugin section to `docs/behavior.md` that records: + +- explicit plugin composition and no implicit Bash; +- the fixed plugin and tool names; +- canonical root and contained cwd; +- fresh non-interactive calls with no supported background persistence and the documented best-effort cleanup limits; +- timeout, run cancellation, best-effort process-group cleanup, bounded final drain, and cancellation propagation; +- bounded separate head-and-tail output with the fixed split and truncation marker and no spill files; +- minimal versus inherited environment policy; +- sequential execution; +- optional top-level approval; +- no automatic child inheritance; +- POSIX-only support and no shell fallback; +- the fact that local Bash is not a sandbox. + +Update only the affected plugin and Bash behavior. Do not change unrelated sections. + +## Implementation steps + +1. Update the affected behavior contract after plan review. +2. Add the frozen `BashPlugin` configuration and static binding. +3. Replace the synchronous subprocess implementation with the private async process runner, owned pipes, and cancellation-safe spawn handoff. +4. Add bounded per-stream head-and-tail collection and bounded final drain. +5. Add process-group timeout, repeated-cancellation-safe cleanup, normal-exit descendant cleanup, and environment policy. +6. Move Bash registration to the plugin and remove the public `BashTool` interface and exports. +7. Replace `tests/unit/test_bash_tool.py` with plugin, process, output, environment, approval, and inheritance tests. +8. Add architecture guards that keep Bash construction out of core and prevent `BashPlugin` from implementing child inheritance. +9. Update README, reference docs, generated site content, examples, and changelog. +10. Add a live Bash plugin journey and run all validation. + +## Tests + +Add focused coverage for: + +### Plugin composition + +- no plugin means no Bash tool; +- `BashPlugin()` contributes one static sequential tool with fixed name and origin; +- binding uses the canonical root and does not create it; +- invalid or mutable constructor values cannot change later bindings; +- string and integer Boolean options, Boolean numeric limits, fractional output limits, and invalid environment entries fail at construction; +- normal duplicate plugin and tool collisions fail; +- direct `BashTool` and `BashArgs` imports and exports are gone; +- the plugin does not inherit into default or named children; +- an explicit child `BashPlugin` works when approval is off; +- approval-required Bash works at the top level, requires a resumable top-level model at harness construction, and fails under existing child approval rules. + +### Process lifecycle + +- success, non-zero exit, signal exit, timeout, and startup failure; +- timeout escalates from TERM to KILL when the command ignores TERM; +- cancelling an active harness run kills the process group and propagates cancellation promptly; +- cancelling a second time during cleanup does not detach cleanup or leave the process group running; +- cancellation paused during process creation still completes spawn handoff, group cleanup, and cancellation propagation; +- normal shell exit cleans up a same-group background descendant; +- a new-session descendant that holds an output pipe cannot keep normal exit, timeout, or cancellation from returning after bounded drain; +- stdout and stderr are drained concurrently with chunk reads, including simultaneous floods larger than 64 KiB with no newlines; +- unsupported platforms fail before pipe creation or spawn through the private capability-check seam; +- missing and non-directory cwd values fail cleanly; +- cwd cannot escape root through absolute paths, `..`, or symlinks. + +### Output + +- small stdout and stderr remain separate and unchanged; +- large streams use the defined head/tail split and marker, report total bytes, and keep the marker outside the retained-byte budget; +- one-byte and odd output limits follow the defined split; +- output floods stay within the configured retained-memory bound and create no spill file; +- split multibyte and invalid UTF-8 decode with replacement; +- timeout and non-zero results keep partial output; +- empty output returns `(no output)`. + +### Environment + +- minimal mode keeps only the named host values plus non-interactive defaults; +- full inheritance is explicit; +- explicit host values override inherited and default values; +- provider API keys are absent in minimal mode; +- inherited `BASH_ENV` and `ENV` are not sourced unless the host explicitly supplies them; +- model arguments cannot provide environment variables; +- string and Boolean model values do not coerce into `timeout`, and whitespace-only commands produce retryable validation results. + +### Limits and integration + +- default timeout, per-call timeout, and maximum clamping; +- invalid plugin limits fail during construction; +- Bash keeps mixed tool batches sequential; +- hooks, tracing, tool-call records, and structured result envelopes use the normal plugin tool path; +- the end-to-end journey uses `BashPlugin` to run a bounded command and verifies that the parent receives its output. + +## Documentation and caller migration + +Update: + +- `README.md` opinion, feature list, and usage example; +- `docs/docs.md` plugin and Bash sections; +- the hand-written `docs/site/index.html` Bash text; +- README-derived `docs/site/about/index.html` through `scripts/build_site.py`; +- `scripts/build_site.py` opinion-tag mapping if the `No bash by default` heading changes; +- `tests/e2e/README.md` with the new journey; +- `CHANGELOG.md` with the breaking removal of public `BashTool` and `BashArgs` and the new environment, cancellation, and output behavior; +- top-level and plugin exports; +- every tracked test, example, and document that uses `BashTool(...).spec()`. + +`READMEV2.md` is an untracked working draft, not a product document for this slice. Do not modify or add it as part of the implementation. `docs/site/explainer/index.html` has no Bash-specific text and is also out of scope unless the implementation changes a statement in that file. + +Use this replacement: + +```python +# Remove +tools=[BashTool(root=".").spec()] + +# Add +plugins=[BashPlugin()] +``` + +Do not add a compatibility wrapper or deprecation period. + +## Validation + +Run: + +```bash +uv run pytest tests/unit/test_bash_plugin.py tests/unit/test_plugins.py tests/unit/test_harness.py tests/unit/test_subagents.py tests/unit/test_approvals.py tests/unit/test_parallel_tools.py tests/unit/test_architecture.py +uv run pytest tests/unit/test_streaming.py tests/unit/test_tracing.py tests/unit/test_tool_retry.py +uv run pytest +uv run ruff check . +uv run pyright +uv run scripts/build_site.py +uv run scripts/build_site.py --check +uv run --env-file .env python tests/e2e/bash_plugin_journey.py +git diff --check +``` + +A credential-based end-to-end skip is not a pass. Report it separately. + +## Success criteria + +- Bash is available only through explicit `BashPlugin` composition. +- Core does not import or construct Bash behavior. +- The plugin uses the canonical root and does not create it during construction or binding. +- Timeout and run cancellation perform bounded best-effort process-group cleanup; repeated cancellation does not detach cleanup. +- Normal exit, timeout, and cancellation return after bounded final drain even when an escaped descendant holds a pipe open. +- Output memory stays bounded while stdout and stderr are drained with chunk reads. +- Minimal environment mode does not expose unrelated host environment variables. +- Bash remains one-shot, non-interactive, sequential, and unavailable to children unless configured there explicitly. +- No background-job interface, persistent shell, PTY, output stream, spill file, shell fallback, command policy, sandbox, or public executor interface is added. +- Focused tests, the full suite, Ruff, Pyright, generated-site checks, and the live journey pass. + +## Out of scope + +- Background command management. +- Persistent shell state or cwd. +- PTYs, interactive programs, or stdin. +- Live tool-output streaming. +- Windows shell support or shell discovery. +- Command allowlists or denylists. +- Filesystem, network, syscall, or credential sandboxing. +- Remote, container, VM, or provider-native execution. +- Model-selected environment, shell, output limits, or sandbox permissions. +- Full-output spill files. +- Automatic child inheritance. +- A public process or executor interface. +- Compatibility aliases or migrations. From e990aed417a25f8758a1182a316ef8a206946a3c Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Wed, 19 Aug 2026 22:11:22 -0400 Subject: [PATCH 15/30] Add explicit Bash plugin --- CHANGELOG.md | 2 + README.md | 17 +- docs/behavior.md | 19 + docs/docs.md | 42 +- docs/site/about/index.html | 4 +- docs/site/index.html | 2 +- tests/e2e/README.md | 1 + tests/e2e/bash_plugin_journey.py | 73 ++ tests/unit/test_architecture.py | 20 + tests/unit/test_bash_plugin.py | 1063 ++++++++++++++++++++++++++++++ tests/unit/test_bash_tool.py | 172 ----- thinharness/__init__.py | 6 +- thinharness/plugins/__init__.py | 2 + thinharness/plugins/bash.py | 610 +++++++++++++++++ thinharness/tools/__init__.py | 3 - thinharness/tools/bash.py | 172 ----- 16 files changed, 1834 insertions(+), 374 deletions(-) create mode 100644 tests/e2e/bash_plugin_journey.py create mode 100644 tests/unit/test_bash_plugin.py delete mode 100644 tests/unit/test_bash_tool.py create mode 100644 thinharness/plugins/bash.py delete mode 100644 thinharness/tools/bash.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 80de3db..9263e73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,14 @@ - Added explicit plugin composition with static and connected contributions, atomic connection rollback, unique plugin names, generic tool origin, and plugin-provided hooks and instructions. - Added `FilesystemPlugin` for the ordered workspace tool surface; `jsonl_search` remains opt-in through this plugin. +- Added `BashPlugin` for explicit one-shot local Bash commands with strict arguments, contained cwd, minimal environment inheritance, bounded separate head-and-tail output, host-capped timeouts, process-group cleanup, and cancellation propagation. - Added `MCPPlugin` for lazy MCP server connection, binding-local server identity, atomic tool discovery, and generic tool origin attribution. - Added `SkillsPlugin` for constructor-time skill discovery, explicit ordered skill-tool selection, static summaries, and shared inherited-child catalogs. - Added `ParallelLlmPlugin` for explicit text-only batch composition with borrowed harness or caller models and plugin-owned string-model provider settings. - Added explicit `SubagentsPlugin` composition for the default child, named child recipes, child-local hooks, additive inheritance, and delegation result shaping. - Added a narrow `ChildHarnessHost` to `PluginContext`, public child request/outcome contracts, and structural `ChildInheritablePlugin.for_child()` rebinding. - Added run-frozen authoritative tool composition provenance for direct-tool inheritance and `subagent.delegation` tracing. +- **Breaking:** Removed the public `BashTool` and `BashArgs` interfaces. Configure `plugins=[BashPlugin()]`; Bash no longer accepts model-selected output limits or direct tool registration. - **Breaking:** Removed `HarnessConfig.mcp_servers`, `McpToolInfo`, and the MCP `ToolKind`; configure one `MCPPlugin` with all harness servers. - **Breaking:** Generic plugin validation now reports MCP tool collisions as duplicate tool names. Use MCP `tool_prefix`, `include_tools`, or `exclude_tools` to prevent collisions. - **Breaking:** Removed `MCPServer.resolve_id()` and post-bind mutation of `server.id`. The public `server.id` remains the base ID; binding-local IDs, including duplicate suffixes, appear in tool origin and result metadata. diff --git a/README.md b/README.md index 073a2bf..1cafad2 100644 --- a/README.md +++ b/README.md @@ -215,7 +215,7 @@ ThinHarness has opinions. They are the reason it stays small. **Purpose-built agents, not universal agents.** ThinHarness is for bounded agent loops, not open-ended interactive assistants like Claude Code or OpenClaw. For business use cases, focused agent loops orchestrated by deterministic code are usually a better fit than sprawling multi-agent systems with broad authority. -**No bash by default.** Purpose-built business agents usually don't need a shell. Bash is a broad security and reliability surface: it gives the model open-ended authority instead of typed, bounded actions. ThinHarness has no implicit tools and exposes Bash only through an opt-in `BashTool` for exploratory runs before the workflow is hardened with typed tools. +**No bash by default.** Purpose-built business agents usually don't need a shell. Bash is a broad security and reliability surface: it gives the model open-ended authority instead of typed, bounded actions. ThinHarness has no implicit tools. Add `BashPlugin()` explicitly for bounded exploratory commands, then harden repeated workflow actions as typed tools. **Search is a top priority.** The `search` tool exposes ripgrep as compact grouped path/line results, tuned for document and business-workflow agents rather than code navigation. There's also a `jsonl_search` variant, because JSONL is the right shape when you're replacing RAG with agent-driven search over structured data: ripgrep row prefiltering, jq-style field projection, `where` filters, range filters, and snippets from large multiline fields. @@ -269,6 +269,19 @@ harness = Harness( MCP tools connect and discover one tool snapshot lazily on `Harness.connect()` or the first run. Install support with `uv add 'thinharness[mcp]'`. +Local Bash is also an explicit plugin: + +```python +from thinharness import BashPlugin + +harness = Harness( + HarnessConfig(root="."), + plugins=[BashPlugin()], +) +``` + +Each call runs a fresh non-interactive shell from a workspace-contained cwd. Bash is sequential, has bounded stdout and stderr, uses a filtered environment by default, and performs best-effort process-group cleanup after normal shell exit, timeout, or run cancellation. It is not a sandbox. + Delegation is also an explicit plugin: ```python @@ -326,7 +339,7 @@ Streaming emits coarse run, model, tool, retry, limit, and subagent events, then - **Filesystem plugin:** explicit `FilesystemPlugin` composition for `read`, `write`, batched exact-replacement `edit`, `search`, `list`, and `glob` with root-scoped path policies. - **JSONL search:** opt-in `jsonl_search` for structured line-delimited data, with ripgrep prefiltering, field projection, equality/contains/regex/range `where` filters, and field-level snippets from large multiline string values. -- **Bash prototype tool:** opt-in `BashTool` for exploratory shell commands. It is lightweight and available only through direct custom registration. +- **Bash plugin:** explicit `BashPlugin` composition for one-shot non-interactive commands with contained cwd, filtered environment, bounded output, timeouts, cancellation cleanup, and optional approval. - **Provider adapters:** built-in OpenAI, Anthropic, and OpenRouter adapters, plus public model/session protocols for implementing another provider. - **Custom typed tools:** define sync or async `ToolSpec` handlers with Pydantic argument models, normalized `ToolResult` envelopes, sequential/approval flags, and per-tool retry settings. - **Structured output:** Pydantic-validated results with native, tool, prompted, and text modes. diff --git a/docs/behavior.md b/docs/behavior.md index 5b32a5d..2cc195c 100644 --- a/docs/behavior.md +++ b/docs/behavior.md @@ -102,6 +102,25 @@ Callers compose optional harness behavior explicitly while independent custom to - PLUGIN-12: Core records authoritative direct, plugin, and delegation composition roles independently of caller-visible `ToolOrigin`; these records control inheritance and delegation tracing and cannot be forged through tool metadata. - PLUGIN-13: Automatic child inheritance is explicit and structural. Only a plugin with synchronous `for_child()` is rebound against the child context; its returned plugin must be valid and keep the expected fixed name. `for_child()` and I/O-free `bind()` can run during parent construction and later child-recipe revalidation, so both operations must be repeatable and side-effect-free. +## Bash Plugin + +### Purpose + +Callers explicitly add one bounded, non-interactive local Bash tool through plugin composition. + +### Requirements + +- BASH-PLUGIN-1: A plain harness has no Bash tool. `BashPlugin` has the fixed name `"bash"` and contributes one sequential tool named `bash`; normal plugin and tool collision rules apply. +- BASH-PLUGIN-2: Bash uses the canonical harness root. A model-selected working directory must resolve to an existing directory inside that root. +- BASH-PLUGIN-3: Each call starts a fresh non-interactive `bash -c` process with no stdin or shared shell state. Persistent background work is not supported. After the direct shell exits, times out, or is cancelled, the plugin performs bounded, best-effort process-group cleanup and final output drain. A descendant that leaves the process group can escape termination, and signalling after shell exit has an unavoidable process-group-ID reuse race. +- BASH-PLUGIN-4: The host controls default and maximum command timeouts. Run cancellation signals the process group, completes bounded cleanup after process handoff despite repeated cancellation, and then propagates cancellation. Cancellation during process creation waits for handoff and cleans up any created process before it propagates. +- BASH-PLUGIN-5: Stdout and stderr are drained concurrently into separate bounded head-and-tail buffers without spill files. Each buffer retains `ceil(limit / 2)` bytes from the start and `floor(limit / 2)` bytes from the end, and inserts `\n... {omitted_bytes} bytes omitted ...\n` between them when truncated. +- BASH-PLUGIN-6: Minimal environment mode inherits only `PATH`, `HOME`, temporary-directory, locale, and timezone values. Full host-environment inheritance is explicit. Both modes remove inherited `BASH_ENV` and `ENV`, set non-interactive defaults, and then apply host-configured environment values. +- BASH-PLUGIN-7: Bash calls execute sequentially. Optional approval uses the existing top-level approval flow. +- BASH-PLUGIN-8: `BashPlugin` does not inherit automatically into children. A child that needs Bash must list its own plugin, subject to existing child approval rules. +- BASH-PLUGIN-9: Bash support is POSIX-only, requires process groups, runs `bash` from `PATH`, and has no shell fallback. +- BASH-PLUGIN-10: Local Bash is not a sandbox. Working-directory containment and environment filtering do not restrict absolute paths, network access, host files, or other host authority. + ## Subagents Plugin ### Purpose diff --git a/docs/docs.md b/docs/docs.md index 0d367b2..f54a5d2 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -98,7 +98,7 @@ Important groups: - `root` defines the run root. `FilesystemPlugin` owns filesystem paths, limits, search settings, and output location. - `model`, `api_key`, `base_url`, `temperature`, `max_tokens`, `effort`, `extra_body`, `request_timeout`, `request_retries`, and `request_retry_backoff` define provider settings. -- The `Harness` constructor's ordered `plugins=` and direct `tools=` inputs define the complete model-callable surface. ThinHarness has no implicit or selected built-in tool path. Filesystem, MCP, skills, parallel LLM, and subagent delegation use explicit plugins. +- The `Harness` constructor's ordered `plugins=` and direct `tools=` inputs define the complete model-callable surface. ThinHarness has no implicit or selected built-in tool path. Filesystem, Bash, MCP, skills, parallel LLM, and subagent delegation use explicit plugins. - `max_model_requests`, `max_tool_calls`, `output_retries`, and `tool_retries` bound the run. - `output_type` and `output_mode` define structured output. - `tracing`, `local_tracing`, and `local_trace_dir` define observability. @@ -209,6 +209,28 @@ harness = Harness( With this configuration, `read` can access `src/app.py` and `tests/test_app.py`, but not `docs/notes.md`. `write` can create or update `outputs/report.md`, but not `src/generated.py`. Omit `read_paths` or `write_paths` to allow that operation anywhere under `root`. +## Bash Plugin + +`BashPlugin` explicitly adds one sequential, non-interactive `bash` tool. A plain harness has no Bash tool. The plugin uses `HarnessConfig.root`; the model can select only an existing cwd contained by that canonical root. + +```python +from thinharness import BashPlugin, Harness, HarnessConfig + + +harness = Harness( + HarnessConfig(root="."), + plugins=[BashPlugin()], +) +``` + +Each call starts a fresh `bash -c` process with no stdin, PTY, shared shell state, or persistent background-job interface. The host configures default and maximum timeouts and a byte limit for each output stream. The model can request only a timeout, which is capped by the host. Stdout and stderr are drained concurrently into separate bounded head-and-tail buffers; omitted middle bytes get a visible marker and are not written to spill files. + +By default, commands inherit only `PATH`, `HOME`, temporary-directory, locale, and timezone values, plus fixed non-interactive defaults. Set `inherit_env=True` to copy the full host environment. In either mode, inherited `BASH_ENV` and `ENV` are removed before explicit host `env` values are applied. + +Timeout and run cancellation signal the command process group, allow one second for termination, and escalate to `SIGKILL`. Cleanup and final pipe drain are bounded, and cancellation then propagates. Normal shell exit also performs best-effort same-group descendant cleanup. A descendant that starts a new session can escape termination, and signalling after direct shell exit has an unavoidable process-group-ID reuse race. Bash is POSIX-only, runs `bash` from `PATH`, does not fall back to another shell, and does not inherit automatically into child harnesses. Configure `requires_approval=True` only for a top-level harness. + +Cwd containment and environment filtering reduce mistakes; they are not a sandbox. Commands can use absolute paths, network access, host files, and other authority available to the local process. Use Bash to explore workflow shape, then promote repeated actions into typed tools. + ## Custom Tools Custom tools are registered as `ToolSpec` objects. A handler may return a `ToolResult`, a string, or JSON-serializable data. The model always receives a JSON envelope with `ok`, `content`, and `metadata`. @@ -267,23 +289,7 @@ The paused result includes: Resume with `resume_approvals(...)`, `stream_approvals(...)`, or `resume_approvals_sync(...)` and one `ApprovalDecision` per pending approval. Approved calls execute through the normal tool machinery, including hooks, tracing, retry accounting, and stream events. Rejected calls do not execute or fire tool hooks; the model receives a failed tool result with `error_type="ApprovalRejected"` and can explain, recover, or request another tool. -Approval-required tools need a resumable model because the harness must continue after the paused assistant tool-call turn. They are not supported inside child harnesses. Configure approval only on direct top-level `ToolSpec` values. - -### Bash Prototype Tool - -`BashTool` is an opt-in custom tool for exploratory agent runs. ThinHarness has no implicit tools; add `BashTool(...).spec()` through direct `tools=` composition. - -```python -from thinharness import BashTool, Harness, HarnessConfig - - -harness = Harness( - HarnessConfig(root="."), - tools=[BashTool(root=".").spec()], -) -``` - -The tool runs one `bash -c` command from a workspace-contained cwd and marks itself sequential because commands may mutate state. The cwd check is not a sandbox: commands can still access absolute paths, network tools, environment variables, and anything the host process can access. The tool has a configured `max_tool_chars` output cap; the model can pass `max_chars` on an individual call only to request a lower cap. The final limit is `min(max_chars, max_tool_chars)`, applied independently to stdout and stderr. Background descendants left by a command are cleaned up when the shell exits; this is not a persistent job runner. Use it to prototype workflow shape, then promote repeated shell logic into typed tools. +Approval-required tools need a resumable model because the harness must continue after the paused assistant tool-call turn. They are not supported inside child harnesses. Configure approval only at the top level, either on a direct `ToolSpec` or with `BashPlugin(requires_approval=True)`. ## Tool Execution Policy diff --git a/docs/site/about/index.html b/docs/site/about/index.html index bcab630..c8a2c08 100644 --- a/docs/site/about/index.html +++ b/docs/site/about/index.html @@ -143,7 +143,7 @@

    Opinions

    ThinHarness has opinions. They are the reason it stays small.

    purpose_built

    Purpose-built agents, not universal agents

    ThinHarness is for bounded agent loops, not open-ended interactive assistants like Claude Code or OpenClaw. For business use cases, focused agent loops orchestrated by deterministic code are usually a better fit than sprawling multi-agent systems with broad authority.

    -
    no_bash

    No bash by default

    Purpose-built business agents usually don't need a shell. Bash is a broad security and reliability surface: it gives the model open-ended authority instead of typed, bounded actions. ThinHarness has no implicit tools and exposes Bash only through an opt-in BashTool for exploratory runs before the workflow is hardened with typed tools.

    +
    no_bash

    No bash by default

    Purpose-built business agents usually don't need a shell. Bash is a broad security and reliability surface: it gives the model open-ended authority instead of typed, bounded actions. ThinHarness has no implicit tools. Add BashPlugin() explicitly for bounded exploratory commands, then harden repeated workflow actions as typed tools.

    search

    Search is a top priority

    The search tool exposes ripgrep as compact grouped path/line results, tuned for document and business-workflow agents rather than code navigation. There's also a jsonl_search variant, because JSONL is the right shape when you're replacing RAG with agent-driven search over structured data: ripgrep row prefiltering, jq-style field projection, where filters, range filters, and snippets from large multiline fields.

    parallel_llm_calls_explicitly_composed

    Parallel LLM calls, explicitly composed

    Fan out from inside the harness when a workflow needs efficient parallel processing or majority vote for reliability. Add ParallelLlmPlugin() for a plain-text batch tool that borrows the harness model, or give the plugin a model string and its own provider settings. For validated structured output per call, instantiate ParallelLlmTool with output_type (a Pydantic model). Each call is stateless, and large batches can write JSON to output_file.

    no_token_streaming

    No token streaming

    Streaming is for workflow progress, not live chatbot text. ThinHarness emits run, model-turn, tool, retry, limit, and subagent events, but it does not stream provider token deltas. Token streaming would add provider-specific plumbing, event merging, cancellation edge cases, and more surface area to keep stable. For workflow-style agents, step-level updates are usually the useful signal.

    @@ -184,7 +184,7 @@

    Features

    Filesystem plugin

    Explicit FilesystemPlugin composition for read, write, batched exact-replacement edit, search, list, and glob with root-scoped path policies.

    JSONL search

    Opt-in jsonl_search for structured line-delimited data, with ripgrep prefiltering, field projection, equality/contains/regex/range where filters, and field-level snippets from large multiline string values.

    -
    Bash prototype tool

    Opt-in BashTool for exploratory shell commands. It is lightweight and available only through direct custom registration.

    +
    Bash plugin

    Explicit BashPlugin composition for one-shot non-interactive commands with contained cwd, filtered environment, bounded output, timeouts, cancellation cleanup, and optional approval.

    Provider adapters

    Built-in OpenAI, Anthropic, and OpenRouter adapters, plus public model/session protocols for implementing another provider.

    Custom typed tools

    Define sync or async ToolSpec handlers with Pydantic argument models, normalized ToolResult envelopes, sequential/approval flags, and per-tool retry settings.

    Structured output

    Pydantic-validated results with native, tool, prompted, and text modes.

    diff --git a/docs/site/index.html b/docs/site/index.html index 236ec82..5ab76f8 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -45,7 +45,7 @@

    A minimal, opinionated agent harness.// 01

    Opinions

    the reason it stays small
    purpose_built

    Purpose-built agents

    ThinHarness is for bounded agent loops inside software you control, not open-ended interactive assistants.

    -
    no_bash

    No bash by default

    Bash stays out of the default tools, with an opt-in BashTool only for prototyping before typed tools.

    +
    no_bash

    No bash by default

    Bash stays out of the default tools. Add BashPlugin explicitly for bounded exploratory commands before typed tools.

    search

    Search is a top priority

    Ripgrep exposed as compact grouped results, tuned for documents and business workflows — plus a custom JSONL search tool for structured corpuses.

    parallel_llm

    Parallel LLM calls, built in

    Add ParallelLlmPlugin to fan out independent prompts with the harness model, or configure a separate batch model.

    no_compaction

    No compaction

    Compaction makes sense for sprawling coding sessions. For business agents the fix is smarter task decomposition and context management

    diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 451c364..de24390 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -13,6 +13,7 @@ Credential-based scripts skip when `CI` is set or when the required provider key Current journeys: - `workspace_tools_journey.py`: filesystem tools plus `jsonl_search`. +- `bash_plugin_journey.py`: explicit Bash composition, bounded output, and parent receipt of the tool result. - `skills_journey.py`: skill discovery, `skill_read`, and `skill_run`. - `control_plane_journey.py`: hooks, sequential execution, and retry-limit behavior. - `structured_output_journey.py`: Pydantic structured output after tool use. diff --git a/tests/e2e/bash_plugin_journey.py b/tests/e2e/bash_plugin_journey.py new file mode 100644 index 0000000..f9a7527 --- /dev/null +++ b/tests/e2e/bash_plugin_journey.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import os +import sys +from pathlib import Path +from tempfile import TemporaryDirectory + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from thinharness import BashPlugin, Harness, HarnessConfig, Hook, ToolResult + +MODEL = os.getenv("E2E_BASH_MODEL", "openai:gpt-5.2") +PROMPT = """ +Use the bash tool exactly once. Run this exact command: + +printf BEGIN; printf '%0200d' 0; printf END + +Read the bounded tool output that you receive. Your final answer must be exactly: +BASH_PLUGIN_DONE BEGIN END +""".strip() + + +def main() -> None: + if _should_skip(MODEL): + return + + with TemporaryDirectory(prefix="thinharness-e2e-bash-") as raw_root: + root = Path(raw_root) + tool_results: list[ToolResult] = [] + + def capture(ctx) -> None: + if ctx.tool_name == "bash": + tool_results.append(ctx.envelope) + + harness = Harness( + HarnessConfig( + root=root, + model=MODEL, + max_model_requests=5, + max_tool_calls=2, + ), + plugins=[BashPlugin(max_output_bytes=64)], + hooks=[Hook("after_tool_call", capture, tools=["bash"])], + ) + + result = harness.run_sync(PROMPT) + + assert len(tool_results) == 1, f"expected one Bash call, saw {len(tool_results)}" + tool_result = tool_results[0] + assert tool_result.ok is True + assert tool_result.metadata["stdout_bytes"] == 208 + assert tool_result.metadata["stdout_truncated"] is True + assert tool_result.content.startswith("stdout:\nBEGIN") + assert "bytes omitted" in tool_result.content + assert "END\nstderr:\n(no output)" in tool_result.content + assert result.text.strip() == "BASH_PLUGIN_DONE BEGIN END" + print(f"PASS bash_plugin_journey model={MODEL}") + + +def _should_skip(model: str) -> bool: + if os.getenv("CI"): + print("SKIP bash_plugin_journey: CI is set") + return True + provider = model.split(":", 1)[0] + env_name = {"openai": "OPENAI_API_KEY", "anthropic": "ANTHROPIC_API_KEY", "openrouter": "OPENROUTER_API_KEY"}[provider] + if not os.getenv(env_name): + print(f"SKIP bash_plugin_journey: {env_name} is not set") + return True + return False + + +if __name__ == "__main__": + main() diff --git a/tests/unit/test_architecture.py b/tests/unit/test_architecture.py index 004fc50..bf03f7b 100644 --- a/tests/unit/test_architecture.py +++ b/tests/unit/test_architecture.py @@ -4,6 +4,26 @@ from pathlib import Path +def test_core_has_no_bash_imports_or_construction() -> None: + """Core stays independent from Bash process behavior.""" + root = Path(__file__).resolve().parents[2] + core_source = (root / "thinharness" / "core.py").read_text(encoding="utf-8") + tools_init = (root / "thinharness" / "tools" / "__init__.py").read_text(encoding="utf-8") + plugin_source = (root / "thinharness" / "plugins" / "bash.py").read_text(encoding="utf-8") + + assert "BashPlugin" not in core_source + assert "plugins.bash" not in core_source + assert "tools.bash" not in core_source + assert "create_subprocess" not in core_source + assert not (root / "thinharness" / "tools" / "bash.py").exists() + assert "BashTool" not in tools_init + assert "BashArgs" not in tools_init + assert "class BashPlugin" in plugin_source + assert "create_subprocess_exec" in plugin_source + assert "def for_child" not in plugin_source + assert "Executor" not in plugin_source + + def test_core_has_no_mcp_imports_or_lifecycle_state() -> None: """Core stays independent from MCP composition and lifecycle details.""" core_path = Path(__file__).resolve().parents[2] / "thinharness" / "core.py" diff --git a/tests/unit/test_bash_plugin.py b/tests/unit/test_bash_plugin.py new file mode 100644 index 0000000..7caec8f --- /dev/null +++ b/tests/unit/test_bash_plugin.py @@ -0,0 +1,1063 @@ +from __future__ import annotations + +import asyncio +import json +import os +import shlex +import signal +import sys +import time +from pathlib import Path +from typing import Any + +import pytest +from fakes import ContextFakeTracer, MultiCallClient, ScriptedModel, ScriptedSession, _fake_openai, slow_tool + +import thinharness +import thinharness.plugins.bash as bash_module +import thinharness.tools as tools_module +from thinharness import ( + ApprovalDecision, + BashPlugin, + Harness, + HarnessConfig, + Hook, + ModelToolCall, + ModelTurn, + SubAgentConfig, + SubagentsPlugin, + ToolOrigin, + ToolResult, + TracingOptions, + call_tool, +) + + +def _call_turn(arguments: dict[str, Any], *, call_id: str = "call_1") -> ModelTurn: + return ModelTurn( + tool_calls=[ModelToolCall(id=call_id, name="bash", arguments=json.dumps(arguments))], + raw={"id": "start"}, + ) + + +async def _run_bash( + root: Path, + arguments: dict[str, Any], + *, + plugin: BashPlugin | None = None, + hooks: list[Hook] | None = None, + tracing: list[TracingOptions] | None = None, +) -> tuple[ToolResult, Any]: + captured: list[ToolResult] = [] + + def on_continue(outputs, _tools, _metadata) -> None: + captured.append(ToolResult.from_json(outputs[0].output)) + + session = ScriptedSession( + start_turn=_call_turn(arguments), + continue_turn=ModelTurn(text="done", raw={"id": "done"}), + on_continue=on_continue, + ) + harness = Harness( + HarnessConfig(root=root), + model=ScriptedModel([session]), + plugins=[plugin or BashPlugin()], + hooks=hooks, + tracing=tracing, + ) + run_result = await harness.run("go") + assert len(captured) == 1 + return captured[0], run_result + + +def _stdout(result: ToolResult) -> str: + if result.content == "(no output)": + return "" + return result.content.split("stdout:\n", 1)[1].split("\nstderr:\n", 1)[0] + + +def _stderr(result: ToolResult) -> str: + if result.content == "(no output)": + return "" + return result.content.split("\nstderr:\n", 1)[1] + + +def _group_exists(pgid: int) -> bool: + try: + os.killpg(pgid, 0) + return True + except ProcessLookupError: + return False + + +async def _wait_for_file(path: Path, timeout: float = 2) -> None: + async with asyncio.timeout(timeout): + while not path.exists(): + await asyncio.sleep(0.01) + + +def test_plugin_is_explicit_static_and_uses_fixed_identity(tmp_path: Path) -> None: + plain = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([])) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[BashPlugin()]) + + assert plain.tools == [] + assert [tool.name for tool in harness.tools] == ["bash"] + tool = harness.tools[0] + assert tool.sequential is True + assert tool.origin == ToolOrigin(plugin="bash", source="bash") + assert tool.requires_approval is False + schema = tool.response_tool()["parameters"] + assert set(schema["properties"]) == {"command", "cwd", "timeout"} + assert schema["additionalProperties"] is False + assert "non-interactive" in tool.description + assert "does not share shell state" in tool.description + assert "best-effort" in tool.description + assert "capped by the host" in tool.description + assert harness.system_instructions() == plain.system_instructions() + + +def test_binding_uses_canonical_root_without_creating_it(tmp_path: Path) -> None: + missing = tmp_path / "missing" + harness = Harness(HarnessConfig(root=missing), model=ScriptedModel([]), plugins=[BashPlugin()]) + + assert not missing.exists() + assert harness.tools[0].name == "bash" + + +def test_plugin_configuration_is_frozen_and_copies_environment() -> None: + environment = {"HOST_VALUE": "before"} + plugin = BashPlugin(env=environment) + environment["HOST_VALUE"] = "after" + + assert plugin.env == {"HOST_VALUE": "before"} + detached = plugin.env + detached["HOST_VALUE"] = "mutated" + assert plugin.env == {"HOST_VALUE": "before"} + with pytest.raises(AttributeError, match="frozen"): + plugin.max_output_bytes = 1 + with pytest.raises(AttributeError, match="fixed"): + plugin.name = "other" + with pytest.raises(AttributeError, match="fixed"): + BashPlugin.name = "other" + + +@pytest.mark.parametrize( + ("kwargs", "error"), + [ + ({"default_timeout": True}, TypeError), + ({"default_timeout": "1"}, TypeError), + ({"default_timeout": float("inf")}, ValueError), + ({"default_timeout": 10**400}, ValueError), + ({"default_timeout": 0}, ValueError), + ({"max_timeout": False}, TypeError), + ({"max_timeout": float("nan")}, ValueError), + ({"default_timeout": 2, "max_timeout": 1}, ValueError), + ({"max_output_bytes": True}, TypeError), + ({"max_output_bytes": 1.5}, TypeError), + ({"max_output_bytes": 0}, ValueError), + ({"inherit_env": 1}, TypeError), + ({"requires_approval": "yes"}, TypeError), + ({"env": []}, TypeError), + ({"env": {1: "value"}}, TypeError), + ({"env": {"NAME": 1}}, TypeError), + ({"env": {"": "value"}}, ValueError), + ({"env": {"A=B": "value"}}, ValueError), + ({"env": {"A\0B": "value"}}, ValueError), + ({"env": {"NAME": "a\0b"}}, ValueError), + ], +) +def test_plugin_constructor_rejects_invalid_values(kwargs: dict[str, Any], error: type[Exception]) -> None: + with pytest.raises(error): + BashPlugin(**kwargs) + + +def test_duplicate_plugin_and_tool_collisions_use_normal_validation(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="duplicate plugin name: bash"): + Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[BashPlugin(), BashPlugin()]) + with pytest.raises(ValueError, match="duplicate tool name: bash"): + Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[BashPlugin()], + tools=[slow_tool("bash", 0)], + ) + + +def test_old_public_bash_interfaces_are_removed() -> None: + assert not hasattr(thinharness, "BashTool") + assert not hasattr(thinharness, "BashArgs") + assert not hasattr(tools_module, "BashTool") + assert not hasattr(tools_module, "BashArgs") + + +async def test_success_keeps_streams_separate_and_reports_metadata(tmp_path: Path) -> None: + result, _ = await _run_bash(tmp_path, {"command": "printf out; printf err >&2"}) + + assert result.ok is True + assert _stdout(result) == "out" + assert _stderr(result) == "err" + assert result.metadata["exit_code"] == 0 + assert result.metadata["timed_out"] is False + assert result.metadata["cwd"] == str(tmp_path) + assert result.metadata["timeout_seconds"] == 30 + assert result.metadata["stdout_bytes"] == 3 + assert result.metadata["stderr_bytes"] == 3 + assert result.metadata["stdout_truncated"] is False + assert result.metadata["stderr_truncated"] is False + assert isinstance(result.metadata["duration_seconds"], float) + + +async def test_one_empty_stream_keeps_both_exact_labels(tmp_path: Path) -> None: + stdout_only, _ = await _run_bash(tmp_path, {"command": "printf out"}) + stderr_only, _ = await _run_bash(tmp_path, {"command": "printf err >&2"}) + + assert stdout_only.content == "stdout:\nout\nstderr:\n(no output)" + assert stderr_only.content == "stdout:\n(no output)\nstderr:\nerr" + + +async def test_nonzero_and_signal_exit_preserve_output(tmp_path: Path) -> None: + failed, _ = await _run_bash(tmp_path, {"command": "printf before; printf nope >&2; exit 7"}) + signalled, _ = await _run_bash(tmp_path, {"command": "printf signal; kill -TERM $$"}) + + assert failed.ok is False + assert failed.metadata["error_type"] == "NonZeroExit" + assert failed.metadata["exit_code"] == 7 + assert _stdout(failed) == "before" + assert _stderr(failed) == "nope" + assert signalled.ok is False + assert signalled.metadata["error_type"] == "NonZeroExit" + assert signalled.metadata["exit_code"] == -signal.SIGTERM + assert signalled.metadata["signal"] == signal.SIGTERM + assert _stdout(signalled) == "signal" + + +async def test_empty_output_uses_single_marker(tmp_path: Path) -> None: + result, _ = await _run_bash(tmp_path, {"command": ":"}) + + assert result.ok is True + assert result.content == "(no output)" + + +async def test_timeout_keeps_partial_output_and_escalates_for_ignored_term(tmp_path: Path) -> None: + started = time.monotonic() + result, _ = await _run_bash( + tmp_path, + {"command": "trap '' TERM; printf before; while :; do sleep 1; done", "timeout": 0.1}, + ) + elapsed = time.monotonic() - started + + assert result.ok is False + assert result.metadata["error_type"] == "Timeout" + assert result.metadata["timed_out"] is True + assert result.metadata["signal"] == signal.SIGKILL + assert _stdout(result) == "before" + assert 1 <= elapsed < 4 + + +async def test_default_timeout_per_call_timeout_and_host_clamp(tmp_path: Path) -> None: + plugin = BashPlugin(default_timeout=0.4, max_timeout=0.6) + default, _ = await _run_bash(tmp_path, {"command": "printf default"}, plugin=plugin) + requested, _ = await _run_bash(tmp_path, {"command": "printf requested", "timeout": 0.2}, plugin=plugin) + clamped, _ = await _run_bash(tmp_path, {"command": "printf clamped", "timeout": 10}, plugin=plugin) + + assert default.metadata["timeout_seconds"] == 0.4 + assert requested.metadata["timeout_seconds"] == 0.2 + assert clamped.metadata["timeout_seconds"] == 0.6 + + +async def test_pipe_setup_set_inheritable_failure_closes_both_descriptors(monkeypatch: pytest.MonkeyPatch) -> None: + descriptors: list[int] = [] + real_pipe = os.pipe + + def tracked_pipe() -> tuple[int, int]: + pair = real_pipe() + descriptors.extend(pair) + return pair + + def fail_inheritable(_fd: int, _inheritable: bool) -> None: + raise OSError("set inheritable sentinel") + + monkeypatch.setattr(bash_module.os, "pipe", tracked_pipe) + monkeypatch.setattr(bash_module.os, "set_inheritable", fail_inheritable) + with pytest.raises(OSError, match="set inheritable sentinel"): + await bash_module._open_owned_pipe() + + assert len(descriptors) == 2 + for fd in descriptors: + with pytest.raises(OSError): + os.fstat(fd) + + +async def test_pipe_setup_fdopen_failure_closes_both_descriptors(monkeypatch: pytest.MonkeyPatch) -> None: + descriptors: list[int] = [] + real_pipe = os.pipe + + def tracked_pipe() -> tuple[int, int]: + pair = real_pipe() + descriptors.extend(pair) + return pair + + def fail_fdopen(*_args, **_kwargs): + raise OSError("fdopen sentinel") + + monkeypatch.setattr(bash_module.os, "pipe", tracked_pipe) + monkeypatch.setattr(bash_module.os, "fdopen", fail_fdopen) + with pytest.raises(OSError, match="fdopen sentinel"): + await bash_module._open_owned_pipe() + + assert len(descriptors) == 2 + for fd in descriptors: + with pytest.raises(OSError): + os.fstat(fd) + + +async def test_pipe_transport_setup_failure_closes_both_descriptors(monkeypatch: pytest.MonkeyPatch) -> None: + descriptors: list[int] = [] + real_pipe = os.pipe + loop = asyncio.get_running_loop() + + def tracked_pipe() -> tuple[int, int]: + pair = real_pipe() + descriptors.extend(pair) + return pair + + async def fail_connect(*_args, **_kwargs): + raise RuntimeError("transport sentinel") + + monkeypatch.setattr(bash_module.os, "pipe", tracked_pipe) + monkeypatch.setattr(loop, "connect_read_pipe", fail_connect) + with pytest.raises(RuntimeError, match="transport sentinel"): + await bash_module._open_owned_pipe() + + assert len(descriptors) == 2 + for fd in descriptors: + with pytest.raises(OSError): + os.fstat(fd) + + +async def test_startup_failure_is_structured(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + async def fail_spawn(*_args, **_kwargs): + raise FileNotFoundError("bash missing") + + monkeypatch.setattr(bash_module, "_spawn_process", fail_spawn) + result, _ = await _run_bash(tmp_path, {"command": "printf never"}) + + assert result.ok is False + assert result.metadata["error_type"] == "ProcessStartError" + assert "bash missing" in result.content + + +@pytest.mark.parametrize("failure", [ValueError("invalid spawn"), NotImplementedError("unsupported spawn")]) +async def test_all_spawn_failures_are_normalized_and_close_owned_pipes( + failure: Exception, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + owned_pipes: list[Any] = [] + write_fds: list[int] = [] + real_open = bash_module._open_owned_pipe + + async def tracked_open(): + pipe, write_fd = await real_open() + owned_pipes.append(pipe) + write_fds.append(write_fd) + return pipe, write_fd + + async def fail_spawn(*_args, **_kwargs): + raise failure + + monkeypatch.setattr(bash_module, "_open_owned_pipe", tracked_open) + monkeypatch.setattr(bash_module, "_spawn_process", fail_spawn) + result, _ = await _run_bash(tmp_path, {"command": "printf never"}) + + assert result.metadata["error_type"] == "ProcessStartError" + assert len(owned_pipes) == 2 + assert all(pipe.file.closed for pipe in owned_pipes) + assert all(pipe.transport.is_closing() for pipe in owned_pipes) + for fd in write_fds: + with pytest.raises(OSError): + os.fstat(fd) + + +async def test_post_spawn_exception_closes_readers_and_pipes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + owned_pipes: list[Any] = [] + reader_tasks: list[asyncio.Task[None]] = [] + real_open = bash_module._open_owned_pipe + real_start_readers = bash_module._start_readers + + async def tracked_open(): + pipe, write_fd = await real_open() + owned_pipes.append(pipe) + return pipe, write_fd + + def tracked_start_readers(pipes, limit): + readers, buffers = real_start_readers(pipes, limit) + reader_tasks.extend(readers) + return readers, buffers + + async def fail_cleanup(*_args, **_kwargs): + raise RuntimeError("cleanup sentinel") + + monkeypatch.setattr(bash_module, "_open_owned_pipe", tracked_open) + monkeypatch.setattr(bash_module, "_start_readers", tracked_start_readers) + monkeypatch.setattr(bash_module, "_terminate_group", fail_cleanup) + result, _ = await _run_bash(tmp_path, {"command": "printf partial"}) + + assert result.metadata["error_type"] == "RuntimeError" + assert "cleanup sentinel" in result.content + assert len(reader_tasks) == 2 + assert all(task.done() for task in reader_tasks) + assert all(pipe.file.closed for pipe in owned_pipes) + assert all(pipe.transport.is_closing() for pipe in owned_pipes) + + +async def test_command_timeout_starts_after_delayed_spawn_handoff( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + real_spawn = bash_module._spawn_process + + async def delayed_spawn(*args, **kwargs): + await asyncio.sleep(0.2) + return await real_spawn(*args, **kwargs) + + monkeypatch.setattr(bash_module, "_spawn_process", delayed_spawn) + started = time.monotonic() + result, _ = await _run_bash(tmp_path, {"command": "sleep 0.02; printf done", "timeout": 0.05}) + + assert result.ok is True + assert _stdout(result) == "done" + assert time.monotonic() - started >= 0.2 + + +@pytest.mark.parametrize("setup_stage", ["environment", "spawn-task"]) +async def test_pre_spawn_setup_failures_close_owned_resources( + setup_stage: str, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + owned_pipes: list[Any] = [] + write_fds: list[int] = [] + real_open = bash_module._open_owned_pipe + + async def tracked_open(): + pipe, write_fd = await real_open() + owned_pipes.append(pipe) + write_fds.append(write_fd) + return pipe, write_fd + + def fail_environment(_config): + raise ValueError("environment sentinel") + + def fail_spawn_task(*_args, **_kwargs): + raise RuntimeError("spawn-task sentinel") + + monkeypatch.setattr(bash_module, "_open_owned_pipe", tracked_open) + if setup_stage == "environment": + monkeypatch.setattr(bash_module, "_command_environment", fail_environment) + else: + monkeypatch.setattr(bash_module, "_spawn_process", fail_spawn_task) + + result, _ = await _run_bash(tmp_path, {"command": "printf never"}) + + assert result.metadata["error_type"] == "ProcessStartError" + assert setup_stage in result.content + assert all(pipe.file.closed for pipe in owned_pipes) + assert all(pipe.transport.is_closing() for pipe in owned_pipes) + for fd in write_fds: + with pytest.raises(OSError): + os.fstat(fd) + + +async def test_unsupported_platform_fails_before_pipe_or_spawn(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + opened = False + + async def unexpected_pipe(): + nonlocal opened + opened = True + raise AssertionError + + monkeypatch.setattr(bash_module, "_supports_process_groups", lambda: False) + monkeypatch.setattr(bash_module, "_open_owned_pipe", unexpected_pipe) + result, _ = await _run_bash(tmp_path, {"command": "printf never"}) + + assert result.metadata["error_type"] == "UnsupportedPlatform" + assert opened is False + + +async def test_cwd_must_exist_be_directory_and_stay_inside_root(tmp_path: Path) -> None: + file_path = tmp_path / "file.txt" + file_path.write_text("x", encoding="utf-8") + outside = tmp_path.parent / f"{tmp_path.name}-outside" + outside.mkdir() + link = tmp_path / "link" + link.symlink_to(outside, target_is_directory=True) + + missing, _ = await _run_bash(tmp_path, {"command": "pwd", "cwd": "missing"}) + not_directory, _ = await _run_bash(tmp_path, {"command": "pwd", "cwd": "file.txt"}) + parent, _ = await _run_bash(tmp_path, {"command": "pwd", "cwd": ".."}) + absolute, _ = await _run_bash(tmp_path, {"command": "pwd", "cwd": str(outside)}) + symlink, _ = await _run_bash(tmp_path, {"command": "pwd", "cwd": "link"}) + + assert missing.metadata["error_type"] == "PathNotFound" + assert not_directory.metadata["error_type"] == "NotADirectory" + assert parent.metadata["error_type"] == "PathValidationError" + assert absolute.metadata["error_type"] == "PathValidationError" + assert symlink.metadata["error_type"] == "PathValidationError" + + +@pytest.mark.parametrize( + ("limit", "expected"), + [ + (1, "a\n... 9 bytes omitted ...\n"), + (5, "abc\n... 5 bytes omitted ...\nij"), + (6, "abc\n... 4 bytes omitted ...\nhij"), + ], +) +async def test_output_uses_fixed_head_tail_split(limit: int, expected: str, tmp_path: Path) -> None: + result, _ = await _run_bash( + tmp_path, + {"command": "printf abcdefghij; printf ABCDEFGHIJ >&2"}, + plugin=BashPlugin(max_output_bytes=limit), + ) + + assert _stdout(result) == expected + assert _stderr(result) == expected.upper().replace("BYTES OMITTED", "bytes omitted") + assert result.metadata["stdout_bytes"] == 10 + assert result.metadata["stderr_bytes"] == 10 + assert result.metadata["stdout_truncated"] is True + assert result.metadata["stderr_truncated"] is True + + +async def test_large_no_newline_floods_are_drained_and_bounded_per_stream(tmp_path: Path) -> None: + command = "(head -c 200000 /dev/zero | tr '\\0' x) & (head -c 220000 /dev/zero | tr '\\0' y >&2) & wait" + result, _ = await _run_bash(tmp_path, {"command": command}, plugin=BashPlugin(max_output_bytes=101)) + stdout_marker = "\n... 199899 bytes omitted ...\n" + stderr_marker = "\n... 219899 bytes omitted ...\n" + + assert result.ok is True + assert result.metadata["stdout_bytes"] == 200_000 + assert result.metadata["stderr_bytes"] == 220_000 + assert result.metadata["stdout_truncated"] is True + assert result.metadata["stderr_truncated"] is True + assert len(_stdout(result)) == 101 + len(stdout_marker) + assert len(_stderr(result)) == 101 + len(stderr_marker) + assert _stdout(result).startswith("x" * 51) + assert _stdout(result).endswith("x" * 50) + assert _stderr(result).startswith("y" * 51) + assert _stderr(result).endswith("y" * 50) + assert list(tmp_path.iterdir()) == [] + + +async def test_exact_output_limit_is_not_truncated(tmp_path: Path) -> None: + result, _ = await _run_bash( + tmp_path, + {"command": "printf abcde; printf ABCDE >&2"}, + plugin=BashPlugin(max_output_bytes=5), + ) + + assert _stdout(result) == "abcde" + assert _stderr(result) == "ABCDE" + assert result.metadata["stdout_bytes"] == 5 + assert result.metadata["stderr_bytes"] == 5 + assert result.metadata["stdout_truncated"] is False + assert result.metadata["stderr_truncated"] is False + + +async def test_invalid_and_split_utf8_decode_with_replacement(tmp_path: Path) -> None: + code = "import os; os.write(1, b'a\\xe2\\x82\\xacb\\xffc')" + result, _ = await _run_bash( + tmp_path, + {"command": f"{shlex.quote(sys.executable)} -c {shlex.quote(code)}"}, + plugin=BashPlugin(max_output_bytes=5), + ) + + assert "�" in _stdout(result) + assert result.metadata["stdout_bytes"] == 7 + assert result.metadata["stdout_truncated"] is True + + +async def test_normal_exit_cleans_same_group_background_descendant(tmp_path: Path) -> None: + pgid_file = tmp_path / "pgid" + child_file = tmp_path / "child" + command = f"echo $$ > {shlex.quote(str(pgid_file))}; sleep 30 & echo $! > {shlex.quote(str(child_file))}" + result, _ = await _run_bash(tmp_path, {"command": command}) + pgid = int(pgid_file.read_text()) + child = int(child_file.read_text()) + + assert result.ok is True + assert not _group_exists(pgid) + with pytest.raises(ProcessLookupError): + os.kill(child, 0) + + +async def test_normal_exit_kills_term_ignoring_same_group_descendant(tmp_path: Path) -> None: + child_file = tmp_path / "child" + code = ( + "import os,signal,time; signal.signal(signal.SIGTERM, signal.SIG_IGN); " + f"open({str(child_file)!r},'w').write(str(os.getpid())); time.sleep(30)" + ) + command = ( + f"{shlex.quote(sys.executable)} -c {shlex.quote(code)} & " + f"while ! test -f {shlex.quote(str(child_file))}; do sleep 0.01; done" + ) + started = time.monotonic() + result, _ = await _run_bash(tmp_path, {"command": command}) + child = int(child_file.read_text()) + + assert result.ok is True + assert time.monotonic() - started >= 1 + with pytest.raises(ProcessLookupError): + os.kill(child, 0) + + +async def test_cancelling_run_kills_group_and_propagates(tmp_path: Path) -> None: + pgid_file = tmp_path / "pgid" + session = ScriptedSession(start_turn=_call_turn({ + "command": f"echo $$ > {shlex.quote(str(pgid_file))}; trap '' TERM; while :; do sleep 1; done", + })) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session]), plugins=[BashPlugin()]) + task = asyncio.create_task(harness.run("go")) + await _wait_for_file(pgid_file) + pgid = int(pgid_file.read_text()) + + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=4) + + assert not _group_exists(pgid) + + +async def test_repeated_cancellation_does_not_detach_cleanup(tmp_path: Path) -> None: + pgid_file = tmp_path / "pgid" + session = ScriptedSession(start_turn=_call_turn({ + "command": f"echo $$ > {shlex.quote(str(pgid_file))}; trap '' TERM; while :; do sleep 1; done", + })) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session]), plugins=[BashPlugin()]) + task = asyncio.create_task(harness.run("go")) + await _wait_for_file(pgid_file) + pgid = int(pgid_file.read_text()) + + task.cancel() + await asyncio.sleep(0.1) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=4) + + assert not _group_exists(pgid) + + +async def test_cleanup_exception_does_not_replace_run_cancellation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pgid_file = tmp_path / "pgid" + owned_pipes: list[Any] = [] + reader_tasks: list[asyncio.Task[None]] = [] + real_open = bash_module._open_owned_pipe + real_start_readers = bash_module._start_readers + real_cleanup = bash_module._cleanup_process + + async def tracked_open(): + pipe, write_fd = await real_open() + owned_pipes.append(pipe) + return pipe, write_fd + + def tracked_start_readers(pipes, limit): + readers, buffers = real_start_readers(pipes, limit) + reader_tasks.extend(readers) + return readers, buffers + + async def failing_cleanup(*args, **kwargs): + await real_cleanup(*args, **kwargs) + raise RuntimeError("cancel cleanup sentinel") + + monkeypatch.setattr(bash_module, "_open_owned_pipe", tracked_open) + monkeypatch.setattr(bash_module, "_start_readers", tracked_start_readers) + monkeypatch.setattr(bash_module, "_cleanup_process", failing_cleanup) + session = ScriptedSession(start_turn=_call_turn({ + "command": f"echo $$ > {shlex.quote(str(pgid_file))}; trap '' TERM; while :; do sleep 1; done", + })) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session]), plugins=[BashPlugin()]) + task = asyncio.create_task(harness.run("go")) + await _wait_for_file(pgid_file) + pgid = int(pgid_file.read_text()) + + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=4) + + assert not _group_exists(pgid) + assert all(task.done() for task in reader_tasks) + assert all(pipe.file.closed for pipe in owned_pipes) + assert all(pipe.transport.is_closing() for pipe in owned_pipes) + + +async def test_cancellation_during_spawn_waits_for_handoff_and_cleans_process( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pgid_file = tmp_path / "pgid" + entered = asyncio.Event() + release = asyncio.Event() + real_spawn = bash_module._spawn_process + spawned_pids: list[int] = [] + + async def delayed_spawn(*args, **kwargs): + entered.set() + await release.wait() + process = await real_spawn(*args, **kwargs) + spawned_pids.append(process.pid) + return process + + monkeypatch.setattr(bash_module, "_spawn_process", delayed_spawn) + session = ScriptedSession(start_turn=_call_turn({ + "command": f"echo $$ > {shlex.quote(str(pgid_file))}; trap '' TERM; while :; do sleep 1; done", + })) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session]), plugins=[BashPlugin()]) + task = asyncio.create_task(harness.run("go")) + await entered.wait() + + task.cancel() + await asyncio.sleep(0.05) + task.cancel() + release.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=4) + + assert len(spawned_pids) == 1 + assert not _group_exists(spawned_pids[0]) + + +async def _escaped_descendant_command(tmp_path: Path, *, shell_tail: str = ":") -> tuple[str, Path]: + pid_file = tmp_path / "escaped-pid" + ready_file = tmp_path / "escaped-ready" + code = ( + "import os,time; os.setsid(); " + f"open({str(pid_file)!r},'w').write(str(os.getpid())); " + f"open({str(ready_file)!r},'w').write('ready'); time.sleep(30)" + ) + command = ( + f"{shlex.quote(sys.executable)} -c {shlex.quote(code)} & " + f"while ! test -f {shlex.quote(str(ready_file))}; do sleep 0.01; done; {shell_tail}" + ) + return command, pid_file + + +def _kill_escaped(pid_file: Path) -> None: + if pid_file.exists(): + try: + os.kill(int(pid_file.read_text()), signal.SIGKILL) + except ProcessLookupError: + pass + + +async def test_escaped_descendant_pipe_has_bounded_normal_drain(tmp_path: Path) -> None: + command, pid_file = await _escaped_descendant_command(tmp_path, shell_tail="printf done") + started = time.monotonic() + try: + result, _ = await _run_bash(tmp_path, {"command": command}) + finally: + _kill_escaped(pid_file) + + assert result.ok is True + assert _stdout(result) == "done" + assert time.monotonic() - started < 3 + + +async def test_escaped_descendant_pipe_has_bounded_timeout_drain(tmp_path: Path) -> None: + command, pid_file = await _escaped_descendant_command(tmp_path, shell_tail="printf before; sleep 30") + started = time.monotonic() + try: + result, _ = await _run_bash(tmp_path, {"command": command, "timeout": 0.1}) + finally: + _kill_escaped(pid_file) + + assert result.metadata["error_type"] == "Timeout" + assert "before" in _stdout(result) + assert time.monotonic() - started < 4 + + +async def test_escaped_descendant_pipe_has_bounded_cancellation_drain(tmp_path: Path) -> None: + command, pid_file = await _escaped_descendant_command(tmp_path, shell_tail="sleep 30") + ready = tmp_path / "escaped-ready" + session = ScriptedSession(start_turn=_call_turn({"command": command})) + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([session]), plugins=[BashPlugin()]) + task = asyncio.create_task(harness.run("go")) + await _wait_for_file(ready) + started = time.monotonic() + try: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=4) + finally: + _kill_escaped(pid_file) + + assert time.monotonic() - started < 3 + + +async def test_minimal_environment_filters_secrets_and_sets_defaults(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("THINHARNESS_SECRET_SENTINEL", "hidden") + monkeypatch.setenv("BASH_ENV", str(tmp_path / "missing-startup")) + result, _ = await _run_bash(tmp_path, {"command": "env"}) + environment = dict(line.split("=", 1) for line in _stdout(result).splitlines() if "=" in line) + + assert "THINHARNESS_SECRET_SENTINEL" not in environment + assert "BASH_ENV" not in environment + assert "ENV" not in environment + assert environment["NO_COLOR"] == "1" + assert environment["TERM"] == "dumb" + assert environment["PAGER"] == "cat" + assert environment["GIT_PAGER"] == "cat" + assert set(environment) <= { + "PATH", "HOME", "TMPDIR", "TMP", "TEMP", "LANG", "LC_ALL", "LC_CTYPE", "TZ", + "NO_COLOR", "TERM", "PAGER", "GIT_PAGER", "PWD", "SHLVL", "_", + } + + +async def test_minimal_environment_allows_explicit_host_overrides(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("THINHARNESS_SECRET_SENTINEL", "hidden") + result, _ = await _run_bash( + tmp_path, + {"command": "env"}, + plugin=BashPlugin(env={"TERM": "minimal-host-term", "EXPLICIT_MINIMAL": "yes"}), + ) + environment = dict(line.split("=", 1) for line in _stdout(result).splitlines() if "=" in line) + + assert "THINHARNESS_SECRET_SENTINEL" not in environment + assert environment["TERM"] == "minimal-host-term" + assert environment["EXPLICIT_MINIMAL"] == "yes" + + +async def test_full_environment_is_explicit_and_host_values_override_defaults(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("THINHARNESS_INHERITED_SENTINEL", "visible") + result, _ = await _run_bash( + tmp_path, + {"command": "env"}, + plugin=BashPlugin(inherit_env=True, env={"TERM": "host-term", "EXPLICIT": "yes"}), + ) + environment = dict(line.split("=", 1) for line in _stdout(result).splitlines() if "=" in line) + + assert environment["THINHARNESS_INHERITED_SENTINEL"] == "visible" + assert environment["TERM"] == "host-term" + assert environment["EXPLICIT"] == "yes" + + +async def test_bash_env_and_env_are_removed_unless_explicit(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + startup = tmp_path / "startup.sh" + startup.write_text("printf 'STARTUP_MARKER\\n'", encoding="utf-8") + monkeypatch.setenv("BASH_ENV", str(startup)) + monkeypatch.setenv("ENV", "inherited-env") + + inherited, _ = await _run_bash(tmp_path, {"command": "env"}, plugin=BashPlugin(inherit_env=True)) + explicit, _ = await _run_bash( + tmp_path, + {"command": "env"}, + plugin=BashPlugin(env={"BASH_ENV": str(startup), "ENV": "explicit-env"}), + ) + inherited_environment = dict(line.split("=", 1) for line in _stdout(inherited).splitlines() if "=" in line) + explicit_environment = dict(line.split("=", 1) for line in _stdout(explicit).splitlines() if "=" in line) + + assert "BASH_ENV" not in inherited_environment + assert "ENV" not in inherited_environment + assert explicit_environment["BASH_ENV"] == str(startup) + assert explicit_environment["ENV"] == "explicit-env" + assert _stdout(explicit).startswith("STARTUP_MARKER\n") + + +@pytest.mark.parametrize( + "arguments", + [ + {}, + {"command": " \t"}, + {"command": 1}, + {"command": "printf ok", "cwd": 1}, + {"command": "printf ok", "timeout": True}, + {"command": "printf ok", "timeout": "1"}, + {"command": "printf ok", "timeout": float("inf")}, + {"command": "printf ok", "timeout": 0}, + {"command": "printf ok", "env": {"SECRET": "x"}}, + ], +) +def test_model_arguments_use_retryable_strict_validation(tmp_path: Path, arguments: dict[str, Any]) -> None: + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[BashPlugin()]) + output = ToolResult.from_json(call_tool(harness.tools[0], arguments)) + + assert output.ok is False + assert output.metadata["error_type"] == "ValidationError" + assert output.metadata["retry"] is True + + +async def test_huge_integer_timeout_is_retryable_through_public_call_paths(tmp_path: Path) -> None: + arguments = {"command": "printf never", "timeout": 10**400} + harness = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([]), plugins=[BashPlugin()]) + direct = ToolResult.from_json(call_tool(harness.tools[0], arguments)) + through_run, _ = await _run_bash(tmp_path, arguments) + + for result in (direct, through_run): + assert result.ok is False + assert result.metadata["error_type"] == "ValidationError" + assert result.metadata["retry"] is True + + +async def test_approval_uses_normal_top_level_pause_and_resume(tmp_path: Path) -> None: + first = ScriptedSession(start_turn=_call_turn({"command": "printf approved"})) + resumed_outputs: list[ToolResult] = [] + resumed = ScriptedSession( + start_turn=ModelTurn(raw={"unused": True}), + continue_turn=ModelTurn(text="done", raw={"id": "done"}), + on_continue=lambda outputs, _tools, _metadata: resumed_outputs.append(ToolResult.from_json(outputs[0].output)), + ) + harness = Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([first, resumed]), + plugins=[BashPlugin(requires_approval=True)], + ) + + paused = await harness.run("go") + assert paused.stop_reason == "approval_required" + assert paused.pending_approvals[0].tool_name == "bash" + result = await harness.resume_approvals( + paused.resume_state, + [ApprovalDecision(call_id="call_1", approved=True)], + ) + + assert result.text == "done" + assert resumed_outputs[0].ok is True + assert _stdout(resumed_outputs[0]) == "approved" + + +def test_approval_required_bash_follows_existing_model_and_child_rules(tmp_path: Path) -> None: + model = ScriptedModel([]) + del model.resume_kind + with pytest.raises(ValueError, match="resumable model"): + Harness( + HarnessConfig(root=tmp_path), + model=model, + plugins=[BashPlugin(requires_approval=True)], + ) + with pytest.raises(ValueError, match="child harnesses"): + Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([]), + plugins=[SubagentsPlugin(agents=[SubAgentConfig( + name="shell", + description="Shell child.", + plugins=[BashPlugin(requires_approval=True)], + )])], + ) + + +def test_bash_does_not_implement_child_inheritance() -> None: + assert not hasattr(BashPlugin(), "for_child") + + +def test_default_and_named_children_do_not_inherit_bash_and_explicit_child_can_use_it(tmp_path: Path) -> None: + observed_tools: list[list[str]] = [] + parent = ScriptedSession( + start_turn=ModelTurn( + tool_calls=[ModelToolCall(id="call_1", name="subagent", arguments='{"task":"check"}')], + raw={"id": "parent"}, + ), + ) + child = ScriptedSession( + start_turn=ModelTurn(text="child", raw={"id": "child"}), + on_start=lambda _prompt, _instructions, tools, _metadata, _previous: observed_tools.append([tool["name"] for tool in tools]), + ) + Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([parent, child]), + plugins=[BashPlugin(), SubagentsPlugin()], + ).run_sync("go") + + assert "bash" not in observed_tools[0] + + observed_tools.clear() + parent = ScriptedSession( + start_turn=ModelTurn( + tool_calls=[ModelToolCall(id="call_2", name="subagent", arguments='{"task":"check","agent":"plain"}')], + raw={"id": "parent"}, + ), + ) + child = ScriptedSession( + start_turn=ModelTurn(text="child", raw={"id": "child"}), + on_start=lambda _prompt, _instructions, tools, _metadata, _previous: observed_tools.append([tool["name"] for tool in tools]), + ) + Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([parent, child]), + plugins=[BashPlugin(), SubagentsPlugin(agents=[SubAgentConfig( + name="plain", + description="Plain child.", + )])], + ).run_sync("go") + + assert "bash" not in observed_tools[0] + + observed_tools.clear() + child_bash_results: list[ToolResult] = [] + parent = ScriptedSession( + start_turn=ModelTurn( + tool_calls=[ModelToolCall(id="call_3", name="subagent", arguments='{"task":"check","agent":"shell"}')], + raw={"id": "parent"}, + ), + ) + child = ScriptedSession( + start_turn=_call_turn({"command": "printf child-bash"}, call_id="child_bash_call"), + continue_turn=ModelTurn(text="child", raw={"id": "child"}), + on_start=lambda _prompt, _instructions, tools, _metadata, _previous: observed_tools.append([tool["name"] for tool in tools]), + on_continue=lambda outputs, _tools, _metadata: child_bash_results.append(ToolResult.from_json(outputs[0].output)), + ) + Harness( + HarnessConfig(root=tmp_path), + model=ScriptedModel([parent, child]), + plugins=[SubagentsPlugin(agents=[SubAgentConfig( + name="shell", + description="Shell child.", + plugins=[BashPlugin()], + )])], + ).run_sync("go") + + assert "bash" in observed_tools[0] + assert len(child_bash_results) == 1 + assert child_bash_results[0].ok is True + assert _stdout(child_bash_results[0]) == "child-bash" + + +def test_mixed_batch_containing_bash_runs_sequentially(tmp_path: Path) -> None: + client = MultiCallClient([("bash", '{"command":"sleep 0.2; printf bash"}'), ("slow", "{}")]) + harness = Harness( + HarnessConfig(root=tmp_path, model="openai:test-model"), + model=_fake_openai(client), + plugins=[BashPlugin()], + tools=[slow_tool("slow", 0.2)], + ) + + started = time.monotonic() + harness.run_sync("go") + + assert time.monotonic() - started >= 0.38 + assert [item["call_id"] for item in client.payloads[1]["input"]] == ["call_1", "call_2"] + + +async def test_bash_uses_normal_hooks_records_and_tracing(tmp_path: Path) -> None: + hook_calls: list[str] = [] + tracer = ContextFakeTracer() + result, run_result = await _run_bash( + tmp_path, + {"command": "printf integrated"}, + hooks=[ + Hook("before_tool_call", lambda ctx: hook_calls.append(f"before:{ctx.tool_name}")), + Hook("after_tool_call", lambda ctx: hook_calls.append(f"after:{ctx.tool_name}")), + ], + tracing=[TracingOptions(tracer=tracer)], + ) + + assert result.ok is True + assert hook_calls == ["before:bash", "after:bash"] + assert ToolResult.from_json(run_result.tool_call_records[0]["output"]).content == result.content + tool_span = next(span for span in tracer.spans if span.name == "execute_tool bash") + assert tool_span.attributes["gen_ai.tool.name"] == "bash" diff --git a/tests/unit/test_bash_tool.py b/tests/unit/test_bash_tool.py deleted file mode 100644 index 0d31df2..0000000 --- a/tests/unit/test_bash_tool.py +++ /dev/null @@ -1,172 +0,0 @@ -from __future__ import annotations - -import json -import time -from pathlib import Path - -import pytest -from fakes import MultiCallClient, ScriptedModel, ScriptedSession, _fake_openai, slow_tool, tool_output - -from thinharness import BashArgs, BashTool, Harness, HarnessConfig, SubAgentConfig, call_tool -from thinharness.providers import ModelToolCall, ModelTurn - - -def test_bash_spec_exposes_expected_schema() -> None: - spec = BashTool().spec() - - assert spec.name == "bash" - assert spec.description == ( - "Run one bash command from a workspace-contained cwd. Intended for exploratory workflows; prefer typed tools for production." - ) - assert spec.parameters is BashArgs - assert spec.sequential is True - schema = spec.response_tool()["parameters"] - assert schema["additionalProperties"] is False - assert schema["properties"]["timeout"]["minimum"] == 1 - assert schema["properties"]["timeout"]["maximum"] == 120 - - -def test_successful_command_captures_output_and_metadata(tmp_path: Path) -> None: - tool = BashTool(tmp_path) - - result = tool.run({"command": "printf out; printf err >&2"}) - - assert result.ok is True - assert "stdout:\nout" in result.content - assert "stderr:\nerr" in result.content - assert result.metadata["exit_code"] == 0 - assert result.metadata["timed_out"] is False - assert result.metadata["cwd"] == str(tmp_path) - assert result.metadata["stdout_truncated"] is False - assert result.metadata["stderr_truncated"] is False - assert isinstance(result.metadata["duration_seconds"], float) - - -def test_non_zero_command_is_tool_failure_with_stderr(tmp_path: Path) -> None: - result = BashTool(tmp_path).run({"command": "printf nope >&2; exit 7"}) - - assert result.ok is False - assert "stderr:\nnope" in result.content - assert result.metadata["exit_code"] == 7 - assert result.metadata["error_type"] == "NonZeroExit" - - -def test_timeout_terminates_command_group_and_returns_metadata(tmp_path: Path) -> None: - result = BashTool(tmp_path).run({"command": "printf before; sleep 2", "timeout": 1}) - - assert result.ok is False - assert result.metadata["timed_out"] is True - assert result.metadata["error_type"] == "Timeout" - assert result.metadata["exit_code"] is not None - assert "before" in result.content - - -def test_timeout_returns_when_command_ignores_sigterm(tmp_path: Path) -> None: - start = time.monotonic() - result = BashTool(tmp_path).run({"command": "trap '' TERM; printf before; sleep 30", "timeout": 1}) - elapsed = time.monotonic() - start - - assert result.ok is False - assert result.metadata["timed_out"] is True - assert elapsed < 5 - assert "before" in result.content - - -def test_cwd_cannot_escape_workspace(tmp_path: Path) -> None: - result = BashTool(tmp_path).run({"command": "pwd", "cwd": ".."}) - - assert result.ok is False - assert result.metadata["error_type"] == "PathValidationError" - - -def test_nonexistent_and_file_cwd_fail_cleanly(tmp_path: Path) -> None: - file_path = tmp_path / "file.txt" - file_path.write_text("x", encoding="utf-8") - - missing = BashTool(tmp_path).run({"command": "pwd", "cwd": "missing"}) - file_result = BashTool(tmp_path).run({"command": "pwd", "cwd": "file.txt"}) - - assert missing.ok is False - assert missing.metadata["error_type"] == "PathNotFound" - assert file_result.ok is False - assert file_result.metadata["error_type"] == "NotADirectory" - - -def test_output_truncation_sets_metadata(tmp_path: Path) -> None: - result = BashTool(tmp_path, max_tool_chars=5).run({"command": "printf 1234567890; printf abcdefghij >&2"}) - - assert result.ok is True - assert "...[truncated]"[:5] in result.content - assert result.metadata["stdout_truncated"] is True - assert result.metadata["stderr_truncated"] is True - - -def test_per_call_max_chars_can_lower_but_not_raise_constructor_cap(tmp_path: Path) -> None: - lower = BashTool(tmp_path, max_tool_chars=100).run({"command": "printf 1234567890", "max_chars": 5}) - clamped = BashTool(tmp_path, max_tool_chars=5).run({"command": "printf 1234567890", "max_chars": 100}) - - assert lower.metadata["stdout_truncated"] is True - assert "12345" not in lower.content - assert clamped.metadata["stdout_truncated"] is True - assert "12345" not in clamped.content - - -def test_invalid_bash_args_return_retry_envelope(tmp_path: Path) -> None: - output = json.loads(call_tool(BashTool(tmp_path).spec(), "{}")) - - assert output["ok"] is False - assert output["metadata"]["error_type"] == "ValidationError" - assert output["metadata"]["retry"] is True - - -def test_bash_is_available_through_explicit_custom_registration(tmp_path: Path) -> None: - call = ModelTurn( - tool_calls=[ModelToolCall(id="call_1", name="bash", arguments='{"command":"printf custom"}')], - raw={"id": "start"}, - ) - captured = {} - - def on_continue(outputs, _tools, _metadata) -> None: - captured["output"] = tool_output(outputs[0].output) - - session = ScriptedSession(start_turn=call, continue_turn=ModelTurn(text="done", raw={"id": "done"}), on_continue=on_continue) - harness = Harness( - HarnessConfig(root=tmp_path), - model=ScriptedModel([session]), - tools=[BashTool(tmp_path).spec()], - ) - - assert [tool["name"] for tool in harness.tool_schemas()] == ["bash"] - assert harness.run_sync("go").text == "done" - assert captured["output"]["ok"] is True - assert "custom" in captured["output"]["content"] - - -def test_bash_is_not_a_builtin_tool(tmp_path: Path) -> None: - default = Harness(HarnessConfig(root=tmp_path), model=ScriptedModel([])) - - assert "bash" not in [tool["name"] for tool in default.tool_schemas()] - with pytest.raises(ValueError, match="SubagentsPlugin"): - HarnessConfig(root=tmp_path, builtin_tools=["bash"]) - - -def test_named_subagent_builtin_selector_is_removed() -> None: - with pytest.raises(ValueError, match="SubAgentConfig.builtin_tools has been removed"): - SubAgentConfig(name="shell", description="Shell helper.", builtin_tools=["bash"], tools=[BashTool().spec()]) - - -def test_mixed_batch_containing_bash_runs_sequentially(tmp_path: Path) -> None: - client = MultiCallClient([("bash", '{"command":"sleep 0.2; printf bash"}'), ("slow", "{}")]) - harness = Harness( - HarnessConfig(root=tmp_path, model="openai:test-model"), - model=_fake_openai(client), - tools=[BashTool(tmp_path).spec(), slow_tool("slow", 0.2)], - ) - - start = time.monotonic() - harness.run_sync("go") - elapsed = time.monotonic() - start - - assert elapsed >= 0.38 - continuation_inputs = client.payloads[1]["input"] - assert [item["call_id"] for item in continuation_inputs] == ["call_1", "call_2"] diff --git a/thinharness/__init__.py b/thinharness/__init__.py index 6e5089f..529fb7b 100644 --- a/thinharness/__init__.py +++ b/thinharness/__init__.py @@ -39,6 +39,7 @@ from .output import NativeOutput, OutputSchema, PromptedOutput, TextOutput, ToolStructuredOutput from .plugins import ( DEFAULT_SUBAGENT_NAME, + BashPlugin, ChildHarnessHost, ChildHarnessOutcome, ChildHarnessRequest, @@ -79,8 +80,6 @@ parse_model_ref, ) from .tools import ( - BashArgs, - BashTool, FilePromptSource, FileTools, InlinePromptSource, @@ -114,12 +113,11 @@ __all__ = [ "__version__", - "BashArgs", + "BashPlugin", "ChildHarnessHost", "ChildHarnessOutcome", "ChildHarnessRequest", "ChildInheritablePlugin", - "BashTool", "FileTools", "FilesystemPlugin", "FilePromptSource", diff --git a/thinharness/plugins/__init__.py b/thinharness/plugins/__init__.py index afe1d75..23a03b7 100644 --- a/thinharness/plugins/__init__.py +++ b/thinharness/plugins/__init__.py @@ -2,6 +2,7 @@ from ..children import ChildHarnessHost, ChildHarnessOutcome, ChildHarnessRequest from .base import ChildInheritablePlugin, Plugin, PluginBinding, PluginConnector, PluginContext, PluginContribution, ToolOrigin +from .bash import BashPlugin from .filesystem import FilesystemPlugin from .mcp import MCPPlugin from .parallel_llm import ParallelLlmPlugin @@ -9,6 +10,7 @@ from .subagents import DEFAULT_SUBAGENT_NAME, SubAgentArgs, SubAgentConfig, SubagentsPlugin __all__ = [ + "BashPlugin", "ChildHarnessHost", "ChildHarnessOutcome", "ChildHarnessRequest", diff --git a/thinharness/plugins/bash.py b/thinharness/plugins/bash.py new file mode 100644 index 0000000..a9d068d --- /dev/null +++ b/thinharness/plugins/bash.py @@ -0,0 +1,610 @@ +"""Explicit bounded local Bash plugin.""" + +from __future__ import annotations + +import asyncio +import math +import os +import signal +import time +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from pydantic import Field, field_validator + +from ..tools.base import PathValidationError, StrictArgs, ToolOrigin, ToolResult, ToolSpec, contained_path +from .base import PluginBinding, PluginContext, PluginContribution + +_BASH_DESCRIPTION = ( + "Run one non-interactive Bash command. The cwd must be inside the workspace. Each call starts a fresh shell and does not share " + "shell state with other calls. Background processes in the same process group are terminated on a best-effort basis. timeout is in " + "seconds and is capped by the host." +) +_ENV_ALLOWLIST = ("PATH", "HOME", "TMPDIR", "TMP", "TEMP", "LANG", "LC_ALL", "LC_CTYPE", "TZ") +_READ_CHUNK_SIZE = 64 * 1024 +_TERMINATE_GRACE_SECONDS = 1.0 +_FINAL_DRAIN_SECONDS = 1.0 +_CANCELLATION_CLEANUP_SECONDS = 4.0 + + +class _BashArgs(StrictArgs): + """Model arguments for one Bash call.""" + + command: str = Field(min_length=1, strict=True) + cwd: str = Field(default=".", strict=True) + timeout: float | None = None + + @field_validator("command") + @classmethod + def _validate_command(cls, value: str) -> str: + if not value.strip(): + raise ValueError("command must contain a non-whitespace character") + return value + + @field_validator("timeout", mode="before") + @classmethod + def _validate_timeout(cls, value: object) -> object: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int | float): + raise ValueError("timeout must be a real integer or float") + try: + converted = float(value) + except OverflowError as exc: + raise ValueError("timeout must be finite and greater than zero") from exc + if not math.isfinite(converted) or converted <= 0: + raise ValueError("timeout must be finite and greater than zero") + return converted + + +@dataclass(frozen=True) +class _BashConfig: + default_timeout: float + max_timeout: float + max_output_bytes: int + inherit_env: bool + env_entries: tuple[tuple[str, str], ...] + requires_approval: bool + + +class _BashPluginMeta(type): + """Keep the Bash plugin name fixed on the class hierarchy.""" + + def __setattr__(cls, attribute: str, value: object) -> None: + if attribute == "name": + raise AttributeError("BashPlugin.name is fixed to 'bash'") + super().__setattr__(attribute, value) + + def __delattr__(cls, attribute: str) -> None: + if attribute == "name": + raise AttributeError("BashPlugin.name is fixed to 'bash'") + super().__delattr__(attribute) + + +class BashPlugin(metaclass=_BashPluginMeta): + """Provide one bounded, non-interactive local Bash tool.""" + + name = "bash" + _config: _BashConfig + _frozen: bool + + def __init_subclass__(cls) -> None: + """Reject subclasses that replace the fixed plugin name.""" + super().__init_subclass__() + if "name" in cls.__dict__: + raise TypeError("BashPlugin subclasses cannot override the fixed name 'bash'") + + def __setattr__(self, attribute: str, value: object) -> None: + """Reject configuration changes after construction.""" + if attribute == "name": + raise AttributeError("BashPlugin.name is fixed to 'bash'") + if getattr(self, "_frozen", False): + raise AttributeError("BashPlugin configuration is frozen") + object.__setattr__(self, attribute, value) + + def __delattr__(self, attribute: str) -> None: + """Reject configuration deletion after construction.""" + if attribute == "name": + raise AttributeError("BashPlugin.name is fixed to 'bash'") + if getattr(self, "_frozen", False): + raise AttributeError("BashPlugin configuration is frozen") + object.__delattr__(self, attribute) + + @property + def env(self) -> dict[str, str]: + """Return a detached copy of the explicit host environment.""" + return dict(self._config.env_entries) + + def __init__( + self, + *, + default_timeout: float = 30, + max_timeout: float = 120, + max_output_bytes: int = 40_000, + inherit_env: bool = False, + env: Mapping[str, str] | None = None, + requires_approval: bool = False, + ) -> None: + default = _validate_positive_real("default_timeout", default_timeout) + maximum = _validate_positive_real("max_timeout", max_timeout) + if default > maximum: + raise ValueError("default_timeout must not exceed max_timeout") + if isinstance(max_output_bytes, bool) or not isinstance(max_output_bytes, int): + raise TypeError("max_output_bytes must be an integer") + if max_output_bytes <= 0: + raise ValueError("max_output_bytes must be greater than zero") + if not isinstance(inherit_env, bool): + raise TypeError("inherit_env must be a boolean") + if not isinstance(requires_approval, bool): + raise TypeError("requires_approval must be a boolean") + if env is not None and not isinstance(env, Mapping): + raise TypeError("env must be a mapping of strings") + env_entries = tuple(_validate_environment(env).items()) if env is not None else () + object.__setattr__(self, "_config", _BashConfig( + default_timeout=default, + max_timeout=maximum, + max_output_bytes=max_output_bytes, + inherit_env=inherit_env, + env_entries=env_entries, + requires_approval=requires_approval, + )) + object.__setattr__(self, "_frozen", True) + + def bind(self, context: PluginContext) -> PluginBinding: + """Build one static tool against the canonical harness root.""" + runner = _BashRunner(context.root, self._config) + tool = ToolSpec( + "bash", + _BASH_DESCRIPTION, + _BashArgs, + runner.run, + sequential=True, + requires_approval=self._config.requires_approval, + origin=ToolOrigin(plugin="bash", source="bash"), + ) + return PluginBinding(static=PluginContribution(tools=(tool,))) + + +def _validate_positive_real(name: str, value: object) -> float: + if isinstance(value, bool) or not isinstance(value, int | float): + raise TypeError(f"{name} must be a real integer or float") + try: + converted = float(value) + except OverflowError as exc: + raise ValueError(f"{name} must be finite and greater than zero") from exc + if not math.isfinite(converted) or converted <= 0: + raise ValueError(f"{name} must be finite and greater than zero") + return converted + + +def _validate_environment(values: Mapping[Any, Any]) -> dict[str, str]: + result: dict[str, str] = {} + for name, value in values.items(): + if not isinstance(name, str) or not isinstance(value, str): + raise TypeError("environment names and values must be strings") + if not name or "=" in name or "\0" in name: + raise ValueError("environment names must be non-empty and contain neither '=' nor NUL") + if "\0" in value: + raise ValueError("environment values must not contain NUL") + result[name] = value + return result + + +@dataclass +class _BoundedBuffer: + """Retain fixed head and tail byte regions while counting all input.""" + + limit: int + total: int = 0 + + def __post_init__(self) -> None: + self._head_limit = (self.limit + 1) // 2 + self._tail_limit = self.limit // 2 + self._head = bytearray() + self._tail = bytearray() + + def add(self, data: bytes) -> None: + self.total += len(data) + head_needed = self._head_limit - len(self._head) + if head_needed > 0: + self._head.extend(data[:head_needed]) + data = data[head_needed:] + if not data or self._tail_limit == 0: + return + if len(data) >= self._tail_limit: + self._tail[:] = data[-self._tail_limit:] + return + overflow = len(self._tail) + len(data) - self._tail_limit + if overflow > 0: + del self._tail[:overflow] + self._tail.extend(data) + + @property + def truncated(self) -> bool: + return self.total > self.limit + + def render(self) -> str: + if not self.truncated: + return bytes(self._head + self._tail).decode("utf-8", errors="replace") + omitted = self.total - len(self._head) - len(self._tail) + marker = f"\n... {omitted} bytes omitted ...\n" + return self._head.decode("utf-8", errors="replace") + marker + self._tail.decode("utf-8", errors="replace") + + +@dataclass +class _OwnedPipe: + """A parent-owned pipe reader and its asyncio transport.""" + + reader: asyncio.StreamReader + transport: asyncio.ReadTransport + file: Any + + def close(self) -> None: + self.transport.close() + self.file.close() + + +class _BashRunner: + """Run local processes for one bound Bash plugin.""" + + def __init__(self, root: Path, config: _BashConfig) -> None: + self._root = root + self._config = config + + async def run(self, args: _BashArgs) -> ToolResult: + """Run one validated Bash command and return a structured outcome.""" + if not _supports_process_groups(): + return ToolResult(False, "BashPlugin requires POSIX process-group support", {"error_type": "UnsupportedPlatform"}) + try: + cwd = contained_path(self._root, args.cwd) + except PathValidationError as exc: + return ToolResult(False, str(exc), {"error_type": "PathValidationError"}) + if not cwd.exists(): + return ToolResult(False, f"cwd not found: {cwd}", {"error_type": "PathNotFound", "cwd": str(cwd)}) + if not cwd.is_dir(): + return ToolResult(False, f"cwd is not a directory: {cwd}", {"error_type": "NotADirectory", "cwd": str(cwd)}) + timeout = min(args.timeout if args.timeout is not None else self._config.default_timeout, self._config.max_timeout) + return await self._execute(args.command, cwd, timeout) + + async def _execute(self, command: str, cwd: Path, timeout: float) -> ToolResult: + pipes: list[_OwnedPipe] = [] + write_fds: list[int] = [] + spawn_task: asyncio.Task[asyncio.subprocess.Process] | None = None + process: asyncio.subprocess.Process | None = None + readers: list[asyncio.Task[None]] = [] + started: float | None = None + try: + try: + stdout_pipe, stdout_write = await _open_owned_pipe() + pipes.append(stdout_pipe) + write_fds.append(stdout_write) + stderr_pipe, stderr_write = await _open_owned_pipe() + pipes.append(stderr_pipe) + write_fds.append(stderr_write) + environment = _command_environment(self._config) + started = time.perf_counter() + spawn_coroutine = _spawn_process(command, cwd, environment, write_fds[0], write_fds[1]) + try: + spawn_task = asyncio.create_task(spawn_coroutine) + except BaseException: + spawn_coroutine.close() + raise + except asyncio.CancelledError: + raise + except Exception as exc: + duration = 0.0 if started is None else time.perf_counter() - started + return self._start_error(exc, cwd, timeout, duration) + + assert spawn_task is not None + try: + process = await asyncio.shield(spawn_task) + except asyncio.CancelledError: + process = await _spawn_after_cancellation(spawn_task) + _close_fds(write_fds) + write_fds.clear() + if process is not None: + readers, _ = _start_readers(pipes, self._config.max_output_bytes) + wait_task = asyncio.create_task(process.wait()) + _signal_group(process.pid, signal.SIGTERM) + cleanup = asyncio.create_task(_cleanup_process( + process.pid, + process, + wait_task, + readers, + pipes, + initial_term_sent=True, + )) + await _finish_cleanup_despite_cancellation(cleanup) + else: + _close_pipes(pipes) + raise + except Exception as exc: + assert started is not None + return self._start_error(exc, cwd, timeout, time.perf_counter() - started) + finally: + if spawn_task.done(): + _close_fds(write_fds) + write_fds.clear() + + assert process is not None + assert started is not None + readers, buffers = _start_readers(pipes, self._config.max_output_bytes) + wait_task = asyncio.create_task(process.wait()) + timed_out = False + try: + try: + await asyncio.wait_for(asyncio.shield(wait_task), timeout=timeout) + except TimeoutError: + timed_out = True + _signal_group(process.pid, signal.SIGTERM) + await _terminate_group(process.pid, initial_term_sent=True) + await _join_process(process, wait_task) + else: + await _terminate_group(process.pid) + await _final_drain(readers, pipes) + except asyncio.CancelledError: + _signal_group(process.pid, signal.SIGTERM) + cleanup = asyncio.create_task(_cleanup_process( + process.pid, + process, + wait_task, + readers, + pipes, + initial_term_sent=True, + )) + await _finish_cleanup_despite_cancellation(cleanup) + raise + + returncode = process.returncode + duration = time.perf_counter() - started + metadata: dict[str, Any] = { + "exit_code": returncode, + "timed_out": timed_out, + "duration_seconds": round(duration, 3), + "cwd": str(cwd), + "timeout_seconds": timeout, + "stdout_bytes": buffers[0].total, + "stderr_bytes": buffers[1].total, + "stdout_truncated": buffers[0].truncated, + "stderr_truncated": buffers[1].truncated, + } + if returncode is not None and returncode < 0: + metadata["signal"] = -returncode + if timed_out: + metadata["error_type"] = "Timeout" + elif returncode != 0: + metadata["error_type"] = "NonZeroExit" + return ToolResult( + not timed_out and returncode == 0, + _format_output(buffers[0].render(), buffers[1].render()), + metadata, + ) + finally: + _close_fds(write_fds) + _close_pipes(pipes) + for reader in readers: + if not reader.done(): + reader.cancel() + if readers: + await asyncio.gather(*readers, return_exceptions=True) + if spawn_task is not None and not spawn_task.done(): + spawn_task.cancel() + await asyncio.gather(spawn_task, return_exceptions=True) + + @staticmethod + def _start_error(exc: BaseException, cwd: Path, timeout: float, duration: float) -> ToolResult: + return ToolResult(False, f"could not start Bash: {type(exc).__name__}: {exc}", { + "error_type": "ProcessStartError", + "timed_out": False, + "duration_seconds": round(duration, 3), + "cwd": str(cwd), + "timeout_seconds": timeout, + }) + + +def _supports_process_groups() -> bool: + return os.name == "posix" and callable(getattr(os, "killpg", None)) + + +async def _spawn_process( + command: str, + cwd: Path, + environment: dict[str, str], + stdout_fd: int, + stderr_fd: int, +) -> asyncio.subprocess.Process: + return await asyncio.create_subprocess_exec( + "bash", + "-c", + command, + cwd=cwd, + env=environment, + stdin=asyncio.subprocess.DEVNULL, + stdout=stdout_fd, + stderr=stderr_fd, + start_new_session=True, + ) + + +def _command_environment(config: _BashConfig) -> dict[str, str]: + if config.inherit_env: + environment = dict(os.environ) + else: + environment = {name: os.environ[name] for name in _ENV_ALLOWLIST if name in os.environ} + environment.pop("BASH_ENV", None) + environment.pop("ENV", None) + environment.update({"NO_COLOR": "1", "TERM": "dumb", "PAGER": "cat", "GIT_PAGER": "cat"}) + environment.update(config.env_entries) + return environment + + +async def _open_owned_pipe() -> tuple[_OwnedPipe, int]: + read_fd, write_fd = os.pipe() + file: Any | None = None + try: + os.set_inheritable(read_fd, False) + os.set_inheritable(write_fd, False) + file = os.fdopen(read_fd, "rb", buffering=0) + read_fd = -1 + reader = asyncio.StreamReader(limit=_READ_CHUNK_SIZE) + protocol = asyncio.StreamReaderProtocol(reader) + transport, _ = await asyncio.get_running_loop().connect_read_pipe(lambda: protocol, file) + return _OwnedPipe(reader, transport, file), write_fd + except BaseException: + if file is not None: + file.close() + elif read_fd >= 0: + os.close(read_fd) + os.close(write_fd) + raise + + +def _start_readers(pipes: list[_OwnedPipe], limit: int) -> tuple[list[asyncio.Task[None]], list[_BoundedBuffer]]: + buffers = [_BoundedBuffer(limit), _BoundedBuffer(limit)] + readers = [asyncio.create_task(_read_pipe(pipe.reader, buffer)) for pipe, buffer in zip(pipes, buffers, strict=True)] + return readers, buffers + + +async def _read_pipe(reader: asyncio.StreamReader, buffer: _BoundedBuffer) -> None: + while chunk := await reader.read(_READ_CHUNK_SIZE): + buffer.add(chunk) + + +async def _spawn_after_cancellation(task: asyncio.Task[asyncio.subprocess.Process]) -> asyncio.subprocess.Process | None: + while not task.done(): + try: + await asyncio.shield(task) + except asyncio.CancelledError: + continue + except Exception: + break + try: + return task.result() + except asyncio.CancelledError: + return None + except Exception: + return None + + +async def _cleanup_process( + pgid: int, + process: asyncio.subprocess.Process, + wait_task: asyncio.Task[int], + readers: list[asyncio.Task[None]], + pipes: list[_OwnedPipe], + *, + initial_term_sent: bool, +) -> None: + try: + await _terminate_group(pgid, initial_term_sent=initial_term_sent) + await _join_process(process, wait_task) + finally: + await _final_drain(readers, pipes) + + +async def _terminate_group(pgid: int, *, initial_term_sent: bool = False) -> None: + if not initial_term_sent and not _signal_group(pgid, signal.SIGTERM): + return + deadline = asyncio.get_running_loop().time() + _TERMINATE_GRACE_SECONDS + while _group_exists(pgid): + remaining = deadline - asyncio.get_running_loop().time() + if remaining <= 0: + break + await asyncio.sleep(min(0.05, remaining)) + if _group_exists(pgid): + _signal_group(pgid, signal.SIGKILL) + + +async def _join_process(process: asyncio.subprocess.Process, wait_task: asyncio.Task[int]) -> None: + if not wait_task.done(): + try: + await asyncio.wait_for(asyncio.shield(wait_task), timeout=_TERMINATE_GRACE_SECONDS) + except TimeoutError: + process.kill() + if not wait_task.done(): + try: + await asyncio.wait_for(asyncio.shield(wait_task), timeout=_TERMINATE_GRACE_SECONDS) + except TimeoutError: + wait_task.cancel() + await asyncio.gather(wait_task, return_exceptions=True) + + +async def _final_drain(readers: list[asyncio.Task[None]], pipes: list[_OwnedPipe]) -> None: + if readers: + await asyncio.wait(readers, timeout=_FINAL_DRAIN_SECONDS) + _close_pipes(pipes) + for reader in readers: + if not reader.done(): + reader.cancel() + if readers: + await asyncio.gather(*readers, return_exceptions=True) + + +async def _finish_cleanup_despite_cancellation(task: asyncio.Task[None]) -> None: + deadline = asyncio.get_running_loop().time() + _CANCELLATION_CLEANUP_SECONDS + while not task.done(): + remaining = deadline - asyncio.get_running_loop().time() + if remaining <= 0: + task.cancel() + break + try: + await asyncio.wait_for(asyncio.shield(task), timeout=remaining) + except asyncio.CancelledError: + continue + except TimeoutError: + task.cancel() + break + except Exception: + break + while not task.done(): + try: + await asyncio.shield(task) + except asyncio.CancelledError: + continue + except Exception: + break + await asyncio.gather(task, return_exceptions=True) + + +def _signal_group(pgid: int, sig: signal.Signals) -> bool: + try: + os.killpg(pgid, sig) + return True + except ProcessLookupError: + return False + except PermissionError: + return False + + +def _group_exists(pgid: int) -> bool: + try: + os.killpg(pgid, 0) + return True + except ProcessLookupError: + return False + except PermissionError: + return True + + +def _close_fds(fds: list[int]) -> None: + for fd in fds: + try: + os.close(fd) + except OSError: + pass + + +def _close_pipes(pipes: list[_OwnedPipe]) -> None: + for pipe in pipes: + pipe.close() + + +def _format_output(stdout: str, stderr: str) -> str: + if not stdout and not stderr: + return "(no output)" + return f"stdout:\n{stdout or '(no output)'}\nstderr:\n{stderr or '(no output)'}" + + +__all__ = ["BashPlugin"] diff --git a/thinharness/tools/__init__.py b/thinharness/tools/__init__.py index b1907d4..650533d 100644 --- a/thinharness/tools/__init__.py +++ b/thinharness/tools/__init__.py @@ -12,7 +12,6 @@ call_tool, contained_path, ) -from .bash import BashArgs, BashTool from .filesystem import FileTools from .jsonl import JsonlFieldSearch, JsonlSearch, JsonlSearchArgs, JsonlWhereFilter from .mcp import MCPDependencyError, MCPError, MCPServer, MCPServerSSE, MCPServerStdio, MCPServerStreamableHTTP @@ -21,8 +20,6 @@ __all__ = [ "FileTools", - "BashArgs", - "BashTool", "Json", "JsonlSearch", "JsonlSearchArgs", diff --git a/thinharness/tools/bash.py b/thinharness/tools/bash.py deleted file mode 100644 index 23b16f2..0000000 --- a/thinharness/tools/bash.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Opt-in bash tool for exploratory workflows.""" - -from __future__ import annotations - -import os -import signal -import subprocess -import tempfile -import time -from pathlib import Path - -from pydantic import Field - -from .base import ( - Json, - PathPolicy, - PathValidationError, - StrictArgs, - ToolResult, - ToolSpec, - _path_error, - coerce_args, -) - -BASH_DESCRIPTION = "Run one bash command from a workspace-contained cwd. Intended for exploratory workflows; prefer typed tools for production." - - -class BashArgs(StrictArgs): - """Arguments for bash.""" - - command: str - cwd: str = "." - timeout: int = Field(default=10, ge=1, le=120) - max_chars: int | None = Field(default=None, ge=1, description="Per-stream stdout/stderr output cap, clamped to the tool cap.") - - -class BashTool: - """Small, explicitly registered bash command tool.""" - - def __init__( - self, - root: str | Path = ".", - *, - max_tool_chars: int = 40_000, - ) -> None: - self.root = Path(root).expanduser().resolve() - self.root.mkdir(parents=True, exist_ok=True) - self.cwd_policy = PathPolicy(self.root, ["."], "bash cwd") - self.max_tool_chars = max_tool_chars - - def spec(self) -> ToolSpec: - """Return the bash tool spec.""" - return ToolSpec("bash", BASH_DESCRIPTION, BashArgs, self.run, sequential=True) - - def run(self, args: BashArgs | Json) -> ToolResult: - """Run one bash command from a contained working directory.""" - args = coerce_args(args, BashArgs) - try: - cwd = self.cwd_policy.resolve(args.cwd) - except PathValidationError as exc: - return _path_error(exc) - if not cwd.exists(): - return ToolResult(False, f"cwd not found: {self._display(cwd)}", {"error_type": "PathNotFound", "cwd": str(cwd)}) - if not cwd.is_dir(): - return ToolResult(False, f"cwd is not a directory: {self._display(cwd)}", {"error_type": "NotADirectory", "cwd": str(cwd)}) - - limit = min(args.max_chars or self.max_tool_chars, self.max_tool_chars) - start = time.perf_counter() - with tempfile.TemporaryFile() as stdout_file, tempfile.TemporaryFile() as stderr_file: - process = subprocess.Popen( - ["bash", "-c", args.command], - cwd=cwd, - stdin=subprocess.DEVNULL, - stdout=stdout_file, - stderr=stderr_file, - start_new_session=True, - ) - timed_out = False - try: - process.wait(timeout=args.timeout) - except subprocess.TimeoutExpired: - timed_out = True - self._terminate_process_group(process) - else: - self._cleanup_process_group(process) - stdout, stdout_truncated = _read_limited_text(stdout_file, limit) - stderr, stderr_truncated = _read_limited_text(stderr_file, limit) - - duration = time.perf_counter() - start - exit_code = process.returncode - ok = not timed_out and exit_code == 0 - metadata: Json = { - "exit_code": exit_code, - "timed_out": timed_out, - "duration_seconds": round(duration, 3), - "cwd": str(cwd), - "stdout_truncated": stdout_truncated, - "stderr_truncated": stderr_truncated, - } - if timed_out: - metadata["error_type"] = "Timeout" - elif exit_code != 0: - metadata["error_type"] = "NonZeroExit" - return ToolResult(ok, _format_output(stdout, stderr), metadata) - - def _display(self, path: Path) -> str: - """Return a workspace-relative display path where possible.""" - try: - return str(path.relative_to(self.root)) or "." - except ValueError: - return str(path) - - @staticmethod - def _terminate_process_group(process: subprocess.Popen[bytes]) -> None: - """Terminate a timed-out process group on POSIX, falling back to kill.""" - BashTool._signal_process_group(process, signal.SIGTERM) - try: - process.wait(timeout=1) - except subprocess.TimeoutExpired: - pass - BashTool._signal_process_group(process, signal.SIGKILL) - try: - process.wait(timeout=1) - except subprocess.TimeoutExpired: - process.kill() - try: - process.wait(timeout=1) - except subprocess.TimeoutExpired: - pass - - @staticmethod - def _cleanup_process_group(process: subprocess.Popen[bytes]) -> None: - """Best-effort cleanup for background descendants left by bash.""" - if not BashTool._signal_process_group(process, signal.SIGTERM): - return - time.sleep(0.05) - BashTool._signal_process_group(process, signal.SIGKILL) - - @staticmethod - def _signal_process_group(process: subprocess.Popen[bytes], sig: signal.Signals) -> bool: - """Signal a process group if it still exists.""" - try: - os.killpg(process.pid, sig) - return True - except ProcessLookupError: - return False - - -def _format_output(stdout: str, stderr: str) -> str: - """Return labeled command output.""" - return f"stdout:\n{stdout}\nstderr:\n{stderr}" - - -def _read_limited_text(handle, limit: int) -> tuple[str, bool]: - """Read a bounded amount of UTF-8-ish text from a binary file handle.""" - handle.seek(0) - max_bytes = limit * 4 - raw = handle.read(max_bytes + 1) - bytes_truncated = len(raw) > max_bytes - text = raw[:max_bytes].decode("utf-8", errors="replace") - text, chars_truncated = _truncate_text(text, limit) - return text, bytes_truncated or chars_truncated - - -def _truncate_text(text: str, limit: int) -> tuple[str, bool]: - """Return bounded output and whether it was truncated.""" - if len(text) <= limit: - return text, False - marker = "...[truncated]" - if limit <= len(marker): - return marker[:limit], True - return f"{text[: limit - len(marker)]}{marker}", True From db140a37f2dbee5f61f9cf57609de536b8d07b6e Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Wed, 19 Aug 2026 22:23:33 -0400 Subject: [PATCH 16/30] docs: approve image support plan --- .plans/42-image-inputs.md | 389 ++++++++++++++++++++++++++++++++++++++ .plans/image-inputs.md | 26 --- 2 files changed, 389 insertions(+), 26 deletions(-) create mode 100644 .plans/42-image-inputs.md delete mode 100644 .plans/image-inputs.md diff --git a/.plans/42-image-inputs.md b/.plans/42-image-inputs.md new file mode 100644 index 0000000..9e8ed74 --- /dev/null +++ b/.plans/42-image-inputs.md @@ -0,0 +1,389 @@ +# ThinHarness image support plan + +Status: Revised after review panel v1. No implementation is authorized by this plan. + +## Goal + +Add first-class image input without expanding ThinHarness into a general media runtime. + +ThinHarness will support ordered text and image content in: + +- user prompts +- custom tool results +- the filesystem plugin through an opt-in image-reading tool +- successful MCP tool results + +OpenAI, Anthropic, and OpenRouter models will receive the same neutral content through provider-specific wire formats. Resume remains self-contained and portable across built-in providers. + +## Non-goals + +- image generation or assistant-produced images +- audio, video, files, or realtime voice +- remote image URLs or URL fetching +- image editing, resizing, OCR, or format conversion +- automatic image loading from Bash output +- model-name tables that predict vision support +- token-by-token streaming +- image-bearing subagent tasks or `parallel_llm` prompts; child task inputs stay text-only + +## Product decisions + +### One neutral content seam + +Add a small leaf module, `thinharness/content.py`, with immutable content blocks: + +```python +@dataclass(frozen=True) +class TextBlock: + text: str + +@dataclass(frozen=True) +class ImageBlock: + data: bytes + media_type: Literal[ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ] + +ContentBlock = TextBlock | ImageBlock +Prompt = str | Sequence[ContentBlock] +``` + +Normalize a string prompt to one `TextBlock` at the public interface. Special-case `str` before sequence handling. Copy caller sequences to tuples on entry and preserve block order. + +Use immutable `bytes` plus an explicit media type as the only public image form. Do not accept `bytearray`, `memoryview`, paths, URLs, or provider-native image dictionaries. Provider adapters encode base64. Give `ImageBlock` a redacted `repr` that never prints image data. + +Reject empty string prompts, empty block sequences, empty text blocks, empty image data, unsupported media types, and unknown block values before the first provider request. Empty strings are accepted today, so this is an explicit behavior change. Do not inspect model names for vision support. A provider or custom model reports that a selected model cannot process images. + +### Prompt behavior + +`Harness.run()`, `stream()`, and `run_sync()` accept `Prompt`. Approval resume does not accept new user content. + +`RunStartContext.prompt` and `UserPromptSubmitContext.prompt` change from `str` to normalized content-block tuples. A hook may replace either field with a string or a valid block sequence; the harness normalizes and validates the replacement after each hook phase. `additional_context` stays `list[str]` and appends one final `TextBlock`. Harness notices remain separate notice transcript entries and append after hook context. The order sent to a provider is: + +1. caller blocks +2. hook context +3. harness notices + +At the provider boundary, an all-text user entry joins its blocks with `\n\n`. This keeps the existing string payload for a normal string prompt plus hook context. A mixed image entry preserves every block boundary and order. + +Rename the public custom-session method `continue_with_user_text` to `continue_with_user_content`. Update all unit and end-to-end fakes. This is a pre-1.0 interface change and avoids keeping a misleading name. + +### Tool-result behavior + +Allow `ToolResult.content` to be either a string or an ordered sequence of `ContentBlock` values. A bare block sequence returned by a handler is also accepted and becomes `ToolResult(ok=True, content=...)`. Normalize before provider serialization. Existing text-only tools continue to expose string content to callers. + +A normalized multimodal tool result keeps: + +- `ok` as the execution outcome +- ordered text and image content +- JSON metadata, including retry guidance, visible to the model + +Keep the current text-only provider output byte-for-byte: one JSON string with `ok`, string `content`, and `metadata`. For an image-bearing result, OpenAI and Anthropic receive a content array with: + +1. one text block containing compact JSON for `{"ok": ..., "metadata": ...}` +2. each `ToolResult.content` block in its original order, mapped to a provider text or image block + +The first block is the multimodal envelope header. It keeps execution state and retry metadata visible without putting image bytes in JSON text. Provider output preserves the tool-call ID and block order. + +`ToolOutput`, tool-call records, and v4 resume entries carry the normalized `ToolResult`, not only a serialized output string. The canonical JSON codec represents image data exactly once. + +Keep `AfterToolCallContext.original_output` and `output` as canonical JSON strings, including base64 image fields, and keep `envelope` as the structured mutation interface. Mutating either `output` or `envelope` updates the other and runs strict canonical validation. Document these hook fields as sensitive and potentially large. Remove every `json.dumps(..., default=str)` path that could stringify an unknown block. + +`BashPlugin` remains text-only. A command that prints an image path returns that path as text. Bash never reads or infers an image. + +### Provider mappings + +Keep existing text-only payload shapes unchanged. Use multimodal arrays only when content contains an image. + +#### OpenAI Responses + +- User text maps to `input_text`. +- User images map to `input_image` with a base64 data URL. +- Image-bearing tool results use the documented `function_call_output.output` content array. The envelope header and tool text map to `input_text`; images map to `input_image`. +- This array support is confirmed by the official [OpenAI function-calling guide](https://developers.openai.com/api/docs/guides/function-calling). Keep one live contract journey because SDK and model behavior can still differ. + +#### Anthropic Messages + +- Text maps to a `text` block. +- Images map to base64 `image` source blocks. +- Tool-result text and images stay inside the matching `tool_result.content` block list. + +#### OpenRouter Chat Completions + +OpenRouter documentation is not consistent about image parts in `role="tool"` messages. Use a portable follow-up user-message projection instead of depending on that shape: + +- User text and images map to `text` and `image_url` content parts. +- Each normal tool message keeps its `tool_call_id` and contains one canonical JSON string. For image-bearing content, image data becomes a descriptor with media type, byte size, and block index; `ok`, text content, and metadata remain in this tool message. +- After all matching tool messages in a parallel batch, add one `role="user"` message. For each tool image in tool-result order, add a label text part, `[tool image call_id= block=]`, immediately followed by its `image_url` part. +- The labelled user message is a provider wire projection only. The durable transcript keeps the neutral tool result, so live and resumed rendering use the same rule. +- A harness notice follows the labelled image parts in the same user message. + +The OpenRouter live journey must verify this projection with a current vision-capable model. + +### Resume state + +Change user transcript content to canonical content-block arrays. Change each tool transcript entry from an output string to `call_id` plus canonical `ok`, `content`, and `metadata` fields. Encode images in resume JSON as: + +```json +{ + "type": "image", + "media_type": "image/png", + "data": "" +} +``` + +Text blocks use `{"type": "text", "text": "..."}`. + +Bump the built-in transcript resume version from 3 to 4. Reject older versions with the existing regenerate-state guidance. Keep the approval envelope version unchanged because it already contains independently versioned provider state. An approval envelope captured before this change fails on resume with `approval state provider_state version 3 is not supported`; test and document this release break. + +Resume stays self-contained. Image bytes therefore appear as base64 and add about 33% encoding overhead. Document resume state and completed results as sensitive and potentially large. + +Cross-provider resume must preserve block order and image bytes. Same-provider resume must continue to preserve native reasoning data. + +### Tracing and events + +Never put image bytes, base64, or data URLs into trace attributes or non-terminal progress events, including when text message capture is enabled. + +Project each image as a descriptor: + +```json +{ + "type": "image", + "media_type": "image/png", + "size_bytes": 12345, + "block_index": 1 +} +``` + +Keep text blocks visible under the existing capture settings. Preserve content order and add each image block's zero-based index to event and trace projections. + +Keep current event field types and text-only values unchanged. For multimodal values, `RunStartedEvent.prompt` and `ToolCallCompletedEvent.output` contain compact redacted JSON strings. `ToolCallCompletedEvent.message` stays a plain text summary and never receives a block sequence. Apply the same projection before writing `gen_ai.prompt` and `gen_ai.tool.call.result` trace attributes. Cover `core.py`, `tool_execution.py`, subagent tracing, and transcript rendering as well as the projection and event modules. + +`RunCompletedEvent.result` remains the same complete result returned by `run()`. It can contain image-bearing resume state and tool-call records. Document this terminal object as sensitive instead of weakening result identity. + +Before creating a provider error event or span error, replace each run-known full base64 value and data URL with `[image data redacted]`. Also redact any complete `data:image/...;base64,...` value found by pattern. Provider adapters must not include serialized request bodies in error messages. + +### Filesystem plugin + +Add an opt-in `read_image` tool. Do not change the text `read` tool and do not add `read_image` to the filesystem plugin's default tool set. + +`read_image` will: + +- reuse the filesystem plugin's read path policy +- read one local file as bounded bytes +- detect PNG, JPEG, GIF, or WebP from file signatures +- return compact metadata as one `TextBlock`, followed by one `ImageBlock` +- reject directories, missing files, path escapes, unknown formats, SVG, and oversized files + +Add a frozen `max_image_bytes` filesystem-plugin setting with a default of `5_000_000`. Require a positive integer. This limit is independent of `max_read_bytes`; exactly the limit is accepted and one byte over is rejected. Do not add Pillow or another image dependency. Validate the full required signature and minimum header length, not only the first magic bytes. The tool validates container format only; providers validate dimensions and model-specific limits. + +`read_image` returns `ToolResult(ok=True, content=[TextBlock(metadata), ImageBlock(...)], metadata=...)`. The compact metadata text comes first and includes the path, media type, and byte size. + +### MCP plugin + +Preserve successful MCP image blocks as `ImageBlock` values in their original order. If `structuredContent` is present, it remains the authoritative text as required by current behavior: emit its canonical JSON as the first `TextBlock`, discard MCP text blocks, then append image blocks and image placeholders from `result.content` in their original relative order. + +A supported media type with valid base64 becomes an `ImageBlock`. Malformed base64 and unsupported image media types become the existing `[image: ]` text placeholder instead of failing the tool call. Continue converting audio, embedded resources, and resource links to text placeholders. + +Protocol-level MCP errors remain text-only retry results. + +## Behavior documentation + +After plan approval and before implementation, update only affected sections of `docs/behavior.md`: + +- add a new image-content section covering public blocks, validation, ordering, provider behavior, tool results, and exclusions +- update Resume State for transcript version 4 and self-contained image data +- update Model Observability Projections for redacted image descriptors +- update Filesystem Plugin for opt-in `read_image` and `max_image_bytes` +- update MCP Client Layer so successful image blocks remain images +- update hook and stream requirements where they currently assume string prompts or outputs + +## Implementation sequence + +### 1. Add and test neutral content types + +Create `thinharness/content.py` with public types, normalization, validation, canonical JSON encoding, canonical JSON decoding, and redacted descriptor helpers. + +Export public content types from `thinharness/__init__.py`. + +Prove that string prompts and text-only tool results retain their current observable behavior. + +### 2. Move the run and hook interfaces to content + +Update: + +- `thinharness/core.py` +- `thinharness/hooks.py` +- `thinharness/turns.py` +- `thinharness/runtime.py` +- `thinharness/events.py` + +Pass normalized content from public entry points to `ModelSession`. Append hook context and notices as text blocks without changing image order. Add redacted run-start event and trace projection in this phase, so the type change cannot expose image data. + +Update custom-model protocols and test fakes to use `continue_with_user_content`. + +### 3. Move tool execution to content + +Update: + +- `thinharness/tools/base.py` +- `thinharness/tool_execution.py` +- after-tool hook handling +- tool-call records + +Keep all existing text tools returning strings. Add multimodal normalization only at the shared tool seam. Define the exact envelope-header serializer and redact tool completion events and trace attributes in this phase. + +### 4. Implement provider serializers + +Update exact request construction in `thinharness/providers.py` for: + +- initial multimodal prompts +- resumed prompts +- mixed text/image tool results +- parallel tool-result batches +- notices added after multimodal content + +Keep text-only provider request payloads unchanged. Implement the decided OpenRouter labelled user-message projection in both live tool continuation and transcript replay. + +### 5. Upgrade transcript resume + +Update transcript entries, codecs, validation, provider transcript renderers, and approval-resume coverage for version 4. + +Test same-provider and cross-provider resume with mixed text and images. + +### 6. Redact observability surfaces + +Audit and complete redaction in: + +- `thinharness/core.py` +- `thinharness/tool_execution.py` +- `thinharness/projections.py` +- `thinharness/tracing.py` +- `thinharness/events.py` +- `thinharness/plugins/subagents.py` +- `scripts/build_transcripts.py` + +Ensure no trace or non-terminal event contains base64 or data URLs. Earlier phases add redaction at each new image-bearing seam; this phase is the full leak audit. + +### 7. Add image-producing built-ins + +Add opt-in filesystem `read_image`, then preserve MCP images. Keep Bash unchanged. + +### 8. Update documentation and release notes + +Update: + +- `README.md` +- `docs/docs.md` +- `CHANGELOG.md` +- API exports and examples + +Version changes follow the normal release process rather than this implementation plan. + +## Tests + +Tests must enter through public run, tool, resume, and plugin interfaces. Provider HTTP handlers are the only mocked boundary. Exact expected payloads must be literal fixtures derived from provider formats, not generated by production serializers. + +### Content contract + +- normalize string prompts to one text block +- preserve mixed block order +- reject empty and unsupported content, including the newly invalid empty string prompt +- detach caller-owned sequences and reject mutable byte containers +- redact `ImageBlock.__repr__` +- keep `thinharness/content.py` independent of core, providers, and tools +- round-trip canonical text and image JSON + +### Provider contracts + +For OpenAI, Anthropic, and OpenRouter: + +- send an initial image prompt +- send interleaved text and images +- continue after a text tool result without changing existing payload shape +- continue after an image tool result while keeping `ok` and metadata model-visible +- preserve several parallel tool results and their call IDs +- keep every existing text-only prompt and tool envelope request body byte-for-byte +- append notices after image content +- migrate both prompt hook fields and normalize valid hook replacements +- surface the normal provider error for a non-vision model + +### Resume + +- same-provider image replay for all built-in providers +- every cross-provider pair, including replay of a foreign tool image through OpenRouter's labelled projection +- mixed user and tool images +- preserved native reasoning on same-provider replay +- malformed base64, wrong keys, unsupported media types, and old versions fail clearly +- approval pause and resume with image-bearing prior state +- a pre-change approval envelope fails with the documented nested provider-state version error + +### Tool behavior + +- custom sync and async tools return image content +- image content survives mutation through both after-tool hook fields +- malformed hook JSON and unknown block values fail strict canonical validation +- failed and retryable image results keep `message` text-only and keep retry metadata visible to the model +- failed and retryable tools remain text-only unless explicitly returned otherwise +- parallel mixed text/image tool batches preserve order +- Bash never auto-loads a printed image path + +### Filesystem and MCP + +- `read_image` is opt-in and absent from defaults +- allowed PNG, JPEG, GIF, and WebP signatures +- truncated headers and files with only spoofed leading magic bytes fail +- exactly `max_image_bytes` succeeds and one byte over fails +- missing file, directory, unreadable file, path escape, and symlink escape +- plugin configuration remains frozen and binding performs no image I/O +- MCP preserves mixed text/image order; audio and resources remain placeholders +- MCP `structuredContent` plus images follows the declared precedence +- malformed MCP base64 and unsupported image media types become placeholders + +### Observability + +- traces preserve text and image descriptors in order +- traces and progress events contain no image bytes, base64, or data URLs +- provider errors cannot leak known base64 or data URLs through `RunFailedEvent` or span errors +- `RunStartedEvent`, `ToolCallCompletedEvent`, subagent traces, and transcript renderers expose descriptors only +- completed results retain full resume state + +### Live validation + +Add an opt-in image journey for each built-in provider. Each journey must prove that a real vision-capable model can: + +1. answer a question about a supplied image +2. inspect an image returned by a custom tool +3. resume from the resulting state + +The OpenRouter journey verifies the labelled user-message image projection. The OpenAI journey verifies the documented `function_call_output.output` content-array contract. + +## Validation commands + +After implementation: + +```bash +uv run ruff check thinharness tests +uv run pytest tests/unit +uv run pyright +``` + +Run relevant provider journeys when credentials are available. Do not require paid live calls in normal CI. + +## Main risks + +- **OpenRouter attribution:** The portable labelled user-message projection keeps call IDs visible but cannot give the image a native tool role. +- **Data leakage:** Existing trace, event, hook, and error paths assume text and may expose base64 unless every projection is updated. +- **State growth:** Self-contained resume and tool records can become large. +- **Interface spread:** Prompt and tool content currently use strings across most core seams. +- **False support claims:** A provider protocol can support images while a selected model does not. +- **Text regressions:** Multimodal normalization must not alter current text-only payloads or tool envelopes. + +## Review decision + +Review panel v1 requested changes. This revision resolves the blocking provider projections, tool envelope, hook mutation, MCP precedence, event redaction, and filesystem limit decisions. + +The plan is ready for human approval. After approval, update `docs/behavior.md` with this contract before implementation starts. diff --git a/.plans/image-inputs.md b/.plans/image-inputs.md deleted file mode 100644 index 81dd4dd..0000000 --- a/.plans/image-inputs.md +++ /dev/null @@ -1,26 +0,0 @@ -# Image Inputs - -ThinHarness should support images by making model input richer while keeping plain text prompts unchanged. The public API should continue to accept `Harness.run("prompt")`, and additionally accept an ordered sequence of input parts such as text plus image content. - -The core type should be provider-neutral: - -```python -UserInput = str | Sequence[UserInputPart] -UserInputPart = TextPart | ImageUrlPart | BinaryImagePart -``` - -`TextPart` can stay optional at first because bare strings inside the sequence are enough for most callers. `ImageUrlPart` should hold a URL, optional MIME type, optional provider metadata, and an optional `force_download` flag. `BinaryImagePart` should hold bytes, MIME type, and optional provider metadata, with helpers such as `from_path(...)` and `from_data_uri(...)`. - -The harness should normalize prompt handling once near the run boundary. Hooks, tracing, stream events, and model session APIs should receive the neutral input shape instead of assuming a string. Text-only callers should see the same behavior and payloads they see today. - -Provider adapters should own wire-format mapping: - -- OpenAI Responses: map text to `input_text` and images to `input_image`; binary images should use data URIs, URL images should use URLs unless `force_download` is set. -- Anthropic Messages: map text to text blocks and images to image blocks; binary images should use base64 source blocks, URL images should use URL source blocks when supported. -- OpenRouter chat completions: map text and images to chat content parts using `text` and `image_url`; binary images should use data URIs. - -Unsupported image cases should fail loudly with a provider error. The harness should not silently stringify images or drop them. - -Keep the first implementation limited to images. Do not include audio, video, PDFs, uploaded provider files, or prompt-cache markers yet. The type shape should leave room for those later, but the code should not implement speculative modalities. - -Tests should cover backward-compatible text prompts, mixed text/image ordering, URL image mapping, binary image mapping, provider-specific unsupported cases, notice appending with rich input, and tracing redaction or placeholder behavior so raw image bytes are not accidentally written into local traces. From 4e2039233cdb25c80ea1aceafd9c03e161868f8d Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Wed, 19 Aug 2026 22:48:27 -0400 Subject: [PATCH 17/30] Add provider-neutral image inputs --- CHANGELOG.md | 2 + README.md | 17 +- assets/thinharness-run-loop.drawio | 4 +- assets/thinharness-run-loop.svg | 2 +- docs/behavior.md | 29 +- docs/docs.md | 33 +- docs/site/assets/thinharness-run-loop.svg | 2 +- docs/site/explainer/index.html | 4 +- tests/e2e/image_inputs_journey.py | 138 +++++++++ tests/e2e/mcp_journey.py | 2 +- tests/unit/fakes.py | 20 +- tests/unit/test_approvals.py | 4 +- tests/unit/test_file_tools.py | 1 + tests/unit/test_harness.py | 4 +- tests/unit/test_hooks.py | 2 +- tests/unit/test_image_inputs.py | 260 ++++++++++++++++ tests/unit/test_mcp.py | 15 +- tests/unit/test_parallel_llm.py | 4 +- tests/unit/test_plugins.py | 2 +- tests/unit/test_providers.py | 38 +-- tests/unit/test_reasoning_fidelity.py | 2 +- tests/unit/test_resume.py | 10 +- tests/unit/test_streaming.py | 2 +- tests/unit/test_structured_output.py | 8 +- tests/unit/test_subagents.py | 2 +- tests/unit/test_tool_retry.py | 8 +- tests/unit/test_tracing.py | 7 +- tests/unit/test_turns.py | 4 +- thinharness/__init__.py | 5 + thinharness/content.py | 171 +++++++++++ thinharness/core.py | 61 ++-- thinharness/hooks.py | 16 +- thinharness/plugins/filesystem.py | 6 + thinharness/projections.py | 33 +- thinharness/providers.py | 354 ++++++++++++++++------ thinharness/runtime.py | 20 +- thinharness/tool_execution.py | 17 +- thinharness/tools/base.py | 97 ++++-- thinharness/tools/filesystem.py | 67 ++++ thinharness/tools/mcp.py | 59 +++- thinharness/tracing.py | 18 +- thinharness/turns.py | 25 +- 42 files changed, 1304 insertions(+), 271 deletions(-) create mode 100644 tests/e2e/image_inputs_journey.py create mode 100644 tests/unit/test_image_inputs.py create mode 100644 thinharness/content.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9263e73..903a084 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Added ordered text and image prompts and tool results for OpenAI, Anthropic, and OpenRouter, with provider-neutral immutable content blocks, redacted observability projections, self-contained version 4 resume state, opt-in filesystem `read_image`, and preserved successful MCP images. +- **Breaking:** Renamed custom `ModelSession.continue_with_user_text(...)` to `continue_with_user_content(...)`; prompt hooks now receive normalized content-block tuples, and built-in transcript resume version 3 state must be regenerated. - Added explicit plugin composition with static and connected contributions, atomic connection rollback, unique plugin names, generic tool origin, and plugin-provided hooks and instructions. - Added `FilesystemPlugin` for the ordered workspace tool surface; `jsonl_search` remains opt-in through this plugin. - Added `BashPlugin` for explicit one-shot local Bash commands with strict arguments, contained cwd, minimal environment inheritance, bounded separate head-and-tail output, host-capped timeouts, process-group cleanup, and cancellation propagation. diff --git a/README.md b/README.md index 1cafad2..c235e12 100644 --- a/README.md +++ b/README.md @@ -256,6 +256,19 @@ asyncio.run(main()) There's a synchronous wrapper too: `Harness(...).run_sync(...)`. +Prompts can contain ordered local text and images: + +```python +from thinharness import ImageBlock, TextBlock + +result = await harness.run([ + TextBlock("Describe this image."), + ImageBlock(open("diagram.png", "rb").read(), "image/png"), +]) +``` + +Supported image types are JPEG, PNG, GIF, and WebP. ThinHarness does not fetch image URLs or infer vision support from model names. + Optional MCP servers use the same plugin composition model: ```python @@ -337,7 +350,7 @@ Streaming emits coarse run, model, tool, retry, limit, and subagent events, then ## Features -- **Filesystem plugin:** explicit `FilesystemPlugin` composition for `read`, `write`, batched exact-replacement `edit`, `search`, `list`, and `glob` with root-scoped path policies. +- **Filesystem plugin:** explicit `FilesystemPlugin` composition for `read`, `write`, batched exact-replacement `edit`, `search`, `list`, and `glob`, plus opt-in bounded `read_image`, with root-scoped path policies. - **JSONL search:** opt-in `jsonl_search` for structured line-delimited data, with ripgrep prefiltering, field projection, equality/contains/regex/range `where` filters, and field-level snippets from large multiline string values. - **Bash plugin:** explicit `BashPlugin` composition for one-shot non-interactive commands with contained cwd, filtered environment, bounded output, timeouts, cancellation cleanup, and optional approval. - **Provider adapters:** built-in OpenAI, Anthropic, and OpenRouter adapters, plus public model/session protocols for implementing another provider. @@ -347,7 +360,7 @@ Streaming emits coarse run, model, tool, retry, limit, and subagent events, then - **Subagents:** explicit `SubagentsPlugin` composition with a default child, ordered named `SubAgentConfig` recipes, additive safe-plugin inheritance, local child hooks, and no recursive delegation. - **Parallel LLM:** explicit `ParallelLlmPlugin` fan-out for batches of independent one-shot prompts, plus `ParallelLlmTool(...).spec()` for renameable or structured tools with explicit model, path, prompt, and provider request settings. - **Skills:** explicit `SkillsPlugin` composition with an ordered `skill_read` and/or `skill_run` selection, plus Python, shell, JavaScript, and Go script runners. -- **Resume:** clean new-turn continuation through self-contained transcript state that can replay across built-in providers and models, preserving native reasoning on same-provider resume and degrading it to text across providers. +- **Resume:** clean new-turn continuation through self-contained transcript state that can replay text and images across built-in providers and models, preserving native reasoning on same-provider resume and degrading it to text across providers. - **MCP:** optional MCP support built on the FastMCP client, including in-process servers via `FastMCPTransport`, with lazy tool discovery and collision checks. - **Parallel tool calls:** same-turn tool batches run concurrently when every called tool is parallel-safe. - **Human approvals:** mark custom tools as approval-required so a run pauses before side effects, returns pending call details plus resume state, then continues after an approve/reject decision. diff --git a/assets/thinharness-run-loop.drawio b/assets/thinharness-run-loop.drawio index 1f941cb..9eb88ad 100644 --- a/assets/thinharness-run-loop.drawio +++ b/assets/thinharness-run-loop.drawio @@ -88,7 +88,7 @@ - + @@ -118,7 +118,7 @@ - + diff --git a/assets/thinharness-run-loop.svg b/assets/thinharness-run-loop.svg index 6ca53e9..aa8d907 100644 --- a/assets/thinharness-run-loop.svg +++ b/assets/thinharness-run-loop.svg @@ -1,3 +1,3 @@ -
    harness/runtime
    provider session
    decision
    approval pause
    tool path
    final result
    retry / stop
    stream event
    Typed event stream
    StreamEvent emitted at each blue dot; contains kind, run_id, and event number
    SETUP
    ONE MODEL TURN
    BRANCH HANDLING INSIDE THE WHILE LOOP
    Entry point
    Harness.stream()
    core.py:305-374
    outer stream flow
    Create run objects
    RunContext
    StreamEmitter
    core.py:405-418
    per-run state
    trace scope
    Open session
    new_session()
    resume_session()
    core.py:448-491
    one active session
    Loop entry
    RunContext.advance_model()
    runtime.py:268
    limits, notices, tracing
    provider request
    Call provider session
    ModelSession
    providers.py:184
    start()
    continue_with_tools()
    continue_with_user_text()
    Model turn
    ModelTurn
    providers.py:56
    text
    tool calls
    raw JSON
    Resolve turn
    resolve_turn_output()
    turns.py:50
    OutputTurnDecision
    chooses next branch
    Approval required?
    _approval_required_calls()
    turns.py:322
    before tool batch executes
    Final result
    RunContext.finalize()
    runtime.py:420
    builds HarnessResult
    fires run_end once
    Approval pause
    pause_for_approval()
    runtime.py:367
    stop_reason="approval_required"
    pending approvals + resume_state
    no tools executed
    Run tools
    ToolBatchExecutor.execute_batch()
    tool_execution.py:63
    runs ordinary tool calls
    preserves model order
    Return tool outputs
    Build ToolOutput[]
    providers.py:97
    model-visible results
    Retry decision
    RunContext.retry_or_fail()
    runtime.py:448
    budget remains or stop
    Stop / raise
    core.py:499-510
    HarnessError
    limit, provider,
    or validation
    Corrective request
    continue_with_tools()
    continue_with_user_text()
    asks model to fix output
    interpret turn
    final
    no approval
    requires approval
    tool batch
    schema retry
    next iteration
    budget remains
    no retry
    next iteration
    Text is not SVG - cannot display
    \ No newline at end of file +
    harness/runtime
    provider session
    decision
    approval pause
    tool path
    final result
    retry / stop
    stream event
    Typed event stream
    StreamEvent emitted at each blue dot; contains kind, run_id, and event number
    SETUP
    ONE MODEL TURN
    BRANCH HANDLING INSIDE THE WHILE LOOP
    Entry point
    Harness.stream()
    core.py:305-374
    outer stream flow
    Create run objects
    RunContext
    StreamEmitter
    core.py:405-418
    per-run state
    trace scope
    Open session
    new_session()
    resume_session()
    core.py:448-491
    one active session
    Loop entry
    RunContext.advance_model()
    runtime.py:268
    limits, notices, tracing
    provider request
    Call provider session
    ModelSession
    providers.py:184
    start()
    continue_with_tools()
    continue_with_user_content()
    Model turn
    ModelTurn
    providers.py:56
    text
    tool calls
    raw JSON
    Resolve turn
    resolve_turn_output()
    turns.py:50
    OutputTurnDecision
    chooses next branch
    Approval required?
    _approval_required_calls()
    turns.py:322
    before tool batch executes
    Final result
    RunContext.finalize()
    runtime.py:420
    builds HarnessResult
    fires run_end once
    Approval pause
    pause_for_approval()
    runtime.py:367
    stop_reason="approval_required"
    pending approvals + resume_state
    no tools executed
    Run tools
    ToolBatchExecutor.execute_batch()
    tool_execution.py:63
    runs ordinary tool calls
    preserves model order
    Return tool outputs
    Build ToolOutput[]
    providers.py:97
    model-visible results
    Retry decision
    RunContext.retry_or_fail()
    runtime.py:448
    budget remains or stop
    Stop / raise
    core.py:499-510
    HarnessError
    limit, provider,
    or validation
    Corrective request
    continue_with_tools()
    continue_with_user_content()
    asks model to fix output
    interpret turn
    final
    no approval
    requires approval
    tool batch
    schema retry
    next iteration
    budget remains
    no retry
    next iteration
    Text is not SVG - cannot display
    \ No newline at end of file diff --git a/docs/behavior.md b/docs/behavior.md index 2cc195c..2b13e4c 100644 --- a/docs/behavior.md +++ b/docs/behavior.md @@ -63,6 +63,21 @@ Use this section only when ordering, lifecycle, concurrency, retries, streaming, - JSONL-TYPED-EQUALITY-4: Non-comparable row values do not match either `eq` or `ne` and increment `compare_warnings` once per candidate row where a typed equality comparison was attempted. - JSONL-TYPED-EQUALITY-5: Invalid typed equality filter definitions fail before scanning rows with `invalid where filter`. +## Image Content + +### Purpose + +ThinHarness accepts ordered text and local image content through one provider-neutral interface while keeping text-only behavior unchanged. + +### Requirements + +- IMAGE-CONTENT-1: `TextBlock`, `ImageBlock`, and `Prompt` are the public content interface. Prompts accept a non-empty string or a non-empty copied sequence of immutable blocks. Text and image data must be non-empty, image data must be `bytes`, and media types are limited to JPEG, PNG, GIF, and WebP. +- IMAGE-CONTENT-2: `Harness.run()`, `stream()`, and `run_sync()` normalize content before the first provider request. Run-start and user-prompt hooks receive normalized block tuples and can replace them with a valid string or block sequence. Caller blocks, hook context, and harness notices remain in that order. +- IMAGE-CONTENT-3: Tool results accept a string or ordered block sequence. Text-only provider payloads remain unchanged. Image-bearing provider payloads preserve execution outcome, metadata, tool-call id, content order, and images through provider-native OpenAI and Anthropic blocks. +- IMAGE-CONTENT-4: OpenRouter keeps a canonical JSON tool message for each result. Tool images are descriptors in that message and are projected after the parallel tool-message batch as labelled user content, with each label immediately before its image and any harness notice after all image parts. +- IMAGE-CONTENT-5: After-tool hook string fields contain canonical JSON, including base64 image fields, while the envelope is the structured mutation interface. Either mutation path is strictly validated and synchronized. These fields, completed results, and resume state can be sensitive and large. +- IMAGE-CONTENT-6: Remote image URLs, image fetching, image generation, other media, conversion, OCR, image-bearing subagent tasks, and image-bearing `parallel_llm` prompts are not supported. Bash output always stays text and does not load image paths. Model image capability errors come from the selected provider or custom model, not model-name checks. + ## Resume State ### Purpose @@ -75,10 +90,11 @@ Built-in provider resume state is a self-contained, provider-agnostic transcript - RESUME-2: `resume_state` is self-contained and does not depend on provider continuation tokens such as OpenAI `previous_response_id`; an OpenAI run that never received a response id is still resumable. - RESUME-3: Resuming on the originating provider preserves native reasoning (Anthropic thinking signatures, OpenAI `encrypted_content`, OpenRouter `reasoning_details`); resuming on a different provider degrades each reasoning part to a leading ``-tagged text block and drops the opaque blob. Native re-emit additionally requires the resuming run to be able to accept the block: OpenAI re-emits the native reasoning item only when the resuming model is reasoning-capable, and Anthropic uses the thinking gate in RESUME-3A; otherwise both use the text fallback. So a reasoning-model capture resumed on a non-reasoning model of the same provider degrades to text. - RESUME-3A: Anthropic resume treats explicit `extra_body["thinking"]` as authoritative: `enabled` and `adaptive` accept signed thinking replay, while `disabled`, unknown, or malformed values suppress native replay. Without an explicit thinking key, `HarnessConfig.effort` implies adaptive thinking; otherwise Anthropic models outside the legacy off-by-default families (`claude-opus-4`, `claude-sonnet-4`, `claude-haiku-4`, and `claude-3`) are assumed to run thinking by default and keep signed thinking blocks on resume. -- RESUME-4: Built-in provider resume state uses `version` 3; version 1 and version 2 state and old provider-native `kind` values are rejected with a regenerate error. +- RESUME-4: Built-in provider resume state uses `version` 4. User and tool entries carry canonical ordered content blocks; image bytes are stored once as base64. Older state and old provider-native `kind` values are rejected with a regenerate error. An approval envelope that contains version 3 provider state fails with `approval state provider_state version 3 is not supported`. - RESUME-5: On resume, the live system prompt from the resuming harness config is re-injected; captured system prompts are not stored or restored. - RESUME-6: A session seeded via `OpenAIResponsesSession.start(prompt, constants, previous_response_id=...)` captures only new transcript entries, so externally seeded prior turns are not present when later resumed from `resume_state`. This is unrelated to reasoning fidelity and is not changed by RESUME-3/RESUME-7. -- RESUME-7: For reasoning-capable OpenAI Responses models the harness requests `include=["reasoning.encrypted_content"]` so reasoning survives resume; non-reasoning models are unaffected. Captured `resume_state` therefore contains encrypted reasoning blobs (OpenAI/OpenRouter) and signed thinking (Anthropic) and should be treated as sensitive, consistent with the local-trace sensitivity note. +- RESUME-7: For reasoning-capable OpenAI Responses models the harness requests `include=["reasoning.encrypted_content"]` so reasoning survives resume; non-reasoning models are unaffected. Captured `resume_state` can contain image bytes with base64 overhead, encrypted reasoning blobs (OpenAI/OpenRouter), and signed thinking (Anthropic), so it must be treated as sensitive and potentially large. +- RESUME-8: Cross-provider and same-provider resume preserve text/image order and exact image bytes. Same-provider replay continues to preserve accepted native reasoning data. ## Plugin Composition @@ -149,12 +165,14 @@ Callers opt into root-scoped workspace tools without making filesystem behavior ### Requirements - FILESYSTEM-PLUGIN-1: `Harness` has no implicit filesystem tools. `FilesystemPlugin` provides `read`, `write`, `edit`, `search`, `list`, and `glob` by default; callers select an ordered subset explicitly. -- FILESYSTEM-PLUGIN-2: `jsonl_search` is an opt-in tool of `FilesystemPlugin` and shares its root, read policy, search process, truncation, and spill-output handling. +- FILESYSTEM-PLUGIN-2: `jsonl_search` and `read_image` are opt-in tools of `FilesystemPlugin`. `read_image` shares the read path policy but is not part of the default tool set and does not change the text-only `read` tool. - FILESYSTEM-PLUGIN-3: `HarnessConfig.root` is the one run root. `FilesystemPlugin` uses that root and cannot configure a different root. - FILESYSTEM-PLUGIN-4: Harness construction and plugin binding do not create the workspace root. A harness without `FilesystemPlugin` has a generic default prompt, adds no workspace-root instruction, and has no workspace filesystem side effect. Observability sinks keep their independent configured storage behavior. - FILESYSTEM-PLUGIN-5: Filesystem limits, output location, search settings, and path policies belong to `FilesystemPlugin`. - FILESYSTEM-PLUGIN-6: Independent custom tools continue to use `tools=[ToolSpec(...)]`; callers do not need to wrap one tool in a plugin. - FILESYSTEM-PLUGIN-7: `FilesystemPlugin` has the runtime-fixed name `"filesystem"`. Its constructor configuration is frozen: mutation of constructor inputs, returned property values, or plugin attributes cannot change later parent or child bindings. +- FILESYSTEM-PLUGIN-8: `read_image` reads one bounded local file, validates complete minimum JPEG, PNG, GIF, or WebP signatures, and returns metadata text followed by one image block. It rejects missing files, directories, unreadable files, path or symlink escapes, unknown formats, SVG, truncated headers, and files over `max_image_bytes`. +- FILESYSTEM-PLUGIN-9: `max_image_bytes` is a positive frozen setting that defaults to 5,000,000 and is independent of `max_read_bytes`; exactly the limit is accepted. Plugin binding performs no image I/O. ## Skills Plugin @@ -283,6 +301,9 @@ Tracing and streaming expose projections of the same neutral per-request model-v - MODEL-OBSERVABILITY-6: Core tracing emits OTel/GenAI-oriented attributes and does not include sink-specific display namespaces. - MODEL-OBSERVABILITY-7: Model spans pin `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.usage.cache_read.input_tokens`, `gen_ai.usage.total_tokens`, `gen_ai.response.model`, and `gen_ai.response.finish_reasons`. `cache_read.input_tokens` carries provider-reported cached input tokens when present and is omitted when unreported. `finish_reasons` is always a list wrapping the normalized reason. `total_tokens` passes through a raw provider `total_tokens` when present and is otherwise computed as input+output only when both are present; partial usage yields no total. - MODEL-OBSERVABILITY-8: Custom `Model` implementations that do not populate normalized `ModelTurn` usage fields keep their `gen_ai.usage.*` span attributes via best-effort extraction from the raw response. +- MODEL-OBSERVABILITY-9: Trace attributes and non-terminal progress events never contain image bytes, base64, or data URLs. Multimodal prompt and tool-result values use compact ordered JSON with visible text and image descriptors containing media type, byte size, and zero-based block index; existing field types and text-only values stay unchanged. +- MODEL-OBSERVABILITY-10: Provider errors and span errors redact run-known image encodings and complete image data URLs. Provider adapters do not include serialized request bodies in error messages. +- MODEL-OBSERVABILITY-11: `ToolCallCompletedEvent.message` remains plain text. `RunCompletedEvent.result` remains identical to the complete run result and can contain sensitive, large image-bearing records and resume state. ## MCP Client Layer @@ -297,7 +318,7 @@ ThinHarness exposes tools from MCP servers through explicit `MCPPlugin` composit - MCP-3: A wrapper owns the FastMCP client built on its transport: nested and concurrent entries share one connection, the final exit closes the transport (terminating a stdio child process), and the same wrapper can reconnect afterwards. One stateful transport object must not be reused across wrappers; reusing the same wrapper across parent, child, or independent harness bindings shares one reference-counted session. - MCP-4: Final close is bounded — the bound comes from FastMCP's `client_disconnect_timeout` setting (default 5 seconds) — and a caller cancellation consumed by transport cleanup is re-raised after cleanup completes. Cancelling a first connection or a final close propagates the cancellation and leaves the wrapper reusable. - MCP-5: `include_tools` and `exclude_tools` match original MCP tool names before prefixing and normalization; `tool_prefix`, schema cleanup, sanitized-name collision errors, and cross-contribution tool collision errors are ThinHarness behavior. Discovered MCP tools are ordinary `ToolSpec` objects with `ToolOrigin(plugin="mcp", source=resolved_server_id, attributes={"tool_name": original_tool_name})`. Tracing reads this origin from the `ToolSpec`, so an after-tool hook cannot erase attribution. Model-visible result metadata uses the same binding-local server id. -- MCP-6: Successful `structuredContent` is returned as a JSON string; text, image, audio, embedded-resource, and resource-link blocks convert in order to model-visible text. A protocol-level tool failure (`isError`) returns a failed `ToolResult` with `error_type="MCPToolError"` and `retry=True`; known transport and protocol failures during a tool call return `error_type="MCPError"`, including when wrapped in an exception group or explicit cause chain — a group whose members are all `Exception`s is normalized when any member's cause chain holds a known failure, even alongside sibling exception noise from teardown. An exception group carrying cancellation or any other non-`Exception` failure propagates, and exceptions with no known failure in their group or cause chain propagate as programming errors. +- MCP-6: Successful supported MCP image blocks with valid base64 remain ordered image blocks. Without `structuredContent`, text and image blocks keep their order. With `structuredContent`, its canonical JSON is the first text block, MCP text blocks are discarded, and images plus image placeholders keep their original relative order. Malformed or unsupported images and audio, embedded resources, and resource links become text placeholders. A protocol-level tool failure (`isError`) returns a text-only failed `ToolResult` with `error_type="MCPToolError"` and `retry=True`; known transport and protocol failures during a tool call return `error_type="MCPError"`, including when wrapped in an exception group or explicit cause chain — a group whose members are all `Exception`s is normalized when any member's cause chain holds a known failure, even alongside sibling exception noise from teardown. An exception group carrying cancellation or any other non-`Exception` failure propagates, and exceptions with no known failure in their group or cause chain propagate as programming errors. - MCP-7: MCP tools and connection details never enter resume state. `MCPPlugin` does not inherit automatically into children. A child that needs MCP lists an explicit `MCPPlugin` in its plugin configuration; that child binding owns its connection lifecycle, while reuse of the same server wrapper keeps the wrapper's reference-counted session behavior. - MCP-8: The base install works without MCP packages: importing ThinHarness and constructing any wrapper or `MCPPlugin` needs no extra, and opening a connection without `mcp` or `fastmcp` raises `MCPDependencyError` with the `thinharness[mcp]` install hint. - MCP-9: `timeout` bounds MCP initialization and HTTP connection establishment; `read_timeout` bounds MCP requests, HTTP reads, and SSE reads. diff --git a/docs/docs.md b/docs/docs.md index f54a5d2..4985259 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -62,7 +62,7 @@ Streaming is coarse turn/tool/run streaming, not token-delta streaming. Provider Stream events are high-level workflow events intended for app consumption: -- `RunStartedEvent.prompt` includes the submitted prompt. +- `RunStartedEvent.prompt` includes the submitted text, or compact ordered JSON with redacted image descriptors for a multimodal prompt. - `ToolCallStartedEvent.arguments` includes the model-requested tool arguments. - `ToolCallCompletedEvent.output` includes model-visible tool output. - Raw provider response JSON is not part of stream events; use `HarnessResult.responses` for raw provider responses after completion. @@ -131,7 +131,18 @@ The core harness has no implicit filesystem tools. Add `FilesystemPlugin()` to g - `list`: list files or directories. - `glob`: find files by glob pattern. -Use the plugin's ordered `tools` list to select a different surface. `jsonl_search` is opt-in: +Use the plugin's ordered `tools` list to select a different surface. `jsonl_search` and `read_image` are opt-in. `read_image` reads a bounded JPEG, PNG, GIF, or WebP file under the read path policy and returns metadata text followed by the image. It does not change the text-only `read` tool. Configure its independent positive byte limit with `max_image_bytes` (default 5,000,000). + +For example: + +```python +harness = Harness( + HarnessConfig(root="."), + plugins=[FilesystemPlugin(tools=["read", "read_image"])], +) +``` + +`jsonl_search` is also opt-in: ```python harness = Harness( @@ -233,7 +244,7 @@ Cwd containment and environment filtering reduce mistakes; they are not a sandbo ## Custom Tools -Custom tools are registered as `ToolSpec` objects. A handler may return a `ToolResult`, a string, or JSON-serializable data. The model always receives a JSON envelope with `ok`, `content`, and `metadata`. +Custom tools are registered as `ToolSpec` objects. A handler may return a `ToolResult`, a string, JSON-serializable data, or an ordered sequence of `TextBlock` and `ImageBlock` values. Text-only results keep the JSON envelope with `ok`, `content`, and `metadata`; image-bearing results use provider-native blocks while keeping `ok` and metadata visible to the model. ```python from pydantic import BaseModel @@ -353,6 +364,12 @@ ThinHarness resolves structured output when the harness is constructed and eager Tool-mode structured output uses a harness-created `final_result` tool. It is not a normal registered tool, does not fire tool hooks, and clean exits through it are not resumable because the provider transcript would contain an unanswered synthetic tool call. +## Image Content + +`Harness.run()`, `stream()`, and `run_sync()` accept a non-empty string or an ordered sequence of `TextBlock` and `ImageBlock` values. Images use immutable bytes with an explicit `image/jpeg`, `image/png`, `image/gif`, or `image/webp` media type. Empty content, unsupported media types, URLs, paths, and mutable byte containers are rejected before provider work. Provider or model errors report unsupported vision models. + +OpenAI, Anthropic, and OpenRouter receive provider-native image parts. OpenRouter projects tool-result images into a labelled follow-up user message because multimodal tool-role support is inconsistent. Bash, subagent tasks, and `parallel_llm` prompts remain text-only. Completed results and resume state can contain full base64 image data, so treat them as sensitive and potentially large. + ## Hooks Hooks are runtime callables registered on a `Harness`. They can observe lifecycle events, append prompt context, cancel selected before-events, or rewrite tool output. @@ -382,7 +399,7 @@ Hook events: - `limit_reached` - `run_end` -`user_prompt_submit`, `before_tool_call`, and `before_subagent_run` are cancellable. `after_tool_call` can rewrite `ctx.output`, but retry control flow is captured before that rewrite. Tool filters apply only to tool events; agent filters apply only to subagent events. +`user_prompt_submit`, `before_tool_call`, and `before_subagent_run` are cancellable. Run-start and prompt-submit hooks receive normalized content-block tuples and can replace the prompt with a string or valid block sequence. `after_tool_call` can rewrite canonical `ctx.output` or structured `ctx.envelope`; either form is strictly validated and keeps the other synchronized. These after-tool fields can contain full base64 image data and can be sensitive and large. Tool filters apply only to tool events; agent filters apply only to subagent events. By default, hook exceptions are logged and the run continues. Set `strict_hooks=True` to make hook exceptions fail the run. @@ -569,7 +586,7 @@ Available wrappers: - `MCPServerSSE` - `MCPServerStreamableHTTP` -ThinHarness only turns MCP tools into harness tools; transport execution and session lifecycle come from the FastMCP client. MCP never inherits automatically into a child. A child that needs MCP lists an explicit `MCPPlugin` in `SubAgentConfig.plugins`, and that child binding owns its connection lifecycle. MCP prompts, resources, sampling, OAuth flows, provider-native MCP, and `.mcp.json` discovery are outside the current scope. +ThinHarness only turns MCP tools into harness tools; transport execution and session lifecycle come from the FastMCP client. Successful supported MCP images remain ordered image blocks. When `structuredContent` exists, its canonical JSON is the authoritative first text block, MCP text blocks are discarded, and image blocks or placeholders keep their relative order. MCP never inherits automatically into a child. A child that needs MCP lists an explicit `MCPPlugin` in `SubAgentConfig.plugins`, and that child binding owns its connection lifecycle. MCP prompts, resources, sampling, OAuth flows, provider-native MCP, and `.mcp.json` discovery are outside the current scope. ## Resume @@ -603,8 +620,8 @@ Budgets span the pause. The paused batch counts against `usage.tool_calls` exact Built-in provider resume details: -- `resume_state["kind"] == "transcript"` and `version == 3`. -- The transcript is provider-agnostic and no longer depends on OpenAI server-side response retention. +- `resume_state["kind"] == "transcript"` and `version == 4`. Older transcript versions must be regenerated; approval envelopes with version 3 nested provider state also fail. +- The transcript is provider-agnostic and no longer depends on OpenAI server-side response retention. Ordered image bytes are self-contained as base64, which adds about 33% encoding overhead. - Provider-specific reasoning chains are preserved on same-provider resume (Anthropic thinking signatures, OpenAI `encrypted_content`, OpenRouter `reasoning_details`) and degraded to a leading ``-tagged text block on cross-provider resume. Anthropic native re-emit also requires extended thinking to be enabled in the resuming run. For reasoning-capable OpenAI models the harness adds `include=["reasoning.encrypted_content"]`, so `resume_state` can contain encrypted reasoning blobs — treat it as sensitive. - Cross-provider resume is supported by the built-in renderers, but real providers may reject foreign-format tool-call ids or malformed tool-call argument JSON. - `OpenAIResponsesSession.start(prompt, constants, previous_response_id=...)` remains available as a low-level escape hatch, but later resume state captures only the new prompt onward, not the externally seeded prior turns. @@ -644,7 +661,7 @@ Local tracing is on by default. It writes plaintext JSONL traces under: ~/.thinharness/traces// ``` -Those traces can include prompts, model outputs, tool arguments, and tool results. Treat them as sensitive local data. +Those traces can include prompt text, model outputs, tool arguments, and tool-result text. Image bytes, base64, and data URLs are replaced with ordered descriptors that contain media type, byte size, and block index. Completed results and resume state still retain full images; treat them as sensitive local data. Disable local trace files with: diff --git a/docs/site/assets/thinharness-run-loop.svg b/docs/site/assets/thinharness-run-loop.svg index 6ca53e9..aa8d907 100644 --- a/docs/site/assets/thinharness-run-loop.svg +++ b/docs/site/assets/thinharness-run-loop.svg @@ -1,3 +1,3 @@ -
    harness/runtime
    provider session
    decision
    approval pause
    tool path
    final result
    retry / stop
    stream event
    Typed event stream
    StreamEvent emitted at each blue dot; contains kind, run_id, and event number
    SETUP
    ONE MODEL TURN
    BRANCH HANDLING INSIDE THE WHILE LOOP
    Entry point
    Harness.stream()
    core.py:305-374
    outer stream flow
    Create run objects
    RunContext
    StreamEmitter
    core.py:405-418
    per-run state
    trace scope
    Open session
    new_session()
    resume_session()
    core.py:448-491
    one active session
    Loop entry
    RunContext.advance_model()
    runtime.py:268
    limits, notices, tracing
    provider request
    Call provider session
    ModelSession
    providers.py:184
    start()
    continue_with_tools()
    continue_with_user_text()
    Model turn
    ModelTurn
    providers.py:56
    text
    tool calls
    raw JSON
    Resolve turn
    resolve_turn_output()
    turns.py:50
    OutputTurnDecision
    chooses next branch
    Approval required?
    _approval_required_calls()
    turns.py:322
    before tool batch executes
    Final result
    RunContext.finalize()
    runtime.py:420
    builds HarnessResult
    fires run_end once
    Approval pause
    pause_for_approval()
    runtime.py:367
    stop_reason="approval_required"
    pending approvals + resume_state
    no tools executed
    Run tools
    ToolBatchExecutor.execute_batch()
    tool_execution.py:63
    runs ordinary tool calls
    preserves model order
    Return tool outputs
    Build ToolOutput[]
    providers.py:97
    model-visible results
    Retry decision
    RunContext.retry_or_fail()
    runtime.py:448
    budget remains or stop
    Stop / raise
    core.py:499-510
    HarnessError
    limit, provider,
    or validation
    Corrective request
    continue_with_tools()
    continue_with_user_text()
    asks model to fix output
    interpret turn
    final
    no approval
    requires approval
    tool batch
    schema retry
    next iteration
    budget remains
    no retry
    next iteration
    Text is not SVG - cannot display
    \ No newline at end of file +
    harness/runtime
    provider session
    decision
    approval pause
    tool path
    final result
    retry / stop
    stream event
    Typed event stream
    StreamEvent emitted at each blue dot; contains kind, run_id, and event number
    SETUP
    ONE MODEL TURN
    BRANCH HANDLING INSIDE THE WHILE LOOP
    Entry point
    Harness.stream()
    core.py:305-374
    outer stream flow
    Create run objects
    RunContext
    StreamEmitter
    core.py:405-418
    per-run state
    trace scope
    Open session
    new_session()
    resume_session()
    core.py:448-491
    one active session
    Loop entry
    RunContext.advance_model()
    runtime.py:268
    limits, notices, tracing
    provider request
    Call provider session
    ModelSession
    providers.py:184
    start()
    continue_with_tools()
    continue_with_user_content()
    Model turn
    ModelTurn
    providers.py:56
    text
    tool calls
    raw JSON
    Resolve turn
    resolve_turn_output()
    turns.py:50
    OutputTurnDecision
    chooses next branch
    Approval required?
    _approval_required_calls()
    turns.py:322
    before tool batch executes
    Final result
    RunContext.finalize()
    runtime.py:420
    builds HarnessResult
    fires run_end once
    Approval pause
    pause_for_approval()
    runtime.py:367
    stop_reason="approval_required"
    pending approvals + resume_state
    no tools executed
    Run tools
    ToolBatchExecutor.execute_batch()
    tool_execution.py:63
    runs ordinary tool calls
    preserves model order
    Return tool outputs
    Build ToolOutput[]
    providers.py:97
    model-visible results
    Retry decision
    RunContext.retry_or_fail()
    runtime.py:448
    budget remains or stop
    Stop / raise
    core.py:499-510
    HarnessError
    limit, provider,
    or validation
    Corrective request
    continue_with_tools()
    continue_with_user_content()
    asks model to fix output
    interpret turn
    final
    no approval
    requires approval
    tool batch
    schema retry
    next iteration
    budget remains
    no retry
    next iteration
    Text is not SVG - cannot display
    \ No newline at end of file diff --git a/docs/site/explainer/index.html b/docs/site/explainer/index.html index 6146eac..d32a414 100644 --- a/docs/site/explainer/index.html +++ b/docs/site/explainer/index.html @@ -211,7 +211,7 @@

    Provider-neutral objects

    NameMeaning ModelProtocol for reusable model configuration. It creates isolated ModelSession objects. - ModelSessionPer-run provider conversation state. Built-in sessions keep native in-run state plus a parallel neutral transcript; dump_state returns the transcript for provider-agnostic resume. All expose three request methods — start, continue_with_tools, and continue_with_user_text — each taking per-run RequestConstants, plus dump_state. + ModelSessionPer-run provider conversation state. Built-in sessions keep native in-run state plus a parallel neutral transcript; dump_state returns the transcript for provider-agnostic resume. All expose three request methods — start, continue_with_tools, and continue_with_user_content — each taking per-run RequestConstants, plus dump_state. ModelTurnNormalized provider response: assistant text, requested ModelToolCall entries, raw provider JSON, plus normalized TokenUsage, finish reason, and response model. ModelToolCallNormalized tool request with id, name, and raw JSON argument string. ToolOutputTool result sent back to the provider so the model can continue after a tool call: call id plus model-visible output string. @@ -731,7 +731,7 @@

    Implementation Deep Dive

    ModelSession Yes - The per-run conversation object. The turn machine calls its start(...), continue_with_tools(...), and continue_with_user_text(...) methods with per-run RequestConstants, plus dump_state(). + The per-run conversation object. The turn machine calls its start(...), continue_with_tools(...), and continue_with_user_content(...) methods with per-run RequestConstants, plus dump_state(). ModelTurn diff --git a/tests/e2e/image_inputs_journey.py b/tests/e2e/image_inputs_journey.py new file mode 100644 index 0000000..16ab554 --- /dev/null +++ b/tests/e2e/image_inputs_journey.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import asyncio +import copy +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from thinharness import ( + AnthropicMessagesModel, + AnthropicProvider, + Harness, + HarnessConfig, + ImageBlock, + OpenAIProvider, + OpenAIResponsesModel, + OpenRouterModel, + OpenRouterProvider, + TextBlock, + ToolResult, + ToolSpec, +) + +ROOT = Path(__file__).resolve().parents[2] +IMAGE = (ROOT / "assets" / "logo-circle.png").read_bytes() + + +class RecordingOpenAI(OpenAIProvider): + def __init__(self) -> None: + super().__init__() + self.payloads: list[dict] = [] + + async def create_response(self, payload): + self.payloads.append(copy.deepcopy(payload)) + return await super().create_response(payload) + + +class RecordingAnthropic(AnthropicProvider): + def __init__(self) -> None: + super().__init__() + self.payloads: list[dict] = [] + + async def create_message(self, payload): + self.payloads.append(copy.deepcopy(payload)) + return await super().create_message(payload) + + +class RecordingOpenRouter(OpenRouterProvider): + def __init__(self) -> None: + super().__init__() + self.payloads: list[dict] = [] + + async def create_chat_completion(self, payload): + self.payloads.append(copy.deepcopy(payload)) + return await super().create_chat_completion(payload) + + +def image_tool() -> ToolSpec: + return ToolSpec( + "inspect_fixture", + "Return the comparison image. Call this exactly once.", + {"type": "object", "properties": {}, "additionalProperties": False}, + lambda _args: ToolResult( + True, + (TextBlock("comparison fixture"), ImageBlock(IMAGE, "image/png")), + {"fixture": "logo-circle.png"}, + ), + ) + + +async def run_provider(label: str, model, payloads: list[dict]) -> None: + harness = Harness( + HarnessConfig(root=ROOT, max_model_requests=5, max_tool_calls=1, local_tracing=False), + model=model, + tools=[image_tool()], + ) + first = await harness.run(( + TextBlock("Describe this image, then call inspect_fixture and compare the two images."), + ImageBlock(IMAGE, "image/png"), + )) + assert first.text + assert first.resume_state is not None + resumed = await harness.run("In one sentence, restate the comparison.", resume_from=first.resume_state) + assert resumed.text + + if label == "openai": + assert any( + isinstance(item.get("output"), list) + for payload in payloads + for item in payload.get("input", []) + if isinstance(item, dict) and item.get("type") == "function_call_output" + ) + if label == "openrouter": + assert any( + part.get("text", "").startswith("[tool image call_id=") + for payload in payloads + for message in payload.get("messages", []) + for part in message.get("content", []) if isinstance(message.get("content"), list) + if isinstance(part, dict) and part.get("type") == "text" + ) + await harness.aclose() + print(f"PASS image_inputs_journey provider={label}") + + +async def main() -> None: + runs = [] + if os.getenv("OPENAI_API_KEY"): + provider = RecordingOpenAI() + runs.append(run_provider( + "openai", + OpenAIResponsesModel(os.getenv("E2E_OPENAI_IMAGE_MODEL", "gpt-5.5"), provider=provider), + provider.payloads, + )) + if os.getenv("ANTHROPIC_API_KEY"): + provider = RecordingAnthropic() + runs.append(run_provider( + "anthropic", + AnthropicMessagesModel(os.getenv("E2E_ANTHROPIC_IMAGE_MODEL", "claude-sonnet-4-5"), provider=provider), + provider.payloads, + )) + if os.getenv("OPENROUTER_API_KEY"): + provider = RecordingOpenRouter() + runs.append(run_provider( + "openrouter", + OpenRouterModel(os.getenv("E2E_OPENROUTER_IMAGE_MODEL", "openai/gpt-4o-mini"), provider=provider), + provider.payloads, + )) + if not runs: + print("SKIP image_inputs_journey: no provider credentials") + return + for run in runs: + await run + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/e2e/mcp_journey.py b/tests/e2e/mcp_journey.py index c174035..f98da7e 100644 --- a/tests/e2e/mcp_journey.py +++ b/tests/e2e/mcp_journey.py @@ -30,7 +30,7 @@ async def continue_with_tools(self, outputs: list[Any], constants: Any, **_kwarg assert "product=42" in outputs[0].output return ModelTurn(text="product=42 MCP_DONE", raw={"id": "done"}) - async def continue_with_user_text(self, text: str, constants: Any, **_kwargs: Any) -> ModelTurn: + async def continue_with_user_content(self, text: str, constants: Any, **_kwargs: Any) -> ModelTurn: raise AssertionError(f"unexpected user continuation: {text!r}, {constants!r}") def dump_state(self) -> None: diff --git a/tests/unit/fakes.py b/tests/unit/fakes.py index fc37082..4b121ca 100644 --- a/tests/unit/fakes.py +++ b/tests/unit/fakes.py @@ -226,9 +226,12 @@ def __init__( async def start(self, prompt, constants, *, previous_response_id=None, notices=None): """Return the scripted start turn.""" + from thinharness.content import normalize_content, text_only_value + + prompt_value = text_only_value(normalize_content(prompt)) or prompt self.notice_calls.append(("start", list(notices or []))) if self.on_start: - self.on_start(prompt, constants.instructions, constants.tools, constants.metadata, previous_response_id) + self.on_start(prompt_value, constants.instructions, constants.tools, constants.metadata, previous_response_id) return self.start_turn async def continue_with_tools(self, outputs, constants, *, notices=None): @@ -239,16 +242,19 @@ async def continue_with_tools(self, outputs, constants, *, notices=None): self.on_continue(outputs, constants.tools, constants.metadata) return self.continue_turn - async def continue_with_user_text(self, text, constants, *, notices=None): - """Return the start turn for a resume (first request on the session) or the continuation turn for a correction.""" + async def continue_with_user_content(self, content, constants, *, notices=None): + """Return the first resumed turn or a scripted correction turn.""" + from thinharness.content import normalize_content, text_only_value + + content_value = text_only_value(normalize_content(content)) or content is_resume = not self.notice_calls - self.notice_calls.append(("continue_with_user_text", list(notices or []))) + self.notice_calls.append(("continue_with_user_content", list(notices or []))) if is_resume: if self.on_start: - self.on_start(text, constants.instructions, constants.tools, constants.metadata, None) + self.on_start(content_value, constants.instructions, constants.tools, constants.metadata, None) return self.start_turn if self.on_continue: - self.on_continue(text, constants.tools, constants.metadata) + self.on_continue(content_value, constants.tools, constants.metadata) return self.continue_turn def dump_state(self): @@ -264,7 +270,7 @@ async def continue_with_tools(self, outputs, constants, *, notices=None): """Never continue after a failed start.""" raise AssertionError("should not continue") - async def continue_with_user_text(self, text, constants, *, notices=None): + async def continue_with_user_content(self, content, constants, *, notices=None): """Never continue after a failed start.""" raise AssertionError("should not continue") diff --git a/tests/unit/test_approvals.py b/tests/unit/test_approvals.py index ed2063c..38b959c 100644 --- a/tests/unit/test_approvals.py +++ b/tests/unit/test_approvals.py @@ -540,7 +540,7 @@ async def test_openai_approval_pause_round_trips_provider_state(tmp_path: Path) assert result.text == "done" assert called == [{"path": "hello.txt"}] assert paused.resume_state["provider_state"]["kind"] == "transcript" - assert paused.resume_state["provider_state"]["version"] == 3 + assert paused.resume_state["provider_state"]["version"] == 4 assert [entry["role"] for entry in paused.resume_state["provider_state"]["entries"]] == ["user", "assistant"] assert "previous_response_id" not in client.payloads[1] assert [item["type"] for item in client.payloads[1]["input"]] == ["message", "function_call", "function_call_output"] @@ -1007,7 +1007,7 @@ async def start(self, prompt, constants, *, previous_response_id=None, notices=N async def continue_with_tools(self, outputs, constants, *, notices=None): return self.turns.pop(0) - async def continue_with_user_text(self, text, constants, *, notices=None): + async def continue_with_user_content(self, text, constants, *, notices=None): return self.turns.pop(0) def dump_state(self): diff --git a/tests/unit/test_file_tools.py b/tests/unit/test_file_tools.py index d259802..feb2c7f 100644 --- a/tests/unit/test_file_tools.py +++ b/tests/unit/test_file_tools.py @@ -26,6 +26,7 @@ def test_file_tool_descriptions_use_defaults(tmp_path: Path) -> None: assert descriptions == { "read": DEFAULT_READ_DESCRIPTION, + "read_image": "Read one local PNG, JPEG, GIF, or WebP image for visual inspection.", "write": DEFAULT_WRITE_DESCRIPTION, "edit": DEFAULT_EDIT_DESCRIPTION, "search": DEFAULT_SEARCH_DESCRIPTION, diff --git a/tests/unit/test_harness.py b/tests/unit/test_harness.py index 4ebe78b..8e5ae09 100644 --- a/tests/unit/test_harness.py +++ b/tests/unit/test_harness.py @@ -676,7 +676,7 @@ async def start(self, prompt, constants, *, previous_response_id=None, notices=N async def continue_with_tools(self, outputs, constants, *, notices=None): raise AssertionError("should not continue") - async def continue_with_user_text(self, text, constants, *, notices=None): + async def continue_with_user_content(self, text, constants, *, notices=None): raise AssertionError("should not continue") model = ScriptedModel([ @@ -770,7 +770,7 @@ async def continue_with_tools(self, outputs, constants, *, notices=None): outputs_seen.extend(output.output for output in outputs) return self.turns.pop(0) - async def continue_with_user_text(self, text, constants, *, notices=None): + async def continue_with_user_content(self, text, constants, *, notices=None): raise AssertionError("unexpected user-text continuation") def dump_state(self): diff --git a/tests/unit/test_hooks.py b/tests/unit/test_hooks.py index e3b3dce..9fefa46 100644 --- a/tests/unit/test_hooks.py +++ b/tests/unit/test_hooks.py @@ -471,7 +471,7 @@ async def start(self, prompt, constants, *, previous_response_id=None, notices=N async def continue_with_tools(self, outputs, constants, *, notices=None): raise HarnessError("bare harness error") - async def continue_with_user_text(self, text, constants, *, notices=None): + async def continue_with_user_content(self, text, constants, *, notices=None): raise HarnessError("bare harness error") harness = Harness( diff --git a/tests/unit/test_image_inputs.py b/tests/unit/test_image_inputs.py new file mode 100644 index 0000000..87133f6 --- /dev/null +++ b/tests/unit/test_image_inputs.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +import base64 +import copy +import json +from pathlib import Path +from typing import Any + +import pytest + +from thinharness import ( + AnthropicMessagesModel, + FilesystemPlugin, + Harness, + HarnessConfig, + ImageBlock, + OpenAIResponsesModel, + OpenRouterModel, + TextBlock, + ToolResult, + ToolSpec, +) +from thinharness.content import content_from_json, content_to_json, normalize_content + +PNG = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR" + b"\x00" * 17 +PNG_B64 = base64.b64encode(PNG).decode("ascii") +PNG_URL = f"data:image/png;base64,{PNG_B64}" + + +class _Provider: + def __init__(self, name: str, responses: list[dict[str, Any]]) -> None: + self.name = name + self.api_key = "test" + self.payloads: list[dict[str, Any]] = [] + self.responses = list(responses) + + async def aclose(self) -> None: + return None + + async def create_response(self, payload: dict[str, Any]) -> dict[str, Any]: + self.payloads.append(copy.deepcopy(payload)) + return self.responses.pop(0) + + async def create_message(self, payload: dict[str, Any]) -> dict[str, Any]: + self.payloads.append(copy.deepcopy(payload)) + return self.responses.pop(0) + + async def create_chat_completion(self, payload: dict[str, Any]) -> dict[str, Any]: + self.payloads.append(copy.deepcopy(payload)) + return self.responses.pop(0) + + +def _config(tmp_path: Path) -> HarnessConfig: + return HarnessConfig(root=tmp_path, system_prompt="sys", local_tracing=False) + + +@pytest.mark.parametrize( + "bad", + [ + "", + [], + [TextBlock("")], + [ImageBlock(b"", "image/png")], + [ImageBlock(b"x", "image/svg+xml")], + ["text"], + ], +) +def test_content_validation_rejects_invalid_public_values(bad: Any) -> None: + with pytest.raises((TypeError, ValueError)): + normalize_content(bad) + + +def test_content_contract_detaches_round_trips_and_redacts() -> None: + caller = [TextBlock("before"), ImageBlock(PNG, "image/png"), TextBlock("after")] + normalized = normalize_content(caller) + caller.clear() + + assert normalized == (TextBlock("before"), ImageBlock(PNG, "image/png"), TextBlock("after")) + assert content_from_json(content_to_json(normalized)) == normalized + assert PNG_B64 not in repr(normalized[1]) + assert "size_bytes=33" in repr(normalized[1]) + + +async def test_public_initial_image_payloads_are_literal(tmp_path: Path) -> None: + prompt = [TextBlock("before"), ImageBlock(PNG, "image/png"), TextBlock("after")] + + openai_provider = _Provider("OpenAI", [{"id": "r1", "output_text": "done"}]) + await Harness(_config(tmp_path), model=OpenAIResponsesModel("gpt-test", provider=openai_provider)).run(prompt) # type: ignore[arg-type] + assert openai_provider.payloads == [{ + "model": "gpt-test", + "input": [{ + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "before"}, + {"type": "input_image", "image_url": PNG_URL}, + {"type": "input_text", "text": "after"}, + ], + }], + "tools": [], + "instructions": "sys", + }] + + anthropic_provider = _Provider("Anthropic", [{"content": [{"type": "text", "text": "done"}]}]) + await Harness(_config(tmp_path), model=AnthropicMessagesModel("claude-test", provider=anthropic_provider)).run(prompt) # type: ignore[arg-type] + assert anthropic_provider.payloads == [{ + "model": "claude-test", + "max_tokens": 16384, + "system": "sys", + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "before"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG_B64}}, + {"type": "text", "text": "after"}, + ], + }], + "tools": [], + "cache_control": {"type": "ephemeral"}, + }] + + openrouter_provider = _Provider("OpenRouter", [{"choices": [{"message": {"role": "assistant", "content": "done"}}]}]) + await Harness(_config(tmp_path), model=OpenRouterModel("vendor/model", provider=openrouter_provider)).run(prompt) # type: ignore[arg-type] + assert openrouter_provider.payloads == [{ + "model": "vendor/model", + "messages": [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": [ + {"type": "text", "text": "before"}, + {"type": "image_url", "image_url": {"url": PNG_URL}}, + {"type": "text", "text": "after"}, + ]}, + ], + "tools": [], + }] + + +async def test_openai_image_tool_result_payload_is_literal(tmp_path: Path) -> None: + provider = _Provider("OpenAI", [ + {"id": "r1", "output": [{"type": "function_call", "call_id": "call_1", "name": "look", "arguments": "{}"}]}, + {"id": "r2", "output_text": "done"}, + ]) + tool = ToolSpec( + "look", + "Return an image.", + {"type": "object", "properties": {}}, + lambda _args: ToolResult(True, [TextBlock("caption"), ImageBlock(PNG, "image/png")], {"source": "fixture"}), + ) + + result = await Harness(_config(tmp_path), model=OpenAIResponsesModel("gpt-test", provider=provider), tools=[tool]).run("go") # type: ignore[arg-type] + + assert provider.payloads[1]["input"] == [{ + "type": "function_call_output", + "call_id": "call_1", + "output": [ + {"type": "input_text", "text": '{"ok":true,"metadata":{"source":"fixture"}}'}, + {"type": "input_text", "text": "caption"}, + {"type": "input_image", "image_url": PNG_URL}, + ], + }] + assert result.resume_state["version"] == 4 + tool_entry = next(entry for entry in result.resume_state["entries"] if entry["role"] == "tool") + assert tool_entry == { + "role": "tool", + "call_id": "call_1", + "ok": True, + "content": [ + {"type": "text", "text": "caption"}, + {"type": "image", "media_type": "image/png", "data": PNG_B64}, + ], + "metadata": {"source": "fixture"}, + } + + +async def test_anthropic_image_tool_result_payload_is_literal(tmp_path: Path) -> None: + provider = _Provider("Anthropic", [ + {"content": [{"type": "tool_use", "id": "call_1", "name": "look", "input": {}}]}, + {"content": [{"type": "text", "text": "done"}]}, + ]) + tool = ToolSpec( + "look", + "Return an image.", + {"type": "object", "properties": {}}, + lambda _args: ToolResult(False, [TextBlock("try another"), ImageBlock(PNG, "image/png")], {"retry": True, "error_type": "Fixture"}), + max_retries=1, + ) + + await Harness(_config(tmp_path), model=AnthropicMessagesModel("claude-test", provider=provider), tools=[tool]).run("go") # type: ignore[arg-type] + + assert provider.payloads[1]["messages"][-1]["content"] == [{ + "type": "tool_result", + "tool_use_id": "call_1", + "content": [ + {"type": "text", "text": '{"ok":false,"metadata":{"retry":true,"error_type":"Fixture"}}'}, + {"type": "text", "text": "try another"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG_B64}}, + ], + }] + + +async def test_openrouter_image_tool_projection_is_labelled_and_literal(tmp_path: Path) -> None: + provider = _Provider("OpenRouter", [ + {"choices": [{"message": {"role": "assistant", "content": "", "tool_calls": [{ + "id": "call_1", + "type": "function", + "function": {"name": "look", "arguments": "{}"}, + }]}}]}, + {"choices": [{"message": {"role": "assistant", "content": "done"}}]}, + ]) + tool = ToolSpec( + "look", + "Return an image.", + {"type": "object", "properties": {}}, + lambda _args: [TextBlock("caption"), ImageBlock(PNG, "image/png")], + ) + + await Harness(_config(tmp_path), model=OpenRouterModel("vendor/model", provider=provider), tools=[tool]).run("go") # type: ignore[arg-type] + + assert provider.payloads[1]["messages"][-2:] == [ + { + "role": "tool", + "tool_call_id": "call_1", + "content": json.dumps({ + "ok": True, + "content": [ + {"type": "text", "text": "caption"}, + {"type": "image", "media_type": "image/png", "size_bytes": 33, "block_index": 1}, + ], + "metadata": {}, + }, ensure_ascii=False), + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "[tool image call_id=call_1 block=1]"}, + {"type": "image_url", "image_url": {"url": PNG_URL}}, + ], + }, + ] + + +def test_filesystem_read_image_is_opt_in_and_bounded(tmp_path: Path) -> None: + image = tmp_path / "sample.png" + image.write_bytes(PNG) + default = FilesystemPlugin().bind(type("Context", (), {"root": tmp_path})()) # type: ignore[arg-type] + assert "read_image" not in [tool.name for tool in default.static.tools] + + binding = FilesystemPlugin(tools=["read_image"], max_image_bytes=len(PNG)).bind(type("Context", (), {"root": tmp_path})()) # type: ignore[arg-type] + result = binding.static.tools[0].handler({"path": "sample.png"}) + assert isinstance(result, ToolResult) + assert result.content == ( + TextBlock('{"path":"sample.png","media_type":"image/png","size_bytes":33}'), + ImageBlock(PNG, "image/png"), + ) + + image.write_bytes(PNG + b"x") + oversized = binding.static.tools[0].handler({"path": "sample.png"}) + assert isinstance(oversized, ToolResult) + assert oversized.ok is False + assert "over max_image_bytes=33" in oversized.message_text() diff --git a/tests/unit/test_mcp.py b/tests/unit/test_mcp.py index 3174293..b70c87e 100644 --- a/tests/unit/test_mcp.py +++ b/tests/unit/test_mcp.py @@ -20,6 +20,7 @@ HarnessConfig, HarnessError, Hook, + ImageBlock, MCPError, MCPPlugin, MCPServer, @@ -31,6 +32,7 @@ PluginContribution, SubAgentConfig, SubagentsPlugin, + TextBlock, ToolOrigin, TracingOptions, ) @@ -196,7 +198,7 @@ async def continue_with_tools(self, outputs, constants, *, notices=None): raise AssertionError("unexpected tool continuation") return self.continue_turns.pop(0) - async def continue_with_user_text(self, text, constants, *, notices=None): + async def continue_with_user_content(self, text, constants, *, notices=None): """Return the scripted turn for a resumed prompt; no tests expect corrections.""" if self.requests_made: raise AssertionError("unexpected user-text correction") @@ -392,7 +394,7 @@ async def test_structured_content_wins_over_blocks(monkeypatch) -> None: ("block_builder", "expected"), [ pytest.param(lambda types: types.TextContent(type="text", text="plain"), "plain", id="text"), - pytest.param(lambda types: types.ImageContent(type="image", data="aGk=", mimeType="image/png"), "[image: image/png]", id="image"), + pytest.param(lambda types: types.ImageContent(type="image", data="aGk=", mimeType="image/png"), (ImageBlock(b"hi", "image/png"),), id="image"), pytest.param(lambda types: types.AudioContent(type="audio", data="aGk=", mimeType="audio/wav"), "[audio: audio/wav]", id="audio"), pytest.param( lambda types: types.EmbeddedResource( @@ -409,7 +411,7 @@ async def test_structured_content_wins_over_blocks(monkeypatch) -> None: ), ], ) -async def test_content_block_conversion(monkeypatch, block_builder, expected: str) -> None: +async def test_content_block_conversion(monkeypatch, block_builder, expected: str | tuple[ImageBlock, ...]) -> None: """Each supported MCP content block keeps its text conversion.""" from mcp import types @@ -442,7 +444,12 @@ async def test_mixed_content_blocks_preserve_order(monkeypatch) -> None: result = await server.call_tool("mixed", {}) - assert result.content == "first\n[image: image/png]\n[resource: file:///data.bin]\nlast" + assert result.content == ( + TextBlock("first"), + ImageBlock(b"hi", "image/png"), + TextBlock("[resource: file:///data.bin]"), + TextBlock("last"), + ) async def test_tool_error_with_structured_content_stays_error_text(monkeypatch) -> None: diff --git a/tests/unit/test_parallel_llm.py b/tests/unit/test_parallel_llm.py index fb6d144..70a5384 100644 --- a/tests/unit/test_parallel_llm.py +++ b/tests/unit/test_parallel_llm.py @@ -99,7 +99,7 @@ async def continue_with_tools(self, outputs, constants, *, notices=None): """Batch sessions never continue.""" raise AssertionError("batch session should not continue") - async def continue_with_user_text(self, text, constants, *, notices=None): + async def continue_with_user_content(self, text, constants, *, notices=None): """Batch sessions never continue.""" raise AssertionError("batch session should not continue") @@ -141,7 +141,7 @@ async def continue_with_tools(self, outputs: list[ToolOutput], constants, *, not payload = json.loads(parsed["content"]) return ModelTurn(text=f"done:{payload['succeeded']}", raw={"id": "done"}) - async def continue_with_user_text(self, text, constants, *, notices=None): + async def continue_with_user_content(self, text, constants, *, notices=None): """Main session never receives user-text continuations.""" raise AssertionError("should not continue with user text") diff --git a/tests/unit/test_plugins.py b/tests/unit/test_plugins.py index 0d37dbd..b7f026f 100644 --- a/tests/unit/test_plugins.py +++ b/tests/unit/test_plugins.py @@ -670,7 +670,7 @@ async def continue_with_tools(self, tool_outputs, constants, **_kwargs): return ModelTurn(tool_calls=[ModelToolCall(id="call_2", name="late", arguments="{}")], raw={"id": "late"}) return ModelTurn(text="done", raw={"id": "done"}) - async def continue_with_user_text(self, text, constants, **_kwargs): + async def continue_with_user_content(self, text, constants, **_kwargs): raise AssertionError("not used") def dump_state(self): diff --git a/tests/unit/test_providers.py b/tests/unit/test_providers.py index f9361d9..67d2bd0 100644 --- a/tests/unit/test_providers.py +++ b/tests/unit/test_providers.py @@ -94,9 +94,9 @@ async def test_model_sessions_advance_independently() -> None: await second.continue_with_tools([ToolOutput(second_turn.tool_calls[0].id, "second result")], constants) assert provider.payloads[2]["messages"][0] == {"role": "user", "content": "first"} - assert provider.payloads[2]["messages"][-1]["content"][0]["content"] == "first result" + assert provider.payloads[2]["messages"][-1]["content"][0]["content"] == '{"ok": true, "content": "first result", "metadata": {}}' assert provider.payloads[3]["messages"][0] == {"role": "user", "content": "second"} - assert provider.payloads[3]["messages"][-1]["content"][0]["content"] == "second result" + assert provider.payloads[3]["messages"][-1]["content"][0]["content"] == '{"ok": true, "content": "second result", "metadata": {}}' async def test_openai_previous_response_id_is_session_scoped() -> None: client = FakeClient() @@ -127,15 +127,15 @@ async def test_openai_appends_notices_to_string_and_tool_inputs() -> None: constants, notices=[notice], ) - await session.continue_with_user_text("fix this", constants, notices=[notice]) + await session.continue_with_user_content("fix this", constants, notices=[notice]) resumed = model.resume_session({ "kind": "transcript", - "version": 3, + "version": 4, "origin_provider": "openai", "origin_model": "gpt-test", - "entries": [{"role": "user", "content": "prior", "notice": False}], + "entries": [{"role": "user", "content": [{"type": "text", "text": "prior"}], "notice": False}], }) - await resumed.continue_with_user_text("follow-up", constants, notices=[notice]) + await resumed.continue_with_user_content("follow-up", constants, notices=[notice]) assert client.payloads[0]["input"].endswith("\nFinal request.\n") assert [item["type"] for item in client.payloads[1]["input"][:-1]] == ["function_call_output", "function_call_output"] @@ -171,7 +171,11 @@ async def test_openai_no_notice_payloads_are_unchanged() -> None: await session.continue_with_tools([ToolOutput("call_1", "ok")], constants) assert client.payloads[0]["input"] == "hi" - assert client.payloads[1]["input"] == [{"type": "function_call_output", "call_id": "call_1", "output": "ok"}] + assert client.payloads[1]["input"] == [{ + "type": "function_call_output", + "call_id": "call_1", + "output": '{"ok": true, "content": "ok", "metadata": {}}', + }] async def test_anthropic_appends_notices_to_messages() -> None: provider = FakeAnthropicProvider() @@ -186,8 +190,8 @@ async def test_anthropic_appends_notices_to_messages() -> None: constants, notices=[notice], ) - await session.continue_with_user_text("fix this", constants, notices=[notice]) - await session.continue_with_user_text("follow-up", constants, notices=[notice]) + await session.continue_with_user_content("fix this", constants, notices=[notice]) + await session.continue_with_user_content("follow-up", constants, notices=[notice]) assert provider.payloads[0]["messages"][0]["content"] == f"hi\n\n\npolicy\n\n\n{_notice_text()}" assert [block["type"] for block in provider.payloads[1]["messages"][-1]["content"][:-1]] == ["tool_result", "tool_result"] @@ -211,8 +215,8 @@ async def test_openrouter_appends_notices_to_messages() -> None: constants, notices=[notice], ) - await session.continue_with_user_text("fix this", constants, notices=[notice]) - await session.continue_with_user_text("follow-up", constants, notices=[notice]) + await session.continue_with_user_content("fix this", constants, notices=[notice]) + await session.continue_with_user_content("follow-up", constants, notices=[notice]) assert provider.payloads[0]["messages"][1]["content"] == f"hi\n\n\npolicy\n\n\n{_notice_text()}" continuation_messages = provider.payloads[1]["messages"] @@ -232,7 +236,7 @@ async def test_resume_replays_preserved_tool_notices() -> None: anthropic_state = json.loads(json.dumps(anthropic_session.dump_state())) assert anthropic_state == json.loads(json.dumps(anthropic_state)) anthropic_resumed = AnthropicMessagesModel("claude-test", provider=anthropic_provider).resume_session(anthropic_state) - await anthropic_resumed.continue_with_user_text("next", constants) + await anthropic_resumed.continue_with_user_content("next", constants) assert anthropic_provider.payloads[2]["messages"][2]["content"][-1] == {"type": "text", "text": _notice_text()} openai_capture = FakeClient() @@ -242,7 +246,7 @@ async def test_resume_replays_preserved_tool_notices() -> None: openai_state = json.loads(json.dumps(openai_session.dump_state())) openai_replay = FakeClient() openai_resumed = OpenAIResponsesModel("gpt-test", provider=openai_replay).resume_session(openai_state) - await openai_resumed.continue_with_user_text("next", constants) + await openai_resumed.continue_with_user_content("next", constants) assert { "type": "message", "role": "user", @@ -255,7 +259,7 @@ async def test_resume_replays_preserved_tool_notices() -> None: await openrouter_session.continue_with_tools([ToolOutput(openrouter_first.tool_calls[0].id, "ok")], constants, notices=[notice]) openrouter_state = json.loads(json.dumps(openrouter_session.dump_state())) openrouter_resumed = OpenRouterModel("openai/test", provider=openrouter_provider).resume_session(openrouter_state) - await openrouter_resumed.continue_with_user_text("next", constants) + await openrouter_resumed.continue_with_user_content("next", constants) assert {"role": "user", "content": _notice_text()} in openrouter_provider.payloads[2]["messages"] async def test_resume_replays_preserved_user_notices() -> None: @@ -269,7 +273,7 @@ async def test_resume_replays_preserved_user_notices() -> None: anthropic_state = json.loads(json.dumps(anthropic_session.dump_state())) anthropic_replay = FakeAnthropicProvider() anthropic_resumed = AnthropicMessagesModel("claude-test", provider=anthropic_replay).resume_session(anthropic_state) - await anthropic_resumed.continue_with_user_text("next", constants) + await anthropic_resumed.continue_with_user_content("next", constants) assert anthropic_replay.payloads[0]["messages"][0]["content"] == f"hi\n\n{_notice_text()}" openai_capture = FakeClient() @@ -279,7 +283,7 @@ async def test_resume_replays_preserved_user_notices() -> None: openai_state = json.loads(json.dumps(openai_session.dump_state())) openai_replay = FakeClient() openai_resumed = OpenAIResponsesModel("gpt-test", provider=openai_replay).resume_session(openai_state) - await openai_resumed.continue_with_user_text("next", constants) + await openai_resumed.continue_with_user_content("next", constants) assert openai_replay.payloads[0]["input"][0]["content"][0]["text"] == f"hi\n\n{_notice_text()}" openrouter_capture = FakeOpenRouterProvider() @@ -289,7 +293,7 @@ async def test_resume_replays_preserved_user_notices() -> None: openrouter_state = json.loads(json.dumps(openrouter_session.dump_state())) openrouter_replay = FakeOpenRouterProvider() openrouter_resumed = OpenRouterModel("openai/test", provider=openrouter_replay).resume_session(openrouter_state) - await openrouter_resumed.continue_with_user_text("next", constants) + await openrouter_resumed.continue_with_user_content("next", constants) assert openrouter_replay.payloads[0]["messages"][1]["content"] == f"hi\n\n{_notice_text()}" def test_openai_native_structured_output_overrides_extra_body_text() -> None: diff --git a/tests/unit/test_reasoning_fidelity.py b/tests/unit/test_reasoning_fidelity.py index 344db0b..b6c821e 100644 --- a/tests/unit/test_reasoning_fidelity.py +++ b/tests/unit/test_reasoning_fidelity.py @@ -427,7 +427,7 @@ async def test_multi_part_reasoning_renders_in_order(tmp_path: Path) -> None: async def test_reasoning_state_round_trips(tmp_path: Path) -> None: state = (await _harness(tmp_path, OpenAIResponsesModel(REASONING_OPENAI_MODEL, provider=ReasoningOpenAIProvider())).run("first")).resume_state - assert state["version"] == 3 + assert state["version"] == 4 assert json.loads(json.dumps(state)) == state assert _assistant_reasoning(state)[0]["signature"] == "enc-blob-1" diff --git a/tests/unit/test_resume.py b/tests/unit/test_resume.py index f73fbc1..85363ce 100644 --- a/tests/unit/test_resume.py +++ b/tests/unit/test_resume.py @@ -113,7 +113,7 @@ async def test_openai_resume_full_replays_transcript_for_followup(tmp_path: Path second = await harness.run("follow-up", resume_from=state) assert first.resume_state["kind"] == "transcript" - assert first.resume_state["version"] == 3 + assert first.resume_state["version"] == 4 assert first.resume_state["origin_provider"] == "openai" assert first.resume_state["origin_model"] == "gpt-test" assert [entry["role"] for entry in first.resume_state["entries"]] == ["user", "assistant", "tool", "assistant"] @@ -324,7 +324,7 @@ def harness() -> Harness: with pytest.raises(HarnessError, match="resume_from must be a dict"): harness().run_sync("follow-up", resume_from="resp_abc") # type: ignore[arg-type] - base_state = {"kind": "transcript", "version": 3, "origin_provider": "anthropic", "origin_model": "claude-test"} + base_state = {"kind": "transcript", "version": 4, "origin_provider": "anthropic", "origin_model": "claude-test"} with pytest.raises(HarnessError, match="resume_from kind None is not supported"): harness().run_sync("follow-up", resume_from={"version": 2, "origin_provider": "anthropic", "origin_model": "claude-test", "entries": []}) with pytest.raises(HarnessError, match="missing required field: 'entries'"): @@ -354,7 +354,7 @@ def test_anthropic_resume_rejects_non_json_tool_arguments() -> None: with pytest.raises(HarnessError, match="resume_from assistant tool call arguments must be JSON"): model.resume_session({ "kind": "transcript", - "version": 3, + "version": 4, "origin_provider": "openrouter", "origin_model": "openai/test", "entries": [{ @@ -429,7 +429,7 @@ def test_resumed_user_prompt_receives_limit_notice(tmp_path: Path) -> None: assert resumed.text == "done" assert [(method, [(notice.limit_kind, notice.remaining) for notice in notices]) for method, notices in resumed_session.notice_calls] == [ - ("continue_with_user_text", [("model_requests", 1)]) + ("continue_with_user_content", [("model_requests", 1)]) ] def test_resumed_user_prompt_runs_prompt_submit_hooks_before_notices(tmp_path: Path) -> None: @@ -761,6 +761,6 @@ async def continue_with_tools(self, outputs, constants, *, notices=None) -> Mode """Reject unexpected tool continuation.""" raise AssertionError("unexpected tool continuation") - async def continue_with_user_text(self, text, constants, *, notices=None) -> ModelTurn: + async def continue_with_user_content(self, text, constants, *, notices=None) -> ModelTurn: """Reject unexpected user-text continuation.""" raise AssertionError("unexpected user-text continuation") diff --git a/tests/unit/test_streaming.py b/tests/unit/test_streaming.py index 1640a25..d91d1b6 100644 --- a/tests/unit/test_streaming.py +++ b/tests/unit/test_streaming.py @@ -53,7 +53,7 @@ async def continue_with_tools(self, outputs, constants, *, notices=None): self.tool_outputs.append(outputs) return self._next_turn() - async def continue_with_user_text(self, text, constants, *, notices=None): + async def continue_with_user_content(self, text, constants, *, notices=None): """Return the first turn for a resume or record a correction and continue.""" is_resume = self.requests_made == 0 self.requests_made += 1 diff --git a/tests/unit/test_structured_output.py b/tests/unit/test_structured_output.py index 1e9d082..a6b24e1 100644 --- a/tests/unit/test_structured_output.py +++ b/tests/unit/test_structured_output.py @@ -265,9 +265,9 @@ def test_tool_mode_invalid_args_retry_uses_tool_output(tmp_path) -> None: seen_messages = [] class RecordingSession(ScriptedSession): - async def continue_with_user_text(self, text, constants, *, notices=None): + async def continue_with_user_content(self, text, constants, *, notices=None): seen_messages.append(text) - return await super().continue_with_user_text(text, constants, notices=notices) + return await super().continue_with_user_content(text, constants, notices=notices) session = RecordingSession( start_turn=ModelTurn(tool_calls=[ModelToolCall(id="call_final", name="final_result", arguments='{"name":"Ada"}')], raw={"id": "bad"}), @@ -353,8 +353,8 @@ def test_limit_notice_dedupes_across_structured_output_retries(tmp_path) -> None assert [(method, [(notice.limit_kind, notice.remaining) for notice in notices]) for method, notices in session.notice_calls] == [ ("start", [("tool_calls", 0)]), - ("continue_with_user_text", []), - ("continue_with_user_text", []), + ("continue_with_user_content", []), + ("continue_with_user_content", []), ] assert session.notice_calls[0][1][0].content == "Tool calls are not available on this run; produce the answer with final_result." diff --git a/tests/unit/test_subagents.py b/tests/unit/test_subagents.py index 329a02f..7e70806 100644 --- a/tests/unit/test_subagents.py +++ b/tests/unit/test_subagents.py @@ -947,7 +947,7 @@ async def start(self, *_args: Any, **_kwargs: Any) -> ModelTurn: async def continue_with_tools(self, *_args: Any, **_kwargs: Any) -> ModelTurn: raise AssertionError("unreachable") - async def continue_with_user_text(self, *_args: Any, **_kwargs: Any) -> ModelTurn: + async def continue_with_user_content(self, *_args: Any, **_kwargs: Any) -> ModelTurn: raise AssertionError("unreachable") def dump_state(self) -> None: diff --git a/tests/unit/test_tool_retry.py b/tests/unit/test_tool_retry.py index 4c25c51..a5c38b9 100644 --- a/tests/unit/test_tool_retry.py +++ b/tests/unit/test_tool_retry.py @@ -44,7 +44,7 @@ async def continue_with_tools(self, outputs, constants, *, notices=None): raise AssertionError("unexpected tool continuation") return self.continue_turns.pop(0) - async def continue_with_user_text(self, text, constants, *, notices=None): + async def continue_with_user_content(self, text, constants, *, notices=None): """No tests in this file expect user-text continuations.""" raise AssertionError("unexpected user-text continuation") @@ -376,7 +376,7 @@ def after(ctx): hooks=[Hook("after_tool_call", after)], ) - with pytest.raises(HarnessError): + with pytest.raises(ValueError, match="valid canonical ToolResult"): harness.run_sync("go") assert seen == ["ModelRetry"] @@ -402,7 +402,7 @@ class AgeArgs(BaseModel): assert seen[0]["retry"] is True -def test_tracing_uses_pre_hook_retry_kind(tmp_path: Path) -> None: +def test_tracing_uses_mutated_retry_kind(tmp_path: Path) -> None: tracer = FakeTracer() def rewrite(ctx): @@ -421,7 +421,7 @@ def rewrite(ctx): harness.run_sync("go") span = next(span for span in tracer.spans if span.name == "execute_tool flaky") - assert span.attributes["error.type"] == "ModelRetry" + assert span.attributes["error.type"] == "Rewritten" def test_subagent_tool_retry_budget_recipes(tmp_path: Path) -> None: diff --git a/tests/unit/test_tracing.py b/tests/unit/test_tracing.py index 6619cec..c0405a9 100644 --- a/tests/unit/test_tracing.py +++ b/tests/unit/test_tracing.py @@ -112,7 +112,7 @@ def test_model_request_delta_includes_rendered_tool_output_notices() -> None: span = FakeSpan("chat", {}) delta = model_request_delta_from_tool_outputs( kind="tool_outputs", - outputs=[ToolOutput(call_id="call_1", output='{"ok":true,"content":"real output"}')], + outputs=[ToolOutput(call_id="call_1", result=ToolResult(True, "real output"))], notices=[ModelNotice(kind="limit_warning", content="notice text", limit_kind="model_requests", remaining=1)], structured_output=None, ) @@ -121,7 +121,7 @@ def test_model_request_delta_includes_rendered_tool_output_notices() -> None: input_messages = json.loads(span.attributes["gen_ai.input.messages"]) notices = json.loads(span.attributes["thinharness.model.notices"]) - assert input_messages[0]["parts"][0]["content"] == '{"ok":true,"content":"real output"}' + assert input_messages[0]["parts"][0]["content"] == '{"ok": true, "content": "real output", "metadata": {}}' assert input_messages[1]["parts"][0]["content"] == '\nnotice text\n' assert "notice text" in span.attributes["gen_ai.prompt"] assert notices[0]["content"] == "notice text" @@ -482,7 +482,8 @@ def test_trace_request_kinds_for_resume_and_output_retries(tmp_path: Path) -> No assert "correction" in kinds assert "resume" in kinds retry_chat = next(span for span in chats if span.attributes.get("thinharness.model.request.kind") == "output_retry_tool") - assert json.loads(retry_chat.attributes["gen_ai.input.messages"])[0]["parts"][0]["content"].startswith("The previous response failed") + retry_content = json.loads(retry_chat.attributes["gen_ai.input.messages"])[0]["parts"][0]["content"] + assert json.loads(retry_content)["content"].startswith("The previous response failed") assert "Final request" in retry_chat.attributes["gen_ai.input.messages"] assert "Final request" in retry_chat.attributes["thinharness.model.notices"] correction_chat = next(span for span in chats if span.attributes.get("thinharness.model.request.kind") == "correction") diff --git a/tests/unit/test_turns.py b/tests/unit/test_turns.py index 3d22ac6..a1a2b1d 100644 --- a/tests/unit/test_turns.py +++ b/tests/unit/test_turns.py @@ -343,8 +343,8 @@ def test_correction_following_resume_uses_same_session(tmp_path: Path) -> None: # The resumed session answers the resume prompt first, then the correction # lands on the same session as a continuation. assert [method for method, _notices in resumed_session.notice_calls] == [ - "continue_with_user_text", - "continue_with_user_text", + "continue_with_user_content", + "continue_with_user_content", ] assert resumed.responses == [{"id": "resumed-bad"}, {"id": "corrected"}] diff --git a/thinharness/__init__.py b/thinharness/__init__.py index 529fb7b..4b4a500 100644 --- a/thinharness/__init__.py +++ b/thinharness/__init__.py @@ -3,6 +3,7 @@ from importlib.metadata import PackageNotFoundError from importlib.metadata import version as _metadata_version +from .content import ContentBlock, ImageBlock, Prompt, TextBlock from .core import Harness, HarnessConfig from .events import ( ApprovalResumedEvent, @@ -114,6 +115,10 @@ __all__ = [ "__version__", "BashPlugin", + "ContentBlock", + "ImageBlock", + "Prompt", + "TextBlock", "ChildHarnessHost", "ChildHarnessOutcome", "ChildHarnessRequest", diff --git a/thinharness/content.py b/thinharness/content.py new file mode 100644 index 0000000..564e92a --- /dev/null +++ b/thinharness/content.py @@ -0,0 +1,171 @@ +"""Provider-neutral text and image content contracts.""" + +from __future__ import annotations + +import base64 +import binascii +import json +import re +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any, Literal, TypeAlias + +ImageMediaType: TypeAlias = Literal["image/jpeg", "image/png", "image/gif", "image/webp"] +SUPPORTED_IMAGE_MEDIA_TYPES = frozenset({"image/jpeg", "image/png", "image/gif", "image/webp"}) + + +@dataclass(frozen=True) +class TextBlock: + """One immutable text content block.""" + + text: str + + +@dataclass(frozen=True, repr=False) +class ImageBlock: + """One immutable local image content block.""" + + data: bytes + media_type: ImageMediaType + + def __repr__(self) -> str: + """Return a representation that never includes image data.""" + return f"ImageBlock(media_type={self.media_type!r}, size_bytes={len(self.data)})" + + +ContentBlock: TypeAlias = TextBlock | ImageBlock +Prompt: TypeAlias = str | Sequence[ContentBlock] +NormalizedContent: TypeAlias = tuple[ContentBlock, ...] + + +def normalize_content(value: Prompt | Any, *, label: str = "content") -> NormalizedContent: + """Copy and validate public text/image content.""" + if isinstance(value, str): + blocks: tuple[Any, ...] = (TextBlock(value),) + elif isinstance(value, Sequence): + blocks = tuple(value) + else: + raise TypeError(f"{label} must be a string or a sequence of content blocks") + if not blocks: + raise ValueError(f"{label} must not be empty") + normalized: list[ContentBlock] = [] + for index, block in enumerate(blocks): + if isinstance(block, TextBlock): + if not isinstance(block.text, str) or not block.text: + raise ValueError(f"{label} text block {index} must not be empty") + normalized.append(block) + continue + if isinstance(block, ImageBlock): + if type(block.data) is not bytes: + raise TypeError(f"{label} image block {index} data must be bytes") + if not block.data: + raise ValueError(f"{label} image block {index} data must not be empty") + if block.media_type not in SUPPORTED_IMAGE_MEDIA_TYPES: + raise ValueError(f"{label} image block {index} has unsupported media type: {block.media_type!r}") + normalized.append(block) + continue + raise TypeError(f"{label} block {index} has unsupported type: {type(block).__name__}") + return tuple(normalized) + + +def content_to_json(blocks: Sequence[ContentBlock]) -> list[dict[str, Any]]: + """Encode validated blocks into the canonical JSON shape.""" + normalized = normalize_content(blocks) + encoded: list[dict[str, Any]] = [] + for block in normalized: + if isinstance(block, TextBlock): + encoded.append({"type": "text", "text": block.text}) + else: + encoded.append({ + "type": "image", + "media_type": block.media_type, + "data": base64.b64encode(block.data).decode("ascii"), + }) + return encoded + + +def content_from_json(value: Any, *, label: str = "content") -> NormalizedContent: + """Strictly decode canonical JSON content blocks.""" + if not isinstance(value, list) or not value: + raise ValueError(f"{label} must be a non-empty array") + blocks: list[ContentBlock] = [] + for index, item in enumerate(value): + if not isinstance(item, dict): + raise ValueError(f"{label} block {index} must be an object") + if item.get("type") == "text": + if set(item) != {"type", "text"} or not isinstance(item.get("text"), str): + raise ValueError(f"{label} text block {index} has wrong shape") + blocks.append(TextBlock(item["text"])) + continue + if item.get("type") == "image": + if set(item) != {"type", "media_type", "data"}: + raise ValueError(f"{label} image block {index} has wrong shape") + media_type = item.get("media_type") + data = item.get("data") + if not isinstance(media_type, str) or not isinstance(data, str): + raise ValueError(f"{label} image block {index} has wrong type") + try: + decoded = base64.b64decode(data, validate=True) + except (binascii.Error, ValueError) as exc: + raise ValueError(f"{label} image block {index} has invalid base64 data") from exc + blocks.append(ImageBlock(decoded, media_type)) # type: ignore[arg-type] + continue + raise ValueError(f"{label} block {index} has unsupported type") + return normalize_content(blocks, label=label) + + +def contains_image(blocks: Sequence[ContentBlock]) -> bool: + """Return whether content includes an image block.""" + return any(isinstance(block, ImageBlock) for block in blocks) + + +def append_text_block(blocks: Sequence[ContentBlock], text: str) -> NormalizedContent: + """Append non-empty text while preserving existing block boundaries.""" + if not text: + return normalize_content(blocks) + return normalize_content((*blocks, TextBlock(text))) + + +def text_only_value(blocks: Sequence[ContentBlock]) -> str | None: + """Join all-text content with the public provider-boundary separator.""" + normalized = normalize_content(blocks) + if contains_image(normalized): + return None + return "\n\n".join(block.text for block in normalized if isinstance(block, TextBlock)) + + +def redacted_content_json(blocks: Sequence[ContentBlock]) -> list[dict[str, Any]]: + """Project blocks without image bytes while preserving order.""" + normalized = normalize_content(blocks) + projected: list[dict[str, Any]] = [] + for index, block in enumerate(normalized): + if isinstance(block, TextBlock): + projected.append({"type": "text", "text": block.text}) + else: + projected.append({ + "type": "image", + "media_type": block.media_type, + "size_bytes": len(block.data), + "block_index": index, + }) + return projected + + +def redacted_content_string(blocks: Sequence[ContentBlock]) -> str: + """Return compact redacted JSON for a multimodal event field.""" + return json.dumps(redacted_content_json(blocks), ensure_ascii=False, separators=(",", ":")) + + +_DATA_URL_RE = re.compile(r"data:image/[^;,\s]+;base64,[A-Za-z0-9+/=]+", re.IGNORECASE) + + +def redact_image_data(text: str, blocks: Sequence[ContentBlock] = ()) -> str: + """Redact complete image encodings from an error message.""" + redacted = _DATA_URL_RE.sub("[image data redacted]", text) + for block in blocks: + if not isinstance(block, ImageBlock): + continue + encoded = base64.b64encode(block.data).decode("ascii") + redacted = redacted.replace(encoded, "[image data redacted]") + redacted = redacted.replace(f"data:{block.media_type};base64,{encoded}", "[image data redacted]") + return redacted diff --git a/thinharness/core.py b/thinharness/core.py index c7db997..4939bc3 100644 --- a/thinharness/core.py +++ b/thinharness/core.py @@ -20,6 +20,7 @@ validate_approval_pause_state, ) from .children import ChildHarnessHost, _ParentChildHarnessHost, _ToolComposition +from .content import NormalizedContent, Prompt, normalize_content, redact_image_data, redacted_content_string, text_only_value from .defaults import DEFAULT_SYSTEM_PROMPT from .events import ( ApprovalResumedEvent, @@ -78,11 +79,12 @@ def _local_tracing_enabled(configured: bool) -> bool: def _classify_run_failure(run_ctx: Any, agent_span: Any, exc: Exception) -> Exception: """Record a run failure and return the exception to raise.""" - agent_span.record_exception(exc) - agent_span.set_error(str(exc), type(exc).__name__) + message = redact_image_data(str(exc), getattr(run_ctx, "image_blocks", ())) + agent_span.record_exception(exc if message == str(exc) else HarnessError(message)) + agent_span.set_error(message, type(exc).__name__) if isinstance(exc, ProviderError): run_ctx.stop_reason = "provider_error" - run_ctx.terminal_error = HarnessError(str(exc)) + run_ctx.terminal_error = HarnessError(message) return run_ctx.terminal_error if isinstance(exc, UnexpectedModelBehavior): run_ctx.stop_reason = "unexpected_model_behavior" @@ -98,6 +100,13 @@ def _classify_run_failure(run_ctx: Any, agent_span: Any, exc: Exception) -> Exce return exc +def _event_prompt(prompt: Prompt) -> str: + """Keep text-only stream values and redact multimodal values.""" + content = normalize_content(prompt, label="prompt") + text = text_only_value(content) + return text if text is not None else redacted_content_string(content) + + class HarnessConfig(BaseModel): """Configuration for Harness.""" @@ -268,7 +277,7 @@ def __init__( self._running = False self._closed = False - async def run(self, prompt: str, *, resume_from: dict[str, Any] | None = None, metadata: Json | None = None) -> HarnessResult: + async def run(self, prompt: Prompt, *, resume_from: dict[str, Any] | None = None, metadata: Json | None = None) -> HarnessResult: """Run one prompt to completion.""" result: HarnessResult | None = None stream = self.stream(prompt, resume_from=resume_from, metadata=metadata) @@ -326,7 +335,7 @@ async def _run_and_close() -> HarnessResult: def stream( self, - prompt: str, + prompt: Prompt, *, resume_from: dict[str, Any] | None = None, metadata: Json | None = None, @@ -396,7 +405,7 @@ def stream_approvals( async def _run_streaming( self, - prompt: str, + prompt: Prompt, *, resume_from: dict[str, Any] | None, approval_state: dict[str, Any] | None, @@ -445,7 +454,7 @@ async def _run_streaming( run_ctx.emit( RunStartedEvent( **run_ctx.stream_base(), - prompt=None if approval_pause is not None else prompt, + prompt=None if approval_pause is not None else _event_prompt(prompt), root=str(self.root), max_model_requests=self.config.max_model_requests, max_tool_calls=self.config.max_tool_calls, @@ -553,7 +562,7 @@ async def _run_streaming( **run_ctx.stream_base(), stop_reason=run_ctx.stop_reason, error_type=type(exc).__name__, - message=str(exc), + message=redact_image_data(str(exc), run_ctx.image_blocks), ) ) raise @@ -563,39 +572,41 @@ async def _run_streaming( async def _prepare_run_start( self, - prompt: str, + prompt: Prompt, run_metadata: Json, run_ctx: Any, agent_span: Any, *, skip_user_prompt: bool = False, - ) -> tuple[str, str]: - """Fire start hooks and return the effective prompt plus instructions.""" - self.hooks.fire( - RunStartContext( - harness=self, - metadata=dict(run_metadata), - prompt=prompt, - root=self.root, - max_model_requests=self.config.max_model_requests, - max_tool_calls=self.config.max_tool_calls, - ) + ) -> tuple[NormalizedContent, str]: + """Fire start hooks and return effective normalized content plus instructions.""" + initial: NormalizedContent = () if skip_user_prompt else normalize_content(prompt, label="prompt") + start_ctx = RunStartContext( + harness=self, + metadata=dict(run_metadata), + prompt=initial, + root=self.root, + max_model_requests=self.config.max_model_requests, + max_tool_calls=self.config.max_tool_calls, ) - effective_prompt = prompt + self.hooks.fire(start_ctx) + effective_prompt = initial if skip_user_prompt else normalize_content(start_ctx.prompt, label="run_start prompt") if not skip_user_prompt: - prompt_ctx = UserPromptSubmitContext(harness=self, metadata=dict(run_metadata), prompt=prompt) + prompt_ctx = UserPromptSubmitContext(harness=self, metadata=dict(run_metadata), prompt=effective_prompt) self.hooks.fire(prompt_ctx) if prompt_ctx.cancelled: reason = prompt_ctx.cancel_reason or "unspecified" run_ctx.stop_reason = "cancelled_by_hook" run_ctx.terminal_error = HarnessError(f"run blocked by hook: {reason}") raise run_ctx.terminal_error - effective_prompt = apply_prompt_context(prompt, prompt_ctx.additional_context) + submitted = normalize_content(prompt_ctx.prompt, label="user_prompt_submit prompt") + effective_prompt = apply_prompt_context(submitted, prompt_ctx.additional_context) + run_ctx.set_prompt_content(effective_prompt) instructions = structured_instructions(self.system_instructions(), self.output_schema) agent_span.for_each( lambda span, option: annotate_agent_start( span, - prompt=prompt, + prompt=effective_prompt, instructions=instructions, capture_messages=option.capture_messages, top_level=not self._is_child_harness, @@ -617,7 +628,7 @@ def _pending_approval_record(self, call: ModelToolCall) -> PendingApproval: """Return the host-facing pending approval shape for one call.""" return PendingApproval(call_id=call.id, tool_name=call.name, arguments=call.arguments) - def run_sync(self, prompt: str, *, resume_from: dict[str, Any] | None = None, metadata: Json | None = None) -> HarnessResult: + def run_sync(self, prompt: Prompt, *, resume_from: dict[str, Any] | None = None, metadata: Json | None = None) -> HarnessResult: """Synchronous wrapper around run.""" if self._running: raise HarnessError("Harness.run is not re-entrant") diff --git a/thinharness/hooks.py b/thinharness/hooks.py index 7b6041f..5ae72c0 100644 --- a/thinharness/hooks.py +++ b/thinharness/hooks.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar, Literal +from .content import ContentBlock, Prompt, TextBlock, normalize_content from .tools.base import Json, ToolEnvelope, ToolResult, ToolSpec from .types import HarnessResult, RunUsage, StopReason @@ -105,7 +106,7 @@ class RunStartContext(HookContext): """Context for a run before the first model request.""" event: ClassVar[HookEvent] = "run_start" - prompt: str + prompt: Prompt root: Path max_model_requests: int max_tool_calls: int | None = None @@ -116,7 +117,7 @@ class UserPromptSubmitContext(HookContext): """Context for the submitted user prompt before querying the model.""" event: ClassVar[HookEvent] = "user_prompt_submit" - prompt: str + prompt: Prompt additional_context: list[str] = field(default_factory=list) cancelled: bool = False cancel_reason: str = "" @@ -241,7 +242,7 @@ def fire_after_tool_call(self, ctx: AfterToolCallContext) -> None: _mark_strict_hook_exception(exc) raise if ctx.output != before_output: - ctx.envelope = ToolResult.from_json(ctx.output) + ctx.envelope = ToolResult.from_json(ctx.output, strict=True) elif ctx.envelope.to_json() != before_envelope: ctx.output = ctx.envelope.to_json() @@ -264,12 +265,13 @@ def _matches(self, hook: Hook, ctx: HookContext) -> bool: return True -def apply_prompt_context(prompt: str, additional_context: list[str]) -> str: - """Append hook-provided context to the submitted prompt.""" +def apply_prompt_context(prompt: Prompt, additional_context: list[str]) -> tuple[ContentBlock, ...]: + """Append hook-provided context to normalized submitted content.""" + content = normalize_content(prompt, label="hook prompt") if not additional_context: - return prompt + return content context = "\n\n".join(additional_context) - return f"{prompt}\n\n\n{context}\n" + return (*content, TextBlock(f"\n{context}\n")) def _handler_name(handler: HookHandler) -> str: diff --git a/thinharness/plugins/filesystem.py b/thinharness/plugins/filesystem.py index 48e0e30..5814634 100644 --- a/thinharness/plugins/filesystem.py +++ b/thinharness/plugins/filesystem.py @@ -19,6 +19,7 @@ class _FilesystemConfig: output_dir: str | Path | None max_read_chars: int max_read_bytes: int + max_image_bytes: int max_tool_chars: int max_search_line_chars: int rg_timeout: int @@ -77,6 +78,7 @@ def __init__( output_dir: str | Path | None = None, max_read_chars: int = 40_000, max_read_bytes: int = 1_000_000, + max_image_bytes: int = 5_000_000, max_tool_chars: int = 40_000, max_search_line_chars: int = 180, rg_timeout: int = 30, @@ -84,6 +86,8 @@ def __init__( read_paths: Sequence[str | Path] | None = None, write_paths: Sequence[str | Path] | None = None, ) -> None: + if not isinstance(max_image_bytes, int) or isinstance(max_image_bytes, bool) or max_image_bytes <= 0: + raise ValueError("max_image_bytes must be a positive integer") if isinstance(tools, (set, frozenset)): raise TypeError("FilesystemPlugin tools must be an ordered sequence, not a set") selected = tuple(_DEFAULT_TOOLS if tools is None else tools) @@ -94,6 +98,7 @@ def __init__( output_dir=output_dir, max_read_chars=max_read_chars, max_read_bytes=max_read_bytes, + max_image_bytes=max_image_bytes, max_tool_chars=max_tool_chars, max_search_line_chars=max_search_line_chars, rg_timeout=rg_timeout, @@ -115,6 +120,7 @@ def bind(self, context: PluginContext) -> PluginBinding: output_dir=config.output_dir, max_read_chars=config.max_read_chars, max_read_bytes=config.max_read_bytes, + max_image_bytes=config.max_image_bytes, max_tool_chars=config.max_tool_chars, max_search_line_chars=config.max_search_line_chars, rg_timeout=config.rg_timeout, diff --git a/thinharness/projections.py b/thinharness/projections.py index 8888c5b..2f633d3 100644 --- a/thinharness/projections.py +++ b/thinharness/projections.py @@ -5,6 +5,7 @@ from dataclasses import dataclass, field from typing import Literal +from .content import Prompt, TextBlock, normalize_content, redacted_content_json, text_only_value from .events import StreamToolCall from .providers import ( AssistantEntry, @@ -14,7 +15,7 @@ ToolResultEntry, TranscriptEntry, UserEntry, - append_notices_to_text, + append_notices_to_content, render_model_notices, ) from .types import Json @@ -35,14 +36,14 @@ class ModelRequestDelta: def model_request_delta_from_prompt( *, kind: Literal["start", "resume", "correction"], - prompt: str, + prompt: Prompt, notices: list[ModelNotice], structured_output: str | None, ) -> ModelRequestDelta: """Build a request delta for a user-text provider continuation.""" return ModelRequestDelta( kind=kind, - entries=[UserEntry(content=append_notices_to_text(prompt, notices))], + entries=[UserEntry(content=append_notices_to_content(normalize_content(prompt), notices))], notices=list(notices), structured_output=structured_output, ) @@ -57,11 +58,11 @@ def model_request_delta_from_tool_outputs( ) -> ModelRequestDelta: """Build a request delta for a tool-output provider continuation.""" entries: list[TranscriptEntry] = [ - ToolResultEntry(call_id=output.call_id, output=output.output) + ToolResultEntry(call_id=output.call_id, result=output.result) for output in outputs ] if notice_text := render_model_notices(notices): - entries.append(UserEntry(content=notice_text, notice=True)) + entries.append(UserEntry(content=(TextBlock(notice_text),), notice=True)) return ModelRequestDelta( kind=kind, entries=entries, @@ -75,14 +76,25 @@ def trace_input_messages_from_entries(entries: list[TranscriptEntry]) -> list[Js messages: list[Json] = [] for entry in entries: if isinstance(entry, UserEntry): - messages.append({"role": "user", "parts": [{"type": "text", "content": entry.content}]}) + text = text_only_value(entry.content) + parts = ( + [{"type": "text", "content": text}] + if text is not None + else [ + {"type": "text", "content": part["text"]} + if part["type"] == "text" + else part + for part in redacted_content_json(entry.content) + ] + ) + messages.append({"role": "user", "parts": parts}) elif isinstance(entry, ToolResultEntry): messages.append({ "role": "tool", "parts": [{ "type": "tool_result", "id": entry.call_id, - "content": entry.output, + "content": entry.result.redacted_json(), }], }) else: @@ -113,13 +125,14 @@ def model_request_input_from_delta(delta: ModelRequestDelta) -> Json | None: """Return the trace display payload for one model-visible request delta.""" if len(delta.entries) == 1 and isinstance(delta.entries[0], UserEntry): content = delta.entries[0].content + projected: str | list[Json] = text_only_value(content) or redacted_content_json(content) if delta.kind in {"start", "resume"}: - return {"prompt": content} + return {"prompt": projected} if delta.kind == "correction": - return {"correction": content} + return {"correction": projected} tool_outputs = [ - {"call_id": entry.call_id, "output": entry.output} + {"call_id": entry.call_id, "output": entry.result.redacted_json()} for entry in delta.entries if isinstance(entry, ToolResultEntry) ] diff --git a/thinharness/providers.py b/thinharness/providers.py index 3121799..b05ef07 100644 --- a/thinharness/providers.py +++ b/thinharness/providers.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import base64 import copy import json import logging @@ -17,7 +18,18 @@ import httpx from pydantic import BaseModel, Field -from .tools.base import Json +from .content import ( + ImageBlock, + NormalizedContent, + Prompt, + TextBlock, + append_text_block, + content_from_json, + content_to_json, + normalize_content, + text_only_value, +) +from .tools.base import Json, ToolResult from .types import HarnessError logger = logging.getLogger(__name__) @@ -91,7 +103,7 @@ class AssistantEntry: class UserEntry: """Provider-neutral user transcript entry.""" - content: str + content: NormalizedContent notice: bool = False @@ -100,7 +112,7 @@ class ToolResultEntry: """Provider-neutral tool-result transcript entry.""" call_id: str - output: str + result: ToolResult TranscriptEntry = AssistantEntry | UserEntry | ToolResultEntry @@ -111,7 +123,19 @@ class ToolOutput: """A normalized local tool output.""" call_id: str - output: str + result: ToolResult + + def __post_init__(self) -> None: + """Normalize direct low-level text outputs into tool envelopes.""" + if isinstance(self.result, str): + self.result = ToolResult(True, self.result) + elif not isinstance(self.result, ToolResult): + raise TypeError("ToolOutput.result must be a ToolResult") + + @property + def output(self) -> str: + """Return the canonical text envelope.""" + return self.result.to_json() @dataclass(frozen=True) @@ -201,7 +225,7 @@ class ModelSession(Protocol): async def start( self, - prompt: str, + prompt: Prompt, constants: RequestConstants, *, previous_response_id: str | None = None, @@ -220,14 +244,14 @@ async def continue_with_tools( """Continue a model run with tool outputs.""" ... - async def continue_with_user_text( + async def continue_with_user_content( self, - text: str, + content: Prompt, constants: RequestConstants, *, notices: list[ModelNotice] | None = None, ) -> ModelTurn: - """Continue a model run with user text (a correction or a resumed prompt).""" + """Continue a model run with user content (a correction or resumed prompt).""" ... def dump_state(self) -> dict[str, Any] | None: @@ -292,10 +316,10 @@ def _retry_delay(base: float, retry_index: int, retry_after: str | None = None) _TRANSCRIPT_ENTRY_KEYS = { "assistant": frozenset({"role", "text", "tool_calls", "reasoning"}), "user": frozenset({"role", "content", "notice"}), - "tool": frozenset({"role", "call_id", "output"}), + "tool": frozenset({"role", "call_id", "ok", "content", "metadata"}), } _REASONING_PART_KEYS = frozenset({"text", "signature", "id", "provider_name", "provider_details"}) -_TRANSCRIPT_VERSION = 3 +_TRANSCRIPT_VERSION = 4 def _validate_resume_state(state: dict[str, Any]) -> list[TranscriptEntry]: @@ -334,9 +358,9 @@ def _transcript_state(*, model: Model, entries: list[TranscriptEntry]) -> dict[s def _transcript_entry_to_dict(entry: TranscriptEntry) -> Json: if isinstance(entry, UserEntry): - return {"role": "user", "content": entry.content, "notice": entry.notice} + return {"role": "user", "content": content_to_json(entry.content), "notice": entry.notice} if isinstance(entry, ToolResultEntry): - return {"role": "tool", "call_id": entry.call_id, "output": entry.output} + return {"role": "tool", "call_id": entry.call_id, **entry.result.to_value()} return { "role": "assistant", "text": entry.text, @@ -370,13 +394,21 @@ def _transcript_entry_from_dict(value: Any) -> TranscriptEntry: if set(value) != _TRANSCRIPT_ENTRY_KEYS[role]: raise HarnessError(f"resume_from entry {role!r} has wrong keys") if role == "user": - if not isinstance(value["content"], str) or type(value["notice"]) is not bool: + if type(value["notice"]) is not bool: raise HarnessError("resume_from user entry has wrong type") - return UserEntry(content=value["content"], notice=value["notice"]) + try: + content = content_from_json(value["content"], label="resume_from user content") + except (TypeError, ValueError) as exc: + raise HarnessError(str(exc)) from exc + return UserEntry(content=content, notice=value["notice"]) if role == "tool": - if not isinstance(value["call_id"], str) or not isinstance(value["output"], str): + if not isinstance(value["call_id"], str): raise HarnessError("resume_from tool entry has wrong type") - return ToolResultEntry(call_id=value["call_id"], output=value["output"]) + try: + result = ToolResult.from_value({key: value[key] for key in ("ok", "content", "metadata")}, label="resume_from tool entry") + except (TypeError, ValueError) as exc: + raise HarnessError(str(exc)) from exc + return ToolResultEntry(call_id=value["call_id"], result=result) if not isinstance(value["text"], str) or not isinstance(value["tool_calls"], list) or not isinstance(value["reasoning"], list): raise HarnessError("resume_from assistant entry has wrong type") return AssistantEntry( @@ -415,9 +447,9 @@ def _model_tool_call_from_dict(value: Any) -> ModelToolCall: def _append_tool_results(transcript: list[TranscriptEntry], outputs: list[ToolOutput], notice_text: str) -> None: - transcript.extend(ToolResultEntry(call_id=output.call_id, output=output.output) for output in outputs) + transcript.extend(ToolResultEntry(call_id=output.call_id, result=copy.deepcopy(output.result)) for output in outputs) if notice_text: - transcript.append(UserEntry(content=notice_text, notice=True)) + transcript.append(UserEntry(content=(TextBlock(notice_text),), notice=True)) def _append_assistant_turn(transcript: list[TranscriptEntry], turn: ModelTurn) -> None: @@ -744,7 +776,7 @@ def __init__(self, model: OpenAIResponsesModel) -> None: async def start( self, - prompt: str, + prompt: Prompt, constants: RequestConstants, *, previous_response_id: str | None = None, @@ -752,10 +784,11 @@ async def start( ) -> ModelTurn: """Start a Responses API run.""" self.previous_response_id = previous_response_id - input_text = append_notices_to_text(prompt, notices) - self.transcript = [UserEntry(content=input_text)] + content = append_notices_to_content(normalize_content(prompt, label="prompt"), notices) + self.transcript = [UserEntry(content=content)] + input_payload = _openai_user_input(content) payload = self.model.build_payload( - input_payload=input_text, + input_payload=input_payload, instructions=constants.instructions, tools=constants.tools, metadata=constants.metadata, @@ -774,7 +807,7 @@ async def continue_with_tools( ) -> ModelTurn: """Continue a Responses API run with function_call_output items.""" input_payload: list[Json] = [ - {"type": "function_call_output", "call_id": output.call_id, "output": output.output} + {"type": "function_call_output", "call_id": output.call_id, "output": _openai_tool_output(output.result)} for output in outputs ] notice_text = render_model_notices(notices) @@ -797,18 +830,19 @@ async def continue_with_tools( payload["previous_response_id"] = self.previous_response_id return await self._complete(payload) - async def continue_with_user_text( + async def continue_with_user_content( self, - text: str, + content: Prompt, constants: RequestConstants, *, notices: list[ModelNotice] | None = None, ) -> ModelTurn: - """Continue a Responses API run with user text.""" - input_text = append_notices_to_text(text, notices) - self.transcript.append(UserEntry(content=input_text)) + """Continue a Responses API run with user content.""" + normalized = append_notices_to_content(normalize_content(content), notices) + self.transcript.append(UserEntry(content=normalized)) + input_payload = _openai_user_input(normalized) payload = self.model.build_payload( - input_payload=self._prepend_replay(input_text), + input_payload=self._prepend_replay(input_payload), instructions=constants.instructions, tools=constants.tools, metadata=constants.metadata, @@ -844,7 +878,7 @@ def _prepend_replay(self, input_payload: str | list[Json]) -> str | list[Json]: replay = _render_openai_transcript(self._pending_replay, encrypted_reasoning_ok=_openai_supports_encrypted_reasoning(self.model.model)) self._pending_replay = None if isinstance(input_payload, str): - return [*replay, _openai_user_item(input_payload)] + return [*replay, _openai_user_item((TextBlock(input_payload),))] return [*replay, *input_payload] @@ -898,7 +932,7 @@ def __init__(self, model: AnthropicMessagesModel) -> None: async def start( self, - prompt: str, + prompt: Prompt, constants: RequestConstants, *, previous_response_id: str | None = None, @@ -908,8 +942,8 @@ async def start( if previous_response_id: raise ProviderError("previous_response_id is only supported by OpenAI Responses") self.system = constants.instructions - content = append_notices_to_text(prompt, notices) - self.messages = [{"role": "user", "content": content}] + content = append_notices_to_content(normalize_content(prompt, label="prompt"), notices) + self.messages = [{"role": "user", "content": _anthropic_user_content(content)}] self.transcript = [UserEntry(content=content)] return await self._complete(tools=constants.tools, metadata=constants.metadata, structured_output=constants.structured_output) @@ -921,7 +955,7 @@ async def continue_with_tools( notices: list[ModelNotice] | None = None, ) -> ModelTurn: """Continue an Anthropic Messages run with tool_result blocks.""" - content = [{"type": "tool_result", "tool_use_id": output.call_id, "content": output.output} for output in outputs] + content = [{"type": "tool_result", "tool_use_id": output.call_id, "content": _anthropic_tool_output(output.result)} for output in outputs] notice_text = render_model_notices(notices) if notice_text: content.append({"type": "text", "text": notice_text}) @@ -933,18 +967,18 @@ async def continue_with_tools( }) return await self._complete(tools=constants.tools, metadata=constants.metadata, structured_output=constants.structured_output) - async def continue_with_user_text( + async def continue_with_user_content( self, - text: str, + content: Prompt, constants: RequestConstants, *, notices: list[ModelNotice] | None = None, ) -> ModelTurn: - """Continue an Anthropic Messages run with user text.""" - content = append_notices_to_text(text, notices) - self.transcript.append(UserEntry(content=content)) + """Continue an Anthropic Messages run with user content.""" + normalized = append_notices_to_content(normalize_content(content), notices) + self.transcript.append(UserEntry(content=normalized)) self._apply_resume(constants.instructions) - self.messages.append({"role": "user", "content": content}) + self.messages.append({"role": "user", "content": _anthropic_user_content(normalized)}) return await self._complete(tools=constants.tools, metadata=constants.metadata, structured_output=constants.structured_output) def dump_state(self) -> dict[str, Any] | None: @@ -1063,7 +1097,7 @@ def __init__(self, model: OpenRouterModel) -> None: async def start( self, - prompt: str, + prompt: Prompt, constants: RequestConstants, *, previous_response_id: str | None = None, @@ -1072,10 +1106,10 @@ async def start( """Start an OpenRouter run.""" if previous_response_id: raise ProviderError("previous_response_id is only supported by OpenAI Responses") - content = append_notices_to_text(prompt, notices) + content = append_notices_to_content(normalize_content(prompt, label="prompt"), notices) self.messages = [ {"role": "system", "content": constants.instructions}, - {"role": "user", "content": content}, + {"role": "user", "content": _openrouter_user_content(content)}, ] self.transcript = [UserEntry(content=content)] return await self._complete(tools=constants.tools, metadata=constants.metadata, structured_output=constants.structured_output) @@ -1092,23 +1126,28 @@ async def continue_with_tools( _append_tool_results(self.transcript, outputs, notice_text) self._apply_resume(constants.instructions) for output in outputs: - self.messages.append({"role": "tool", "tool_call_id": output.call_id, "content": output.output}) - if notice_text: + self.messages.append({"role": "tool", "tool_call_id": output.call_id, "content": _openrouter_tool_output_json(output.result)}) + image_parts = _openrouter_tool_image_parts(outputs) + if image_parts: + if notice_text: + image_parts.append({"type": "text", "text": notice_text}) + self.messages.append({"role": "user", "content": image_parts}) + elif notice_text: self.messages.append({"role": "user", "content": notice_text}) return await self._complete(tools=constants.tools, metadata=constants.metadata, structured_output=constants.structured_output) - async def continue_with_user_text( + async def continue_with_user_content( self, - text: str, + content: Prompt, constants: RequestConstants, *, notices: list[ModelNotice] | None = None, ) -> ModelTurn: - """Continue an OpenRouter run with user text.""" - content = append_notices_to_text(text, notices) - self.transcript.append(UserEntry(content=content)) + """Continue an OpenRouter run with user content.""" + normalized = append_notices_to_content(normalize_content(content), notices) + self.transcript.append(UserEntry(content=normalized)) self._apply_resume(constants.instructions) - self.messages.append({"role": "user", "content": content}) + self.messages.append({"role": "user", "content": _openrouter_user_content(normalized)}) return await self._complete(tools=constants.tools, metadata=constants.metadata, structured_output=constants.structured_output) def dump_state(self) -> dict[str, Any] | None: @@ -1269,6 +1308,117 @@ def append_notices_to_text(text: str, notices: list[ModelNotice] | None) -> str: return text if not notice_text else f"{text}\n\n{notice_text}" +def append_notices_to_content(content: NormalizedContent, notices: list[ModelNotice] | None) -> NormalizedContent: + """Append notices as one final text block.""" + return append_text_block(content, render_model_notices(notices)) + + +def _data_url(block: ImageBlock) -> str: + """Encode one image as a provider data URL.""" + return f"data:{block.media_type};base64,{base64.b64encode(block.data).decode('ascii')}" + + +def _openai_content_parts(content: NormalizedContent, *, text_type: str = "input_text", image_type: str = "input_image") -> list[Json]: + """Map neutral content to OpenAI Responses content parts.""" + return [ + {"type": text_type, "text": block.text} + if isinstance(block, TextBlock) + else {"type": image_type, "image_url": _data_url(block)} + for block in content + ] + + +def _openai_user_input(content: NormalizedContent) -> str | list[Json]: + """Keep all-text input scalar and use a message item for images.""" + text = text_only_value(content) + return text if text is not None else [_openai_user_item(content)] + + +def _openai_tool_output(result: ToolResult) -> str | list[Json]: + """Map one canonical result to Responses function-call output.""" + if not result.has_image: + return result.to_json() + header = json.dumps({"ok": result.ok, "metadata": result.metadata}, ensure_ascii=False, separators=(",", ":")) + return [{"type": "input_text", "text": header}, *_openai_content_parts(result.blocks)] + + +def _anthropic_content_parts(content: NormalizedContent) -> list[Json]: + """Map neutral content to Anthropic content blocks.""" + return [ + {"type": "text", "text": block.text} + if isinstance(block, TextBlock) + else { + "type": "image", + "source": { + "type": "base64", + "media_type": block.media_type, + "data": base64.b64encode(block.data).decode("ascii"), + }, + } + for block in content + ] + + +def _anthropic_user_content(content: NormalizedContent) -> str | list[Json]: + """Keep all-text messages scalar and use blocks for images.""" + return text_only_value(content) or _anthropic_content_parts(content) + + +def _anthropic_tool_output(result: ToolResult) -> str | list[Json]: + """Map one canonical result to an Anthropic tool-result value.""" + if not result.has_image: + return result.to_json() + header = json.dumps({"ok": result.ok, "metadata": result.metadata}, ensure_ascii=False, separators=(",", ":")) + return [{"type": "text", "text": header}, *_anthropic_content_parts(result.blocks)] + + +def _openrouter_content_parts(content: NormalizedContent) -> list[Json]: + """Map neutral content to OpenRouter chat parts.""" + return [ + {"type": "text", "text": block.text} + if isinstance(block, TextBlock) + else {"type": "image_url", "image_url": {"url": _data_url(block)}} + for block in content + ] + + +def _openrouter_user_content(content: NormalizedContent) -> str | list[Json]: + """Keep all-text messages scalar and use parts for images.""" + return text_only_value(content) or _openrouter_content_parts(content) + + +def _openrouter_tool_output_json(result: ToolResult) -> str: + """Serialize a tool result with image descriptors instead of image bytes.""" + if not result.has_image: + return result.to_json() + projected: list[Json] = [] + for index, block in enumerate(result.blocks): + if isinstance(block, TextBlock): + projected.append({"type": "text", "text": block.text}) + else: + projected.append({ + "type": "image", + "media_type": block.media_type, + "size_bytes": len(block.data), + "block_index": index, + }) + return json.dumps({"ok": result.ok, "content": projected, "metadata": result.metadata}, ensure_ascii=False) + + +def _openrouter_tool_image_parts(outputs: list[ToolOutput]) -> list[Json]: + """Build labelled user-message parts for an ordered tool batch.""" + parts: list[Json] = [] + for output in outputs: + for index, block in enumerate(output.result.blocks): + if not isinstance(block, ImageBlock): + continue + parts.extend([ + {"type": "text", "text": f"[tool image call_id={output.call_id} block={index}]"}, + {"type": "image_url", "image_url": {"url": _data_url(block)}}, + ]) + return parts + + def _thinking_fallback(text: str) -> str: """Render reasoning text as a degraded cross-provider thinking block.""" return f"\n{text}\n" @@ -1297,10 +1447,7 @@ def _render_anthropic_transcript(entries: list[TranscriptEntry], *, thinking_ena while index < len(entries): entry = entries[index] if isinstance(entry, UserEntry): - if entry.notice: - messages.append({"role": "user", "content": [{"type": "text", "text": entry.content}]}) - else: - messages.append({"role": "user", "content": entry.content}) + messages.append({"role": "user", "content": _anthropic_user_content(entry.content)}) index += 1 continue if isinstance(entry, AssistantEntry): @@ -1328,12 +1475,12 @@ def _render_anthropic_transcript(entries: list[TranscriptEntry], *, thinking_ena while index < len(entries) and isinstance(entries[index], ToolResultEntry): tool_entry = entries[index] assert isinstance(tool_entry, ToolResultEntry) - content.append({"type": "tool_result", "tool_use_id": tool_entry.call_id, "content": tool_entry.output}) + content.append({"type": "tool_result", "tool_use_id": tool_entry.call_id, "content": _anthropic_tool_output(tool_entry.result)}) index += 1 if index < len(entries): notice_entry = entries[index] if isinstance(notice_entry, UserEntry) and notice_entry.notice: - content.append({"type": "text", "text": notice_entry.content}) + content.extend(_anthropic_content_parts(notice_entry.content)) index += 1 messages.append({"role": "user", "content": content}) return messages @@ -1342,46 +1489,69 @@ def _render_anthropic_transcript(entries: list[TranscriptEntry], *, thinking_ena def _render_openrouter_transcript(entries: list[TranscriptEntry]) -> list[Json]: """Render neutral transcript entries as OpenRouter chat history.""" messages: list[Json] = [] - for entry in entries: + index = 0 + while index < len(entries): + entry = entries[index] if isinstance(entry, UserEntry): - messages.append({"role": "user", "content": entry.content}) - elif isinstance(entry, ToolResultEntry): - messages.append({"role": "tool", "tool_call_id": entry.call_id, "content": entry.output}) - else: - message: Json = {"role": "assistant"} - reasoning_details = [ - part.provider_details - for part in entry.reasoning - if part.provider_name == "openrouter" and part.provider_details is not None - ] - fallback_blocks = [ - _thinking_fallback(part.text) - for part in entry.reasoning - if not (part.provider_name == "openrouter" and part.provider_details is not None) and part.text + messages.append({"role": "user", "content": _openrouter_user_content(entry.content)}) + index += 1 + continue + if isinstance(entry, ToolResultEntry): + outputs: list[ToolOutput] = [] + while index < len(entries) and isinstance(entries[index], ToolResultEntry): + tool_entry = entries[index] + assert isinstance(tool_entry, ToolResultEntry) + output = ToolOutput(tool_entry.call_id, tool_entry.result) + outputs.append(output) + messages.append({"role": "tool", "tool_call_id": output.call_id, "content": _openrouter_tool_output_json(output.result)}) + index += 1 + notice: UserEntry | None = None + if index < len(entries): + candidate = entries[index] + if isinstance(candidate, UserEntry) and candidate.notice: + notice = candidate + index += 1 + image_parts = _openrouter_tool_image_parts(outputs) + if image_parts: + if notice is not None: + image_parts.extend(_openrouter_content_parts(notice.content)) + messages.append({"role": "user", "content": image_parts}) + elif notice is not None: + messages.append({"role": "user", "content": _openrouter_user_content(notice.content)}) + continue + message: Json = {"role": "assistant"} + reasoning_details = [ + part.provider_details + for part in entry.reasoning + if part.provider_name == "openrouter" and part.provider_details is not None + ] + fallback_blocks = [ + _thinking_fallback(part.text) + for part in entry.reasoning + if not (part.provider_name == "openrouter" and part.provider_details is not None) and part.text + ] + if reasoning_details: + message["reasoning_details"] = reasoning_details + text = "\n\n".join([*fallback_blocks, *([entry.text] if entry.text else [])]) + if text: + message["content"] = text + if entry.tool_calls: + message["tool_calls"] = [ + {"id": call.id, "type": "function", "function": {"name": call.name, "arguments": call.arguments}} + for call in entry.tool_calls ] - if reasoning_details: - message["reasoning_details"] = reasoning_details - text = "\n\n".join([*fallback_blocks, *([entry.text] if entry.text else [])]) - if text: - message["content"] = text - if entry.tool_calls: - message["tool_calls"] = [ - { - "id": call.id, - "type": "function", - "function": {"name": call.name, "arguments": call.arguments}, - } - for call in entry.tool_calls - ] - if not text and not entry.tool_calls: - message["content"] = "" - messages.append(message) + if not text and not entry.tool_calls: + message["content"] = "" + messages.append(message) + index += 1 return messages -def _openai_user_item(text: str) -> Json: +def _openai_user_item(content: NormalizedContent) -> Json: """Render one Responses API user message item.""" - return {"type": "message", "role": "user", "content": [{"type": "input_text", "text": text}]} + text = text_only_value(content) + parts = [{"type": "input_text", "text": text}] if text is not None else _openai_content_parts(content) + return {"type": "message", "role": "user", "content": parts} def _render_openai_transcript(entries: list[TranscriptEntry], *, encrypted_reasoning_ok: bool = False) -> list[Json]: @@ -1391,7 +1561,7 @@ def _render_openai_transcript(entries: list[TranscriptEntry], *, encrypted_reaso if isinstance(entry, UserEntry): items.append(_openai_user_item(entry.content)) elif isinstance(entry, ToolResultEntry): - items.append({"type": "function_call_output", "call_id": entry.call_id, "output": entry.output}) + items.append({"type": "function_call_output", "call_id": entry.call_id, "output": _openai_tool_output(entry.result)}) else: for part in entry.reasoning: if encrypted_reasoning_ok and part.provider_name == "openai" and part.signature and part.id: diff --git a/thinharness/runtime.py b/thinharness/runtime.py index 4954642..5a5ac85 100644 --- a/thinharness/runtime.py +++ b/thinharness/runtime.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Literal, Protocol, cast from .approvals import build_approval_envelope +from .content import ImageBlock, NormalizedContent, Prompt, redact_image_data from .events import ( HarnessStreamEvent, LimitWarningEvent, @@ -166,7 +167,7 @@ class RunContext: """Mutable state for one harness run.""" harness: Harness - prompt: str + prompt: Prompt metadata: Json usage: RunUsage responses: list[Json] = field(default_factory=list) @@ -181,6 +182,16 @@ class RunContext: agent_span: _TraceSpan | None = None stream: RunStreamContext | None = None emitter: StreamEmitter | None = None + image_blocks: list[ImageBlock] = field(default_factory=list) + + def set_prompt_content(self, content: NormalizedContent) -> None: + """Record normalized prompt images for later error redaction.""" + self.prompt = content + self.image_blocks.extend(block for block in content if isinstance(block, ImageBlock)) + + def record_tool_result_images(self, result: Any) -> None: + """Record tool-result images for later error redaction.""" + self.image_blocks.extend(block for block in result.blocks if isinstance(block, ImageBlock)) def stream_base(self) -> dict[str, Any]: """Return common event metadata for this run.""" @@ -271,7 +282,7 @@ async def advance_model( *, request_kind: ModelRequestKind, structured_output: str | None, - prompt: str | None = None, + prompt: NormalizedContent | None = None, tool_outputs: list[ToolOutput] | None = None, output_retry: bool = False, ) -> tuple[ModelTurn, OutputTurnDecision]: @@ -335,8 +346,9 @@ async def advance_model( self.usage.output_tokens += turn.usage.output_tokens or 0 self.usage.cached_tokens += turn.usage.cached_tokens or 0 except Exception as exc: - model_span.record_exception(exc) - model_span.set_error(str(exc), type(exc).__name__) + message = redact_image_data(str(exc), self.image_blocks) + model_span.record_exception(HarnessError(message)) + model_span.set_error(message, type(exc).__name__) raise model_span.for_each( lambda span, option: annotate_model_span( diff --git a/thinharness/tool_execution.py b/thinharness/tool_execution.py index 1453ad8..df4a8a2 100644 --- a/thinharness/tool_execution.py +++ b/thinharness/tool_execution.py @@ -89,11 +89,13 @@ async def execute_batch( results = await self._run_calls_concurrently(calls, indices) records = [] for call, execution in zip(calls, results, strict=True): - record = {"call": {"id": call.id, "name": call.name, "arguments": call.arguments}, "output": execution.output} + record = {"call": {"id": call.id, "name": call.name, "arguments": call.arguments}, "result": execution.envelope.to_value()} + if not execution.envelope.has_image: + record["output"] = execution.output if execution.cancelled: record["cancelled"] = True records.append(record) - outputs = [ToolOutput(call.id, execution.output) for call, execution in zip(calls, results, strict=True)] + outputs = [ToolOutput(call.id, execution.envelope) for call, execution in zip(calls, results, strict=True)] return records, outputs, results def _should_run_sequentially(self, calls: list[ModelToolCall]) -> bool: @@ -216,11 +218,14 @@ async def execute_one(self, call: ModelToolCall, index: int) -> ToolCallExecutio self.harness.hooks.fire_after_tool_call(after) output = after.output envelope = after.envelope + retry_kind = None if cancelled else envelope.retry_kind() + self.run_context.record_tool_result_images(envelope) + projected_output = envelope.redacted_json() self._annotate_special_tool(span, call.name, envelope, composition) span.set_attribute_where( lambda option: option.capture_tool_results, "gen_ai.tool.call.result", - serialize_attribute_value(output), + serialize_attribute_value(projected_output), ) if retry_kind is not None: span.set_error(f'Tool "{call.name}" failed', retry_kind) @@ -249,7 +254,7 @@ async def execute_one(self, call: ModelToolCall, index: int) -> ToolCallExecutio error_type=type(exc).__name__, message=str(exc), duration_ms=(time.perf_counter() - start) * 1000, - output=output, + output=(envelope.redacted_json() if envelope is not None else output), ) ) raise @@ -279,9 +284,9 @@ def _emit_completed( cancelled=cancelled, retry_kind=retry_kind, error_type=envelope.error_type(), - message=envelope.content if not envelope.ok else None, + message=envelope.message_text() if not envelope.ok else None, duration_ms=duration_ms, - output=output, + output=envelope.redacted_json(), ) ) diff --git a/thinharness/tools/base.py b/thinharness/tools/base.py index 726789e..5c8ac9d 100644 --- a/thinharness/tools/base.py +++ b/thinharness/tools/base.py @@ -15,6 +15,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError +from ..content import ContentBlock, ImageBlock, TextBlock, content_from_json, content_to_json, normalize_content, redacted_content_json from ..types import Json ToolHandler = Callable[[Any], Any | Awaitable[Any]] @@ -72,39 +73,89 @@ class ToolResult: """Structured internal tool output envelope.""" ok: bool - content: str + content: str | Sequence[ContentBlock] metadata: Json = field(default_factory=dict) + def __post_init__(self) -> None: + """Detach and validate multimodal content.""" + if not isinstance(self.ok, bool): + raise TypeError("ToolResult.ok must be a bool") + if not isinstance(self.metadata, dict): + raise TypeError("ToolResult.metadata must be a dict") + if isinstance(self.content, str): + if not self.content: + raise ValueError("ToolResult.content must not be empty") + else: + self.content = normalize_content(self.content, label="ToolResult.content") + + @property + def blocks(self) -> tuple[ContentBlock, ...]: + """Return content in normalized block form.""" + return normalize_content(self.content, label="ToolResult.content") + + @property + def has_image(self) -> bool: + """Return whether this result contains image content.""" + return any(isinstance(block, ImageBlock) for block in self.blocks) + + def to_value(self) -> Json: + """Return the canonical JSON-compatible envelope value.""" + content: str | list[Json] + if isinstance(self.content, str): + content = self.content + else: + content = content_to_json(self.blocks) + return {"ok": self.ok, "content": content, "metadata": self.metadata} + @classmethod - def from_json(cls, output: str) -> ToolResult: - """Parse a provider-facing tool output string into an envelope.""" + def from_value(cls, parsed: Any, *, label: str = "tool output") -> ToolResult: + """Strictly decode one canonical envelope value.""" + if not isinstance(parsed, dict) or set(parsed) != {"ok", "content", "metadata"}: + raise ValueError(f"{label} has wrong keys") + if not isinstance(parsed["ok"], bool) or not isinstance(parsed["metadata"], dict): + raise ValueError(f"{label} has wrong type") + content = parsed["content"] + if isinstance(content, str): + if not content: + raise ValueError(f"{label} content must not be empty") + return cls(parsed["ok"], content, parsed["metadata"]) + return cls(parsed["ok"], content_from_json(content, label=f"{label} content"), parsed["metadata"]) + + @classmethod + def from_json(cls, output: str, *, strict: bool = False) -> ToolResult: + """Parse a canonical provider-facing tool output string.""" try: parsed = json.loads(output) - except json.JSONDecodeError: - return cls(False, output, {"error_type": "InvalidToolOutput"}) - if not isinstance(parsed, dict): + return cls.from_value(parsed) + except (json.JSONDecodeError, TypeError, ValueError): + if strict: + raise ValueError("tool output is not a valid canonical ToolResult") from None return cls(False, output, {"error_type": "InvalidToolOutput"}) - ok = parsed.get("ok") - content = parsed.get("content") - metadata = parsed.get("metadata") - return cls( - ok if isinstance(ok, bool) else False, - content if isinstance(content, str) else output, - metadata if isinstance(metadata, dict) else {}, - ) def to_json(self) -> str: - """Serialize the envelope for a provider-facing tool output.""" - return json.dumps( - {"ok": self.ok, "content": self.content, "metadata": self.metadata}, - ensure_ascii=False, - default=str, - ) + """Serialize the canonical provider-facing tool envelope.""" + return json.dumps(self.to_value(), ensure_ascii=False) def as_json(self) -> str: - """Serialize the envelope for compatibility with existing callers.""" + """Serialize the canonical provider-facing tool envelope.""" return self.to_json() + def message_text(self) -> str: + """Return a plain-text summary suitable for progress events.""" + if isinstance(self.content, str): + return self.content + return "\n\n".join(block.text for block in self.blocks if isinstance(block, TextBlock)) + + def redacted_json(self) -> str: + """Return the canonical envelope without image bytes.""" + if not self.has_image: + return self.to_json() + return json.dumps( + {"ok": self.ok, "content": redacted_content_json(self.blocks), "metadata": self.metadata}, + ensure_ascii=False, + separators=(",", ":"), + ) + def retry_kind(self) -> str | None: """Return the retry error type if this envelope asks the model to retry.""" error_type = self.error_type() @@ -219,7 +270,9 @@ def _normalize_result(result: Any) -> ToolEnvelope: return result if isinstance(result, str): return ToolResult(True, result) - return ToolResult(True, json.dumps(result, indent=2, sort_keys=True, default=str)) + if isinstance(result, Sequence) and all(isinstance(block, (TextBlock, ImageBlock)) for block in result): + return ToolResult(True, result) + return ToolResult(True, json.dumps(result, indent=2, sort_keys=True)) def _retry_envelope(error_type: str, message: str, *, errors: list[Json] | None = None) -> ToolEnvelope: diff --git a/thinharness/tools/filesystem.py b/thinharness/tools/filesystem.py index 0f418fa..894f97f 100644 --- a/thinharness/tools/filesystem.py +++ b/thinharness/tools/filesystem.py @@ -4,6 +4,7 @@ import heapq import itertools +import json import subprocess import time import uuid @@ -13,6 +14,7 @@ from pydantic import Field +from ..content import ImageBlock, TextBlock from ..defaults import ( DEFAULT_EDIT_DESCRIPTION, DEFAULT_EDIT_INSTRUCTIONS, @@ -60,6 +62,12 @@ class ReadArgs(StrictArgs): max_chars: int | None = Field(default=None, ge=1) +class ReadImageArgs(StrictArgs): + """Arguments for read_image.""" + + path: str + + class WriteArgs(StrictArgs): """Arguments for write.""" @@ -124,6 +132,7 @@ def __init__( output_dir: str | Path | None = None, max_read_chars: int = 40_000, max_read_bytes: int = 1_000_000, + max_image_bytes: int = 5_000_000, max_tool_chars: int = 40_000, max_search_line_chars: int = 180, rg_timeout: int = 30, @@ -141,7 +150,10 @@ def __init__( self.read_policy = PathPolicy(self.root, read_paths, "read") self.write_policy = PathPolicy(self.root, write_paths, "write") self.max_read_chars = max_read_chars + if not isinstance(max_image_bytes, int) or isinstance(max_image_bytes, bool) or max_image_bytes <= 0: + raise ValueError("max_image_bytes must be a positive integer") self.max_read_bytes = max_read_bytes + self.max_image_bytes = max_image_bytes self.max_tool_chars = max_tool_chars self.max_search_line_chars = max_search_line_chars self.rg_timeout = rg_timeout @@ -164,6 +176,12 @@ def specs(self) -> list[ToolSpec]: """Return built-in filesystem tool specs.""" return [ ToolSpec("read", DEFAULT_READ_DESCRIPTION, ReadArgs, self.read, instructions=DEFAULT_READ_INSTRUCTIONS), + ToolSpec( + "read_image", + "Read one local PNG, JPEG, GIF, or WebP image for visual inspection.", + ReadImageArgs, + self.read_image, + ), ToolSpec("write", DEFAULT_WRITE_DESCRIPTION, WriteArgs, self.write, sequential=True, instructions=DEFAULT_WRITE_INSTRUCTIONS), ToolSpec("edit", DEFAULT_EDIT_DESCRIPTION, EditArgs, self.edit, sequential=True, instructions=DEFAULT_EDIT_INSTRUCTIONS), ToolSpec("search", DEFAULT_SEARCH_DESCRIPTION, SearchArgs, self.search, instructions=DEFAULT_SEARCH_INSTRUCTIONS), @@ -219,6 +237,36 @@ def read(self, args: ReadArgs | Json) -> ToolResult: result.metadata.update({"path": str(path), "total_lines": total_lines, "returned_lines": len(selected), "size_bytes": size}) return result + def read_image(self, args: ReadImageArgs | Json) -> ToolResult: + """Read one bounded contained image without format conversion.""" + args = coerce_args(args, ReadImageArgs) + try: + path = self._resolve_read_path(args.path) + except PathValidationError as exc: + return _path_error(exc) + display = self._display(path) + try: + if not path.exists(): + return ToolResult(False, f"file not found: {display}", {"path": str(path)}) + if path.is_dir(): + return ToolResult(False, f"path is a directory: {display}", {"path": str(path)}) + size = path.stat().st_size + if size > self.max_image_bytes: + return ToolResult( + False, + f"image is {size} bytes, over max_image_bytes={self.max_image_bytes}", + {"path": str(path), "size_bytes": size, "max_image_bytes": self.max_image_bytes}, + ) + data = path.read_bytes() + except OSError as exc: + return ToolResult(False, f"{type(exc).__name__}: {exc}", {"path": str(path), "error_type": type(exc).__name__}) + media_type = _detect_image_media_type(data) + if media_type is None: + return ToolResult(False, f"unsupported or invalid image format: {display}", {"path": str(path), "size_bytes": len(data)}) + metadata: Json = {"path": str(path), "media_type": media_type, "size_bytes": len(data)} + summary = json.dumps({"path": display, "media_type": media_type, "size_bytes": len(data)}, ensure_ascii=False, separators=(",", ":")) + return ToolResult(True, (TextBlock(summary), ImageBlock(data, media_type)), metadata) # type: ignore[arg-type] + def write(self, args: WriteArgs | Json) -> ToolResult: """Write a contained UTF-8 file.""" args = coerce_args(args, WriteArgs) @@ -579,6 +627,25 @@ def _truncate(self, text: str, *, prefix: str, max_chars: int | None = None) -> ) +def _detect_image_media_type(data: bytes) -> str | None: + """Detect supported image containers from complete minimum signatures.""" + if len(data) >= 33 and data.startswith(b"\x89PNG\r\n\x1a\n") and data[8:12] == b"\x00\x00\x00\r" and data[12:16] == b"IHDR": + return "image/png" + if len(data) >= 4 and data.startswith(b"\xff\xd8\xff") and data.endswith(b"\xff\xd9"): + return "image/jpeg" + if len(data) >= 13 and data[:6] in {b"GIF87a", b"GIF89a"}: + return "image/gif" + if ( + len(data) >= 20 + and data.startswith(b"RIFF") + and data[8:12] == b"WEBP" + and data[12:16] in {b"VP8 ", b"VP8L", b"VP8X"} + and int.from_bytes(data[4:8], "little") == len(data) - 8 + ): + return "image/webp" + return None + + # ============================================================================= # Tool plumbing # ============================================================================= diff --git a/thinharness/tools/mcp.py b/thinharness/tools/mcp.py index 37d465c..f4c17e2 100644 --- a/thinharness/tools/mcp.py +++ b/thinharness/tools/mcp.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio +import base64 +import binascii import copy import json import re @@ -11,6 +13,7 @@ import httpx +from ..content import ImageBlock, TextBlock from .base import Json, ToolOrigin, ToolResult, ToolSpec _INSTALL_HINT = "Install MCP support with: pip install thinharness[mcp]" @@ -163,9 +166,12 @@ async def call_tool(self, name: str, arguments: Json, *, server_id: str | None = ) structured_content = getattr(result, "structuredContent", None) if structured_content is not None: - content = json.dumps(structured_content, ensure_ascii=False) + structured = TextBlock(json.dumps(structured_content, ensure_ascii=False)) + content = (structured, *_content_to_blocks(result.content, include_text=False)) else: - content = _content_to_text(result.content) + content = _content_to_blocks(result.content, include_text=True) + if all(isinstance(block, TextBlock) for block in content): + return ToolResult(True, "\n".join(block.text for block in content if isinstance(block, TextBlock)), base_metadata) return ToolResult(True, content, base_metadata) @@ -362,7 +368,7 @@ def _clean_mcp_schema(schema: Any, tool_name: str) -> Json: def _content_to_text(blocks: list[Any]) -> str: - """Convert MCP content blocks to model-visible text.""" + """Convert MCP content blocks to text for protocol-level failures.""" parts: list[str] = [] for block in blocks: block_type = getattr(block, "type", "") @@ -373,11 +379,48 @@ def _content_to_text(blocks: list[Any]) -> str: elif block_type == "audio": parts.append(f"[audio: {getattr(block, 'mimeType', 'unknown')}]") elif block_type in {"resource", "resource_link"}: - uri = getattr(block, "uri", None) - resource = getattr(block, "resource", None) - if uri is None and resource is not None: - uri = getattr(resource, "uri", None) - parts.append(f"[resource: {uri or 'unknown'}]") + parts.append(_resource_placeholder(block)) else: parts.append(str(block)) return "\n".join(parts) + + +def _content_to_blocks(blocks: list[Any], *, include_text: bool) -> tuple[TextBlock | ImageBlock, ...]: + """Preserve supported MCP images and ordered text placeholders.""" + parts: list[TextBlock | ImageBlock] = [] + for block in blocks: + block_type = getattr(block, "type", "") + if block_type == "text": + if include_text: + text = str(getattr(block, "text", "")) + if text: + parts.append(TextBlock(text)) + continue + if block_type == "image": + media_type = str(getattr(block, "mimeType", "unknown")) + data = getattr(block, "data", "") + if media_type in {"image/jpeg", "image/png", "image/gif", "image/webp"} and isinstance(data, str): + try: + decoded = base64.b64decode(data, validate=True) + except (binascii.Error, ValueError): + decoded = b"" + if decoded: + parts.append(ImageBlock(decoded, media_type)) # type: ignore[arg-type] + continue + parts.append(TextBlock(f"[image: {media_type}]")) + elif block_type == "audio": + parts.append(TextBlock(f"[audio: {getattr(block, 'mimeType', 'unknown')}]")) + elif block_type in {"resource", "resource_link"}: + parts.append(TextBlock(_resource_placeholder(block))) + else: + parts.append(TextBlock(str(block))) + return tuple(parts) + + +def _resource_placeholder(block: Any) -> str: + """Return the existing resource placeholder text.""" + uri = getattr(block, "uri", None) + resource = getattr(block, "resource", None) + if uri is None and resource is not None: + uri = getattr(resource, "uri", None) + return f"[resource: {uri or 'unknown'}]" diff --git a/thinharness/tracing.py b/thinharness/tracing.py index 46db21e..396b246 100644 --- a/thinharness/tracing.py +++ b/thinharness/tracing.py @@ -1,9 +1,9 @@ """OpenTelemetry-compatible tracing helpers. Model input messages are constructed from provider-neutral request deltas, -never from provider payloads. For top-level runs, the agent span stores the raw -caller prompt while the first model span stores the effective prompt after -hooks. OTel GenAI message shapes follow the semantic convention as retrieved +never from provider payloads. Agent and model spans store effective prompt +text plus redacted image descriptors after hooks. OTel GenAI message shapes +follow the semantic convention as retrieved on 2026-05-19: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/. """ @@ -25,6 +25,7 @@ from pydantic import BaseModel, ConfigDict +from .content import NormalizedContent, redacted_content_string, text_only_value from .projections import ModelRequestDelta, model_request_input_from_delta, trace_input_messages_from_entries, trace_output_messages_from_assistant from .providers import TokenUsage, extract_finish_reason, extract_response_model, extract_token_usage from .tools.base import Json @@ -509,7 +510,7 @@ def annotate_model_span(span: _SpanAdapter, turn: Any, *, capture_messages: bool def annotate_agent_start( span: _SpanAdapter, *, - prompt: str, + prompt: NormalizedContent, instructions: str, capture_messages: bool, top_level: bool, @@ -517,13 +518,14 @@ def annotate_agent_start( """Write opt-in agent input attributes before provider work runs.""" if not capture_messages: return + projected_prompt = text_only_value(prompt) or redacted_content_string(prompt) if top_level: span.set_attributes({ - "gen_ai.prompt": prompt, + "gen_ai.prompt": projected_prompt, "gen_ai.system_instructions": serialize_attribute_value([{"type": "text", "content": instructions}]), }) else: - span.set_attribute("gen_ai.prompt", prompt) + span.set_attribute("gen_ai.prompt", projected_prompt) def annotate_agent_result( @@ -557,9 +559,9 @@ def serialize_attribute_value(value: Any) -> str | None: if isinstance(value, str): return value try: - return json.dumps(value, ensure_ascii=False, default=str) + return json.dumps(value, ensure_ascii=False) except TypeError: - return str(value) + return "[unserializable value redacted]" def _usage_attributes(raw: Json, usage: TokenUsage | None) -> Json: diff --git a/thinharness/turns.py b/thinharness/turns.py index 1ff6aa1..bcca219 100644 --- a/thinharness/turns.py +++ b/thinharness/turns.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal +from .content import NormalizedContent, normalize_content from .events import ToolCallCompletedEvent, ToolCallStartedEvent from .output import FINAL_RESULT_TOOL_NAME, OutputSchema, OutputValidationError, ResolvedOutputMode from .providers import ModelSession, ModelToolCall, ModelTurn, RequestConstants, ToolOutput @@ -42,7 +43,7 @@ class TurnStart: """First-turn production input for one run.""" kind: Literal["start", "resume", "approval_resume"] - prompt: str = "" + prompt: NormalizedContent = () approval_pause: ApprovalPause | None = None approval_decisions: dict[str, ApprovalDecision] | None = None @@ -122,17 +123,17 @@ async def advance_until_terminal( output_mode = _trace_output_mode(harness.output_schema) require_dump_state = harness._model_supports_approval_resume() - async def send_user_text( - text: str, + async def send_user_content( + content: NormalizedContent, *, kind: Literal["resume", "correction"], output_retry: bool = False, ) -> tuple[ModelTurn, OutputTurnDecision]: - """Continue the run with user text.""" + """Continue the run with user content.""" return await run_ctx.advance_model( - lambda notices: session.continue_with_user_text(text, constants, notices=notices), + lambda notices: session.continue_with_user_content(content, constants, notices=notices), request_kind=kind, - prompt=text, + prompt=content, structured_output=output_mode, output_retry=output_retry, ) @@ -160,7 +161,7 @@ async def send_tool_outputs( structured_output=output_mode, ) elif start.kind == "resume": - turn, decision = await send_user_text(start.prompt, kind="resume") + turn, decision = await send_user_content(start.prompt, kind="resume") else: assert start.approval_pause is not None assert start.approval_decisions is not None @@ -184,7 +185,7 @@ async def send_tool_outputs( retry_message = decision.retry_message run_ctx.emit_retry_event("structured_output", retry_message, final_id) turn, decision = await send_tool_outputs( - [ToolOutput(final_id, retry_message)], + [ToolOutput(final_id, ToolResult(True, retry_message))], kind="output_retry_tool", output_retry=True, ) @@ -193,7 +194,7 @@ async def send_tool_outputs( run_ctx.retry_or_fail() retry_message = decision.retry_message run_ctx.emit_retry_event("structured_output", retry_message, decision.retry_call_id) - turn, decision = await send_user_text(retry_message, kind="correction", output_retry=True) + turn, decision = await send_user_content(normalize_content(retry_message), kind="correction", output_retry=True) continue if decision.kind == "unexpected": raise UnexpectedModelBehavior(decision.unexpected_message) @@ -269,7 +270,8 @@ def _reject_approval_call( message = "Tool call was rejected by a human reviewer." if decision.reason: message = f"{message}\nReason: {decision.reason}" - output = ToolResult(False, message, {"error_type": "ApprovalRejected"}).to_json() + envelope = ToolResult(False, message, {"error_type": "ApprovalRejected"}) + output = envelope.to_json() start = time.perf_counter() assert run_ctx.tracer is not None with run_ctx.tracer.tool(tool_name=call.name, call_id=call.id, arguments=call.arguments) as span: @@ -299,10 +301,11 @@ def _reject_approval_call( return ( { "call": {"id": call.id, "name": call.name, "arguments": call.arguments}, + "result": envelope.to_value(), "output": output, "approval": {"approved": False, "reason": decision.reason}, }, - ToolOutput(call.id, output), + ToolOutput(call.id, envelope), ) From d80d6a70332c02c3c89ac7bc4cabf8bf26f183b3 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Wed, 19 Aug 2026 23:20:39 -0400 Subject: [PATCH 18/30] Fix image input review findings --- docs/behavior.md | 11 +- docs/docs.md | 8 +- docs/site/explainer/index.html | 2 +- tests/e2e/image_inputs_journey.py | 30 +- tests/unit/test_harness.py | 4 +- tests/unit/test_image_inputs.py | 678 +++++++++++++++++++++++++++++- tests/unit/test_mcp.py | 61 +++ tests/unit/test_providers.py | 45 +- tests/unit/test_tracing.py | 2 +- tests/unit/test_turns.py | 4 +- thinharness/core.py | 11 +- thinharness/projections.py | 9 +- thinharness/providers.py | 65 ++- thinharness/runtime.py | 21 +- thinharness/tool_execution.py | 8 +- thinharness/tools/base.py | 49 ++- thinharness/tools/filesystem.py | 45 +- thinharness/tools/mcp.py | 17 +- thinharness/tracing.py | 6 +- thinharness/turns.py | 2 +- 20 files changed, 961 insertions(+), 117 deletions(-) diff --git a/docs/behavior.md b/docs/behavior.md index 2b13e4c..96add5b 100644 --- a/docs/behavior.md +++ b/docs/behavior.md @@ -73,9 +73,10 @@ ThinHarness accepts ordered text and local image content through one provider-ne - IMAGE-CONTENT-1: `TextBlock`, `ImageBlock`, and `Prompt` are the public content interface. Prompts accept a non-empty string or a non-empty copied sequence of immutable blocks. Text and image data must be non-empty, image data must be `bytes`, and media types are limited to JPEG, PNG, GIF, and WebP. - IMAGE-CONTENT-2: `Harness.run()`, `stream()`, and `run_sync()` normalize content before the first provider request. Run-start and user-prompt hooks receive normalized block tuples and can replace them with a valid string or block sequence. Caller blocks, hook context, and harness notices remain in that order. -- IMAGE-CONTENT-3: Tool results accept a string or ordered block sequence. Text-only provider payloads remain unchanged. Image-bearing provider payloads preserve execution outcome, metadata, tool-call id, content order, and images through provider-native OpenAI and Anthropic blocks. +- IMAGE-CONTENT-3: Tool results accept a string, including an empty string, or a non-empty ordered block sequence. Empty and ordinary JSON lists remain ordinary JSON tool data. Invalid or non-serializable handler results become failed tool envelopes. Text-only provider payloads remain unchanged. Image-bearing provider payloads preserve execution outcome, metadata, tool-call id, content order, and images through provider-native OpenAI and Anthropic blocks. - IMAGE-CONTENT-4: OpenRouter keeps a canonical JSON tool message for each result. Tool images are descriptors in that message and are projected after the parallel tool-message batch as labelled user content, with each label immediately before its image and any harness notice after all image parts. -- IMAGE-CONTENT-5: After-tool hook string fields contain canonical JSON, including base64 image fields, while the envelope is the structured mutation interface. Either mutation path is strictly validated and synchronized. These fields, completed results, and resume state can be sensitive and large. +- IMAGE-CONTENT-5: After-tool hook string fields contain canonical JSON, including base64 image fields, while the envelope is the structured mutation interface. Either mutation path is strictly validated and synchronized. Retry classification uses the final post-hook envelope. These fields, completed results, and resume state can be sensitive and large. +- IMAGE-CONTENT-5A: Every tool-call record contains canonical structured `result` and string `output` fields. Image-bearing `output` is the canonical redacted projection, while `result` retains the complete image data once. - IMAGE-CONTENT-6: Remote image URLs, image fetching, image generation, other media, conversion, OCR, image-bearing subagent tasks, and image-bearing `parallel_llm` prompts are not supported. Bash output always stays text and does not load image paths. Model image capability errors come from the selected provider or custom model, not model-name checks. ## Resume State @@ -171,7 +172,7 @@ Callers opt into root-scoped workspace tools without making filesystem behavior - FILESYSTEM-PLUGIN-5: Filesystem limits, output location, search settings, and path policies belong to `FilesystemPlugin`. - FILESYSTEM-PLUGIN-6: Independent custom tools continue to use `tools=[ToolSpec(...)]`; callers do not need to wrap one tool in a plugin. - FILESYSTEM-PLUGIN-7: `FilesystemPlugin` has the runtime-fixed name `"filesystem"`. Its constructor configuration is frozen: mutation of constructor inputs, returned property values, or plugin attributes cannot change later parent or child bindings. -- FILESYSTEM-PLUGIN-8: `read_image` reads one bounded local file, validates complete minimum JPEG, PNG, GIF, or WebP signatures, and returns metadata text followed by one image block. It rejects missing files, directories, unreadable files, path or symlink escapes, unknown formats, SVG, truncated headers, and files over `max_image_bytes`. +- FILESYSTEM-PLUGIN-8: `read_image` reads at most `max_image_bytes + 1` bytes from one regular local file, validates complete minimum JPEG, PNG, GIF, or WebP signatures, and returns metadata text followed by one image block. It rejects missing files, non-regular files, unreadable files, path or symlink escapes, unknown formats, SVG, truncated headers, and files over `max_image_bytes`; harmless trailing JPEG or WebP data is accepted. - FILESYSTEM-PLUGIN-9: `max_image_bytes` is a positive frozen setting that defaults to 5,000,000 and is independent of `max_read_bytes`; exactly the limit is accepted. Plugin binding performs no image I/O. ## Skills Plugin @@ -302,7 +303,7 @@ Tracing and streaming expose projections of the same neutral per-request model-v - MODEL-OBSERVABILITY-7: Model spans pin `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.usage.cache_read.input_tokens`, `gen_ai.usage.total_tokens`, `gen_ai.response.model`, and `gen_ai.response.finish_reasons`. `cache_read.input_tokens` carries provider-reported cached input tokens when present and is omitted when unreported. `finish_reasons` is always a list wrapping the normalized reason. `total_tokens` passes through a raw provider `total_tokens` when present and is otherwise computed as input+output only when both are present; partial usage yields no total. - MODEL-OBSERVABILITY-8: Custom `Model` implementations that do not populate normalized `ModelTurn` usage fields keep their `gen_ai.usage.*` span attributes via best-effort extraction from the raw response. - MODEL-OBSERVABILITY-9: Trace attributes and non-terminal progress events never contain image bytes, base64, or data URLs. Multimodal prompt and tool-result values use compact ordered JSON with visible text and image descriptors containing media type, byte size, and zero-based block index; existing field types and text-only values stay unchanged. -- MODEL-OBSERVABILITY-10: Provider errors and span errors redact run-known image encodings and complete image data URLs. Provider adapters do not include serialized request bodies in error messages. +- MODEL-OBSERVABILITY-10: Provider errors and span errors redact image encodings restored from resume or approval state as well as new run-known images and complete image data URLs. Redacted failures use a sanitized harness exception; failures that need no redaction preserve their original exception type. Provider adapters do not include serialized request bodies in error messages. - MODEL-OBSERVABILITY-11: `ToolCallCompletedEvent.message` remains plain text. `RunCompletedEvent.result` remains identical to the complete run result and can contain sensitive, large image-bearing records and resume state. ## MCP Client Layer @@ -318,7 +319,7 @@ ThinHarness exposes tools from MCP servers through explicit `MCPPlugin` composit - MCP-3: A wrapper owns the FastMCP client built on its transport: nested and concurrent entries share one connection, the final exit closes the transport (terminating a stdio child process), and the same wrapper can reconnect afterwards. One stateful transport object must not be reused across wrappers; reusing the same wrapper across parent, child, or independent harness bindings shares one reference-counted session. - MCP-4: Final close is bounded — the bound comes from FastMCP's `client_disconnect_timeout` setting (default 5 seconds) — and a caller cancellation consumed by transport cleanup is re-raised after cleanup completes. Cancelling a first connection or a final close propagates the cancellation and leaves the wrapper reusable. - MCP-5: `include_tools` and `exclude_tools` match original MCP tool names before prefixing and normalization; `tool_prefix`, schema cleanup, sanitized-name collision errors, and cross-contribution tool collision errors are ThinHarness behavior. Discovered MCP tools are ordinary `ToolSpec` objects with `ToolOrigin(plugin="mcp", source=resolved_server_id, attributes={"tool_name": original_tool_name})`. Tracing reads this origin from the `ToolSpec`, so an after-tool hook cannot erase attribution. Model-visible result metadata uses the same binding-local server id. -- MCP-6: Successful supported MCP image blocks with valid base64 remain ordered image blocks. Without `structuredContent`, text and image blocks keep their order. With `structuredContent`, its canonical JSON is the first text block, MCP text blocks are discarded, and images plus image placeholders keep their original relative order. Malformed or unsupported images and audio, embedded resources, and resource links become text placeholders. A protocol-level tool failure (`isError`) returns a text-only failed `ToolResult` with `error_type="MCPToolError"` and `retry=True`; known transport and protocol failures during a tool call return `error_type="MCPError"`, including when wrapped in an exception group or explicit cause chain — a group whose members are all `Exception`s is normalized when any member's cause chain holds a known failure, even alongside sibling exception noise from teardown. An exception group carrying cancellation or any other non-`Exception` failure propagates, and exceptions with no known failure in their group or cause chain propagate as programming errors. +- MCP-6: Successful supported MCP image blocks with valid base64 remain ordered image blocks. Without `structuredContent`, text and image blocks keep their order and non-image media become placeholders. With `structuredContent`, its canonical JSON is the first text block, all MCP text, audio, resource, and resource-link blocks are discarded, and only images plus image placeholders keep their original relative order. Malformed or unsupported images become text placeholders. A protocol-level tool failure (`isError`) returns a text-only failed `ToolResult` with `error_type="MCPToolError"` and `retry=True`; known transport and protocol failures during a tool call return `error_type="MCPError"`, including when wrapped in an exception group or explicit cause chain — a group whose members are all `Exception`s is normalized when any member's cause chain holds a known failure, even alongside sibling exception noise from teardown. An exception group carrying cancellation or any other non-`Exception` failure propagates, and exceptions with no known failure in their group or cause chain propagate as programming errors. - MCP-7: MCP tools and connection details never enter resume state. `MCPPlugin` does not inherit automatically into children. A child that needs MCP lists an explicit `MCPPlugin` in its plugin configuration; that child binding owns its connection lifecycle, while reuse of the same server wrapper keeps the wrapper's reference-counted session behavior. - MCP-8: The base install works without MCP packages: importing ThinHarness and constructing any wrapper or `MCPPlugin` needs no extra, and opening a connection without `mcp` or `fastmcp` raises `MCPDependencyError` with the `thinharness[mcp]` install hint. - MCP-9: `timeout` bounds MCP initialization and HTTP connection establishment; `read_timeout` bounds MCP requests, HTTP reads, and SSE reads. diff --git a/docs/docs.md b/docs/docs.md index 4985259..16c74eb 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -64,7 +64,7 @@ Stream events are high-level workflow events intended for app consumption: - `RunStartedEvent.prompt` includes the submitted text, or compact ordered JSON with redacted image descriptors for a multimodal prompt. - `ToolCallStartedEvent.arguments` includes the model-requested tool arguments. -- `ToolCallCompletedEvent.output` includes model-visible tool output. +- `ToolCallCompletedEvent.output` includes the exact text-only tool output or a compact canonical redacted projection for image-bearing output. - Raw provider response JSON is not part of stream events; use `HarnessResult.responses` for raw provider responses after completion. - `ModelMessageEvent.text` includes assistant text from the completed provider turn. - Child subagent events are flattened by default; set `include_subagents=False` to keep only the parent `subagent` tool lifecycle. @@ -131,7 +131,7 @@ The core harness has no implicit filesystem tools. Add `FilesystemPlugin()` to g - `list`: list files or directories. - `glob`: find files by glob pattern. -Use the plugin's ordered `tools` list to select a different surface. `jsonl_search` and `read_image` are opt-in. `read_image` reads a bounded JPEG, PNG, GIF, or WebP file under the read path policy and returns metadata text followed by the image. It does not change the text-only `read` tool. Configure its independent positive byte limit with `max_image_bytes` (default 5,000,000). +Use the plugin's ordered `tools` list to select a different surface. `jsonl_search` and `read_image` are opt-in. `read_image` reads at most `max_image_bytes + 1` bytes from a regular JPEG, PNG, GIF, or WebP file under the read path policy and returns metadata text followed by the image. It does not change the text-only `read` tool. Configure its independent positive byte limit with `max_image_bytes` (default 5,000,000). For example: @@ -586,7 +586,7 @@ Available wrappers: - `MCPServerSSE` - `MCPServerStreamableHTTP` -ThinHarness only turns MCP tools into harness tools; transport execution and session lifecycle come from the FastMCP client. Successful supported MCP images remain ordered image blocks. When `structuredContent` exists, its canonical JSON is the authoritative first text block, MCP text blocks are discarded, and image blocks or placeholders keep their relative order. MCP never inherits automatically into a child. A child that needs MCP lists an explicit `MCPPlugin` in `SubAgentConfig.plugins`, and that child binding owns its connection lifecycle. MCP prompts, resources, sampling, OAuth flows, provider-native MCP, and `.mcp.json` discovery are outside the current scope. +ThinHarness only turns MCP tools into harness tools; transport execution and session lifecycle come from the FastMCP client. Successful supported MCP images remain ordered image blocks. When `structuredContent` exists, its canonical JSON is the authoritative first text block; text, audio, resource, and resource-link blocks are discarded; and only image blocks or image placeholders keep their relative order. MCP never inherits automatically into a child. A child that needs MCP lists an explicit `MCPPlugin` in `SubAgentConfig.plugins`, and that child binding owns its connection lifecycle. MCP prompts, resources, sampling, OAuth flows, provider-native MCP, and `.mcp.json` discovery are outside the current scope. ## Resume @@ -707,7 +707,7 @@ Each tracing sink owns its capture policy. External spans can exist without reco - `text`: final model text. - `output`: parsed structured output, if configured. - `responses`: raw provider responses. -- `tool_call_records`: normalized tool call and output records. +- `tool_call_records`: normalized tool call records with canonical structured `result` and string `output`; image-bearing `output` is redacted while `result` retains complete image data. - `usage`: model request counts, tool call counts, cancellations, retry counters, and run token totals (`input_tokens`/`output_tokens`). - `stop_reason`: terminal reason. - `resume_state`: opaque continuation state when the run is cleanly resumable. diff --git a/docs/site/explainer/index.html b/docs/site/explainer/index.html index d32a414..19b9820 100644 --- a/docs/site/explainer/index.html +++ b/docs/site/explainer/index.html @@ -326,7 +326,7 @@

    Retry semantics

  • A handler can raise ModelRetry to ask the model to retry with a hint.
  • Ordinary handler exceptions become failed tool results but are not retryable unless metadata says so.
  • Tool retry budgets are per tool name per run, not per individual call id.
  • -
  • after_tool_call hooks can rewrite output text, but retry control flow is captured before that mutation.
  • +
  • after_tool_call hooks can rewrite canonical output or the structured envelope; retry control flow uses the final validated envelope.
  • diff --git a/tests/e2e/image_inputs_journey.py b/tests/e2e/image_inputs_journey.py index 16ab554..e2d3999 100644 --- a/tests/e2e/image_inputs_journey.py +++ b/tests/e2e/image_inputs_journey.py @@ -3,7 +3,9 @@ import asyncio import copy import os +import struct import sys +import zlib from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[2])) @@ -24,7 +26,26 @@ ) ROOT = Path(__file__).resolve().parents[2] -IMAGE = (ROOT / "assets" / "logo-circle.png").read_bytes() + + +def _known_image() -> bytes: + """Return a 64x32 PNG whose left half is red and right half is blue.""" + width, height = 64, 32 + row = b"\x00" + (b"\xff\x00\x00" * (width // 2)) + (b"\x00\x00\xff" * (width // 2)) + raw = row * height + + def chunk(kind: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw)) + + chunk(b"IEND", b"") + ) + + +IMAGE = _known_image() class RecordingOpenAI(OpenAIProvider): @@ -65,7 +86,7 @@ def image_tool() -> ToolSpec: lambda _args: ToolResult( True, (TextBlock("comparison fixture"), ImageBlock(IMAGE, "image/png")), - {"fixture": "logo-circle.png"}, + {"fixture": "red-blue.png"}, ), ) @@ -77,10 +98,11 @@ async def run_provider(label: str, model, payloads: list[dict]) -> None: tools=[image_tool()], ) first = await harness.run(( - TextBlock("Describe this image, then call inspect_fixture and compare the two images."), + TextBlock("Name the color on the left and the color on the right, then call inspect_fixture and compare the two images."), ImageBlock(IMAGE, "image/png"), )) - assert first.text + assert "red" in first.text.lower() + assert "blue" in first.text.lower() assert first.resume_state is not None resumed = await harness.run("In one sentence, restate the comparison.", resume_from=first.resume_state) assert resumed.text diff --git a/tests/unit/test_harness.py b/tests/unit/test_harness.py index 8e5ae09..8431768 100644 --- a/tests/unit/test_harness.py +++ b/tests/unit/test_harness.py @@ -103,7 +103,7 @@ def test_classify_run_failure_preserves_exception_ladder_semantics() -> None: (ValueError("plain failed"), "error", ValueError, True), ] for exc, stop_reason, raised_type, same_exception in cases: - run_ctx = SimpleNamespace(stop_reason="end_turn", terminal_error=None) + run_ctx = SimpleNamespace(stop_reason="end_turn", terminal_error=None, image_blocks=[]) span = _FailureSpan() raised = _classify_run_failure(run_ctx, span, exc) @@ -119,7 +119,7 @@ def test_classify_run_failure_preserves_exception_ladder_semantics() -> None: def test_classify_run_failure_preserves_existing_harness_stop_reason() -> None: existing = HarnessError("blocked by hook") exc = HarnessError("strict hook failure") - run_ctx = SimpleNamespace(stop_reason="cancelled_by_hook", terminal_error=existing) + run_ctx = SimpleNamespace(stop_reason="cancelled_by_hook", terminal_error=existing, image_blocks=[]) span = _FailureSpan() raised = _classify_run_failure(run_ctx, span, exc) diff --git a/tests/unit/test_image_inputs.py b/tests/unit/test_image_inputs.py index 87133f6..47815de 100644 --- a/tests/unit/test_image_inputs.py +++ b/tests/unit/test_image_inputs.py @@ -3,24 +3,37 @@ import base64 import copy import json +from datetime import datetime from pathlib import Path from typing import Any import pytest +from fakes import FakeTracer +from pydantic import BaseModel from thinharness import ( AnthropicMessagesModel, + ApprovalDecision, + BashPlugin, FilesystemPlugin, Harness, HarnessConfig, + HarnessError, + Hook, ImageBlock, OpenAIResponsesModel, OpenRouterModel, + RunFailedEvent, + RunStartedEvent, TextBlock, + ToolCallCompletedEvent, ToolResult, ToolSpec, + TracingOptions, + call_tool, ) from thinharness.content import content_from_json, content_to_json, normalize_content +from thinharness.providers import ProviderError PNG = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR" + b"\x00" * 17 PNG_B64 = base64.b64encode(PNG).decode("ascii") @@ -211,7 +224,12 @@ async def test_openrouter_image_tool_projection_is_labelled_and_literal(tmp_path "look", "Return an image.", {"type": "object", "properties": {}}, - lambda _args: [TextBlock("caption"), ImageBlock(PNG, "image/png")], + lambda _args: ToolResult( + False, + [TextBlock("caption"), ImageBlock(PNG, "image/png")], + {"retry": True, "error_type": "Fixture"}, + ), + max_retries=1, ) await Harness(_config(tmp_path), model=OpenRouterModel("vendor/model", provider=provider), tools=[tool]).run("go") # type: ignore[arg-type] @@ -221,12 +239,12 @@ async def test_openrouter_image_tool_projection_is_labelled_and_literal(tmp_path "role": "tool", "tool_call_id": "call_1", "content": json.dumps({ - "ok": True, + "ok": False, "content": [ {"type": "text", "text": "caption"}, {"type": "image", "media_type": "image/png", "size_bytes": 33, "block_index": 1}, ], - "metadata": {}, + "metadata": {"retry": True, "error_type": "Fixture"}, }, ensure_ascii=False), }, { @@ -258,3 +276,657 @@ def test_filesystem_read_image_is_opt_in_and_bounded(tmp_path: Path) -> None: assert isinstance(oversized, ToolResult) assert oversized.ok is False assert "over max_image_bytes=33" in oversized.message_text() + + +def _resume_state( + origin: str, + *, + include_tool_image: bool = True, + include_user_image: bool = True, +) -> dict[str, Any]: + user_content: list[dict[str, Any]] = [{"type": "text", "text": "prior user"}] + if include_user_image: + user_content.append({"type": "image", "media_type": "image/png", "data": PNG_B64}) + entries: list[dict[str, Any]] = [ + {"role": "user", "content": user_content, "notice": False}, + ] + if include_tool_image: + entries.extend([ + { + "role": "assistant", + "text": "", + "tool_calls": [{"id": "call_foreign", "name": "look", "arguments": "{}"}], + "reasoning": [], + }, + { + "role": "tool", + "call_id": "call_foreign", + "ok": True, + "content": [ + {"type": "text", "text": "prior tool"}, + {"type": "image", "media_type": "image/png", "data": PNG_B64}, + ], + "metadata": {"source": "resume"}, + }, + ]) + entries.append({"role": "assistant", "text": "prior done", "tool_calls": [], "reasoning": []}) + return { + "kind": "transcript", + "version": 4, + "origin_provider": origin, + "origin_model": "source-model", + "entries": entries, + } + + +def _model_for(provider_name: str, provider: _Provider): + if provider_name == "openai": + return OpenAIResponsesModel("target", provider=provider) # type: ignore[arg-type] + if provider_name == "anthropic": + return AnthropicMessagesModel("target", provider=provider) # type: ignore[arg-type] + return OpenRouterModel("target", provider=provider) # type: ignore[arg-type] + + +def _final_response(provider_name: str) -> dict[str, Any]: + if provider_name == "openai": + return {"id": "done", "output_text": "done"} + if provider_name == "anthropic": + return {"content": [{"type": "text", "text": "done"}]} + return {"choices": [{"message": {"role": "assistant", "content": "done"}}]} + + +@pytest.mark.parametrize("origin", ["openai", "anthropic", "openrouter"]) +@pytest.mark.parametrize("target", ["openai", "anthropic", "openrouter"]) +async def test_every_provider_pair_replays_user_and_tool_images(tmp_path: Path, origin: str, target: str) -> None: + provider = _Provider(target, [_final_response(target)]) + result = await Harness(_config(tmp_path), model=_model_for(target, provider)).run( + "follow-up", + resume_from=_resume_state(origin), + ) + + assert result.text == "done" + rendered = json.dumps(provider.payloads[0], ensure_ascii=False) + assert PNG_B64 in rendered + if target == "openrouter": + assert "[tool image call_id=call_foreign block=1]" in rendered + + +@pytest.mark.parametrize( + "mutate, message", + [ + (lambda state: state["entries"][0]["content"][1].update(data="***"), "invalid base64"), + (lambda state: state["entries"][0]["content"][1].update(media_type="image/svg+xml"), "unsupported media type"), + (lambda state: state["entries"][0]["content"][1].update(extra=True), "wrong shape"), + (lambda state: state.update(version=3), "version 3"), + ], +) +def test_malformed_v4_image_state_fails_clearly(tmp_path: Path, mutate, message: str) -> None: + state = _resume_state("openai", include_tool_image=False) + mutate(state) + provider = _Provider("OpenAI", []) + harness = Harness(_config(tmp_path), model=OpenAIResponsesModel("target", provider=provider)) # type: ignore[arg-type] + + with pytest.raises(Exception, match=message): + harness.run_sync("follow-up", resume_from=state) + assert provider.payloads == [] + + +@pytest.mark.parametrize( + "value, expected", + [ + ("", ""), + ([], "[]"), + (None, "null"), + ([1, "two"], '[\n 1,\n "two"\n]'), + ({"answer": 42}, '{\n "answer": 42\n}'), + ], +) +def test_sync_tool_result_normalization_preserves_json_values(value: Any, expected: str) -> None: + spec = ToolSpec("value", "Return value.", {"type": "object", "properties": {}}, lambda _args: value) + result = ToolResult.from_json(call_tool(spec, {}), strict=True) + + assert result.ok is True + assert result.content == expected + + +@pytest.mark.parametrize("value", [Path("not-json"), datetime(2026, 1, 1)]) +def test_nonserializable_sync_tool_results_become_failed_envelopes(value: Any) -> None: + spec = ToolSpec("bad", "Return invalid value.", {"type": "object", "properties": {}}, lambda _args: value) + result = ToolResult.from_json(call_tool(spec, {}), strict=True) + + assert result.ok is False + assert result.metadata["error_type"] == "InvalidToolResult" + + +async def test_async_invalid_metadata_and_empty_content_stay_at_tool_boundary(tmp_path: Path) -> None: + async def invalid(_args): + return ToolResult(True, "", {"bad": Path("not-json")}) + + provider = _Provider("OpenAI", [ + {"id": "r1", "output": [{"type": "function_call", "call_id": "call_1", "name": "bad", "arguments": "{}"}]}, + {"id": "r2", "output_text": "done"}, + ]) + result = await Harness( + _config(tmp_path), + model=OpenAIResponsesModel("target", provider=provider), # type: ignore[arg-type] + tools=[ToolSpec("bad", "Bad metadata.", {"type": "object", "properties": {}}, invalid)], + ).run("go") + + envelope = result.tool_call_records[0]["result"] + assert envelope["ok"] is False + assert envelope["metadata"]["error_type"] == "InvalidToolResult" + + +def test_image_and_text_tool_records_have_stable_keys(tmp_path: Path) -> None: + provider = _Provider("OpenAI", [ + {"id": "r1", "output": [ + {"type": "function_call", "call_id": "text", "name": "text", "arguments": "{}"}, + {"type": "function_call", "call_id": "image", "name": "image", "arguments": "{}"}, + ]}, + {"id": "r2", "output_text": "done"}, + ]) + harness = Harness( + _config(tmp_path), + model=OpenAIResponsesModel("target", provider=provider), # type: ignore[arg-type] + tools=[ + ToolSpec("text", "Text.", {"type": "object", "properties": {}}, lambda _args: "ok"), + ToolSpec("image", "Image.", {"type": "object", "properties": {}}, lambda _args: [ImageBlock(PNG, "image/png")]), + ], + ) + result = harness.run_sync("go") + text_record, image_record = result.tool_call_records + + assert set(text_record) == set(image_record) == {"call", "result", "output"} + assert PNG_B64 not in image_record["output"] + assert "size_bytes" in image_record["output"] + + +async def test_prompt_hooks_replace_content_and_append_context_after_images(tmp_path: Path) -> None: + def start(ctx): + ctx.prompt = "start replacement" + + def submit(ctx): + assert ctx.prompt == (TextBlock("start replacement"),) + ctx.prompt = (TextBlock("hook text"), ImageBlock(PNG, "image/png")) + ctx.additional_context.append("policy") + + provider = _Provider("OpenAI", [{"id": "done", "output_text": "done"}]) + await Harness( + _config(tmp_path), + model=OpenAIResponsesModel("target", provider=provider), # type: ignore[arg-type] + hooks=[Hook("run_start", start), Hook("user_prompt_submit", submit)], + ).run("caller") + + content = provider.payloads[0]["input"][0]["content"] + assert content == [ + {"type": "input_text", "text": "hook text"}, + {"type": "input_image", "image_url": PNG_URL}, + {"type": "input_text", "text": "\npolicy\n"}, + ] + + +async def test_image_events_and_tool_trace_are_redacted(tmp_path: Path) -> None: + tracer = FakeTracer() + provider = _Provider("OpenAI", [ + {"id": "r1", "output": [{"type": "function_call", "call_id": "image", "name": "image", "arguments": "{}"}]}, + {"id": "r2", "output_text": "done"}, + ]) + harness = Harness( + _config(tmp_path), + model=OpenAIResponsesModel("target", provider=provider), # type: ignore[arg-type] + tools=[ToolSpec("image", "Image.", {"type": "object", "properties": {}}, lambda _args: [ImageBlock(PNG, "image/png")])], + tracing=[TracingOptions(tracer=tracer, capture_messages=True, capture_tool_results=True)], + ) + events = [] + async for event in harness.stream((TextBlock("go"), ImageBlock(PNG, "image/png"))): + events.append(event) + + started = next(event for event in events if isinstance(event, RunStartedEvent)) + completed = next(event for event in events if isinstance(event, ToolCallCompletedEvent)) + assert PNG_B64 not in started.prompt + assert PNG_B64 not in completed.output + trace_text = json.dumps([span.attributes for span in tracer.spans], ensure_ascii=False) + assert PNG_B64 not in trace_text + assert PNG_URL not in trace_text + + +class _FailingProvider(_Provider): + def __init__(self, name: str = "OpenAI", *, leak: bool = True) -> None: + super().__init__(name, []) + self.leak = leak + + async def create_response(self, payload: dict[str, Any]) -> dict[str, Any]: + self.payloads.append(copy.deepcopy(payload)) + message = f"provider echoed {PNG_B64} and {PNG_URL}" if self.leak else "plain provider failure" + raise ProviderError(message) + + +@pytest.mark.parametrize( + "state", + [ + _resume_state("openai", include_tool_image=False), + _resume_state("openai", include_user_image=False), + ], +) +async def test_resumed_provider_failures_redact_user_and_tool_images(tmp_path: Path, state: dict[str, Any]) -> None: + tracer = FakeTracer() + provider = _FailingProvider() + harness = Harness( + _config(tmp_path), + model=OpenAIResponsesModel("target", provider=provider), + tracing=[TracingOptions(tracer=tracer, capture_messages=True)], + ) + events = [] + with pytest.raises(HarnessError): + async for event in harness.stream("follow-up", resume_from=copy.deepcopy(state)): + events.append(event) + + failed = next(event for event in events if isinstance(event, RunFailedEvent)) + assert PNG_B64 not in failed.message + assert PNG_URL not in failed.message + recorded = " ".join(str(exc) for span in tracer.spans for exc in span.exceptions) + attributes = json.dumps([span.attributes for span in tracer.spans], ensure_ascii=False) + assert PNG_B64 not in recorded + attributes + assert PNG_URL not in recorded + attributes + + +async def test_text_only_provider_failure_preserves_recorded_exception_type(tmp_path: Path) -> None: + tracer = FakeTracer() + provider = _FailingProvider(leak=False) + harness = Harness( + _config(tmp_path), + model=OpenAIResponsesModel("target", provider=provider), + tracing=[TracingOptions(tracer=tracer)], + ) + + with pytest.raises(HarnessError): + await harness.run("go") + model_span = next(span for span in tracer.spans if span.name.startswith("chat ")) + assert any(isinstance(exc, ProviderError) for exc in model_span.exceptions) + + +class _ApprovalFailingProvider(_Provider): + def __init__(self) -> None: + super().__init__("OpenAI", []) + self.calls = 0 + + async def create_response(self, payload: dict[str, Any]) -> dict[str, Any]: + self.payloads.append(copy.deepcopy(payload)) + self.calls += 1 + if self.calls == 1: + return { + "id": "pause", + "output": [{"type": "function_call", "call_id": "approve", "name": "approved", "arguments": "{}"}], + } + raise ProviderError(f"approval provider echoed {PNG_B64} and {PNG_URL}") + + +async def test_image_approval_resume_with_capture_redacts_failure(tmp_path: Path) -> None: + tracer = FakeTracer() + provider = _ApprovalFailingProvider() + tool = ToolSpec( + "approved", + "Approval tool.", + {"type": "object", "properties": {}}, + lambda _args: "approved", + requires_approval=True, + ) + harness = Harness( + _config(tmp_path), + model=OpenAIResponsesModel("target", provider=provider), # type: ignore[arg-type] + tools=[tool], + tracing=[TracingOptions(tracer=tracer, capture_messages=True)], + ) + paused = await harness.run((TextBlock("approve"), ImageBlock(PNG, "image/png"))) + assert paused.stop_reason == "approval_required" + + old_state = copy.deepcopy(paused.resume_state) + old_state["provider_state"]["version"] = 3 + with pytest.raises(Exception, match="approval state provider_state version 3 is not supported"): + await Harness( + _config(tmp_path), + model=OpenAIResponsesModel("target", provider=_Provider("OpenAI", [])), # type: ignore[arg-type] + tools=[tool], + ).resume_approvals(old_state, [ApprovalDecision("approve", True)]) + + events = [] + with pytest.raises(HarnessError): + async for event in harness.stream_approvals( + paused.resume_state, + [ApprovalDecision("approve", True)], + ): + events.append(event) + failed = next(event for event in events if isinstance(event, RunFailedEvent)) + trace_text = json.dumps([span.attributes for span in tracer.spans], ensure_ascii=False) + exceptions = " ".join(str(exc) for span in tracer.spans for exc in span.exceptions) + assert PNG_B64 not in failed.message + trace_text + exceptions + assert PNG_URL not in failed.message + trace_text + exceptions + + +@pytest.mark.parametrize("field", ["output", "envelope"]) +async def test_after_tool_hook_mutates_image_through_both_fields(tmp_path: Path, field: str) -> None: + def mutate(ctx): + replacement = ToolResult(True, (TextBlock("mutated"), ImageBlock(PNG + b"m", "image/png")), {"hook": field}) + if field == "output": + ctx.output = replacement.to_json() + else: + ctx.envelope = replacement + + provider = _Provider("OpenAI", [ + {"id": "r1", "output": [{"type": "function_call", "call_id": "image", "name": "image", "arguments": "{}"}]}, + {"id": "r2", "output_text": "done"}, + ]) + await Harness( + _config(tmp_path), + model=OpenAIResponsesModel("target", provider=provider), # type: ignore[arg-type] + tools=[ToolSpec("image", "Image.", {"type": "object", "properties": {}}, lambda _args: [ImageBlock(PNG, "image/png")])], + hooks=[Hook("after_tool_call", mutate)], + ).run("go") + + output = provider.payloads[1]["input"][0]["output"] + assert output[0] == {"type": "input_text", "text": f'{{"ok":true,"metadata":{{"hook":"{field}"}}}}'} + assert output[1] == {"type": "input_text", "text": "mutated"} + assert output[2]["image_url"].endswith(base64.b64encode(PNG + b"m").decode("ascii")) + + +async def test_parallel_async_image_tools_preserve_model_order(tmp_path: Path) -> None: + async def first(_args): + return [TextBlock("first"), ImageBlock(PNG + b"1", "image/png")] + + async def second(_args): + return [TextBlock("second"), ImageBlock(PNG + b"2", "image/png")] + + provider = _Provider("OpenRouter", [ + {"choices": [{"message": {"role": "assistant", "content": "", "tool_calls": [ + {"id": "one", "type": "function", "function": {"name": "first", "arguments": "{}"}}, + {"id": "two", "type": "function", "function": {"name": "second", "arguments": "{}"}}, + ]}}]}, + {"choices": [{"message": {"role": "assistant", "content": "done"}}]}, + ]) + await Harness( + _config(tmp_path), + model=OpenRouterModel("target", provider=provider), # type: ignore[arg-type] + tools=[ + ToolSpec("first", "First.", {"type": "object", "properties": {}}, first), + ToolSpec("second", "Second.", {"type": "object", "properties": {}}, second), + ], + ).run("go") + + labels = [ + part["text"] + for part in provider.payloads[1]["messages"][-1]["content"] + if part["type"] == "text" + ] + assert labels == ["[tool image call_id=one block=1]", "[tool image call_id=two block=1]"] + + +async def test_notice_follows_image_content(tmp_path: Path) -> None: + provider = _Provider("OpenRouter", [_final_response("openrouter")]) + await Harness( + HarnessConfig(root=tmp_path, system_prompt="sys", local_tracing=False, max_model_requests=1), + model=OpenRouterModel("target", provider=provider), # type: ignore[arg-type] + ).run((TextBlock("go"), ImageBlock(PNG, "image/png"))) + + parts = provider.payloads[0]["messages"][-1]["content"] + assert parts[-2]["type"] == "image_url" + assert parts[-1]["type"] == "text" + assert parts[-1]["text"].startswith('') + + +@pytest.mark.parametrize( + ("media_type", "data"), + [ + ("image/png", PNG), + ("image/jpeg", b"\xff\xd8\xff\xe0body\xff\xd9trailing"), + ("image/gif", b"GIF89a" + b"\x00" * 7), + ("image/webp", b"RIFF" + (12).to_bytes(4, "little") + b"WEBPVP8 " + b"\x00" * 4 + b"trailing"), + ], +) +def test_read_image_accepts_supported_signatures_and_trailing_data( + tmp_path: Path, + media_type: str, + data: bytes, +) -> None: + path = tmp_path / "image.bin" + path.write_bytes(data) + binding = FilesystemPlugin(tools=["read_image"], max_image_bytes=len(data)).bind( + type("Context", (), {"root": tmp_path})(), # type: ignore[arg-type] + ) + + result = binding.static.tools[0].handler({"path": "image.bin"}) + + assert isinstance(result, ToolResult) + assert result.ok is True + assert result.metadata["media_type"] == media_type + + +@pytest.mark.parametrize( + "data", + [ + b"\x89PNG\r\n\x1a\n", + b"\xff\xd8\xffleading only", + b"GIF89a", + b"RIFF\x04\x00\x00\x00WEBP", + ], +) +def test_read_image_rejects_truncated_or_spoofed_headers(tmp_path: Path, data: bytes) -> None: + (tmp_path / "bad.bin").write_bytes(data) + binding = FilesystemPlugin(tools=["read_image"]).bind( + type("Context", (), {"root": tmp_path})(), # type: ignore[arg-type] + ) + + result = binding.static.tools[0].handler({"path": "bad.bin"}) + + assert isinstance(result, ToolResult) + assert result.ok is False + assert "unsupported or invalid" in result.message_text() + + +def test_read_image_rejects_non_regular_file_without_opening(tmp_path: Path) -> None: + directory = tmp_path / "directory" + directory.mkdir() + binding = FilesystemPlugin(tools=["read_image"]).bind( + type("Context", (), {"root": tmp_path})(), # type: ignore[arg-type] + ) + + result = binding.static.tools[0].handler({"path": "directory"}) + + assert isinstance(result, ToolResult) + assert result.ok is False + assert "not a regular file" in result.message_text() + + +def test_filesystem_image_settings_validate_without_binding_io(tmp_path: Path, monkeypatch) -> None: + for invalid in (0, -1, True, 1.5): + with pytest.raises(ValueError, match="positive integer"): + FilesystemPlugin(max_image_bytes=invalid) # type: ignore[arg-type] + + plugin = FilesystemPlugin(tools=["read_image"], max_image_bytes=7) + monkeypatch.setattr(Path, "open", lambda *_args, **_kwargs: pytest.fail("binding performed image I/O")) + binding = plugin.bind(type("Context", (), {"root": tmp_path})()) # type: ignore[arg-type] + assert binding.static.tools[0].name == "read_image" + + +class _Answer(BaseModel): + value: str + + +def _structured_responses(provider_name: str) -> list[dict[str, Any]]: + if provider_name == "openai": + return [ + {"id": "bad", "output": [{"type": "function_call", "call_id": "final_bad", "name": "final_result", "arguments": "{}"}]}, + {"id": "good", "output": [{"type": "function_call", "call_id": "final_good", "name": "final_result", "arguments": '{"value":"ok"}'}]}, + ] + if provider_name == "anthropic": + return [ + {"content": [{"type": "tool_use", "id": "final_bad", "name": "final_result", "input": {}}]}, + {"content": [{"type": "tool_use", "id": "final_good", "name": "final_result", "input": {"value": "ok"}}]}, + ] + return [ + {"choices": [{"message": {"role": "assistant", "content": "", "tool_calls": [{ + "id": "final_bad", + "type": "function", + "function": {"name": "final_result", "arguments": "{}"}, + }]}}]}, + {"choices": [{"message": {"role": "assistant", "content": "", "tool_calls": [{ + "id": "final_good", + "type": "function", + "function": {"name": "final_result", "arguments": '{"value":"ok"}'}, + }]}}]}, + ] + + +@pytest.mark.parametrize("provider_name", ["openai", "anthropic", "openrouter"]) +async def test_structured_output_retry_keeps_plain_wire_text(tmp_path: Path, provider_name: str) -> None: + provider = _Provider(provider_name, _structured_responses(provider_name)) + config = HarnessConfig( + root=tmp_path, + system_prompt="sys", + local_tracing=False, + output_type=_Answer, + output_mode="tool", + ) + result = await Harness(config, model=_model_for(provider_name, provider)).run("answer") + assert result.output == _Answer(value="ok") + + if provider_name == "openai": + wire = provider.payloads[1]["input"][0]["output"] + elif provider_name == "anthropic": + wire = provider.payloads[1]["messages"][-1]["content"][0]["content"] + else: + wire = provider.payloads[1]["messages"][-1]["content"] + assert isinstance(wire, str) + assert wire.startswith("The previous response failed structured output validation.") + assert not wire.startswith("{") + + +async def test_initial_image_provider_failure_is_redacted_everywhere(tmp_path: Path) -> None: + tracer = FakeTracer() + provider = _FailingProvider() + harness = Harness( + _config(tmp_path), + model=OpenAIResponsesModel("target", provider=provider), + tracing=[TracingOptions(tracer=tracer, capture_messages=True)], + ) + events = [] + with pytest.raises(HarnessError): + async for event in harness.stream((TextBlock("go"), ImageBlock(PNG, "image/png"))): + events.append(event) + + failed = next(event for event in events if isinstance(event, RunFailedEvent)) + trace_text = json.dumps([span.attributes for span in tracer.spans], ensure_ascii=False) + exceptions = " ".join(str(exc) for span in tracer.spans for exc in span.exceptions) + assert PNG_B64 not in failed.message + trace_text + exceptions + assert PNG_URL not in failed.message + trace_text + exceptions + + +async def test_async_empty_string_tool_result_succeeds(tmp_path: Path) -> None: + async def empty(_args): + return "" + + provider = _Provider("OpenAI", [ + {"id": "r1", "output": [{"type": "function_call", "call_id": "empty", "name": "empty", "arguments": "{}"}]}, + {"id": "r2", "output_text": "done"}, + ]) + result = await Harness( + _config(tmp_path), + model=OpenAIResponsesModel("target", provider=provider), # type: ignore[arg-type] + tools=[ToolSpec("empty", "Empty.", {"type": "object", "properties": {}}, empty)], + ).run("go") + + assert result.tool_call_records[0]["result"] == {"ok": True, "content": "", "metadata": {}} + + +async def test_bash_printed_image_path_stays_text_only(tmp_path: Path) -> None: + (tmp_path / "sample.png").write_bytes(PNG) + provider = _Provider("OpenAI", [ + { + "id": "r1", + "output": [{ + "type": "function_call", + "call_id": "bash", + "name": "bash", + "arguments": '{"command":"printf sample.png"}', + }], + }, + {"id": "r2", "output_text": "done"}, + ]) + result = await Harness( + _config(tmp_path), + model=OpenAIResponsesModel("target", provider=provider), # type: ignore[arg-type] + plugins=[BashPlugin()], + ).run("print path") + + content = result.tool_call_records[0]["result"]["content"] + assert isinstance(content, str) + assert "sample.png" in content + assert PNG_B64 not in result.tool_call_records[0]["output"] + + +def test_read_image_rejects_symlink_escape(tmp_path: Path) -> None: + outside = tmp_path.parent / f"{tmp_path.name}-outside.png" + outside.write_bytes(PNG) + (tmp_path / "escape.png").symlink_to(outside) + binding = FilesystemPlugin(tools=["read_image"]).bind( + type("Context", (), {"root": tmp_path})(), # type: ignore[arg-type] + ) + + result = binding.static.tools[0].handler({"path": "escape.png"}) + + assert isinstance(result, ToolResult) + assert result.ok is False + assert "escapes root" in result.message_text() + + +def test_read_image_reports_unreadable_file(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "image.png" + path.write_bytes(PNG) + binding = FilesystemPlugin(tools=["read_image"]).bind( + type("Context", (), {"root": tmp_path})(), # type: ignore[arg-type] + ) + original_open = Path.open + + def denied(self, *args, **kwargs): + if self == path: + raise PermissionError("denied") + return original_open(self, *args, **kwargs) + + monkeypatch.setattr(Path, "open", denied) + result = binding.static.tools[0].handler({"path": "image.png"}) + + assert isinstance(result, ToolResult) + assert result.ok is False + assert result.metadata["error_type"] == "PermissionError" + + +async def test_image_approval_resume_succeeds_with_message_capture(tmp_path: Path) -> None: + tracer = FakeTracer() + provider = _Provider("OpenAI", [ + { + "id": "pause", + "output": [{"type": "function_call", "call_id": "approve", "name": "approved", "arguments": "{}"}], + }, + {"id": "done", "output_text": "done"}, + ]) + tool = ToolSpec( + "approved", + "Approval tool.", + {"type": "object", "properties": {}}, + lambda _args: "approved", + requires_approval=True, + ) + harness = Harness( + _config(tmp_path), + model=OpenAIResponsesModel("target", provider=provider), # type: ignore[arg-type] + tools=[tool], + tracing=[TracingOptions(tracer=tracer, capture_messages=True)], + ) + paused = await harness.run((TextBlock("approve"), ImageBlock(PNG, "image/png"))) + + result = await harness.resume_approvals( + paused.resume_state, + [ApprovalDecision("approve", True)], + ) + + assert result.text == "done" + assert any(span.name.startswith("chat ") for span in tracer.spans) diff --git a/tests/unit/test_mcp.py b/tests/unit/test_mcp.py index b70c87e..9aae281 100644 --- a/tests/unit/test_mcp.py +++ b/tests/unit/test_mcp.py @@ -1673,3 +1673,64 @@ async def list_tools(self, *, server_id: str | None = None) -> list[ToolSpec]: assert events == [] assert tracer.spans == [] + + +async def test_structured_content_keeps_only_ordered_images_and_image_placeholders(monkeypatch) -> None: + from mcp import types + + scripted = types.CallToolResult( + content=[ + types.TextContent(type="text", text="discard text"), + types.AudioContent(type="audio", data="aGk=", mimeType="audio/wav"), + types.ImageContent(type="image", data="aGk=", mimeType="image/png"), + types.EmbeddedResource( + type="resource", + resource=types.TextResourceContents(uri="file:///discard.txt", text="discard"), + ), + types.ImageContent(type="image", data="***", mimeType="image/jpeg"), + types.ResourceLink(type="resource_link", uri="https://example.com/discard", name="discard"), + types.ImageContent(type="image", data="aGk=", mimeType="image/svg+xml"), + ], + structuredContent={"answer": 42}, + isError=False, + ) + server = scripted_server(monkeypatch, {"mixed": _schema()}, {"mixed": scripted}) + + result = await server.call_tool("mixed", {}) + + assert result.content == ( + TextBlock('{"answer": 42}'), + ImageBlock(b"hi", "image/png"), + TextBlock("[image: image/jpeg]"), + TextBlock("[image: image/svg+xml]"), + ) + + +@pytest.mark.parametrize( + ("data", "media_type"), + [("***", "image/png"), ("aGk=", "image/svg+xml")], +) +async def test_mcp_malformed_or_unsupported_images_become_placeholders(monkeypatch, data: str, media_type: str) -> None: + from mcp import types + + scripted = types.CallToolResult( + content=[types.ImageContent(type="image", data=data, mimeType=media_type)], + isError=False, + ) + server = scripted_server(monkeypatch, {"image": _schema()}, {"image": scripted}) + + result = await server.call_tool("image", {}) + + assert result.content == f"[image: {media_type}]" + + +async def test_empty_successful_mcp_content_remains_successful(monkeypatch) -> None: + from mcp import types + + scripted = types.CallToolResult(content=[], isError=False) + server = scripted_server(monkeypatch, {"empty": _schema()}, {"empty": scripted}) + + result = await server.call_tool("empty", {}) + + assert result.ok is True + assert result.content == "" diff --git a/tests/unit/test_providers.py b/tests/unit/test_providers.py index 67d2bd0..b290e48 100644 --- a/tests/unit/test_providers.py +++ b/tests/unit/test_providers.py @@ -26,6 +26,7 @@ OpenRouterModel, OpenRouterProvider, RequestConstants, + ToolResult, parse_model_ref, ) from thinharness.providers import ( @@ -36,13 +37,17 @@ ToolOutput, _retry_after_seconds, _retry_delay, - append_notices_to_text, extract_finish_reason, extract_token_usage, render_model_notices, ) +def _tool_output(call_id: str, content: str) -> ToolOutput: + """Return one normalized text tool output.""" + return ToolOutput(call_id, ToolResult(True, content)) + + def _notice() -> ModelNotice: """Return a reusable test notice.""" return ModelNotice(kind="limit_warning", content="Final request.", limit_kind="model_requests", remaining=1) @@ -72,14 +77,12 @@ def test_model_notice_rendering_is_deterministic() -> None: second = ModelNotice(kind="limit_warning", content="One tool call remains.", limit_kind="tool_calls", remaining=1) assert render_model_notices(None) == "" - assert append_notices_to_text("hi", None) == "hi" assert render_model_notices([first]) == '\nFinal request.\n' assert render_model_notices([first, second]) == ( '\nFinal request.\n' "\n\n" '\nOne tool call remains.\n' ) - assert append_notices_to_text("hi", [first]) == 'hi\n\n\nFinal request.\n' async def test_model_sessions_advance_independently() -> None: provider = FakeAnthropicProvider() @@ -90,8 +93,8 @@ async def test_model_sessions_advance_independently() -> None: first_turn = await first.start("first", constants) second_turn = await second.start("second", constants) - await first.continue_with_tools([ToolOutput(first_turn.tool_calls[0].id, "first result")], constants) - await second.continue_with_tools([ToolOutput(second_turn.tool_calls[0].id, "second result")], constants) + await first.continue_with_tools([_tool_output(first_turn.tool_calls[0].id, "first result")], constants) + await second.continue_with_tools([_tool_output(second_turn.tool_calls[0].id, "second result")], constants) assert provider.payloads[2]["messages"][0] == {"role": "user", "content": "first"} assert provider.payloads[2]["messages"][-1]["content"][0]["content"] == '{"ok": true, "content": "first result", "metadata": {}}' @@ -106,7 +109,7 @@ async def test_openai_previous_response_id_is_session_scoped() -> None: second = model.new_session() await first.start("first", constants, previous_response_id="existing") - await first.continue_with_tools([ToolOutput("call_1", "ok")], constants) + await first.continue_with_tools([_tool_output("call_1", "ok")], constants) await second.start("second", constants) assert client.payloads[0]["previous_response_id"] == "existing" @@ -123,7 +126,7 @@ async def test_openai_appends_notices_to_string_and_tool_inputs() -> None: first = await session.start("hi", constants, notices=[notice]) await session.continue_with_tools( - [ToolOutput(first.tool_calls[0].id, "ok"), ToolOutput("call_2", "second")], + [_tool_output(first.tool_calls[0].id, "ok"), _tool_output("call_2", "second")], constants, notices=[notice], ) @@ -168,7 +171,7 @@ async def test_openai_no_notice_payloads_are_unchanged() -> None: constants = _constants() await session.start("hi", constants) - await session.continue_with_tools([ToolOutput("call_1", "ok")], constants) + await session.continue_with_tools([_tool_output("call_1", "ok")], constants) assert client.payloads[0]["input"] == "hi" assert client.payloads[1]["input"] == [{ @@ -186,7 +189,7 @@ async def test_anthropic_appends_notices_to_messages() -> None: first = await session.start("hi\n\n\npolicy\n", constants, notices=[notice]) await session.continue_with_tools( - [ToolOutput(first.tool_calls[0].id, "ok"), ToolOutput("toolu_2", "second")], + [_tool_output(first.tool_calls[0].id, "ok"), _tool_output("toolu_2", "second")], constants, notices=[notice], ) @@ -211,7 +214,7 @@ async def test_openrouter_appends_notices_to_messages() -> None: first = await session.start("hi\n\n\npolicy\n", constants, notices=[notice]) await session.continue_with_tools( - [ToolOutput(first.tool_calls[0].id, "ok"), ToolOutput("call_2", "second")], + [_tool_output(first.tool_calls[0].id, "ok"), _tool_output("call_2", "second")], constants, notices=[notice], ) @@ -232,7 +235,7 @@ async def test_resume_replays_preserved_tool_notices() -> None: anthropic_provider = FakeAnthropicProvider() anthropic_session = AnthropicMessagesModel("claude-test", provider=anthropic_provider).new_session() anthropic_first = await anthropic_session.start("hi", constants) - await anthropic_session.continue_with_tools([ToolOutput(anthropic_first.tool_calls[0].id, "ok")], constants, notices=[notice]) + await anthropic_session.continue_with_tools([_tool_output(anthropic_first.tool_calls[0].id, "ok")], constants, notices=[notice]) anthropic_state = json.loads(json.dumps(anthropic_session.dump_state())) assert anthropic_state == json.loads(json.dumps(anthropic_state)) anthropic_resumed = AnthropicMessagesModel("claude-test", provider=anthropic_provider).resume_session(anthropic_state) @@ -242,7 +245,7 @@ async def test_resume_replays_preserved_tool_notices() -> None: openai_capture = FakeClient() openai_session = OpenAIResponsesModel("gpt-test", provider=openai_capture).new_session() openai_first = await openai_session.start("hi", constants) - await openai_session.continue_with_tools([ToolOutput(openai_first.tool_calls[0].id, "ok")], constants, notices=[notice]) + await openai_session.continue_with_tools([_tool_output(openai_first.tool_calls[0].id, "ok")], constants, notices=[notice]) openai_state = json.loads(json.dumps(openai_session.dump_state())) openai_replay = FakeClient() openai_resumed = OpenAIResponsesModel("gpt-test", provider=openai_replay).resume_session(openai_state) @@ -256,7 +259,7 @@ async def test_resume_replays_preserved_tool_notices() -> None: openrouter_provider = FakeOpenRouterProvider() openrouter_session = OpenRouterModel("openai/test", provider=openrouter_provider).new_session() openrouter_first = await openrouter_session.start("hi", constants) - await openrouter_session.continue_with_tools([ToolOutput(openrouter_first.tool_calls[0].id, "ok")], constants, notices=[notice]) + await openrouter_session.continue_with_tools([_tool_output(openrouter_first.tool_calls[0].id, "ok")], constants, notices=[notice]) openrouter_state = json.loads(json.dumps(openrouter_session.dump_state())) openrouter_resumed = OpenRouterModel("openai/test", provider=openrouter_provider).resume_session(openrouter_state) await openrouter_resumed.continue_with_user_content("next", constants) @@ -269,7 +272,7 @@ async def test_resume_replays_preserved_user_notices() -> None: anthropic_capture = FakeAnthropicProvider() anthropic_session = AnthropicMessagesModel("claude-test", provider=anthropic_capture).new_session() anthropic_first = await anthropic_session.start("hi", constants, notices=[notice]) - await anthropic_session.continue_with_tools([ToolOutput(anthropic_first.tool_calls[0].id, "ok")], constants) + await anthropic_session.continue_with_tools([_tool_output(anthropic_first.tool_calls[0].id, "ok")], constants) anthropic_state = json.loads(json.dumps(anthropic_session.dump_state())) anthropic_replay = FakeAnthropicProvider() anthropic_resumed = AnthropicMessagesModel("claude-test", provider=anthropic_replay).resume_session(anthropic_state) @@ -279,7 +282,7 @@ async def test_resume_replays_preserved_user_notices() -> None: openai_capture = FakeClient() openai_session = OpenAIResponsesModel("gpt-test", provider=openai_capture).new_session() openai_first = await openai_session.start("hi", constants, notices=[notice]) - await openai_session.continue_with_tools([ToolOutput(openai_first.tool_calls[0].id, "ok")], constants) + await openai_session.continue_with_tools([_tool_output(openai_first.tool_calls[0].id, "ok")], constants) openai_state = json.loads(json.dumps(openai_session.dump_state())) openai_replay = FakeClient() openai_resumed = OpenAIResponsesModel("gpt-test", provider=openai_replay).resume_session(openai_state) @@ -289,7 +292,7 @@ async def test_resume_replays_preserved_user_notices() -> None: openrouter_capture = FakeOpenRouterProvider() openrouter_session = OpenRouterModel("openai/test", provider=openrouter_capture).new_session() openrouter_first = await openrouter_session.start("hi", constants, notices=[notice]) - await openrouter_session.continue_with_tools([ToolOutput(openrouter_first.tool_calls[0].id, "ok")], constants) + await openrouter_session.continue_with_tools([_tool_output(openrouter_first.tool_calls[0].id, "ok")], constants) openrouter_state = json.loads(json.dumps(openrouter_session.dump_state())) openrouter_replay = FakeOpenRouterProvider() openrouter_resumed = OpenRouterModel("openai/test", provider=openrouter_replay).resume_session(openrouter_state) @@ -404,7 +407,7 @@ def handler(request: httpx.Request) -> httpx.Response: first = await session.start("hi", constants) assert first.tool_calls[0].name == "echo" - second = await session.continue_with_tools([ToolOutput(first.tool_calls[0].id, "ok")], constants) + second = await session.continue_with_tools([_tool_output(first.tool_calls[0].id, "ok")], constants) assert second.text == "done" assert calls[0][1]["tools"][0]["input_schema"]["type"] == "object" @@ -428,7 +431,7 @@ async def test_anthropic_requests_opt_into_prompt_caching() -> None: constants = _constants(ECHO_TOOLS) first = await session.start("hi", constants) - await session.continue_with_tools([ToolOutput(first.tool_calls[0].id, "ok")], constants) + await session.continue_with_tools([_tool_output(first.tool_calls[0].id, "ok")], constants) assert [payload["cache_control"] for payload in provider.payloads] == [{"type": "ephemeral"}] * 2 @@ -502,7 +505,7 @@ def handler(request: httpx.Request) -> httpx.Response: first = await session.start("hi", constants) assert first.tool_calls[0].id == "call_1" - second = await session.continue_with_tools([ToolOutput("call_1", "ok")], constants) + second = await session.continue_with_tools([_tool_output("call_1", "ok")], constants) assert second.text == "done" assert calls[0][1]["tools"][0]["function"]["name"] == "echo" @@ -577,7 +580,7 @@ async def test_anthropic_notice_payload_live() -> None: ] tools = [{"type": "function", "name": "echo", "description": "Echo", "parameters": {"type": "object", "properties": {"value": {"type": "string"}}}}] try: - turn = await session.continue_with_tools([ToolOutput("toolu_live", "ok")], _constants(tools), notices=[sentinel_notice]) + turn = await session.continue_with_tools([_tool_output("toolu_live", "ok")], _constants(tools), notices=[sentinel_notice]) finally: await provider.aclose() @@ -602,7 +605,7 @@ async def test_openrouter_notice_payload_live() -> None: ] tools = [{"type": "function", "name": "echo", "description": "Echo", "parameters": {"type": "object", "properties": {"value": {"type": "string"}}}}] try: - turn = await session.continue_with_tools([ToolOutput("call_live", "ok")], _constants(tools), notices=[_notice()]) + turn = await session.continue_with_tools([_tool_output("call_live", "ok")], _constants(tools), notices=[_notice()]) finally: await provider.aclose() diff --git a/tests/unit/test_tracing.py b/tests/unit/test_tracing.py index c0405a9..d0ce3d2 100644 --- a/tests/unit/test_tracing.py +++ b/tests/unit/test_tracing.py @@ -483,7 +483,7 @@ def test_trace_request_kinds_for_resume_and_output_retries(tmp_path: Path) -> No assert "resume" in kinds retry_chat = next(span for span in chats if span.attributes.get("thinharness.model.request.kind") == "output_retry_tool") retry_content = json.loads(retry_chat.attributes["gen_ai.input.messages"])[0]["parts"][0]["content"] - assert json.loads(retry_content)["content"].startswith("The previous response failed") + assert retry_content.startswith("The previous response failed") assert "Final request" in retry_chat.attributes["gen_ai.input.messages"] assert "Final request" in retry_chat.attributes["thinharness.model.notices"] correction_chat = next(span for span in chats if span.attributes.get("thinharness.model.request.kind") == "correction") diff --git a/tests/unit/test_turns.py b/tests/unit/test_turns.py index a1a2b1d..514c560 100644 --- a/tests/unit/test_turns.py +++ b/tests/unit/test_turns.py @@ -7,7 +7,7 @@ from fakes import ScriptedProvider, ScriptedSession from pydantic import BaseModel -from thinharness import Harness, HarnessConfig, ModelMessageEvent, RequestConstants, ToolSpec, UnexpectedModelBehavior +from thinharness import Harness, HarnessConfig, ModelMessageEvent, RequestConstants, ToolResult, ToolSpec, UnexpectedModelBehavior from thinharness.approvals import ApprovalPause, ApprovalToolCall from thinharness.providers import ModelToolCall, ModelTurn, ToolOutput from thinharness.tracing import RunTracer @@ -89,7 +89,7 @@ def __init__(self, tool_map: dict | None = None, cancelled_ids: set[str] | None async def execute_batch(self, calls, tool_indices=None): self.batches.append([call.id for call in calls]) records = [{"call": {"id": call.id, "name": call.name, "arguments": call.arguments}, "output": "ok"} for call in calls] - outputs = [ToolOutput(call.id, "ok") for call in calls] + outputs = [ToolOutput(call.id, ToolResult(True, "ok")) for call in calls] executions = [SimpleNamespace(cancelled=call.id in self.cancelled_ids, retry_kind=None) for call in calls] return records, outputs, executions diff --git a/thinharness/core.py b/thinharness/core.py index 4939bc3..54fabd5 100644 --- a/thinharness/core.py +++ b/thinharness/core.py @@ -58,6 +58,7 @@ StructuredOutputRequest, infer_model, model_capabilities, + session_image_blocks, ) from .tools.base import ToolOrigin, ToolSpec from .tracing import ( @@ -79,12 +80,12 @@ def _local_tracing_enabled(configured: bool) -> bool: def _classify_run_failure(run_ctx: Any, agent_span: Any, exc: Exception) -> Exception: """Record a run failure and return the exception to raise.""" - message = redact_image_data(str(exc), getattr(run_ctx, "image_blocks", ())) + message = redact_image_data(str(exc), run_ctx.image_blocks) agent_span.record_exception(exc if message == str(exc) else HarnessError(message)) agent_span.set_error(message, type(exc).__name__) - if isinstance(exc, ProviderError): + if isinstance(exc, ProviderError) or getattr(exc, "_thinharness_provider_error", False): run_ctx.stop_reason = "provider_error" - run_ctx.terminal_error = HarnessError(message) + run_ctx.terminal_error = exc if isinstance(exc, HarnessError) else HarnessError(message) return run_ctx.terminal_error if isinstance(exc, UnexpectedModelBehavior): run_ctx.stop_reason = "unexpected_model_behavior" @@ -503,6 +504,8 @@ async def _run_streaming( raise run_ctx.terminal_error session = cast(ResumableModel, self.model).resume_session(resume_from) first_turn_kind = "resume" + if session is not None: + run_ctx.register_image_blocks(session_image_blocks(session)) conversation_id = str(run_metadata.get("conversation_id")) if run_metadata.get("conversation_id") else None with run_tracer.agent(conversation_id=conversation_id) as agent_span: run_ctx.agent_span = agent_span @@ -606,7 +609,7 @@ async def _prepare_run_start( agent_span.for_each( lambda span, option: annotate_agent_start( span, - prompt=effective_prompt, + prompt=None if skip_user_prompt else effective_prompt, instructions=instructions, capture_messages=option.capture_messages, top_level=not self._is_child_harness, diff --git a/thinharness/projections.py b/thinharness/projections.py index 2f633d3..14f7678 100644 --- a/thinharness/projections.py +++ b/thinharness/projections.py @@ -58,7 +58,7 @@ def model_request_delta_from_tool_outputs( ) -> ModelRequestDelta: """Build a request delta for a tool-output provider continuation.""" entries: list[TranscriptEntry] = [ - ToolResultEntry(call_id=output.call_id, result=output.result) + ToolResultEntry(call_id=output.call_id, result=output.result, wire_output=output.wire_output) for output in outputs ] if notice_text := render_model_notices(notices): @@ -94,7 +94,7 @@ def trace_input_messages_from_entries(entries: list[TranscriptEntry]) -> list[Js "parts": [{ "type": "tool_result", "id": entry.call_id, - "content": entry.result.redacted_json(), + "content": entry.wire_output if entry.wire_output is not None else entry.result.redacted_json(), }], }) else: @@ -132,7 +132,10 @@ def model_request_input_from_delta(delta: ModelRequestDelta) -> Json | None: return {"correction": projected} tool_outputs = [ - {"call_id": entry.call_id, "output": entry.result.redacted_json()} + { + "call_id": entry.call_id, + "output": entry.wire_output if entry.wire_output is not None else entry.result.redacted_json(), + } for entry in delta.entries if isinstance(entry, ToolResultEntry) ] diff --git a/thinharness/providers.py b/thinharness/providers.py index b05ef07..f3f47ef 100644 --- a/thinharness/providers.py +++ b/thinharness/providers.py @@ -113,6 +113,7 @@ class ToolResultEntry: call_id: str result: ToolResult + wire_output: str | None = None TranscriptEntry = AssistantEntry | UserEntry | ToolResultEntry @@ -124,18 +125,17 @@ class ToolOutput: call_id: str result: ToolResult + wire_output: str | None = None def __post_init__(self) -> None: - """Normalize direct low-level text outputs into tool envelopes.""" - if isinstance(self.result, str): - self.result = ToolResult(True, self.result) - elif not isinstance(self.result, ToolResult): + """Require the normalized tool-result contract.""" + if not isinstance(self.result, ToolResult): raise TypeError("ToolOutput.result must be a ToolResult") @property def output(self) -> str: - """Return the canonical text envelope.""" - return self.result.to_json() + """Return the exact provider-facing text when one is required.""" + return self.wire_output if self.wire_output is not None else self.result.to_json() @dataclass(frozen=True) @@ -447,11 +447,32 @@ def _model_tool_call_from_dict(value: Any) -> ModelToolCall: def _append_tool_results(transcript: list[TranscriptEntry], outputs: list[ToolOutput], notice_text: str) -> None: - transcript.extend(ToolResultEntry(call_id=output.call_id, result=copy.deepcopy(output.result)) for output in outputs) + transcript.extend( + ToolResultEntry( + call_id=output.call_id, + result=copy.deepcopy(output.result), + wire_output=output.wire_output, + ) + for output in outputs + ) if notice_text: transcript.append(UserEntry(content=(TextBlock(notice_text),), notice=True)) +def session_image_blocks(session: ModelSession) -> list[ImageBlock]: + """Return image blocks restored into a built-in model session.""" + transcript = getattr(session, "transcript", None) + if not isinstance(transcript, list): + return [] + images: list[ImageBlock] = [] + for entry in transcript: + if isinstance(entry, UserEntry): + images.extend(block for block in entry.content if isinstance(block, ImageBlock)) + elif isinstance(entry, ToolResultEntry): + images.extend(block for block in entry.result.blocks if isinstance(block, ImageBlock)) + return images + + def _append_assistant_turn(transcript: list[TranscriptEntry], turn: ModelTurn) -> None: transcript.append( AssistantEntry( @@ -807,7 +828,11 @@ async def continue_with_tools( ) -> ModelTurn: """Continue a Responses API run with function_call_output items.""" input_payload: list[Json] = [ - {"type": "function_call_output", "call_id": output.call_id, "output": _openai_tool_output(output.result)} + { + "type": "function_call_output", + "call_id": output.call_id, + "output": output.wire_output if output.wire_output is not None else _openai_tool_output(output.result), + } for output in outputs ] notice_text = render_model_notices(notices) @@ -955,7 +980,14 @@ async def continue_with_tools( notices: list[ModelNotice] | None = None, ) -> ModelTurn: """Continue an Anthropic Messages run with tool_result blocks.""" - content = [{"type": "tool_result", "tool_use_id": output.call_id, "content": _anthropic_tool_output(output.result)} for output in outputs] + content = [ + { + "type": "tool_result", + "tool_use_id": output.call_id, + "content": output.wire_output if output.wire_output is not None else _anthropic_tool_output(output.result), + } + for output in outputs + ] notice_text = render_model_notices(notices) if notice_text: content.append({"type": "text", "text": notice_text}) @@ -1126,7 +1158,11 @@ async def continue_with_tools( _append_tool_results(self.transcript, outputs, notice_text) self._apply_resume(constants.instructions) for output in outputs: - self.messages.append({"role": "tool", "tool_call_id": output.call_id, "content": _openrouter_tool_output_json(output.result)}) + self.messages.append({ + "role": "tool", + "tool_call_id": output.call_id, + "content": output.wire_output if output.wire_output is not None else _openrouter_tool_output_json(output.result), + }) image_parts = _openrouter_tool_image_parts(outputs) if image_parts: if notice_text: @@ -1302,12 +1338,6 @@ def render_model_notices(notices: list[ModelNotice] | None) -> str: ) -def append_notices_to_text(text: str, notices: list[ModelNotice] | None) -> str: - """Append rendered notices to provider text input.""" - notice_text = render_model_notices(notices) - return text if not notice_text else f"{text}\n\n{notice_text}" - - def append_notices_to_content(content: NormalizedContent, notices: list[ModelNotice] | None) -> NormalizedContent: """Append notices as one final text block.""" return append_text_block(content, render_model_notices(notices)) @@ -1447,7 +1477,8 @@ def _render_anthropic_transcript(entries: list[TranscriptEntry], *, thinking_ena while index < len(entries): entry = entries[index] if isinstance(entry, UserEntry): - messages.append({"role": "user", "content": _anthropic_user_content(entry.content)}) + user_content = _anthropic_content_parts(entry.content) if entry.notice else _anthropic_user_content(entry.content) + messages.append({"role": "user", "content": user_content}) index += 1 continue if isinstance(entry, AssistantEntry): diff --git a/thinharness/runtime.py b/thinharness/runtime.py index 5a5ac85..aad98b7 100644 --- a/thinharness/runtime.py +++ b/thinharness/runtime.py @@ -26,7 +26,7 @@ model_request_delta_from_tool_outputs, stream_tool_calls_from_assistant, ) -from .providers import ModelNotice, ModelSession, ModelTurn, ToolOutput +from .providers import ModelNotice, ModelSession, ModelTurn, ProviderError, ToolOutput from .tracing import ( RunTracer, annotate_agent_result, @@ -189,9 +189,13 @@ def set_prompt_content(self, content: NormalizedContent) -> None: self.prompt = content self.image_blocks.extend(block for block in content if isinstance(block, ImageBlock)) + def register_image_blocks(self, blocks: Sequence[ImageBlock]) -> None: + """Register known images for later error redaction.""" + self.image_blocks.extend(blocks) + def record_tool_result_images(self, result: Any) -> None: """Record tool-result images for later error redaction.""" - self.image_blocks.extend(block for block in result.blocks if isinstance(block, ImageBlock)) + self.register_image_blocks([block for block in result.blocks if isinstance(block, ImageBlock)]) def stream_base(self) -> dict[str, Any]: """Return common event metadata for this run.""" @@ -347,9 +351,16 @@ async def advance_model( self.usage.cached_tokens += turn.usage.cached_tokens or 0 except Exception as exc: message = redact_image_data(str(exc), self.image_blocks) - model_span.record_exception(HarnessError(message)) - model_span.set_error(message, type(exc).__name__) - raise + if message == str(exc): + model_span.record_exception(exc) + model_span.set_error(message, type(exc).__name__) + raise + sanitized = HarnessError(message) + if isinstance(exc, ProviderError): + sanitized.__dict__["_thinharness_provider_error"] = True + model_span.record_exception(sanitized) + model_span.set_error(message, type(sanitized).__name__) + raise sanitized from exc model_span.for_each( lambda span, option: annotate_model_span( span, diff --git a/thinharness/tool_execution.py b/thinharness/tool_execution.py index df4a8a2..37628df 100644 --- a/thinharness/tool_execution.py +++ b/thinharness/tool_execution.py @@ -89,9 +89,11 @@ async def execute_batch( results = await self._run_calls_concurrently(calls, indices) records = [] for call, execution in zip(calls, results, strict=True): - record = {"call": {"id": call.id, "name": call.name, "arguments": call.arguments}, "result": execution.envelope.to_value()} - if not execution.envelope.has_image: - record["output"] = execution.output + record = { + "call": {"id": call.id, "name": call.name, "arguments": call.arguments}, + "result": execution.envelope.to_value(), + "output": execution.envelope.redacted_json(), + } if execution.cancelled: record["cancelled"] = True records.append(record) diff --git a/thinharness/tools/base.py b/thinharness/tools/base.py index 5c8ac9d..86a1d3f 100644 --- a/thinharness/tools/base.py +++ b/thinharness/tools/base.py @@ -82,21 +82,20 @@ def __post_init__(self) -> None: raise TypeError("ToolResult.ok must be a bool") if not isinstance(self.metadata, dict): raise TypeError("ToolResult.metadata must be a dict") - if isinstance(self.content, str): - if not self.content: - raise ValueError("ToolResult.content must not be empty") - else: + if not isinstance(self.content, str): self.content = normalize_content(self.content, label="ToolResult.content") @property def blocks(self) -> tuple[ContentBlock, ...]: - """Return content in normalized block form.""" + """Return content in block form, including an empty string block.""" + if isinstance(self.content, str): + return (TextBlock(self.content),) return normalize_content(self.content, label="ToolResult.content") @property def has_image(self) -> bool: """Return whether this result contains image content.""" - return any(isinstance(block, ImageBlock) for block in self.blocks) + return not isinstance(self.content, str) and any(isinstance(block, ImageBlock) for block in self.blocks) def to_value(self) -> Json: """Return the canonical JSON-compatible envelope value.""" @@ -116,8 +115,6 @@ def from_value(cls, parsed: Any, *, label: str = "tool output") -> ToolResult: raise ValueError(f"{label} has wrong type") content = parsed["content"] if isinstance(content, str): - if not content: - raise ValueError(f"{label} content must not be empty") return cls(parsed["ok"], content, parsed["metadata"]) return cls(parsed["ok"], content_from_json(content, label=f"{label} content"), parsed["metadata"]) @@ -265,14 +262,26 @@ def _prepare_args(spec: ToolSpec, raw_args: str | Json) -> ToolEnvelope | Any: def _normalize_result(result: Any) -> ToolEnvelope: - """Normalize a tool handler result to a structured JSON envelope.""" + """Normalize and validate a tool handler result.""" if isinstance(result, ToolResult): - return result - if isinstance(result, str): - return ToolResult(True, result) - if isinstance(result, Sequence) and all(isinstance(block, (TextBlock, ImageBlock)) for block in result): - return ToolResult(True, result) - return ToolResult(True, json.dumps(result, indent=2, sort_keys=True)) + envelope = result + elif isinstance(result, str): + envelope = ToolResult(True, result) + elif isinstance(result, Sequence) and result and all(isinstance(block, (TextBlock, ImageBlock)) for block in result): + envelope = ToolResult(True, result) + else: + envelope = ToolResult(True, json.dumps(result, indent=2, sort_keys=True)) + envelope.to_json() + return envelope + + +def _invalid_result_envelope(exc: Exception) -> ToolEnvelope: + """Return a failed envelope for an unsupported handler result.""" + return ToolResult( + False, + f"Invalid tool result: {type(exc).__name__}: {exc}", + {"error_type": "InvalidToolResult"}, + ) def _retry_envelope(error_type: str, message: str, *, errors: list[Json] | None = None) -> ToolEnvelope: @@ -333,7 +342,10 @@ def call_tool(spec: ToolSpec, raw_args: str | Json) -> str: "async handler requires harness execution", {"error_type": "AsyncHandlerInSyncContext"}, ).to_json() - return _normalize_result(result).to_json() + try: + return _normalize_result(result).to_json() + except (TypeError, ValueError) as exc: + return _invalid_result_envelope(exc).to_json() async def _invoke_tool(spec: ToolSpec, raw_args: str | Json) -> ToolEnvelope: @@ -359,7 +371,10 @@ async def _invoke_tool(spec: ToolSpec, raw_args: str | Json) -> ToolEnvelope: if getattr(exc, "_thinharness_strict_hook", False): raise return ToolResult(False, f"{type(exc).__name__}: {exc}", {"error_type": type(exc).__name__}) - return _normalize_result(result) + try: + return _normalize_result(result) + except (TypeError, ValueError) as exc: + return _invalid_result_envelope(exc) def _is_async_callable(handler: ToolHandler) -> bool: diff --git a/thinharness/tools/filesystem.py b/thinharness/tools/filesystem.py index 894f97f..1238a05 100644 --- a/thinharness/tools/filesystem.py +++ b/thinharness/tools/filesystem.py @@ -5,6 +5,8 @@ import heapq import itertools import json +import os +import stat import subprocess import time import uuid @@ -248,16 +250,18 @@ def read_image(self, args: ReadImageArgs | Json) -> ToolResult: try: if not path.exists(): return ToolResult(False, f"file not found: {display}", {"path": str(path)}) - if path.is_dir(): - return ToolResult(False, f"path is a directory: {display}", {"path": str(path)}) - size = path.stat().st_size - if size > self.max_image_bytes: + if not path.is_file(): + return ToolResult(False, f"path is not a regular file: {display}", {"path": str(path)}) + with path.open("rb") as handle: + if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode): + return ToolResult(False, f"path is not a regular file: {display}", {"path": str(path)}) + data = handle.read(self.max_image_bytes + 1) + if len(data) > self.max_image_bytes: return ToolResult( False, - f"image is {size} bytes, over max_image_bytes={self.max_image_bytes}", - {"path": str(path), "size_bytes": size, "max_image_bytes": self.max_image_bytes}, + f"image is over max_image_bytes={self.max_image_bytes}", + {"path": str(path), "size_bytes": len(data), "max_image_bytes": self.max_image_bytes}, ) - data = path.read_bytes() except OSError as exc: return ToolResult(False, f"{type(exc).__name__}: {exc}", {"path": str(path), "error_type": type(exc).__name__}) media_type = _detect_image_media_type(data) @@ -631,21 +635,32 @@ def _detect_image_media_type(data: bytes) -> str | None: """Detect supported image containers from complete minimum signatures.""" if len(data) >= 33 and data.startswith(b"\x89PNG\r\n\x1a\n") and data[8:12] == b"\x00\x00\x00\r" and data[12:16] == b"IHDR": return "image/png" - if len(data) >= 4 and data.startswith(b"\xff\xd8\xff") and data.endswith(b"\xff\xd9"): + if ( + len(data) >= 6 + and data.startswith(b"\xff\xd8\xff") + and data[3] not in {0x00, 0xFF} + and b"\xff\xd9" in data[4:] + ): return "image/jpeg" if len(data) >= 13 and data[:6] in {b"GIF87a", b"GIF89a"}: return "image/gif" - if ( - len(data) >= 20 - and data.startswith(b"RIFF") - and data[8:12] == b"WEBP" - and data[12:16] in {b"VP8 ", b"VP8L", b"VP8X"} - and int.from_bytes(data[4:8], "little") == len(data) - 8 - ): + if _valid_webp_header(data): return "image/webp" return None +def _valid_webp_header(data: bytes) -> bool: + """Return whether the first WebP chunk fits its declared RIFF container.""" + if len(data) < 20 or not data.startswith(b"RIFF") or data[8:12] != b"WEBP": + return False + if data[12:16] not in {b"VP8 ", b"VP8L", b"VP8X"}: + return False + riff_end = int.from_bytes(data[4:8], "little") + 8 + chunk_size = int.from_bytes(data[16:20], "little") + chunk_end = 20 + chunk_size + (chunk_size % 2) + return chunk_end <= riff_end <= len(data) + + # ============================================================================= # Tool plumbing # ============================================================================= diff --git a/thinharness/tools/mcp.py b/thinharness/tools/mcp.py index f4c17e2..cc3b7f8 100644 --- a/thinharness/tools/mcp.py +++ b/thinharness/tools/mcp.py @@ -167,7 +167,7 @@ async def call_tool(self, name: str, arguments: Json, *, server_id: str | None = structured_content = getattr(result, "structuredContent", None) if structured_content is not None: structured = TextBlock(json.dumps(structured_content, ensure_ascii=False)) - content = (structured, *_content_to_blocks(result.content, include_text=False)) + content = (structured, *_content_to_blocks(result.content, include_text=False, images_only=True)) else: content = _content_to_blocks(result.content, include_text=True) if all(isinstance(block, TextBlock) for block in content): @@ -385,8 +385,13 @@ def _content_to_text(blocks: list[Any]) -> str: return "\n".join(parts) -def _content_to_blocks(blocks: list[Any], *, include_text: bool) -> tuple[TextBlock | ImageBlock, ...]: - """Preserve supported MCP images and ordered text placeholders.""" +def _content_to_blocks( + blocks: list[Any], + *, + include_text: bool, + images_only: bool = False, +) -> tuple[TextBlock | ImageBlock, ...]: + """Preserve supported MCP images and selected ordered placeholders.""" parts: list[TextBlock | ImageBlock] = [] for block in blocks: block_type = getattr(block, "type", "") @@ -408,11 +413,11 @@ def _content_to_blocks(blocks: list[Any], *, include_text: bool) -> tuple[TextBl parts.append(ImageBlock(decoded, media_type)) # type: ignore[arg-type] continue parts.append(TextBlock(f"[image: {media_type}]")) - elif block_type == "audio": + elif not images_only and block_type == "audio": parts.append(TextBlock(f"[audio: {getattr(block, 'mimeType', 'unknown')}]")) - elif block_type in {"resource", "resource_link"}: + elif not images_only and block_type in {"resource", "resource_link"}: parts.append(TextBlock(_resource_placeholder(block))) - else: + elif not images_only: parts.append(TextBlock(str(block))) return tuple(parts) diff --git a/thinharness/tracing.py b/thinharness/tracing.py index 396b246..907d987 100644 --- a/thinharness/tracing.py +++ b/thinharness/tracing.py @@ -510,7 +510,7 @@ def annotate_model_span(span: _SpanAdapter, turn: Any, *, capture_messages: bool def annotate_agent_start( span: _SpanAdapter, *, - prompt: NormalizedContent, + prompt: NormalizedContent | None, instructions: str, capture_messages: bool, top_level: bool, @@ -518,13 +518,13 @@ def annotate_agent_start( """Write opt-in agent input attributes before provider work runs.""" if not capture_messages: return - projected_prompt = text_only_value(prompt) or redacted_content_string(prompt) + projected_prompt = None if prompt is None else text_only_value(prompt) or redacted_content_string(prompt) if top_level: span.set_attributes({ "gen_ai.prompt": projected_prompt, "gen_ai.system_instructions": serialize_attribute_value([{"type": "text", "content": instructions}]), }) - else: + elif projected_prompt is not None: span.set_attribute("gen_ai.prompt", projected_prompt) diff --git a/thinharness/turns.py b/thinharness/turns.py index bcca219..9d6093c 100644 --- a/thinharness/turns.py +++ b/thinharness/turns.py @@ -185,7 +185,7 @@ async def send_tool_outputs( retry_message = decision.retry_message run_ctx.emit_retry_event("structured_output", retry_message, final_id) turn, decision = await send_tool_outputs( - [ToolOutput(final_id, ToolResult(True, retry_message))], + [ToolOutput(final_id, ToolResult(True, retry_message), wire_output=retry_message)], kind="output_retry_tool", output_retry=True, ) From 51c52198bea3e690da58b2d5afb97766c7579a08 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Wed, 19 Aug 2026 23:51:53 -0400 Subject: [PATCH 19/30] Fix final image input review findings --- CHANGELOG.md | 1 + docs/behavior.md | 10 +- docs/docs.md | 6 +- docs/site/explainer/index.html | 2 +- tests/e2e/image_inputs_journey.py | 31 ++- tests/unit/test_hooks.py | 34 ++++ tests/unit/test_image_inputs.py | 320 +++++++++++++++++++++++++++++- tests/unit/test_mcp.py | 23 +++ tests/unit/test_streaming.py | 2 +- tests/unit/test_tool_retry.py | 49 ++++- thinharness/core.py | 92 ++++++--- thinharness/hooks.py | 21 +- thinharness/providers.py | 37 +++- thinharness/runtime.py | 11 +- thinharness/tool_execution.py | 46 ++++- thinharness/tools/base.py | 14 +- thinharness/tools/mcp.py | 4 +- 17 files changed, 626 insertions(+), 77 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 903a084..4ce9c49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Added ordered text and image prompts and tool results for OpenAI, Anthropic, and OpenRouter, with provider-neutral immutable content blocks, redacted observability projections, self-contained version 4 resume state, opt-in filesystem `read_image`, and preserved successful MCP images. - **Breaking:** Renamed custom `ModelSession.continue_with_user_text(...)` to `continue_with_user_content(...)`; prompt hooks now receive normalized content-block tuples, and built-in transcript resume version 3 state must be regenerated. +- **Breaking:** `ToolOutput` now requires `ToolOutput(call_id, ToolResult(...))`; direct string output construction is no longer accepted. - Added explicit plugin composition with static and connected contributions, atomic connection rollback, unique plugin names, generic tool origin, and plugin-provided hooks and instructions. - Added `FilesystemPlugin` for the ordered workspace tool surface; `jsonl_search` remains opt-in through this plugin. - Added `BashPlugin` for explicit one-shot local Bash commands with strict arguments, contained cwd, minimal environment inheritance, bounded separate head-and-tail output, host-capped timeouts, process-group cleanup, and cancellation propagation. diff --git a/docs/behavior.md b/docs/behavior.md index 96add5b..3ea580b 100644 --- a/docs/behavior.md +++ b/docs/behavior.md @@ -72,10 +72,10 @@ ThinHarness accepts ordered text and local image content through one provider-ne ### Requirements - IMAGE-CONTENT-1: `TextBlock`, `ImageBlock`, and `Prompt` are the public content interface. Prompts accept a non-empty string or a non-empty copied sequence of immutable blocks. Text and image data must be non-empty, image data must be `bytes`, and media types are limited to JPEG, PNG, GIF, and WebP. -- IMAGE-CONTENT-2: `Harness.run()`, `stream()`, and `run_sync()` normalize content before the first provider request. Run-start and user-prompt hooks receive normalized block tuples and can replace them with a valid string or block sequence. Caller blocks, hook context, and harness notices remain in that order. +- IMAGE-CONTENT-2: `Harness.run()`, `stream()`, and `run_sync()` normalize content before run hooks and the first provider request. Invalid caller prompts and invalid run-start or user-prompt hook replacements fail as `HarnessError` with the validation detail. Run-start and user-prompt hooks receive normalized block tuples and can replace them with a valid string or block sequence. Caller blocks, hook context, and harness notices remain in that order. - IMAGE-CONTENT-3: Tool results accept a string, including an empty string, or a non-empty ordered block sequence. Empty and ordinary JSON lists remain ordinary JSON tool data. Invalid or non-serializable handler results become failed tool envelopes. Text-only provider payloads remain unchanged. Image-bearing provider payloads preserve execution outcome, metadata, tool-call id, content order, and images through provider-native OpenAI and Anthropic blocks. - IMAGE-CONTENT-4: OpenRouter keeps a canonical JSON tool message for each result. Tool images are descriptors in that message and are projected after the parallel tool-message batch as labelled user content, with each label immediately before its image and any harness notice after all image parts. -- IMAGE-CONTENT-5: After-tool hook string fields contain canonical JSON, including base64 image fields, while the envelope is the structured mutation interface. Either mutation path is strictly validated and synchronized. Retry classification uses the final post-hook envelope. These fields, completed results, and resume state can be sensitive and large. +- IMAGE-CONTENT-5: After-tool hook string fields contain canonical JSON, including base64 image fields, while the envelope is the structured mutation interface. Either mutation path is validated and synchronized within each hook's error policy. A non-strict invalid mutation is logged and rolled back; a strict invalid mutation raises `HarnessError`. Retry control flow, budget use, retry events, and retry span status use the pre-hook classification, while hooks can change model-visible output. These fields, completed results, and resume state can be sensitive and large. - IMAGE-CONTENT-5A: Every tool-call record contains canonical structured `result` and string `output` fields. Image-bearing `output` is the canonical redacted projection, while `result` retains the complete image data once. - IMAGE-CONTENT-6: Remote image URLs, image fetching, image generation, other media, conversion, OCR, image-bearing subagent tasks, and image-bearing `parallel_llm` prompts are not supported. Bash output always stays text and does not load image paths. Model image capability errors come from the selected provider or custom model, not model-name checks. @@ -91,7 +91,7 @@ Built-in provider resume state is a self-contained, provider-agnostic transcript - RESUME-2: `resume_state` is self-contained and does not depend on provider continuation tokens such as OpenAI `previous_response_id`; an OpenAI run that never received a response id is still resumable. - RESUME-3: Resuming on the originating provider preserves native reasoning (Anthropic thinking signatures, OpenAI `encrypted_content`, OpenRouter `reasoning_details`); resuming on a different provider degrades each reasoning part to a leading ``-tagged text block and drops the opaque blob. Native re-emit additionally requires the resuming run to be able to accept the block: OpenAI re-emits the native reasoning item only when the resuming model is reasoning-capable, and Anthropic uses the thinking gate in RESUME-3A; otherwise both use the text fallback. So a reasoning-model capture resumed on a non-reasoning model of the same provider degrades to text. - RESUME-3A: Anthropic resume treats explicit `extra_body["thinking"]` as authoritative: `enabled` and `adaptive` accept signed thinking replay, while `disabled`, unknown, or malformed values suppress native replay. Without an explicit thinking key, `HarnessConfig.effort` implies adaptive thinking; otherwise Anthropic models outside the legacy off-by-default families (`claude-opus-4`, `claude-sonnet-4`, `claude-haiku-4`, and `claude-3`) are assumed to run thinking by default and keep signed thinking blocks on resume. -- RESUME-4: Built-in provider resume state uses `version` 4. User and tool entries carry canonical ordered content blocks; image bytes are stored once as base64. Older state and old provider-native `kind` values are rejected with a regenerate error. An approval envelope that contains version 3 provider state fails with `approval state provider_state version 3 is not supported`. +- RESUME-4: Built-in provider resume state uses `version` 4. User and tool entries carry canonical ordered content blocks; image bytes are stored once as base64. A tool entry also stores nullable `wire_output` when its exact provider-facing text differs from the canonical result, and replay uses that text byte-for-byte. Older state and old provider-native `kind` values are rejected with a regenerate error. An approval envelope that contains version 3 provider state fails with `approval state provider_state version 3 is not supported`. - RESUME-5: On resume, the live system prompt from the resuming harness config is re-injected; captured system prompts are not stored or restored. - RESUME-6: A session seeded via `OpenAIResponsesSession.start(prompt, constants, previous_response_id=...)` captures only new transcript entries, so externally seeded prior turns are not present when later resumed from `resume_state`. This is unrelated to reasoning fidelity and is not changed by RESUME-3/RESUME-7. - RESUME-7: For reasoning-capable OpenAI Responses models the harness requests `include=["reasoning.encrypted_content"]` so reasoning survives resume; non-reasoning models are unaffected. Captured `resume_state` can contain image bytes with base64 overhead, encrypted reasoning blobs (OpenAI/OpenRouter), and signed thinking (Anthropic), so it must be treated as sensitive and potentially large. @@ -303,8 +303,8 @@ Tracing and streaming expose projections of the same neutral per-request model-v - MODEL-OBSERVABILITY-7: Model spans pin `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.usage.cache_read.input_tokens`, `gen_ai.usage.total_tokens`, `gen_ai.response.model`, and `gen_ai.response.finish_reasons`. `cache_read.input_tokens` carries provider-reported cached input tokens when present and is omitted when unreported. `finish_reasons` is always a list wrapping the normalized reason. `total_tokens` passes through a raw provider `total_tokens` when present and is otherwise computed as input+output only when both are present; partial usage yields no total. - MODEL-OBSERVABILITY-8: Custom `Model` implementations that do not populate normalized `ModelTurn` usage fields keep their `gen_ai.usage.*` span attributes via best-effort extraction from the raw response. - MODEL-OBSERVABILITY-9: Trace attributes and non-terminal progress events never contain image bytes, base64, or data URLs. Multimodal prompt and tool-result values use compact ordered JSON with visible text and image descriptors containing media type, byte size, and zero-based block index; existing field types and text-only values stay unchanged. -- MODEL-OBSERVABILITY-10: Provider errors and span errors redact image encodings restored from resume or approval state as well as new run-known images and complete image data URLs. Redacted failures use a sanitized harness exception; failures that need no redaction preserve their original exception type. Provider adapters do not include serialized request bodies in error messages. -- MODEL-OBSERVABILITY-11: `ToolCallCompletedEvent.message` remains plain text. `RunCompletedEvent.result` remains identical to the complete run result and can contain sensitive, large image-bearing records and resume state. +- MODEL-OBSERVABILITY-10: Provider, hook, and span errors redact image encodings restored from resume or approval state as well as caller prompts, tool results, valid hook replacements, and complete image data URLs. Redacted provider failures keep `ProviderError` classification and status code in public events and spans while recording only the sanitized message. Failures that need no redaction preserve their original exception type. Provider adapters do not include serialized request bodies in error messages. +- MODEL-OBSERVABILITY-11: `ToolCallCompletedEvent.message` remains plain text. The agent span records the redacted normalized caller prompt, while the first model span records the effective post-hook prompt. Approval resume adds no prompt annotation. `RunCompletedEvent.result` remains identical to the complete run result and can contain sensitive, large image-bearing records and resume state. ## MCP Client Layer diff --git a/docs/docs.md b/docs/docs.md index 16c74eb..d04172e 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -399,7 +399,7 @@ Hook events: - `limit_reached` - `run_end` -`user_prompt_submit`, `before_tool_call`, and `before_subagent_run` are cancellable. Run-start and prompt-submit hooks receive normalized content-block tuples and can replace the prompt with a string or valid block sequence. `after_tool_call` can rewrite canonical `ctx.output` or structured `ctx.envelope`; either form is strictly validated and keeps the other synchronized. These after-tool fields can contain full base64 image data and can be sensitive and large. Tool filters apply only to tool events; agent filters apply only to subagent events. +`user_prompt_submit`, `before_tool_call`, and `before_subagent_run` are cancellable. Run-start and prompt-submit hooks receive normalized content-block tuples and can replace the prompt with a string or valid block sequence. Invalid prompt replacements fail as `HarnessError`. `after_tool_call` can rewrite canonical `ctx.output` or structured `ctx.envelope`; either form is validated and keeps the other synchronized. A malformed non-strict mutation is logged and rolled back, while a malformed strict mutation fails as `HarnessError`. Tool retry control flow and budgets use the result classification from before after-tool hooks; hooks can change model-visible output but cannot create or suppress the current retry. These after-tool fields can contain full base64 image data and can be sensitive and large. Tool filters apply only to tool events; agent filters apply only to subagent events. By default, hook exceptions are logged and the run continues. Set `strict_hooks=True` to make hook exceptions fail the run. @@ -621,7 +621,7 @@ Budgets span the pause. The paused batch counts against `usage.tool_calls` exact Built-in provider resume details: - `resume_state["kind"] == "transcript"` and `version == 4`. Older transcript versions must be regenerated; approval envelopes with version 3 nested provider state also fail. -- The transcript is provider-agnostic and no longer depends on OpenAI server-side response retention. Ordered image bytes are self-contained as base64, which adds about 33% encoding overhead. +- The transcript is provider-agnostic and no longer depends on OpenAI server-side response retention. Ordered image bytes are self-contained as base64, which adds about 33% encoding overhead. Exact structured-output retry wire text is also stored and replayed byte-for-byte when it differs from the canonical tool result. - Provider-specific reasoning chains are preserved on same-provider resume (Anthropic thinking signatures, OpenAI `encrypted_content`, OpenRouter `reasoning_details`) and degraded to a leading ``-tagged text block on cross-provider resume. Anthropic native re-emit also requires extended thinking to be enabled in the resuming run. For reasoning-capable OpenAI models the harness adds `include=["reasoning.encrypted_content"]`, so `resume_state` can contain encrypted reasoning blobs — treat it as sensitive. - Cross-provider resume is supported by the built-in renderers, but real providers may reject foreign-format tool-call ids or malformed tool-call argument JSON. - `OpenAIResponsesSession.start(prompt, constants, previous_response_id=...)` remains available as a low-level escape hatch, but later resume state captures only the new prompt onward, not the externally seeded prior turns. @@ -661,7 +661,7 @@ Local tracing is on by default. It writes plaintext JSONL traces under: ~/.thinharness/traces// ``` -Those traces can include prompt text, model outputs, tool arguments, and tool-result text. Image bytes, base64, and data URLs are replaced with ordered descriptors that contain media type, byte size, and block index. Completed results and resume state still retain full images; treat them as sensitive local data. +Those traces can include prompt text, model outputs, tool arguments, and tool-result text. The agent span shows the normalized raw caller prompt, while the first model span shows the effective prompt after hooks; approval resume adds no prompt. Image bytes, base64, and data URLs are replaced with ordered descriptors that contain media type, byte size, and block index. Provider failures keep their `ProviderError` classification and status code after message redaction. Completed results and resume state still retain full images; treat them as sensitive local data. Disable local trace files with: diff --git a/docs/site/explainer/index.html b/docs/site/explainer/index.html index 19b9820..b098ef3 100644 --- a/docs/site/explainer/index.html +++ b/docs/site/explainer/index.html @@ -326,7 +326,7 @@

    Retry semantics

  • A handler can raise ModelRetry to ask the model to retry with a hint.
  • Ordinary handler exceptions become failed tool results but are not retryable unless metadata says so.
  • Tool retry budgets are per tool name per run, not per individual call id.
  • -
  • after_tool_call hooks can rewrite canonical output or the structured envelope; retry control flow uses the final validated envelope.
  • +
  • after_tool_call hooks can rewrite canonical output or the structured envelope; retry control flow uses the pre-hook classification, while invalid non-strict mutations are rolled back.
  • diff --git a/tests/e2e/image_inputs_journey.py b/tests/e2e/image_inputs_journey.py index e2d3999..d77709b 100644 --- a/tests/e2e/image_inputs_journey.py +++ b/tests/e2e/image_inputs_journey.py @@ -28,10 +28,10 @@ ROOT = Path(__file__).resolve().parents[2] -def _known_image() -> bytes: - """Return a 64x32 PNG whose left half is red and right half is blue.""" +def _known_image(left: bytes, right: bytes) -> bytes: + """Return a 64x32 PNG with two known solid-color halves.""" width, height = 64, 32 - row = b"\x00" + (b"\xff\x00\x00" * (width // 2)) + (b"\x00\x00\xff" * (width // 2)) + row = b"\x00" + (left * (width // 2)) + (right * (width // 2)) raw = row * height def chunk(kind: bytes, data: bytes) -> bytes: @@ -45,7 +45,8 @@ def chunk(kind: bytes, data: bytes) -> bytes: ) -IMAGE = _known_image() +IMAGE = _known_image(b"\xff\x00\x00", b"\x00\x00\xff") +TOOL_IMAGE = _known_image(b"\x00\xff\x00", b"\xff\xff\x00") class RecordingOpenAI(OpenAIProvider): @@ -85,8 +86,8 @@ def image_tool() -> ToolSpec: {"type": "object", "properties": {}, "additionalProperties": False}, lambda _args: ToolResult( True, - (TextBlock("comparison fixture"), ImageBlock(IMAGE, "image/png")), - {"fixture": "red-blue.png"}, + (TextBlock("comparison fixture"), ImageBlock(TOOL_IMAGE, "image/png")), + {"fixture": "green-yellow.png"}, ), ) @@ -98,11 +99,17 @@ async def run_provider(label: str, model, payloads: list[dict]) -> None: tools=[image_tool()], ) first = await harness.run(( - TextBlock("Name the color on the left and the color on the right, then call inspect_fixture and compare the two images."), + TextBlock( + "For the supplied image, name the left and right colors. Then call inspect_fixture and name its left and right colors. " + "Report all four facts." + ), ImageBlock(IMAGE, "image/png"), )) assert "red" in first.text.lower() assert "blue" in first.text.lower() + assert "green" in first.text.lower() + assert "yellow" in first.text.lower() + assert any(record["call"]["name"] == "inspect_fixture" for record in first.tool_call_records) assert first.resume_state is not None resumed = await harness.run("In one sentence, restate the comparison.", resume_from=first.resume_state) assert resumed.text @@ -114,6 +121,16 @@ async def run_provider(label: str, model, payloads: list[dict]) -> None: for item in payload.get("input", []) if isinstance(item, dict) and item.get("type") == "function_call_output" ) + if label == "anthropic": + assert any( + block.get("type") == "image" + for payload in payloads + for message in payload.get("messages", []) + for item in message.get("content", []) if isinstance(message.get("content"), list) + if isinstance(item, dict) and item.get("type") == "tool_result" and isinstance(item.get("content"), list) + for block in item["content"] + if isinstance(block, dict) + ) if label == "openrouter": assert any( part.get("text", "").startswith("[tool image call_id=") diff --git a/tests/unit/test_hooks.py b/tests/unit/test_hooks.py index 9fefa46..ce2ef4d 100644 --- a/tests/unit/test_hooks.py +++ b/tests/unit/test_hooks.py @@ -357,6 +357,40 @@ def fail(ctx): with pytest.raises(RuntimeError, match="after failed"): harness.run_sync("go") +@pytest.mark.parametrize("field", ["output", "envelope"]) +@pytest.mark.parametrize("strict", [False, True]) +def test_after_tool_hook_invalid_mutation_uses_hook_error_policy(field: str, strict: bool) -> None: + original = ToolResult(True, "original", {"stable": True}) + + def mutate(ctx: AfterToolCallContext) -> None: + if field == "output": + ctx.output = "not canonical json" + else: + ctx.envelope.metadata = {"bad": Path("not-json")} + + registry = HookRegistry([Hook("after_tool_call", mutate)], strict_hooks=strict) + ctx = AfterToolCallContext( + harness=None, # type: ignore[arg-type] + call_id="call_1", + tool_name="raw", + arguments="{}", + original_output=original.to_json(), + output=original.to_json(), + envelope=original, + duration_ms=0, + ) + + if strict: + with pytest.raises(HarnessError, match="canonical output validation failed"): + registry.fire_after_tool_call(ctx) + else: + registry.fire_after_tool_call(ctx) + + if not strict: + assert ctx.output == ToolResult(True, "original", {"stable": True}).to_json() + assert ctx.envelope == ToolResult(True, "original", {"stable": True}) + + def test_after_tool_hook_envelope_uses_normalized_invalid_output() -> None: seen = [] registry = HookRegistry([ diff --git a/tests/unit/test_image_inputs.py b/tests/unit/test_image_inputs.py index 47815de..fbdce01 100644 --- a/tests/unit/test_image_inputs.py +++ b/tests/unit/test_image_inputs.py @@ -83,6 +83,63 @@ def test_content_validation_rejects_invalid_public_values(bad: Any) -> None: normalize_content(bad) +@pytest.mark.parametrize("bad", ["", [], [TextBlock("")], [TextBlock("ok"), object()]]) +async def test_public_async_prompt_validation_raises_harness_error_and_emits_failure(tmp_path: Path, bad: Any) -> None: + provider = _Provider("OpenAI", []) + harness = Harness(_config(tmp_path), model=OpenAIResponsesModel("target", provider=provider)) # type: ignore[arg-type] + events = [] + + with pytest.raises(HarnessError): + async for event in harness.stream(bad): + events.append(event) + + failed = next(event for event in events if isinstance(event, RunFailedEvent)) + assert failed.error_type == "HarnessError" + assert failed.stop_reason == "error" + assert "prompt" in failed.message + assert provider.payloads == [] + + direct_provider = _Provider("OpenAI", []) + direct = Harness(_config(tmp_path), model=OpenAIResponsesModel("target", provider=direct_provider)) # type: ignore[arg-type] + with pytest.raises(HarnessError, match="prompt"): + await direct.run(bad) + assert direct_provider.payloads == [] + + +@pytest.mark.parametrize("bad", ["", [], [TextBlock("")], [TextBlock("ok"), object()]]) +def test_public_run_sync_prompt_validation_raises_harness_error(tmp_path: Path, bad: Any) -> None: + provider = _Provider("OpenAI", []) + harness = Harness(_config(tmp_path), model=OpenAIResponsesModel("target", provider=provider)) # type: ignore[arg-type] + + with pytest.raises(HarnessError, match="prompt"): + harness.run_sync(bad) + + assert provider.payloads == [] + + +@pytest.mark.parametrize("event", ["run_start", "user_prompt_submit"]) +@pytest.mark.parametrize("strict", [False, True]) +async def test_invalid_prompt_hook_replacement_is_harness_error( + tmp_path: Path, + event: str, + strict: bool, +) -> None: + def invalidate(ctx) -> None: + ctx.prompt = [] + + provider = _Provider("OpenAI", []) + harness = Harness( + HarnessConfig(root=tmp_path, system_prompt="sys", local_tracing=False, strict_hooks=strict), + model=OpenAIResponsesModel("target", provider=provider), # type: ignore[arg-type] + hooks=[Hook(event, invalidate)], # type: ignore[arg-type] + ) + + with pytest.raises(HarnessError, match=f"{event} prompt must not be empty"): + await harness.run("caller") + + assert provider.payloads == [] + + def test_content_contract_detaches_round_trips_and_redacts() -> None: caller = [TextBlock("before"), ImageBlock(PNG, "image/png"), TextBlock("after")] normalized = normalize_content(caller) @@ -182,6 +239,7 @@ async def test_openai_image_tool_result_payload_is_literal(tmp_path: Path) -> No {"type": "image", "media_type": "image/png", "data": PNG_B64}, ], "metadata": {"source": "fixture"}, + "wire_output": None, } @@ -307,6 +365,7 @@ def _resume_state( {"type": "image", "media_type": "image/png", "data": PNG_B64}, ], "metadata": {"source": "resume"}, + "wire_output": None, }, ]) entries.append({"role": "assistant", "text": "prior done", "tool_calls": [], "reasoning": []}) @@ -351,6 +410,46 @@ async def test_every_provider_pair_replays_user_and_tool_images(tmp_path: Path, assert "[tool image call_id=call_foreign block=1]" in rendered +async def test_same_provider_openai_resume_combines_native_reasoning_and_image(tmp_path: Path) -> None: + state = { + "kind": "transcript", + "version": 4, + "origin_provider": "openai", + "origin_model": "o3-source", + "entries": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "inspect"}, + {"type": "image", "media_type": "image/png", "data": PNG_B64}, + ], + "notice": False, + }, + { + "role": "assistant", + "text": "prior", + "tool_calls": [], + "reasoning": [{ + "text": "", + "signature": "encrypted-reasoning", + "id": "rs_1", + "provider_name": "openai", + }], + }, + ], + } + provider = _Provider("OpenAI", [{"id": "done", "output_text": "done"}]) + + await Harness(_config(tmp_path), model=OpenAIResponsesModel("o3-target", provider=provider)).run( # type: ignore[arg-type] + "follow-up", + resume_from=state, + ) + + payload = provider.payloads[0] + assert any(item.get("type") == "reasoning" and item.get("encrypted_content") == "encrypted-reasoning" for item in payload["input"]) + assert PNG_URL in json.dumps(payload, ensure_ascii=False) + + @pytest.mark.parametrize( "mutate, message", [ @@ -465,6 +564,30 @@ def submit(ctx): ] +async def test_agent_trace_keeps_raw_image_prompt_and_model_trace_uses_hook_context(tmp_path: Path) -> None: + tracer = FakeTracer() + + def add_context(ctx) -> None: + ctx.additional_context.append("effective-only policy") + + provider = _Provider("OpenAI", [{"id": "done", "output_text": "done"}]) + prompt = (TextBlock("caller"), ImageBlock(PNG, "image/png")) + await Harness( + _config(tmp_path), + model=OpenAIResponsesModel("target", provider=provider), # type: ignore[arg-type] + hooks=[Hook("user_prompt_submit", add_context)], + tracing=[TracingOptions(tracer=tracer, capture_messages=True)], + ).run(prompt) + + agent = next(span for span in tracer.spans if span.name.startswith("invoke_agent ")) + model = next(span for span in tracer.spans if span.name.startswith("chat ")) + assert "effective-only policy" not in agent.attributes["gen_ai.prompt"] + assert "effective-only policy" in model.attributes["gen_ai.input.messages"] + combined = json.dumps([agent.attributes, model.attributes], ensure_ascii=False) + assert PNG_B64 not in combined + assert PNG_URL not in combined + + async def test_image_events_and_tool_trace_are_redacted(tmp_path: Path) -> None: tracer = FakeTracer() provider = _Provider("OpenAI", [ @@ -490,6 +613,97 @@ async def test_image_events_and_tool_trace_are_redacted(tmp_path: Path) -> None: assert PNG_URL not in trace_text +@pytest.mark.parametrize("event", ["run_start", "user_prompt_submit"]) +async def test_strict_prompt_hook_failure_redacts_known_image_data(tmp_path: Path, event: str) -> None: + tracer = FakeTracer() + + def fail(_ctx) -> None: + raise RuntimeError(f"hook echoed {PNG_B64} and {PNG_URL}") + + provider = _Provider("OpenAI", []) + harness = Harness( + HarnessConfig(root=tmp_path, system_prompt="sys", local_tracing=False, strict_hooks=True), + model=OpenAIResponsesModel("target", provider=provider), # type: ignore[arg-type] + hooks=[Hook(event, fail)], # type: ignore[arg-type] + tracing=[TracingOptions(tracer=tracer, capture_messages=True)], + ) + events = [] + + with pytest.raises(HarnessError): + async for stream_event in harness.stream((TextBlock("go"), ImageBlock(PNG, "image/png"))): + events.append(stream_event) + + failed = next(item for item in events if isinstance(item, RunFailedEvent)) + observed = failed.message + json.dumps([span.attributes for span in tracer.spans], ensure_ascii=False) + observed += " ".join(str(exc) for span in tracer.spans for exc in span.exceptions) + assert failed.error_type == "RuntimeError" + assert PNG_B64 not in observed + assert PNG_URL not in observed + + +async def test_strict_after_tool_hook_failure_registers_and_redacts_original_and_replacement_images(tmp_path: Path) -> None: + replacement_data = PNG + b"replacement" + replacement_b64 = base64.b64encode(replacement_data).decode("ascii") + replacement_url = f"data:image/png;base64,{replacement_b64}" + tracer = FakeTracer() + + def fail(ctx) -> None: + ctx.output = ToolResult(True, (ImageBlock(replacement_data, "image/png"),), {}).to_json() + raise RuntimeError(f"hook echoed {PNG_B64} {PNG_URL} {replacement_b64} {replacement_url}") + + provider = _Provider("OpenAI", [{ + "id": "tool", + "output": [{"type": "function_call", "call_id": "image", "name": "image", "arguments": "{}"}], + }]) + harness = Harness( + HarnessConfig(root=tmp_path, system_prompt="sys", local_tracing=False, strict_hooks=True), + model=OpenAIResponsesModel("target", provider=provider), # type: ignore[arg-type] + tools=[ToolSpec("image", "Image.", {"type": "object", "properties": {}}, lambda _args: (ImageBlock(PNG, "image/png"),))], + hooks=[Hook("after_tool_call", fail)], + tracing=[TracingOptions(tracer=tracer, capture_tool_results=True)], + ) + events = [] + + with pytest.raises(HarnessError): + async for event in harness.stream("go"): + events.append(event) + + completed = next(item for item in events if isinstance(item, ToolCallCompletedEvent)) + failed = next(item for item in events if isinstance(item, RunFailedEvent)) + observed = json.dumps([completed.output, completed.message, failed.message, [span.attributes for span in tracer.spans]], ensure_ascii=False) + observed += " ".join(str(exc) for span in tracer.spans for exc in span.exceptions) + for secret in (PNG_B64, PNG_URL, replacement_b64, replacement_url): + assert secret not in observed + + +async def test_redacted_tool_projection_removes_repeated_image_data_from_text_and_metadata(tmp_path: Path) -> None: + tracer = FakeTracer() + repeated = f"repeated {PNG_B64} and {PNG_URL}" + provider = _Provider("OpenAI", [ + {"id": "tool", "output": [{"type": "function_call", "call_id": "image", "name": "image", "arguments": "{}"}]}, + {"id": "done", "output_text": "done"}, + ]) + harness = Harness( + _config(tmp_path), + model=OpenAIResponsesModel("target", provider=provider), # type: ignore[arg-type] + tools=[ToolSpec( + "image", + "Image.", + {"type": "object", "properties": {}}, + lambda _args: ToolResult(True, (TextBlock(repeated), ImageBlock(PNG, "image/png")), {"echo": repeated}), + )], + tracing=[TracingOptions(tracer=tracer, capture_tool_results=True)], + ) + events = [event async for event in harness.stream("go")] + + completed = next(event for event in events if isinstance(event, ToolCallCompletedEvent)) + tool_span = next(span for span in tracer.spans if span.name == "execute_tool image") + observed = completed.output + str(tool_span.attributes.get("gen_ai.tool.call.result", "")) + assert PNG_B64 not in observed + assert PNG_URL not in observed + assert observed.count("[image data redacted]") >= 2 + + class _FailingProvider(_Provider): def __init__(self, name: str = "OpenAI", *, leak: bool = True) -> None: super().__init__(name, []) @@ -498,7 +712,7 @@ def __init__(self, name: str = "OpenAI", *, leak: bool = True) -> None: async def create_response(self, payload: dict[str, Any]) -> dict[str, Any]: self.payloads.append(copy.deepcopy(payload)) message = f"provider echoed {PNG_B64} and {PNG_URL}" if self.leak else "plain provider failure" - raise ProviderError(message) + raise ProviderError(message, status_code=422) @pytest.mark.parametrize( @@ -530,6 +744,52 @@ async def test_resumed_provider_failures_redact_user_and_tool_images(tmp_path: P assert PNG_URL not in recorded + attributes +@pytest.mark.parametrize("leak", [False, True]) +async def test_provider_failure_redaction_keeps_public_classification_and_status(tmp_path: Path, leak: bool) -> None: + tracer = FakeTracer() + provider = _FailingProvider(leak=leak) + harness = Harness( + _config(tmp_path), + model=OpenAIResponsesModel("target", provider=provider), + tracing=[TracingOptions(tracer=tracer, capture_messages=True)], + ) + events = [] + + with pytest.raises(HarnessError) as exc_info: + async for event in harness.stream((TextBlock("go"), ImageBlock(PNG, "image/png"))): + events.append(event) + + failed = next(event for event in events if isinstance(event, RunFailedEvent)) + model_span = next(span for span in tracer.spans if span.name.startswith("chat ")) + assert failed.error_type == "ProviderError" + assert model_span.attributes["error.type"] == "ProviderError" + assert exc_info.value.__dict__["status_code"] == 422 + if leak: + assert failed.message == "provider echoed [image data redacted] and [image data redacted]" + else: + assert failed.message == "plain provider failure" + + +async def test_non_vision_provider_error_keeps_provider_classification(tmp_path: Path) -> None: + class NonVisionProvider(_Provider): + async def create_response(self, payload: dict[str, Any]) -> dict[str, Any]: + self.payloads.append(copy.deepcopy(payload)) + raise ProviderError("selected model does not support image inputs", status_code=400) + + provider = NonVisionProvider("OpenAI", []) + harness = Harness(_config(tmp_path), model=OpenAIResponsesModel("text-only", provider=provider)) # type: ignore[arg-type] + events = [] + + with pytest.raises(HarnessError) as exc_info: + async for event in harness.stream((TextBlock("look"), ImageBlock(PNG, "image/png"))): + events.append(event) + + failed = next(event for event in events if isinstance(event, RunFailedEvent)) + assert failed.error_type == "ProviderError" + assert failed.message == "selected model does not support image inputs" + assert exc_info.value.__dict__["status_code"] == 400 + + async def test_text_only_provider_failure_preserves_recorded_exception_type(tmp_path: Path) -> None: tracer = FakeTracer() provider = _FailingProvider(leak=False) @@ -800,6 +1060,64 @@ async def test_structured_output_retry_keeps_plain_wire_text(tmp_path: Path, pro assert not wire.startswith("{") +@pytest.mark.parametrize("provider_name", ["openai", "anthropic", "openrouter"]) +async def test_structured_output_retry_wire_text_persists_and_replays_exactly(tmp_path: Path, provider_name: str) -> None: + live_provider = _Provider(provider_name, _structured_responses(provider_name)) + model = _model_for(provider_name, live_provider) + session = model.new_session() + model.new_session = lambda: session # type: ignore[method-assign] + config = HarnessConfig( + root=tmp_path, + system_prompt="sys", + local_tracing=False, + output_type=_Answer, + output_mode="tool", + ) + result = await Harness(config, model=model).run("answer") + assert result.output == _Answer(value="ok") + state = session.dump_state() + assert state is not None + retry_entry = next(entry for entry in state["entries"] if entry["role"] == "tool" and entry["call_id"] == "final_bad") + live_wire = retry_entry["wire_output"] + assert isinstance(live_wire, str) + assert live_wire.startswith("The previous response failed structured output validation.") + + replay_provider = _Provider(provider_name, [_final_response(provider_name)]) + await Harness(_config(tmp_path), model=_model_for(provider_name, replay_provider)).run("follow-up", resume_from=state) + payload = replay_provider.payloads[0] + if provider_name == "openai": + replay_wire = next(item["output"] for item in payload["input"] if item.get("type") == "function_call_output" and item.get("call_id") == "final_bad") + elif provider_name == "anthropic": + replay_wire = next( + block["content"] + for message in payload["messages"] + if isinstance(message.get("content"), list) + for block in message["content"] + if block.get("type") == "tool_result" and block.get("tool_use_id") == "final_bad" + ) + else: + replay_wire = next( + message["content"] + for message in payload["messages"] + if message.get("role") == "tool" and message.get("tool_call_id") == "final_bad" + ) + assert replay_wire == live_wire + + +@pytest.mark.parametrize("mutation, message", [("wrong_type", "wrong type"), ("missing", "wrong keys")]) +def test_resume_tool_wire_output_decodes_strictly(tmp_path: Path, mutation: str, message: str) -> None: + state = _resume_state("openai") + tool_entry = next(entry for entry in state["entries"] if entry["role"] == "tool") + if mutation == "wrong_type": + tool_entry["wire_output"] = 3 + else: + del tool_entry["wire_output"] + harness = Harness(_config(tmp_path), model=OpenAIResponsesModel("target", provider=_Provider("OpenAI", []))) # type: ignore[arg-type] + + with pytest.raises(HarnessError, match=message): + harness.run_sync("follow-up", resume_from=state) + + async def test_initial_image_provider_failure_is_redacted_everywhere(tmp_path: Path) -> None: tracer = FakeTracer() provider = _FailingProvider() diff --git a/tests/unit/test_mcp.py b/tests/unit/test_mcp.py index 9aae281..dc0a2e8 100644 --- a/tests/unit/test_mcp.py +++ b/tests/unit/test_mcp.py @@ -1724,6 +1724,29 @@ async def test_mcp_malformed_or_unsupported_images_become_placeholders(monkeypat assert result.content == f"[image: {media_type}]" +@pytest.mark.parametrize( + "texts, expected", + [ + (["", "a"], "\na"), + (["a", "", "b"], "a\n\nb"), + (["a", ""], "a\n"), + ], +) +async def test_successful_mcp_text_preserves_empty_block_positions(monkeypatch, texts: list[str], expected: str) -> None: + from mcp import types + + scripted = types.CallToolResult( + content=[types.TextContent(type="text", text=text) for text in texts], + isError=False, + ) + server = scripted_server(monkeypatch, {"text": _schema()}, {"text": scripted}) + + result = await server.call_tool("text", {}) + + assert result.ok is True + assert result.content == expected + + async def test_empty_successful_mcp_content_remains_successful(monkeypatch) -> None: from mcp import types diff --git a/tests/unit/test_streaming.py b/tests/unit/test_streaming.py index d91d1b6..69658b8 100644 --- a/tests/unit/test_streaming.py +++ b/tests/unit/test_streaming.py @@ -242,7 +242,7 @@ async def test_stream_failure_yields_failed_event_then_raises(tmp_path: Path) -> failed = [event for event in events if isinstance(event, RunFailedEvent)] assert len(failed) == 1 assert failed[0].stop_reason == "provider_error" - assert failed[0].error_type == "HarnessError" + assert failed[0].error_type == "ProviderError" async def test_stream_subagent_events_include_parent_ids(tmp_path: Path) -> None: diff --git a/tests/unit/test_tool_retry.py b/tests/unit/test_tool_retry.py index a5c38b9..d4391eb 100644 --- a/tests/unit/test_tool_retry.py +++ b/tests/unit/test_tool_retry.py @@ -376,7 +376,7 @@ def after(ctx): hooks=[Hook("after_tool_call", after)], ) - with pytest.raises(ValueError, match="valid canonical ToolResult"): + with pytest.raises(HarnessError, match="exceeded max_retries=0"): harness.run_sync("go") assert seen == ["ModelRetry"] @@ -402,7 +402,49 @@ class AgeArgs(BaseModel): assert seen[0]["retry"] is True -def test_tracing_uses_mutated_retry_kind(tmp_path: Path) -> None: +def test_after_tool_hook_cannot_create_retry_control_flow(tmp_path: Path) -> None: + def add_retry(ctx: AfterToolCallContext) -> None: + ctx.envelope.metadata.update({"error_type": "HookRetry", "retry": True}) + ctx.output = ctx.envelope.to_json() + + client = MultiCallClient([("ok", "{}")]) + harness = Harness( + HarnessConfig(root=tmp_path, model="openai:test-model", tool_retries=0), + model=_fake_openai(client), + tools=[ToolSpec("ok", "Ok", {"type": "object", "properties": {}}, lambda _args: "done")], + hooks=[Hook("after_tool_call", add_retry)], + ) + + result = harness.run_sync("go") + + assert len(client.payloads) == 2 + assert result.usage.tool_retries == {} + assert tool_output(client.payloads[1]["input"][0]["output"])["metadata"] == { + "error_type": "HookRetry", + "retry": True, + } + + +def test_after_tool_hook_cannot_suppress_retry_budget(tmp_path: Path) -> None: + def remove_retry(ctx: AfterToolCallContext) -> None: + ctx.envelope.metadata = {} + ctx.output = ctx.envelope.to_json() + + session = SequenceSession(ModelTurn(tool_calls=[_call("flaky", "{}")], raw={"id": "start"})) + harness = Harness( + HarnessConfig(root=tmp_path, tool_retries=0), + model=ScriptedModel([session]), + tools=[ToolSpec("flaky", "Flaky", {"type": "object", "properties": {}}, lambda _args: (_ for _ in ()).throw(ModelRetry("again")))], + hooks=[Hook("after_tool_call", remove_retry)], + ) + + with pytest.raises(HarnessError, match="exceeded max_retries=0"): + harness.run_sync("go") + + assert session.tool_outputs == [] + + +def test_tracing_and_control_flow_use_pre_hook_retry_kind(tmp_path: Path) -> None: tracer = FakeTracer() def rewrite(ctx): @@ -421,7 +463,8 @@ def rewrite(ctx): harness.run_sync("go") span = next(span for span in tracer.spans if span.name == "execute_tool flaky") - assert span.attributes["error.type"] == "Rewritten" + assert span.attributes["error.type"] == "ModelRetry" + assert tool_output(client.payloads[1]["input"][0]["output"])["metadata"]["error_type"] == "Rewritten" def test_subagent_tool_retry_budget_recipes(tmp_path: Path) -> None: diff --git a/thinharness/core.py b/thinharness/core.py index 54fabd5..6183615 100644 --- a/thinharness/core.py +++ b/thinharness/core.py @@ -20,7 +20,7 @@ validate_approval_pause_state, ) from .children import ChildHarnessHost, _ParentChildHarnessHost, _ToolComposition -from .content import NormalizedContent, Prompt, normalize_content, redact_image_data, redacted_content_string, text_only_value +from .content import ImageBlock, NormalizedContent, Prompt, normalize_content, redact_image_data, redacted_content_string, text_only_value from .defaults import DEFAULT_SYSTEM_PROMPT from .events import ( ApprovalResumedEvent, @@ -78,32 +78,70 @@ def _local_tracing_enabled(configured: bool) -> bool: return configured and not disabled +def _error_type(exc: BaseException) -> str: + """Return the stable public error classification for an exception.""" + value = getattr(exc, "_thinharness_error_type", None) + return value if isinstance(value, str) else type(exc).__name__ + + +def _sanitized_failure(exc: Exception, message: str) -> Exception: + """Return an exception with a safe message and stable classification.""" + if message == str(exc): + return exc + if isinstance(exc, ProviderError): + sanitized_provider = ProviderError(message, status_code=exc.status_code) + sanitized_provider.__dict__["_thinharness_sanitized"] = True + return sanitized_provider + sanitized = HarnessError(message) + sanitized.__dict__["_thinharness_error_type"] = _error_type(exc) + sanitized.__dict__["_thinharness_sanitized"] = True + if getattr(exc, "_thinharness_strict_hook", False): + sanitized.__dict__["_thinharness_strict_hook"] = True + return sanitized + + def _classify_run_failure(run_ctx: Any, agent_span: Any, exc: Exception) -> Exception: """Record a run failure and return the exception to raise.""" message = redact_image_data(str(exc), run_ctx.image_blocks) - agent_span.record_exception(exc if message == str(exc) else HarnessError(message)) - agent_span.set_error(message, type(exc).__name__) - if isinstance(exc, ProviderError) or getattr(exc, "_thinharness_provider_error", False): + safe_exc = _sanitized_failure(exc, message) + error_type = _error_type(safe_exc) + agent_span.record_exception(safe_exc) + agent_span.set_error(message, error_type) + if isinstance(safe_exc, ProviderError) or getattr(safe_exc, "_thinharness_provider_error", False): run_ctx.stop_reason = "provider_error" - run_ctx.terminal_error = exc if isinstance(exc, HarnessError) else HarnessError(message) - return run_ctx.terminal_error - if isinstance(exc, UnexpectedModelBehavior): + terminal = HarnessError(message) + terminal.__dict__["_thinharness_error_type"] = error_type + status_code = getattr(safe_exc, "status_code", None) + if status_code is not None: + terminal.__dict__["status_code"] = status_code + if getattr(safe_exc, "_thinharness_sanitized", False): + terminal.__dict__["_thinharness_sanitized"] = True + run_ctx.terminal_error = terminal + return terminal + if isinstance(safe_exc, UnexpectedModelBehavior): run_ctx.stop_reason = "unexpected_model_behavior" - run_ctx.terminal_error = run_ctx.terminal_error or exc - return exc - if isinstance(exc, HarnessError): - run_ctx.terminal_error = run_ctx.terminal_error or exc + run_ctx.terminal_error = run_ctx.terminal_error or safe_exc + return safe_exc + if isinstance(safe_exc, HarnessError): + run_ctx.terminal_error = run_ctx.terminal_error or safe_exc if run_ctx.stop_reason == "end_turn": run_ctx.stop_reason = "error" - return exc + return safe_exc run_ctx.stop_reason = "error" - run_ctx.terminal_error = exc - return exc + run_ctx.terminal_error = safe_exc + return safe_exc + + +def _normalize_public_prompt(prompt: Prompt, *, label: str) -> NormalizedContent: + """Normalize a public or hook prompt as a harness validation error.""" + try: + return normalize_content(prompt, label=label) + except (TypeError, ValueError) as exc: + raise HarnessError(str(exc)) from exc -def _event_prompt(prompt: Prompt) -> str: +def _event_prompt(content: NormalizedContent) -> str: """Keep text-only stream values and redact multimodal values.""" - content = normalize_content(prompt, label="prompt") text = text_only_value(content) return text if text is not None else redacted_content_string(content) @@ -439,15 +477,17 @@ async def _run_streaming( else: run_metadata = dict(metadata or {}) usage = RunUsage() + raw_prompt: NormalizedContent = () if approval_pause is not None else _normalize_public_prompt(prompt, label="prompt") run_ctx = RunContext( harness=self, - prompt=prompt, + prompt=raw_prompt, metadata=run_metadata, usage=usage, tracer=run_tracer, stream=stream_context, emitter=emitter, ) + run_ctx.register_image_blocks(tuple(block for block in raw_prompt if isinstance(block, ImageBlock))) if approval_pause is not None: run_ctx.responses = restored_responses run_ctx.tool_call_records = restored_records @@ -455,7 +495,7 @@ async def _run_streaming( run_ctx.emit( RunStartedEvent( **run_ctx.stream_base(), - prompt=None if approval_pause is not None else _event_prompt(prompt), + prompt=None if approval_pause is not None else _event_prompt(raw_prompt), root=str(self.root), max_model_requests=self.config.max_model_requests, max_tool_calls=self.config.max_tool_calls, @@ -511,7 +551,7 @@ async def _run_streaming( run_ctx.agent_span = agent_span try: effective_prompt, instructions = await self._prepare_run_start( - prompt, + raw_prompt, run_metadata, run_ctx, agent_span, @@ -552,6 +592,8 @@ async def _run_streaming( failure = _classify_run_failure(run_ctx, agent_span, exc) if failure is exc: raise + if getattr(failure, "_thinharness_sanitized", False): + raise failure from None raise failure from exc finally: run_ctx.fire_run_end_once() @@ -564,7 +606,7 @@ async def _run_streaming( RunFailedEvent( **run_ctx.stream_base(), stop_reason=run_ctx.stop_reason, - error_type=type(exc).__name__, + error_type=_error_type(exc), message=redact_image_data(str(exc), run_ctx.image_blocks), ) ) @@ -575,7 +617,7 @@ async def _run_streaming( async def _prepare_run_start( self, - prompt: Prompt, + prompt: NormalizedContent, run_metadata: Json, run_ctx: Any, agent_span: Any, @@ -583,7 +625,7 @@ async def _prepare_run_start( skip_user_prompt: bool = False, ) -> tuple[NormalizedContent, str]: """Fire start hooks and return effective normalized content plus instructions.""" - initial: NormalizedContent = () if skip_user_prompt else normalize_content(prompt, label="prompt") + initial: NormalizedContent = () if skip_user_prompt else prompt start_ctx = RunStartContext( harness=self, metadata=dict(run_metadata), @@ -593,7 +635,7 @@ async def _prepare_run_start( max_tool_calls=self.config.max_tool_calls, ) self.hooks.fire(start_ctx) - effective_prompt = initial if skip_user_prompt else normalize_content(start_ctx.prompt, label="run_start prompt") + effective_prompt = initial if skip_user_prompt else _normalize_public_prompt(start_ctx.prompt, label="run_start prompt") if not skip_user_prompt: prompt_ctx = UserPromptSubmitContext(harness=self, metadata=dict(run_metadata), prompt=effective_prompt) self.hooks.fire(prompt_ctx) @@ -602,14 +644,14 @@ async def _prepare_run_start( run_ctx.stop_reason = "cancelled_by_hook" run_ctx.terminal_error = HarnessError(f"run blocked by hook: {reason}") raise run_ctx.terminal_error - submitted = normalize_content(prompt_ctx.prompt, label="user_prompt_submit prompt") + submitted = _normalize_public_prompt(prompt_ctx.prompt, label="user_prompt_submit prompt") effective_prompt = apply_prompt_context(submitted, prompt_ctx.additional_context) run_ctx.set_prompt_content(effective_prompt) instructions = structured_instructions(self.system_instructions(), self.output_schema) agent_span.for_each( lambda span, option: annotate_agent_start( span, - prompt=None if skip_user_prompt else effective_prompt, + prompt=None if skip_user_prompt else initial, instructions=instructions, capture_messages=option.capture_messages, top_level=not self._is_child_harness, diff --git a/thinharness/hooks.py b/thinharness/hooks.py index 5ae72c0..a3e186f 100644 --- a/thinharness/hooks.py +++ b/thinharness/hooks.py @@ -3,6 +3,7 @@ from __future__ import annotations import contextvars +import copy import logging from collections.abc import Callable from dataclasses import dataclass, field @@ -11,7 +12,7 @@ from .content import ContentBlock, Prompt, TextBlock, normalize_content from .tools.base import Json, ToolEnvelope, ToolResult, ToolSpec -from .types import HarnessResult, RunUsage, StopReason +from .types import HarnessError, HarnessResult, RunUsage, StopReason _CURRENT_TOOL_CALL: contextvars.ContextVar[Json | None] = contextvars.ContextVar("thinharness_current_tool_call", default=None) _CURRENT_TOOL_RUNTIME: contextvars.ContextVar[dict[str, Any] | None] = contextvars.ContextVar("thinharness_current_tool_runtime", default=None) @@ -231,20 +232,28 @@ def fire_after_tool_call(self, ctx: AfterToolCallContext) -> None: if not self._matches(hook, ctx): continue before_output = ctx.output - before_envelope = ctx.envelope.to_json() + before_envelope = copy.deepcopy(ctx.envelope) + handler_completed = False try: hook.handler(ctx) + handler_completed = True + if ctx.output != before_output: + ctx.envelope = ToolResult.from_json(ctx.output, strict=True) + elif ctx.envelope.to_json() != before_envelope.to_json(): + ctx.output = ctx.envelope.to_json() except Exception as exc: name = _handler_name(hook.handler) logger.warning("hook handler failed for event %s: %s", ctx.event, name) logger.debug("hook handler traceback for event %s: %s", ctx.event, name, exc_info=True) if self.strict_hooks: + if handler_completed: + failure = HarnessError(f"after_tool_call hook canonical output validation failed: {exc}") + _mark_strict_hook_exception(failure) + raise failure from None _mark_strict_hook_exception(exc) raise - if ctx.output != before_output: - ctx.envelope = ToolResult.from_json(ctx.output, strict=True) - elif ctx.envelope.to_json() != before_envelope: - ctx.output = ctx.envelope.to_json() + ctx.output = before_output + ctx.envelope = before_envelope def validate_filters(self, *, agent_names: set[str]) -> None: """Raise for agent filters that do not match registered names.""" diff --git a/thinharness/providers.py b/thinharness/providers.py index f3f47ef..0c58d46 100644 --- a/thinharness/providers.py +++ b/thinharness/providers.py @@ -316,7 +316,7 @@ def _retry_delay(base: float, retry_index: int, retry_after: str | None = None) _TRANSCRIPT_ENTRY_KEYS = { "assistant": frozenset({"role", "text", "tool_calls", "reasoning"}), "user": frozenset({"role", "content", "notice"}), - "tool": frozenset({"role", "call_id", "ok", "content", "metadata"}), + "tool": frozenset({"role", "call_id", "ok", "content", "metadata", "wire_output"}), } _REASONING_PART_KEYS = frozenset({"text", "signature", "id", "provider_name", "provider_details"}) _TRANSCRIPT_VERSION = 4 @@ -360,7 +360,10 @@ def _transcript_entry_to_dict(entry: TranscriptEntry) -> Json: if isinstance(entry, UserEntry): return {"role": "user", "content": content_to_json(entry.content), "notice": entry.notice} if isinstance(entry, ToolResultEntry): - return {"role": "tool", "call_id": entry.call_id, **entry.result.to_value()} + wire_output = entry.wire_output + if wire_output == entry.result.to_json(): + wire_output = None + return {"role": "tool", "call_id": entry.call_id, **entry.result.to_value(), "wire_output": wire_output} return { "role": "assistant", "text": entry.text, @@ -402,13 +405,13 @@ def _transcript_entry_from_dict(value: Any) -> TranscriptEntry: raise HarnessError(str(exc)) from exc return UserEntry(content=content, notice=value["notice"]) if role == "tool": - if not isinstance(value["call_id"], str): + if not isinstance(value["call_id"], str) or (value["wire_output"] is not None and not isinstance(value["wire_output"], str)): raise HarnessError("resume_from tool entry has wrong type") try: result = ToolResult.from_value({key: value[key] for key in ("ok", "content", "metadata")}, label="resume_from tool entry") except (TypeError, ValueError) as exc: raise HarnessError(str(exc)) from exc - return ToolResultEntry(call_id=value["call_id"], result=result) + return ToolResultEntry(call_id=value["call_id"], result=result, wire_output=value["wire_output"]) if not isinstance(value["text"], str) or not isinstance(value["tool_calls"], list) or not isinstance(value["reasoning"], list): raise HarnessError("resume_from assistant entry has wrong type") return AssistantEntry( @@ -1348,12 +1351,12 @@ def _data_url(block: ImageBlock) -> str: return f"data:{block.media_type};base64,{base64.b64encode(block.data).decode('ascii')}" -def _openai_content_parts(content: NormalizedContent, *, text_type: str = "input_text", image_type: str = "input_image") -> list[Json]: +def _openai_content_parts(content: NormalizedContent) -> list[Json]: """Map neutral content to OpenAI Responses content parts.""" return [ - {"type": text_type, "text": block.text} + {"type": "input_text", "text": block.text} if isinstance(block, TextBlock) - else {"type": image_type, "image_url": _data_url(block)} + else {"type": "input_image", "image_url": _data_url(block)} for block in content ] @@ -1506,7 +1509,11 @@ def _render_anthropic_transcript(entries: list[TranscriptEntry], *, thinking_ena while index < len(entries) and isinstance(entries[index], ToolResultEntry): tool_entry = entries[index] assert isinstance(tool_entry, ToolResultEntry) - content.append({"type": "tool_result", "tool_use_id": tool_entry.call_id, "content": _anthropic_tool_output(tool_entry.result)}) + content.append({ + "type": "tool_result", + "tool_use_id": tool_entry.call_id, + "content": tool_entry.wire_output if tool_entry.wire_output is not None else _anthropic_tool_output(tool_entry.result), + }) index += 1 if index < len(entries): notice_entry = entries[index] @@ -1532,9 +1539,13 @@ def _render_openrouter_transcript(entries: list[TranscriptEntry]) -> list[Json]: while index < len(entries) and isinstance(entries[index], ToolResultEntry): tool_entry = entries[index] assert isinstance(tool_entry, ToolResultEntry) - output = ToolOutput(tool_entry.call_id, tool_entry.result) + output = ToolOutput(tool_entry.call_id, tool_entry.result, wire_output=tool_entry.wire_output) outputs.append(output) - messages.append({"role": "tool", "tool_call_id": output.call_id, "content": _openrouter_tool_output_json(output.result)}) + messages.append({ + "role": "tool", + "tool_call_id": output.call_id, + "content": output.wire_output if output.wire_output is not None else _openrouter_tool_output_json(output.result), + }) index += 1 notice: UserEntry | None = None if index < len(entries): @@ -1592,7 +1603,11 @@ def _render_openai_transcript(entries: list[TranscriptEntry], *, encrypted_reaso if isinstance(entry, UserEntry): items.append(_openai_user_item(entry.content)) elif isinstance(entry, ToolResultEntry): - items.append({"type": "function_call_output", "call_id": entry.call_id, "output": _openai_tool_output(entry.result)}) + items.append({ + "type": "function_call_output", + "call_id": entry.call_id, + "output": entry.wire_output if entry.wire_output is not None else _openai_tool_output(entry.result), + }) else: for part in entry.reasoning: if encrypted_reasoning_ok and part.provider_name == "openai" and part.signature and part.id: diff --git a/thinharness/runtime.py b/thinharness/runtime.py index aad98b7..dc90b1d 100644 --- a/thinharness/runtime.py +++ b/thinharness/runtime.py @@ -355,12 +355,15 @@ async def advance_model( model_span.record_exception(exc) model_span.set_error(message, type(exc).__name__) raise - sanitized = HarnessError(message) if isinstance(exc, ProviderError): - sanitized.__dict__["_thinharness_provider_error"] = True + sanitized: Exception = ProviderError(message, status_code=exc.status_code) + else: + sanitized = HarnessError(message) + sanitized.__dict__["_thinharness_error_type"] = type(exc).__name__ + sanitized.__dict__["_thinharness_sanitized"] = True model_span.record_exception(sanitized) - model_span.set_error(message, type(sanitized).__name__) - raise sanitized from exc + model_span.set_error(message, type(exc).__name__) + raise sanitized from None model_span.for_each( lambda span, option: annotate_model_span( span, diff --git a/thinharness/tool_execution.py b/thinharness/tool_execution.py index 37628df..bbfdb63 100644 --- a/thinharness/tool_execution.py +++ b/thinharness/tool_execution.py @@ -3,11 +3,13 @@ from __future__ import annotations import asyncio +import copy import time from dataclasses import dataclass from typing import TYPE_CHECKING from .children import _ToolComposition +from .content import redact_image_data from .events import ( _CURRENT_STREAM_EMITTER, ToolCallCompletedEvent, @@ -23,6 +25,7 @@ from .providers import ModelToolCall, ToolOutput from .tools.base import Json, ToolEnvelope, ToolResult, ToolSpec, _invoke_tool from .tracing import RunTracer, serialize_attribute_value +from .types import HarnessError if TYPE_CHECKING: from .core import Harness @@ -177,6 +180,7 @@ async def execute_one(self, call: ModelToolCall, index: int) -> ToolCallExecutio envelope: ToolEnvelope | None = None output: str | None = None retry_kind: str | None = None + after: AfterToolCallContext | None = None try: spec = self.tool_map.get(str(call.name)) before = BeforeToolCallContext( @@ -206,6 +210,7 @@ async def execute_one(self, call: ModelToolCall, index: int) -> ToolCallExecutio envelope = await self._call_output(call.name, call.arguments) output = envelope.to_json() retry_kind = None if cancelled else envelope.retry_kind() + self.run_context.record_tool_result_images(envelope) after = AfterToolCallContext( harness=self.harness, metadata=dict(self.run_context.metadata), @@ -214,13 +219,12 @@ async def execute_one(self, call: ModelToolCall, index: int) -> ToolCallExecutio arguments=call.arguments, original_output=output, output=output, - envelope=envelope, + envelope=copy.deepcopy(envelope), duration_ms=(time.perf_counter() - start) * 1000, ) self.harness.hooks.fire_after_tool_call(after) output = after.output envelope = after.envelope - retry_kind = None if cancelled else envelope.retry_kind() self.run_context.record_tool_result_images(envelope) projected_output = envelope.redacted_json() self._annotate_special_tool(span, call.name, envelope, composition) @@ -244,6 +248,19 @@ async def execute_one(self, call: ModelToolCall, index: int) -> ToolCallExecutio completed_emitted = True return ToolCallExecution(envelope=envelope, output=output, cancelled=cancelled, retry_kind=retry_kind) except Exception as exc: + if after is not None: + self._register_valid_hook_images(after) + message = redact_image_data(str(exc), self.run_context.image_blocks) + failure: Exception = exc + if message != str(exc): + failure = HarnessError(message) + failure.__dict__["_thinharness_error_type"] = type(exc).__name__ + failure.__dict__["_thinharness_sanitized"] = True + if getattr(exc, "_thinharness_strict_hook", False): + failure.__dict__["_thinharness_strict_hook"] = True + error_type = getattr(failure, "_thinharness_error_type", type(failure).__name__) + span.record_exception(failure) + span.set_error(message, error_type) if not completed_emitted: self.run_context.emit( ToolCallCompletedEvent( @@ -253,19 +270,38 @@ async def execute_one(self, call: ModelToolCall, index: int) -> ToolCallExecutio ok=False, cancelled=cancelled, retry_kind=retry_kind, - error_type=type(exc).__name__, - message=str(exc), + error_type=error_type, + message=message, duration_ms=(time.perf_counter() - start) * 1000, output=(envelope.redacted_json() if envelope is not None else output), ) ) - raise + if failure is exc: + raise + raise failure from None finally: lease.active = False _CURRENT_STREAM_EMITTER.reset(emitter_token) _CURRENT_TOOL_RUNTIME.reset(runtime_token) _CURRENT_TOOL_CALL.reset(call_token) + def _register_valid_hook_images(self, ctx: AfterToolCallContext) -> None: + """Register valid hook replacement images before failure observability.""" + if isinstance(ctx.envelope, ToolResult): + try: + ctx.envelope.to_json() + except (TypeError, ValueError): + pass + else: + self.run_context.record_tool_result_images(ctx.envelope) + if not isinstance(ctx.output, str): + return + try: + parsed = ToolResult.from_json(ctx.output, strict=True) + except (TypeError, ValueError): + return + self.run_context.record_tool_result_images(parsed) + def _emit_completed( self, *, diff --git a/thinharness/tools/base.py b/thinharness/tools/base.py index 86a1d3f..46f584f 100644 --- a/thinharness/tools/base.py +++ b/thinharness/tools/base.py @@ -15,7 +15,16 @@ from pydantic import BaseModel, ConfigDict, ValidationError -from ..content import ContentBlock, ImageBlock, TextBlock, content_from_json, content_to_json, normalize_content, redacted_content_json +from ..content import ( + ContentBlock, + ImageBlock, + TextBlock, + content_from_json, + content_to_json, + normalize_content, + redact_image_data, + redacted_content_json, +) from ..types import Json ToolHandler = Callable[[Any], Any | Awaitable[Any]] @@ -147,11 +156,12 @@ def redacted_json(self) -> str: """Return the canonical envelope without image bytes.""" if not self.has_image: return self.to_json() - return json.dumps( + projection = json.dumps( {"ok": self.ok, "content": redacted_content_json(self.blocks), "metadata": self.metadata}, ensure_ascii=False, separators=(",", ":"), ) + return redact_image_data(projection, self.blocks) def retry_kind(self) -> str | None: """Return the retry error type if this envelope asks the model to retry.""" diff --git a/thinharness/tools/mcp.py b/thinharness/tools/mcp.py index cc3b7f8..096676e 100644 --- a/thinharness/tools/mcp.py +++ b/thinharness/tools/mcp.py @@ -397,9 +397,7 @@ def _content_to_blocks( block_type = getattr(block, "type", "") if block_type == "text": if include_text: - text = str(getattr(block, "text", "")) - if text: - parts.append(TextBlock(text)) + parts.append(TextBlock(str(getattr(block, "text", "")))) continue if block_type == "image": media_type = str(getattr(block, "mimeType", "unknown")) From 17eb82ddf0e3554ca6333f3d461b2df04574b957 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Thu, 20 Aug 2026 21:13:24 -0400 Subject: [PATCH 20/30] Prepare 0.7.0 release --- .github/workflows/pages.yml | 56 - CHANGELOG.md | 2 +- README.md | 384 +---- docs/docs.md | 2 - docs/site/about/index.html | 231 --- docs/site/assets/ThinHarness.svg | 6 - docs/site/assets/agno-a.svg | 4 - docs/site/assets/apple-touch-icon.png | Bin 2720 -> 0 bytes docs/site/assets/favicon.svg | 11 - docs/site/assets/github-mark.svg | 3 - docs/site/assets/site.css | 1838 --------------------- docs/site/assets/thinharness-mark.svg | 11 - docs/site/assets/thinharness-run-loop.svg | 3 - docs/site/examples/index.html | 637 ------- docs/site/explainer/index.html | 960 ----------- docs/site/index.html | 88 - pyproject.toml | 5 +- scripts/build_site.py | 420 ----- scripts/build_transcripts.py | 5 +- uv.lock | 15 +- 20 files changed, 49 insertions(+), 4632 deletions(-) delete mode 100644 .github/workflows/pages.yml delete mode 100644 docs/site/about/index.html delete mode 100644 docs/site/assets/ThinHarness.svg delete mode 100644 docs/site/assets/agno-a.svg delete mode 100644 docs/site/assets/apple-touch-icon.png delete mode 100644 docs/site/assets/favicon.svg delete mode 100644 docs/site/assets/github-mark.svg delete mode 100644 docs/site/assets/site.css delete mode 100644 docs/site/assets/thinharness-mark.svg delete mode 100644 docs/site/assets/thinharness-run-loop.svg delete mode 100644 docs/site/examples/index.html delete mode 100644 docs/site/explainer/index.html delete mode 100644 docs/site/index.html delete mode 100644 scripts/build_site.py diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml deleted file mode 100644 index c25900e..0000000 --- a/.github/workflows/pages.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Pages - -on: - push: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: pages - cancel-in-progress: false - -jobs: - build: - name: Build site - runs-on: ubuntu-latest - - steps: - - name: Check out repository - uses: actions/checkout@v6 - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: "3.12" - - - name: Set up uv - uses: astral-sh/setup-uv@v8.1.0 - with: - enable-cache: true - cache-dependency-glob: uv.lock - - - name: Check generated site - run: uv run scripts/build_site.py --check - - - name: Upload Pages artifact - uses: actions/upload-pages-artifact@v4 - with: - path: docs/site - - deploy: - name: Deploy site - needs: build - runs-on: ubuntu-latest - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ce9c49..7909c3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 0.7.0 - 2026-08-20 - Added ordered text and image prompts and tool results for OpenAI, Anthropic, and OpenRouter, with provider-neutral immutable content blocks, redacted observability projections, self-contained version 4 resume state, opt-in filesystem `read_image`, and preserved successful MCP images. - **Breaking:** Renamed custom `ModelSession.continue_with_user_text(...)` to `continue_with_user_content(...)`; prompt hooks now receive normalized content-block tuples, and built-in transcript resume version 3 state must be regenerated. diff --git a/README.md b/README.md index c235e12..ad0d6a6 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@


    - A minimal, opinionated agent harness — + A compact, SDK-only agent harness:
    - focused scope, straightforward code, easy to fork. + maximum performance, minimum framework code

    @@ -18,216 +18,48 @@ -## Why this exists +## Why keep it small -*ThinHarness is for building agents with a defined job and a bounded set of tools, using a refreshingly simple framework that's easy to inspect and customize.* +Minimal agent harnesses are becoming more common. [Pi](https://github.com/earendil-works/pi) has shown how far a focused, token-efficient harness can go, and [Deep Agents](https://github.com/langchain-ai/deepagents) is close behind for cost/performance on [Composio's benchmark](https://composio.dev/content/best-agent-harness-deepseek-v4-flash). Vercel recently released [fx](https://github.com/vercel-labs/fx), which is deliberately tiny and embeddable. ThinHarness takes the same direction to its limit: how little framework code can you keep without giving up capability or performance? -Production agents rarely stop at framework configuration. Things like orchestration, permissions, user/session storage, and deployment become specific to the application and its users. +Larger harnesses are larger for good reasons. They support more providers, storage systems, sandboxes, durable jobs, deployment targets, integrations, and interactive interfaces. ThinHarness makes fewer promises. The core owns the model and tool loop and the behavior that must stay consistent across every run. Optional capabilities live in explicit plugins. -ThinHarness exists for the gap between building the agent loop yourself and adopting a large agent runtime where the loop comes bundled with assumptions you don’t need and can’t easily change. +For a side project, that means less framework code to learn, configure, debug, update, and carry in a fork. It also means the project can become complete. ThinHarness does not need to keep growing into an agent platform after it has the features its agents need. -It owns a focused set of agent-loop primitives that generalize well and are tedious to rebuild, leaving the rest of the application stack for you to own. +## Benchmarking -I started building ThinHarness after running into this gap in practice. Filesystem-enabled agents are simple yet powerful, but you mostly get them by adopting a large framework with layers of abstraction. I usually needed only a small slice of the functionality, but that slice came with coupled assumptions that didn't match my application. Making it fit meant writing enough wrappers, adapters, and fixes that I ended up owning framework-shaped code anyway. +Benchmarking is in progress. - +ThinHarness powers [Retrodict](https://github.com/ryanbbrown/Retrodict), a specialized ARC-AGI-3 agent that leads the reported cost-performance frontier for public ARC-AGI-3 harnesses. Its official competition-mode scorecard reports 99.86% mean RHAE across all 25 public games, with all 183 levels solved. See the [ARC-AGI Community Leaderboard submission](https://github.com/arcprize/ARC-AGI-Community-Leaderboard/tree/main/submissions/retrodict). This is application evidence, not an isolated comparison of generic harness loops. -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    LibraryLOC1Tool
    retries2
    SubagentsSkillsFS
    tools
    OTel
    tracing
    ThinHarness8,035
    - -  Claude Agent SDK - 8,2633⚠️
    - -  smolagents - 9,840⚠️
    - -  deepagents - 17,6644
    - -  AWS Strands - 32,526⚠️
    - -  Microsoft
    - Agent Framework -
    41,331
    - -  Pydantic AI - 59,087
    - -  Google ADK - 65,799⚠️
    - -  OpenAI Agents SDK - 73,796⚠️⚠️
    - -  Agno - 113,477⚠️⚠️
    -

    * Table focuses on harness-level features that differentiate the libraries. All listed also support MCP, lifecycle hooks, multi-turn conversations, structured output, and human-in-the-loop. It intentionally does not compare framework/platform features like vector DB integrations, hosted deployment, memory/session stores, or broad SaaS connectors.

    - -

    - 1. LOC excludes anything that is not the core agent harness framework. See raw README source comments for exact commands.
    - 2. Tool retries: a documented primitive (e.g. Pydantic AI's ModelRetry) that lets tools signal "model passed bad args — retry with this feedback," distinct from generic exception propagation.
    - 3. Claude Agent SDK shells out to the Claude Code CLI binary, which is 200k+ LOC.
    - 4. deepagents is a thin wrapper over LangChain/LangGraph; effective import surface is ≈112k LOC.
    -

    - -
    - -
    - -See [docs/table.md](docs/table.md) for per-cell rationale and how the LOC numbers are measured. - -## Opinions - -ThinHarness has opinions. They are the reason it stays small. - -**Purpose-built agents, not universal agents.** ThinHarness is for bounded agent loops, not open-ended interactive assistants like Claude Code or OpenClaw. For business use cases, focused agent loops orchestrated by deterministic code are usually a better fit than sprawling multi-agent systems with broad authority. - -**No bash by default.** Purpose-built business agents usually don't need a shell. Bash is a broad security and reliability surface: it gives the model open-ended authority instead of typed, bounded actions. ThinHarness has no implicit tools. Add `BashPlugin()` explicitly for bounded exploratory commands, then harden repeated workflow actions as typed tools. - -**Search is a top priority.** The `search` tool exposes ripgrep as compact grouped path/line results, tuned for document and business-workflow agents rather than code navigation. There's also a `jsonl_search` variant, because JSONL is the right shape when you're replacing RAG with agent-driven search over structured data: ripgrep row prefiltering, jq-style field projection, `where` filters, range filters, and snippets from large multiline fields. +## Features -**Parallel LLM calls, explicitly composed.** Fan out from inside the harness when a workflow needs efficient parallel processing or majority vote for reliability. Add `ParallelLlmPlugin()` for a plain-text batch tool that borrows the harness model, or give the plugin a model string and its own provider settings. For validated structured output per call, instantiate `ParallelLlmTool` with `output_type` (a Pydantic model). Each call is stateless, and large batches can write JSON to `output_file`. +The core contains the run behavior every configuration shares. Plugins add complete capabilities without making them implicit dependencies. -**No token streaming.** Streaming is for workflow progress, not live chatbot text. ThinHarness emits run, model-turn, tool, retry, limit, and subagent events, but it does not stream provider token deltas. Token streaming would add provider-specific plumbing, event merging, cancellation edge cases, and more surface area to keep stable. For workflow-style agents, step-level updates are usually the useful signal. +### Core -**Three providers, no matrix.** ThinHarness ships small provider classes for OpenAI, Anthropic, and OpenRouter. If your gateway speaks one of those protocols, you swap a base URL and move on. If not, the provider classes are small enough to fork or replace, and ignoring the bundled ones costs you nothing +- **Plugin composition:** explicit Python plugins can contribute tools, instructions, hooks, and connected resources. +- **Provider adapters:** built-in OpenAI, Anthropic, and OpenRouter adapters, plus public model and session protocols for implementing another provider. +- **Custom typed tools:** define sync or async `ToolSpec` handlers with Pydantic argument models, normalized `ToolResult` envelopes, parallel and approval flags, and per-tool retry settings. +- **Structured output:** Pydantic-validated results with native, tool, prompted, and text modes. +- **Text and images:** ordered text and image input in prompts and tool results, with JPEG, PNG, GIF, and WebP support across the built-in providers. +- **Resume:** self-contained transcript state can replay text and images across built-in providers and models, preserve native reasoning on same-provider resume, and degrade it to text across providers. +- **Parallel tool calls:** same-turn tool batches run concurrently when every called tool is parallel-safe. +- **Human approvals:** approval-required tools pause before side effects and return the pending call plus the state needed to continue after an approve or reject decision. +- **Tool retries:** tools raise `ModelRetry` to send structured feedback to the model and retry within a per-tool budget. +- **Limits and notices:** request, tool-call, output-retry, and tool-retry budgets bound each run; near-limit guidance can warn the model before a budget is exhausted. +- **Hooks and events:** lifecycle hooks can inspect or intercept prompts, tool calls, subagents, limits, and run boundaries; async streaming emits coarse run, model, tool, retry, limit, and subagent events. +- **Tracing:** local plaintext JSONL traces plus OpenTelemetry-compatible spans for runs, provider calls, tools, and subagents. -**No compaction.** Compaction is a workaround for context windows filling up across long, accumulating runs — useful for interactive coding sessions that sprawl over hours. For SDK-based business agents, the right answer to "context is getting big" is almost always better task decomposition: shorter runs, separate harness instances, narrower subagents. +### Plugins -**No deployment layer.** Agents still need serving, auth, durable jobs, user/session storage, and deployment in production. ThinHarness does not try to own that stack. A bundled deployment layer might work for some teams, but it will miss plenty of real production shapes; instead of adding more code and more options, ThinHarness leaves that application stack for you to own. +- **Filesystem:** root-scoped `read`, `write`, batched exact-replacement `edit`, `search`, `list`, and `glob`, plus opt-in bounded `read_image`. +- **JSONL search:** an opt-in `FilesystemPlugin` tool for structured search over line-delimited data, with ripgrep prefiltering, field projection, equality, contains, regex, and range filters, plus field-level snippets from large multiline values. +- **Bash:** one-shot non-interactive commands with a contained working directory, filtered environment, bounded output, timeouts, cancellation cleanup, and optional approval. It is not a sandbox. +- **MCP:** `MCPPlugin` support built on the FastMCP client, including in-process servers, lazy tool discovery, and collision checks. +- **Subagents:** a default child, named child configurations, explicit safe-plugin inheritance, local child hooks, and no recursive delegation. +- **Parallel LLM:** batches of independent one-shot prompts, with an optional separate model, structured results, and explicit read and write paths. +- **Skills:** ordered `skill_read` and `skill_run` tools, with Python, shell, JavaScript, and Go script runners. ## Install @@ -237,7 +69,7 @@ uv add thinharness # or pip install thinharness Requires Python 3.11+. -## Use +## Quick start ```python import asyncio @@ -256,148 +88,20 @@ asyncio.run(main()) There's a synchronous wrapper too: `Harness(...).run_sync(...)`. -Prompts can contain ordered local text and images: - -```python -from thinharness import ImageBlock, TextBlock - -result = await harness.run([ - TextBlock("Describe this image."), - ImageBlock(open("diagram.png", "rb").read(), "image/png"), -]) -``` - -Supported image types are JPEG, PNG, GIF, and WebP. ThinHarness does not fetch image URLs or infer vision support from model names. - -Optional MCP servers use the same plugin composition model: - -```python -from thinharness import MCPPlugin, MCPServerStdio - -harness = Harness( - HarnessConfig(root="."), - plugins=[MCPPlugin(servers=[MCPServerStdio("python", ["server.py"])])], -) -``` - -MCP tools connect and discover one tool snapshot lazily on `Harness.connect()` or the first run. Install support with `uv add 'thinharness[mcp]'`. - -Local Bash is also an explicit plugin: - -```python -from thinharness import BashPlugin - -harness = Harness( - HarnessConfig(root="."), - plugins=[BashPlugin()], -) -``` - -Each call runs a fresh non-interactive shell from a workspace-contained cwd. Bash is sequential, has bounded stdout and stderr, uses a filtered environment by default, and performs best-effort process-group cleanup after normal shell exit, timeout, or run cancellation. It is not a sandbox. - -Delegation is also an explicit plugin: - -```python -from thinharness import SubAgentConfig, SubagentsPlugin - -harness = Harness( - HarnessConfig(root="."), - plugins=[SubagentsPlugin(agents=[ - SubAgentConfig( - name="reviewer", - description="Reviews one draft.", - system_prompt="Return concise issues.", - ) - ])], -) -``` - -Omit `agent` in a `subagent` call to use the default child. Named children can add tools and plugins or set `inherit_parent=True` to rebind safe parent plugins and inherit the active run's frozen direct tools. - -Skills and plain-text parallel batches are explicit plugins too: - -```python -from thinharness import ParallelLlmPlugin, SkillsPlugin - -harness = Harness( - HarnessConfig(root="."), - plugins=[ - # Relative skill directories use the process working directory. - SkillsPlugin(".agents/skills", tools=["skill_read"]), - # Parallel paths use HarnessConfig.root; no model means borrow the harness model. - ParallelLlmPlugin(read_paths=["inputs"], write_paths=["outputs"]), - ], -) -``` - -Built-in provider requests retry transient HTTP failures three times by default. Configure the shared policy with `request_retries` and `request_retry_backoff` on `HarnessConfig`. - -If an injected `http_client` owns retries, set `request_retries=0` on the provider. This prevents nested retry policies from multiplying attempts. - -For workflow visibility, use `Harness.stream(...)`: - -```python -from thinharness import RunCompletedEvent - -async for event in harness.stream("Process these records."): - if event.kind == "tool_call_started": - print(event.tool_name) - if isinstance(event, RunCompletedEvent): - result = event.result -``` - -Streaming emits coarse run, model, tool, retry, limit, and subagent events, then finishes with the same `HarnessResult` returned by `run()`. - -## Features - -- **Filesystem plugin:** explicit `FilesystemPlugin` composition for `read`, `write`, batched exact-replacement `edit`, `search`, `list`, and `glob`, plus opt-in bounded `read_image`, with root-scoped path policies. -- **JSONL search:** opt-in `jsonl_search` for structured line-delimited data, with ripgrep prefiltering, field projection, equality/contains/regex/range `where` filters, and field-level snippets from large multiline string values. -- **Bash plugin:** explicit `BashPlugin` composition for one-shot non-interactive commands with contained cwd, filtered environment, bounded output, timeouts, cancellation cleanup, and optional approval. -- **Provider adapters:** built-in OpenAI, Anthropic, and OpenRouter adapters, plus public model/session protocols for implementing another provider. -- **Custom typed tools:** define sync or async `ToolSpec` handlers with Pydantic argument models, normalized `ToolResult` envelopes, sequential/approval flags, and per-tool retry settings. -- **Structured output:** Pydantic-validated results with native, tool, prompted, and text modes. -- **Hooks:** lifecycle and tool-call interception for prompt submission, tool calls, subagents, limits, and run boundaries. -- **Subagents:** explicit `SubagentsPlugin` composition with a default child, ordered named `SubAgentConfig` recipes, additive safe-plugin inheritance, local child hooks, and no recursive delegation. -- **Parallel LLM:** explicit `ParallelLlmPlugin` fan-out for batches of independent one-shot prompts, plus `ParallelLlmTool(...).spec()` for renameable or structured tools with explicit model, path, prompt, and provider request settings. -- **Skills:** explicit `SkillsPlugin` composition with an ordered `skill_read` and/or `skill_run` selection, plus Python, shell, JavaScript, and Go script runners. -- **Resume:** clean new-turn continuation through self-contained transcript state that can replay text and images across built-in providers and models, preserving native reasoning on same-provider resume and degrading it to text across providers. -- **MCP:** optional MCP support built on the FastMCP client, including in-process servers via `FastMCPTransport`, with lazy tool discovery and collision checks. -- **Parallel tool calls:** same-turn tool batches run concurrently when every called tool is parallel-safe. -- **Human approvals:** mark custom tools as approval-required so a run pauses before side effects, returns pending call details plus resume state, then continues after an approve/reject decision. -- **Event streaming:** async coarse-grained run, model, tool, retry, limit, and subagent events for workflow visibility. -- **Tool retries:** tools raise `ModelRetry` to send structured feedback back to the model and trigger a retry within a per-tool budget. -- **Limits and notices:** configured request, tool-call, output-retry, and tool-retry budgets bound each run; near-limit guidance can warn the model before request or tool-call budgets are exhausted. -- **Tracing:** local plaintext JSONL traces plus OpenTelemetry-compatible spans for runs, provider calls, tools, and subagents. - -## Examples - -Three agents built on ThinHarness, from a self-contained demo to a benchmark run to one I use live. - -### 1. Web Research Report - -A market-landscape research agent that plans, runs batched Exa search, triages and fetches sources, extracts structured source notes with `parallel_llm`, drafts a report, and runs a `citation_critic` subagent before finalizing. - -It isn't meant to be a state-of-the-art research agent; it's a worked example showing the harness drive a non-trivial agentic loop correctly — real multi-tool use across 15 model turns, ending in a reasonable cited report. The full run is browsable on the [docs site](https://ryanbbrown.com/thinharness/examples). - -### 2. LongMemEval-V2 Reproduction - -I ran ThinHarness on a retrieval-heavy 127 question subset of a benchmark for long-term agent memory, and did a local reproduction of the benchmark's optimized harness on the same subset. - -- **Performance:** Matched-or-better accuracy (74.0% vs 72.4% on the 127 dynamic questions) with ~46% less token usage (62M vs. 116M). See [my fork](https://github.com/ryanbbrown/LongMemEval-V2) for more details. -- **Simpler Setup:** ThinHarness only used its filesystem tools (with `jsonl_search` doing the heavy lifting), while the benchmark harness was a full Codex instance with shell and a custom Python tool designed for the task. - -### 3. Personal Opinions Agent - -An agent I run live for myself, inspired by [this Substack post](https://blog.kunchenguid.com/p/everyone-should-have-an-opinionsmd). On a schedule, it reads my Readwise highlights + surrounding context, draws on what it's learned in past runs, and proposes conceptual changes to a durable `OPINIONS.md`. - -Approval happens over Telegram, and it can be simple accept/reject or involve multi-turn revision + discussion. Under the hood it exercises filesystem tools over a JSONL corpus, native structured output, long-term memory, a custom validation tool, and resumable conversations that pick up across each round. See [here](https://github.com/ryanbbrown/opinions-agent) for the code. +## Size -## Status +These are source lines of code in the smallest first-party opinionated configuration for each harness. The count includes required first-party runtime packages and excludes tests, documentation, examples, hosted services, and unrelated optional interfaces where the project structure makes that separation possible. -Pre-1.0. APIs may shift, but I don't expect dramatic changes. Forking is a real option, not just a theoretical one: the codebase is small enough that pulling upstream changes into your fork by hand stays cheap. Each major feature (MCP, subagents, jsonl_search, parallel_llm, skills) lives in its own file with no hidden dependencies. If you don't use one, that's even less code to worry about. If you want to delete it entirely, that's a one-shot 10-word prompt to a coding agent. +| Harness | Source LOC | What is counted | +| --- | --- | --- | +| **ThinHarness** | **10,230** | 6,073 core + 4,157 bundled plugins and tools | +| [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) | 72,737 | The base and headless profiles + their first-party package dependency closure | +| [Pydantic AI Coder](https://github.com/pydantic/pydantic-ai-harness) | 85,633 | Pydantic AI runtime + the Coder capability and everything it composes | +| [Pi](https://github.com/earendil-works/pi) | 95,276 | `pi-coding-agent` + its first-party workspace dependency closure | +| [Deep Agents](https://github.com/langchain-ai/deepagents) | 118,977 | Deep Agents + its required LangChain and LangGraph runtime | -ThinHarness was built with coding agents, but isn't vibe-coded. I have used it, iterated on it, and reviewed its design + behavior. The [website](https://ryanbbrown.com/thinharness/) includes a [codebase explainer](https://ryanbbrown.com/thinharness/explainer) that has gone through many versions so my mental model of the codebase stays up-to-date. +LOC is not a quality or performance score. It measures how much framework code comes with the comparable agent configuration. Moving code from a core package into required plugins does not make that configuration smaller, so the table counts both. ## License -MIT. See [LICENSE](LICENSE). +MIT. See [LICENSE](LICENSE). \ No newline at end of file diff --git a/docs/docs.md b/docs/docs.md index d04172e..fa42f6f 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -2,8 +2,6 @@ ThinHarness is a small SDK for purpose-built agent loops. The host application chooses the model, tools, limits, context, output contract, and lifecycle hooks. The model gets enough room to plan and use tools, but the run stays bounded by configuration. -For code ownership and the run-loop mental model, see `docs/site/explainer.html`. This file is the user-facing API guide. - ## Install ```bash diff --git a/docs/site/about/index.html b/docs/site/about/index.html deleted file mode 100644 index c8a2c08..0000000 --- a/docs/site/about/index.html +++ /dev/null @@ -1,231 +0,0 @@ - - - - - -About — ThinHarness - - - - - - - -
    -
    -
    - ThinHarness -

    A minimal, opinionated agent harness — focused scope, straightforward code, easy to fork.

    - -
    - - - -
    -
    // why this exists
    -

    Why this exists

    -

    ThinHarness is for building agents with a defined job and a bounded set of tools, using a refreshingly simple framework that's easy to inspect and customize.

    Production agents rarely stop at framework configuration. Things like orchestration, permissions, user/session storage, and deployment become specific to the application and its users.

    ThinHarness exists for the gap between building the agent loop yourself and adopting a large agent runtime where the loop comes bundled with assumptions you don’t need and can’t easily change.

    It owns a focused set of agent-loop primitives that generalize well and are tedious to rebuild, leaving the rest of the application stack for you to own.

    I started building ThinHarness after running into this gap in practice. Filesystem-enabled agents are simple yet powerful, but you mostly get them by adopting a large framework with layers of abstraction. I usually needed only a small slice of the functionality, but that slice came with coupled assumptions that didn't match my application. Making it fit meant writing enough wrappers, adapters, and fixes that I ended up owning framework-shaped code anyway.

    -
    - -
    -
    // loc comparison
    -

    How small, exactly

    -

    Framework-only LOC. Each row strips non-framework code (platform/deployment, voice/realtime, eval suites, UI/CLI, wire protocols) from the upstream package. Provider implementations stay in.

    -
    - yes - partial - no -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    LibraryLOC1Tool
    retries2
    Sub-
    agents
    SkillsFS
    tools
    OTel
    tracing
    ThinHarness
    8,035
    Claude Agent SDK
    8,2633
    smolagents
    9,840
    deepagents
    17,6644
    AWS Strands
    32,526
    Microsoft Agent Framework
    41,331
    Pydantic AI
    59,087
    Google ADK
    65,799
    OpenAI Agents SDK
    73,796
    Agno
    113,477
    -
    -

    Table focuses on harness-level features that differentiate the libraries. All listed also support MCP, lifecycle hooks, multi-turn conversations, structured output, and human-in-the-loop. It intentionally does not compare framework/platform features like vector DB integrations, hosted deployment, memory/session stores, or broad SaaS connectors.

    -
    -

    1. LOC excludes anything that is not the core agent harness framework. See raw README source comments for exact commands.

    -

    2. Tool retries: a documented primitive (e.g. Pydantic AI's ModelRetry) that lets tools signal "model passed bad args — retry with this feedback," distinct from generic exception propagation.

    -

    3. Claude Agent SDK shells out to the Claude Code CLI binary, which is 200k+ LOC.

    -

    4. deepagents is a thin wrapper over LangChain/LangGraph; effective import surface is ≈112k LOC.

    -
    -

    See docs/table.md for per-cell rationale and how the LOC numbers are measured.

    -
    - -
    -
    // opinions
    -

    Opinions

    -

    ThinHarness has opinions. They are the reason it stays small.

    -
    -
    purpose_built

    Purpose-built agents, not universal agents

    ThinHarness is for bounded agent loops, not open-ended interactive assistants like Claude Code or OpenClaw. For business use cases, focused agent loops orchestrated by deterministic code are usually a better fit than sprawling multi-agent systems with broad authority.

    -
    no_bash

    No bash by default

    Purpose-built business agents usually don't need a shell. Bash is a broad security and reliability surface: it gives the model open-ended authority instead of typed, bounded actions. ThinHarness has no implicit tools. Add BashPlugin() explicitly for bounded exploratory commands, then harden repeated workflow actions as typed tools.

    -
    search

    Search is a top priority

    The search tool exposes ripgrep as compact grouped path/line results, tuned for document and business-workflow agents rather than code navigation. There's also a jsonl_search variant, because JSONL is the right shape when you're replacing RAG with agent-driven search over structured data: ripgrep row prefiltering, jq-style field projection, where filters, range filters, and snippets from large multiline fields.

    -
    parallel_llm_calls_explicitly_composed

    Parallel LLM calls, explicitly composed

    Fan out from inside the harness when a workflow needs efficient parallel processing or majority vote for reliability. Add ParallelLlmPlugin() for a plain-text batch tool that borrows the harness model, or give the plugin a model string and its own provider settings. For validated structured output per call, instantiate ParallelLlmTool with output_type (a Pydantic model). Each call is stateless, and large batches can write JSON to output_file.

    -
    no_token_streaming

    No token streaming

    Streaming is for workflow progress, not live chatbot text. ThinHarness emits run, model-turn, tool, retry, limit, and subagent events, but it does not stream provider token deltas. Token streaming would add provider-specific plumbing, event merging, cancellation edge cases, and more surface area to keep stable. For workflow-style agents, step-level updates are usually the useful signal.

    -
    providers

    Three providers, no matrix

    ThinHarness ships small provider classes for OpenAI, Anthropic, and OpenRouter. If your gateway speaks one of those protocols, you swap a base URL and move on. If not, the provider classes are small enough to fork or replace, and ignoring the bundled ones costs you nothing.

    -
    no_compaction

    No compaction

    Compaction is a workaround for context windows filling up across long, accumulating runs — useful for interactive coding sessions that sprawl over hours. For SDK-based business agents, the right answer to "context is getting big" is almost always better task decomposition: shorter runs, separate harness instances, narrower subagents.

    -
    no_deployment

    No deployment layer

    Agents still need serving, auth, durable jobs, user/session storage, and deployment in production. ThinHarness does not try to own that stack. A bundled deployment layer might work for some teams, but it will miss plenty of real production shapes; instead of adding more code and more options, ThinHarness leaves that application stack for you to own.

    -
    -
    - -
    -
    // install
    -

    Install

    -
    $ uv add thinharness  # or pip install thinharness
    -

    Requires Python 3.11+.

    -
    - -
    -
    // use
    -

    Use

    -
    import asyncio
    -from thinharness import FilesystemPlugin, Harness, HarnessConfig
    -
    -async def main():
    -    async with Harness(
    -        HarnessConfig(root=".", model="openai:gpt-5.5"),
    -        plugins=[FilesystemPlugin(tools=["read"])],
    -    ) as harness:
    -        result = await harness.run("Read README.md and summarize it.")
    -        print(result.text)
    -
    -asyncio.run(main())
    -

    There's a synchronous wrapper too: Harness(...).run_sync(...).

    -
    - -
    -
    // features
    -

    Features

    -
    -
    Filesystem plugin

    Explicit FilesystemPlugin composition for read, write, batched exact-replacement edit, search, list, and glob with root-scoped path policies.

    -
    JSONL search

    Opt-in jsonl_search for structured line-delimited data, with ripgrep prefiltering, field projection, equality/contains/regex/range where filters, and field-level snippets from large multiline string values.

    -
    Bash plugin

    Explicit BashPlugin composition for one-shot non-interactive commands with contained cwd, filtered environment, bounded output, timeouts, cancellation cleanup, and optional approval.

    -
    Provider adapters

    Built-in OpenAI, Anthropic, and OpenRouter adapters, plus public model/session protocols for implementing another provider.

    -
    Custom typed tools

    Define sync or async ToolSpec handlers with Pydantic argument models, normalized ToolResult envelopes, sequential/approval flags, and per-tool retry settings.

    -
    Structured output

    Pydantic-validated results with native, tool, prompted, and text modes.

    -
    Hooks

    Lifecycle and tool-call interception for prompt submission, tool calls, subagents, limits, and run boundaries.

    -
    Subagents

    Explicit SubagentsPlugin composition with a default child, ordered named SubAgentConfig recipes, additive safe-plugin inheritance, local child hooks, and no recursive delegation.

    -
    Parallel LLM

    Explicit ParallelLlmPlugin fan-out for batches of independent one-shot prompts, plus ParallelLlmTool(...).spec() for renameable or structured tools with explicit model, path, prompt, and provider request settings.

    -
    Skills

    Explicit SkillsPlugin composition with an ordered skill_read and/or skill_run selection, plus Python, shell, JavaScript, and Go script runners.

    -
    Resume

    Clean new-turn continuation through self-contained transcript state that can replay across built-in providers and models, preserving native reasoning on same-provider resume and degrading it to text across providers.

    -
    MCP

    Optional MCP support built on the FastMCP client, including in-process servers via FastMCPTransport, with lazy tool discovery and collision checks.

    -
    Parallel tool calls

    Same-turn tool batches run concurrently when every called tool is parallel-safe.

    -
    Human approvals

    Mark custom tools as approval-required so a run pauses before side effects, returns pending call details plus resume state, then continues after an approve/reject decision.

    -
    Event streaming

    Async coarse-grained run, model, tool, retry, limit, and subagent events for workflow visibility.

    -
    Tool retries

    Tools raise ModelRetry to send structured feedback back to the model and trigger a retry within a per-tool budget.

    -
    Limits and notices

    Configured request, tool-call, output-retry, and tool-retry budgets bound each run; near-limit guidance can warn the model before request or tool-call budgets are exhausted.

    -
    Tracing

    Local plaintext JSONL traces plus OpenTelemetry-compatible spans for runs, provider calls, tools, and subagents.

    -
    -
    - -
    -
    // status
    -

    Status

    -

    Pre-1.0. APIs may shift, but I don't expect dramatic changes. Forking is a real option, not just a theoretical one: the codebase is small enough that pulling upstream changes into your fork by hand stays cheap. Each major feature (MCP, subagents, jsonl_search, parallel_llm, skills) lives in its own file with no hidden dependencies. If you don't use one, that's even less code to worry about. If you want to delete it entirely, that's a one-shot 10-word prompt to a coding agent.

    -
    - -
    -
    // license
    -

    License

    -

    MIT. See LICENSE.

    -
    -
    -
    - -
    - -
    - - diff --git a/docs/site/assets/ThinHarness.svg b/docs/site/assets/ThinHarness.svg deleted file mode 100644 index 3e1608d..0000000 --- a/docs/site/assets/ThinHarness.svg +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/docs/site/assets/agno-a.svg b/docs/site/assets/agno-a.svg deleted file mode 100644 index 0e07b94..0000000 --- a/docs/site/assets/agno-a.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/docs/site/assets/apple-touch-icon.png b/docs/site/assets/apple-touch-icon.png deleted file mode 100644 index 5cab3e7452e423c7cbcb0a1cc225dc0d2b843ef9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2720 zcmc&$`#Tei7vBtfO^6g0uQx*!Q6slTGUf6jMy``?bDzs%avdtUL{YBEB`mp==9Y!= z3Tap>mofH+HqAzi70SE*iSPG0&pDs-!?`@?`JD4SC;hzRIVlJf0ssJ{Y;BM(LTvu6 ze~1e$Vp#ly5X3LrpF;w6e@l5MF%JNMxY;5tU1J_`83|eP8!7|e1PXj(8AorM{msdD z2{P(wZUOsHK-H~#G);P6(D4VSwVs}TD!(mb(9qCwQho0m*0YfNVXl-jGfdv$BRY~M!l{h?^1UJK5m_HXKQMtVM)Rn)#U@-oc>VM^L0IJ*L z5c<-JVtd4)$&v)1=Sb864!~U8EBOiq=+{=%a?R9Dq6l1?iB}7Ug{wFZ!kk|xpux_Q zaRe?!vs{BS2$!HUEWlKeE<1T#!U3GkXLFzzfklYIM7NKFKI#PEEfflYGm-`%)_4#j zH+>()o&u+=W1?RkIO>21z&{6JL?n~-syfY=@}vPyfNlel)0mHQkR_{iPyk#f$Obl? z+j?k4iDa?@Lz_^-X5D-0#ruihevPegMR`Ufquj)KWR80K_IZ2~G-m%B9R^u0W|)yQ z&nt6SySs1t40+p-66<%}WkuWnRw_jYK3&{b;?{X98jllz3A$RC+8G3}I-bUv{1%8=S8FUk0KRL8!A)JaS7wjWb~9C%=zyQaVl#*9N*Z<DVb6iKYN9(xIDe@o52uvVE7-HSVH|BG-rpO2P2#p0zNx#2uz zNhZsd7EG9vJ`N32p5wR=t8;bvGItu^?yapR6Tm|Wk)Dg>?I;T8G@4q0d=?>67=qF5 zL^Ms5ZEg zDAPvG)X;lU7%ax`X7{I$bt{8!f61{`2&*VLPaWGe08FrUU6lEETgt4}WsMe9<-rM| z;;DDkD`?mVT7kd*9UD0a7fK+Pft9TO=MLx5sl65#!SH*$;Q z9B#^9hbtr1CGHBnLrT(Wxi=jxjWEW-3Z`O=d;k8WW~(SdOdXe%URn{C{{@rFz>WF# zG@{8K9oGZtbxZpi+do6T441f%<5Uy)&YR0qRtX6RC-Wayhw`qmZYlUOr~J#r{CZNv zd=hU+IohzgUTEwuIT;#RX*Mgm%gXo0YH!a~*VeuTh6M!JaLT%zVwRf*AiL%zMq!RP zw@gd2vRvUuyh(9!am-4mV9eK{r@|^N0GrKwn>e6?e-ZG$vbZ6RR-S=9si$p&V0Uzw z-j-2)ycqB_)(d`+_!`)26mi;9Ic6w&eF&z83>W3#g( z@ASHrU1{_*$HJm(h+aYiM<96tDYiZN##>U3g z26DxDnuK>t(0*NA-DL*3?FE(U(G_okV~q@7t*J#3AAhsIlvr~w+UT_9VP3DK-U7F6 z4zoD)Fl4T)RLJNF{}JVyzl+9}Zsch?T;_~77#J9YFHbhP`a{j*jm_}9g>M6@{sY-6 zdk&t=GQoY(zp=L9I#D7fR+`e^e>PvoGj~I=)ZpaFwLw_URm9n^eESMfYfJm{^3DHG zNZIykJ7je`T)@DCgmm7_A};v~*?w{^r>3jRoa5|bn7H%3cQ+^VSclfeh7y6SC!Y9V zi;f&4^-&K57g(j_%5=a!Xd|CZ0EwyQkAGjIF@%_k9S+}nU%fdcB|lua(RRdgnXA7K zcJrU5c^XTW>&nr8%s;0>@q-5+1o&af0ANz7t$J$$W?a0qZIFBddbY}oye^ZGI(xrI zSW&ClTbW<8M_ksnw|@up&LUh8%Y-ps>uAcQFB$89q?+HV)V+=+c4*YqA}_}xRN;?q z6!LyRWneB1j>BHxF3uzB!4;$t*Z2?~UZ>j$*iOAh6@ORBE~ElD zldyNyiJ99>luQ2?%{&1QUfD{`b)Q(w<}}#E9l2ka|1d+TW9kg3ecOamLU9Xp(2)Db zj8b67#h#X#uweOyTQIeyG+kW#9a3t*!R3($^K zyFsmRIOEgJb>EzwoW5?r+%cOLOvlO(S!$lfKmc7`Fdm-p-6hh{`yxRBa(!-*{a(6igqak`axM$u^3th zpH@?Z9B4WWl=7q=N;i{!4Lk*|&xwnyf)#9&(1Bkm-2fvv4K(K4mXcgt;>_|Dv1Qxe^H5`c(>}47?fu6O!cTHc&{H b)ug`)j6Yp+PrfF6IRUoTj>yI{ekuO}QVAE! diff --git a/docs/site/assets/favicon.svg b/docs/site/assets/favicon.svg deleted file mode 100644 index e28faca..0000000 --- a/docs/site/assets/favicon.svg +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - diff --git a/docs/site/assets/github-mark.svg b/docs/site/assets/github-mark.svg deleted file mode 100644 index a605d37..0000000 --- a/docs/site/assets/github-mark.svg +++ /dev/null @@ -1,3 +0,0 @@ - diff --git a/docs/site/assets/site.css b/docs/site/assets/site.css deleted file mode 100644 index c62ed3d..0000000 --- a/docs/site/assets/site.css +++ /dev/null @@ -1,1838 +0,0 @@ -/* ============================================================ - ThinHarness — shared site theme (Direction B · "Terminal") - Cool blue-grey · monospace-forward · forest-green accent - ============================================================ */ - -@import url('https://fonts.googleapis.com/css2?family=JetBrains+Mono:wght@400;500;600;700&family=Inter+Tight:wght@400;500;600;700&display=swap'); - -:root{ - --bg:#eceff2; - --panel:#ffffff; - --panel-2:#f5f7f9; - --ink:#19222b; - --ink-soft:#3a4650; - --muted:#69757f; - --line:#d5dde3; - --line-2:#e3e9ed; - --green:#1f7a4d; - --green-bright:#16924f; - --green-deep:#185f3c; - --green-wash:#e4f0e9; - --green-line:#c4ddcd; - --term-bg:#16201b; - --term-ink:#d6e6d8; - --amber:#946a14; - --amber-wash:#f6efda; - --red:#9d3a35; - --red-wash:#f6e6e4; - --radius:10px; - --sans:"Inter Tight",system-ui,-apple-system,sans-serif; - --mono:"JetBrains Mono",ui-monospace,SFMono-Regular,Menlo,monospace; -} - -/* ---- grid background, applied to ---- */ -.th{ - margin:0; - background:var(--bg); - color:var(--ink); - font-family:var(--sans); - line-height:1.55; - -webkit-font-smoothing:antialiased; - background-image: - linear-gradient(var(--line-2) 1px,transparent 1px), - linear-gradient(90deg,var(--line-2) 1px,transparent 1px); - background-size:64px 64px; - background-position:center top; -} -.th *{box-sizing:border-box;} - -/* ============================================================ - Shared header / nav - ============================================================ */ -.site-header{ - position:sticky;top:0;z-index:50; - background:rgba(236,239,242,.86); - backdrop-filter:blur(8px); - -webkit-backdrop-filter:blur(8px); - border-bottom:1px solid var(--line); -} -.site-header__inner{ - max-width:1180px;margin:0 auto;padding:13px 28px; - display:flex;align-items:center;justify-content:space-between;gap:24px; -} -.site-logo{display:inline-flex;align-items:center;line-height:0;} -.site-logo img{height:28px;display:block;} -.site-nav{display:flex;align-items:center;gap:4px;flex-wrap:wrap;} -.site-nav a{ - font-family:var(--mono);font-size:13px;color:var(--ink-soft); - text-decoration:none;padding:6px 11px;border-radius:7px; - border:1px solid transparent;white-space:nowrap;transition:all .14s ease; -} -.site-nav a:hover{color:var(--green-deep);border-color:var(--line);background:var(--panel);} -.site-nav a.is-active{color:var(--green-deep);background:var(--green-wash);border-color:var(--green-line);} -.site-nav a.ext{color:var(--muted);} -.site-nav a.ext:hover{color:var(--green-deep);} - -/* ============================================================ - Shared footer - ============================================================ */ -.site-footer{ - border-top:1px solid var(--line); - margin-top:8px; -} -.site-footer__inner{ - max-width:1180px;margin:0 auto;padding:26px 28px 56px; - display:flex;align-items:center;justify-content:space-between;gap:20px;flex-wrap:wrap; - font-family:var(--mono);font-size:12.5px;color:var(--muted); -} -.site-footer__inner a{color:var(--muted);text-decoration:none;} -.site-footer__inner a:hover{color:var(--green-deep);} -.site-footer .links{display:flex;gap:18px;flex-wrap:wrap;} -.site-footer .mono-mark{display:flex;align-items:center;gap:10px;} -.site-footer .mono-mark .dot{width:7px;height:7px;border-radius:50%;background:var(--green-bright);} - -@media(max-width:640px){ - .site-header__inner{flex-direction:column;align-items:flex-start;gap:12px;} - .site-nav{gap:2px;} - .site-nav a{padding:5px 8px;font-size:12px;} -} - -/* ============================================================ - Home page - ============================================================ */ - -body.page-home { -font-size:16px; -} - -body.page-home .wrap a { -color:var(--green);text-decoration:none; -} - -body.page-home .wrap { -max-width:1180px;margin:0 auto;padding:0 28px; -} - -/* hero */ - -body.page-home .hero { -padding:72px 0 44px;display:grid;grid-template-columns:1.12fr .88fr;gap:48px;align-items:center; -} - -body.page-home .badge { -display:inline-flex;align-items:center;gap:8px;font-family:var(--mono);font-size:12px; - color:var(--green-deep);background:var(--green-wash);border:1px solid var(--green-line); - border-radius:999px;padding:4px 12px;margin-bottom:22px;white-space:nowrap; -} - -body.page-home .badge .dot { -width:7px;height:7px;border-radius:50%;background:var(--green-bright); -} - -body.page-home .badge a { -color:var(--green-deep);text-decoration:none; -} - -body.page-home .badge a:hover { -text-decoration:underline; -} - -body.page-home .badge .badge-icon-link { -display:inline-flex;align-items:center;justify-content:center;width:16px;height:16px;vertical-align:-3px; -} - -body.page-home .badge .badge-icon-link:hover { -text-decoration:none; -} - -body.page-home .badge .badge-icon-link img { -width:16px;height:16px;display:block;opacity:.86; -} - -body.page-home .badge .badge-icon-link:hover img { -opacity:1; -} - -body.page-home h1.tagline { -font-size:46px;line-height:1.08;letter-spacing:-0.028em;font-weight:700;margin:0 0 20px; -} - -body.page-home .tagline .hl { -color:var(--green); -} - -body.page-home .lede { -color:var(--ink-soft);font-size:18px;max-width:480px;margin:0 0 26px; -} - -body.page-home .hero-cta { -display:flex;gap:10px 20px;align-items:center;flex-wrap:wrap;font-family:var(--mono);font-size:13.5px; -} - -body.page-home .hero-cta a { -color:var(--green-deep);white-space:nowrap; -} - -body.page-home .hero-cta .arrow { -transition:transform .15s;display:inline-block; -} - -body.page-home .hero-cta a:hover .arrow { -transform:translateX(3px); -} - -/* terminal block */ - -body.page-home .term { -background:var(--term-bg);border-radius:12px;overflow:hidden; - box-shadow:0 24px 50px rgba(20,30,24,.20);border:1px solid #0e1612; -} - -body.page-home .term .bar { -display:flex;align-items:center;gap:7px;padding:12px 15px;background:#1c2a22;border-bottom:1px solid #0e1612; -} - -body.page-home .term .bar i { -width:11px;height:11px;border-radius:50%;display:block;background:#33473b; -} - -body.page-home .term .bar .t { -margin-left:8px;font-family:var(--mono);font-size:11.5px;color:#7f9a8a; -} - -body.page-home .term .body { -padding:20px;font-family:var(--mono);font-size:14px;line-height:1.95;color:var(--term-ink); -} - -body.page-home .term .body .c { -color:#5f8c72; -} - -body.page-home .term .body .p { -color:#5a7d68; -} - -body.page-home .term .body .g { -color:#7fd39a; -} - -body.page-home .term .body .w { -color:#e9f3eb; -} - -body.page-home .term .body .copy { -float:right;font-size:10.5px;letter-spacing:.07em;text-transform:uppercase; - color:#6f8c7b;cursor:pointer;border:1px solid #2c4034;border-radius:5px;padding:2px 7px; -} - -body.page-home .term .body .copy:hover { -color:#aee7c0;border-color:#3f5b49; -} - -body.page-home .band { -display:grid;grid-template-columns:repeat(3,1fr);background:var(--panel); - border:1px solid var(--line);border-radius:11px;overflow:hidden;margin-top:14px; -} - -body.page-home .band .cell { -padding:16px 20px;border-right:1px solid var(--line-2); -} - -body.page-home .band .cell:last-child { -border-right:0; -} - -body.page-home .band .cell .k { -font-family:var(--mono);font-size:11px;text-transform:uppercase;letter-spacing:.07em;color:var(--muted); -} - -body.page-home .band .cell .v { -font-family:var(--mono);font-size:25px;font-weight:600;margin-top:5px;color:var(--ink); -} - -body.page-home .band .cell .v span { -color:var(--green); -} - -/* section heads */ - -body.page-home .sechead { -display:flex;align-items:baseline;gap:14px;margin-bottom:26px; -} - -body.page-home .sechead .num { -font-family:var(--mono);font-size:13px;color:var(--green); -} - -body.page-home .sechead h2 { -font-size:25px;font-weight:700;letter-spacing:-0.02em;margin:0; -} - -body.page-home .sechead .rule { -flex:1;height:1px;background:var(--line); -} - -body.page-home .sechead .meta { -font-family:var(--mono);font-size:12px;color:var(--muted); -} - -/* opinions */ - -body.page-home .opinions { -padding:60px 0 18px; -} - -body.page-home .oplist { -display:grid;grid-template-columns:1fr 1fr;gap:1px;background:var(--line); - border:1px solid var(--line);border-radius:12px;overflow:hidden; -} - -body.page-home .op { -background:var(--panel);padding:22px 24px;transition:background .14s; -} - -body.page-home .op:hover { -background:var(--panel-2); -} - -body.page-home .op .tag { -font-family:var(--mono);font-size:12px;color:var(--green); -} - -body.page-home .op h3 { -font-size:17px;font-weight:600;margin:9px 0 6px;letter-spacing:-0.01em; -} - -body.page-home .op p { -margin:0;color:var(--ink-soft);font-size:14.5px;line-height:1.5; -} - -/* explore */ - -body.page-home .explore { -padding:58px 0 30px; -} - -body.page-home .xrow { -display:grid;grid-template-columns:1fr 1fr 1fr;gap:14px; -} - -body.page-home .x { -display:flex;flex-direction:column;background:var(--panel);border:1px solid var(--line); - border-radius:12px;padding:22px;transition:border-color .15s,transform .15s,box-shadow .15s; -} - -body.page-home .x:hover { -border-color:var(--green);transform:translateY(-3px);box-shadow:0 16px 32px rgba(25,34,43,.07); -} - -body.page-home .x .path { -font-family:var(--mono);font-size:12px;color:var(--muted); -} - -body.page-home .x h4 { -font-size:19px;font-weight:600;margin:11px 0 7px;color:var(--ink);letter-spacing:-0.01em; -} - -body.page-home .x p { -margin:0 0 16px;color:var(--muted);font-size:14.5px;flex:1; -} - -body.page-home .x .go { -font-family:var(--mono);font-size:13px;color:var(--green-deep); -} - -@media(max-width:880px) { -body.page-home .hero { -grid-template-columns:1fr;gap:30px; -} - -body.page-home .oplist, body.page-home .xrow { -grid-template-columns:1fr; -} - -body.page-home h1.tagline { -font-size:38px; -} -} - -/* ============================================================ - About page - ============================================================ */ - -body.page-about { -font-size:16px; -} - -body.page-about .wrap a { -color:var(--green);text-decoration:none; -} - -body.page-about .wrap a:hover { -text-decoration:underline; -} - -body.page-about .wrap { -max-width:1080px;margin:0 auto;padding:0 28px; -} - -body.page-about .doc { -max-width:860px;margin:0 auto; -} - -/* hero */ - -body.page-about .doc-hero { -padding:64px 0 30px;text-align:center; -} - -body.page-about .doc-hero .bigmark { -height:46px;margin-bottom:26px; -} - -body.page-about .doc-hero .tag { -font-size:21px;color:var(--ink-soft);max-width:560px;margin:0 auto;line-height:1.5; -} - -body.page-about .doc-hero .tag b { -color:var(--ink);font-weight:600; -} - -body.page-about .badges { -display:flex;justify-content:center;gap:8px;flex-wrap:wrap;margin-top:24px; -} - -body.page-about .badges a { -display:inline-flex;align-items:center;gap:7px;font-family:var(--mono);font-size:12px; - color:var(--ink-soft);background:var(--panel);border:1px solid var(--line);border-radius:7px;padding:5px 11px;text-decoration:none;white-space:nowrap; -} - -body.page-about .badges a:hover { -border-color:var(--green-line);color:var(--green-deep); -} - -body.page-about .badges .b-dot { -width:7px;height:7px;border-radius:50%; -} - -body.page-about .badges .ci .b-dot { -background:var(--green-bright); -} - -body.page-about .badges .lic .b-dot { -background:#3b76c4; -} - -body.page-about .badges .pypi .b-dot { -background:var(--amber); -} - -/* on-page nav */ - -body.page-about .toc { -display:flex;flex-wrap:wrap;gap:6px;justify-content:center;margin:30px 0 8px; - padding:14px 0;border-top:1px solid var(--line);border-bottom:1px solid var(--line); -} - -body.page-about .toc a { -font-family:var(--mono);font-size:12.5px;color:var(--muted);padding:4px 9px;border-radius:6px;text-decoration:none; -} - -body.page-about .toc a:hover { -color:var(--green-deep);background:var(--green-wash); -} - -/* sections */ - -body.page-about section { -padding:46px 0 6px;scroll-margin-top:80px; -} - -body.page-about .eyebrow { -font-family:var(--mono);font-size:12.5px;color:var(--green);letter-spacing:.04em;margin-bottom:8px; -} - -body.page-about h2 { -font-size:28px;font-weight:700;letter-spacing:-0.025em;margin:0 0 18px; -} - -body.page-about h3 { -font-size:18px;font-weight:600;margin:0 0 8px;letter-spacing:-0.01em; -} - -body.page-about p { -margin:0 0 16px;color:var(--ink-soft);font-size:16px;line-height:1.66; -} - -body.page-about .doc strong { -color:var(--ink); -} - -body.page-about code { -font-family:var(--mono);font-size:.88em;background:var(--panel-2);border:1px solid var(--line-2); - border-radius:5px;padding:1px 5px;color:var(--green-deep); -} - -/* terminal / code blocks */ - -body.page-about pre { -margin:0 0 16px;background:var(--term-bg);border-radius:12px;border:1px solid #0e1612; - padding:18px 20px;overflow-x:auto;font-family:var(--mono);font-size:13.5px;line-height:1.75;color:var(--term-ink); - box-shadow:0 16px 36px rgba(20,30,24,.14); -} - -body.page-about pre code { -background:none;border:0;padding:0;color:inherit;font-size:inherit; -} - -body.page-about pre .c { -color:#5f8c72; -} - -body.page-about pre .k { -color:#8fd0a6; -} - -body.page-about pre .s { -color:#cfe6d4; -} - -body.page-about pre .p { -color:#5a7d68; -} - -body.page-about .install-line { -display:flex;align-items:center;gap:14px;background:var(--term-bg);color:#e9f3eb; - border-radius:11px;padding:15px 20px;font-family:var(--mono);font-size:15px;margin-bottom:10px; - border:1px solid #0e1612; -} - -body.page-about .install-line .p { -color:#5a7d68; -} - -body.page-about .install-line .c { -color:#5f8c72; -} - -body.page-about .req { -font-family:var(--mono);font-size:13px;color:var(--muted);margin-bottom:0; -} - -/* comparison table */ - -body.page-about .table-note { -font-family:var(--mono);font-size:12px;color:var(--muted);margin:0 0 14px; -} - -body.page-about .table-wrap { -border:1px solid var(--line);border-radius:12px;overflow:hidden;background:var(--panel); - margin-bottom:14px;overflow-x:auto; -} - -body.page-about table.loc { -width:100%;border-collapse:collapse;font-size:14px;min-width:760px; -} - -body.page-about table.loc th, body.page-about table.loc td { -padding:11px 12px;text-align:center;border-bottom:1px solid var(--line-2); -} - -body.page-about table.loc thead th { -background:var(--panel-2);font-family:var(--mono);font-size:11px;font-weight:600; - text-transform:uppercase;letter-spacing:.04em;color:var(--muted);border-bottom:1px solid var(--line); - vertical-align:bottom; -} - -body.page-about table.loc th.lib, body.page-about table.loc td.lib { -text-align:left; -} - -body.page-about table.loc td.lib { -font-weight:500;color:var(--ink); -} - -body.page-about table.loc td.loc-n { -font-family:var(--mono);font-weight:600;color:var(--ink);text-align:right; -} - -body.page-about table.loc tbody tr:last-child td { -border-bottom:0; -} - -body.page-about table.loc tbody tr:hover { -background:var(--panel-2); -} - -body.page-about table.loc tr.me { -background:var(--green-wash)!important; -} - -body.page-about table.loc tr.me td { -border-bottom-color:var(--green-line); -} - -body.page-about table.loc tr.me td.lib { -color:var(--green-deep);font-weight:700; -} - -body.page-about table.loc tr.me td.loc-n { -color:var(--green-deep); -} - -body.page-about .lib-cell { -display:flex;align-items:center;gap:9px; -} - -body.page-about .lib-cell img { -width:18px;height:18px;opacity:.85;flex:none; -} - -body.page-about .lib-cell img.thinharness-table-logo { -width:auto;height:13px; -} - -body.page-about .lib-cell sup { -color:var(--muted);font-size:10px; -} - -body.page-about .mark { -font-family:var(--mono);font-size:15px;line-height:1; -} - -body.page-about .mark.y { -color:var(--green-bright); -} - -body.page-about .mark.n { -color:#c2ccd3; -} - -body.page-about .mark.p { -color:var(--amber); -} - -body.page-about tr.me .mark.y { -color:var(--green-deep); -} - -body.page-about .legend { -display:flex;gap:18px;flex-wrap:wrap;font-family:var(--mono);font-size:12px;color:var(--muted);margin-bottom:18px; -} - -body.page-about .legend span { -display:inline-flex;align-items:center;gap:6px; -} - -body.page-about .footnotes { -font-size:13px;color:var(--muted);line-height:1.6;border-left:2px solid var(--line);padding-left:16px; -} - -body.page-about .footnotes p { -font-size:13px;color:var(--muted);margin:0 0 8px; -} - -/* opinions */ - -body.page-about .op-list { -display:grid;gap:1px;background:var(--line);border:1px solid var(--line);border-radius:12px;overflow:hidden; -} - -body.page-about .op-item { -background:var(--panel);padding:22px 24px; -} - -body.page-about .op-item .tag { -font-family:var(--mono);font-size:12px;color:var(--green); -} - -body.page-about .op-item h3 { -margin:8px 0 7px; -} - -body.page-about .op-item p { -margin:0;font-size:15px; -} - -/* features */ - -body.page-about .feat { -display:grid;grid-template-columns:1fr 1fr;gap:1px;background:var(--line);border:1px solid var(--line);border-radius:12px;overflow:hidden; -} - -body.page-about .feat .f { -background:var(--panel);padding:18px 20px; -} - -body.page-about .feat .f .ft { -font-family:var(--mono);font-size:13px;color:var(--green-deep);font-weight:600;margin-bottom:5px; -} - -body.page-about .feat .f p { -margin:0;font-size:14px;color:var(--ink-soft);line-height:1.55; -} - -body.page-about .callout { -background:var(--amber-wash);border:1px solid #e7d6a8;border-radius:10px;padding:16px 18px;margin-bottom:16px; -} - -body.page-about .callout p { -margin:0;font-size:14.5px;color:#5e4a18; -} - -body.page-about .source-link { -font-family:var(--mono);font-size:13px;color:var(--muted); -} - -body.page-about .table-note-summary { -margin-bottom:18px; -} - -body.page-about .source-note { -margin-top:16px; -} - -@media(max-width:760px) { -body.page-about .feat { -grid-template-columns:1fr; -} - -body.page-about h2 { -font-size:24px; -} -} - -/* ============================================================ - Explainer page - ============================================================ */ - -:root { ---bg: #eceff2; - --paper: #ffffff; - --ink: #19222b; - --muted: #69757f; - --line: #d5dde3; - --soft: #e3e9ed; - --blue: #1f7a4d; - --green: #185f3c; - --red: #9d3a35; - --gold: #946a14; - --code-bg: #16201b; - --code-ink: #d6e6d8; - --shadow: 0 16px 38px rgba(25, 34, 43, 0.08); - --radius: 10px; - --mono: "JetBrains Mono", ui-monospace, SFMono-Regular, Menlo, monospace; - --sans: "Inter Tight", system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; -} - -body.page-explainer * { -box-sizing: border-box; -} - -html:has(body.page-explainer) { -scroll-behavior: smooth; -} - -body.page-explainer { -margin: 0; - background: var(--bg); - color: var(--ink); - font-family: var(--sans); - line-height: 1.55; - -webkit-font-smoothing: antialiased; - background-image: - linear-gradient(#e3e9ed 1px, transparent 1px), - linear-gradient(90deg, #e3e9ed 1px, transparent 1px); - background-size: 64px 64px; - background-position: center top; -} - -body.page-explainer .page a { -color: var(--blue); -} - -body.page-explainer .page { -max-width: 1180px; - margin: 0 auto; - padding: 32px 22px 56px; -} - -body.page-explainer .page > header { -padding: 30px 0 24px; - border-bottom: 1px solid var(--line); -} - -body.page-explainer .eyebrow { -color: var(--green); - font-family: var(--mono); - font-size: 13px; - text-transform: none; - letter-spacing: 0.02em; - font-weight: 500; -} - -body.page-explainer .page h1 { -margin: 10px 0 12px; - font-size: 40px; - line-height: 1.06; - letter-spacing: -0.025em; - font-weight: 700; -} - -body.page-explainer .lede { -max-width: 820px; - color: #3a4650; - font-size: 18px; - margin: 0; -} - -body.page-explainer .page > header nav { -display: flex; - flex-wrap: wrap; - gap: 8px; - margin: 22px 0 0; -} - -body.page-explainer .page > header nav a { -display: inline-block; - border: 1px solid var(--line); - background: var(--paper); - color: var(--ink-soft, #3a4650); - text-decoration: none; - padding: 7px 11px; - border-radius: 7px; - font-size: 13px; - font-family: var(--mono); - transition: all .14s ease; -} - -body.page-explainer .page > header nav a:hover { -color: var(--green); - border-color: #c4ddcd; - background: #e4f0e9; -} - -body.page-explainer .page section { -margin: 34px 0; -} - -body.page-explainer .page h2 { -margin: 0 0 14px; - font-size: 25px; - line-height: 1.2; - letter-spacing: 0; -} - -body.page-explainer .page h3 { -margin: 0 0 10px; - font-size: 18px; - letter-spacing: 0; -} - -body.page-explainer .page p { -margin: 0 0 12px; -} - -body.page-explainer .grid { -display: grid; - gap: 14px; -} - -body.page-explainer .grid.cols-2 { -grid-template-columns: repeat(2, minmax(0, 1fr)); -} - -body.page-explainer .grid.cols-3 { -grid-template-columns: repeat(3, minmax(0, 1fr)); -} - -body.page-explainer .grid-spaced { -margin-top: 16px; -} - -body.page-explainer .table-block { -margin: 22px 0 20px; -} - -body.page-explainer .table-block h3 { -margin-bottom: 12px; -} - -body.page-explainer .grid > * { -min-width: 0; -} - -body.page-explainer .panel { -background: var(--paper); - border: 1px solid var(--line); - border-radius: var(--radius); - padding: 16px; - box-shadow: var(--shadow); - min-width: 0; -} - -body.page-explainer .plain-panel { -background: var(--paper); - border: 1px solid var(--line); - border-radius: var(--radius); - padding: 16px; - min-width: 0; -} - -body.page-explainer .kpi { -display: grid; - gap: 4px; - min-height: 104px; -} - -body.page-explainer .kpi strong { -font-size: 30px; - line-height: 1; -} - -body.page-explainer .kpi span { -color: var(--muted); - font-size: 14px; -} - -body.page-explainer .tagrow { -display: flex; - flex-wrap: wrap; - gap: 6px; - margin-top: 10px; -} - -body.page-explainer .tag { -display: inline-flex; - align-items: center; - min-height: 24px; - border-radius: 999px; - padding: 2px 8px; - background: var(--soft); - border: 1px solid var(--line); - color: #34414c; - font-size: 12px; - font-weight: 650; -} - -body.page-explainer .callout { -border-left: 4px solid var(--blue); - background: #e4f0e9; - padding: 12px 14px; - border-radius: 0 6px 6px 0; - margin: 14px 0; -} - -body.page-explainer .callout.warning { -border-left-color: var(--gold); - background: #f6efda; -} - -body.page-explainer .page table { -width: 100%; - max-width: 100%; - display: block; - overflow-x: auto; - border-collapse: collapse; - background: var(--paper); - border: 1px solid var(--line); - border-radius: var(--radius); - overflow: hidden; - font-size: 14px; -} - -body.page-explainer .page th, body.page-explainer .page td { -padding: 10px 11px; - border-bottom: 1px solid var(--line); - text-align: left; - vertical-align: top; -} - -body.page-explainer .page th { -background: #f5f7f9; - color: #2a343d; - font-size: 12px; - text-transform: uppercase; - letter-spacing: 0.05em; -} - -body.page-explainer .page tr:last-child td { -border-bottom: 0; -} - -body.page-explainer .page code { -font-family: var(--mono); - font-size: 0.92em; - background: #eef2f4; - color: var(--green); - padding: 1px 5px; - border-radius: 5px; -} - -body.page-explainer .page pre { -margin: 0; - max-width: 100%; - background: var(--code-bg); - color: var(--code-ink); - border-radius: var(--radius); - padding: 14px; - overflow-x: auto; - font-family: var(--mono); - font-size: 13px; - line-height: 1.45; - white-space: pre; -} - -body.page-explainer .page pre code { -background: transparent; - color: inherit; - padding: 0; - border-radius: 0; - font-size: inherit; -} - -body.page-explainer .page pre.tree-map { -background: #ffffff; - color: var(--green); - border: 1px solid var(--line); - box-shadow: inset 4px 0 0 var(--green); -} - -body.page-explainer .page pre.tree-map code { -color: inherit; -} - -body.page-explainer .page pre.tree-map::selection, body.page-explainer .page pre.tree-map code::selection { -background: #235b8e; - color: #ffffff; -} - -body.page-explainer .diagram { -width: 100%; - height: auto; - background: var(--paper); - border: 1px solid var(--line); - border-radius: var(--radius); - box-shadow: var(--shadow); -} - -body.page-explainer .diagram-frame { -margin: 0; - overflow-x: auto; - background: var(--paper); - border: 1px solid var(--line); - border-radius: var(--radius); - box-shadow: var(--shadow); - padding: 12px; -} - -body.page-explainer .diagram-frame img { -display: block; - width: 100%; - min-width: 920px; - height: auto; -} - -body.page-explainer .diagram-frame figcaption { -color: var(--muted); - font-size: 13px; - margin-top: 10px; -} - -body.page-explainer .svg-title { -font: 700 14px var(--sans); fill: #1d252c; -} - -body.page-explainer .svg-text { -font: 13px var(--sans); fill: #2f3b44; -} - -body.page-explainer .svg-small { -font: 11px var(--sans); fill: #52616d; -} - -body.page-explainer .svg-box { -fill: #fffdfa; stroke: #bcb5a8; stroke-width: 1.2; -} - -body.page-explainer .svg-core { -fill: #e9f1f6; stroke: #235b8e; -} - -body.page-explainer .svg-tool { -fill: #eaf4ec; stroke: #2f6f50; -} - -body.page-explainer .svg-ext { -fill: #f7eed4; stroke: #8b6b22; -} - -body.page-explainer .svg-err { -fill: #f7e7e4; stroke: #a43e3e; -} - -body.page-explainer .svg-decision { -fill: #fff7d8; stroke: #8b6b22; stroke-width: 1.2; -} - -body.page-explainer .arrow { -stroke: #4d5963; stroke-width: 1.5; fill: none; marker-end: url(#arrowhead); -} - -body.page-explainer .arrow-dashed { -stroke-dasharray: 5 4; -} - -body.page-explainer .step-dot { -fill: #1d252c; -} - -body.page-explainer .step-num { -font: 700 11px var(--sans); fill: #ffffff; text-anchor: middle; dominant-baseline: central; -} - -body.page-explainer .page details { -background: var(--paper); - border: 1px solid var(--line); - border-radius: var(--radius); - padding: 0; -} - -body.page-explainer .page details + details { -margin-top: 10px; -} - -body.page-explainer .page summary { -cursor: pointer; - padding: 12px 14px; - font-weight: 750; -} - -body.page-explainer .page details .inside { -padding: 0 14px 14px; -} - -body.page-explainer .page ul, body.page-explainer .page ol { -margin-top: 0; padding-left: 22px; -} - -body.page-explainer .page li { -margin: 6px 0; -} - -body.page-explainer .page > footer { -border-top: 1px solid var(--line); - padding-top: 18px; - color: var(--muted); - font-size: 13px; -} - -@media (max-width: 780px) { -body.page-explainer .page { -padding: 24px 16px 42px; -} - -body.page-explainer .page h1 { -font-size: 32px; -} - -body.page-explainer .grid.cols-2, body.page-explainer .grid.cols-3 { -grid-template-columns: 1fr; -} - -body.page-explainer .page table { -font-size: 13px; -} - -body.page-explainer .page th, body.page-explainer .page td { -padding: 8px; -} -} - -@media print { -body.page-explainer { -background: white; -} - -body.page-explainer .page { -max-width: none; padding: 18px; -} - -body.page-explainer .page > header nav { -display: none; -} - -body.page-explainer .panel, body.page-explainer .diagram { -box-shadow: none; -} - -body.page-explainer .page pre { -white-space: pre-wrap; background: #f0f0f0; color: #111; -} -} - -/* ============================================================ - Transcripts page - ============================================================ */ - -:root { ---bg: #eceff2; - --panel: #ffffff; - --ink: #19222b; - --muted: #69757f; - --line: #d5dde3; - --accent: #1f7a4d; - --agent: #e4f0e9; - --assistant: #eaf2f8; - --tool: #f6efda; - --subagent: #f3ece2; - --error: #f6e6e4; - --ok: #1f7a4d; - --bad: #9d3a35; - --code-bg: #f5f7f9; - --radius: 10px; - --mono: "JetBrains Mono", ui-monospace, SFMono-Regular, Menlo, monospace; - --sans: "Inter Tight", system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; -} - -body.page-transcripts * { -box-sizing: border-box; -} - -body.page-transcripts { -margin: 0; - background: var(--bg); - color: var(--ink); - font-family: var(--sans); - font-size: 15px; - line-height: 1.45; - -webkit-font-smoothing: antialiased; - background-image: - linear-gradient(#e3e9ed 1px, transparent 1px), - linear-gradient(90deg, #e3e9ed 1px, transparent 1px); - background-size: 64px 64px; -} - -body.page-transcripts header.page-title { -width: min(1180px, calc(100vw - 48px)); - margin: 0 auto; - padding: 30px 0 24px; - border-bottom: 1px solid var(--line); -} - -body.page-transcripts .page-title .eyebrow { -font-family: var(--mono); - font-size: 13px; - color: var(--accent); - margin-bottom: 10px; - letter-spacing: 0.02em; - font-weight: 500; -} - -body.page-transcripts h1 { -margin: 0 0 4px; font-size: 23px; letter-spacing: 0; -} - -body.page-transcripts .page-title h1 { -margin: 0 0 12px; - font-size: 40px; - line-height: 1.06; - letter-spacing: -0.025em; - font-weight: 700; -} - -body.page-transcripts .page-title .muted { -max-width: 820px; - color: #3a4650; - font-size: 18px; -} - -body.page-transcripts h2 { -margin: 0 0 4px; font-size: 21px; -} - -body.page-transcripts h3 { -margin: 0 0 6px; font-size: 14px; color: var(--muted); -} - -body.page-transcripts button, body.page-transcripts input, body.page-transcripts select { -font: inherit; - border: 1px solid var(--line); - border-radius: 6px; - background: #fff; - color: var(--ink); - padding: 8px 10px; -} - -body.page-transcripts button { -cursor: pointer; -} - -body.page-transcripts button:focus, body.page-transcripts input:focus, body.page-transcripts select:focus, body.page-transcripts summary:focus { -outline: 2px solid var(--accent); - outline-offset: 2px; -} - -body.page-transcripts .muted { -color: var(--muted); -} - -body.page-transcripts .layout { -display: grid; - grid-template-columns: 310px minmax(0, 1fr); - min-height: calc(100vh - 50px); -} - -body.page-transcripts .transcript-layout-single { -grid-template-columns: minmax(0, 1fr); - min-height: auto; -} - -body.page-transcripts .transcript-layout-single main { -width: min(1180px, calc(100vw - 48px)); - margin: 0 auto; -} - -body.page-transcripts aside { -background: var(--panel); - border-right: 1px solid var(--line); - padding: 16px; - position: sticky; - top: 50px; - height: calc(100vh - 50px); - overflow: auto; -} - -body.page-transcripts main { -padding: 18px 24px 40px; min-width: 0; -} - -body.page-transcripts .controls { -display: grid; gap: 10px; margin-bottom: 14px; -} - -body.page-transcripts .transcript-toolbar { -grid-template-columns: minmax(260px, 1fr) 190px auto; - align-items: center; - background: var(--panel); - border: 1px solid var(--line); - border-radius: var(--radius); - padding: 12px; -} - -body.page-transcripts .agent-list { -display: grid; gap: 8px; -} - -body.page-transcripts .agent-button { -width: 100%; - text-align: left; - display: grid; - gap: 2px; - border-radius: var(--radius); -} - -body.page-transcripts .agent-button.active { -background: var(--accent); - border-color: var(--accent); - color: #fff; -} - -body.page-transcripts .agent-button.active .muted { -color: #d3eade; -} - -body.page-transcripts .stats { -display: grid; - grid-template-columns: repeat(6, minmax(0, 1fr)); - gap: 10px; - margin-bottom: 14px; -} - -body.page-transcripts .stat { -background: var(--panel); - border: 1px solid var(--line); - border-radius: var(--radius); - padding: 12px; -} - -body.page-transcripts .stat strong { -display: block; font-size: 20px; -} - -body.page-transcripts .agent-card { -background: var(--panel); - border: 1px solid var(--line); - border-radius: var(--radius); - padding: 16px; - margin-bottom: 18px; -} - -body.page-transcripts .agent-head { -display: flex; - justify-content: space-between; - gap: 16px; - padding-bottom: 12px; - margin-bottom: 14px; - border-bottom: 1px solid var(--line); -} - -body.page-transcripts .audit-spec { -border: 1px solid var(--line); - border-radius: var(--radius); - background: #fbfcfd; - padding: 12px; - margin-bottom: 12px; - display: grid; - gap: 14px; -} - -body.page-transcripts .audit-section h3 { -margin-bottom: 8px; - color: var(--ink); - font-size: 13px; - text-transform: uppercase; - letter-spacing: .02em; -} - -body.page-transcripts .audit-section ol { -margin: 0; - padding-left: 22px; -} - -body.page-transcripts .audit-section li + li { -margin-top: 4px; -} - -body.page-transcripts .system-prompt { -margin-top: 16px; -} - -body.page-transcripts .tool-overview { -border: 1px solid var(--line); - border-radius: var(--radius); - background: #fbfcfd; - padding: 12px; - margin: 12px 0; -} - -body.page-transcripts .tool-overview h3 { -margin: 0 0 10px; - color: var(--ink); - font-size: 13px; - text-transform: uppercase; - letter-spacing: .02em; -} - -body.page-transcripts .tool-overview-grid { -display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); - gap: 10px; -} - -body.page-transcripts .tool-overview-item { -border: 1px solid var(--line-2); - border-radius: 8px; - background: #fff; - padding: 10px; - min-width: 0; -} - -body.page-transcripts .tool-overview-item h4 { -margin: 0 0 7px; - font-size: 14px; - color: var(--ink); -} - -body.page-transcripts .tool-overview-item .plain-text { -margin-top: 8px; - color: var(--ink-soft); - font-size: 13px; -} - -body.page-transcripts .pill-row { -display: flex; flex-wrap: wrap; gap: 6px; -} - -body.page-transcripts .pill { -display: inline-flex; - align-items: center; - border: 1px solid var(--line); - border-radius: 999px; - padding: 3px 8px; - color: var(--muted); - background: #fff; - font-size: 12px; -} - -body.page-transcripts .pill.ok { -color: var(--ok); border-color: #b8d9bf; background: #f0faf2; -} - -body.page-transcripts .pill.error { -color: var(--bad); border-color: #e3b9b7; background: var(--error); -} - -body.page-transcripts .transcript { -display: grid; - gap: 12px; - margin-top: 16px; -} - -body.page-transcripts .event { -border: 1px solid var(--line); - border-radius: var(--radius); - background: #fff; - overflow: hidden; - min-width: 0; -} - -body.page-transcripts .event.error { -border-color: #e0b4b2; background: var(--error); -} - -body.page-transcripts .event-head { -display: flex; - justify-content: space-between; - gap: 12px; - align-items: center; - padding: 10px 12px; - border-bottom: 1px solid var(--line); -} - -body.page-transcripts .event-title { -display: flex; gap: 8px; align-items: center; min-width: 0; -} - -body.page-transcripts .badge { -font-family: var(--mono); - text-transform: uppercase; - font-size: 11px; - border-radius: 5px; - padding: 3px 6px; - white-space: nowrap; -} - -body.page-transcripts .badge.agent { -background: var(--agent); color: var(--ok); -} - -body.page-transcripts .badge.chat { -background: var(--assistant); color: #075a71; -} - -body.page-transcripts .badge.tool { -background: var(--tool); color: #765700; -} - -body.page-transcripts .badge.subagent { -background: var(--subagent); color: #7a4618; -} - -body.page-transcripts .badge.error { -background: var(--error); color: var(--bad); -} - -body.page-transcripts .duration { -color: var(--muted); - font-family: var(--mono); - font-size: 12px; - white-space: nowrap; -} - -body.page-transcripts .event-body { -padding: 12px; display: grid; gap: 12px; min-width: 0; -} - -body.page-transcripts .bubble { -border: 1px solid var(--line); - border-radius: var(--radius); - padding: 11px 12px; - background: #fff; - min-width: 0; - overflow-wrap: anywhere; -} - -body.page-transcripts .bubble.assistant { -background: var(--assistant); border-color: #c8e3eb; -} - -body.page-transcripts .bubble.tool-call { -background: var(--tool); border-color: #ecdca6; -} - -body.page-transcripts .bubble.tool-result { -background: #fffdf4; border-color: #ecdca6; -} - -body.page-transcripts .bubble.subagent { -background: var(--subagent); border-color: #edcfb8; -} - -body.page-transcripts .bubble.final { -background: #effaf2; border-color: #badbc1; -} - -body.page-transcripts .bubble.context { -background: #f8fafb; -} - -body.page-transcripts .write-input .plain-text { -font-family: var(--mono); - font-size: 12px; - line-height: 1.5; - max-height: 480px; - overflow: auto; - background: rgba(255,255,255,.58); - border: 1px solid rgba(214,179,65,.35); - border-radius: 6px; - padding: 10px; -} - -body.page-transcripts .write-path { -font-family: var(--mono); - font-size: 12px; - color: #5f4705; - background: rgba(255,255,255,.6); - border: 1px solid rgba(214,179,65,.35); - border-radius: 6px; - padding: 6px 8px; - margin-bottom: 8px; - overflow-wrap: anywhere; -} - -body.page-transcripts .compact-call > div:not(.label) > strong, -body.page-transcripts .compact-call > div:not(.label) > .muted { -display: block; - min-width: 0; - overflow-wrap: anywhere; -} - -body.page-transcripts .label { -display: flex; - justify-content: space-between; - gap: 10px; - margin-bottom: 6px; - color: var(--muted); - font-size: 12px; - text-transform: uppercase; - letter-spacing: .02em; - font-weight: 700; -} - -body.page-transcripts .message-list { -display: grid; gap: 8px; min-width: 0; -} - -body.page-transcripts .message { -border-left: 3px solid var(--line); - padding: 7px 9px; - background: #fff; - border-radius: 0 6px 6px 0; - min-width: 0; -} - -body.page-transcripts .message.user { -border-color: #7da8b6; -} - -body.page-transcripts .message.assistant { -border-color: #6baec2; -} - -body.page-transcripts .message.tool { -border-color: #d6b341; -} - -body.page-transcripts .tool-result-list { -display: grid; gap: 6px; min-width: 0; -} - -body.page-transcripts .tool-result-row summary { -display: flex; - justify-content: space-between; - gap: 12px; - align-items: center; -} - -body.page-transcripts .tool-result-row .summary-main { -min-width: 0; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; -} - -body.page-transcripts .tool-result-row .call-id { -color: var(--muted); - font-family: var(--mono); - font-size: 11px; - white-space: nowrap; -} - -body.page-transcripts .role { -display: block; - color: var(--muted); - font-size: 12px; - font-weight: 700; - text-transform: uppercase; - margin-bottom: 4px; -} - -body.page-transcripts details { -border: 1px solid var(--line); - border-radius: 6px; - background: #fff; - min-width: 0; -} - -body.page-transcripts summary { -cursor: pointer; - padding: 8px 10px; - color: var(--muted); - font-weight: 650; - overflow-wrap: anywhere; -} - -body.page-transcripts .details-body { -padding: 0 10px 10px; - min-width: 0; -} - -body.page-transcripts .output-details .details-body { -padding-top: 0; -} - -body.page-transcripts .output-details pre { -max-height: 720px; -} - -body.page-transcripts .nested-events { -background: #fbfcfd; - border-color: var(--line); -} - -body.page-transcripts .nested-events > summary { -font-family: var(--mono); - font-size: 12px; - color: var(--ink-soft); -} - -body.page-transcripts .nested-events-body { -display: grid; - gap: 10px; - padding: 0 10px 10px 16px; - border-left: 2px solid var(--line); - margin-left: 10px; -} - -body.page-transcripts .nested-events-body .event { -box-shadow: none; -} - -body.page-transcripts pre { -margin: 0; - padding: 10px; - background: var(--code-bg); - color: #10202b; - border-radius: 6px; - overflow: auto; - max-height: 560px; - white-space: pre-wrap; - word-break: break-word; - font-family: var(--mono); - font-size: 12px; - line-height: 1.45; -} - -body.page-transcripts .plain-text { -white-space: pre-wrap; - word-break: break-word; -} - -body.page-transcripts .empty { -color: var(--muted); padding: 12px; -} - -body.page-transcripts .layout footer { -margin-top: 24px; color: var(--muted); font-size: 12px; -} - -@media (max-width: 860px) { -body.page-transcripts header.page-title { -width: calc(100vw - 32px); - padding: 24px 0 22px; -} - -body.page-transcripts .page-title h1 { -font-size: 32px; -} - -body.page-transcripts .layout { -grid-template-columns: 1fr; -} - -body.page-transcripts aside { -position: static; - height: auto; - border-right: 0; - border-bottom: 1px solid var(--line); -} - -body.page-transcripts .stats { -grid-template-columns: repeat(2, minmax(0, 1fr)); -} - -body.page-transcripts .tool-overview-grid { -grid-template-columns: 1fr; -} - -body.page-transcripts .agent-head, body.page-transcripts .event-head { -display: grid; -} -} - -@media print { -body.page-transcripts header.page-title, body.page-transcripts aside { -position: static; -} - -body.page-transcripts .layout { -display: block; -} - -body.page-transcripts .controls { -display: none; -} - -body.page-transcripts .event, body.page-transcripts .agent-card { -break-inside: avoid; -} - -body.page-transcripts { -background: #fff; -} -} - -/* ---- examples page: LongMemEval / transcript toggle ---- */ -.ex-toggle{display:flex;justify-content:center;gap:8px;width:min(1180px,calc(100vw - 48px));margin:18px auto 6px;} -.ex-toggle button{font-family:var(--mono);font-size:13px;padding:8px 16px;border:1px solid var(--line);border-radius:8px;background:var(--panel);color:var(--ink-soft);cursor:pointer;transition:color .15s,background .15s,border-color .15s;} -.ex-toggle button:hover{color:var(--green-deep);border-color:var(--green-line);} -.ex-toggle button.is-active{color:var(--green-deep);background:var(--green-wash);border-color:var(--green-line);font-weight:600;} -.ex-panel[hidden]{display:none !important;} -.md-render{width:min(860px,calc(100vw - 48px));margin:0 auto;padding:14px 0 64px;color:var(--ink);font-family:var(--sans);line-height:1.62;} -.md-render .md-eyebrow{font-family:var(--mono);font-size:13px;color:var(--green);letter-spacing:.04em;font-weight:500;margin:6px 0 6px;} -.md-render h1{font-size:30px;margin:0 0 8px;letter-spacing:-.02em;font-weight:700;} -.md-render h2{font-size:20px;margin:30px 0 10px;color:var(--green-deep);border-bottom:1px solid var(--line);padding-bottom:6px;} -.md-render h3{font-size:16px;margin:20px 0 8px;} -.md-render p{margin:10px 0;color:var(--ink-soft);} -.md-render ul{margin:10px 0;padding-left:22px;color:var(--ink-soft);} -.md-render li{margin:7px 0;} -.md-render a{color:var(--green);text-decoration:underline;text-underline-offset:2px;} -.md-render code{font-family:var(--mono);font-size:.86em;background:var(--green-wash);padding:1px 5px;border-radius:4px;color:var(--green-deep);} -.md-render strong{color:var(--ink);} -.md-table-wrap{overflow-x:auto;margin:14px 0;border:1px solid var(--line);border-radius:8px;max-width:620px;} -.md-render table{width:100%;border-collapse:collapse;font-size:13px;} -.md-render th,.md-render td{padding:8px 12px;border-bottom:1px solid var(--line);text-align:left;vertical-align:top;} -.md-render td:first-child,.md-render th:first-child{font-family:var(--mono);font-size:12px;color:var(--ink);white-space:nowrap;} -.md-render thead th{background:var(--green-wash);color:var(--green-deep);font-weight:600;} -.md-render tbody tr:last-child td{border-bottom:none;} diff --git a/docs/site/assets/thinharness-mark.svg b/docs/site/assets/thinharness-mark.svg deleted file mode 100644 index 36dd8cb..0000000 --- a/docs/site/assets/thinharness-mark.svg +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - diff --git a/docs/site/assets/thinharness-run-loop.svg b/docs/site/assets/thinharness-run-loop.svg deleted file mode 100644 index aa8d907..0000000 --- a/docs/site/assets/thinharness-run-loop.svg +++ /dev/null @@ -1,3 +0,0 @@ - - -
    harness/runtime
    provider session
    decision
    approval pause
    tool path
    final result
    retry / stop
    stream event
    Typed event stream
    StreamEvent emitted at each blue dot; contains kind, run_id, and event number
    SETUP
    ONE MODEL TURN
    BRANCH HANDLING INSIDE THE WHILE LOOP
    Entry point
    Harness.stream()
    core.py:305-374
    outer stream flow
    Create run objects
    RunContext
    StreamEmitter
    core.py:405-418
    per-run state
    trace scope
    Open session
    new_session()
    resume_session()
    core.py:448-491
    one active session
    Loop entry
    RunContext.advance_model()
    runtime.py:268
    limits, notices, tracing
    provider request
    Call provider session
    ModelSession
    providers.py:184
    start()
    continue_with_tools()
    continue_with_user_content()
    Model turn
    ModelTurn
    providers.py:56
    text
    tool calls
    raw JSON
    Resolve turn
    resolve_turn_output()
    turns.py:50
    OutputTurnDecision
    chooses next branch
    Approval required?
    _approval_required_calls()
    turns.py:322
    before tool batch executes
    Final result
    RunContext.finalize()
    runtime.py:420
    builds HarnessResult
    fires run_end once
    Approval pause
    pause_for_approval()
    runtime.py:367
    stop_reason="approval_required"
    pending approvals + resume_state
    no tools executed
    Run tools
    ToolBatchExecutor.execute_batch()
    tool_execution.py:63
    runs ordinary tool calls
    preserves model order
    Return tool outputs
    Build ToolOutput[]
    providers.py:97
    model-visible results
    Retry decision
    RunContext.retry_or_fail()
    runtime.py:448
    budget remains or stop
    Stop / raise
    core.py:499-510
    HarnessError
    limit, provider,
    or validation
    Corrective request
    continue_with_tools()
    continue_with_user_content()
    asks model to fix output
    interpret turn
    final
    no approval
    requires approval
    tool batch
    schema retry
    next iteration
    budget remains
    no retry
    next iteration
    Text is not SVG - cannot display
    \ No newline at end of file diff --git a/docs/site/examples/index.html b/docs/site/examples/index.html deleted file mode 100644 index 3a207a4..0000000 --- a/docs/site/examples/index.html +++ /dev/null @@ -1,637 +0,0 @@ - - - - - - Examples — ThinHarness - - - - - - -
    -
    // examples
    -

    Examples

    -
    Real ThinHarness runs. Toggle between the LongMemEval-V2 benchmark write-up and the Web Research transcript.
    -
    -
    - - -
    -
    -
    // benchmark reproduction

    LongMemEval-V2 - ThinHarness Benchmark

    -

    Context

    -

    I used a fork of the LongMemEval-V2 benchmark to test whether ThinHarness is a strong general-purpose agent harness on a nontrivial non-coding task: information retrieval over long trajectory haystacks.

    -

    I scoped the comparison to the 127 dynamic questions in the small tier: dynamic-environment and dynamic-environment-abs across the web and enterprise domains. After reviewing the benchmark categories, this subset looked like the best test for my purposes: given a long history of interaction traces, can the memory layer efficiently and accurately find the state change, UI behavior, or environment fact needed to answer the question?

    -

    Run Details

    -

    To get a local baseline for more detailed metrics than the leaderboard provides, I ran AgentRunbook-C with the resumable wrapper in evaluation/scripts/run_agentrunbook_c_dynamic.sh. That script plans the 127-question dynamic set, runs one question per output directory, and can be re-run to fill only missing questions. Running one question at a time is slower but was helpful when encountering intermittent issues running on my local machine. The matching ThinHarness wrapper is evaluation/scripts/run_thinharness_dynamic.sh.

    -

    This is a best-faith reproduction rather than an exact reproduction of the paper. The paper setup expects a local Qwen/Qwen3.5-9B reader deployment; I used qwen/qwen3.5-9b through OpenRouter. The paper uses Codex v0.117.0 for Codex and AgentRunbook-C; my rerun used the local Codex CLI available, codex-cli 0.141.0. Both the AgentRunbook-C rerun and ThinHarness run used gpt-5.4-mini with xhigh reasoning for the query-time memory agent and gpt-5.2 for the evaluator.

    -

    For the ThinHarness run, I used only its generic built-in filesystem tools (read, search, jsonl_search, list, and glob). I did restructure the memory files into a JSONL-friendly corpus so its generic jsonl_search tool could work well; that seems acceptable here because AgentRunbook-C also creates a custom trajectory structure rather than using the vanilla Codex raw layout. I tried to keep the query-time system prompt as close as practical to AgentRunbook-C, changing the tool instructions only where ThinHarness needed to know how to use its built-in tools.

    -

    Results

    -

    Across all 127 dynamic questions in the small tier:

    -
    MetricAgentRunbook-C rerunThinHarness
    Dynamic score72.4% (92/127)74.0% (94/127)
    Non-abstention86.0% (74/86)84.9% (73/86)
    Abstention43.9% (18/41)51.2% (21/41)
    Memory query time151.9s avg, 129.1s median99.7s avg, 87.9s median
    Memory-agent tokens / usage114.77M input, 1.32M output, 116.09M total60.14M input, 2.10M output, 62.24M total
    Dynamic-subset LAFS3.769.73 (+5.96)
    -

    The 72.4% accuracy for AgentRunbook-C matches the paper, but I would not treat this single consolidated run as a statistically signficant claim that ThinHarness has higher accuracy than AgentRunbook-C--I saw meaningful variance on a portion of the questions when doing targeted reruns. The result does make me reasonably confident that ThinHarness at least matches AgentRunbook-C's performance on this slice, and the published leaderboard reference for vanilla Codex is materially lower than both.

    -

    The memory query time is the harness-measured time around memory.query(...): it includes the query-time memory retrieval agent, but not the downstream reader, scorer, or prior runtime input generation. The timing comparison isn't perfect (local Codex CLI vs. OpenAI API), but the 46.4% lower token usage indicates that ~34% time savings is probably in the right ballpark. Note that the paper only provides a single aggregate query time figure across all questions, 108.3s, which is far lower than the 151.9s above but includes all questions in the small tier (some of which may have been faster).

    -
    - - - - - - - diff --git a/docs/site/explainer/index.html b/docs/site/explainer/index.html deleted file mode 100644 index b098ef3..0000000 --- a/docs/site/explainer/index.html +++ /dev/null @@ -1,960 +0,0 @@ - - - - - - Codebase Explainer — ThinHarness - - - - - - -
    -
    -
    // codebase guide
    -

    A mental model for the whole repository

    -

    - ThinHarness is a focused agent harness with a small runtime surface. The same loop runs no matter which - provider adapter is used: create a model session, ask the model for a turn, convert - provider-specific responses into ThinHarness objects, execute local tools, send tool - results back, and stop when the final-answer code says the run is complete. -

    - -
    - -
    -

    Snapshot

    -
    -
    - 24 - runtime Python files under thinharness/, including the tools package. -
    -
    - 8,035 - README-stated framework LOC, intentionally small enough to inspect, adapt, and fork. -
    -
    - 8 - hook events covering run start, prompt submit, tools, subagents, limits, and run end. -
    -
    -
    -

    - ThinHarness separates reusable setup from per-run state. HarnessConfig is Pydantic - configuration; Harness owns configured runtime objects; RunContext owns one - invocation's mutable state; provider ModelSession objects own provider conversation state; - ModelTurn and ToolSpec are small dataclass contracts passed across those boundaries. -

    -
    -
    -

    Where this page starts

    -

    - For positioning, quick-start usage, and feature comparison, read README.md. This page focuses on - runtime architecture and code ownership. -

    -
    -
    - -
    -

    Run Loop

    -
    - SVG flowchart showing the ThinHarness run loop from Harness.run through RunContext, ModelSession, ModelTurn resolution, tool execution, approval pause, retries, and HarnessResult construction. -
    - Tool calls execute through the same local tool path, then their normalized outputs are returned to the provider before the loop resolves the next model turn. -
    -
    - -
    -
    -

    Happy path

    -
      -
    1. Harness.run() checks that the harness can run, creates or resumes a ModelSession, and builds RunContext.
    2. -
    3. RunContext.advance_model() makes one provider call with limit checks, notices, usage accounting, and model tracing around it.
    4. -
    5. The provider adapter returns a ModelTurn, ThinHarness's common Python object for model text, tool calls, and raw provider JSON.
    6. -
    7. resolve_turn_output() reads that turn against OutputSchema and chooses the next action: final result, run tools, structured-output retry, or failure.
    8. -
    9. If any requested tool requires approval, the harness pauses before the batch runs and returns stop_reason="approval_required" with pending approval records and an approval resume envelope.
    10. -
    11. If the decision is continue, ToolBatchExecutor runs ordinary tool calls and sends ordered ToolOutput values back through continue_with_tools().
    12. -
    13. If the decision is final, RunContext.finalize() builds HarnessResult, attaches resume state when valid, annotates tracing, and fires run_end.
    14. -
    -
    -
    -

    Failure and limit path

    -
      -
    • Provider errors become HarnessError with stop_reason="provider_error".
    • -
    • Structured-output validation can request a corrective model turn until output_retries is exhausted.
    • -
    • Retryable tool envelopes increment per-tool retry counters and can stop the run with tool_retries_exceeded.
    • -
    • Tool batches that would exceed max_tool_calls are rejected before local execution.
    • -
    • run_end is guarded so success, errors, hook cancellation, and external cancellation fire it at most once.
    • -
    -
    -
    -
    - -
    -

    Streaming

    -

    - Harness.stream(...) is the public progress API, and Harness.run(...) is implemented - by consuming that stream until the top-level RunCompletedEvent arrives. Streaming does not change - the run loop; it exposes coarse lifecycle events from the same execution path. -

    -
    -
    -

    What streams

    -
      -
    • Run start, provider request start, complete model turns, tool call start/completion, retry opportunities, limit warnings, run completion, and run failure.
    • -
    • Provider calls still return complete model turns. This is workflow progress streaming, not token-delta streaming.
    • -
    • The final successful event carries the full HarnessResult, including provider responses, tool records, usage, stop reason, output, and resume state.
    • -
    -
    -
    -

    Payload policy

    -
      -
    • Stream events include high-level prompt, tool argument, and model-visible tool result payloads for app-facing workflow visibility.
    • -
    • Raw provider response JSON is not included in stream events. Use the terminal HarnessResult.responses after completion when provider raw responses are needed.
    • -
    • StreamOptions can hide child subagent events, but it does not hide model text or expose raw provider payloads.
    • -
    • Nested work is correlated with run_id, parent_run_id, parent_tool_call_id, agent_name, and per-stream sequence.
    • -
    -
    -
    -
    -

    - stream() starts eagerly when called and owns an in-process event queue. If a caller may stop - before the terminal event, use the stream as an async context manager or call aclose(); closing - the stream cancels the underlying run task and lets the harness be reused cleanly. -

    -
    -
    - -
    -

    Repository File Map

    -
    .
    -|-- thinharness/
    -|   |-- __init__.py                  public API exports
    -|   |-- core.py                      HarnessConfig, Harness, run-loop coordination
    -|   |-- runtime.py                   RunContext and the provider-call wrapper
    -|   |-- turns.py                     turn state machine and output turn decisions
    -|   |-- types.py                     leaf run result, usage, errors, stop reasons, Json alias
    -|   |-- tool_execution.py            tool batch execution and one-call hook/tracing flow
    -|   |-- providers.py                 provider transports, model adapters, session state
    -|   |-- output.py                    structured-output schemas and validation
    -|   |-- hooks.py                     hook dataclasses, registry, context variables
    -|   |-- children.py                  narrow child-harness host and execution
    -|   |-- tracing.py                   OTel-compatible spans and local JSONL tracing
    -|   |-- defaults.py                  default system and tool instructions
    -|   |-- plugins/
    -|   |   |-- base.py                  plugin context, binding, contribution contracts
    -|   |   |-- filesystem.py            FilesystemPlugin composition and root policy
    -|   |   |-- skills.py                SkillsPlugin catalog and tool selection
    -|   |   |-- parallel_llm.py           ParallelLlmPlugin model and path composition
    -|   |   `-- mcp.py                   MCPPlugin connection and discovery lifecycle
    -|   `-- tools/
    -|       |-- __init__.py              tool package exports
    -|       |-- base.py                  ToolSpec, ToolResult, path policy, invocation
    -|       |-- filesystem.py            FileTools: read, write, edit, search, list, glob
    -|       |-- jsonl.py                 JsonlSearch and JSONL where/projection helpers
    -|       |-- search_support.py        shared ripgrep parsing and glob validation
    -|       |-- mcp.py                   optional MCP transports and tool conversion
    -|       |-- parallel_llm.py          ParallelLlmTool batch completions
    -|       `-- skills.py                SkillRegistry, skill_read, skill_run
    -|-- tests/                           pytest coverage for every feature area
    -|-- e2e/                             live-provider journeys, intentionally outside CI
    -|-- examples/                        shared scenario registry plus thin agent entrypoints
    -|-- docs/                            decisions, user docs, site files, releasing notes
    -|-- README.md                        motivation, usage, comparison table
    -`-- pyproject.toml                   package metadata, deps, ruff, pyright, pytest
    -
    - -
    -

    Core Objects

    -
    -
    -

    Harness-facing objects

    - - - - - - - - - - -
    NameMeaningRelationship
    HarnessConfigPydantic setup model: root, model ref, limits, output mode, tracing, and provider settings.Configures a Harness.
    HarnessLong-lived configured runner. It owns the model object, resolved tool map, plugin bindings, hooks, and tracing configuration.Creates a fresh RunContext for each run.
    RequestConstantsFrozen per-run request constants: instructions, tool schemas, metadata, and the structured-output request. Built once after run-start hooks and plugin connection, so the toolset is frozen for the run.Passed to every ModelSession request by the turn machine in turns.py.
    RunContextInternal state for one Harness.run(...): responses, tool records, usage, retry/notice state, terminal error, stop reason, tracing span, and final result.References the reusable Harness, but is not stored on it after the run.
    HarnessResultFinal run result: final text, parsed structured output, raw provider responses, tool call records, usage, stop reason, and resume state.Receives finalized state from RunContext.
    RunUsageCounts model requests, tool calls, cancelled tool calls, output retries, per-tool retry counts, and run token totals (input_tokens/output_tokens).Per-run counter owned by RunContext and returned in HarnessResult.
    -
    -
    -

    Provider-neutral objects

    - - - - - - - - - - -
    NameMeaning
    ModelProtocol for reusable model configuration. It creates isolated ModelSession objects.
    ModelSessionPer-run provider conversation state. Built-in sessions keep native in-run state plus a parallel neutral transcript; dump_state returns the transcript for provider-agnostic resume. All expose three request methods — start, continue_with_tools, and continue_with_user_content — each taking per-run RequestConstants, plus dump_state.
    ModelTurnNormalized provider response: assistant text, requested ModelToolCall entries, raw provider JSON, plus normalized TokenUsage, finish reason, and response model.
    ModelToolCallNormalized tool request with id, name, and raw JSON argument string.
    ToolOutputTool result sent back to the provider so the model can continue after a tool call: call id plus model-visible output string.
    ModelNoticeProvider-neutral model input notice for run-budget warnings, including remaining model requests and remaining tool calls.
    -
    -
    - -
    - -
    -

    Default Tools

    -

    - Every model-callable tool is represented by ToolSpec, a dataclass that packages - the tool definition sent to the model with the Python callable that executes it. A spec includes - a name, description, JSON schema or Pydantic argument model, handler callable, sequential flag, - metadata, optional retry budget, and approval flag. - The result sent back to the provider is always a JSON envelope with ok, content, and metadata. -

    - -
    -
    -

    What the handler is

    -

    - ToolSpec.handler is the callable the harness invokes after it parses and validates - the model's JSON arguments. It can be a plain function, a bound method, a callable object, - or an async callable. The harness calls it as handler(args), then converts - a ToolResult, string, or JSON-serializable value into the output envelope sent back to the provider. -

    -
    -
    -

    Class to ToolSpec handoff

    -

    - Built-in tool modules often use classes to hold shared state, but the class itself is not - the model-callable tool. For example, FileTools.specs() creates several - ToolSpec objects whose handlers are bound methods such as self.read, - self.write, and self.search. Each bound method carries the configured - root path, path policies, limits, and spill behavior from that FileTools instance. -

    -
    -
    -

    Sequential flag

    -

    - sequential=True means calls involving that tool force the current model-emitted - tool-call batch to run serially instead of concurrently. The flag is used only inside ThinHarness, - not sent to the model as part of the tool JSON schema. With the default batch policy, one sequential - tool makes the whole batch run in model order. -

    -
    -
    -

    Plugin composition

    -

    - ThinHarness composes filesystem, skills, MCP, parallel LLM, and subagent behavior through explicit plugins. Plugins receive - the canonical root, configured model, and narrow child host, then contribute ordered tools, instructions, hooks, agent names, or connected state. - The core has no implicit or selected built-in tool path. -

    -
    -
    - -
    -

    Tool surfaces

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    SurfaceClass or ownerHow it enters the harnessImportant behavior
    FilesystemFilesystemPlugin over FileToolsplugins=[FilesystemPlugin(...)] binds static ToolSpec values without importing filesystem code into core.The default set is read, write, edit, search, list, glob; jsonl_search is opt-in. Mutating tools are sequential.
    Plugin-providedSkillsPlugin, ParallelLlmPlugin, MCPPlugin, and SubagentsPluginExplicit plugin contributions are normalized into the same ToolSpec map with plugin origin attribution.Skills and parallel LLM contribute static tools; MCP connects lazily and contributes one discovered snapshot.
    DelegationSubagentsPluginExplicit plugin composition contributes one ordinary subagent tool and static agent names.The plugin owns child recipes; the neutral child host owns isolated execution and authoritative delegation provenance.
    CustomCaller-provided ToolSpecRegistered at construction with tools=[...] or later with add_tool(); both populate the same runtime tool map.Sync handlers run in worker threads. Async handlers run directly. Pydantic args turn validation failures into retry envelopes. Human approval is opt-in through requires_approval=True.
    -
    - -
    -
    -

    Invocation path

    -
      -
    • ToolCallExecutor.execute_one(call) sets the current tool-call context.
    • -
    • before_tool_call hooks can cancel before local execution.
    • -
    • Arguments are parsed from JSON and validated with Pydantic when the tool has an argument model.
    • -
    • The handler runs directly when async, or in a worker thread when sync.
    • -
    • The result is normalized into the standard {"ok", "content", "metadata"} envelope.
    • -
    • after_tool_call hooks can rewrite model-visible output before tracing records the result.
    • -
    -
    -
    -

    Retry semantics

    -
      -
    • Malformed JSON args, non-object args, and Pydantic validation errors return retryable envelopes.
    • -
    • A handler can raise ModelRetry to ask the model to retry with a hint.
    • -
    • Ordinary handler exceptions become failed tool results but are not retryable unless metadata says so.
    • -
    • Tool retry budgets are per tool name per run, not per individual call id.
    • -
    • after_tool_call hooks can rewrite canonical output or the structured envelope; retry control flow uses the pre-hook classification, while invalid non-strict mutations are rolled back.
    • -
    -
    -
    -
    - -
    -

    Providers and Sessions

    -

    - Provider classes own auth, base URL, HTTP client setup/cleanup, and request posting. - Model classes own static settings and create session objects. Session classes own mutable conversation state. -

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    ProviderTransport classModel classSession stateStructured-output default
    OpenAI ResponsesOpenAIProviderOpenAIResponsesModelLive previous_response_id chaining plus neutral transcript statenative: ask OpenAI directly for JSON-schema output.
    Anthropic MessagesAnthropicProviderAnthropicMessagesModelLive system/messages plus neutral transcript statetool: use the harness-created final_result tool because Anthropic native JSON-schema output is not supported here.
    OpenRouter chat completionsOpenRouterProviderOpenRouterModelLive chat messages plus neutral transcript statetool by default; explicit native mode is passed through as OpenRouter response_format.
    - -
    -

    - Resume state is intentionally provider-owned and strictly validated. It checks kind, version, model, - known fields, field types, unknown keys, and JSON serializability. It does not verify that tools or system - prompts match the original run; callers own that compatibility. -

    -
    - -
    - Provider payload conversion helpers worth knowing -
    -
      -
    • infer_model("provider:model") selects the adapter and creates provider/settings objects.
    • -
    • ModelNotice values are rendered with render_model_notices() and inserted into provider input by each session method; prompt starts/corrections append text, while tool continuations may add notice content after tool outputs.
    • -
    • _responses_tool_to_anthropic() and _responses_tool_to_chat() convert the common function-tool schema.
    • -
    • _extract_responses_tool_calls(), _extract_anthropic_tool_calls(), and _extract_chat_tool_calls() convert provider-specific tool calls into ModelToolCall objects.
    • -
    -
    -
    -
    - -
    -

    Lifecycle And Observability

    -

    - Hooks and tracing are runtime surfaces, not model-callable tools. The harness works with no registered hooks, - but the hook points are part of the run lifecycle. Tracing records the lifecycle through spans; it should not - change control flow. -

    -
    -
    -

    Hooks

    -

    - Hooks are runtime-only callables registered as Hook(event, handler, tools=None, agents=None). - Dispatch is synchronous and ordered, so earlier hooks can deterministically cancel or mutate context before - later hooks run. Tool filters only apply to tool events; agent filters only apply to subagent events. - Limit and retry logic is not implemented through hooks; hard limit events notify hooks after the runtime - detects the limit condition. -

    - - - - - - - - - - - - -
    EventCan cancel?Can mutate?Filter
    run_startNoNoNone
    user_prompt_submitYesAdd prompt contextNone
    before_tool_callYesNoTool name
    after_tool_callNoRewrite model-visible outputTool name
    before_subagent_runYesNoAgent name
    after_subagent_runNoNoAgent name
    limit_reachedNoNoNone
    run_endNoNoNone
    -
    -
    -

    Tracing

    -

    - RunTracer opens agent, model, and tool spans across every configured TracingOptions sink. - Local tracing is enabled by default for top-level harnesses unless THINHARNESS_DISABLE_LOCAL_TRACING disables it. - Trace attributes follow the OpenTelemetry GenAI semantic conventions used by the local tracing implementation. -

    -
      -
    • Local traces are JSONL span records under the configured trace directory.
    • -
    • OTLP tracing is optional through the tracing extra.
    • -
    • Span creation and content capture are separate: external tracing can record spans without recording prompts, tool args, or tool results unless those capture flags are enabled.
    • -
    -
    -
    -
    - -
    -

    Extras

    -

    - These surfaces are not required for ordinary harness runs. They still use the same Harness and - ToolSpec machinery, but applications can ignore them unless they need specialized search, - skills, delegation, MCP, or one-shot model fan-out. -

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    ExtraHow it enters the runBoundary / behavior
    JSONL searchFileTools.__init__ creates self.jsonl, and FileTools.specs() includes self.jsonl.spec().Specialized search over large JSONL content stores: ripgrep row prefiltering, field paths, equality/contains/regex/range filters, projection, field-level snippets, row limits, and truncation through FileTools spill behavior.
    SkillsSkillsPlugin(..., tools=[...]) discovers one catalog at construction and explicitly selects skill_read, skill_run, or both.Relative directories use the process cwd. Catalog metadata and summary stay fixed, while reads and scripts use live discovered files.
    SubagentsSubagentsPlugin contributes the subagent tool and passes an opaque recipe to a narrow core child host.The parent tool call awaits a fresh child. Safe plugins rebind through for_child(), direct tools come from the frozen run snapshot, and the child's disabled host blocks recursion.
    MCPConfigured server objects connect lazily and discover tools into live ToolSpec objects.MCP does not inherit into children. A named child lists an explicit MCPPlugin in its plugin configuration, then discovers tools through that child binding's connection lifecycle.
    Parallel LLMParallelLlmPlugin(...) contributes the standard text-only tool; custom renameable or structured batches use ParallelLlmTool(...).spec().The plugin borrows the harness model by default or uses explicit model ownership rules. Paths stay under the canonical harness root.
    -
    - -
    -

    Recommended Reading Path

    -
      -
    1. README.md for motivation and constraints: reusable loop primitives, small runtime surface, provider-agnostic, no shell by default.
    2. -
    3. thinharness/tools/base.py to understand ToolSpec, ToolResult, argument validation, retry envelopes, and path policy.
    4. -
    5. thinharness/providers.py through the common dataclasses/protocols, then skim each provider session.
    6. -
    7. thinharness/output.py so final-answer decisions make sense before reading the loop.
    8. -
    9. thinharness/core.py, thinharness/turns.py, and thinharness/runtime.py together. Read Harness.__init__, then Harness.run, then advance_until_terminal and RunContext.advance_model.
    10. -
    11. thinharness/tool_execution.py to see how a model-emitted batch becomes ordered provider outputs.
    12. -
    13. thinharness/tools/filesystem.py to understand workspace tools that FilesystemPlugin can expose.
    14. -
    15. thinharness/hooks.py and thinharness/tracing.py to understand lifecycle callbacks and run observability.
    16. -
    17. Pick extras as needed: thinharness/tools/jsonl.py, thinharness/plugins/subagents.py, thinharness/children.py, thinharness/tools/mcp.py, thinharness/tools/skills.py, and thinharness/tools/parallel_llm.py.
    18. -
    19. Use tests as executable documentation. Start with tests/test_harness.py, then the feature-specific test file for whatever you are changing.
    20. -
    -
    - -
    -

    Implementation Deep Dive

    -

    - This section is for code ownership: the details you need to answer why the code is organized this way, - where behavior lives, and which boundaries are deliberate. It focuses on the current implementation: - runtime ownership, tool execution, structured turn resolution, provider/session boundaries, tracing, - resume, subagents, MCP, search support, and parallel LLM. -

    - -
    - 1. Code Shape Primer: Pydantic, dataclasses, classes, functions, and protocols -
    -

    - ThinHarness uses different Python object types for different responsibilities. The short rule is: - serializable setup uses Pydantic, runtime records use dataclasses, long-lived state holders use ordinary - classes, and callable boundaries use functions or protocols. -

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    ShapeUsed forWhyExamples
    Pydantic modelCaller-owned configuration and typed argument/output validation.It validates user input, can produce JSON schema, and is reasonable to serialize or inspect.HarnessConfig, SubAgentConfig, tool args such as ReadArgs, structured output types.
    DataclassRuntime objects, provider-independent records, and small objects that carry a decision.These are Python-side values, often carrying callables, raw provider data, or mutable run state.ToolSpec, ToolResult, ModelTurn, RunUsage, hook contexts, OutputTurnDecision.
    Ordinary classObjects that own state, configuration, setup/cleanup, or a family of methods.The instance owns durable state; individual methods can still be exposed through small dataclasses such as ToolSpec.Harness, FileTools, provider classes, model/session classes, MCPServer, ParallelLlmTool.
    Callable/functionExecution hooks and model-callable tool handlers.The harness only needs something it can call after preparing context or arguments.ToolSpec.handler, hook handlers, nested handlers inside ParallelLlmTool.spec().
    ProtocolProvider-neutral interfaces.Different providers can implement the same required methods without inheriting from the same base class.Model, ModelSession, tracer-like objects accepted by tracing.
    -

    - Inheritance is intentionally light. Provider session classes implement the same session protocol, but the core - loop mostly uses composition: Harness owns a model, tool specs, plugin bindings, hooks, and tracing options. - Plugin and direct tool objects hand callable handlers to ordinary ToolSpec values. -

    -
    -
    - -
    - 2. Run Loop Ownership: core.py, turns.py, runtime.py, and tool_execution.py -
    -

    - The run loop is divided by ownership. core.py coordinates the public run, - turns.py owns the turn state machine that drives one run to a terminal result, - runtime.py owns one run's mutable state and the repeated wrapper around each provider call, and - tool_execution.py owns model-requested tool batches and the hook/tracing flow for one tool call. -

    - - - - - - - - - - - - - - - - - - - - - - - - -
    FileOwnsOwned elsewhere
    core.pyHarness construction, public run API, running/closed checks, provider session selection, per-run RequestConstants construction, and run-failure classification.turns.py owns the turn dispatch loop; providers.py owns provider API payload details; tool_execution.py owns one tool call's hook/tracing flow; runtime.py owns repeated provider-call wrapper code.
    runtime.pyOne run's mutable state: responses, usage, tool records, limit notices, terminal result/error, stop reason, resume attachment, run_end guard.providers.py owns provider-specific API request formats; tool_execution.py owns individual tool invocation mechanics.
    tool_execution.pyBatch execution policy, hook/tracing flow for one tool call, current tool-call context, output parsing, retry-kind capture.output.py owns structured final-answer validation; turns.py chooses whether the next provider call starts, continues with tools, retries output, or stops.
    turns.pyClassifies a returned ModelTurn as final, continue with tools, retry via user message, retry via tool output, or unexpected, and dispatches on that decision: it owns the loop that drives a run to a terminal result or approval pause.runtime.py owns retry-budget exhaustion; providers.py owns transport calls; core.py owns exception classification around the single machine call.
    -

    - The important handoff is turns.advance_until_terminal(...) plus RunContext.advance_model(...). - The turn machine chooses whether to start, resume, send tool outputs, or send a correction, and - supplies the matching provider-session method call with the run's RequestConstants. The run-loop - diagram may label the second case as "continue" for space, but the code path is continue_with_tools(...). - runtime.py wraps that callable with limit checks, usage accounting, - model tracing, notice computation, and output turn resolution. -

    -

    - This keeps provider API payload knowledge in providers.py while keeping the repeated provider-call wrapper - in one place. -

    -
    -
    - -
    - 3. Approval Pause Internals: from pending tool call to resumed batch -
    -

    - Approval-required tools are a loop primitive rather than a special model output format. A custom - ToolSpec can set requires_approval=True. When a model turn asks for any such - tool, core.py pauses before the batch reaches ToolBatchExecutor, so neither the - approval-required call nor any normal sibling call has side effects yet. -

    - - - - - - - - - - - - - - - - - - - - -
    PieceRole in approval flow
    approvals.pyBuilds and validates the approval_pause envelope, restores usage/history, and validates that host decisions exactly cover approval-required call ids.
    RunContext.pause_for_approval()Captures provider resume payload, pending tool batch, usage, responses, tool records, emitted limit-warning keys, and metadata before emitting RunCompletedEvent.
    Harness.resume_approvals()Restores the logical run, resumes the provider session, emits ApprovalResumedEvent, validates the approval-required tools still exist, and processes the paused batch.
    ToolBatchExecutorRuns approved calls and normal sibling calls through the standard hook, tracing, retry, and output-ordering machinery.
    -

    - Rejected calls bypass tool hooks and do not execute. They still produce ordered tool outputs for the provider: - a failed ToolResult with error_type="ApprovalRejected". From the model's perspective, - it requested tools and then received tool results on the next turn; it never sees the host pause itself. -

    -

    - The paused batch counts against usage.tool_calls at pause time and is not counted again during - resume. The post-resume result contains the whole logical run history, not only the second half of the run. - This is why approval envelopes are larger than plain resume state: they carry provider transcript state, prior responses, and - accounting as well as the provider checkpoint. -

    -
    -
    - -
    - 4. Tool Execution Internals: from model tool call to provider follow-up -
    -

    - A provider returns one ModelTurn. That object has text, a tool_calls list, and raw - provider JSON. The tool_calls field is a list[ModelToolCall], where each ModelToolCall - has the provider call id, tool name, and raw JSON argument string. -

    -

    - When resolve_turn_output() decides the turn should continue with ordinary tools, the harness executes - the whole model-emitted batch and sends one ordered set of ToolOutput values back through - continue_with_tools(...). -

    -
      -
    • ToolBatchExecutor chooses serial execution if any requested tool is marked sequential=True; otherwise it can run calls concurrently.
    • -
    • ToolCallExecutor sets the current tool-call context, then runs before_tool_call hooks that may cancel the call.
    • -
    • The executor parses JSON arguments, validates them with Pydantic when a model exists, and calls the async handler directly or the sync handler in a worker thread.
    • -
    • Handler output is normalized into the standard {"ok", "content", "metadata"} tool envelope.
    • -
    • Retry metadata is captured before after_tool_call hooks can rewrite the output text the model sees.
    • -
    • Tracing records one tool span per call, while tool_call_records and provider-facing ToolOutput values preserve the original model order.
    • -
    -

    - These are the main branches after a model asks for a tool call. They determine whether ThinHarness asks the - model to repair its request, reports a normal tool failure, or continues the provider conversation. -

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    CaseWhat happensWhy it matters
    Bad JSON or bad Pydantic argsReturns a failed tool envelope with metadata.retry=true.The model passed arguments in a fixable bad format, so the harness can ask it to retry.
    Handler raises ModelRetryReturns a retryable envelope with the handler's hint.Tool code can classify a domain-level mistake as model-repairable.
    Handler raises an ordinary exceptionReturns ok=false without metadata.retry=true.The model can still call tools later, but this result does not ask it to repair the same call and does not increment the tool retry budget.
    after_tool_call rewrites outputThe model sees the rewritten text, but retry control flow was captured first.Hooks own presentation; the harness owns retry budget accounting.
    Any called tool has sequential=TrueThe whole current batch runs serially in model order.Mutating tools avoid race conditions without partitioning the batch into smaller dependency groups.
    -
    -
    - -
    - 5. Provider and Session Handoff: reusable models, fresh sessions, common turns -
    -

    - Provider adapters are deliberately layered. Provider transport classes own auth, base URL, timeout, and HTTP - client setup/cleanup. Model classes own reusable static settings. Session classes own mutable conversation state. -

    -

    - In plain terms: a Model is the reusable object you pass into Harness(..., model=...). It knows - which provider/model/settings to use, but it should not hold the live conversation transcript. For each - Harness.run(...), it creates a fresh ModelSession; that session is the object the harness - actually talks to during the run. -

    - - - - - - - - - - - - - - - - - - - - - - - - -
    LayerMutable?Responsibility
    Provider transportHTTP client setup/cleanup onlyPost HTTP requests, wrap HTTP/transport errors, close owned clients.
    ModelNo provider transcriptThe reusable object passed to Harness. It creates a fresh session with new_session(), or a resumed session with resume_session(...) when resume is supported.
    ModelSessionYesThe per-run conversation object. The turn machine calls its start(...), continue_with_tools(...), and continue_with_user_content(...) methods with per-run RequestConstants, plus dump_state().
    ModelTurnNoCommon result object: final text extracted from the response, requested tool calls, and raw provider JSON.
    -

    - The core loop should not know OpenAI Responses, Anthropic Messages, or OpenRouter Chat Completions API formats. - It receives a ModelTurn and applies the same output resolution, tool execution, tracing, hook, limit, - and retry logic regardless of provider. -

    -

    - Provider-specific translation stays in providers.py: common function tools become Anthropic - input_schema tools or Chat Completions function tools; native structured output requests become - provider-specific API fields; notice text is rendered into each provider's input format. -

    -
    -
    - -
    - 6. Structured Output: one resolver, several delivery strategies -
    -

    - Structured output is not bolted onto every provider separately. The harness builds an OutputSchema - from the caller's output spec, then providers translate the provider-native request details when native mode is used. - resolve_turn_output() decides what one returned ModelTurn means. -

    - - - - - - - - - - - - - - - - - - - - - - - - -
    ModeHow the model is guidedHow the final result is recognized
    textNo schema payload and no harness-created tool.Final assistant text populates both text and output.
    nativeThe provider API is asked directly for JSON-schema output.A final native-output turn has no ordinary tool calls. Earlier turns may still request ordinary tools; final assistant text is parsed and validated through Pydantic.
    toolA harness-created final_result function tool is exposed to the provider.Exactly one final_result call, with no sibling tool calls in that same turn, completes the run and builds the final result.
    promptedThe harness appends JSON-schema instructions to the prompt/instructions instead of using provider-native schema output or final_result.A final prompted-output turn has no ordinary tool calls. Earlier turns may still request ordinary tools; final assistant text is parsed and validated.
    -

    - final_result is harness-created, not a normal user tool. It is reserved only when structured tool mode is active, - is not exposed in self.tools, does not fire tool hooks, does not count as usage.tool_calls, - and makes the exit non-resumable because the provider transcript would contain an unanswered synthetic tool call. - Native and prompted structured-output exits can still be resumable when the provider session can dump clean resume state. -

    -

    - Non-object output schemas such as lists are wrapped under a single value argument for tool mode because - function tool arguments must be JSON objects. Validation uses Pydantic's public TypeAdapter API and - local schema cleanup. The local implementation is intentionally narrower than Pydantic AI: it does not stream - partial structured objects, treat a Python function signature as the output schema, or run custom output validator hooks. -

    -

    - Invalid structured output creates corrective model requests until output_retries is exhausted. - Retry-budget exhaustion is handled by the caller of the resolver: Harness.run() turns it into a run failure, while - ParallelLlmTool turns it into a per-entry failure. -

    -
    -
    - -
    - 7. State, limits, retry budgets, and resume -
    -

    - Harness is reusable setup; RunContext is one invocation's state. That distinction is - the main reason repeated runs on the same harness do not leak responses, usage, limit warnings, or resume bookkeeping. -

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    ConceptWhere it livesImportant detail
    Model request limitRunContext and RunUsage.model_requestsCounts provider calls, including corrective structured-output requests after the limit check allows them.
    Tool call limitRunContext.check_tool_limit and RunUsage.tool_callsCounts model-requested ordinary tool calls before execution, so hook-blocked calls still count.
    Cancelled tool callsRunUsage.cancelled_tool_callsTracked separately after before_tool_call hooks cancel calls; cancellation does not erase the requested-call count.
    Tool retry budgetRunUsage.tool_retries, a dict keyed by tool name and updated by RunContext.check_tool_retry_limits(...)Two calls to the same tool share the same retry counter. This is coarse by design.
    Output retry budgetBudget: HarnessConfig.output_retries. Current count: RunUsage.output_retries.Counts corrective model requests after invalid structured output, not total validation attempts. RunContext.retry_or_fail() checks the budget before the corrective request is made.
    Near-limit notices_compute_limit_notices(...) returns provider-facing ModelNotice values; RunContext.emitted_limit_warnings records which warning thresholds already went out.Computed from the current RunUsage.model_requests and RunUsage.tool_calls before the next provider request. A warning for the same budget threshold is sent once per run, not repeated on every later provider call.
    Resume stateProvider-agnostic transcript state copied into HarnessResult.resume_state while building the final result.Built-in providers emit kind="transcript", version=3, origin diagnostics, and neutral user/assistant/tool entries. Callers can store and pass it back, but should not edit or construct it.
    Approval pause stateHarness-level approval_pause envelope copied into HarnessResult.resume_state when stop_reason="approval_required".Wraps provider transcript state plus pending batch, run history, usage, emitted limit-warning keys, and metadata. It must be resumed with resume_approvals(), not resume_from.
    -

    - resume_from starts a new turn from a previous clean result. It is not a failed-request retry, - interrupted-tool continuation, or transcript repair mechanism. Errors, cancellation, limit exits, tool retry - exhaustion, output validation failure, unexpected model behavior, and tool-mode final_result exits - intentionally produce no checkpoint. -

    -

    - Resume also carries model reasoning. Each built-in session keeps the provider's native reasoning parts in the - neutral transcript, so resuming on the same provider replays them verbatim — Anthropic signed thinking - blocks, OpenAI encrypted_content, OpenRouter reasoning_details. An opaque blob cannot - be replayed to a different provider, so cross-provider resume degrades every reasoning part to a leading - <thinking>-tagged text block and drops the blob. Native re-emit also requires the resuming - request to be able to accept the block: -

    - - - - - - - - - - - - - - - - - - - -
    ProviderNative reasoning in resume stateRe-emits natively only when
    OpenAI Responsesencrypted_content, captured via include=["reasoning.encrypted_content"] on reasoning-capable modelsthe resuming model is reasoning-capable; otherwise the text fallback is used
    Anthropic Messagessigned thinking / redacted_thinking blocksextended thinking is enabled on the resuming run; otherwise the text fallback is used
    OpenRouter chat completionsreasoning_detailsresuming on OpenRouter — no additional capability gate
    -

    - Because resume_state can therefore hold encrypted reasoning blobs and signed thinking, treat it as - sensitive, like the local traces it mirrors. -

    -
    -
    - -
    - 8. Extras Internals: JSONL search, skills, subagents, MCP, and parallel LLM -
    -

    - Extras are specialized capabilities that an application can ignore unless it uses that feature. They still - enter the run through existing ToolSpec, Harness, and provider-session boundaries. -

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    ExtraWhere responsibility changes handsBoundary / behavior
    JSONL searchFileTools owns the JsonlSearch instance and exposes its spec with the filesystem built-ins.search_support.py holds shared ripgrep parsing, glob validation, containment filtering, and search-root helpers used by both text search and JSONL search; jsonl.py owns structured field projection, range filters, and field snippet rendering.
    SubagentsSubagentsPlugin owns the delegation tool and child recipes; children.py implements the narrow host.Child runs start fresh, safe plugins rebind, direct tools use the frozen parent-run snapshot, and a disabled child host blocks recursion.
    MCPConfigured server objects connect lazily and discover tools into live ToolSpec objects.MCP does not inherit automatically. A child lists an explicit MCPPlugin and discovers tools in its own harness lifecycle.
    SkillsSkillsPlugin owns constructor-time discovery and ordered tool selection over SkillRegistry.The catalog and summary are static, discovered file content stays live, and inherited children rebind the exact parent plugin temporarily.
    Parallel LLMParallelLlmPlugin composes a normal text-only ToolSpec over independent one-shot model calls.It borrows a harness or caller model, or owns per-call providers for model strings. Custom ParallelLlmTool can rename the tool or opt into structured output.
    -
    -
    -
    - -
    -

    Part of the ThinHarness documentation site · last updated 2026-06-05. The source focus is current project runtime and docs; generated caches, build outputs, and vendor reference checkouts are treated as non-runtime material.

    -
    -
    - - - - diff --git a/docs/site/index.html b/docs/site/index.html deleted file mode 100644 index 5ab76f8..0000000 --- a/docs/site/index.html +++ /dev/null @@ -1,88 +0,0 @@ - - - - - -ThinHarness — a minimal, opinionated agent harness - - - - - - - -
    -
    -
    -
    v0.4 · pre-1.0 · MIT licensed ·
    -

    A minimal, opinionated agent harness.

    -

    Build agents without adopting a stack you don’t need. Focused scope, straightforward code, easy to fork.

    - -
    -
    -
    -
    install.sh
    -
    copy$ uv add thinharness
    # or: pip install thinharness
    # requires python 3.11+

    resolved · 24 files · 8,035 LOC
    -
    -
    -
    - -
    -
    // 01

    Opinions

    the reason it stays small
    -
    -
    purpose_built

    Purpose-built agents

    ThinHarness is for bounded agent loops inside software you control, not open-ended interactive assistants.

    -
    no_bash

    No bash by default

    Bash stays out of the default tools. Add BashPlugin explicitly for bounded exploratory commands before typed tools.

    -
    search

    Search is a top priority

    Ripgrep exposed as compact grouped results, tuned for documents and business workflows — plus a custom JSONL search tool for structured corpuses.

    -
    parallel_llm

    Parallel LLM calls, built in

    Add ParallelLlmPlugin to fan out independent prompts with the harness model, or configure a separate batch model.

    -
    no_compaction

    No compaction

    Compaction makes sense for sprawling coding sessions. For business agents the fix is smarter task decomposition and context management

    -
    no_deployment

    No deployment layer

    Serving, auth, durable jobs, and session storage stay yours. ThinHarness owns the agent loop, not the production stack around it.

    -
    -
    - -
    -
    // 02

    Explore

    - -
    -
    - -
    - -
    - - - - diff --git a/pyproject.toml b/pyproject.toml index 0ad22a8..8a4dfab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "thinharness" -version = "0.6.0" +version = "0.7.0" description = "Minimal plugin-based agent harness with provider-backed Responses-like models." readme = "README.md" requires-python = ">=3.11" @@ -92,6 +92,3 @@ dev = [ "pytest-cov>=7", "ruff>=0.14", ] -docs = [ - "markdown>=3.6", -] diff --git a/scripts/build_site.py b/scripts/build_site.py deleted file mode 100644 index 44715ff..0000000 --- a/scripts/build_site.py +++ /dev/null @@ -1,420 +0,0 @@ -from __future__ import annotations - -import argparse -import difflib -import html -import re -from html.parser import HTMLParser -from pathlib import Path - -REPO_ROOT = Path(__file__).resolve().parents[1] -README = REPO_ROOT / "README.md" -SITE_DIR = REPO_ROOT / "docs" / "site" -ABOUT = SITE_DIR / "about" / "index.html" -GITHUB_ROOT = "https://github.com/ryanbbrown/thinharness" - - -def section(markdown: str, title: str) -> str: - pattern = re.compile(rf"^## {re.escape(title)}\n(?P.*?)(?=^## |\Z)", re.M | re.S) - match = pattern.search(markdown) - if not match: - raise ValueError(f"README section not found: {title}") - return match.group("body").strip() - - -def inline_markdown(text: str) -> str: - placeholders: list[str] = [] - - def hold(value: str) -> str: - placeholders.append(value) - return f"\0{len(placeholders) - 1}\0" - - text = re.sub(r"`([^`]+)`", lambda m: hold(f"{html.escape(m.group(1))}"), text) - text = re.sub(r"\[([^\]]+)\]\([^)]+\)", lambda m: m.group(1), text) - escaped = html.escape(text, quote=False) - escaped = re.sub(r"\*\*([^*]+)\*\*", r"\1", escaped) - escaped = re.sub(r"\*([^*]+)\*", r"\1", escaped) - for index, value in enumerate(placeholders): - escaped = escaped.replace(f"\0{index}\0", value) - return escaped - - -def paragraphs(markdown: str) -> list[str]: - return [block.replace("\n", " ") for block in markdown.split("\n\n") if block.strip()] - - -def slug_for_opinion(title: str) -> str: - known_tags = { - "Purpose-built agents, not universal agents": "purpose_built", - "No bash by default": "no_bash", - "Search is a top priority": "search", - "Parallel LLM calls, built in": "parallel_llm", - "Three providers, no matrix": "providers", - "No compaction": "no_compaction", - "No deployment layer": "no_deployment", - } - if title in known_tags: - return known_tags[title] - return re.sub(r"[^a-z0-9]+", "_", title.lower()).strip("_") or "opinion" - - -def opinions_from_readme(markdown: str) -> list[tuple[str, str, str]]: - body = section(markdown, "Opinions") - result: list[tuple[str, str, str]] = [] - for match in re.finditer(r"\*\*([^*]+?)\.\*\* ([\s\S]*?)(?=\n\n\*\*|\Z)", body): - title = match.group(1) - text = match.group(2).strip().replace("\n", " ") - if title == "Three providers, no matrix" and not text.endswith("."): - text += "." - result.append((slug_for_opinion(title), title, inline_markdown(text))) - return result - - -def features_from_readme(markdown: str) -> list[tuple[str, str]]: - body = section(markdown, "Features") - result: list[tuple[str, str]] = [] - for match in re.finditer(r"^- \*\*([^*]+):\*\* (.+)$", body, re.M): - title, text = match.groups() - if title == "Limit notices": - text = text.replace( - "Notices are harness-owned model input, not hooks or callbacks; parent and child runs compute them from their own local budgets.", - "Harness-owned model input — parent and child runs compute them from their own local budgets.", - ) - result.append((title, sentence_case(inline_markdown(text)))) - return result - - -def sentence_case(text: str) -> str: - if not text: - return text - return text[0].upper() + text[1:] - - -class TableParser(HTMLParser): - def __init__(self) -> None: - super().__init__(convert_charrefs=True) - self.rows: list[list[dict[str, str]]] = [] - self._row: list[dict[str, str]] | None = None - self._cell: dict[str, str] | None = None - self._text: list[str] = [] - self._sup: list[str] = [] - self._in_sup = False - - def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - if tag == "tr": - self._row = [] - elif tag in {"td", "th"} and self._row is not None: - self._cell = {"text": "", "img": "", "sup": ""} - self._text = [] - self._sup = [] - self._in_sup = False - elif tag == "img" and self._cell is not None: - attrs_dict = dict(attrs) - self._cell["img"] = attrs_dict.get("src") or "" - elif tag == "sup" and self._cell is not None: - self._in_sup = True - elif tag == "br" and self._cell is not None: - self._text.append(" ") - - def handle_endtag(self, tag: str) -> None: - if tag == "sup" and self._cell is not None: - self._in_sup = False - elif tag in {"td", "th"} and self._cell is not None and self._row is not None: - self._cell["text"] = normalize_table_text("".join(self._text)) - self._cell["sup"] = normalize_table_text("".join(self._sup)) - self._row.append(self._cell) - self._cell = None - elif tag == "tr" and self._row is not None: - if self._row: - self.rows.append(self._row) - self._row = None - - def handle_data(self, data: str) -> None: - if self._cell is None: - return - if self._in_sup: - self._sup.append(data) - else: - self._text.append(data) - - -def normalize_table_text(text: str) -> str: - return re.sub(r"\s+", " ", text.replace("\xa0", " ")).strip() - - -def readme_table_rows(markdown: str) -> list[list[dict[str, str]]]: - table_match = re.search(r".*?
    ", markdown, re.S) - if not table_match: - raise ValueError("README comparison table not found") - parser = TableParser() - parser.feed(table_match.group(0)) - return parser.rows - - -def mark(value: str) -> str: - return { - "✅": '', - "❌": '', - "⚠️": '', - }[value] - - -def library_cell(cell: dict[str, str], asset_prefix: str = "assets/") -> str: - name = cell["text"] - superscript = f"{html.escape(cell['sup'])}" if cell.get("sup") else "" - display_name = html.escape(name) - if name == "ThinHarness": - return f'
    ThinHarness
    ' - if name == "Agno": - return f'
    Agno
    ' - img = cell["img"] - img_src = img if re.match(r"https?://", img) else f"{asset_prefix}{img}" - return f'
    {display_name}{superscript}
    ' - - -def comparison_table(markdown: str, asset_prefix: str = "assets/") -> str: - rows = readme_table_rows(markdown) - body_rows = [] - for row in rows[1:]: - library = row[0] - loc = row[1] - loc_sup = f'{html.escape(loc["sup"])}' if loc.get("sup") else "" - css = ' class="me"' if library["text"] == "ThinHarness" else "" - marks = "".join(f"{mark(cell['text'])}" for cell in row[2:]) - body_rows.append( - f""" - {library_cell(library, asset_prefix)} - {html.escape(loc["text"])}{loc_sup} - {marks} - """ - ) - return "\n".join(body_rows) - - -def code_highlight(code: str) -> str: - escaped = html.escape(code) - escaped = re.sub(r"\b(import|from|async def|async with|as|await)\b", r'\1', escaped) - escaped = re.sub(r'("[^&]+?")', r'\1', escaped) - return escaped.replace(""", '"') - - -def fenced_code(markdown: str, language: str) -> str: - match = re.search(rf"```{language}\n(.*?)\n```", markdown, re.S) - if not match: - raise ValueError(f"{language} code block not found") - return match.group(1) - - -def render_about(markdown: str) -> str: - asset_prefix = "../assets/" - why = paragraphs(section(markdown, "Why this exists").split("