From fe7a7de5874ffbc4ba290e97aae6a863205dd4f0 Mon Sep 17 00:00:00 2001 From: cc Date: Thu, 4 Jun 2026 13:41:32 -0700 Subject: [PATCH 01/15] spec: openspec init --- .gitignore | 2 + docs/coding-agents/index.md | 1 + docs/development/openspec.md | 102 ++++++++++++++ docs/docs.json | 1 + openspec/config.yaml | 45 ++++++ openspec/schemas/dimos-capability/schema.yaml | 128 ++++++++++++++++++ .../dimos-capability/templates/design.md | 35 +++++ .../dimos-capability/templates/docs.md | 19 +++ .../dimos-capability/templates/proposal.md | 32 +++++ .../dimos-capability/templates/spec.md | 16 +++ .../dimos-capability/templates/tasks.md | 15 ++ 11 files changed, 396 insertions(+) create mode 100644 docs/development/openspec.md create mode 100644 openspec/config.yaml create mode 100644 openspec/schemas/dimos-capability/schema.yaml create mode 100644 openspec/schemas/dimos-capability/templates/design.md create mode 100644 openspec/schemas/dimos-capability/templates/docs.md create mode 100644 openspec/schemas/dimos-capability/templates/proposal.md create mode 100644 openspec/schemas/dimos-capability/templates/spec.md create mode 100644 openspec/schemas/dimos-capability/templates/tasks.md diff --git a/.gitignore b/.gitignore index 42bdddfa45..787163e787 100644 --- a/.gitignore +++ b/.gitignore @@ -63,8 +63,10 @@ yolo11n.pt # symlink one of .envrc.* if you'd like to use .envrc .claude +.opencode/ **/CLAUDE.md .direnv/ +.omo/ /logs diff --git a/docs/coding-agents/index.md b/docs/coding-agents/index.md index ff778ac5cf..5ac7c854a7 100644 --- a/docs/coding-agents/index.md +++ b/docs/coding-agents/index.md @@ -3,6 +3,7 @@ ├── worktrees.md (creating provisioned worktrees with `bin/worktree`) ├── style.md (code style guidelines for dimos) ├── testing.md (docs about writing tests) +├── ../development/openspec.md (OpenSpec behavior-spec workflow) ├── docs (these are docs about writing docs) │   ├── codeblocks.md │   ├── doclinks.md diff --git a/docs/development/openspec.md b/docs/development/openspec.md new file mode 100644 index 0000000000..280eb0f57e --- /dev/null +++ b/docs/development/openspec.md @@ -0,0 +1,102 @@ +# OpenSpec Workflow + +DimOS uses OpenSpec as the checked-in planning layer for behavior changes. OpenSpec artifacts live under `openspec/` and should describe what the system is supposed to do, why it is changing, and how contributors or agents should validate the work. + +## Terminology + +Keep these two meanings separate: + +- **OpenSpec capability spec**: Markdown requirements under `openspec/specs//spec.md`. These describe observable behavior and acceptance scenarios. +- **DimOS Spec**: Python Protocol/RPC contracts in files like `dimos/navigation/navigation_spec.py` or `dimos/manipulation/control/arm_driver_spec.py`. These describe module interfaces for code wiring. + +Use "OpenSpec capability spec" in prose when there is any chance of confusion. + +## Schema + +The project uses the `dimos-capability` schema configured in `openspec/config.yaml`. + +The artifact flow is: + +```text +proposal + ├── specs + ├── design + └── docs + └── tasks +``` + +| Artifact | Purpose | +|---|---| +| `proposal.md` | Intent, scope, affected DimOS surfaces, and capability impact. | +| `specs//spec.md` | Behavior-first requirements and scenarios. | +| `design.md` | Module, stream, blueprint, skill/MCP, safety, and rollout decisions. | +| `docs.md` | Documentation impact and doc validation plan. | +| `tasks.md` | Implementation, docs, verification, and manual QA checklist. | + +## When to create a change + +Create an OpenSpec change when work changes observable behavior, public CLI/API/MCP behavior, robot behavior, hardware/simulation/replay workflows, docs that users rely on, or cross-module architecture. + +Do not create a change for a purely mechanical refactor, typo fix, or internal cleanup unless it changes behavior or needs cross-session planning context. + +## Writing specs + +OpenSpec capability specs are behavior contracts, not implementation plans. + +Good spec content: + +- User- or developer-visible behavior. +- Public CLI/API/MCP tool behavior. +- Stream or message behavior that downstream modules rely on. +- Robot safety constraints and hardware/simulation/replay expectations. +- Scenarios that can be tested or manually verified. + +Avoid in specs: + +- Private class/function names. +- Generated-file mechanics. +- Library choices and wiring details. +- Step-by-step implementation tasks. + +Put those details in `design.md` or `tasks.md`. + +## Capability names + +Prefer behavior-domain names over code names. Useful starting points: + +- `module-system` +- `blueprint-composition` +- `cli-lifecycle` +- `agent-skills-mcp` +- `configuration` +- `navigation-stack` +- `manipulation-stack` +- `hardware-adapters` +- `simulation-replay` +- `documentation-system` + +Add specs progressively as changes need them. Do not try to backfill the whole project at once. + +## Validation + +Use OpenSpec validation before implementation and before archiving: + +```bash skip +openspec schema validate dimos-capability +openspec validate +openspec templates --json +``` + +For documentation changes, also run the relevant doc checks from [Writing Docs](/docs/development/writing_docs.md): + +```bash skip +md-babel-py run +``` + +When a change touches blueprint names, module-level blueprint variables, or module registry inputs, run: + +```bash skip +pytest dimos/robot/test_all_blueprints_generation.py +``` + +Then run focused tests for the changed code and manually QA through the actual surface: CLI command, MCP tool, HTTP API, simulation/replay blueprint, hardware procedure, or library driver. diff --git a/docs/docs.json b/docs/docs.json index 58da2ff6a1..f0064c9ab9 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -144,6 +144,7 @@ "group": "Development", "pages": [ "development/conventions", + "development/openspec", "development/testing", "development/docker", "development/grid_testing", diff --git a/openspec/config.yaml b/openspec/config.yaml new file mode 100644 index 0000000000..62a72bba63 --- /dev/null +++ b/openspec/config.yaml @@ -0,0 +1,45 @@ +schema: dimos-capability + +context: | + DimOS is a robotics operating system for generalist robots. Modules communicate + through typed streams (`In[T]`, `Out[T]`) over LCM, SHM, ROS, DDS, or other + transports. Blueprints compose modules into runnable robot stacks. Skills are + `@skill`-annotated RPC methods exposed to agents and MCP clients. + + Terminology boundary: + - "OpenSpec spec" means a behavior specification under `openspec/specs/`. + - "DimOS Spec" means a Python Protocol/RPC contract in `*_spec.py` files, + usually inheriting `dimos.spec.utils.Spec` and `typing.Protocol`. + Keep these separate. OpenSpec specs describe observable behavior; DimOS Specs + describe code-level module interfaces. + + OpenSpec specs should capture current behavior, user/developer-visible + outcomes, public CLI/API/tool surfaces, robot safety constraints, and testable + scenarios. Put implementation choices, class names, module wiring, generated + registry updates, and rollout details in `design.md` or `tasks.md`. + + Documentation lives in: + - `docs/usage/` for user-facing concepts and APIs. + - `docs/capabilities/` for capability and platform guides. + - `docs/development/` for contributor process. + - `docs/coding-agents/` and `AGENTS.md` for coding-agent guidance. + +rules: + proposal: + - "Identify affected DimOS surfaces: modules, streams, blueprints, CLI, skills/MCP, docs, hardware, simulation, replay, or generated registries." + - Use capability names that match behavior domains, not Python class names. + - Mark hardware safety or public API/CLI changes explicitly. + specs: + - Write behavior-first requirements; avoid implementation detail unless it is externally observable. + - Every requirement must include at least one `#### Scenario:` block with concrete observable outcomes. + - Use "OpenSpec capability spec" when prose might otherwise be confused with DimOS Python `Spec` Protocols. + design: + - Call out DimOS `Spec` Protocols, adapter Protocols, blueprint composition, stream names/types, and skill/MCP exposure when relevant. + - Mention generated files and required regeneration commands, especially `pytest dimos/robot/test_all_blueprints_generation.py` for blueprint registry changes. + - Include hardware/simulation/replay assumptions and safety constraints for robot-facing work. + docs: + - List user-facing docs, contributor docs, coding-agent docs, and AGENTS.md updates required by the change. + - Include documentation validation commands for changed docs, such as `doclinks` and `md-babel-py run ` where applicable. + tasks: + - Include verification tasks for OpenSpec validation, relevant pytest targets, type checks when needed, and manual QA through the user-facing surface. + - Add registry generation tasks when blueprint names, module classes, or generated registry inputs change. diff --git a/openspec/schemas/dimos-capability/schema.yaml b/openspec/schemas/dimos-capability/schema.yaml new file mode 100644 index 0000000000..fedb7964ee --- /dev/null +++ b/openspec/schemas/dimos-capability/schema.yaml @@ -0,0 +1,128 @@ +name: dimos-capability +version: 1 +description: DimOS capability workflow - proposal → specs/design/docs → tasks +artifacts: + - id: proposal + generates: proposal.md + description: DimOS change proposal covering intent, scope, capability impact, and affected robot/software surfaces + template: proposal.md + instruction: | + Create the proposal document that establishes WHY this change is needed and what DimOS behavior it affects. + + Sections: + - **Why**: 1-2 concise paragraphs on the problem or opportunity. Explain why the change matters now. + - **What Changes**: Bullet list of added, modified, or removed behavior. Mark public API/CLI or hardware-safety breaking changes with **BREAKING**. + - **Affected DimOS Surfaces**: Identify modules, streams, blueprints, CLI commands, skills/MCP tools, docs, hardware, simulation, replay, generated registries, or external protocols touched by the change. + - **Capabilities**: Identify which OpenSpec capability specs will be created or modified: + - **New Capabilities**: List behavior domains introduced by the change. Each becomes `specs//spec.md`. Use kebab-case names (for example, `agent-skills-mcp`, `blueprint-composition`, `manipulation-stack`). + - **Modified Capabilities**: List existing `openspec/specs//` entries whose requirements change. Only include spec-level behavior changes, not implementation-only refactors. + - **Impact**: Summarize user/developer impact, compatibility risks, dependency changes, documentation updates, and test/QA scope. + + Keep proposals concise. Do not include line-by-line implementation details; put architecture and rollout decisions in `design.md`. + requires: [] + - id: specs + generates: specs/**/*.md + description: Behavior-first OpenSpec capability delta specifications + template: spec.md + instruction: | + Create OpenSpec capability specs that define WHAT DimOS should do, not how it is implemented. + + Create one delta spec file per capability listed in proposal.md: + - New capabilities: use `specs//spec.md` with the exact kebab-case name from the proposal. + - Modified capabilities: use the existing folder from `openspec/specs//`. + + Use these delta sections as `##` headers: + - **ADDED Requirements**: New externally observable behavior. + - **MODIFIED Requirements**: Changed behavior. Include the full updated requirement block, not a partial patch. + - **REMOVED Requirements**: Deprecated behavior. Include **Reason** and **Migration**. + - **RENAMED Requirements**: Name-only changes. Use FROM:/TO: format. + + Requirement format: + - Use `### Requirement: `. + - Use SHALL/MUST for normative requirements. + - Include at least one `#### Scenario: ` per requirement. Scenario headings MUST use exactly four `#` characters. + - Prefer `- **GIVEN**`, `- **WHEN**`, `- **THEN**`, and `- **AND**` bullets. + - Cover happy path plus meaningful edge/error/safety cases. + + DimOS-specific guidance: + - Specify user/developer-visible behavior, robot outcomes, CLI behavior, skill/MCP tool behavior, stream contracts, safety constraints, and compatibility expectations. + - Avoid Python class names, private module internals, transport implementation choices, and generated-file details unless those details are observable API contracts. + - Use "OpenSpec capability spec" in prose when needed to avoid confusion with DimOS Python `Spec` Protocols. + - If the behavior only changes implementation and not observable requirements, do not create a spec delta. + requires: + - proposal + - id: design + generates: design.md + description: DimOS technical design and architecture decisions + template: design.md + instruction: | + Create the design document that explains HOW the change should be implemented in DimOS. + + Include design.md for cross-module changes, new robot/hardware integration, new public interfaces, new dependencies, safety-sensitive behavior, generated registry changes, or unclear architecture. + + Sections: + - **Context**: Current state, relevant modules/blueprints/docs, and constraints. + - **Goals / Non-Goals**: What the design achieves and explicitly excludes. + - **DimOS Architecture**: Modules, streams, transports, blueprints, RPC/module refs, DimOS `Spec` Protocols, adapter Protocols, skills/MCP exposure, CLI entry points, and generated registries involved. + - **Decisions**: Key choices with rationale and alternatives considered. + - **Safety / Simulation / Replay**: Hardware assumptions, sim/replay behavior, safety constraints, and manual QA surface. + - **Risks / Trade-offs**: Known risks and mitigations. + - **Migration / Rollout**: Compatibility, generated files, docs, and deployment steps. + - **Open Questions**: Outstanding decisions or unknowns. + + Reference proposal.md for intent and specs for behavior. Keep line-by-line work in tasks.md. + requires: + - proposal + - id: docs + generates: docs.md + description: Documentation impact plan for user, contributor, and coding-agent docs + template: docs.md + instruction: | + Create the documentation impact plan for the change. + + Sections: + - **User-Facing Docs**: Updates under `docs/usage/`, `docs/capabilities/`, `docs/platforms/`, or README files. + - **Contributor Docs**: Updates under `docs/development/`. + - **Coding-Agent Docs**: Updates under `docs/coding-agents/` or `AGENTS.md`. + - **Doc Validation**: Commands needed for changed docs, such as `doclinks`, `md-babel-py run `, and `bin/gen-diagrams`. + - **No Docs Needed**: If no docs are needed, explain why. + + Match `docs/development/writing_docs.md`: contributor-only docs belong in `docs/development`; user-facing behavior belongs in `docs/usage` or `docs/capabilities`. + requires: + - proposal + - id: tasks + generates: tasks.md + description: Implementation, validation, docs, and manual-QA checklist + template: tasks.md + instruction: | + Create the implementation checklist. The apply phase parses checkbox format, so every actionable task MUST use `- [ ]`. + + Guidelines: + - Group tasks under numbered `##` headings. + - Each task must be `- [ ] X.Y Task description`. + - Keep tasks small enough to complete in one focused session. + - Order tasks by dependency. + - Include docs and validation tasks from docs.md. + - Include generated registry tasks when blueprints or module registry inputs change. + - Include manual QA through the actual user surface: CLI, TUI, HTTP API, MCP tool, simulation/replay blueprint, hardware procedure, or library driver. + + Typical DimOS validation tasks: + - Run `openspec validate `. + - Run focused pytest targets for changed modules. + - Run `pytest dimos/robot/test_all_blueprints_generation.py` when blueprint registry output may change. + - Run docs validation commands for changed docs. + - Run lints/types when the touched area requires them. + + Reference specs for WHAT, design for HOW, and docs.md for documentation work. + requires: + - specs + - design + - docs +apply: + requires: + - tasks + tracks: tasks.md + instruction: | + Read proposal.md, specs, design.md, docs.md, and tasks.md before editing code. + Work through pending tasks, mark checkboxes complete as they finish, and keep artifacts current when implementation changes the plan. + Verify with OpenSpec validation, focused tests, docs checks, and manual QA through the relevant DimOS surface. diff --git a/openspec/schemas/dimos-capability/templates/design.md b/openspec/schemas/dimos-capability/templates/design.md new file mode 100644 index 0000000000..25031ceb8b --- /dev/null +++ b/openspec/schemas/dimos-capability/templates/design.md @@ -0,0 +1,35 @@ +## Context + + + +## Goals / Non-Goals + +**Goals:** + + +**Non-Goals:** + + +## DimOS Architecture + + + +## Decisions + + + +## Safety / Simulation / Replay + + + +## Risks / Trade-offs + + + +## Migration / Rollout + + + +## Open Questions + + diff --git a/openspec/schemas/dimos-capability/templates/docs.md b/openspec/schemas/dimos-capability/templates/docs.md new file mode 100644 index 0000000000..d274aed653 --- /dev/null +++ b/openspec/schemas/dimos-capability/templates/docs.md @@ -0,0 +1,19 @@ +## User-Facing Docs + + + +## Contributor Docs + + + +## Coding-Agent Docs + + + +## Doc Validation + + + +## No Docs Needed + + diff --git a/openspec/schemas/dimos-capability/templates/proposal.md b/openspec/schemas/dimos-capability/templates/proposal.md new file mode 100644 index 0000000000..98d409e8de --- /dev/null +++ b/openspec/schemas/dimos-capability/templates/proposal.md @@ -0,0 +1,32 @@ +## Why + + + +## What Changes + + + +## Affected DimOS Surfaces + + +- Modules/streams: +- Blueprints/CLI: +- Skills/MCP: +- Hardware/simulation/replay: +- Docs/generated registries: + +## Capabilities + +### New Capabilities + +- ``: + +### Modified Capabilities + +- ``: + +## Impact + + diff --git a/openspec/schemas/dimos-capability/templates/spec.md b/openspec/schemas/dimos-capability/templates/spec.md new file mode 100644 index 0000000000..afc0c1ff58 --- /dev/null +++ b/openspec/schemas/dimos-capability/templates/spec.md @@ -0,0 +1,16 @@ +## ADDED Requirements + +### Requirement: + + +#### Scenario: +- **GIVEN** +- **WHEN** +- **THEN** +- **AND** + + diff --git a/openspec/schemas/dimos-capability/templates/tasks.md b/openspec/schemas/dimos-capability/templates/tasks.md new file mode 100644 index 0000000000..b38fcdfabb --- /dev/null +++ b/openspec/schemas/dimos-capability/templates/tasks.md @@ -0,0 +1,15 @@ +## 1. Implementation + +- [ ] 1.1 +- [ ] 1.2 + +## 2. Documentation + +- [ ] 2.1 + +## 3. Verification + +- [ ] 3.1 Run `openspec validate ` +- [ ] 3.2 Run focused tests for changed code +- [ ] 3.3 Run docs validation commands for changed docs +- [ ] 3.4 Manually QA through the relevant DimOS surface (CLI, MCP, simulation/replay, hardware procedure, HTTP API, or library driver) From 76158b261a9a4c3f0509bc7a218db7cfe1010e44 Mon Sep 17 00:00:00 2001 From: cc Date: Mon, 8 Jun 2026 16:20:39 -0700 Subject: [PATCH 02/15] chore: revert change to doc folder --- docs/coding-agents/index.md | 1 - docs/development/openspec.md | 102 ----------------------------------- docs/docs.json | 1 - 3 files changed, 104 deletions(-) delete mode 100644 docs/development/openspec.md diff --git a/docs/coding-agents/index.md b/docs/coding-agents/index.md index 5ac7c854a7..ff778ac5cf 100644 --- a/docs/coding-agents/index.md +++ b/docs/coding-agents/index.md @@ -3,7 +3,6 @@ ├── worktrees.md (creating provisioned worktrees with `bin/worktree`) ├── style.md (code style guidelines for dimos) ├── testing.md (docs about writing tests) -├── ../development/openspec.md (OpenSpec behavior-spec workflow) ├── docs (these are docs about writing docs) │   ├── codeblocks.md │   ├── doclinks.md diff --git a/docs/development/openspec.md b/docs/development/openspec.md deleted file mode 100644 index 280eb0f57e..0000000000 --- a/docs/development/openspec.md +++ /dev/null @@ -1,102 +0,0 @@ -# OpenSpec Workflow - -DimOS uses OpenSpec as the checked-in planning layer for behavior changes. OpenSpec artifacts live under `openspec/` and should describe what the system is supposed to do, why it is changing, and how contributors or agents should validate the work. - -## Terminology - -Keep these two meanings separate: - -- **OpenSpec capability spec**: Markdown requirements under `openspec/specs//spec.md`. These describe observable behavior and acceptance scenarios. -- **DimOS Spec**: Python Protocol/RPC contracts in files like `dimos/navigation/navigation_spec.py` or `dimos/manipulation/control/arm_driver_spec.py`. These describe module interfaces for code wiring. - -Use "OpenSpec capability spec" in prose when there is any chance of confusion. - -## Schema - -The project uses the `dimos-capability` schema configured in `openspec/config.yaml`. - -The artifact flow is: - -```text -proposal - ├── specs - ├── design - └── docs - └── tasks -``` - -| Artifact | Purpose | -|---|---| -| `proposal.md` | Intent, scope, affected DimOS surfaces, and capability impact. | -| `specs//spec.md` | Behavior-first requirements and scenarios. | -| `design.md` | Module, stream, blueprint, skill/MCP, safety, and rollout decisions. | -| `docs.md` | Documentation impact and doc validation plan. | -| `tasks.md` | Implementation, docs, verification, and manual QA checklist. | - -## When to create a change - -Create an OpenSpec change when work changes observable behavior, public CLI/API/MCP behavior, robot behavior, hardware/simulation/replay workflows, docs that users rely on, or cross-module architecture. - -Do not create a change for a purely mechanical refactor, typo fix, or internal cleanup unless it changes behavior or needs cross-session planning context. - -## Writing specs - -OpenSpec capability specs are behavior contracts, not implementation plans. - -Good spec content: - -- User- or developer-visible behavior. -- Public CLI/API/MCP tool behavior. -- Stream or message behavior that downstream modules rely on. -- Robot safety constraints and hardware/simulation/replay expectations. -- Scenarios that can be tested or manually verified. - -Avoid in specs: - -- Private class/function names. -- Generated-file mechanics. -- Library choices and wiring details. -- Step-by-step implementation tasks. - -Put those details in `design.md` or `tasks.md`. - -## Capability names - -Prefer behavior-domain names over code names. Useful starting points: - -- `module-system` -- `blueprint-composition` -- `cli-lifecycle` -- `agent-skills-mcp` -- `configuration` -- `navigation-stack` -- `manipulation-stack` -- `hardware-adapters` -- `simulation-replay` -- `documentation-system` - -Add specs progressively as changes need them. Do not try to backfill the whole project at once. - -## Validation - -Use OpenSpec validation before implementation and before archiving: - -```bash skip -openspec schema validate dimos-capability -openspec validate -openspec templates --json -``` - -For documentation changes, also run the relevant doc checks from [Writing Docs](/docs/development/writing_docs.md): - -```bash skip -md-babel-py run -``` - -When a change touches blueprint names, module-level blueprint variables, or module registry inputs, run: - -```bash skip -pytest dimos/robot/test_all_blueprints_generation.py -``` - -Then run focused tests for the changed code and manually QA through the actual surface: CLI command, MCP tool, HTTP API, simulation/replay blueprint, hardware procedure, or library driver. diff --git a/docs/docs.json b/docs/docs.json index f0064c9ab9..58da2ff6a1 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -144,7 +144,6 @@ "group": "Development", "pages": [ "development/conventions", - "development/openspec", "development/testing", "development/docker", "development/grid_testing", From 4e25297e72a260e5dcba0365941aceaee3a72993 Mon Sep 17 00:00:00 2001 From: cc Date: Sun, 19 Jul 2026 22:44:26 -0700 Subject: [PATCH 03/15] add mattskill --- docs/agents/domain.md | 60 ++++++++++++++++++++++++++++++++++ docs/agents/issue-tracker.md | 62 ++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 docs/agents/domain.md create mode 100644 docs/agents/issue-tracker.md diff --git a/docs/agents/domain.md b/docs/agents/domain.md new file mode 100644 index 0000000000..e1de27973a --- /dev/null +++ b/docs/agents/domain.md @@ -0,0 +1,60 @@ +# DimOS agent domain context + +## Context loading + +Before working on a change, load the repository context in this order: + +1. Read `AGENTS.md` and follow its applicable instructions. +2. Read `openspec/config.yaml` for the OpenSpec schema, terminology, and rules. +3. Read the relevant files under `openspec/specs/`. +4. Read the root `CONTEXT.md` if it exists. +5. Read relevant records under `docs/adr/` if that directory exists. + +`CONTEXT.md` and `docs/adr/` are optional. If either is absent, continue +silently; do not report the absence as an error. Select specs and ADRs based on +the affected behavior and implementation surface rather than reading +unrelated material. + +## Two meanings of “spec” + +Keep these terms separate: + +- An **OpenSpec spec** is a behavior specification under `openspec/specs/`. + It describes observable behavior, user or developer outcomes, public + interfaces, safety constraints, and testable scenarios. +- A **DimOS Python Spec Protocol** is a code-level interface contract, usually + a `Protocol` inheriting from `dimos.spec.utils.Spec`, often found in a + `*_spec.py` file. It describes module RPCs and injected interfaces. + +An OpenSpec spec is not a Python Protocol, and a Python Protocol does not +replace an OpenSpec behavioral requirement. Keep implementation details such as +class names, module wiring, stream types, generated registries, and rollout +steps in the OpenSpec change design or tasks unless they are externally +observable. + +## Work layout + +Organize work through this chain: + +```text +Linear issue -> OpenSpec change -> implementation tasks -> pull request +``` + +Linear provides intake and tracking. The OpenSpec change is the source of truth +for the behavioral change, design, and tasks. The pull request implements and +reviews those tasks. Keep the identifiers and links aligned across all three +artifacts; any Linear link edit requires user confirmation before it is made. + +When a task affects behavior, update the relevant OpenSpec change and, where +appropriate, the corresponding spec under `openspec/specs/`. Include concrete +scenarios for behavioral requirements. Call out DimOS Python Spec Protocols, +blueprint composition, streams, skills/MCP exposure, generated files, and +hardware, simulation, or replay assumptions in design and task material when +they are relevant. + +## Conflicting guidance + +Surface conflicts between an ADR and an OpenSpec spec explicitly. Do not +silently reconcile, overwrite, or guess which decision applies. Report the +conflict, identify the affected behavior or implementation, and ask for the +decision or update the authoritative document only when instructed. diff --git a/docs/agents/issue-tracker.md b/docs/agents/issue-tracker.md new file mode 100644 index 0000000000..c0db692d0f --- /dev/null +++ b/docs/agents/issue-tracker.md @@ -0,0 +1,62 @@ +# Issue tracking with Linear + +## Workspace + +DimOS work is tracked in the **DIM** team in Linear: + + + +Access Linear through the configured Linear MCP. Do not assume that a local +copy, an unconfigured client, or a direct API call is an alternative source of +truth. + +## Confirmation policy + +User confirmation is required immediately before **every** Linear edit. This +includes, without limitation: + +- creating an issue; +- changing any issue field, including title, description, assignee, priority, + project, or due date; +- adding, removing, or changing labels; +- posting comments; +- changing state or making any other state transition; and +- adding, removing, or changing links. + +Reading Linear is not an edit. Before an edit, state exactly what will change +and wait for explicit user confirmation. One confirmation does not authorize +later edits, even when they concern the same issue or change. + +## Linking convention + +Keep the work chain navigable: + +```text +Linear issue <-> openspec/changes/ <-> pull request +``` + +Use the OpenSpec change ID as the stable identifier in the relationship. Link +the Linear issue to the relevant OpenSpec change and link the pull request to +both when the tools support those links. If a link must be created or changed, +it is a Linear edit and requires confirmation under the policy above. + +## Source of truth and workflow + +Linear is the intake and tracking system. It records requests, ownership, +status, discussion, and delivery progress. OpenSpec is the source of truth for +the behavioral change, its design, and its implementation tasks. The pull +request is the review and delivery vehicle. + +Use this sequence: + +1. Capture or find the Linear issue in the DIM team. +2. Create or update `openspec/changes//` for the proposed behavior, + design, and tasks. +3. Implement the tasks and keep the OpenSpec change current. +4. Open the pull request and connect it to the issue and OpenSpec change. +5. Reflect progress in Linear only after confirming each requested edit. + +Do not use a Linear description, comment, or state as a substitute for an +OpenSpec requirement, design decision, or task. If Linear and OpenSpec +disagree about behavior, treat OpenSpec as authoritative and surface the +discrepancy to the user rather than silently choosing a version. From f9f660a480bb08c69e0d0b98c0c8d76df180c661 Mon Sep 17 00:00:00 2001 From: cc Date: Wed, 5 Aug 2026 13:12:00 -0700 Subject: [PATCH 04/15] feat: add frozen recording agent evaluation --- .github/workflows/ci.yml | 18 + dimos/agents/code_policy_core.py | 666 ++++++ dimos/agents/code_policy_server.py | 345 ++++ dimos/agents/mcp/mcp_adapter.py | 2 +- dimos/agents/mcp/test_mcp_adapter.py | 29 + dimos/agents/test_code_policy_core.py | 108 + dimos/agents/test_code_policy_server.py | 110 + dimos/benchmark/agent_eval/artifacts.py | 78 + dimos/benchmark/agent_eval/auth.py | 25 + dimos/benchmark/agent_eval/base.py | 26 + dimos/benchmark/agent_eval/case.py | 232 +++ dimos/benchmark/agent_eval/engine.py | 262 +++ dimos/benchmark/agent_eval/interfaces.py | 106 + dimos/benchmark/agent_eval/json.py | 29 + dimos/benchmark/agent_eval/pi.py | 56 + dimos/benchmark/agent_eval/pi_adapter.py | 302 +++ dimos/benchmark/agent_eval/pi_process.py | 429 ++++ dimos/benchmark/agent_eval/progress.py | 84 + dimos/benchmark/agent_eval/single_case.py | 245 +++ dimos/benchmark/agent_eval/store.py | 285 +++ dimos/benchmark/agent_eval/test_case.py | 148 ++ dimos/benchmark/agent_eval/test_engine.py | 281 +++ .../agent_eval/test_import_boundaries.py | 43 + dimos/benchmark/agent_eval/test_json.py | 26 + dimos/benchmark/agent_eval/test_pi_adapter.py | 198 ++ dimos/benchmark/agent_eval/test_pi_process.py | 229 +++ .../benchmark/agent_eval/test_single_case.py | 124 ++ dimos/benchmark/agent_eval/test_store.py | 160 ++ .../README.md | 17 + .../case.json | 31 + .../private/oracle.json | 9 + dimos/benchmark/short_horizon_qa/eval.py | 412 ++++ dimos/benchmark/short_horizon_qa/models.py | 72 + dimos/benchmark/short_horizon_qa/prepare.py | 266 +++ dimos/benchmark/short_horizon_qa/service.py | 122 ++ dimos/benchmark/short_horizon_qa/test_eval.py | 223 ++ .../short_horizon_qa/test_hongkong_eval.py | 98 + .../short_horizon_qa/test_prepare.py | 219 ++ dimos/cli/dimos.py | 2 + dimos/cli/eval.py | 201 ++ dimos/cli/test_eval.py | 267 +++ dimos/memory2/observationstore/sqlite.py | 17 +- dimos/memory2/registry.py | 22 +- dimos/memory2/store/frozen.py | 80 + dimos/memory2/store/sqlite.py | 28 +- dimos/memory2/store/test_frozen.py | 101 + dimos/memory2/stream.py | 25 +- dimos/memory2/type/filter.py | 10 + dimos/memory2/utils/sqlite.py | 17 +- docs/capabilities/agents/evaluation.md | 94 + docs/capabilities/agents/index.md | 2 + docs/development/testing.md | 42 + .../extract-frozen-qa-eval/.openspec.yaml | 2 + .../changes/extract-frozen-qa-eval/README.md | 3 + .../changes/extract-frozen-qa-eval/design.md | 112 + .../changes/extract-frozen-qa-eval/docs.md | 42 + .../extract-frozen-qa-eval/proposal.md | 42 + .../specs/frozen-agent-evaluation/spec.md | 107 + .../specs/frozen-memory-views/spec.md | 53 + .../standalone-code-policy-runtime/spec.md | 73 + .../changes/extract-frozen-qa-eval/tasks.md | 75 + packages/pi-code-policy-adapter/.gitignore | 6 + packages/pi-code-policy-adapter/README.md | 16 + .../pi-code-policy-adapter/package-lock.json | 1825 +++++++++++++++++ packages/pi-code-policy-adapter/package.json | 24 + .../src/code-policy-main.ts | 241 +++ .../src/code-policy-protocol.ts | 122 ++ .../src/code-policy-session.ts | 74 + .../pi-code-policy-adapter/src/session.ts | 352 ++++ .../test/code-policy-main.test.ts | 123 ++ .../test/code-policy-protocol.test.ts | 31 + .../test/code-policy-session.test.ts | 40 + .../test/session.test.ts | 229 +++ .../tsconfig.build.json | 13 + packages/pi-code-policy-adapter/tsconfig.json | 18 + .../pi-code-policy-adapter/tsconfig.test.json | 14 + pyproject.toml | 8 + uv.lock | 27 + 78 files changed, 10676 insertions(+), 19 deletions(-) create mode 100644 dimos/agents/code_policy_core.py create mode 100644 dimos/agents/code_policy_server.py create mode 100644 dimos/agents/mcp/test_mcp_adapter.py create mode 100644 dimos/agents/test_code_policy_core.py create mode 100644 dimos/agents/test_code_policy_server.py create mode 100644 dimos/benchmark/agent_eval/artifacts.py create mode 100644 dimos/benchmark/agent_eval/auth.py create mode 100644 dimos/benchmark/agent_eval/base.py create mode 100644 dimos/benchmark/agent_eval/case.py create mode 100644 dimos/benchmark/agent_eval/engine.py create mode 100644 dimos/benchmark/agent_eval/interfaces.py create mode 100644 dimos/benchmark/agent_eval/json.py create mode 100644 dimos/benchmark/agent_eval/pi.py create mode 100644 dimos/benchmark/agent_eval/pi_adapter.py create mode 100644 dimos/benchmark/agent_eval/pi_process.py create mode 100644 dimos/benchmark/agent_eval/progress.py create mode 100644 dimos/benchmark/agent_eval/single_case.py create mode 100644 dimos/benchmark/agent_eval/store.py create mode 100644 dimos/benchmark/agent_eval/test_case.py create mode 100644 dimos/benchmark/agent_eval/test_engine.py create mode 100644 dimos/benchmark/agent_eval/test_import_boundaries.py create mode 100644 dimos/benchmark/agent_eval/test_json.py create mode 100644 dimos/benchmark/agent_eval/test_pi_adapter.py create mode 100644 dimos/benchmark/agent_eval/test_pi_process.py create mode 100644 dimos/benchmark/agent_eval/test_single_case.py create mode 100644 dimos/benchmark/agent_eval/test_store.py create mode 100644 dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/README.md create mode 100644 dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/case.json create mode 100644 dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/private/oracle.json create mode 100644 dimos/benchmark/short_horizon_qa/eval.py create mode 100644 dimos/benchmark/short_horizon_qa/models.py create mode 100644 dimos/benchmark/short_horizon_qa/prepare.py create mode 100644 dimos/benchmark/short_horizon_qa/service.py create mode 100644 dimos/benchmark/short_horizon_qa/test_eval.py create mode 100644 dimos/benchmark/short_horizon_qa/test_hongkong_eval.py create mode 100644 dimos/benchmark/short_horizon_qa/test_prepare.py create mode 100644 dimos/cli/eval.py create mode 100644 dimos/cli/test_eval.py create mode 100644 dimos/memory2/store/frozen.py create mode 100644 dimos/memory2/store/test_frozen.py create mode 100644 docs/capabilities/agents/evaluation.md create mode 100644 openspec/changes/extract-frozen-qa-eval/.openspec.yaml create mode 100644 openspec/changes/extract-frozen-qa-eval/README.md create mode 100644 openspec/changes/extract-frozen-qa-eval/design.md create mode 100644 openspec/changes/extract-frozen-qa-eval/docs.md create mode 100644 openspec/changes/extract-frozen-qa-eval/proposal.md create mode 100644 openspec/changes/extract-frozen-qa-eval/specs/frozen-agent-evaluation/spec.md create mode 100644 openspec/changes/extract-frozen-qa-eval/specs/frozen-memory-views/spec.md create mode 100644 openspec/changes/extract-frozen-qa-eval/specs/standalone-code-policy-runtime/spec.md create mode 100644 openspec/changes/extract-frozen-qa-eval/tasks.md create mode 100644 packages/pi-code-policy-adapter/.gitignore create mode 100644 packages/pi-code-policy-adapter/README.md create mode 100644 packages/pi-code-policy-adapter/package-lock.json create mode 100644 packages/pi-code-policy-adapter/package.json create mode 100644 packages/pi-code-policy-adapter/src/code-policy-main.ts create mode 100644 packages/pi-code-policy-adapter/src/code-policy-protocol.ts create mode 100644 packages/pi-code-policy-adapter/src/code-policy-session.ts create mode 100644 packages/pi-code-policy-adapter/src/session.ts create mode 100644 packages/pi-code-policy-adapter/test/code-policy-main.test.ts create mode 100644 packages/pi-code-policy-adapter/test/code-policy-protocol.test.ts create mode 100644 packages/pi-code-policy-adapter/test/code-policy-session.test.ts create mode 100644 packages/pi-code-policy-adapter/test/session.test.ts create mode 100644 packages/pi-code-policy-adapter/tsconfig.build.json create mode 100644 packages/pi-code-policy-adapter/tsconfig.json create mode 100644 packages/pi-code-policy-adapter/tsconfig.test.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d980c9d8ab..aeeb558ca3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,6 +70,24 @@ jobs: - name: Run pre-commit uses: pre-commit/action@v3.0.1 + pi-code-policy-adapter: + timeout-minutes: 10 + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: '22.19.0' + cache: npm + cache-dependency-path: packages/pi-code-policy-adapter/package-lock.json + - name: Install adapter dependencies + run: npm ci --prefix packages/pi-code-policy-adapter + - name: Test adapter + run: npm test --prefix packages/pi-code-policy-adapter + rust: timeout-minutes: 20 runs-on: ubuntu-latest diff --git a/dimos/agents/code_policy_core.py b/dimos/agents/code_policy_core.py new file mode 100644 index 0000000000..4265e579f9 --- /dev/null +++ b/dimos/agents/code_policy_core.py @@ -0,0 +1,666 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module-independent persistent Python policy session.""" + +from __future__ import annotations + +import base64 +from datetime import UTC, datetime +import os +import re +import threading +import time +from typing import Annotated, Any, Literal +from uuid import uuid4 + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + +MAX_EXECUTION_TIMEOUT_S = 110.0 +DEFAULT_STARTUP_TIMEOUT_S = 10.0 +DEFAULT_INTERRUPT_GRACE_S = 2.0 +DEFAULT_OUTPUT_LIMIT = 32_000 +_RECORDING_PATH_ENV = "DIMOS_CODE_POLICY_RECORDING_PATH" +_DERIVED_RECORDING_PATH_ENV = "DIMOS_CODE_POLICY_DERIVED_RECORDING_PATH" +_MEMORY_CUTOFF_ENV = "DIMOS_CODE_POLICY_MEMORY_CUTOFF" +_CONNECT_APP_ENV = "DIMOS_CODE_POLICY_CONNECT_APP" +_TRUNCATION_MARKER = "\n... [output truncated]" +_ANSI_ESCAPE_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") + +SessionId = Annotated[str, Field(pattern=r"^code_policy_session_[0-9a-f]{32}$")] +ExecutionId = Annotated[str, Field(pattern=r"^code_policy_call_[0-9a-f]{32}$")] +ExecutionStatus = Literal[ + "busy", + "completed", + "execution-failed", + "invalid-request", + "kernel-start-failed", + "module-stopped", + "python-error", + "timed-out", +] +ObserverAvailability = Literal["ready", "replaced", "stopped", "unavailable"] + + +class _EvidenceModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class FrozenMemoryEnvironment(_EvidenceModel): + kind: Literal["frozen_memory"] = "frozen_memory" + recording_path: str = Field(min_length=1) + derived_recording_path: str = Field(min_length=1) + memory_cutoff_timestamp: float + + +class LiveDimosEnvironment(_EvidenceModel): + kind: Literal["live_dimos"] = "live_dimos" + recording_path: str = Field(min_length=1) + + +CodePolicyEnvironment = FrozenMemoryEnvironment | LiveDimosEnvironment + + +class CodePolicySessionConfig(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + environment: CodePolicyEnvironment + output_limit: int = DEFAULT_OUTPUT_LIMIT + startup_timeout_s: float = DEFAULT_STARTUP_TIMEOUT_S + interrupt_grace_s: float = DEFAULT_INTERRUPT_GRACE_S + + @model_validator(mode="after") + def limits_are_valid(self) -> CodePolicySessionConfig: + if self.output_limit < 0: + raise ValueError("output_limit must be non-negative") + if self.startup_timeout_s <= 0 or self.interrupt_grace_s <= 0: + raise ValueError("CodePolicy timeouts must be positive") + return self + + +class CodePolicySessionReceipt(_EvidenceModel): + session_id: SessionId + reset_at: datetime + previous_session_id: SessionId | None + + +class CodePolicyExecutionRecord(_EvidenceModel): + execution_id: ExecutionId + session_id: SessionId + source: str + requested_timeout_s: float + started_at: datetime + finished_at: datetime + monotonic_duration_s: Annotated[float, Field(ge=0)] + status: ExecutionStatus + jupyter_message_id: str | None + jupyter_execution_count: int | None + output: str + transcript: str + interrupt_attempted: bool + interrupt_recovered: bool + kernel_restarted: bool + namespace_preserved: bool + remote_work_may_continue: bool + + +class CodePolicyObserverDescriptor(_EvidenceModel): + transport: Literal["tcp", "ipc"] + ip: str + iopub_port: Annotated[int, Field(gt=0)] + signature_scheme: str + key_base64: str + code_policy_session_id: SessionId + jupyter_client_session_id: str + kernel_generation: Annotated[int, Field(gt=0)] + + +class CodePolicyObserverState(_EvidenceModel): + availability: ObserverAvailability + code_policy_session_id: SessionId + kernel_generation: Annotated[int, Field(ge=0)] + descriptor: CodePolicyObserverDescriptor | None + + +class CodePolicyObserverProbeReceipt(_EvidenceModel): + message_id: str + code_policy_session_id: SessionId + kernel_generation: Annotated[int, Field(gt=0)] + + +class _BoundedTextOutput: + def __init__(self, limit: int) -> None: + self._limit = max(0, limit) + self._parts: list[str] = [] + self._length = 0 + self._truncated = False + + def __call__(self, message: dict[str, Any]) -> None: + message_type = message.get("header", {}).get("msg_type") + content = message.get("content", {}) + if message_type == "stream": + self._append(str(content.get("text", ""))) + elif message_type in {"execute_result", "display_data"}: + text = content.get("data", {}).get("text/plain") + if text is not None: + self._append(str(text)) + elif message_type == "error": + traceback = content.get("traceback") + if isinstance(traceback, list): + self._append("\n".join(str(line) for line in traceback)) + else: + self._append(f"{content.get('ename', 'Error')}: {content.get('evalue', '')}") + + def text(self) -> str: + return "".join(self._parts) + + def _append(self, value: str) -> None: + if not value or self._truncated: + return + value = _ANSI_ESCAPE_RE.sub("", value) + remaining = self._limit - self._length + if len(value) <= remaining: + self._parts.append(value) + self._length += len(value) + return + marker = _TRUNCATION_MARKER[:remaining] + content_limit = max(0, remaining - len(marker)) + self._parts.append(value[:content_limit] + marker) + self._length = self._limit + self._truncated = True + + +def _load_kernel_manager() -> type[Any]: + try: + from jupyter_client.manager import KernelManager + except ImportError as exc: + raise RuntimeError( + "Code-policy execution requires the agents extra: uv sync --extra agents" + ) from exc + return KernelManager + + +def _bootstrap_source() -> str: + return f""" +import os as _os +from dimos.memory2.store.sqlite import SqliteStore as _SqliteStore + +_cutoff = _os.environ.get({_MEMORY_CUTOFF_ENV!r}) +if _cutoff is None: + memory = _SqliteStore( + path=_os.environ[{_RECORDING_PATH_ENV!r}], must_exist=True, read_only=True + ) + memory.start() +else: + from dimos.memory2.store.frozen import FrozenMemoryStore as _FrozenMemoryStore + + _source = _SqliteStore( + path=_os.environ[{_RECORDING_PATH_ENV!r}], must_exist=True, read_only=True + ) + _derived = _SqliteStore( + path=_os.environ[{_DERIVED_RECORDING_PATH_ENV!r}], must_exist=True, read_only=True + ) + memory = _FrozenMemoryStore( + source=_source, derived=_derived, through_timestamp=float(_cutoff) + ) + memory.start() + del _FrozenMemoryStore, _source, _derived + +if _os.environ.get({_CONNECT_APP_ENV!r}, "1") == "1": + from dimos.porcelain.dimos import Dimos as _Dimos + + app = _Dimos.connect() + del _Dimos + +del _os, _SqliteStore, _cutoff +""" + + +class CodePolicySession: + """Execute trusted agent-authored Python in one persistent kernel.""" + + def __init__(self, config: CodePolicySessionConfig) -> None: + self.config = config + self._execution_lock = threading.Lock() + self._records_lock = threading.Lock() + self._kernel_lock = threading.RLock() + self._kernel_manager: Any = None + self._kernel_client: Any = None + self._kernel_generation = 0 + self._session_id: str = _new_session_id() + self._session_reset_at = _utc_now() + self._stopped = True + self._execution_records: list[CodePolicyExecutionRecord] = [] + + def start(self) -> None: + self._stopped = False + + def python_exec(self, code: str, timeout_s: float = MAX_EXECUTION_TIMEOUT_S) -> str: + started_at = _utc_now() + started_monotonic = time.monotonic() + if self._stopped: + transcript = "Code Policy Module stopped" + self._record( + code, + timeout_s, + started_at, + started_monotonic, + status="module-stopped", + transcript=transcript, + ) + return transcript + if not 0 < timeout_s <= MAX_EXECUTION_TIMEOUT_S: + transcript = ( + f"Invalid timeout_s={timeout_s!r}; expected a value in " + f"(0, {MAX_EXECUTION_TIMEOUT_S:g}]" + ) + self._record( + code, + timeout_s, + started_at, + started_monotonic, + status="invalid-request", + transcript=transcript, + ) + return transcript + if not self._execution_lock.acquire(blocking=False): + transcript = "Code Policy Module busy: another python_exec call is active" + self._record( + code, + timeout_s, + started_at, + started_monotonic, + status="busy", + transcript=transcript, + ) + return transcript + logger.info("Code policy execution started", source=code, timeout_s=timeout_s) + try: + try: + client = self._ensure_kernel() + except Exception as exc: + logger.exception("Code policy kernel failed to start") + transcript = f"Code policy kernel failed to start: {type(exc).__name__}: {exc}" + self._record( + code, + timeout_s, + started_at, + started_monotonic, + status="kernel-start-failed", + transcript=transcript, + ) + return transcript + output = _BoundedTextOutput(self.config.output_limit) + try: + reply = client.execute_interactive( + code, + allow_stdin=False, + output_hook=output, + store_history=True, + timeout=timeout_s, + ) + except TimeoutError: + interrupt_recovered, kernel_restarted = self._recover_from_timeout() + if interrupt_recovered: + transcript = ( + f"Execution timed out after {timeout_s:.1f}s and was interrupted. " + "The Python namespace was preserved. Remote RPC work may still be " + "running; it was not cancelled." + ) + else: + transcript = ( + f"Execution timed out after {timeout_s:.1f}s. The kernel did not " + "recover from interruption and was restarted; the Python namespace " + "was reset. Remote RPC work may still be running; it was not cancelled." + ) + self._record( + code, + timeout_s, + started_at, + started_monotonic, + status="timed-out", + output=output.text(), + transcript=transcript, + interrupt_attempted=True, + interrupt_recovered=interrupt_recovered, + kernel_restarted=kernel_restarted, + namespace_preserved=interrupt_recovered, + remote_work_may_continue=True, + ) + return transcript + except Exception as exc: + self._shutdown_kernel(reason=type(exc).__name__) + logger.exception("Code policy execution failed") + transcript = ( + f"Code policy execution failed: {type(exc).__name__}: {exc}. " + "The Python namespace was reset." + ) + self._record( + code, + timeout_s, + started_at, + started_monotonic, + status="execution-failed", + output=output.text(), + transcript=transcript, + kernel_restarted=True, + ) + return transcript + duration_s = time.monotonic() - started_monotonic + transcript = _format_reply(reply, output.text(), duration_s) + content = reply.get("content", {}) + status: ExecutionStatus = ( + "completed" if content.get("status") == "ok" else "python-error" + ) + self._record( + code, + timeout_s, + started_at, + started_monotonic, + status=status, + output=output.text(), + transcript=transcript, + jupyter_message_id=reply.get("parent_header", {}).get("msg_id"), + jupyter_execution_count=content.get("execution_count"), + ) + return transcript + finally: + self._execution_lock.release() + + def reset_session(self) -> CodePolicySessionReceipt: + if not self._execution_lock.acquire(blocking=False): + raise RuntimeError("cannot reset code policy while python_exec is active") + try: + previous_session_id = self._session_id + self._shutdown_kernel(reason="session reset") + self._session_id = _new_session_id() + self._session_reset_at = _utc_now() + return CodePolicySessionReceipt( + session_id=self._session_id, + reset_at=self._session_reset_at, + previous_session_id=previous_session_id, + ) + finally: + self._execution_lock.release() + + def get_session_receipt(self) -> CodePolicySessionReceipt: + return CodePolicySessionReceipt( + session_id=self._session_id, + reset_at=self._session_reset_at, + previous_session_id=None, + ) + + def get_execution_records( + self, session_id: str | None = None + ) -> tuple[CodePolicyExecutionRecord, ...]: + with self._records_lock: + records = tuple(self._execution_records) + if session_id is None: + return records + return tuple(record for record in records if record.session_id == session_id) + + def prepare_observer(self) -> CodePolicyObserverState: + if self._stopped: + return self._observer_state("stopped") + self._ensure_kernel() + return self._observer_state("ready") + + def get_observer_state(self, known_generation: int | None = None) -> CodePolicyObserverState: + if self._stopped: + return self._observer_state("stopped") + with self._kernel_lock: + manager = self._kernel_manager + client = self._kernel_client + generation = self._kernel_generation + is_ready = manager is not None and client is not None and bool(manager.is_alive()) + if not is_ready: + return self._observer_state("unavailable") + availability: ObserverAvailability = ( + "replaced" + if known_generation is not None and known_generation != generation + else "ready" + ) + return self._observer_state(availability) + + def issue_observer_probe(self, kernel_generation: int) -> CodePolicyObserverProbeReceipt: + if self._stopped: + raise RuntimeError("code policy session is stopped") + if not self._execution_lock.acquire(blocking=False): + raise RuntimeError("cannot probe while python_exec is active") + try: + client = self._ensure_kernel() + with self._kernel_lock: + if kernel_generation != self._kernel_generation: + raise RuntimeError( + "code policy kernel generation changed before readiness probe" + ) + message_id = client.execute( + "None", silent=True, store_history=False, allow_stdin=False + ) + return CodePolicyObserverProbeReceipt( + message_id=message_id, + code_policy_session_id=self._session_id, + kernel_generation=self._kernel_generation, + ) + finally: + self._execution_lock.release() + + def interrupt_active(self) -> bool: + if self._execution_lock.acquire(blocking=False): + self._execution_lock.release() + return False + manager = self._kernel_manager + if manager is None or not manager.is_alive(): + return False + manager.interrupt_kernel() + return True + + def stop(self) -> None: + self._stopped = True + self._shutdown_kernel(reason="session stop") + + def _record( + self, + source: str, + timeout_s: float, + started_at: datetime, + started_monotonic: float, + *, + status: ExecutionStatus, + output: str = "", + transcript: str, + jupyter_message_id: str | None = None, + jupyter_execution_count: int | None = None, + interrupt_attempted: bool = False, + interrupt_recovered: bool = False, + kernel_restarted: bool = False, + namespace_preserved: bool = True, + remote_work_may_continue: bool = False, + ) -> None: + record = CodePolicyExecutionRecord( + execution_id=f"code_policy_call_{uuid4().hex}", + session_id=self._session_id, + source=source, + requested_timeout_s=timeout_s, + started_at=started_at, + finished_at=_utc_now(), + monotonic_duration_s=max(0.0, time.monotonic() - started_monotonic), + status=status, + jupyter_message_id=jupyter_message_id, + jupyter_execution_count=jupyter_execution_count, + output=output, + transcript=transcript, + interrupt_attempted=interrupt_attempted, + interrupt_recovered=interrupt_recovered, + kernel_restarted=kernel_restarted, + namespace_preserved=namespace_preserved, + remote_work_may_continue=remote_work_may_continue, + ) + with self._records_lock: + self._execution_records.append(record) + + def _ensure_kernel(self) -> Any: + with self._kernel_lock: + manager = self._kernel_manager + client = self._kernel_client + if manager is not None and client is not None and manager.is_alive(): + return client + self._shutdown_kernel(reason="kernel unavailable") + manager_type = _load_kernel_manager() + manager = manager_type(kernel_name="python3") + client = None + try: + env = os.environ.copy() + environment = self.config.environment + env[_RECORDING_PATH_ENV] = environment.recording_path + if isinstance(environment, FrozenMemoryEnvironment): + env[_CONNECT_APP_ENV] = "0" + env[_DERIVED_RECORDING_PATH_ENV] = environment.derived_recording_path + env[_MEMORY_CUTOFF_ENV] = str(environment.memory_cutoff_timestamp) + else: + env[_CONNECT_APP_ENV] = "1" + env.pop(_DERIVED_RECORDING_PATH_ENV, None) + env.pop(_MEMORY_CUTOFF_ENV, None) + manager.start_kernel(env=env) + client = manager.client() + client.start_channels() + client.wait_for_ready(timeout=self.config.startup_timeout_s) + self._bootstrap(client) + except Exception: + if client is not None: + client.stop_channels() + try: + manager.shutdown_kernel(now=True) + except Exception: + logger.exception("Failed to stop an uninitialized code policy kernel") + raise + self._kernel_manager = manager + self._kernel_client = client + self._kernel_generation += 1 + logger.info("Code policy kernel started", environment=self.config.environment.kind) + return client + + def _bootstrap(self, client: Any) -> None: + reply = client.execute_interactive( + _bootstrap_source(), + allow_stdin=False, + output_hook=lambda _message: None, + silent=True, + store_history=False, + timeout=self.config.startup_timeout_s, + ) + content = reply.get("content", {}) + if content.get("status") != "ok": + name = content.get("ename", "KernelBootstrapError") + value = content.get("evalue", "unknown bootstrap failure") + raise RuntimeError(f"{name}: {value}") + + def _recover_from_timeout(self) -> tuple[bool, bool]: + manager = self._kernel_manager + client = self._kernel_client + if manager is None or client is None: + return False, False + try: + manager.interrupt_kernel() + client.wait_for_ready(timeout=self.config.interrupt_grace_s) + logger.info("Code policy kernel recovered after interrupt") + return True, False + except Exception: + logger.warning("Code policy kernel did not recover after interrupt") + try: + manager.restart_kernel(now=True) + client.wait_for_ready(timeout=self.config.startup_timeout_s) + self._bootstrap(client) + with self._kernel_lock: + self._kernel_generation += 1 + logger.info("Code policy kernel restarted after failed interrupt") + return False, True + except Exception: + logger.exception("Code policy kernel failed to restart") + self._shutdown_kernel(reason="restart failure") + return False, True + + def _shutdown_kernel(self, *, reason: str) -> None: + with self._kernel_lock: + manager = self._kernel_manager + client = self._kernel_client + self._kernel_manager = None + self._kernel_client = None + if manager is None and client is None: + return + try: + if manager is not None: + manager.shutdown_kernel(now=True) + except Exception: + logger.exception("Failed to stop code policy kernel") + finally: + if client is not None: + client.stop_channels() + logger.info("Code policy kernel stopped", reason=reason) + + def _observer_state(self, availability: ObserverAvailability) -> CodePolicyObserverState: + descriptor: CodePolicyObserverDescriptor | None = None + with self._kernel_lock: + generation = self._kernel_generation + manager = self._kernel_manager + client = self._kernel_client + if availability in {"ready", "replaced"}: + if manager is None or client is None or not manager.is_alive(): + availability = "unavailable" + else: + connection = manager.get_connection_info() + key = connection["key"] + if isinstance(key, str): + key = key.encode() + descriptor = CodePolicyObserverDescriptor( + transport=connection["transport"], + ip=connection["ip"], + iopub_port=connection["iopub_port"], + signature_scheme=connection["signature_scheme"], + key_base64=base64.b64encode(key).decode("ascii"), + code_policy_session_id=self._session_id, + jupyter_client_session_id=client.session.session, + kernel_generation=generation, + ) + return CodePolicyObserverState( + availability=availability, + code_policy_session_id=self._session_id, + kernel_generation=generation, + descriptor=descriptor, + ) + + +def _new_session_id() -> str: + return f"code_policy_session_{uuid4().hex}" + + +def _utc_now() -> datetime: + return datetime.now(UTC) + + +def _format_reply(reply: dict[str, Any], output: str, duration_s: float) -> str: + content = reply.get("content", {}) + status = content.get("status", "unknown") + execution_count = content.get("execution_count", "?") + state = "completed" if status == "ok" else "failed" + body = output.rstrip() + if not body and status != "ok": + body = f"{content.get('ename', 'Error')}: {content.get('evalue', '')}".rstrip() + if not body: + body = "(completed)" + return f"In [{execution_count}] {state} in {duration_s:.2f}s\n\n{body}" diff --git a/dimos/agents/code_policy_server.py b/dimos/agents/code_policy_server.py new file mode 100644 index 0000000000..0a3db174c4 --- /dev/null +++ b/dimos/agents/code_policy_server.py @@ -0,0 +1,345 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Standalone one-tool MCP host for :class:`CodePolicySession`.""" + +from __future__ import annotations + +import argparse +import asyncio +import socket +import subprocess +import sys +import threading +from typing import Any + +from fastapi import FastAPI +from fastapi.responses import JSONResponse +import requests +from starlette.requests import Request +import uvicorn + +from dimos.agents.code_policy_core import ( + MAX_EXECUTION_TIMEOUT_S, + CodePolicySession, + CodePolicySessionConfig, + FrozenMemoryEnvironment, + LiveDimosEnvironment, +) +from dimos.agents.mcp.mcp_adapter import McpAdapter + +PYTHON_EXEC_DESCRIPTION = """Execute one synchronous Python program in the persistent policy session. + +The trusted, unsandboxed session preloads `memory` for observations and, in a live +environment, `app` for deployed DimOS RPCs. Imports, functions, variables, and +mutations persist until the host resets the session. +""" + +PYTHON_EXEC_TOOL = { + "name": "python_exec", + "description": PYTHON_EXEC_DESCRIPTION, + "inputSchema": { + "type": "object", + "properties": { + "code": {"type": "string"}, + "timeout_s": {"type": "number", "default": MAX_EXECUTION_TIMEOUT_S}, + }, + "required": ["code"], + "additionalProperties": False, + }, +} + + +class StandaloneCodePolicyServer: + """Own a CodePolicy session and serve it directly over MCP.""" + + def __init__( + self, + config: CodePolicySessionConfig, + *, + host: str = "127.0.0.1", + port: int = 0, + ) -> None: + self.config = config + self.host = host + self.port = port + self.session = CodePolicySession(config) + self.app = FastAPI() + self._server: uvicorn.Server | None = None + self._thread: threading.Thread | None = None + self._socket: socket.socket | None = None + self.shutdown_requested = threading.Event() + self._install_routes() + + @property + def mcp_url(self) -> str: + if self.port <= 0: + raise RuntimeError("standalone CodePolicy server has not started") + return f"http://{self.host}:{self.port}/mcp" + + @property + def control_url(self) -> str: + if self.port <= 0: + raise RuntimeError("standalone CodePolicy server has not started") + return f"http://{self.host}:{self.port}/control" + + def start(self) -> None: + if self._thread is not None: + raise RuntimeError("standalone CodePolicy server already started") + self.session.start() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((self.host, self.port)) + sock.listen(2048) + self.port = int(sock.getsockname()[1]) + self._socket = sock + server = uvicorn.Server(uvicorn.Config(self.app, log_level="warning", access_log=False)) + self._server = server + + def serve() -> None: + asyncio.run(server.serve(sockets=[sock])) + + self._thread = threading.Thread( + target=serve, + name=f"code-policy-mcp-{self.port}", + daemon=True, + ) + self._thread.start() + if not McpAdapter(self.mcp_url, timeout=2).wait_for_ready(timeout=10, interval=0.05): + self.stop() + raise TimeoutError("standalone CodePolicy MCP server did not become ready") + + def stop(self) -> None: + server = self._server + thread = self._thread + self._server = None + self._thread = None + if server is not None: + server.should_exit = True + if thread is not None: + thread.join(timeout=5) + if self._socket is not None: + self._socket.close() + self._socket = None + self.session.stop() + + def run_forever(self) -> None: + self.start() + try: + while not self.shutdown_requested.wait(0.2): + thread = self._thread + if thread is None or not thread.is_alive(): + raise RuntimeError("standalone CodePolicy MCP server stopped unexpectedly") + except KeyboardInterrupt: + pass + finally: + self.stop() + + def _install_routes(self) -> None: + @self.app.post("/mcp") + async def mcp_endpoint(request: Request) -> JSONResponse: + try: + body = await request.json() + except Exception: + return JSONResponse(_error(None, -32700, "Parse error"), status_code=400) + return JSONResponse(await self._handle_mcp(body)) + + @self.app.post("/control/{operation}") + async def control_endpoint(operation: str, request: Request) -> JSONResponse: + body: dict[str, Any] = {} + if request.headers.get("content-length") not in {None, "0"}: + body = await request.json() + if operation == "receipt": + value: Any = self.session.get_session_receipt().model_dump(mode="json") + elif operation == "reset": + value = self.session.reset_session().model_dump(mode="json") + elif operation == "interrupt": + value = {"interrupted": self.session.interrupt_active()} + elif operation == "records": + records = self.session.get_execution_records(body.get("session_id")) + value = [record.model_dump(mode="json") for record in records] + elif operation == "shutdown": + self.shutdown_requested.set() + value = {"accepted": True} + else: + return JSONResponse({"error": f"unknown control operation: {operation}"}, 404) + return JSONResponse(value) + + async def _handle_mcp(self, body: Any) -> dict[str, Any]: + if not isinstance(body, dict): + return _error(None, -32600, "Invalid request") + request_id = body.get("id") + method = body.get("method") + if method == "initialize": + return _result( + request_id, + { + "protocolVersion": "2025-11-25", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "dimos-code-policy", "version": "1.0.0"}, + }, + ) + if method == "tools/list": + return _result(request_id, {"tools": [PYTHON_EXEC_TOOL]}) + if method != "tools/call": + return _error(request_id, -32601, f"Unknown: {method}") + params = body.get("params") or {} + if params.get("name") != "python_exec": + return _result(request_id, _text(f"Tool not found: {params.get('name', '')}")) + arguments = params.get("arguments") or {} + if not isinstance(arguments, dict) or set(arguments) - {"code", "timeout_s"}: + return _result(request_id, _text("Invalid python_exec arguments")) + code = arguments.get("code") + timeout_s = arguments.get("timeout_s", MAX_EXECUTION_TIMEOUT_S) + if not isinstance(code, str) or not code: + return _result(request_id, _text("python_exec code must be a non-empty string")) + if isinstance(timeout_s, bool) or not isinstance(timeout_s, (int, float)): + return _result(request_id, _text("python_exec timeout_s must be numeric")) + transcript = await asyncio.to_thread(self.session.python_exec, code, float(timeout_s)) + return _result(request_id, _text(transcript)) + + +class StandaloneCodePolicyProcess: + """Runner-owned standalone process plus private control client.""" + + def __init__(self, config: CodePolicySessionConfig) -> None: + self.config = config + self.port = _available_port() + self.mcp_url = f"http://127.0.0.1:{self.port}/mcp" + self.control_url = f"http://127.0.0.1:{self.port}/control" + self.process: subprocess.Popen[str] | None = None + + def start(self, timeout_s: float = 10.0) -> None: + if self.process is not None: + raise RuntimeError("standalone CodePolicy process already started") + self.process = subprocess.Popen( + ( + sys.executable, + "-m", + "dimos.agents.code_policy_server", + "--config-json", + self.config.model_dump_json(), + "--port", + str(self.port), + ), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if not McpAdapter(self.mcp_url, timeout=2).wait_for_ready(timeout=timeout_s, interval=0.05): + self.close() + raise TimeoutError("standalone CodePolicy process did not become ready") + + def receipt(self) -> dict[str, Any]: + value = self._control("receipt") + if not isinstance(value, dict): + raise TypeError("CodePolicy receipt response is not an object") + return value + + def reset(self) -> dict[str, Any]: + value = self._control("reset") + if not isinstance(value, dict): + raise TypeError("CodePolicy reset response is not an object") + return value + + def records(self, session_id: str | None = None) -> list[dict[str, Any]]: + value = self._control("records", {"session_id": session_id}) + if not isinstance(value, list): + raise TypeError("CodePolicy records response is not a list") + return value + + def interrupt(self) -> bool: + return bool(self._control("interrupt").get("interrupted")) + + def close(self) -> None: + process = self.process + self.process = None + if process is None: + return + if process.poll() is None: + try: + self._control("shutdown") + process.wait(timeout=5) + except Exception: + process.terminate() + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=2) + + def _control(self, operation: str, body: dict[str, Any] | None = None) -> Any: + response = requests.post(f"{self.control_url}/{operation}", json=body or {}, timeout=5) + response.raise_for_status() + return response.json() + + def __enter__(self) -> StandaloneCodePolicyProcess: + self.start() + return self + + def __exit__(self, *_args: Any) -> None: + self.close() + + +def _result(request_id: Any, result: Any) -> dict[str, Any]: + return {"jsonrpc": "2.0", "id": request_id, "result": result} + + +def _error(request_id: Any, code: int, message: str) -> dict[str, Any]: + return {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}} + + +def _text(value: str) -> dict[str, Any]: + return {"content": [{"type": "text", "text": value}]} + + +def _available_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m dimos.agents.code_policy_server") + parser.add_argument("--config-json") + parser.add_argument("--port", type=int, default=0) + environment = parser.add_mutually_exclusive_group() + environment.add_argument("--live-memory") + environment.add_argument("--frozen-source") + parser.add_argument("--derived-memory") + parser.add_argument("--cutoff-timestamp", type=float) + args = parser.parse_args(argv) + if args.config_json: + config = CodePolicySessionConfig.model_validate_json(args.config_json) + elif args.live_memory: + config = CodePolicySessionConfig( + environment=LiveDimosEnvironment(recording_path=args.live_memory) + ) + elif args.frozen_source and args.derived_memory and args.cutoff_timestamp is not None: + config = CodePolicySessionConfig( + environment=FrozenMemoryEnvironment( + recording_path=args.frozen_source, + derived_recording_path=args.derived_memory, + memory_cutoff_timestamp=args.cutoff_timestamp, + ) + ) + else: + parser.error("provide --config-json, --live-memory, or all frozen source arguments") + StandaloneCodePolicyServer(config, port=args.port).run_forever() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dimos/agents/mcp/mcp_adapter.py b/dimos/agents/mcp/mcp_adapter.py index 213bf71e23..dc41b801fc 100644 --- a/dimos/agents/mcp/mcp_adapter.py +++ b/dimos/agents/mcp/mcp_adapter.py @@ -116,7 +116,7 @@ def wait_for_ready(self, timeout: float = 10.0, interval: float = 0.5) -> bool: ) if resp.status_code == 200: return True - except requests.ConnectionError: + except (requests.ConnectionError, requests.ReadTimeout): pass time.sleep(interval) return False diff --git a/dimos/agents/mcp/test_mcp_adapter.py b/dimos/agents/mcp/test_mcp_adapter.py new file mode 100644 index 0000000000..d1de1975f4 --- /dev/null +++ b/dimos/agents/mcp/test_mcp_adapter.py @@ -0,0 +1,29 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests + +from dimos.agents.mcp.mcp_adapter import McpAdapter + + +def test_wait_for_ready_retries_read_timeout(mocker) -> None: + ready = mocker.Mock(status_code=200) + post = mocker.patch( + "dimos.agents.mcp.mcp_adapter.requests.post", + side_effect=[requests.ReadTimeout("busy"), ready], + ) + adapter = McpAdapter("http://localhost:9990/mcp") + + assert adapter.wait_for_ready(timeout=1.0, interval=0.0) + assert post.call_count == 2 diff --git a/dimos/agents/test_code_policy_core.py b/dimos/agents/test_code_policy_core.py new file mode 100644 index 0000000000..17c542dc2f --- /dev/null +++ b/dimos/agents/test_code_policy_core.py @@ -0,0 +1,108 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from dimos.agents.code_policy_core import ( + CodePolicySession, + CodePolicySessionConfig, + FrozenMemoryEnvironment, + LiveDimosEnvironment, +) +from dimos.memory2.store.sqlite import SqliteStore + + +def test_plain_session_persists_and_resets_without_module(mocker, tmp_path: Path) -> None: + mocker.patch("dimos.agents.code_policy_core._bootstrap_source", return_value="pass") + session = CodePolicySession( + CodePolicySessionConfig( + environment=LiveDimosEnvironment(recording_path=str(tmp_path / "unused.db")) + ) + ) + session.start() + try: + assert "[1]" in session.python_exec("items = [1]\nitems") + assert "[1, 2]" in session.python_exec("items.append(2)\nitems") + first = session.get_session_receipt() + second = session.reset_session() + assert second.previous_session_id == first.session_id + assert "NameError" in session.python_exec("items") + finally: + session.stop() + + +def test_frozen_session_bootstrap_exposes_memory_without_app(tmp_path: Path) -> None: + source_path = tmp_path / "source.db" + derived_path = tmp_path / "derived.db" + with SqliteStore(path=str(source_path)) as source: + source.stream("messages", str).append("before", ts=1.0) + source.stream("messages", str).append("after", ts=3.0) + with SqliteStore(path=str(derived_path)) as derived: + derived.stream("global_map", str).append("map", ts=2.0) + + session = CodePolicySession( + CodePolicySessionConfig( + environment=FrozenMemoryEnvironment( + recording_path=str(source_path), + derived_recording_path=str(derived_path), + memory_cutoff_timestamp=2.0, + ) + ) + ) + session.start() + try: + result = session.python_exec( + "([item.data for item in memory.streams.messages], " + "memory.streams.global_map.last().data, 'app' in globals())" + ) + assert "(['before'], 'map', False)" in result + finally: + session.stop() + + +def test_live_read_only_memory_observes_committed_writer_data(mocker, tmp_path: Path) -> None: + path = tmp_path / "live.db" + with SqliteStore(path=str(path)) as writer: + writer.stream("events", int).append(1, ts=1.0) + + bootstrap = f""" +from dimos.memory2.store.sqlite import SqliteStore +memory = SqliteStore(path={str(path)!r}, must_exist=True, read_only=True) +memory.start() +""" + mocker.patch("dimos.agents.code_policy_core._bootstrap_source", return_value=bootstrap) + session = CodePolicySession( + CodePolicySessionConfig(environment=LiveDimosEnvironment(recording_path=str(path))) + ) + session.start() + try: + assert "\n\n1" in session.python_exec("memory.streams.events.count()") + with SqliteStore(path=str(path)) as writer: + writer.stream("events", int).append(2, ts=2.0) + assert "\n\n2" in session.python_exec("memory.streams.events.count()") + mutation = session.python_exec("memory.streams.events.append(3)") + assert "PermissionError" in mutation + finally: + session.stop() + + +def test_frozen_environment_requires_all_fields() -> None: + with pytest.raises(ValueError): + FrozenMemoryEnvironment.model_validate( + {"kind": "frozen_memory", "recording_path": "source.db"} + ) diff --git a/dimos/agents/test_code_policy_server.py b/dimos/agents/test_code_policy_server.py new file mode 100644 index 0000000000..afceb93b99 --- /dev/null +++ b/dimos/agents/test_code_policy_server.py @@ -0,0 +1,110 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from pathlib import Path + +import requests + +from dimos.agents.code_policy_core import ( + CodePolicySessionConfig, + FrozenMemoryEnvironment, +) +from dimos.agents.code_policy_server import ( + StandaloneCodePolicyProcess, + StandaloneCodePolicyServer, +) +from dimos.agents.mcp.mcp_adapter import McpAdapter +from dimos.benchmark.agent_eval.pi_adapter import inspect_python_exec_inventory +from dimos.core.module import Module +from dimos.memory2.store.sqlite import SqliteStore + + +def _config(tmp_path: Path) -> CodePolicySessionConfig: + source = tmp_path / "source.db" + derived = tmp_path / "derived.db" + with SqliteStore(path=str(source)) as store: + store.stream("messages", str).append("visible", ts=1.0) + with SqliteStore(path=str(derived)) as store: + store.stream("global_map", str).append("map", ts=1.0) + return CodePolicySessionConfig( + environment=FrozenMemoryEnvironment( + recording_path=str(source), + derived_recording_path=str(derived), + memory_cutoff_timestamp=1.0, + ) + ) + + +def test_standalone_server_has_exact_direct_mcp_surface(tmp_path: Path) -> None: + server = StandaloneCodePolicyServer(_config(tmp_path)) + assert not isinstance(server, Module) + server.start() + adapter = McpAdapter(server.mcp_url, timeout=5) + try: + tools = adapter.list_tools() + assert [tool["name"] for tool in tools] == ["python_exec"] + inspect_python_exec_inventory(server.mcp_url, tools) + first = adapter.call_tool_text("python_exec", {"code": "items = [1]\nitems"}) + second = adapter.call_tool_text("python_exec", {"code": "items.append(2)\nitems"}) + assert "[1]" in first + assert "[1, 2]" in second + receipt = server.session.get_session_receipt() + assert len(server.session.get_execution_records(receipt.session_id)) == 2 + finally: + server.stop() + assert adapter.wait_for_down(timeout=2, interval=0.05) + + +def test_standalone_process_control_resets_and_collects_records(tmp_path: Path) -> None: + process = StandaloneCodePolicyProcess(_config(tmp_path)) + process.start() + adapter = McpAdapter(process.mcp_url, timeout=5) + try: + receipt = process.receipt() + result = adapter.call_tool_text( + "python_exec", {"code": "memory.streams.messages.last().data"} + ) + assert "visible" in result + assert "app" not in adapter.call_tool_text("python_exec", {"code": "sorted(globals())"}) + assert len(process.records(receipt["session_id"])) == 2 + reset = process.reset() + assert reset["previous_session_id"] == receipt["session_id"] + assert reset["session_id"] != receipt["session_id"] + finally: + process.close() + assert process.process is None + + +def test_server_stops_after_execution_start_failure(tmp_path: Path) -> None: + config = CodePolicySessionConfig( + environment=FrozenMemoryEnvironment( + recording_path=str(tmp_path / "missing.db"), + derived_recording_path=str(tmp_path / "also-missing.db"), + memory_cutoff_timestamp=1.0, + ) + ) + server = StandaloneCodePolicyServer(config) + server.start() + adapter = McpAdapter(server.mcp_url, timeout=5) + result = adapter.call_tool_text("python_exec", {"code": "1 + 1"}) + assert "failed to start" in result + server.stop() + try: + requests.post(server.mcp_url, timeout=0.2) + except (requests.ConnectionError, requests.ReadTimeout): + pass + else: + raise AssertionError("standalone service remained reachable after stop") diff --git a/dimos/benchmark/agent_eval/artifacts.py b/dimos/benchmark/agent_eval/artifacts.py new file mode 100644 index 0000000000..a33c2d4d19 --- /dev/null +++ b/dimos/benchmark/agent_eval/artifacts.py @@ -0,0 +1,78 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generic records for immutable evaluation evidence.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Annotated, Literal + +from pydantic import Field, JsonValue, model_validator + +from dimos.benchmark.agent_eval.base import BaseEvalModel + +NonEmpty = Annotated[str, Field(min_length=1)] +Sha256 = Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")] +AttemptId = Annotated[str, Field(pattern=r"^attempt_[0-9a-f]{32}$")] +OperationId = Annotated[str, Field(pattern=r"^operation_[0-9a-f]{32}$")] +CodePolicySessionId = Annotated[str, Field(pattern=r"^code_policy_session_[0-9a-f]{32}$")] + + +class ArtifactReference(BaseEvalModel): + record_type: Literal["artifact-reference"] = "artifact-reference" + path: NonEmpty + sha256: Sha256 + size_bytes: Annotated[int, Field(ge=0)] + + @model_validator(mode="after") + def path_is_relative(self) -> ArtifactReference: + if self.path.startswith("/") or ".." in self.path.split("/"): + raise ValueError("artifact path must be attempt-relative") + return self + + +class LifecycleEvent(BaseEvalModel): + record_type: Literal["agent-eval-lifecycle-event"] = "agent-eval-lifecycle-event" + sequence: Annotated[int, Field(ge=1)] + attempt_id: AttemptId + operation_id: OperationId | None = None + occurred_at: datetime + monotonic_offset_s: Annotated[float, Field(ge=0)] + kind: NonEmpty + payload: dict[str, JsonValue] = Field(default_factory=dict) + + +class NormalizedOutcome(BaseEvalModel): + """Generic terminal record retained for attempt-store callers.""" + + record_type: Literal["agent-eval-outcome"] = "agent-eval-outcome" + attempt_id: AttemptId + attempt_status: Literal["completed", "failed"] + task_result: Literal["passed", "failed", "not_evaluated"] + terminal_stage: NonEmpty + reason: NonEmpty + required_artifacts_complete: bool + finished_at: datetime + duration_s: Annotated[float, Field(ge=0)] + + @model_validator(mode="after") + def infrastructure_and_task_states_are_consistent(self) -> NormalizedOutcome: + if self.attempt_status == "failed" and self.task_result != "not_evaluated": + raise ValueError("failed infrastructure cannot report a task result") + if self.attempt_status == "completed" and self.task_result == "not_evaluated": + raise ValueError("completed evaluation must report pass or fail") + if self.attempt_status == "completed" and not self.required_artifacts_complete: + raise ValueError("completed evaluation requires complete artifacts") + return self diff --git a/dimos/benchmark/agent_eval/auth.py b/dimos/benchmark/agent_eval/auth.py new file mode 100644 index 0000000000..9b29d17211 --- /dev/null +++ b/dimos/benchmark/agent_eval/auth.py @@ -0,0 +1,25 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Runtime-only Pi credential transport without benchmark dependencies.""" + +from dataclasses import dataclass +from typing import Literal + + +@dataclass(frozen=True) +class RuntimeCredential: + auth_mode: Literal["subscription", "environment"] + binding_name: str + value: str | None diff --git a/dimos/benchmark/agent_eval/base.py b/dimos/benchmark/agent_eval/base.py new file mode 100644 index 0000000000..4fe7a15771 --- /dev/null +++ b/dimos/benchmark/agent_eval/base.py @@ -0,0 +1,26 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared Pydantic policy for serialized evaluation contracts.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict + + +class BaseEvalModel(BaseModel): + """Strict immutable base for serialized evaluation contracts.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + schema_version: Literal["1.0"] = "1.0" diff --git a/dimos/benchmark/agent_eval/case.py b/dimos/benchmark/agent_eval/case.py new file mode 100644 index 0000000000..4aec9021b0 --- /dev/null +++ b/dimos/benchmark/agent_eval/case.py @@ -0,0 +1,232 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Canonical source/task/interaction/validator contracts for agent evaluation.""" + +from __future__ import annotations + +import hashlib +import math +from pathlib import PurePosixPath +from typing import Annotated, Literal + +from pydantic import Field, JsonValue, model_validator + +from dimos.benchmark.agent_eval.base import BaseEvalModel +from dimos.benchmark.agent_eval.json import canonical_json + +NonEmpty = Annotated[str, Field(min_length=1)] +Sha256 = Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")] +NormalizedProgress = Annotated[float, Field(ge=0.0, le=1.0, allow_inf_nan=False)] + + +class FrozenRecordingSource(BaseEvalModel): + kind: Literal["frozen_memory"] = "frozen_memory" + recording: NonEmpty + progress: NormalizedProgress + bundle_manifest_sha256: Sha256 | None = None + + @model_validator(mode="after") + def progress_is_finite(self) -> FrozenRecordingSource: + if not math.isfinite(self.progress): + raise ValueError("recording progress must be finite") + return self + + +SourceSpec = FrozenRecordingSource + + +class IntegerQuestionTask(BaseEvalModel): + kind: Literal["integer_question"] = "integer_question" + prompt: NonEmpty + answer_marker: Literal["ANSWER:"] = "ANSWER:" + + +TaskSpec = IntegerQuestionTask + + +class FrozenCodePolicyInteraction(BaseEvalModel): + kind: Literal["frozen_code_policy"] = "frozen_code_policy" + driver_revision: NonEmpty + session_lifetime: Literal["one_attempt"] = "one_attempt" + + +InteractionSpec = FrozenCodePolicyInteraction + + +class ExactIntegerValidatorRef(BaseEvalModel): + kind: Literal["exact_integer"] = "exact_integer" + revision: NonEmpty + private_path: NonEmpty + private_sha256: Sha256 + + @model_validator(mode="after") + def private_path_is_relative(self) -> ExactIntegerValidatorRef: + path = PurePosixPath(self.private_path) + if path.is_absolute() or not path.parts or ".." in path.parts: + raise ValueError("validator private_path must be a safe relative path") + return self + + +ValidatorRef = ExactIntegerValidatorRef + + +class PublicEvalCase(BaseEvalModel): + """Agent-safe case projection; it intentionally cannot carry a validator.""" + + case_id: NonEmpty + source: SourceSpec + task: TaskSpec + interaction: InteractionSpec + + +class EvalCase(BaseEvalModel): + """Compiled private case binding all four semantic contracts.""" + + case_id: NonEmpty + source: SourceSpec + task: TaskSpec + interaction: InteractionSpec + validator: ValidatorRef + fingerprint: Sha256 + + @classmethod + def compile( + cls, + *, + case_id: str, + source: SourceSpec, + task: TaskSpec, + interaction: InteractionSpec, + validator: ValidatorRef, + ) -> EvalCase: + payload = _case_payload(case_id, source, task, interaction, validator) + return cls( + case_id=case_id, + source=source, + task=task, + interaction=interaction, + validator=validator, + fingerprint=hashlib.sha256(canonical_json(payload)).hexdigest(), + ) + + @model_validator(mode="after") + def fingerprint_matches_payload(self) -> EvalCase: + payload = _case_payload( + self.case_id, + self.source, + self.task, + self.interaction, + self.validator, + ) + expected = hashlib.sha256(canonical_json(payload)).hexdigest() + if self.fingerprint != expected: + raise ValueError("evaluation case fingerprint does not match its contracts") + return self + + def public_projection(self) -> PublicEvalCase: + return PublicEvalCase( + case_id=self.case_id, + source=self.source, + task=self.task, + interaction=self.interaction, + ) + + +class AgentCondition(BaseEvalModel): + agent_id: NonEmpty + adapter: NonEmpty + model: NonEmpty + thinking_level: NonEmpty + + +class RuntimeBinding(BaseEvalModel): + runtime_id: NonEmpty + parameters: dict[str, JsonValue] = Field(default_factory=dict) + + +class AttemptRequest(BaseEvalModel): + case: EvalCase + agent: AgentCondition + runtime: RuntimeBinding + seed: int | None = None + + +class AgentOutcome(BaseEvalModel): + final_text: str + tool_call_count: int = Field(ge=0) + terminal_reason: NonEmpty + agent_session_id: NonEmpty | None = None + interaction_session_id: NonEmpty | None = None + + +class Prediction(BaseEvalModel): + case_id: NonEmpty + attempt_id: NonEmpty + agent_session_id: NonEmpty + interaction_session_id: NonEmpty + parser_revision: NonEmpty + final_text: str + status: Literal["parsed", "invalid"] + integer_answer: int | None = None + diagnostic: NonEmpty | None = None + + @model_validator(mode="after") + def answer_matches_status(self) -> Prediction: + if self.status == "parsed" and (self.integer_answer is None or self.diagnostic is not None): + raise ValueError("parsed prediction requires only integer_answer") + if self.status == "invalid" and ( + self.integer_answer is not None or self.diagnostic is None + ): + raise ValueError("invalid prediction requires only diagnostic") + return self + + +class PrivateScore(BaseEvalModel): + case_id: NonEmpty + attempt_id: NonEmpty + validator_revision: NonEmpty + passed: bool + prediction_status: Literal["parsed", "invalid"] + + +class EvalOutcome(BaseEvalModel): + attempt_id: NonEmpty + attempt_status: Literal["completed", "failed"] + task_result: Literal["passed", "failed", "not_evaluated"] + reason: NonEmpty + + @model_validator(mode="after") + def states_are_consistent(self) -> EvalOutcome: + if self.attempt_status == "failed" and self.task_result != "not_evaluated": + raise ValueError("failed infrastructure cannot claim a task result") + if self.attempt_status == "completed" and self.task_result == "not_evaluated": + raise ValueError("completed evaluation must report passed or failed") + return self + + +def _case_payload( + case_id: str, + source: SourceSpec, + task: TaskSpec, + interaction: InteractionSpec, + validator: ValidatorRef, +) -> dict[str, JsonValue]: + return { + "case_id": case_id, + "source": source.model_dump(mode="json"), + "task": task.model_dump(mode="json"), + "interaction": interaction.model_dump(mode="json"), + "validator": validator.model_dump(mode="json"), + } diff --git a/dimos/benchmark/agent_eval/engine.py b/dimos/benchmark/agent_eval/engine.py new file mode 100644 index 0000000000..f03aa5b763 --- /dev/null +++ b/dimos/benchmark/agent_eval/engine.py @@ -0,0 +1,262 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared single-attempt engine for canonical agent-evaluation cases.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, ConfigDict, JsonValue + +from dimos.benchmark.agent_eval.artifacts import ArtifactReference +from dimos.benchmark.agent_eval.case import AttemptRequest, EvalOutcome, PrivateScore +from dimos.benchmark.agent_eval.interfaces import ( + AgentAdapter, + AttemptContext, + EvidenceSink, + InteractionDriver, + SourceDriver, + ValidatorDriver, + ValidatorSession, +) +from dimos.benchmark.agent_eval.store import AttemptStore + + +class EngineResult(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + attempt_path: Path + outcome: EvalOutcome + artifacts: tuple[ArtifactReference, ...] + + +class AttemptEvidence(EvidenceSink): + def __init__(self, store: AttemptStore) -> None: + self.store = store + self.artifacts: list[ArtifactReference] = [] + + def event(self, kind: str, payload: dict[str, JsonValue] | None = None) -> None: + self.store.append_event(kind, payload=payload) + + def artifact(self, relative_path: str, value: BaseModel | JsonValue | bytes | str) -> None: + self.artifacts.append(self.store.write_artifact(relative_path, value)) + + def reference(self, relative_path: str) -> None: + self.artifacts.append(_reference(self.store.path, relative_path)) + + +class AttemptEngine: + """Coordinate source, private validator, interaction, evidence, and cleanup.""" + + def __init__( + self, + *, + request: AttemptRequest, + output_root: Path, + source: SourceDriver, + interaction: InteractionDriver, + validator: ValidatorDriver, + agent: AgentAdapter, + ) -> None: + self.request = request + self.output_root = output_root + self.source = source + self.interaction = interaction + self.validator = validator + self.agent = agent + + def run(self) -> EngineResult: + store = AttemptStore(self.output_root) + result: EngineResult | None = None + close_error: Exception | None = None + try: + result = self._run_reserved(store) + finally: + try: + store.close() + except Exception as exc: + close_error = exc + if close_error is not None: + return EngineResult( + attempt_path=store.path, + outcome=EvalOutcome( + attempt_id=store.attempt_id, + attempt_status="failed", + task_result="not_evaluated", + reason=f"attempt store cleanup failed: {type(close_error).__name__}: {close_error}"[ + :1024 + ], + ), + artifacts=result.artifacts if result is not None else (), + ) + assert result is not None + return result + + def _run_reserved(self, store: AttemptStore) -> EngineResult: + evidence = AttemptEvidence(store) + context = AttemptContext( + attempt_id=store.attempt_id, + path=store.path, + request=self.request, + ) + validator_session: ValidatorSession | None = None + agent_outcome = None + outcome: EvalOutcome + completed = False + passed = False + reason = "infrastructure failure" + try: + evidence.event("attempt-created") + evidence.artifact("case.private.v1.json", self.request.case) + evidence.artifact("case.public.v1.json", self.request.case.public_projection()) + prepared = self.source.prepare( + source=self.request.case.source, + context=context, + evidence=evidence, + ) + evidence.event("source-prepared") + validator_session = self.validator.prepare( + case=self.request.case, + prepared_source=prepared, + context=context, + evidence=evidence, + ) + evidence.event("validator-prepared") + agent_outcome = self.interaction.run( + case=self.request.case, + prepared_source=prepared, + agent=self.agent, + context=context, + evidence=evidence, + ) + evidence.artifact("agent-outcome.v1.json", agent_outcome) + evidence.event("interaction-completed") + score = validator_session.evaluate(agent_outcome) + _validate_score(score, store.attempt_id, self.request.case.case_id) + evidence.artifact("score.private.v1.json", score) + evidence.event("validation-completed", {"passed": score.passed}) + completed = True + passed = score.passed + reason = "validator passed" if passed else "validator failed" + except KeyboardInterrupt: + reason = "user interrupted" + _safe_event(evidence, "attempt-interrupted") + except Exception as exc: + reason = f"{type(exc).__name__}: {exc}"[:1024] + _safe_event(evidence, "infrastructure-failure", {"diagnostic": reason}) + cleanup_errors = self._cleanup(validator_session) + if cleanup_errors: + _safe_event(evidence, "cleanup-failure", {"diagnostic": "; ".join(cleanup_errors)}) + completed = False + passed = False + reason = "; ".join(cleanup_errors) + try: + evidence.reference("events.jsonl") + evidence.artifact( + "attempt-manifest.v1.json", + { + "schema_version": "1.0", + "attempt_id": store.attempt_id, + "case_id": self.request.case.case_id, + "case_fingerprint": self.request.case.fingerprint, + "agent": self.request.agent.model_dump(mode="json"), + "runtime": self.request.runtime.model_dump(mode="json"), + "agent_session_id": ( + agent_outcome.agent_session_id if agent_outcome is not None else None + ), + "interaction_session_id": ( + agent_outcome.interaction_session_id if agent_outcome is not None else None + ), + "artifacts": [ + artifact.model_dump(mode="json") for artifact in evidence.artifacts + ], + }, + ) + except Exception as exc: + completed = False + passed = False + reason = f"attempt finalization failed: {type(exc).__name__}: {exc}"[:1024] + _safe_event(evidence, "finalization-failure", {"diagnostic": reason}) + outcome = EvalOutcome( + attempt_id=store.attempt_id, + attempt_status="completed" if completed else "failed", + task_result=("passed" if passed else "failed") if completed else "not_evaluated", + reason=reason, + ) + try: + evidence.artifacts.append(store.write_eval_outcome(outcome)) + except Exception as exc: + outcome = EvalOutcome( + attempt_id=store.attempt_id, + attempt_status="failed", + task_result="not_evaluated", + reason=f"terminal publication failed: {type(exc).__name__}: {exc}"[:1024], + ) + _safe_event( + evidence, + "terminal-publication-failure", + {"diagnostic": outcome.reason}, + ) + return EngineResult( + attempt_path=store.path, + outcome=outcome, + artifacts=tuple(evidence.artifacts), + ) + + def _cleanup(self, validator_session: ValidatorSession | None) -> list[str]: + errors: list[str] = [] + resources: tuple[tuple[str, Any], ...] = ( + ("agent", self.agent), + ("interaction", self.interaction), + ("validator", validator_session), + ("source", self.source), + ) + for name, resource in resources: + if resource is None: + continue + try: + resource.close() + except Exception as exc: + errors.append(f"{name}: {type(exc).__name__}: {exc}"[:1024]) + return errors + + +def _validate_score(score: PrivateScore, attempt_id: str, case_id: str) -> None: + if score.attempt_id != attempt_id or score.case_id != case_id: + raise ValueError("validator score identity mismatch") + + +def _reference(root: Path, relative_path: str) -> ArtifactReference: + import hashlib + + data = (root / relative_path).read_bytes() + return ArtifactReference( + path=relative_path, + sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + ) + + +def _safe_event( + evidence: AttemptEvidence, + kind: str, + payload: dict[str, JsonValue] | None = None, +) -> None: + try: + evidence.event(kind, payload) + except Exception: + # Evidence storage is already failing; cleanup and lock release remain mandatory. + return diff --git a/dimos/benchmark/agent_eval/interfaces.py b/dimos/benchmark/agent_eval/interfaces.py new file mode 100644 index 0000000000..35b98982cd --- /dev/null +++ b/dimos/benchmark/agent_eval/interfaces.py @@ -0,0 +1,106 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backend-neutral interfaces used by canonical agent-evaluation attempts.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Protocol + +from pydantic import BaseModel, ConfigDict, JsonValue + +from dimos.benchmark.agent_eval.case import ( + AgentOutcome, + AttemptRequest, + EvalCase, + PrivateScore, + SourceSpec, + TaskSpec, +) + + +class InterfaceModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class AttemptContext(InterfaceModel): + attempt_id: str + path: Path + request: AttemptRequest + + +class PreparedSource(InterfaceModel): + public: dict[str, JsonValue] + receipt: dict[str, JsonValue] + private_handle: Any = None + + +class AgentAdapter(Protocol): + def run( + self, *, task: TaskSpec, context: AttemptContext, interface: Any = None + ) -> AgentOutcome: ... + + def close(self) -> None: ... + + +class EvidenceSink(Protocol): + def event(self, kind: str, payload: dict[str, JsonValue] | None = None) -> None: ... + + def artifact(self, relative_path: str, value: BaseModel | JsonValue | bytes | str) -> None: ... + + def reference(self, relative_path: str) -> None: ... + + +class SourceDriver(Protocol): + def prepare( + self, + *, + source: SourceSpec, + context: AttemptContext, + evidence: EvidenceSink, + ) -> PreparedSource: ... + + def close(self) -> None: ... + + +class InteractionDriver(Protocol): + def run( + self, + *, + case: EvalCase, + prepared_source: PreparedSource, + agent: AgentAdapter, + context: AttemptContext, + evidence: EvidenceSink, + ) -> AgentOutcome: ... + + def close(self) -> None: ... + + +class ValidatorSession(Protocol): + def evaluate(self, outcome: AgentOutcome) -> PrivateScore: ... + + def close(self) -> None: ... + + +class ValidatorDriver(Protocol): + def prepare( + self, + *, + case: EvalCase, + prepared_source: PreparedSource, + context: AttemptContext, + evidence: EvidenceSink, + ) -> ValidatorSession: ... diff --git a/dimos/benchmark/agent_eval/json.py b/dimos/benchmark/agent_eval/json.py new file mode 100644 index 0000000000..527804362d --- /dev/null +++ b/dimos/benchmark/agent_eval/json.py @@ -0,0 +1,29 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Deterministic JSON encoding for evaluation fingerprints and artifacts.""" + +import json +from typing import Any + + +def canonical_json(value: Any) -> bytes: + """Encode JSON with stable ordering, compact separators, and UTF-8 bytes.""" + return json.dumps( + value, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") diff --git a/dimos/benchmark/agent_eval/pi.py b/dimos/benchmark/agent_eval/pi.py new file mode 100644 index 0000000000..75aea78c21 --- /dev/null +++ b/dimos/benchmark/agent_eval/pi.py @@ -0,0 +1,56 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backend-neutral Pi code-policy session contracts.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Protocol + +from pydantic import BaseModel, ConfigDict, Field + +from dimos.benchmark.agent_eval.artifacts import ArtifactReference +from dimos.benchmark.agent_eval.pi_adapter import CodePolicyCallLog, McpBinding + + +class PiTurn(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + final_text: str = "" + policy_call_count: int = Field(ge=0) + + +class PiSession(Protocol): + session_id: str + + def prompt(self, prompt: str, timeout_s: float) -> PiTurn: ... + + def abort(self, timeout_s: float) -> None: ... + + def dispose(self) -> None: ... + + def artifact_references(self) -> tuple[ArtifactReference, ...]: ... + + +class PiSessionFactory(Protocol): + def create( + self, + *, + attempt_path: Path, + public_prompt: str, + code_policy_session_id: str, + call_log: CodePolicyCallLog, + mcp: McpBinding, + ) -> PiSession: ... diff --git a/dimos/benchmark/agent_eval/pi_adapter.py b/dimos/benchmark/agent_eval/pi_adapter.py new file mode 100644 index 0000000000..b4ebd816cd --- /dev/null +++ b/dimos/benchmark/agent_eval/pi_adapter.py @@ -0,0 +1,302 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""One-tool Pi facade over an attached DimOS MCP server. + +The attached server is intentionally allowed to expose the normal robot skill +inventory. This module records that inventory but admits only the exact +``python_exec`` schema into the Pi model-facing session. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from datetime import UTC, datetime +import hashlib +import json +import os +from pathlib import Path +import threading +import time +from typing import Any, Protocol +from uuid import uuid4 + +from pydantic import BaseModel, ConfigDict, Field, JsonValue + +from dimos.agents.code_policy_core import MAX_EXECUTION_TIMEOUT_S +from dimos.benchmark.agent_eval.artifacts import ( + AttemptId, + CodePolicySessionId, + NonEmpty, +) +from dimos.benchmark.agent_eval.json import canonical_json + +PYTHON_EXEC_TOOL_NAME = "python_exec" +PI_TOOL_NAMES = (PYTHON_EXEC_TOOL_NAME,) +_EXPECTED_DESCRIPTION_PREFIX = ( + "Execute one synchronous Python program in the persistent policy session." +) + + +class McpBinding(Protocol): + """Minimum MCP behavior used by the one-tool facade.""" + + def wait_for_ready(self, timeout: float) -> bool: ... + + def list_tools(self) -> list[dict[str, Any]]: ... + + def call_tool(self, name: str, arguments: dict[str, Any] | None = None) -> dict[str, Any]: ... + + +class PiAdapterModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + schema_version: str = "1.0" + + +class McpInventoryReceipt(PiAdapterModel): + record_type: str = "mcp-inventory-receipt" + endpoint: NonEmpty + observed_tools: tuple[dict[str, JsonValue], ...] + python_exec_schema_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + +class CodePolicyCallRecord(PiAdapterModel): + record_type: str = "code-policy-call" + call_id: NonEmpty + attempt_id: AttemptId + pi_session_id: NonEmpty + code_policy_session_id: CodePolicySessionId + tool_name: str + arguments: dict[str, JsonValue] + requested_at: datetime + completed_at: datetime + monotonic_duration_s: float = Field(ge=0) + ok: bool + result: dict[str, JsonValue] | None = None + error: NonEmpty | None = None + + +class ToolInventoryError(ValueError): + """The attached MCP inventory cannot safely back the Pi facade.""" + + +def inspect_python_exec_inventory( + endpoint: str, + tools: Sequence[Mapping[str, Any]], +) -> McpInventoryReceipt: + """Validate one exact code-policy tool while retaining the full inventory.""" + observed = tuple(_json_tool(tool) for tool in tools) + matches = [tool for tool in observed if tool.get("name") == PYTHON_EXEC_TOOL_NAME] + if len(matches) != 1: + raise ToolInventoryError("MCP inventory must contain exactly one python_exec tool") + schema = matches[0].get("inputSchema") + description = matches[0].get("description") + if not isinstance(schema, dict) or not _is_python_exec_schema(schema): + raise ToolInventoryError("python_exec input schema is incompatible") + if ( + not isinstance(description, str) + or not description.startswith(_EXPECTED_DESCRIPTION_PREFIX) + or "trusted, unsandboxed" not in description + ): + raise ToolInventoryError("python_exec description is incompatible") + return McpInventoryReceipt( + endpoint=endpoint, + observed_tools=observed, + python_exec_schema_sha256=hashlib.sha256(canonical_json(schema)).hexdigest(), + ) + + +def wait_for_python_exec( + endpoint: str, + mcp: McpBinding, + timeout_s: float, +) -> McpInventoryReceipt: + if timeout_s <= 0: + raise ValueError("MCP readiness timeout must be positive") + if not mcp.wait_for_ready(timeout_s): + raise TimeoutError(f"MCP server did not become ready at {endpoint}") + return inspect_python_exec_inventory(endpoint, mcp.list_tools()) + + +class CodePolicyCallLog: + """Append-only durable evidence for calls forwarded on Pi's behalf.""" + + def __init__(self, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + self.path = path + self._descriptor = os.open( + path, + os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, + 0o600, + ) + self._lock = threading.Lock() + self._closed = False + + def append(self, record: CodePolicyCallRecord) -> None: + encoded = canonical_json(record.model_dump(mode="json")) + b"\n" + with self._lock: + if self._closed: + raise RuntimeError("code-policy call log is closed") + view = memoryview(encoded) + while view: + view = view[os.write(self._descriptor, view) :] + os.fsync(self._descriptor) + + def close(self) -> None: + with self._lock: + if self._closed: + return + self._closed = True + os.close(self._descriptor) + + def __enter__(self) -> CodePolicyCallLog: + return self + + def __exit__(self, *_args: Any) -> None: + self.close() + + +class PythonExecBroker: + """Forward the sole Pi tool and bind evidence to both session identities.""" + + def __init__( + self, + *, + attempt_id: str, + pi_session_id: str, + code_policy_session_id: str, + mcp: McpBinding, + call_log: CodePolicyCallLog, + clock: Callable[[], float] = time.monotonic, + ) -> None: + self.attempt_id = attempt_id + self.pi_session_id = pi_session_id + self.code_policy_session_id = code_policy_session_id + self.mcp = mcp + self.call_log = call_log + self.clock = clock + self.call_count = 0 + + def request(self, tool_name: str, arguments: Mapping[str, Any]) -> dict[str, Any]: + if tool_name != PYTHON_EXEC_TOOL_NAME: + raise PermissionError(f"Pi tool {tool_name!r} is not permitted") + safe_arguments = _validate_arguments(arguments) + requested_at = datetime.now(UTC) + started = self.clock() + self.call_count += 1 + result: dict[str, Any] | None = None + error: str | None = None + try: + result = self.mcp.call_tool(PYTHON_EXEC_TOOL_NAME, safe_arguments) + if not isinstance(result, dict): + raise TypeError("MCP tool result must be an object") + return result + except Exception as exc: + error = _bounded_diagnostic(exc) + raise + finally: + safe_result = _json_object(result) if result is not None else None + self.call_log.append( + CodePolicyCallRecord( + call_id=f"pi_tool_call_{uuid4().hex}", + attempt_id=self.attempt_id, + pi_session_id=self.pi_session_id, + code_policy_session_id=self.code_policy_session_id, + tool_name=PYTHON_EXEC_TOOL_NAME, + arguments=safe_arguments, + requested_at=requested_at, + completed_at=datetime.now(UTC), + monotonic_duration_s=max(0.0, self.clock() - started), + ok=error is None, + result=safe_result, + error=error, + ) + ) + + +def credential_binding_sha256( + auth_mode: str, + binding_name: str, + credential: str | bytes | None = None, +) -> str: + """Return a domain-separated binding digest without retaining the secret.""" + if not auth_mode or not binding_name: + raise ValueError("authentication mode and binding name are required") + digest = hashlib.sha256() + digest.update(b"dimos-agent-eval-credential-binding-v1\0") + digest.update(auth_mode.encode()) + digest.update(b"\0") + digest.update(binding_name.encode()) + if credential is not None: + digest.update(b"\0") + digest.update( + hashlib.sha256( + credential.encode() if isinstance(credential, str) else credential + ).digest() + ) + return digest.hexdigest() + + +def _is_python_exec_schema(schema: Mapping[str, Any]) -> bool: + properties = schema.get("properties") + if ( + schema.get("type") != "object" + or schema.get("required") != ["code"] + or not isinstance(properties, dict) + or set(properties) != {"code", "timeout_s"} + ): + return False + code = properties["code"] + timeout = properties["timeout_s"] + return ( + isinstance(code, dict) + and code.get("type") == "string" + and isinstance(timeout, dict) + and timeout.get("type") == "number" + and timeout.get("default") == MAX_EXECUTION_TIMEOUT_S + ) + + +def _validate_arguments(arguments: Mapping[str, Any]) -> dict[str, JsonValue]: + if set(arguments) - {"code", "timeout_s"}: + raise ValueError("python_exec arguments contain unknown fields") + code = arguments.get("code") + timeout = arguments.get("timeout_s", MAX_EXECUTION_TIMEOUT_S) + if not isinstance(code, str) or not code: + raise ValueError("python_exec code must be a non-empty string") + if isinstance(timeout, bool) or not isinstance(timeout, (float, int)): + raise ValueError("python_exec timeout_s must be numeric") + if not 0 < float(timeout) <= MAX_EXECUTION_TIMEOUT_S: + raise ValueError("python_exec timeout_s is outside the supported range") + return {"code": code, "timeout_s": float(timeout)} + + +def _json_tool(tool: Mapping[str, Any]) -> dict[str, JsonValue]: + return _json_object(dict(tool)) + + +def _json_object(value: Mapping[str, Any]) -> dict[str, JsonValue]: + try: + encoded = json.dumps(value, allow_nan=False) + decoded = json.loads(encoded) + except (TypeError, ValueError) as exc: + raise ValueError("value is not strict JSON") from exc + if not isinstance(decoded, dict): + raise ValueError("value must be a JSON object") + return decoded + + +def _bounded_diagnostic(exc: Exception) -> str: + message = f"{type(exc).__name__}: {exc}".replace("\r", " ").replace("\n", " ") + return message[:1024] or type(exc).__name__ diff --git a/dimos/benchmark/agent_eval/pi_process.py b/dimos/benchmark/agent_eval/pi_process.py new file mode 100644 index 0000000000..f1a8f8f356 --- /dev/null +++ b/dimos/benchmark/agent_eval/pi_process.py @@ -0,0 +1,429 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Process binding for the interactive one-tool Pi SDK host.""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import queue +import subprocess +import threading +import time +from typing import IO, Any +from uuid import uuid4 + +from dimos.benchmark.agent_eval.artifacts import ArtifactReference +from dimos.benchmark.agent_eval.auth import RuntimeCredential +from dimos.benchmark.agent_eval.pi import PiTurn +from dimos.benchmark.agent_eval.pi_adapter import ( + CodePolicyCallLog, + McpBinding, + PythonExecBroker, +) +from dimos.benchmark.agent_eval.progress import ( + AssistantTextProgress, + FinalResponseProgress, + ProgressSink, + StatusProgress, + ToolEndProgress, + ToolStartProgress, + emit_progress, +) + +_PROTOCOL_VERSION = 1 +_MAX_FRAME_BYTES = 64 * 1024 +_MAX_STDERR_BYTES = 64 * 1024 +_MAX_PROGRESS_BYTES = 4 * 1024 + + +class NodePiSessionFactory: + def __init__( + self, + *, + command: tuple[str, ...], + credential: RuntimeCredential, + model: str, + thinking_level: str, + startup_timeout_s: float, + progress: ProgressSink | None = None, + ) -> None: + if not command or startup_timeout_s <= 0: + raise ValueError("Pi adapter command and startup timeout are required") + if model != "gpt-5.6-luna" or thinking_level != "medium": + raise ValueError("the pinned Pi adapter supports only gpt-5.6-luna/medium") + self.command = command + self.credential = credential + self.startup_timeout_s = startup_timeout_s + self.progress = progress + + def create( + self, + *, + attempt_path: Path, + public_prompt: str, + code_policy_session_id: str, + call_log: CodePolicyCallLog, + mcp: McpBinding, + ) -> NodePiSession: + session_id = f"pi_session_{uuid4().hex}" + broker = PythonExecBroker( + attempt_id=attempt_path.name, + pi_session_id=session_id, + code_policy_session_id=code_policy_session_id, + mcp=mcp, + call_log=call_log, + ) + return NodePiSession( + command=self.command, + credential=self.credential, + attempt_path=attempt_path, + session_id=session_id, + initial_prompt=public_prompt, + broker=broker, + startup_timeout_s=self.startup_timeout_s, + progress=self.progress, + ) + + +class NodePiSession: + def __init__( + self, + *, + command: tuple[str, ...], + credential: RuntimeCredential, + attempt_path: Path, + session_id: str, + initial_prompt: str, + broker: PythonExecBroker, + startup_timeout_s: float, + progress: ProgressSink | None, + ) -> None: + self.session_id = session_id + self.policy_call_count = 0 + self._attempt_path = attempt_path + self._broker = broker + self._frames: queue.Queue[dict[str, Any] | BaseException] = queue.Queue() + self._write_lock = threading.Lock() + self._closed_evidence: dict[str, Any] | None = None + self._disposed = False + self._progress = progress + self._stderr_path = attempt_path / "pi-adapter.stderr.log" + self._stderr = bytearray() + self._process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + cwd=attempt_path, + env=_adapter_environment(credential, attempt_path), + ) + assert ( + self._process.stdin is not None + and self._process.stdout is not None + and self._process.stderr is not None + ) + self._stdin: IO[str] = self._process.stdin + self._reader = threading.Thread( + target=self._read_frames, + args=(self._process.stdout,), + name=f"pi-adapter-reader-{session_id}", + daemon=True, + ) + self._stderr_reader = threading.Thread( + target=self._read_stderr, + args=(self._process.stderr,), + name=f"pi-adapter-stderr-{session_id}", + daemon=True, + ) + self._reader.start() + self._stderr_reader.start() + emit_progress(self._progress, StatusProgress(channel="pi", message="session starting")) + self._send( + { + "version": _PROTOCOL_VERSION, + "type": "session_start", + "id": session_id, + "initial_prompt": initial_prompt, + "thinking_level": "medium", + } + ) + try: + started = self._await("session_started", session_id, startup_timeout_s) + except BaseException: + self._terminate_process() + raise + if started.get("tools") != ["python_exec"]: + self.dispose() + raise RuntimeError("Pi adapter activated an unexpected tool inventory") + emit_progress(self._progress, StatusProgress(channel="pi", message="session started")) + + def prompt(self, prompt: str, timeout_s: float) -> PiTurn: + if self._disposed: + raise RuntimeError("Pi session is disposed") + turn_id = f"turn_{uuid4().hex}" + self._send( + { + "version": _PROTOCOL_VERSION, + "type": "prompt", + "id": turn_id, + "text": prompt, + } + ) + frame = self._await("turn_complete", turn_id, timeout_s) + count = frame.get("policy_call_count") + if not isinstance(count, int) or count < self.policy_call_count: + raise RuntimeError("Pi adapter returned an invalid policy-call count") + self.policy_call_count = count + final_text = frame.get("final_text") + text = final_text if isinstance(final_text, str) else "" + emit_progress(self._progress, FinalResponseProgress(text=_bounded_progress(text))) + return PiTurn( + final_text=text, + policy_call_count=count, + ) + + def abort(self, timeout_s: float) -> None: + del timeout_s + if not self._disposed and self._process.poll() is None: + self._send({"version": _PROTOCOL_VERSION, "type": "abort"}) + + def dispose(self) -> None: + if self._disposed: + return + self._disposed = True + try: + if self._process.poll() is None: + self._send({"version": _PROTOCOL_VERSION, "type": "dispose"}) + try: + frame = self._await("session_closed", self.session_id, 5.0) + evidence = frame.get("evidence") + if isinstance(evidence, dict): + self._closed_evidence = evidence + except (RuntimeError, TimeoutError): + self._process.terminate() + self._process.wait(timeout=2.0) + except subprocess.TimeoutExpired: + self._terminate_process() + finally: + self._reader.join(timeout=2.0) + self._stderr_reader.join(timeout=2.0) + self._stderr_path.write_bytes(bytes(self._stderr)) + + def _terminate_process(self) -> None: + self._disposed = True + if self._process.poll() is None: + self._process.terminate() + try: + self._process.wait(timeout=2.0) + except subprocess.TimeoutExpired: + self._process.kill() + self._process.wait(timeout=2.0) + self._reader.join(timeout=2.0) + self._stderr_reader.join(timeout=2.0) + self._stderr_path.write_bytes(bytes(self._stderr)) + + def artifact_references(self) -> tuple[ArtifactReference, ...]: + if not self._disposed: + raise RuntimeError("Pi session evidence is available only after disposal") + relative_paths = ["pi-adapter.stderr.log"] + evidence = self._closed_evidence or {} + session_path = evidence.get("relative_path") + if evidence.get("persisted") is True and isinstance(session_path, str): + relative_paths.append(session_path) + for key in ("system_prompt", "initial_prompt"): + prompt = evidence.get(key) + if isinstance(prompt, dict) and isinstance(prompt.get("relative_path"), str): + relative_paths.append(prompt["relative_path"]) + return tuple(_artifact(self._attempt_path, path) for path in relative_paths) + + def _read_frames(self, output: IO[str]) -> None: + try: + for line in output: + if len(line.encode()) > _MAX_FRAME_BYTES: + raise RuntimeError("Pi adapter frame exceeds limit") + frame = json.loads(line) + if not isinstance(frame, dict) or frame.get("version") != _PROTOCOL_VERSION: + raise RuntimeError("invalid Pi adapter frame") + if frame.get("type") == "tool_call": + self._handle_tool_call(frame) + elif frame.get("type") == "transcript": + self._handle_transcript(frame) + else: + self._frames.put(frame) + except BaseException as exc: + self._frames.put(exc) + + def _handle_tool_call(self, frame: dict[str, Any]) -> None: + call_id = frame.get("id") + tool = frame.get("tool") + params = frame.get("params") + if ( + not isinstance(call_id, str) + or not isinstance(tool, str) + or not isinstance(params, dict) + ): + raise RuntimeError("malformed Pi tool call") + code = params.get("code") + if isinstance(code, str) and code: + emit_progress( + self._progress, + ToolStartProgress(code=_bounded_progress(code)), + ) + started = time.monotonic() + try: + result = self._broker.request(tool, params) + text = _mcp_text(result) + emit_progress( + self._progress, + ToolEndProgress( + ok=True, + result=_bounded_progress(text), + duration_seconds=max(0.0, time.monotonic() - started), + ), + ) + reply = { + "version": _PROTOCOL_VERSION, + "type": "tool_reply", + "id": call_id, + "ok": True, + "result": text, + } + except Exception as exc: + diagnostic = f"{type(exc).__name__}: {exc}" + emit_progress( + self._progress, + ToolEndProgress( + ok=False, + result=_bounded_progress(diagnostic), + duration_seconds=max(0.0, time.monotonic() - started), + ), + ) + reply = { + "version": _PROTOCOL_VERSION, + "type": "tool_reply", + "id": call_id, + "ok": False, + "error": diagnostic[:1024], + } + self._send(reply) + + def _handle_transcript(self, frame: dict[str, Any]) -> None: + event = frame.get("event") + if event == "assistant_text_delta": + delta = frame.get("delta") + if isinstance(delta, str) and delta: + emit_progress( + self._progress, + AssistantTextProgress(delta=_bounded_progress(delta)), + ) + elif event == "agent_start": + emit_progress(self._progress, StatusProgress(channel="pi", message="agent started")) + elif event == "turn_start": + emit_progress(self._progress, StatusProgress(channel="pi", message="turn started")) + elif event == "agent_end": + emit_progress(self._progress, StatusProgress(channel="pi", message="agent finished")) + + def _read_stderr(self, stderr: IO[str]) -> None: + for chunk in iter(lambda: stderr.read(4096), ""): + remaining = _MAX_STDERR_BYTES - len(self._stderr) + if remaining > 0: + self._stderr.extend(chunk.encode()[:remaining]) + + def _send(self, frame: dict[str, Any]) -> None: + encoded = json.dumps(frame, allow_nan=False, separators=(",", ":")) + if len(encoded.encode()) > _MAX_FRAME_BYTES: + raise ValueError("outbound Pi adapter frame exceeds limit") + with self._write_lock: + self._stdin.write(encoded + "\n") + self._stdin.flush() + + def _await(self, frame_type: str, frame_id: str, timeout_s: float) -> dict[str, Any]: + deadline = time.monotonic() + timeout_s + deferred: list[dict[str, Any]] = [] + try: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError(f"Pi adapter timed out waiting for {frame_type}") + try: + item = self._frames.get(timeout=remaining) + except queue.Empty as exc: + raise TimeoutError(f"Pi adapter timed out waiting for {frame_type}") from exc + if isinstance(item, BaseException): + raise RuntimeError(f"Pi adapter reader failed: {item}") from item + if item.get("type") == "protocol_error": + raise RuntimeError(str(item.get("error", "Pi adapter protocol error"))) + if item.get("type") == frame_type and item.get("id") == frame_id: + return item + deferred.append(item) + finally: + for item in deferred: + self._frames.put(item) + + +def _adapter_environment( + credential: RuntimeCredential, + attempt_path: Path, +) -> dict[str, str]: + env = { + "PATH": os.environ.get("PATH", ""), + "PI_SPATIAL_AGENT_CWD": str(attempt_path), + "PI_SPATIAL_SESSION_DIR": "pi-session", + } + if credential.auth_mode == "subscription": + env["PI_SPATIAL_AUTH_MODE"] = "codex-oauth" + env["PI_SPATIAL_AUTH_PATH"] = credential.binding_name + elif credential.auth_mode == "environment" and credential.value: + env["PI_SPATIAL_AUTH_MODE"] = "openai-api-key" + env["OPENAI_API_KEY"] = credential.value + else: + raise ValueError("unsupported or incomplete Pi credential binding") + return env + + +def _mcp_text(result: dict[str, Any]) -> str: + content = result.get("content") + if not isinstance(content, list) or not content: + return "" + first = content[0] + if isinstance(first, dict): + text = first.get("text") + if isinstance(text, str): + return text + return json.dumps(first, allow_nan=False, separators=(",", ":"))[:32_000] + + +def _bounded_progress(value: str) -> str: + encoded = value.encode() + if len(encoded) <= _MAX_PROGRESS_BYTES: + return value + marker = "\n… [truncated]" + keep = _MAX_PROGRESS_BYTES - len(marker.encode()) + return encoded[:keep].decode(errors="ignore") + marker + + +def _artifact(root: Path, relative_path: str) -> ArtifactReference: + if not relative_path or relative_path.startswith("/") or ".." in relative_path.split("/"): + raise ValueError("Pi evidence path is not attempt-relative") + data = (root / relative_path).read_bytes() + return ArtifactReference( + path=relative_path, + sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + ) diff --git a/dimos/benchmark/agent_eval/progress.py b/dimos/benchmark/agent_eval/progress.py new file mode 100644 index 0000000000..f8e58ad149 --- /dev/null +++ b/dimos/benchmark/agent_eval/progress.py @@ -0,0 +1,84 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Presentation-only progress contracts for interactive evaluation runs.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class ProgressModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class StatusProgress(ProgressModel): + kind: Literal["status"] = "status" + channel: Literal["eval", "pi"] + message: str = Field(min_length=1) + + +class CaseHeaderProgress(ProgressModel): + kind: Literal["case_header"] = "case_header" + case_id: str = Field(min_length=1) + source: str = Field(min_length=1) + progress: float | None = Field(default=None, ge=0, le=1) + question: str = Field(min_length=1) + + +class AssistantTextProgress(ProgressModel): + kind: Literal["assistant_text"] = "assistant_text" + delta: str = Field(min_length=1) + + +class ToolStartProgress(ProgressModel): + kind: Literal["tool_start"] = "tool_start" + code: str = Field(min_length=1) + + +class ToolEndProgress(ProgressModel): + kind: Literal["tool_end"] = "tool_end" + ok: bool + result: str + duration_seconds: float = Field(ge=0) + + +class FinalResponseProgress(ProgressModel): + kind: Literal["final_response"] = "final_response" + text: str + + +EvalProgress = Annotated[ + StatusProgress + | CaseHeaderProgress + | AssistantTextProgress + | ToolStartProgress + | ToolEndProgress + | FinalResponseProgress, + Field(discriminator="kind"), +] +ProgressSink = Callable[[EvalProgress], None] + + +def emit_progress(sink: ProgressSink | None, event: EvalProgress) -> None: + """Notify a presentation observer without allowing it to affect evaluation.""" + if sink is None: + return + try: + sink(event) + except Exception: + return diff --git a/dimos/benchmark/agent_eval/single_case.py b/dimos/benchmark/agent_eval/single_case.py new file mode 100644 index 0000000000..52b7a1867b --- /dev/null +++ b/dimos/benchmark/agent_eval/single_case.py @@ -0,0 +1,245 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""One immutable case bound to typed local agent execution configuration.""" + +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +import time +from typing import Annotated, Literal, TypeVar + +from pydantic import Field + +from dimos.benchmark.agent_eval.auth import RuntimeCredential +from dimos.benchmark.agent_eval.base import BaseEvalModel +from dimos.benchmark.agent_eval.case import ( + AgentCondition, + AgentOutcome, + EvalCase, + FrozenRecordingSource, + Prediction, + RuntimeBinding, +) +from dimos.benchmark.agent_eval.pi_adapter import credential_binding_sha256 +from dimos.benchmark.agent_eval.pi_process import NodePiSessionFactory +from dimos.benchmark.agent_eval.progress import ( + CaseHeaderProgress, + ProgressSink, + StatusProgress, + emit_progress, +) +from dimos.benchmark.short_horizon_qa.eval import ( + load_exact_integer_oracle, + run_frozen_case, +) +from dimos.benchmark.short_horizon_qa.prepare import prepare_bundle +from dimos.constants import CACHE_DIR, STATE_DIR + +DEFAULT_MODEL: Literal["gpt-5.6-luna"] = "gpt-5.6-luna" +DEFAULT_OUTPUT_ROOT = STATE_DIR / "evals" +DEFAULT_CODEX_AUTH_PATH = Path.home() / ".pi" / "agent" / "auth.json" +DEFAULT_OPENAI_API_KEY_ENV = "OPENAI_API_KEY" +SINGLE_CASE_TURN_TIMEOUT_SECONDS = 600.0 +EvalModelT = TypeVar("EvalModelT", bound=BaseEvalModel) + + +class CodexOAuthConfig(BaseEvalModel): + mode: Literal["codex-oauth"] = "codex-oauth" + path: Path | None = None + + +class OpenAIApiKeyConfig(BaseEvalModel): + mode: Literal["openai-api-key"] = "openai-api-key" + env: str = Field(default=DEFAULT_OPENAI_API_KEY_ENV, min_length=1) + + +AgentAuthConfig = Annotated[ + CodexOAuthConfig | OpenAIApiKeyConfig, + Field(discriminator="mode"), +] + + +class PiAgentConfig(BaseEvalModel): + backend: Literal["pi"] = "pi" + model: Literal["gpt-5.6-luna"] = DEFAULT_MODEL + thinking_level: Literal["medium"] = "medium" + auth: AgentAuthConfig = Field(default_factory=CodexOAuthConfig) + + +class EvalRunConfig(BaseEvalModel): + agent: PiAgentConfig = Field(default_factory=PiAgentConfig) + + +class CompactEvalResult(BaseEvalModel): + attempt_id: str + case_id: str + source: str + progress: float | None + question: str + attempt_status: Literal["completed", "failed"] + task_result: Literal["passed", "failed", "not_evaluated"] + reason: str + prediction_status: Literal["parsed", "invalid"] | None = None + integer_answer: int | None = None + agent: AgentCondition + tool_call_count: int = Field(ge=0) + duration_seconds: float = Field(ge=0) + artifact_path: Path + + +def execute_single_case( + case_path: Path, + *, + config: EvalRunConfig, + output_root: Path = DEFAULT_OUTPUT_ROOT, + progress: ProgressSink | None = None, +) -> CompactEvalResult: + """Preflight and execute exactly one static frozen-memory case.""" + path = case_path.expanduser().resolve() + emit_progress(progress, StatusProgress(channel="eval", message="loading case")) + case = EvalCase.model_validate_json(path.read_bytes()) + if not isinstance(case.source, FrozenRecordingSource): + raise ValueError("single-case CLI currently supports frozen-memory cases only") + task = case.task + emit_progress( + progress, + CaseHeaderProgress( + case_id=case.case_id, + source=case.source.recording, + progress=case.source.progress, + question=getattr(task, "prompt", ""), + ), + ) + + # Resolve all private case material before starting Pi. + emit_progress(progress, StatusProgress(channel="eval", message="verifying validator")) + load_exact_integer_oracle(case, path.parent) + emit_progress(progress, StatusProgress(channel="eval", message="preparing frozen memory")) + bundle = _materialize_frozen_memory(case) + emit_progress(progress, StatusProgress(channel="eval", message="frozen memory ready")) + credential, binding_digest = _resolve_credential(config.agent.auth) + adapter = _adapter_entrypoint() + condition = AgentCondition( + agent_id="pi-code-policy", + adapter="pi-node", + model=config.agent.model, + thinking_level=config.agent.thinking_level, + ) + runtime = RuntimeBinding( + runtime_id="local-standalone-code-policy", + parameters={ + "auth_mode": config.agent.auth.mode, + "credential_binding_sha256": binding_digest, + "turn_timeout_seconds": SINGLE_CASE_TURN_TIMEOUT_SECONDS, + }, + ) + factory = NodePiSessionFactory( + command=("node", str(adapter)), + credential=credential, + model=config.agent.model, + thinking_level=config.agent.thinking_level, + startup_timeout_s=180.0, + progress=progress, + ) + emit_progress(progress, StatusProgress(channel="eval", message="starting attempt")) + started = time.monotonic() + engine_result = run_frozen_case( + case=case, + bundle=bundle, + private_root=path.parent, + output_root=output_root.expanduser(), + pi_factory=factory, + agent_condition=condition, + runtime_binding=runtime, + turn_timeout_s=SINGLE_CASE_TURN_TIMEOUT_SECONDS, + ) + duration = time.monotonic() - started + emit_progress(progress, StatusProgress(channel="eval", message="attempt finished")) + prediction = _optional_model(engine_result.attempt_path / "prediction.v1.json", Prediction) + agent_outcome = _optional_model( + engine_result.attempt_path / "agent-outcome.v1.json", AgentOutcome + ) + return CompactEvalResult( + attempt_id=engine_result.outcome.attempt_id, + case_id=case.case_id, + source=case.source.recording, + progress=case.source.progress, + question=getattr(task, "prompt", ""), + attempt_status=engine_result.outcome.attempt_status, + task_result=engine_result.outcome.task_result, + reason=engine_result.outcome.reason, + prediction_status=prediction.status if prediction is not None else None, + integer_answer=prediction.integer_answer if prediction is not None else None, + agent=condition, + tool_call_count=agent_outcome.tool_call_count if agent_outcome is not None else 0, + duration_seconds=duration, + artifact_path=engine_result.attempt_path, + ) + + +def _resolve_credential(auth: AgentAuthConfig) -> tuple[RuntimeCredential, str]: + if isinstance(auth, CodexOAuthConfig): + configured = auth.path or Path( + os.environ.get("PI_SPATIAL_AUTH_PATH", DEFAULT_CODEX_AUTH_PATH) + ) + path = configured.expanduser().resolve() + if not path.is_file(): + raise FileNotFoundError( + f"Codex OAuth credential not found at {path}; use --agent.auth.path" + ) + material = path.read_bytes() + return ( + RuntimeCredential(auth_mode="subscription", binding_name=str(path), value=None), + credential_binding_sha256("subscription", str(path), material), + ) + value = os.environ.get(auth.env) + if not value: + raise ValueError(f"credential environment variable {auth.env!r} is unset") + return ( + RuntimeCredential(auth_mode="environment", binding_name=auth.env, value=value), + credential_binding_sha256("environment", auth.env, value), + ) + + +def _materialize_frozen_memory(case: EvalCase) -> Path: + source = case.source + assert isinstance(source, FrozenRecordingSource) + key = hashlib.sha256(source.model_dump_json().encode()).hexdigest()[:24] + bundle = CACHE_DIR / "agent_eval" / "frozen_memory" / key + if not (bundle / "manifest.v1.json").is_file(): + bundle.parent.mkdir(parents=True, exist_ok=True) + prepare_bundle(source.recording, [], bundle, progress=[source.progress]) + return bundle + + +def _adapter_entrypoint() -> Path: + path = ( + Path(__file__).resolve().parents[3] + / "packages" + / "pi-code-policy-adapter" + / "dist" + / "code-policy-main.js" + ) + if not path.is_file(): + raise FileNotFoundError( + "Pi adapter is not built; run npm run build in packages/pi-code-policy-adapter" + ) + return path + + +def _optional_model(path: Path, model: type[EvalModelT]) -> EvalModelT | None: + return model.model_validate_json(path.read_bytes()) if path.is_file() else None diff --git a/dimos/benchmark/agent_eval/store.py b/dimos/benchmark/agent_eval/store.py new file mode 100644 index 0000000000..8b87cd6285 --- /dev/null +++ b/dimos/benchmark/agent_eval/store.py @@ -0,0 +1,285 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exclusive, append-only, non-overwriting storage for one local attempt.""" + +from __future__ import annotations + +from datetime import UTC, datetime +import fcntl +import hashlib +import os +from pathlib import Path +import time +from typing import Any +from uuid import uuid4 + +from pydantic import BaseModel, JsonValue + +from dimos.benchmark.agent_eval.artifacts import ( + ArtifactReference, + AttemptId, + LifecycleEvent, + NormalizedOutcome, + OperationId, +) +from dimos.benchmark.agent_eval.case import EvalOutcome +from dimos.benchmark.agent_eval.json import canonical_json + + +class AttemptAlreadyActiveError(RuntimeError): + pass + + +class AttemptStore: + """Own a target-wide lock and one fresh immutable attempt directory.""" + + def __init__(self, output_root: Path, attempt_id: str | None = None) -> None: + self.output_root = output_root.resolve() + self.attempt_id: AttemptId = attempt_id or f"attempt_{uuid4().hex}" + self.path = self.output_root / self.attempt_id + self._lock_fd = -1 + self._events_fd = -1 + self._started_monotonic = time.monotonic() + self._event_sequence = 0 + self._last_event_offset = 0.0 + self._closed = False + self._reserve() + + def _reserve(self) -> None: + self.output_root.mkdir(parents=True, exist_ok=True) + lock_path = self.output_root / ".attached-target.lock" + self._lock_fd = os.open(lock_path, os.O_RDWR | os.O_CREAT | os.O_CLOEXEC, 0o600) + try: + fcntl.flock(self._lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + os.close(self._lock_fd) + self._lock_fd = -1 + raise AttemptAlreadyActiveError( + f"another attempt is active for {self.output_root}" + ) from exc + try: + self.path.mkdir(mode=0o700) + except BaseException: + self.close() + raise + try: + self._events_fd = os.open( + self.path / "events.jsonl", + os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, + 0o600, + ) + except BaseException: + self.close() + raise + + def append_event( + self, + kind: str, + *, + operation_id: str | None = None, + payload: dict[str, JsonValue] | None = None, + ) -> LifecycleEvent: + self._require_open() + offset = max( + self._last_event_offset, + time.monotonic() - self._started_monotonic, + ) + self._event_sequence += 1 + event = LifecycleEvent( + sequence=self._event_sequence, + attempt_id=self.attempt_id, + operation_id=operation_id, + occurred_at=datetime.now(UTC), + monotonic_offset_s=offset, + kind=kind, + payload=payload or {}, + ) + encoded = canonical_json(event.model_dump(mode="json")) + b"\n" + _write_all(self._events_fd, encoded) + os.fsync(self._events_fd) + self._last_event_offset = offset + return event + + def write_artifact( + self, relative_path: str, value: BaseModel | JsonValue | bytes | str + ) -> ArtifactReference: + self._require_open() + path = self._resolve_relative(relative_path) + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(value, BaseModel): + data = canonical_json(value.model_dump(mode="json")) + b"\n" + elif isinstance(value, bytes): + data = value + elif isinstance(value, str): + data = value.encode("utf-8") + else: + data = canonical_json(value) + b"\n" + descriptor = os.open( + path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, + 0o600, + ) + try: + _write_all(descriptor, data) + os.fsync(descriptor) + except BaseException: + os.close(descriptor) + path.unlink(missing_ok=True) + raise + else: + os.close(descriptor) + _fsync_directory(path.parent) + return ArtifactReference( + path=relative_path, + sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + ) + + def write_outcome(self, outcome: NormalizedOutcome) -> ArtifactReference: + self._require_open() + if outcome.attempt_id != self.attempt_id: + raise ValueError("outcome attempt identity mismatch") + relative_path = "outcome.v1.json" + final_path = self.path / relative_path + temp_path = self.path / f".outcome.v1.json.tmp-{uuid4().hex}" + data = canonical_json(outcome.model_dump(mode="json")) + b"\n" + descriptor = os.open( + temp_path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, + 0o600, + ) + try: + _write_all(descriptor, data) + os.fsync(descriptor) + os.close(descriptor) + descriptor = -1 + os.link(temp_path, final_path) + _fsync_directory(self.path) + finally: + if descriptor >= 0: + os.close(descriptor) + temp_path.unlink(missing_ok=True) + return ArtifactReference( + path=relative_path, + sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + ) + + def write_eval_outcome(self, outcome: EvalOutcome) -> ArtifactReference: + """Atomically retain the backend-neutral terminal outcome.""" + if outcome.attempt_id != self.attempt_id: + raise ValueError("outcome attempt identity mismatch") + return self._write_terminal("outcome.v1.json", outcome) + + def _write_terminal(self, relative_path: str, value: BaseModel) -> ArtifactReference: + self._require_open() + final_path = self.path / relative_path + temp_path = self.path / f".{relative_path}.tmp-{uuid4().hex}" + data = canonical_json(value.model_dump(mode="json")) + b"\n" + descriptor = os.open( + temp_path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, + 0o600, + ) + try: + _write_all(descriptor, data) + os.fsync(descriptor) + os.close(descriptor) + descriptor = -1 + os.link(temp_path, final_path) + _fsync_directory(self.path) + finally: + if descriptor >= 0: + os.close(descriptor) + temp_path.unlink(missing_ok=True) + return ArtifactReference( + path=relative_path, + sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + ) + + def verify_artifacts(self, artifacts: tuple[ArtifactReference, ...]) -> bool: + """Return whether every admitted reference still matches retained bytes.""" + for artifact in artifacts: + try: + path = self._resolve_relative(artifact.path) + data = path.read_bytes() + except (OSError, ValueError): + return False + if ( + len(data) != artifact.size_bytes + or hashlib.sha256(data).hexdigest() != artifact.sha256 + ): + return False + return True + + def _resolve_relative(self, relative_path: str) -> Path: + if not relative_path or relative_path.startswith("/") or ".." in relative_path.split("/"): + raise ValueError("artifact path must be attempt-relative") + result = self.path / relative_path + if result.resolve().parent != self.path and self.path not in result.resolve().parents: + raise ValueError("artifact path escapes attempt directory") + return result + + def _require_open(self) -> None: + if self._closed: + raise RuntimeError("attempt store is closed") + + def close(self) -> None: + if self._closed: + return + self._closed = True + first_error: OSError | None = None + if self._events_fd >= 0: + try: + os.close(self._events_fd) + except OSError as exc: + first_error = exc + finally: + self._events_fd = -1 + if self._lock_fd >= 0: + try: + fcntl.flock(self._lock_fd, fcntl.LOCK_UN) + os.close(self._lock_fd) + except OSError as exc: + first_error = first_error or exc + finally: + self._lock_fd = -1 + if first_error is not None: + raise first_error + + def __enter__(self) -> AttemptStore: + return self + + def __exit__(self, *_args: Any) -> None: + self.close() + + +def new_operation_id() -> OperationId: + return f"operation_{uuid4().hex}" + + +def _write_all(descriptor: int, data: bytes) -> None: + view = memoryview(data) + while view: + view = view[os.write(descriptor, view) :] + + +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) diff --git a/dimos/benchmark/agent_eval/test_case.py b/dimos/benchmark/agent_eval/test_case.py new file mode 100644 index 0000000000..6fe35212b6 --- /dev/null +++ b/dimos/benchmark/agent_eval/test_case.py @@ -0,0 +1,148 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json + +from pydantic import ValidationError +import pytest + +from dimos.benchmark.agent_eval.case import ( + AgentCondition, + AttemptRequest, + EvalCase, + EvalOutcome, + ExactIntegerValidatorRef, + FrozenCodePolicyInteraction, + FrozenRecordingSource, + IntegerQuestionTask, + Prediction, + RuntimeBinding, +) + + +def _case(prompt: str = "How many rooms?") -> EvalCase: + return EvalCase.compile( + case_id="office-room-count", + source=FrozenRecordingSource(recording="office", progress=1.0), + task=IntegerQuestionTask(prompt=prompt), + interaction=FrozenCodePolicyInteraction(driver_revision="v1"), + validator=ExactIntegerValidatorRef( + revision="exact-v1", + private_path="private/oracle.json", + private_sha256="a" * 64, + ), + ) + + +def test_compiled_case_has_stable_fingerprint_and_public_projection() -> None: + first = _case() + second = _case() + + assert first.fingerprint == second.fingerprint + public = first.public_projection().model_dump(mode="json") + assert "validator" not in public + assert "private/oracle.json" not in json.dumps(public) + assert "a" * 64 not in json.dumps(public) + + +def test_case_fingerprint_binds_task_and_private_validator() -> None: + assert _case("How many rooms?").fingerprint != _case("Count the rooms.").fingerprint + encoded = _case().model_dump(mode="json") + encoded["fingerprint"] = "0" * 64 + with pytest.raises(ValidationError, match="fingerprint"): + EvalCase.model_validate(encoded) + + +def test_case_rejects_missing_contract_and_unknown_discriminator() -> None: + encoded = _case().model_dump(mode="json") + del encoded["validator"] + with pytest.raises(ValidationError): + EvalCase.model_validate(encoded) + + encoded = _case().model_dump(mode="json") + encoded["interaction"]["kind"] = "prompt_dump" + with pytest.raises(ValidationError, match="frozen_code_policy"): + EvalCase.model_validate(encoded) + + +@pytest.mark.parametrize("progress", [-0.1, 1.1, float("inf"), float("nan")]) +def test_frozen_source_rejects_invalid_progress(progress: float) -> None: + with pytest.raises(ValidationError): + FrozenRecordingSource(recording="office", progress=progress) + + +def test_one_source_can_back_independent_tasks() -> None: + first = _case("Question one") + second = _case("Question two") + assert first.source == second.source + assert first.task != second.task + assert first.fingerprint != second.fingerprint + + +def test_attempt_request_keeps_runtime_out_of_case_identity() -> None: + case = _case() + request = AttemptRequest( + case=case, + agent=AgentCondition( + agent_id="pi", + adapter="pi-node", + model="gpt-5.6-luna", + thinking_level="medium", + ), + runtime=RuntimeBinding(runtime_id="local", parameters={"port": 10090}), + ) + assert request.case.fingerprint == case.fingerprint + + +def test_prediction_status_is_strict() -> None: + common = { + "case_id": "case", + "attempt_id": "attempt", + "agent_session_id": "pi", + "interaction_session_id": "code-policy", + "parser_revision": "v1", + "final_text": "ANSWER: 4", + } + Prediction(**common, status="parsed", integer_answer=4) + Prediction(**common, status="invalid", diagnostic="missing marker") + with pytest.raises(ValidationError): + Prediction(**common, status="parsed", diagnostic="bad") + + +@pytest.mark.parametrize( + ("attempt_status", "task_result", "valid"), + [ + ("completed", "passed", True), + ("completed", "failed", True), + ("completed", "not_evaluated", False), + ("failed", "not_evaluated", True), + ("failed", "failed", False), + ], +) +def test_outcome_separates_operation_from_task( + attempt_status: str, task_result: str, valid: bool +) -> None: + values = { + "attempt_id": "attempt", + "attempt_status": attempt_status, + "task_result": task_result, + "reason": "test", + } + if valid: + EvalOutcome.model_validate(values) + else: + with pytest.raises(ValidationError): + EvalOutcome.model_validate(values) diff --git a/dimos/benchmark/agent_eval/test_engine.py b/dimos/benchmark/agent_eval/test_engine.py new file mode 100644 index 0000000000..897f6e270d --- /dev/null +++ b/dimos/benchmark/agent_eval/test_engine.py @@ -0,0 +1,281 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from dimos.benchmark.agent_eval.case import ( + AgentCondition, + AgentOutcome, + AttemptRequest, + EvalCase, + ExactIntegerValidatorRef, + FrozenCodePolicyInteraction, + FrozenRecordingSource, + IntegerQuestionTask, + PrivateScore, + RuntimeBinding, +) +from dimos.benchmark.agent_eval.engine import AttemptEngine +from dimos.benchmark.agent_eval.interfaces import AttemptContext, PreparedSource +from dimos.benchmark.agent_eval.store import AttemptStore + + +def _request() -> AttemptRequest: + case = EvalCase.compile( + case_id="case", + source=FrozenRecordingSource(recording="recording", progress=1.0), + task=IntegerQuestionTask(prompt="Question"), + interaction=FrozenCodePolicyInteraction(driver_revision="v1"), + validator=ExactIntegerValidatorRef( + revision="v1", + private_path="private/oracle.json", + private_sha256="a" * 64, + ), + ) + return AttemptRequest( + case=case, + agent=AgentCondition( + agent_id="agent", + adapter="fake", + model="fake", + thinking_level="off", + ), + runtime=RuntimeBinding(runtime_id="local"), + ) + + +class FakeAgent: + def __init__(self, fail: bool = False, cleanup_fail: bool = False) -> None: + self.fail = fail + self.cleanup_fail = cleanup_fail + self.closed = False + + def run(self, *, task: Any, context: AttemptContext, interface: Any = None) -> AgentOutcome: + del task, context, interface + if self.fail: + raise RuntimeError("agent failed") + return AgentOutcome( + final_text="ANSWER: 4", tool_call_count=1, terminal_reason="agent completed" + ) + + def close(self) -> None: + self.closed = True + if self.cleanup_fail: + raise RuntimeError("agent cleanup failed") + + +class FakeSource: + def __init__(self, fail: bool = False) -> None: + self.fail = fail + self.closed = False + + def prepare(self, *, source: Any, context: AttemptContext, evidence: Any) -> PreparedSource: + del source, context + evidence.artifact("source-receipt.v1.json", {"ready": True}) + if self.fail: + raise RuntimeError("source failed") + return PreparedSource(public={"ready": True}, receipt={"source": "fake"}) + + def close(self) -> None: + self.closed = True + + +class FakeInteraction: + def __init__(self, fail: bool = False) -> None: + self.fail = fail + self.closed = False + + def run( + self, + *, + case: EvalCase, + prepared_source: PreparedSource, + agent: FakeAgent, + context: AttemptContext, + evidence: Any, + ) -> AgentOutcome: + del case, prepared_source, evidence + if self.fail: + raise RuntimeError("interaction failed") + return agent.run(task=context.request.case.task, context=context) + + def close(self) -> None: + self.closed = True + + +class FakeValidatorSession: + def __init__( + self, context: AttemptContext, *, passed: bool, evaluate_fail: bool = False + ) -> None: + self.context = context + self.passed = passed + self.evaluate_fail = evaluate_fail + self.closed = False + + def evaluate(self, outcome: AgentOutcome) -> PrivateScore: + del outcome + if self.evaluate_fail: + raise RuntimeError("validation failed") + return PrivateScore( + case_id=self.context.request.case.case_id, + attempt_id=self.context.attempt_id, + validator_revision="v1", + passed=self.passed, + prediction_status="parsed" if self.passed else "invalid", + ) + + def close(self) -> None: + self.closed = True + + +class FakeValidator: + def __init__( + self, *, passed: bool = True, prepare_fail: bool = False, evaluate_fail: bool = False + ) -> None: + self.passed = passed + self.prepare_fail = prepare_fail + self.evaluate_fail = evaluate_fail + + def prepare( + self, + *, + case: EvalCase, + prepared_source: PreparedSource, + context: AttemptContext, + evidence: Any, + ) -> FakeValidatorSession: + del case, prepared_source, evidence + if self.prepare_fail: + raise RuntimeError("validator prepare failed") + return FakeValidatorSession(context, passed=self.passed, evaluate_fail=self.evaluate_fail) + + +def _run(tmp_path: Path, **kwargs: Any): + source = FakeSource(fail=kwargs.get("source_fail", False)) + interaction = FakeInteraction(fail=kwargs.get("interaction_fail", False)) + agent = FakeAgent( + fail=kwargs.get("agent_fail", False), + cleanup_fail=kwargs.get("cleanup_fail", False), + ) + validator = FakeValidator( + passed=kwargs.get("passed", True), + prepare_fail=kwargs.get("validator_prepare_fail", False), + evaluate_fail=kwargs.get("validator_evaluate_fail", False), + ) + result = AttemptEngine( + request=_request(), + output_root=tmp_path, + source=source, + interaction=interaction, + validator=validator, + agent=agent, + ).run() + return result, source, interaction, agent + + +@pytest.mark.parametrize("passed", [True, False]) +def test_engine_completes_pass_and_wrong_answer(tmp_path: Path, passed: bool) -> None: + result, source, interaction, agent = _run(tmp_path, passed=passed) + assert result.outcome.attempt_status == "completed" + assert result.outcome.task_result == ("passed" if passed else "failed") + assert (result.attempt_path / "score.private.v1.json").is_file() + assert source.closed and interaction.closed and agent.closed + + +@pytest.mark.parametrize( + "failure", + [ + "source_fail", + "interaction_fail", + "agent_fail", + "validator_prepare_fail", + "validator_evaluate_fail", + ], +) +def test_engine_retains_failed_prefix_and_not_evaluated(tmp_path: Path, failure: str) -> None: + result, source, interaction, agent = _run(tmp_path, **{failure: True}) + assert result.outcome.attempt_status == "failed" + assert result.outcome.task_result == "not_evaluated" + assert (result.attempt_path / "outcome.v1.json").is_file() + assert (result.attempt_path / "events.jsonl").is_file() + assert source.closed and interaction.closed and agent.closed + + +def test_cleanup_failure_invalidates_completed_attempt(tmp_path: Path) -> None: + result, _, _, _ = _run(tmp_path, cleanup_fail=True) + assert result.outcome.attempt_status == "failed" + assert result.outcome.task_result == "not_evaluated" + assert "cleanup" in result.outcome.reason + + +def test_engine_writes_exactly_one_terminal_outcome(tmp_path: Path) -> None: + result, _, _, _ = _run(tmp_path) + assert [path.name for path in result.attempt_path.glob("outcome*")] == ["outcome.v1.json"] + + +def test_manifest_write_failure_returns_failed_attempt_and_releases_lock( + tmp_path: Path, mocker +) -> None: + original = AttemptStore.write_artifact + + def fail_manifest(store, relative_path, value): + if relative_path == "attempt-manifest.v1.json": + raise OSError("manifest disk failure") + return original(store, relative_path, value) + + mocker.patch.object(AttemptStore, "write_artifact", autospec=True, side_effect=fail_manifest) + + result, _, _, _ = _run(tmp_path) + + assert result.outcome.attempt_status == "failed" + assert "finalization" in result.outcome.reason + with AttemptStore(tmp_path) as subsequent: + assert subsequent.path != result.attempt_path + + +def test_terminal_publication_failure_returns_failed_attempt_and_releases_lock( + tmp_path: Path, mocker +) -> None: + mocker.patch.object( + AttemptStore, + "write_eval_outcome", + autospec=True, + side_effect=OSError("terminal link failure"), + ) + + result, _, _, _ = _run(tmp_path) + + assert result.outcome.attempt_status == "failed" + assert "terminal publication" in result.outcome.reason + assert not (result.attempt_path / "outcome.v1.json").exists() + with AttemptStore(tmp_path) as subsequent: + assert subsequent.path != result.attempt_path + + +def test_event_fsync_failure_still_cleans_resources_and_releases_lock( + tmp_path: Path, mocker +) -> None: + mocker.patch("dimos.benchmark.agent_eval.store.os.fsync", side_effect=OSError("fsync failed")) + + result, source, interaction, agent = _run(tmp_path) + + assert result.outcome.attempt_status == "failed" + assert source.closed and interaction.closed and agent.closed + with AttemptStore(tmp_path) as subsequent: + assert subsequent.path != result.attempt_path diff --git a/dimos/benchmark/agent_eval/test_import_boundaries.py b/dimos/benchmark/agent_eval/test_import_boundaries.py new file mode 100644 index 0000000000..b367c668e2 --- /dev/null +++ b/dimos/benchmark/agent_eval/test_import_boundaries.py @@ -0,0 +1,43 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ast +from pathlib import Path + +FORBIDDEN_PREFIXES = ( + "dimos.benchmark.dimsim", + "dimos.benchmark.spatial", +) + + +def test_focused_evaluation_slice_has_no_live_benchmark_imports() -> None: + package = Path(__file__).parent + violations: list[str] = [] + + for path in sorted(package.glob("*.py")): + if path.name.startswith("test_"): + continue + tree = ast.parse(path.read_text(), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom) and node.module is not None: + imports = [node.module] + else: + continue + for imported in imports: + if imported.startswith(FORBIDDEN_PREFIXES): + violations.append(f"{path.name}: {imported}") + + assert violations == [] diff --git a/dimos/benchmark/agent_eval/test_json.py b/dimos/benchmark/agent_eval/test_json.py new file mode 100644 index 0000000000..73d3739492 --- /dev/null +++ b/dimos/benchmark/agent_eval/test_json.py @@ -0,0 +1,26 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from dimos.benchmark.agent_eval.json import canonical_json + + +def test_canonical_json_is_sorted_compact_utf8() -> None: + assert canonical_json({"z": "café", "a": [2, 1]}) == (b'{"a":[2,1],"z":"caf\xc3\xa9"}') + + +def test_canonical_json_rejects_nonfinite_numbers() -> None: + with pytest.raises(ValueError, match="JSON compliant"): + canonical_json({"value": float("nan")}) diff --git a/dimos/benchmark/agent_eval/test_pi_adapter.py b/dimos/benchmark/agent_eval/test_pi_adapter.py new file mode 100644 index 0000000000..7db3b0a124 --- /dev/null +++ b/dimos/benchmark/agent_eval/test_pi_adapter.py @@ -0,0 +1,198 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json + +import pytest + +from dimos.agents.mcp.mcp_server import _handle_tools_list +from dimos.benchmark.agent_eval.pi_adapter import ( + PI_TOOL_NAMES, + CodePolicyCallLog, + PythonExecBroker, + ToolInventoryError, + credential_binding_sha256, + inspect_python_exec_inventory, + wait_for_python_exec, +) +from dimos.core.module import SkillInfo + +ATTEMPT_ID = "attempt_" + "a" * 32 +PI_SESSION_ID = "pi_session_" + "b" * 32 +POLICY_SESSION_ID = "code_policy_session_" + "c" * 32 + + +def _python_exec_tool() -> dict[str, object]: + return { + "name": "python_exec", + "description": ( + "Execute one synchronous Python program in the persistent policy session.\n\n" + "The trusted, unsandboxed session preloads `app` for deployed DimOS RPCs." + ), + "inputSchema": { + "type": "object", + "properties": { + "code": {"title": "Code", "type": "string"}, + "timeout_s": { + "default": 110.0, + "title": "Timeout S", + "type": "number", + }, + }, + "required": ["code"], + }, + } + + +class FakeMcp: + def __init__( + self, + tools: list[dict[str, object]] | None = None, + result: dict[str, object] | None = None, + ) -> None: + self.tools = tools or [_python_exec_tool()] + self.result = result or {"content": [{"type": "text", "text": "done"}]} + self.calls: list[tuple[str, dict[str, object]]] = [] + self.ready = True + + def wait_for_ready(self, timeout: float) -> bool: + assert timeout > 0 + return self.ready + + def list_tools(self) -> list[dict[str, object]]: + return self.tools + + def call_tool(self, name: str, arguments: dict[str, object] | None = None) -> dict[str, object]: + self.calls.append((name, arguments or {})) + return self.result + + +def test_inventory_retains_additional_tools_but_admits_only_python_exec() -> None: + tools = [ + _python_exec_tool(), + { + "name": "move", + "description": "Direct robot motion", + "inputSchema": {"type": "object", "properties": {}}, + }, + ] + + receipt = inspect_python_exec_inventory("http://localhost/mcp", tools) + + assert [tool["name"] for tool in receipt.observed_tools] == ["python_exec", "move"] + assert PI_TOOL_NAMES == ("python_exec",) + + +@pytest.mark.parametrize( + "tools", + [ + [], + [_python_exec_tool(), _python_exec_tool()], + [{**_python_exec_tool(), "description": "changed"}], + [ + { + **_python_exec_tool(), + "inputSchema": { + "type": "object", + "properties": {"code": {"type": "string"}}, + "required": ["code"], + }, + } + ], + ], +) +def test_inventory_rejects_missing_duplicate_or_changed_tool( + tools: list[dict[str, object]], +) -> None: + with pytest.raises(ToolInventoryError): + inspect_python_exec_inventory("http://localhost/mcp", tools) + + +def test_readiness_timeout_is_infrastructure_failure() -> None: + mcp = FakeMcp() + mcp.ready = False + + with pytest.raises(TimeoutError): + wait_for_python_exec("http://localhost/mcp", mcp, 0.1) + + +def test_broker_forwards_one_tool_and_records_both_sessions(tmp_path) -> None: + mcp = FakeMcp() + path = tmp_path / "code-policy-calls.jsonl" + with CodePolicyCallLog(path) as call_log: + broker = PythonExecBroker( + attempt_id=ATTEMPT_ID, + pi_session_id=PI_SESSION_ID, + code_policy_session_id=POLICY_SESSION_ID, + mcp=mcp, + call_log=call_log, + ) + result = broker.request("python_exec", {"code": "print('hello')"}) + + record = json.loads(path.read_text()) + assert result == mcp.result + assert mcp.calls == [("python_exec", {"code": "print('hello')", "timeout_s": 110.0})] + assert record["attempt_id"] == ATTEMPT_ID + assert record["pi_session_id"] == PI_SESSION_ID + assert record["code_policy_session_id"] == POLICY_SESSION_ID + assert record["ok"] is True + + +def test_broker_rejects_every_other_tool_without_forwarding(tmp_path) -> None: + mcp = FakeMcp() + with CodePolicyCallLog(tmp_path / "calls.jsonl") as call_log: + broker = PythonExecBroker( + attempt_id=ATTEMPT_ID, + pi_session_id=PI_SESSION_ID, + code_policy_session_id=POLICY_SESSION_ID, + mcp=mcp, + call_log=call_log, + ) + with pytest.raises(PermissionError): + broker.request("move", {"x": 1}) + assert mcp.calls == [] + + +def test_credentials_do_not_enter_records_or_diagnostics(tmp_path) -> None: + secret = "sk-super-secret-value" + digest = credential_binding_sha256("environment", "OPENAI_API_KEY", secret) + mcp = FakeMcp(result={"content": [{"type": "text", "text": "safe"}]}) + path = tmp_path / "calls.jsonl" + with CodePolicyCallLog(path) as call_log: + broker = PythonExecBroker( + attempt_id=ATTEMPT_ID, + pi_session_id=PI_SESSION_ID, + code_policy_session_id=POLICY_SESSION_ID, + mcp=mcp, + call_log=call_log, + ) + broker.request("python_exec", {"code": "1 + 1"}) + + retained = path.read_text() + digest + assert secret not in retained + assert digest == credential_binding_sha256("environment", "OPENAI_API_KEY", secret) + + +def test_inventory_rejects_non_json_tool() -> None: + skill = SkillInfo( + class_name="Bad", + func_name="python_exec", + args_schema=json.dumps({"type": "object"}), + ) + tool = _handle_tools_list(1, [skill])["result"]["tools"][0] + tool["not_json"] = object() + with pytest.raises(ValueError, match="strict JSON"): + inspect_python_exec_inventory("http://localhost/mcp", [tool]) diff --git a/dimos/benchmark/agent_eval/test_pi_process.py b/dimos/benchmark/agent_eval/test_pi_process.py new file mode 100644 index 0000000000..7bd58ddc31 --- /dev/null +++ b/dimos/benchmark/agent_eval/test_pi_process.py @@ -0,0 +1,229 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import hashlib +import sys + +import pytest + +from dimos.benchmark.agent_eval.auth import RuntimeCredential +from dimos.benchmark.agent_eval.pi_adapter import CodePolicyCallLog +from dimos.benchmark.agent_eval.pi_process import NodePiSessionFactory +from dimos.benchmark.agent_eval.progress import ( + AssistantTextProgress, + FinalResponseProgress, + ToolEndProgress, + ToolStartProgress, +) + +_HOST = r""" +import json, pathlib, sys +def send(value): + print(json.dumps(value, separators=(",", ":")), flush=True) +start = json.loads(sys.stdin.readline()) +send({"version":1,"type":"session_started","id":start["id"],"tools":["python_exec"]}) +for line in sys.stdin: + frame = json.loads(line) + if frame["type"] == "prompt": + send({"version":1,"type":"transcript","event":"agent_start"}) + send({"version":1,"type":"transcript","event":"assistant_text_delta","delta":"Checking memory"}) + send({"version":1,"type":"transcript","event":"thinking_delta","delta":"private reasoning"}) + send({"version":1,"type":"tool_call","id":"tool-1","tool":"python_exec","params":{"code":"1 + 1"}}) + reply = json.loads(sys.stdin.readline()) + assert reply["type"] == "tool_reply" and reply["ok"] + send({"version":1,"type":"turn_complete","id":frame["id"],"policy_call_count":1,"final_text":"done"}) + elif frame["type"] == "dispose": + pathlib.Path("pi-session").mkdir() + pathlib.Path("pi-prompt").mkdir() + pathlib.Path("pi-session/native.jsonl").write_text('{"type":"session"}\n') + pathlib.Path("pi-prompt/system.txt").write_text("system") + pathlib.Path("pi-prompt/initial.txt").write_text(start["initial_prompt"]) + send({"version":1,"type":"session_closed","id":start["id"],"evidence":{ + "state":"complete","persisted":True,"relative_path":"pi-session/native.jsonl", + "system_prompt":{"relative_path":"pi-prompt/system.txt","byte_count":6,"sha256":"0"*64}, + "initial_prompt":{"relative_path":"pi-prompt/initial.txt","byte_count":len(start["initial_prompt"]),"sha256":"1"*64} + }}) + break +""" + + +class _Mcp: + def __init__(self, result: str = "2") -> None: + self.result = result + + def wait_for_ready(self, timeout: float) -> bool: + return True + + def list_tools(self): + return [] + + def call_tool(self, name, arguments=None): + assert name == "python_exec" + return {"content": [{"type": "text", "text": self.result}]} + + +def test_node_pi_process_roundtrip_and_native_evidence(tmp_path) -> None: + attempt = tmp_path / ("attempt_" + "a" * 32) + attempt.mkdir() + calls = CodePolicyCallLog(attempt / "code-policy-calls.jsonl") + progress = [] + factory = NodePiSessionFactory( + command=(sys.executable, "-c", _HOST), + credential=RuntimeCredential( + auth_mode="environment", + binding_name="OPENAI_API_KEY", + value="secret", + ), + model="gpt-5.6-luna", + thinking_level="medium", + startup_timeout_s=2.0, + progress=progress.append, + ) + + session = factory.create( + attempt_path=attempt, + public_prompt="Navigate to the bathtub.", + code_policy_session_id="code_policy_session_" + "b" * 32, + call_log=calls, + mcp=_Mcp(), + ) + turn = session.prompt("Navigate to the bathtub.", 2.0) + session.dispose() + calls.close() + + assert turn.policy_call_count == 1 + assert any( + isinstance(event, AssistantTextProgress) and event.delta == "Checking memory" + for event in progress + ) + assert any(isinstance(event, ToolStartProgress) and event.code == "1 + 1" for event in progress) + assert any( + isinstance(event, ToolEndProgress) and event.ok and event.result == "2" + for event in progress + ) + assert any( + isinstance(event, FinalResponseProgress) and event.text == "done" for event in progress + ) + assert "private reasoning" not in repr(progress) + references = session.artifact_references() + assert {item.path for item in references} == { + "pi-adapter.stderr.log", + "pi-session/native.jsonl", + "pi-prompt/system.txt", + "pi-prompt/initial.txt", + } + retained = (attempt / "code-policy-calls.jsonl").read_text() + assert "secret" not in retained + assert hashlib.sha256((attempt / "pi-session/native.jsonl").read_bytes()).hexdigest() + + +def test_progress_observer_failure_does_not_fail_turn(tmp_path) -> None: + attempt = tmp_path / ("attempt_" + "e" * 32) + attempt.mkdir() + calls = CodePolicyCallLog(attempt / "code-policy-calls.jsonl") + + def broken_progress(_event) -> None: + raise RuntimeError("presentation failed") + + factory = NodePiSessionFactory( + command=(sys.executable, "-c", _HOST), + credential=RuntimeCredential( + auth_mode="environment", + binding_name="OPENAI_API_KEY", + value="secret", + ), + model="gpt-5.6-luna", + thinking_level="medium", + startup_timeout_s=2.0, + progress=broken_progress, + ) + session = factory.create( + attempt_path=attempt, + public_prompt="Count rooms.", + code_policy_session_id="code_policy_session_" + "f" * 32, + call_log=calls, + mcp=_Mcp(), + ) + try: + turn = session.prompt("Count rooms.", 2.0) + assert turn.final_text == "done" + finally: + session.dispose() + calls.close() + + +def test_tool_result_progress_is_bounded(tmp_path) -> None: + attempt = tmp_path / ("attempt_" + "1" * 32) + attempt.mkdir() + calls = CodePolicyCallLog(attempt / "code-policy-calls.jsonl") + progress = [] + factory = NodePiSessionFactory( + command=(sys.executable, "-c", _HOST), + credential=RuntimeCredential( + auth_mode="environment", + binding_name="OPENAI_API_KEY", + value="secret", + ), + model="gpt-5.6-luna", + thinking_level="medium", + startup_timeout_s=2.0, + progress=progress.append, + ) + session = factory.create( + attempt_path=attempt, + public_prompt="Count rooms.", + code_policy_session_id="code_policy_session_" + "2" * 32, + call_log=calls, + mcp=_Mcp("x" * 10_000), + ) + try: + session.prompt("Count rooms.", 2.0) + finally: + session.dispose() + calls.close() + + result = next(event.result for event in progress if isinstance(event, ToolEndProgress)) + assert len(result.encode()) <= 4 * 1024 + assert result.endswith("… [truncated]") + + +def test_node_pi_process_reaps_child_when_startup_times_out(tmp_path) -> None: + attempt = tmp_path / ("attempt_" + "c" * 32) + attempt.mkdir() + calls = CodePolicyCallLog(attempt / "code-policy-calls.jsonl") + factory = NodePiSessionFactory( + command=(sys.executable, "-c", "import time; time.sleep(60)"), + credential=RuntimeCredential( + auth_mode="environment", + binding_name="OPENAI_API_KEY", + value="secret", + ), + model="gpt-5.6-luna", + thinking_level="medium", + startup_timeout_s=0.05, + ) + + with pytest.raises(TimeoutError, match="session_started"): + factory.create( + attempt_path=attempt, + public_prompt="Navigate to the bathtub.", + code_policy_session_id="code_policy_session_" + "d" * 32, + call_log=calls, + mcp=_Mcp(), + ) + + calls.close() + assert (attempt / "pi-adapter.stderr.log").read_bytes() == b"" diff --git a/dimos/benchmark/agent_eval/test_single_case.py b/dimos/benchmark/agent_eval/test_single_case.py new file mode 100644 index 0000000000..5af3d4aad6 --- /dev/null +++ b/dimos/benchmark/agent_eval/test_single_case.py @@ -0,0 +1,124 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import json + +import pytest + +from dimos.benchmark.agent_eval.case import ( + EvalCase, + ExactIntegerValidatorRef, + FrozenCodePolicyInteraction, + FrozenRecordingSource, + IntegerQuestionTask, +) +import dimos.benchmark.agent_eval.single_case as single_case +from dimos.benchmark.agent_eval.single_case import ( + EvalRunConfig, + OpenAIApiKeyConfig, + _resolve_credential, + execute_single_case, +) +from dimos.benchmark.short_horizon_qa.eval import load_exact_integer_oracle + + +def test_run_config_round_trips_through_pydantic() -> None: + configured = EvalRunConfig() + + decoded = EvalRunConfig.model_validate_json(configured.model_dump_json()) + + assert decoded == configured + assert decoded.agent.backend == "pi" + assert decoded.agent.model == "gpt-5.6-luna" + assert decoded.agent.auth.mode == "codex-oauth" + + +def test_api_key_auth_uses_named_environment_without_serializing_secret(monkeypatch) -> None: + monkeypatch.setenv("EVAL_TEST_KEY", "private-value") + auth = OpenAIApiKeyConfig(env="EVAL_TEST_KEY") + + credential, digest = _resolve_credential(auth) + + assert credential.value == "private-value" + assert len(digest) == 64 + assert "private-value" not in auth.model_dump_json() + assert "private-value" not in digest + + +def test_private_validator_resolves_relative_to_case_directory(tmp_path) -> None: + private = tmp_path / "private" + private.mkdir() + oracle = private / "oracle.json" + oracle.write_text( + json.dumps( + { + "schema_version": "1.0", + "expected_count": 4, + "counting_policy": "Count enclosed rooms.", + "rooms": [], + "reviewed_by": ["reviewer"], + } + ) + ) + case = EvalCase.compile( + case_id="case", + source=FrozenRecordingSource(recording="recording", progress=1.0), + task=IntegerQuestionTask(prompt="How many rooms?"), + interaction=FrozenCodePolicyInteraction(driver_revision="v1"), + validator=ExactIntegerValidatorRef( + revision="v1", + private_path="private/oracle.json", + private_sha256=hashlib.sha256(oracle.read_bytes()).hexdigest(), + ), + ) + + loaded = load_exact_integer_oracle(case, tmp_path) + + assert loaded.expected_count == 4 + + +def test_single_case_emits_public_question_before_private_preflight(tmp_path, monkeypatch) -> None: + case = EvalCase.compile( + case_id="case", + source=FrozenRecordingSource(recording="recording", progress=1.0), + task=IntegerQuestionTask(prompt="How many rooms?"), + interaction=FrozenCodePolicyInteraction(driver_revision="v1"), + validator=ExactIntegerValidatorRef( + revision="v1", + private_path="private/oracle.json", + private_sha256="0" * 64, + ), + ) + case_path = tmp_path / "case.json" + case_path.write_text(case.model_dump_json()) + events = [] + + def stop_at_private_preflight(*args, **kwargs): + raise RuntimeError("stop after public header") + + monkeypatch.setattr(single_case, "load_exact_integer_oracle", stop_at_private_preflight) + + with pytest.raises(RuntimeError, match="stop after public header"): + execute_single_case(case_path, config=EvalRunConfig(), progress=events.append) + + assert [event.kind for event in events] == ["status", "case_header", "status"] + header = events[1] + assert header.model_dump() == { + "kind": "case_header", + "case_id": "case", + "source": "recording", + "progress": 1.0, + "question": "How many rooms?", + } diff --git a/dimos/benchmark/agent_eval/test_store.py b/dimos/benchmark/agent_eval/test_store.py new file mode 100644 index 0000000000..c39b679d7c --- /dev/null +++ b/dimos/benchmark/agent_eval/test_store.py @@ -0,0 +1,160 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import UTC, datetime +import json +from pathlib import Path + +import pytest + +from dimos.benchmark.agent_eval.artifacts import NormalizedOutcome +from dimos.benchmark.agent_eval.store import ( + AttemptAlreadyActiveError, + AttemptStore, + new_operation_id, +) + + +def _outcome( + attempt_id: str, + *, + attempt_status: str = "completed", + task_result: str = "passed", + complete: bool = True, + reason: str = "native evaluator terminal result", +) -> NormalizedOutcome: + return NormalizedOutcome( + attempt_id=attempt_id, + attempt_status=attempt_status, + task_result=task_result, + terminal_stage="terminal", + reason=reason, + required_artifacts_complete=complete, + finished_at=datetime.now(UTC), + duration_s=1.0, + ) + + +@pytest.mark.parametrize("task_result", ["passed", "failed"]) +def test_completed_pass_and_fail_write_atomic_outcome(tmp_path: Path, task_result: str) -> None: + with AttemptStore(tmp_path) as store: + reference = store.write_outcome(_outcome(store.attempt_id, task_result=task_result)) + + assert reference.path == "outcome.v1.json" + assert store.verify_artifacts((reference,)) + payload = json.loads((store.path / reference.path).read_text()) + assert payload["attempt_status"] == "completed" + assert payload["task_result"] == task_result + + +def test_infrastructure_failure_is_not_evaluated_and_retains_partial_evidence( + tmp_path: Path, +) -> None: + with AttemptStore(tmp_path) as store: + event = store.append_event( + "reset-failed", + operation_id=new_operation_id(), + payload={"stage": "reset"}, + ) + diagnostic = store.write_artifact("diagnostics/reset.txt", "reset timed out") + outcome = store.write_outcome( + _outcome( + store.attempt_id, + attempt_status="failed", + task_result="not_evaluated", + complete=False, + reason="reset timed out", + ) + ) + + assert event.sequence == 1 + assert store.verify_artifacts((diagnostic, outcome)) + + +def test_interrupted_store_retains_events_and_releases_target_lock( + tmp_path: Path, +) -> None: + first = AttemptStore(tmp_path) + first.append_event("interrupted") + first_path = first.path + first.close() + + with AttemptStore(tmp_path) as second: + assert second.path != first_path + assert (first_path / "events.jsonl").read_text() + + +def test_missing_or_changed_artifact_is_detected(tmp_path: Path) -> None: + with AttemptStore(tmp_path) as store: + missing = store.write_artifact("partial.json", {"retained": True}) + (store.path / missing.path).unlink() + + assert not store.verify_artifacts((missing,)) + + +def test_existing_attempt_directory_is_never_reused(tmp_path: Path) -> None: + attempt_id = "attempt_" + "1" * 32 + (tmp_path / attempt_id).mkdir(parents=True) + + with pytest.raises(FileExistsError): + AttemptStore(tmp_path, attempt_id) + + +def test_concurrent_attempt_against_same_target_is_rejected(tmp_path: Path) -> None: + first = AttemptStore(tmp_path) + try: + with pytest.raises(AttemptAlreadyActiveError): + AttemptStore(tmp_path) + finally: + first.close() + + +def test_events_are_append_only_correlated_and_monotonic(tmp_path: Path) -> None: + with AttemptStore(tmp_path) as store: + operation_id = new_operation_id() + first = store.append_event("reset-started", operation_id=operation_id) + second = store.append_event("reset-finished", operation_id=operation_id) + + records = [json.loads(line) for line in (store.path / "events.jsonl").read_text().splitlines()] + assert [record["sequence"] for record in records] == [1, 2] + assert first.operation_id == second.operation_id == operation_id + assert first.monotonic_offset_s <= second.monotonic_offset_s + assert all(record["attempt_id"] == store.attempt_id for record in records) + + +def test_outcome_is_non_overwriting(tmp_path: Path) -> None: + with AttemptStore(tmp_path) as store: + store.write_outcome(_outcome(store.attempt_id)) + + with pytest.raises(FileExistsError): + store.write_outcome( + _outcome(store.attempt_id, task_result="failed", reason="different") + ) + + +def test_outcome_write_failure_keeps_partial_evidence_and_no_outcome( + tmp_path: Path, monkeypatch +) -> None: + with AttemptStore(tmp_path) as store: + evidence = store.write_artifact("task.v1.json", {"task": "public"}) + + def fail_link(_source: Path, _destination: Path) -> None: + raise OSError("simulated link failure") + + monkeypatch.setattr("dimos.benchmark.agent_eval.store.os.link", fail_link) + with pytest.raises(OSError, match="simulated link failure"): + store.write_outcome(_outcome(store.attempt_id)) + + assert store.verify_artifacts((evidence,)) + assert not (store.path / "outcome.v1.json").exists() diff --git a/dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/README.md b/dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/README.md new file mode 100644 index 0000000000..a6a132141a --- /dev/null +++ b/dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/README.md @@ -0,0 +1,17 @@ +# Hong Kong office room-count CLI smoke case + +This case exercises the real `go2_hongkong_office` recording at progress `1.0` +through the standalone evaluation CLI. + +Its expected count is the synthetic sentinel `0`. It validates CLI and runtime +plumbing only and is not the benchmark room-count oracle. Do not interpret a +failed task score as an agent or mapping regression. + +The authoritative case remains incomplete until a human-authored room inventory, +counting policy, and independent review establish the expected count. + +The credentialed API-key smoke was exercised on 2026-08-04 after adding live +progress. The final operational attempt completed in 312.9 seconds with 40 +successful `python_exec` broker calls and a parsed `ANSWER: 8`. Its task score +was expectedly failed because this fixture's synthetic oracle is `0`; the result +must not be used as the authoritative room count. diff --git a/dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/case.json b/dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/case.json new file mode 100644 index 0000000000..69fcc00ef1 --- /dev/null +++ b/dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/case.json @@ -0,0 +1,31 @@ +{ + "schema_version": "1.0", + "case_id": "go2-hongkong-office-room-count-smoke", + "source": { + "schema_version": "1.0", + "kind": "frozen_memory", + "recording": "go2_hongkong_office", + "progress": 1.0, + "bundle_manifest_sha256": null + }, + "task": { + "schema_version": "1.0", + "kind": "integer_question", + "prompt": "How many rooms in total?", + "answer_marker": "ANSWER:" + }, + "interaction": { + "schema_version": "1.0", + "kind": "frozen_code_policy", + "driver_revision": "standalone-frozen-v1", + "session_lifetime": "one_attempt" + }, + "validator": { + "schema_version": "1.0", + "kind": "exact_integer", + "revision": "synthetic-cli-smoke-v1", + "private_path": "private/oracle.json", + "private_sha256": "e60d4c73c7b3d75a85358f209a15e5fc4e5a4191395bf9d799be42db5fa1f196" + }, + "fingerprint": "5a55ee32257d7acb1f9918f327cfe9253a632619fad40db7f9ad164b69568e8e" +} diff --git a/dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/private/oracle.json b/dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/private/oracle.json new file mode 100644 index 0000000000..e92000e3bc --- /dev/null +++ b/dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/private/oracle.json @@ -0,0 +1,9 @@ +{ + "schema_version": "1.0", + "expected_count": 0, + "counting_policy": "Synthetic CLI plumbing sentinel only; this is not the Hong Kong office room-count oracle.", + "rooms": [], + "reviewed_by": [ + "synthetic-cli-smoke-fixture" + ] +} diff --git a/dimos/benchmark/short_horizon_qa/eval.py b/dimos/benchmark/short_horizon_qa/eval.py new file mode 100644 index 0000000000..f09eb278be --- /dev/null +++ b/dimos/benchmark/short_horizon_qa/eval.py @@ -0,0 +1,412 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Frozen Memory2 QA drivers for the canonical agent-evaluation engine.""" + +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +from pathlib import Path +import re +from typing import Any, cast + +from pydantic import Field, JsonValue + +from dimos.agents.code_policy_core import ( + CodePolicySessionConfig, + FrozenMemoryEnvironment, +) +from dimos.agents.code_policy_server import StandaloneCodePolicyProcess +from dimos.agents.mcp.mcp_adapter import McpAdapter +from dimos.benchmark.agent_eval.base import BaseEvalModel +from dimos.benchmark.agent_eval.case import ( + AgentCondition, + AgentOutcome, + AttemptRequest, + EvalCase, + ExactIntegerValidatorRef, + FrozenCodePolicyInteraction, + FrozenRecordingSource, + IntegerQuestionTask, + Prediction, + PrivateScore, + RuntimeBinding, + SourceSpec, +) +from dimos.benchmark.agent_eval.engine import AttemptEngine, EngineResult +from dimos.benchmark.agent_eval.interfaces import ( + AgentAdapter, + AttemptContext, + EvidenceSink, + PreparedSource, + ValidatorSession, +) +from dimos.benchmark.agent_eval.pi import PiSession, PiSessionFactory +from dimos.benchmark.agent_eval.pi_adapter import ( + CodePolicyCallLog, + wait_for_python_exec, +) +from dimos.benchmark.short_horizon_qa.service import load_bundle + +_ANSWER_LINE = re.compile(r"(?m)^ANSWER:\s*") +_TERMINAL_INTEGER = re.compile(r"(?:^|\n)ANSWER:\s*(-?\d+)\s*\Z") + + +class RoomOracleEntry(BaseEvalModel): + label: str = Field(min_length=1) + evidence: tuple[str, ...] = Field(min_length=1) + + +class ExactIntegerOracle(BaseEvalModel): + expected_count: int = Field(ge=0) + counting_policy: str = Field(min_length=1) + rooms: tuple[RoomOracleEntry, ...] + reviewed_by: tuple[str, ...] = Field(min_length=1) + + +class FrozenMemorySourceDriver: + def __init__(self, bundle: Path) -> None: + self.bundle = bundle + + def prepare( + self, + *, + source: SourceSpec, + context: AttemptContext, + evidence: EvidenceSink, + ) -> PreparedSource: + del context + if not isinstance(source, FrozenRecordingSource): + raise TypeError("frozen source driver requires FrozenRecordingSource") + manifest, cutoff, source_path, derived_path = load_bundle( + self.bundle, progress=source.progress + ) + if source_path.stem != source.recording: + raise ValueError("prepared recording does not match the authored source") + if ( + source.bundle_manifest_sha256 is not None + and source.bundle_manifest_sha256 + != hashlib.sha256((self.bundle / "manifest.v1.json").read_bytes()).hexdigest() + ): + raise ValueError("prepared manifest digest does not match the case") + evidence.artifact("source-manifest.v1.json", manifest) + receipt: dict[str, JsonValue] = { + "recording": source.recording, + "progress": source.progress, + "cutoff_seconds": cutoff.cutoff_seconds, + "cutoff_timestamp": cutoff.cutoff_timestamp, + "source_sha256": manifest.source_sha256, + "derived_sha256": manifest.derived_sha256, + } + return PreparedSource( + public={"recording": source.recording, "progress": source.progress}, + receipt=receipt, + private_handle={ + "source_path": str(source_path), + "derived_path": str(derived_path), + "cutoff_timestamp": cutoff.cutoff_timestamp, + }, + ) + + def close(self) -> None: + return None + + +@dataclass(frozen=True) +class CodePolicyAgentInterface: + mcp: McpAdapter + session_id: str + call_log: CodePolicyCallLog + evidence: EvidenceSink + + +class PiCodePolicyAgent(AgentAdapter): + def __init__(self, factory: PiSessionFactory, *, turn_timeout_s: float = 180.0) -> None: + self.factory = factory + self.turn_timeout_s = turn_timeout_s + self._session: PiSession | None = None + self._interface: CodePolicyAgentInterface | None = None + + def run( + self, + *, + task: Any, + context: AttemptContext, + interface: Any = None, + ) -> AgentOutcome: + if not isinstance(task, IntegerQuestionTask): + raise TypeError("frozen Pi agent requires an integer question task") + if not isinstance(interface, CodePolicyAgentInterface): + raise TypeError("frozen Pi agent requires a CodePolicy interface") + prompt = ( + f"{task.prompt}\n\n" + "Use the provided python_exec tool and the read-only `memory` API to " + "answer from the recording. End your final response with exactly " + f"`{task.answer_marker} `." + ) + session = self.factory.create( + attempt_path=context.path, + public_prompt=prompt, + code_policy_session_id=interface.session_id, + call_log=interface.call_log, + mcp=interface.mcp, + ) + self._session = session + self._interface = interface + turn = session.prompt(prompt, self.turn_timeout_s) + return AgentOutcome( + final_text=turn.final_text, + tool_call_count=turn.policy_call_count, + terminal_reason="pi turn completed", + agent_session_id=session.session_id, + interaction_session_id=interface.session_id, + ) + + def close(self) -> None: + session = self._session + interface = self._interface + self._session = None + self._interface = None + if session is None: + return + session.dispose() + if interface is not None: + for artifact in session.artifact_references(): + interface.evidence.reference(artifact.path) + + +class FrozenCodePolicyInteractionDriver: + def __init__(self, *, readiness_timeout_s: float = 10.0) -> None: + self.readiness_timeout_s = readiness_timeout_s + self._process: StandaloneCodePolicyProcess | None = None + self._call_log: CodePolicyCallLog | None = None + self._evidence: EvidenceSink | None = None + self._session_id: str | None = None + + def run( + self, + *, + case: EvalCase, + prepared_source: PreparedSource, + agent: AgentAdapter, + context: AttemptContext, + evidence: EvidenceSink, + ) -> AgentOutcome: + if not isinstance(case.interaction, FrozenCodePolicyInteraction): + raise TypeError("frozen interaction driver received an incompatible case") + handle = prepared_source.private_handle + if not isinstance(handle, dict): + raise TypeError("frozen source did not provide a private binding") + config = CodePolicySessionConfig( + environment=FrozenMemoryEnvironment( + recording_path=str(handle["source_path"]), + derived_recording_path=str(handle["derived_path"]), + memory_cutoff_timestamp=float(handle["cutoff_timestamp"]), + ) + ) + process = StandaloneCodePolicyProcess(config) + process.start(self.readiness_timeout_s) + self._process = process + self._evidence = evidence + adapter = McpAdapter(process.mcp_url, timeout=120) + inventory = wait_for_python_exec(process.mcp_url, adapter, self.readiness_timeout_s) + evidence.artifact("mcp-inventory.v1.json", inventory) + receipt = process.receipt() + session_id = str(receipt["session_id"]) + self._session_id = session_id + evidence.artifact("code-policy-session.v1.json", receipt) + call_log = CodePolicyCallLog(context.path / "code-policy-calls.jsonl") + self._call_log = call_log + return agent.run( + task=case.task, + context=context, + interface=CodePolicyAgentInterface( + mcp=adapter, + session_id=session_id, + call_log=call_log, + evidence=evidence, + ), + ) + + def close(self) -> None: + process = self._process + call_log = self._call_log + evidence = self._evidence + session_id = self._session_id + self._process = None + self._call_log = None + self._evidence = None + self._session_id = None + if call_log is not None: + call_log.close() + if evidence is not None: + evidence.reference("code-policy-calls.jsonl") + if process is not None: + try: + if evidence is not None: + evidence.artifact( + "code-policy-records.v1.json", + cast("JsonValue", process.records(session_id)), + ) + finally: + process.close() + + +class ExactIntegerValidatorDriver: + def __init__(self, private_root: Path) -> None: + self.private_root = private_root.resolve() + + def prepare( + self, + *, + case: EvalCase, + prepared_source: PreparedSource, + context: AttemptContext, + evidence: EvidenceSink, + ) -> ValidatorSession: + del prepared_source + oracle = load_exact_integer_oracle(case, self.private_root) + reference = case.validator + evidence.artifact("oracle.private.v1.json", oracle) + return _ExactIntegerValidatorSession( + case=case, + context=context, + evidence=evidence, + oracle=oracle, + revision=reference.revision, + ) + + +class _ExactIntegerValidatorSession: + def __init__( + self, + *, + case: EvalCase, + context: AttemptContext, + evidence: EvidenceSink, + oracle: ExactIntegerOracle, + revision: str, + ) -> None: + self.case = case + self.context = context + self.evidence = evidence + self.oracle = oracle + self.revision = revision + + def evaluate(self, outcome: AgentOutcome) -> PrivateScore: + if outcome.agent_session_id is None or outcome.interaction_session_id is None: + raise ValueError("agent outcome is missing session identities") + prediction = parse_integer_prediction( + case_id=self.case.case_id, + attempt_id=self.context.attempt_id, + agent_session_id=outcome.agent_session_id, + interaction_session_id=outcome.interaction_session_id, + final_text=outcome.final_text, + ) + self.evidence.artifact("prediction.v1.json", prediction) + passed = ( + prediction.status == "parsed" + and prediction.integer_answer == self.oracle.expected_count + ) + return PrivateScore( + case_id=self.case.case_id, + attempt_id=self.context.attempt_id, + validator_revision=self.revision, + passed=passed, + prediction_status=prediction.status, + ) + + def close(self) -> None: + return None + + +def load_exact_integer_oracle(case: EvalCase, private_root: Path) -> ExactIntegerOracle: + """Resolve and verify a case-relative private oracle before agent dispatch.""" + reference = case.validator + if not isinstance(reference, ExactIntegerValidatorRef): + raise TypeError("exact integer validator requires ExactIntegerValidatorRef") + root = private_root.resolve() + path = (root / reference.private_path).resolve() + if root not in path.parents: + raise ValueError("private oracle path escapes its case directory") + data = path.read_bytes() + if hashlib.sha256(data).hexdigest() != reference.private_sha256: + raise ValueError("private oracle digest does not match the case") + return ExactIntegerOracle.model_validate_json(data) + + +def parse_integer_prediction( + *, + case_id: str, + attempt_id: str, + agent_session_id: str, + interaction_session_id: str, + final_text: str, +) -> Prediction: + markers = _ANSWER_LINE.findall(final_text) + match = _TERMINAL_INTEGER.search(final_text) + if len(markers) != 1 or match is None: + return Prediction( + case_id=case_id, + attempt_id=attempt_id, + agent_session_id=agent_session_id, + interaction_session_id=interaction_session_id, + parser_revision="marked-integer-v1", + final_text=final_text, + status="invalid", + diagnostic="expected exactly one terminal ANSWER: marker", + ) + return Prediction( + case_id=case_id, + attempt_id=attempt_id, + agent_session_id=agent_session_id, + interaction_session_id=interaction_session_id, + parser_revision="marked-integer-v1", + final_text=final_text, + status="parsed", + integer_answer=int(match.group(1)), + ) + + +def run_frozen_case( + *, + case: EvalCase, + bundle: Path, + private_root: Path, + output_root: Path, + pi_factory: PiSessionFactory, + agent_condition: AgentCondition | None = None, + runtime_binding: RuntimeBinding | None = None, + turn_timeout_s: float = 180.0, +) -> EngineResult: + request = AttemptRequest( + case=case, + agent=agent_condition + or AgentCondition( + agent_id="pi-code-policy", + adapter="pi-node", + model="gpt-5.6-luna", + thinking_level="medium", + ), + runtime=runtime_binding or RuntimeBinding(runtime_id="local-standalone-code-policy"), + ) + return AttemptEngine( + request=request, + output_root=output_root, + source=FrozenMemorySourceDriver(bundle), + interaction=FrozenCodePolicyInteractionDriver(), + validator=ExactIntegerValidatorDriver(private_root), + agent=PiCodePolicyAgent(pi_factory, turn_timeout_s=turn_timeout_s), + ).run() diff --git a/dimos/benchmark/short_horizon_qa/models.py b/dimos/benchmark/short_horizon_qa/models.py new file mode 100644 index 0000000000..47703b8533 --- /dev/null +++ b/dimos/benchmark/short_horizon_qa/models.py @@ -0,0 +1,72 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Immutable records describing a prepared frozen-memory bundle.""" + +from __future__ import annotations + +import math +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class FrozenQaModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class MapperSettings(FrozenQaModel): + voxel_size_m: float = Field(default=0.05, gt=0) + block_count: int = Field(default=2_000_000, gt=0) + device: str = "CUDA:0" + carve_columns: bool = True + frame_id: str = Field(default="world", min_length=1) + emit_every: int = Field(default=5, gt=0) + + +class StreamBoundary(FrozenQaModel): + name: str = Field(min_length=1) + count: int = Field(ge=0) + last_observation_id: int | None + last_timestamp: float | None + + +class CutoffRecord(FrozenQaModel): + cutoff_seconds: float = Field(ge=0) + cutoff_timestamp: float + normalized_progress: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False) + stream_boundaries: tuple[StreamBoundary, ...] + map_observation_id: int + map_timestamp: float + map_frame_count: int = Field(gt=0) + + @model_validator(mode="after") + def progress_is_finite(self) -> CutoffRecord: + if self.normalized_progress is not None and not math.isfinite(self.normalized_progress): + raise ValueError("normalized progress must be finite") + return self + + +class FrozenMemoryManifest(FrozenQaModel): + record_type: Literal["frozen-memory-bundle"] = "frozen-memory-bundle" + schema_version: Literal["1.0"] = "1.0" + source_path: str = Field(min_length=1) + source_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + source_size_bytes: int = Field(gt=0) + recording_start_timestamp: float + recording_end_timestamp: float + derived_path: Literal["derived.db"] = "derived.db" + derived_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + mapper: MapperSettings + cutoffs: tuple[CutoffRecord, ...] = Field(min_length=1) diff --git a/dimos/benchmark/short_horizon_qa/prepare.py b/dimos/benchmark/short_horizon_qa/prepare.py new file mode 100644 index 0000000000..0a3b8ced74 --- /dev/null +++ b/dimos/benchmark/short_horizon_qa/prepare.py @@ -0,0 +1,266 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Prepare reusable runtime-map snapshots for frozen Memory2 cutoffs.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +from pathlib import Path +import sqlite3 +import tempfile +from typing import Any, NamedTuple + +from dimos.benchmark.short_horizon_qa.models import ( + CutoffRecord, + FrozenMemoryManifest, + MapperSettings, + StreamBoundary, +) +from dimos.mapping.voxels.module import VoxelMapTransformer +from dimos.memory2.cli.dataset import resolve_dataset +from dimos.memory2.store.sqlite import SqliteStore +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 + +MANIFEST_NAME = "manifest.v1.json" +DERIVED_NAME = "derived.db" + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def prepare_bundle( + recording: str | Path, + cutoff_seconds: list[float] | None, + output: Path, + *, + progress: list[float] | None = None, + mapper: MapperSettings = MapperSettings(), +) -> FrozenMemoryManifest: + """Build one derived map sidecar without copying the source recording.""" + cutoffs = _validate_seconds(cutoff_seconds or []) + progresses = _validate_progress(progress or []) + if not cutoffs and not progresses: + raise ValueError("At least one cutoff in seconds or normalized progress is required") + if output.exists(): + raise FileExistsError(f"Output already exists: {output}") + + source_path = resolve_dataset(recording).resolve() + if not source_path.is_file(): + raise FileNotFoundError(source_path) + wal_path = Path(f"{source_path}-wal") + if wal_path.exists() and wal_path.stat().st_size > 0: + raise ValueError("Source recording has an active WAL and is not immutable") + + output.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=f".{output.name}-", dir=output.parent) as temporary: + temporary_path = Path(temporary) + manifest = _prepare_into(source_path, cutoffs, progresses, temporary_path, mapper) + os.replace(temporary_path, output) + return manifest + + +def _prepare_into( + source_path: Path, + cutoffs: list[float], + progresses: list[float], + output: Path, + mapper: MapperSettings, +) -> FrozenMemoryManifest: + derived_path = output / DERIVED_NAME + with SqliteStore(path=str(source_path), must_exist=True, read_only=True) as source: + if "global_map" in source.list_streams(): + raise ValueError("Source recording already contains a global_map stream") + ranges = _stream_ranges(source) + if not ranges: + raise ValueError("Source recording contains no observations") + recording_start = min(item[1] for item in ranges.values()) + recording_end = max(item[2] for item in ranges.values()) + duration = recording_end - recording_start + selections = [_CutoffSelection(seconds=value, progress=None) for value in cutoffs] + [ + _CutoffSelection( + seconds=resolve_progress(value, recording_start, recording_end) - recording_start, + progress=value, + ) + for value in progresses + ] + selections.sort(key=lambda item: (item.seconds, item.progress is None)) + absolute_cutoffs = [recording_start + item.seconds for item in selections] + if absolute_cutoffs[-1] > recording_end: + raise ValueError( + f"Cutoff {selections[-1].seconds}s exceeds recording duration {duration:.3f}s" + ) + cutoff_maps = _write_maps(source, derived_path, absolute_cutoffs, mapper) + records = tuple( + CutoffRecord( + cutoff_seconds=selection.seconds, + cutoff_timestamp=absolute, + normalized_progress=selection.progress, + stream_boundaries=_stream_boundaries(source, absolute), + map_observation_id=map_obs.id, + map_timestamp=map_obs.ts, + map_frame_count=int(map_obs.tags["frame_count"]), + ) + for selection, absolute, map_obs in zip( + selections, absolute_cutoffs, cutoff_maps, strict=True + ) + ) + + _seal_sqlite(derived_path) + manifest = FrozenMemoryManifest( + source_path=str(source_path), + source_sha256=file_sha256(source_path), + source_size_bytes=source_path.stat().st_size, + recording_start_timestamp=recording_start, + recording_end_timestamp=recording_end, + derived_sha256=file_sha256(derived_path), + mapper=mapper, + cutoffs=records, + ) + (output / MANIFEST_NAME).write_text( + json.dumps(manifest.model_dump(mode="json"), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return manifest + + +class _CutoffSelection(NamedTuple): + seconds: float + progress: float | None + + +def resolve_progress(progress: float, recording_start: float, recording_end: float) -> float: + """Resolve normalized progress to an exact timestamp over a sealed range.""" + if not math.isfinite(progress) or not 0 <= progress <= 1: + raise ValueError("Normalized progress must be finite and within [0, 1]") + if recording_end < recording_start: + raise ValueError("Recording end precedes recording start") + if progress == 0: + return recording_start + if progress == 1: + return recording_end + return recording_start + progress * (recording_end - recording_start) + + +def _validate_seconds(values: list[float]) -> list[float]: + if any(not math.isfinite(value) or value < 0 for value in values): + raise ValueError("Cutoffs must be finite and non-negative") + return sorted(set(values)) + + +def _validate_progress(values: list[float]) -> list[float]: + for value in values: + resolve_progress(value, 0.0, 1.0) + return sorted(set(values)) + + +def _stream_ranges(source: SqliteStore) -> dict[str, tuple[int, float, float]]: + result: dict[str, tuple[int, float, float]] = {} + for name in source.list_streams(): + stream = source.stream(name) + count = stream.count() + if count: + start, end = stream.get_time_range() + result[name] = (count, start, end) + return result + + +def _stream_boundaries(source: SqliteStore, cutoff: float) -> tuple[StreamBoundary, ...]: + boundaries: list[StreamBoundary] = [] + for name in source.list_streams(): + bounded = source.stream(name).through(cutoff) + count = bounded.count() + last = bounded.last() if count else None + boundaries.append( + StreamBoundary( + name=name, + count=count, + last_observation_id=last.id if last is not None else None, + last_timestamp=last.ts if last is not None else None, + ) + ) + return tuple(sorted(boundaries, key=lambda item: item.name)) + + +def _write_maps( + source: SqliteStore, + derived_path: Path, + cutoffs: list[float], + mapper: MapperSettings, +) -> list[Any]: + if "lidar" not in source.list_streams(): + raise ValueError("Source recording has no lidar stream") + lidar = source.stream("lidar", PointCloud2).as_read_only() + first = next(iter(lidar), None) + if first is None: + raise ValueError("No lidar observations exist before the final cutoff") + if first.data.frame_id != mapper.frame_id: + raise ValueError( + f"LiDAR frame {first.data.frame_id!r} does not match mapper frame {mapper.frame_id!r}" + ) + transformer = VoxelMapTransformer( + emit_every=mapper.emit_every, + voxel_size=mapper.voxel_size_m, + block_count=mapper.block_count, + device=mapper.device, + carve_columns=mapper.carve_columns, + frame_id=mapper.frame_id, + show_startup_log=False, + ) + emissions = iter(lidar.transform(transformer)) + try: + latest = next(emissions, None) + if latest is None: + raise ValueError("Mapper produced no global map") + + selected: list[Any] = [] + with SqliteStore(path=str(derived_path)) as derived: + target = derived.stream("global_map", PointCloud2) + stored_by_source_id: dict[int, Any] = {} + following = next(emissions, None) + for cutoff in cutoffs: + while following is not None and following.ts <= cutoff: + latest = following + following = next(emissions, None) + if latest.ts > cutoff: + raise ValueError(f"No runtime map was emitted by cutoff {cutoff}") + source_key = latest.id + stored = stored_by_source_id.get(source_key) + if stored is None: + stored = target.append( + latest.data, + ts=latest.ts, + tags={**latest.tags, "source_observation_id": source_key}, + ) + stored_by_source_id[source_key] = stored + selected.append(stored) + return selected + finally: + close = getattr(emissions, "close", None) + if close is not None: + close() + + +def _seal_sqlite(path: Path) -> None: + with sqlite3.connect(path) as connection: + connection.execute("PRAGMA wal_checkpoint(TRUNCATE)") + connection.execute("PRAGMA journal_mode=DELETE") diff --git a/dimos/benchmark/short_horizon_qa/service.py b/dimos/benchmark/short_horizon_qa/service.py new file mode 100644 index 0000000000..b372aedeea --- /dev/null +++ b/dimos/benchmark/short_horizon_qa/service.py @@ -0,0 +1,122 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline MCP service exposing CodePolicy over one frozen memory cutoff.""" + +from __future__ import annotations + +import math +from pathlib import Path + +from dimos.agents.code_policy_core import ( + CodePolicySessionConfig, + FrozenMemoryEnvironment, +) +from dimos.agents.code_policy_server import StandaloneCodePolicyServer +from dimos.benchmark.short_horizon_qa.models import CutoffRecord, FrozenMemoryManifest +from dimos.benchmark.short_horizon_qa.prepare import ( + DERIVED_NAME, + MANIFEST_NAME, + file_sha256, +) + + +def load_bundle( + bundle: Path, + cutoff_seconds: float | None = None, + *, + progress: float | None = None, + verify_integrity: bool = True, +) -> tuple[FrozenMemoryManifest, CutoffRecord, Path, Path]: + """Validate a prepared bundle and resolve one exact configured cutoff.""" + if (cutoff_seconds is None) == (progress is None): + raise ValueError("Select exactly one cutoff in seconds or normalized progress") + bundle = bundle.resolve() + manifest_path = bundle / MANIFEST_NAME + if not manifest_path.is_file(): + raise FileNotFoundError(manifest_path) + manifest = FrozenMemoryManifest.model_validate_json(manifest_path.read_text(encoding="utf-8")) + source_path = Path(manifest.source_path) + derived_path = bundle / DERIVED_NAME + if not source_path.is_file(): + raise FileNotFoundError(source_path) + if not derived_path.is_file(): + raise FileNotFoundError(derived_path) + + if progress is not None: + matches = [ + cutoff + for cutoff in manifest.cutoffs + if cutoff.normalized_progress is not None + and math.isclose(cutoff.normalized_progress, progress, rel_tol=0.0, abs_tol=1e-12) + ] + requested = f"progress {progress}" + available = ", ".join( + str(item.normalized_progress) + for item in manifest.cutoffs + if item.normalized_progress is not None + ) + else: + assert cutoff_seconds is not None + matches = [ + cutoff + for cutoff in manifest.cutoffs + if math.isclose(cutoff.cutoff_seconds, cutoff_seconds, rel_tol=0.0, abs_tol=1e-9) + ] + requested = f"cutoff {cutoff_seconds}s" + available = ", ".join(str(item.cutoff_seconds) for item in manifest.cutoffs) + if len(matches) != 1: + raise ValueError( + f"Requested {requested} is not unique in the bundle. Available: {available}" + ) + + if verify_integrity: + if source_path.stat().st_size != manifest.source_size_bytes: + raise ValueError(f"Source recording size changed: {source_path}") + if file_sha256(source_path) != manifest.source_sha256: + raise ValueError(f"Source recording hash changed: {source_path}") + if file_sha256(derived_path) != manifest.derived_sha256: + raise ValueError(f"Derived recording hash changed: {derived_path}") + return manifest, matches[0], source_path, derived_path + + +def frozen_qa_config( + source_path: Path, + derived_path: Path, + cutoff: CutoffRecord, +) -> CodePolicySessionConfig: + """Build the module-independent session configuration for one cutoff.""" + return CodePolicySessionConfig( + environment=FrozenMemoryEnvironment( + recording_path=str(source_path), + derived_recording_path=str(derived_path), + memory_cutoff_timestamp=cutoff.cutoff_timestamp, + ) + ) + + +def serve_bundle( + bundle: Path, + cutoff_seconds: float | None = None, + *, + progress: float | None = None, + mcp_port: int = 9990, +) -> None: + """Run the frozen QA MCP endpoint until interrupted.""" + _, cutoff, source_path, derived_path = load_bundle(bundle, cutoff_seconds, progress=progress) + server = StandaloneCodePolicyServer( + frozen_qa_config(source_path, derived_path, cutoff), + port=mcp_port, + ) + server.run_forever() diff --git a/dimos/benchmark/short_horizon_qa/test_eval.py b/dimos/benchmark/short_horizon_qa/test_eval.py new file mode 100644 index 0000000000..13e5f785b7 --- /dev/null +++ b/dimos/benchmark/short_horizon_qa/test_eval.py @@ -0,0 +1,223 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import numpy as np +import open3d as o3d +import pytest + +from dimos.benchmark.agent_eval.case import ( + EvalCase, + ExactIntegerValidatorRef, + FrozenCodePolicyInteraction, + FrozenRecordingSource, + IntegerQuestionTask, + RuntimeBinding, +) +from dimos.benchmark.agent_eval.pi import PiTurn +from dimos.benchmark.agent_eval.pi_adapter import ( + PythonExecBroker, + credential_binding_sha256, +) +from dimos.benchmark.short_horizon_qa.eval import ( + parse_integer_prediction, + run_frozen_case, +) +from dimos.benchmark.short_horizon_qa.models import MapperSettings +from dimos.benchmark.short_horizon_qa.prepare import prepare_bundle +from dimos.memory2.store.sqlite import SqliteStore +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 + + +def _cloud(x: float, ts: float) -> PointCloud2: + cloud = o3d.geometry.PointCloud() + cloud.points = o3d.utility.Vector3dVector(np.asarray([[x, 0.0, 0.5]])) + return PointCloud2(cloud, frame_id="world", ts=ts) + + +class ScriptedPiSession: + def __init__(self, broker: PythonExecBroker, final_text: str) -> None: + self.session_id = "pi_session_scripted" + self.broker = broker + self.final_text = final_text + + def prompt(self, prompt: str, timeout_s: float) -> PiTurn: + del prompt, timeout_s + self.broker.request( + "python_exec", + { + "code": "(memory.streams.lidar.count(), " + "memory.streams.global_map.last().tags['frame_count'], " + "'app' in globals())" + }, + ) + return PiTurn(final_text=self.final_text, policy_call_count=1) + + def abort(self, timeout_s: float) -> None: + del timeout_s + + def dispose(self) -> None: + return None + + def artifact_references(self): + return () + + +class ScriptedPiFactory: + def __init__(self, final_text: str) -> None: + self.final_text = final_text + self.public_prompts: list[str] = [] + + def create( + self, + *, + attempt_path: Path, + public_prompt: str, + code_policy_session_id: str, + call_log, + mcp, + ) -> ScriptedPiSession: + self.public_prompts.append(public_prompt) + return ScriptedPiSession( + PythonExecBroker( + attempt_id=attempt_path.name, + pi_session_id="pi_session_scripted", + code_policy_session_id=code_policy_session_id, + mcp=mcp, + call_log=call_log, + ), + self.final_text, + ) + + +@pytest.mark.parametrize( + ("text", "status", "answer"), + [ + ("I counted them.\nANSWER: 4", "parsed", 4), + ("ANSWER: -2", "parsed", -2), + ("The answer is 4", "invalid", None), + ("ANSWER: 3\nthen maybe\nANSWER: 4", "invalid", None), + ("ANSWER: 4\nextra", "invalid", None), + ], +) +def test_marked_integer_parser(text: str, status: str, answer: int | None) -> None: + prediction = parse_integer_prediction( + case_id="case", + attempt_id="attempt", + agent_session_id="pi", + interaction_session_id="policy", + final_text=text, + ) + assert prediction.status == status + assert prediction.integer_answer == answer + + +@pytest.mark.parametrize( + ("final_text", "task_result"), + [("Used memory.\nANSWER: 2", "passed"), ("ANSWER: 3", "failed"), ("2", "failed")], +) +def test_real_standalone_frozen_attempt_scores_scripted_pi( + tmp_path: Path, final_text: str, task_result: str +) -> None: + recording = tmp_path / "recording.db" + with SqliteStore(path=str(recording)) as store: + lidar = store.stream("lidar", PointCloud2) + for index in range(5): + ts = 100.0 + index + lidar.append(_cloud(float(index), ts), ts=ts) + bundle = tmp_path / "bundle" + prepare_bundle( + recording, + [], + bundle, + progress=[1.0], + mapper=MapperSettings(device="CPU:0"), + ) + private_root = tmp_path / "validators" + oracle_path = private_root / "private" / "oracle.json" + oracle_path.parent.mkdir(parents=True) + oracle_sentinel = "ORACLE_PRIVATE_SENTINEL_7f9d" + credential_sentinel = "CREDENTIAL_PRIVATE_SENTINEL_a13c" + oracle = { + "schema_version": "1.0", + "expected_count": 2, + "counting_policy": oracle_sentinel, + "rooms": [ + {"schema_version": "1.0", "label": "one", "evidence": ["test"]}, + {"schema_version": "1.0", "label": "two", "evidence": ["test"]}, + ], + "reviewed_by": ["test-reviewer"], + } + oracle_path.write_text(json.dumps(oracle)) + digest = hashlib.sha256(oracle_path.read_bytes()).hexdigest() + case = EvalCase.compile( + case_id="recording-room-count", + source=FrozenRecordingSource(recording="recording", progress=1.0), + task=IntegerQuestionTask(prompt="How many rooms in total?"), + interaction=FrozenCodePolicyInteraction(driver_revision="v1"), + validator=ExactIntegerValidatorRef( + revision="exact-v1", + private_path="private/oracle.json", + private_sha256=digest, + ), + ) + + factory = ScriptedPiFactory(final_text) + result = run_frozen_case( + case=case, + bundle=bundle, + private_root=private_root, + output_root=tmp_path / "attempts", + pi_factory=factory, + runtime_binding=RuntimeBinding( + runtime_id="local-standalone-code-policy", + parameters={ + "credential_binding_sha256": credential_binding_sha256( + "environment", "TEST_API_KEY", credential_sentinel + ) + }, + ), + ) + + assert result.outcome.attempt_status == "completed" + assert result.outcome.task_result == task_result + assert (result.attempt_path / "prediction.v1.json").is_file() + assert (result.attempt_path / "score.private.v1.json").is_file() + assert (result.attempt_path / "attempt-manifest.v1.json").is_file() + calls = (result.attempt_path / "code-policy-calls.jsonl").read_text().splitlines() + assert len(calls) == 1 + records = json.loads((result.attempt_path / "code-policy-records.v1.json").read_text()) + assert "False" in records[0]["output"] + + # The private oracle is intentionally retained in its private artifact. Every + # model-facing or public/runtime surface must remain sentinel-free. + public_surfaces = [ + case.public_projection().model_dump_json(), + *factory.public_prompts, + result.outcome.model_dump_json(), + ] + for path in result.attempt_path.rglob("*"): + if path.is_file() and path.name not in { + "oracle.private.v1.json", + "score.private.v1.json", + }: + public_surfaces.append(path.read_text(errors="replace")) + serialized = "\n".join(public_surfaces) + assert oracle_sentinel not in serialized + assert credential_sentinel not in serialized diff --git a/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py b/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py new file mode 100644 index 0000000000..bbe24b1e38 --- /dev/null +++ b/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py @@ -0,0 +1,98 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Self-hosted mechanics gate over the real Hong Kong office recording. + +The zero-valued oracle in this test is deliberately synthetic and must never be +used as the north-star room-count oracle. It validates plumbing only. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from dimos.benchmark.agent_eval.case import ( + EvalCase, + ExactIntegerValidatorRef, + FrozenCodePolicyInteraction, + FrozenRecordingSource, + IntegerQuestionTask, +) +from dimos.benchmark.short_horizon_qa.eval import run_frozen_case +from dimos.benchmark.short_horizon_qa.models import MapperSettings +from dimos.benchmark.short_horizon_qa.prepare import prepare_bundle +from dimos.benchmark.short_horizon_qa.test_eval import ScriptedPiFactory +from dimos.utils.data import get_data + + +@pytest.mark.self_hosted +def test_real_hongkong_recording_standalone_scripted_mechanics(tmp_path: Path) -> None: + recording = get_data("go2_hongkong_office.db") + bundle = tmp_path / "bundle" + manifest = prepare_bundle( + recording, + [], + bundle, + progress=[1.0], + mapper=MapperSettings(device="CPU:0"), + ) + private_root = tmp_path / "validators" + oracle_path = private_root / "private" / "mechanics-only.json" + oracle_path.parent.mkdir(parents=True) + oracle_path.write_text( + json.dumps( + { + "schema_version": "1.0", + "expected_count": 0, + "counting_policy": "Synthetic mechanics value; not a room oracle.", + "rooms": [], + "reviewed_by": ["self-hosted-mechanics-test"], + } + ) + ) + case = EvalCase.compile( + case_id="hongkong-office-mechanics-only", + source=FrozenRecordingSource( + recording="go2_hongkong_office", + progress=1.0, + bundle_manifest_sha256=hashlib.sha256( + (bundle / "manifest.v1.json").read_bytes() + ).hexdigest(), + ), + task=IntegerQuestionTask(prompt="How many rooms in total?"), + interaction=FrozenCodePolicyInteraction(driver_revision="v1"), + validator=ExactIntegerValidatorRef( + revision="mechanics-only-v1", + private_path="private/mechanics-only.json", + private_sha256=hashlib.sha256(oracle_path.read_bytes()).hexdigest(), + ), + ) + + result = run_frozen_case( + case=case, + bundle=bundle, + private_root=private_root, + output_root=tmp_path / "attempts", + pi_factory=ScriptedPiFactory("ANSWER: 0"), + ) + + assert result.outcome.task_result == "passed" + assert manifest.cutoffs[0].normalized_progress == 1.0 + assert manifest.cutoffs[0].map_frame_count == 4235 + records = json.loads((result.attempt_path / "code-policy-records.v1.json").read_text()) + assert "False" in records[0]["output"] diff --git a/dimos/benchmark/short_horizon_qa/test_prepare.py b/dimos/benchmark/short_horizon_qa/test_prepare.py new file mode 100644 index 0000000000..93c8b073e0 --- /dev/null +++ b/dimos/benchmark/short_horizon_qa/test_prepare.py @@ -0,0 +1,219 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import open3d as o3d +import pytest + +from dimos.agents.code_policy_core import FrozenMemoryEnvironment +from dimos.benchmark.short_horizon_qa.models import MapperSettings +from dimos.benchmark.short_horizon_qa.prepare import ( + file_sha256, + prepare_bundle, + resolve_progress, +) +from dimos.benchmark.short_horizon_qa.service import frozen_qa_config, load_bundle +from dimos.memory2.store.frozen import FrozenMemoryStore +from dimos.memory2.store.sqlite import SqliteStore +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 + + +def _point_cloud(x: float, *, frame_id: str = "world", ts: float) -> PointCloud2: + cloud = o3d.geometry.PointCloud() + cloud.points = o3d.utility.Vector3dVector(np.asarray([[x, 0.0, 0.5]])) + return PointCloud2(cloud, frame_id=frame_id, ts=ts) + + +@pytest.fixture +def recording(tmp_path: Path) -> Path: + path = tmp_path / "recording.db" + with SqliteStore(path=str(path)) as store: + lidar = store.stream("lidar", PointCloud2) + odom = store.stream("odom", int) + for index in range(10): + timestamp = 100.0 + index + lidar.append(_point_cloud(index * 0.2, ts=timestamp), ts=timestamp) + odom.append(index, ts=timestamp) + return path + + +def test_prepare_builds_reusable_runtime_maps_without_changing_source( + recording: Path, tmp_path: Path +) -> None: + output = tmp_path / "bundle" + before = file_sha256(recording) + + manifest = prepare_bundle( + recording, + [4.0, 9.0], + output, + mapper=MapperSettings(device="CPU:0"), + ) + + assert file_sha256(recording) == before + assert [item.map_frame_count for item in manifest.cutoffs] == [5, 10] + assert [item.map_timestamp for item in manifest.cutoffs] == [104.0, 109.0] + assert (output / "derived.db").is_file() + encoded = json.loads((output / "manifest.v1.json").read_text()) + assert encoded["source_sha256"] == before + + with FrozenMemoryStore( + SqliteStore(path=str(recording), must_exist=True, read_only=True), + derived=SqliteStore(path=str(output / "derived.db"), must_exist=True, read_only=True), + through_timestamp=104.0, + ) as memory: + assert memory.streams.lidar.count() == 5 + assert memory.streams.odom.last().data == 4 + assert memory.streams.global_map.last().tags["frame_count"] == 5 + assert len(memory.streams.global_map.last().data) == 5 + + +def test_prepare_reuses_one_derived_map_for_nearby_cutoffs(recording: Path, tmp_path: Path) -> None: + output = tmp_path / "bundle" + + manifest = prepare_bundle( + recording, + [4.0, 4.5], + output, + mapper=MapperSettings(device="CPU:0"), + ) + + assert manifest.cutoffs[0].map_observation_id == manifest.cutoffs[1].map_observation_id + with SqliteStore(path=str(output / "derived.db"), must_exist=True, read_only=True) as derived: + assert derived.stream("global_map").count() == 1 + + +def test_progress_resolves_exact_end_and_persists_provenance( + recording: Path, tmp_path: Path +) -> None: + output = tmp_path / "progress-bundle" + manifest = prepare_bundle( + recording, + [], + output, + progress=[4 / 9, 1.0, 1.0], + mapper=MapperSettings(device="CPU:0"), + ) + + assert [item.normalized_progress for item in manifest.cutoffs] == [4 / 9, 1.0] + assert [item.cutoff_timestamp for item in manifest.cutoffs] == [104.0, 109.0] + assert manifest.cutoffs[-1].stream_boundaries[0].count == 10 + loaded_manifest, cutoff, _, _ = load_bundle(output, progress=1.0) + assert cutoff.cutoff_timestamp == loaded_manifest.recording_end_timestamp + + encoded = json.loads((output / "manifest.v1.json").read_text()) + assert encoded["cutoffs"][-1]["normalized_progress"] == 1.0 + + +def test_progress_resolution_has_exact_endpoints_and_linear_interior() -> None: + assert resolve_progress(0.0, 100.0, 109.0) == 100.0 + assert resolve_progress(0.5, 100.0, 110.0) == 105.0 + assert resolve_progress(1.0, 100.0, 109.0) == 109.0 + + +@pytest.mark.parametrize("progress", [-0.1, 1.1, float("inf"), float("nan")]) +def test_prepare_rejects_invalid_progress(recording: Path, tmp_path: Path, progress: float) -> None: + with pytest.raises(ValueError, match="progress|Progress"): + prepare_bundle( + recording, + [], + tmp_path / "bundle", + progress=[progress], + mapper=MapperSettings(device="CPU:0"), + ) + + +def test_progress_before_first_map_preserves_runtime_emission_rule( + recording: Path, tmp_path: Path +) -> None: + with pytest.raises(ValueError, match="No runtime map was emitted"): + prepare_bundle( + recording, + [], + tmp_path / "bundle", + progress=[0.0], + mapper=MapperSettings(device="CPU:0"), + ) + + +def test_prepare_rejects_cutoff_before_first_runtime_emission( + recording: Path, tmp_path: Path +) -> None: + output = tmp_path / "bundle" + + with pytest.raises(ValueError, match="No runtime map was emitted"): + prepare_bundle( + recording, + [2.0], + output, + mapper=MapperSettings(device="CPU:0"), + ) + + assert not output.exists() + + +def test_prepare_rejects_non_world_lidar(tmp_path: Path) -> None: + recording = tmp_path / "sensor-frame.db" + with SqliteStore(path=str(recording)) as store: + lidar = store.stream("lidar", PointCloud2) + for index in range(5): + timestamp = 100.0 + index + lidar.append(_point_cloud(float(index), frame_id="lidar", ts=timestamp), ts=timestamp) + + with pytest.raises(ValueError, match="does not match mapper frame"): + prepare_bundle( + recording, + [4.0], + tmp_path / "bundle", + mapper=MapperSettings(device="CPU:0"), + ) + + +def test_bundle_loads_into_standalone_code_policy_config(recording: Path, tmp_path: Path) -> None: + output = tmp_path / "bundle" + prepare_bundle( + recording, + [4.0], + output, + mapper=MapperSettings(device="CPU:0"), + ) + + _, cutoff, source_path, derived_path = load_bundle(output, 4.0) + config = frozen_qa_config(source_path, derived_path, cutoff) + assert isinstance(config.environment, FrozenMemoryEnvironment) + assert config.environment.recording_path == str(recording.resolve()) + assert config.environment.derived_recording_path == str(derived_path) + assert config.environment.memory_cutoff_timestamp == 104.0 + + +def test_bundle_integrity_check_rejects_changed_derived_recording( + recording: Path, tmp_path: Path +) -> None: + output = tmp_path / "bundle" + prepare_bundle( + recording, + [4.0], + output, + mapper=MapperSettings(device="CPU:0"), + ) + with (output / "derived.db").open("ab") as stream: + stream.write(b"tampered") + + with pytest.raises(ValueError, match="Derived recording hash changed"): + load_bundle(output, 4.0) diff --git a/dimos/cli/dimos.py b/dimos/cli/dimos.py index ee6a92f457..0f50caaaea 100644 --- a/dimos/cli/dimos.py +++ b/dimos/cli/dimos.py @@ -50,6 +50,7 @@ from dimos.agents.mcp.mcp_adapter import McpAdapter, McpError from dimos.cli.cache import app as cache_app +from dimos.cli.eval import app as eval_app from dimos.cli.hardware_cli import app as hardware_app from dimos.cli.shell import shell from dimos.constants import CONFIG_DIR, LOG_DIR @@ -178,6 +179,7 @@ def callback(**kwargs) -> None: # type: ignore[no-untyped-def] main.add_typer(piper_app, name="piper") main.command()(shell) main.add_typer(cache_app, name="cache") +main.add_typer(eval_app, name="eval") def _with_relay_bridge(blueprint: Blueprint) -> Blueprint: diff --git a/dimos/cli/eval.py b/dimos/cli/eval.py new file mode 100644 index 0000000000..478b1c6dfe --- /dev/null +++ b/dimos/cli/eval.py @@ -0,0 +1,201 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dependency-light shell for the immutable single-case evaluation CLI.""" + +from __future__ import annotations + +import os +from pathlib import Path +import threading +from typing import Any, Literal + +import typer + +app = typer.Typer(help="Run immutable agent evaluation cases", no_args_is_help=True) + + +def execute_single_case(*args: Any, **kwargs: Any) -> Any: + """Import and dispatch the evaluation runtime only when ``eval run`` executes.""" + try: + from dimos.benchmark.agent_eval.single_case import execute_single_case as execute + except ModuleNotFoundError as exc: + raise RuntimeError( + "Evaluation dependencies are missing; run `uv sync --extra agents`" + ) from exc + + return execute(*args, **kwargs) + + +@app.command("run") +def run( + case: Path = typer.Argument(..., exists=True, dir_okay=False, readable=True), + agent_backend: Literal["pi"] = typer.Option("pi", "--agent.backend"), + agent_model: Literal["gpt-5.6-luna"] = typer.Option("gpt-5.6-luna", "--agent.model"), + thinking_level: Literal["medium"] = typer.Option("medium", "--agent.thinking-level"), + auth_mode: Literal["codex-oauth", "openai-api-key"] | None = typer.Option( + None, + "--agent.auth.mode", + help="Auth mode; inferred from auth options or OPENAI_API_KEY when omitted", + ), + auth_path: Path | None = typer.Option(None, "--agent.auth.path"), + auth_env: str | None = typer.Option(None, "--agent.auth.env"), + output: Path | None = typer.Option(None, "--output"), + json_output: bool = typer.Option(False, "--json", help="Print compact JSON"), + quiet: bool = typer.Option(False, "--quiet", help="Suppress live evaluation progress"), +) -> None: + """Run one static evaluation case synchronously.""" + from dimos.benchmark.agent_eval.single_case import ( + DEFAULT_OPENAI_API_KEY_ENV, + CodexOAuthConfig, + EvalRunConfig, + OpenAIApiKeyConfig, + PiAgentConfig, + ) + + if auth_mode is None: + if auth_path is not None and auth_env is not None: + raise typer.BadParameter( + "--agent.auth.path and --agent.auth.env select different auth modes" + ) + if auth_path is not None: + auth_mode = "codex-oauth" + elif auth_env is not None or os.environ.get(DEFAULT_OPENAI_API_KEY_ENV): + auth_mode = "openai-api-key" + else: + auth_mode = "codex-oauth" + auth: CodexOAuthConfig | OpenAIApiKeyConfig + if auth_mode == "codex-oauth": + if auth_env is not None: + raise typer.BadParameter("--agent.auth.env requires --agent.auth.mode=openai-api-key") + auth = CodexOAuthConfig(path=auth_path) + else: + if auth_path is not None: + raise typer.BadParameter("--agent.auth.path requires --agent.auth.mode=codex-oauth") + auth = OpenAIApiKeyConfig(env=auth_env or DEFAULT_OPENAI_API_KEY_ENV) + config = EvalRunConfig( + agent=PiAgentConfig( + backend=agent_backend, + model=agent_model, + thinking_level=thinking_level, + auth=auth, + ) + ) + renderer = None if quiet else ProgressRenderer() + try: + result = ( + execute_single_case(case, config=config, progress=renderer) + if output is None + else execute_single_case( + case, + config=config, + output_root=output, + progress=renderer, + ) + ) + except Exception as exc: + if renderer is not None: + renderer.finish() + typer.echo(f"Evaluation preflight failed: {type(exc).__name__}: {exc}", err=True) + raise typer.Exit(2) from exc + if renderer is not None: + renderer.finish() + typer.echo(result.model_dump_json() if json_output else format_result(result)) + if result.attempt_status == "failed": + raise typer.Exit(1) + + +def format_result(result: Any) -> str: + """Render the compact typed result without exposing private oracle material.""" + if result.attempt_status == "failed": + heading = "! Evaluation not evaluated" + elif result.task_result == "passed": + heading = "✓ Evaluation passed" + else: + heading = "✗ Evaluation failed" + source = result.source + if result.progress is not None: + source += f" @ {result.progress * 100:g}%" + answer = str(result.integer_answer) if result.integer_answer is not None else "—" + rows = ( + ("Case", result.case_id), + ("Source", source), + ("Question", result.question), + ("Answer", answer), + ("Result", result.task_result), + ( + "Agent", + f"{result.agent.agent_id} · {result.agent.model} · {result.agent.thinking_level}", + ), + ("Tool calls", str(result.tool_call_count)), + ("Duration", f"{result.duration_seconds:.1f}s"), + ("Attempt", result.attempt_id), + ("Artifacts", str(result.artifact_path)), + ) + body = "\n".join(f" {label:<10} {value}" for label, value in rows) + return f"{heading}\n\n{body}" + + +class ProgressRenderer: + """Thread-safe concise terminal renderer for best-effort evaluation progress.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._assistant_open = False + self._saw_assistant_text = False + + def __call__(self, event: Any) -> None: + with self._lock: + if event.kind == "assistant_text": + if not self._assistant_open: + typer.echo("[pi] ", err=True, nl=False) + self._assistant_open = True + typer.echo(event.delta, err=True, nl=False) + self._saw_assistant_text = True + return + self._end_assistant_line() + if event.kind == "case_header": + source = event.source + if event.progress is not None: + source += f" @ {event.progress * 100:g}%" + typer.echo("[eval] Session", err=True) + typer.echo(f" {'Case':<10} {event.case_id}", err=True) + typer.echo(f" {'Source':<10} {source}", err=True) + typer.echo(f" {'Question':<10} {event.question}", err=True) + typer.echo(f" {'Answer':<10} pending", err=True) + elif event.kind == "status": + typer.echo(f"[{event.channel}] {event.message}", err=True) + elif event.kind == "tool_start": + typer.echo("[python_exec] call", err=True) + typer.echo(_indent(event.code), err=True) + elif event.kind == "tool_end": + status = "ok" if event.ok else "error" + typer.echo(f"[python_exec] {status} ({event.duration_seconds:.1f}s)", err=True) + if event.result: + typer.echo(_indent(event.result), err=True) + elif event.kind == "final_response" and not self._saw_assistant_text: + typer.echo(f"[pi] {event.text}", err=True) + + def finish(self) -> None: + with self._lock: + self._end_assistant_line() + + def _end_assistant_line(self) -> None: + if self._assistant_open: + typer.echo("", err=True) + self._assistant_open = False + + +def _indent(value: str) -> str: + return "\n".join(f" {line}" for line in value.splitlines()) diff --git a/dimos/cli/test_eval.py b/dimos/cli/test_eval.py new file mode 100644 index 0000000000..d75b9ee901 --- /dev/null +++ b/dimos/cli/test_eval.py @@ -0,0 +1,267 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import builtins +import json +from pathlib import Path +import subprocess +import sys +import textwrap + +import pytest +from typer.testing import CliRunner + +from dimos.benchmark.agent_eval.case import AgentCondition +from dimos.benchmark.agent_eval.progress import ( + AssistantTextProgress, + CaseHeaderProgress, + StatusProgress, + ToolEndProgress, + ToolStartProgress, +) +from dimos.benchmark.agent_eval.single_case import CompactEvalResult +from dimos.cli.dimos import main +import dimos.cli.eval as eval_cli + + +def _result( + tmp_path: Path, *, status: str = "completed", task: str = "passed" +) -> CompactEvalResult: + return CompactEvalResult( + attempt_id="attempt_" + "a" * 32, + case_id="hongkong-room-count", + source="go2_hongkong_office", + progress=1.0, + question="How many rooms in total?", + attempt_status=status, + task_result=task, + reason="validator passed" if status == "completed" else "infrastructure failed", + prediction_status="parsed" if status == "completed" else None, + integer_answer=4 if status == "completed" else None, + agent=AgentCondition( + agent_id="pi-code-policy", + adapter="pi-node", + model="gpt-5.6-luna", + thinking_level="medium", + ), + tool_call_count=7, + duration_seconds=42.75, + artifact_path=tmp_path / "attempt", + ) + + +def _case(tmp_path: Path) -> Path: + path = tmp_path / "case.json" + path.write_text("{}") + return path + + +def test_eval_run_uses_typed_defaults_and_separates_progress(tmp_path, monkeypatch) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + captured = {} + + def execute(path, *, config, progress, **kwargs): + captured.update(path=path, config=config, progress=progress, kwargs=kwargs) + progress(StatusProgress(channel="eval", message="loading case")) + progress( + CaseHeaderProgress( + case_id="hongkong-room-count", + source="go2_hongkong_office", + progress=1.0, + question="How many rooms in total?", + ) + ) + progress(AssistantTextProgress(delta="Inspecting memory")) + progress(ToolStartProgress(code="memory.streams()")) + progress(ToolEndProgress(ok=True, result="['lidar']", duration_seconds=0.25)) + return _result(tmp_path) + + monkeypatch.setattr(eval_cli, "execute_single_case", execute) + result = CliRunner().invoke(main, ["eval", "run", str(_case(tmp_path))]) + + assert result.exit_code == 0, result.output + assert captured["config"].agent.auth.mode == "codex-oauth" + assert "✓ Evaluation passed" in result.stdout + assert "go2_hongkong_office @ 100%" in result.stdout + assert "[eval] loading case" in result.stderr + assert "[pi] Inspecting memory" in result.stderr + assert "[python_exec] ok (0.2s)" in result.stderr + + +def test_eval_run_auth_inference_and_explicit_precedence(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "credential-sentinel") + captured = [] + + def execute(*args, **kwargs): + captured.append(kwargs["config"]) + return _result(tmp_path) + + monkeypatch.setattr(eval_cli, "execute_single_case", execute) + runner = CliRunner() + automatic = runner.invoke(main, ["eval", "run", str(_case(tmp_path))]) + explicit = runner.invoke( + main, + ["eval", "run", str(_case(tmp_path)), "--agent.auth.mode=codex-oauth"], + ) + + assert automatic.exit_code == explicit.exit_code == 0 + assert captured[0].agent.auth.mode == "openai-api-key" + assert captured[1].agent.auth.mode == "codex-oauth" + assert "credential-sentinel" not in automatic.output + explicit.output + + +def test_eval_run_accepts_dotted_options_and_json(tmp_path, monkeypatch) -> None: + captured = {} + + def execute(*args, **kwargs): + captured.update(kwargs) + return _result(tmp_path) + + monkeypatch.setattr(eval_cli, "execute_single_case", execute) + output = tmp_path / "results" + result = CliRunner().invoke( + main, + [ + "eval", + "run", + str(_case(tmp_path)), + "--agent.backend=pi", + "--agent.model=gpt-5.6-luna", + "--agent.auth.mode=openai-api-key", + "--agent.auth.env=MY_OPENAI_KEY", + f"--output={output}", + "--json", + ], + ) + + assert result.exit_code == 0, result.output + assert captured["config"].agent.auth.env == "MY_OPENAI_KEY" + assert captured["output_root"] == output + assert json.loads(result.stdout)["task_result"] == "passed" + assert "private" not in result.stdout + + +def test_eval_run_quiet_and_exit_codes(tmp_path, monkeypatch) -> None: + observed = [] + + def failed_attempt(*args, **kwargs): + observed.append(kwargs["progress"]) + return _result(tmp_path, status="failed", task="not_evaluated") + + monkeypatch.setattr(eval_cli, "execute_single_case", failed_attempt) + failed = CliRunner().invoke(main, ["eval", "run", str(_case(tmp_path)), "--quiet"]) + + def preflight(*args, **kwargs): + raise FileNotFoundError("adapter build missing") + + monkeypatch.setattr(eval_cli, "execute_single_case", preflight) + preflight_result = CliRunner().invoke(main, ["eval", "run", str(_case(tmp_path))]) + + assert failed.exit_code == 1 + assert observed == [None] + assert failed.stderr == "" + assert preflight_result.exit_code == 2 + assert "Evaluation preflight failed: FileNotFoundError" in preflight_result.stderr + + +def test_eval_rejects_invalid_auth_combinations_and_semantic_override(tmp_path) -> None: + runner = CliRunner() + case = _case(tmp_path) + conflicting = runner.invoke( + main, + ["eval", "run", str(case), "--agent.auth.path=x", "--agent.auth.env=Y"], + ) + semantic = runner.invoke(main, ["eval", "run", str(case), "--source.recording=other"]) + + assert conflicting.exit_code == 2 + assert "select different auth" in conflicting.stderr + assert "modes" in conflicting.stderr + assert semantic.exit_code == 2 + assert "No such option" in semantic.stderr + + +def test_eval_help_is_typed_and_rejects_unsupported_model(tmp_path) -> None: + runner = CliRunner() + help_result = runner.invoke(main, ["eval", "run", "--help"]) + unsupported = runner.invoke( + main, + ["eval", "run", str(_case(tmp_path)), "--agent.model=unreviewed-model"], + ) + + assert help_result.exit_code == 0 + assert "--agent.model" in help_result.stdout + assert "gpt-5.6-luna" in help_result.stdout + assert "--agent.thinking-level" in help_result.stdout + assert unsupported.exit_code == 2 + assert "unreviewed-model" in unsupported.stderr + + +def test_eval_semantic_failure_is_a_successful_attempt(tmp_path, monkeypatch) -> None: + monkeypatch.setattr( + eval_cli, + "execute_single_case", + lambda *args, **kwargs: _result(tmp_path, status="completed", task="failed"), + ) + + result = CliRunner().invoke(main, ["eval", "run", str(_case(tmp_path))]) + + assert result.exit_code == 0 + assert "Evaluation failed" in result.stdout + + +def test_lazy_runtime_import_has_actionable_missing_agents_error(monkeypatch) -> None: + original_import = builtins.__import__ + + def fail_single_case(name, *args, **kwargs): + if name == "dimos.benchmark.agent_eval.single_case": + raise ModuleNotFoundError("No module named 'fastapi'") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", fail_single_case) + + with pytest.raises(RuntimeError, match="uv sync --extra agents"): + eval_cli.execute_single_case(Path("case.json"), config=None) + + +def test_base_cli_help_imports_without_agents_only_modules() -> None: + script = textwrap.dedent( + """ + import sys + + class BlockAgentsImports: + def find_spec(self, fullname, path=None, target=None): + if fullname.split('.')[0] in { + 'fastapi', 'ipykernel', 'jupyter_client', 'uvicorn' + }: + raise RuntimeError(f'agents-only import attempted: {fullname}') + return None + + sys.meta_path.insert(0, BlockAgentsImports()) + from typer.testing import CliRunner + from dimos.cli.dimos import main + + result = CliRunner().invoke(main, ['--help']) + assert result.exit_code == 0, result.output + assert 'eval' in result.stdout + """ + ) + + completed = subprocess.run( + [sys.executable, "-c", script], + check=False, + capture_output=True, + text=True, + ) + + assert completed.returncode == 0, completed.stdout + completed.stderr diff --git a/dimos/memory2/observationstore/sqlite.py b/dimos/memory2/observationstore/sqlite.py index 31c6a25ea0..ef7e4a1774 100644 --- a/dimos/memory2/observationstore/sqlite.py +++ b/dimos/memory2/observationstore/sqlite.py @@ -30,6 +30,7 @@ BeforeFilter, NearFilter, TagsFilter, + ThroughFilter, TimeRangeFilter, ) from dimos.memory2.type.observation import _UNLOADED, Observation, PoseTuple @@ -71,6 +72,8 @@ def _compile_filter(f: Filter, stream: str, prefix: str = "") -> tuple[str, list return (f"{prefix}ts > ?", [f.t]) if isinstance(f, BeforeFilter): return (f"{prefix}ts < ?", [f.t]) + if isinstance(f, ThroughFilter): + return (f"{prefix}ts <= ?", [f.t]) if isinstance(f, TimeRangeFilter): return (f"{prefix}ts >= ? AND {prefix}ts <= ?", [f.t1, f.t2]) if isinstance(f, AtFilter): @@ -209,6 +212,7 @@ class SqliteObservationStoreConfig(ObservationStoreConfig): blob_store_conn_match: bool = Field(default=False, exclude=True) page_size: int = 256 path: str | None = None + read_only: bool = False @model_validator(mode="after") def _conn_xor_path(self) -> SqliteObservationStoreConfig: @@ -249,9 +253,18 @@ def __init__(self, **kwargs: Any) -> None: def start(self) -> None: if self._conn is None: assert self._path is not None - disposable, self._conn = open_disposable_sqlite_connection(self._path) + disposable, self._conn = open_disposable_sqlite_connection( + self._path, read_only=self.config.read_only + ) self.register_disposable(disposable) - self._ensure_tables() + if self.config.read_only: + found = self._conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (self._name,) + ).fetchone() + if found is None: + raise KeyError(f"Stream table {self._name!r} does not exist") + else: + self._ensure_tables() def _ensure_tables(self) -> None: """Create the metadata table and R*Tree index if they don't exist.""" diff --git a/dimos/memory2/registry.py b/dimos/memory2/registry.py index a5707a8cea..ba464ebb7f 100644 --- a/dimos/memory2/registry.py +++ b/dimos/memory2/registry.py @@ -41,6 +41,7 @@ def deserialize_component(data: dict[str, Any]) -> Any: class RegistryStoreConfig(BaseConfig): conn: sqlite3.Connection = Field(exclude=True) + read_only: bool = False class RegistryStore(Configurable): @@ -51,13 +52,20 @@ class RegistryStore(Configurable): def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) self._conn: sqlite3.Connection = self.config.conn - self._conn.execute( - "CREATE TABLE IF NOT EXISTS _streams (" - " name TEXT PRIMARY KEY," - " config TEXT NOT NULL" - ")" - ) - self._conn.commit() + if self.config.read_only: + found = self._conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='_streams'" + ).fetchone() + if found is None: + raise ValueError("SQLite recording has no Memory2 stream registry") + else: + self._conn.execute( + "CREATE TABLE IF NOT EXISTS _streams (" + " name TEXT PRIMARY KEY," + " config TEXT NOT NULL" + ")" + ) + self._conn.commit() def get(self, name: str) -> dict[str, Any] | None: row = self._conn.execute("SELECT config FROM _streams WHERE name = ?", (name,)).fetchone() diff --git a/dimos/memory2/store/frozen.py b/dimos/memory2/store/frozen.py new file mode 100644 index 0000000000..27158a3b28 --- /dev/null +++ b/dimos/memory2/store/frozen.py @@ -0,0 +1,80 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Read-only Memory2 view bounded by one frozen timestamp.""" + +from __future__ import annotations + +from typing import Any, TypeVar, cast + +from dimos.core.resource import CompositeResource +from dimos.memory2.store.base import Store, StreamAccessor +from dimos.memory2.stream import Stream + +T = TypeVar("T") + + +class FrozenMemoryStore(CompositeResource): + """Overlay source and derived stores while hiding observations after a cutoff.""" + + def __init__( + self, + source: Store, + *, + through_timestamp: float, + derived: Store | None = None, + ) -> None: + super().__init__() + self.through_timestamp = through_timestamp + self._source = self.register_disposable(source) + self._derived = self.register_disposable(derived) if derived is not None else None + self._streams: dict[str, Stream[Any]] = {} + source_names = set(source.list_streams()) + derived_names = set(derived.list_streams()) if derived is not None else set() + collisions = source_names & derived_names + if collisions: + names = ", ".join(sorted(collisions)) + raise ValueError(f"Source and derived stores contain overlapping streams: {names}") + self._source_names = source_names + self._derived_names = derived_names + + @property + def streams(self) -> StreamAccessor[Stream[Any]]: + return StreamAccessor(self) + + def list_streams(self) -> list[str]: + return sorted(self._source_names | self._derived_names) + + def summary(self) -> str: + """Describe only observations visible through the frozen boundary.""" + return "\n".join(stream.summary() for _, stream in self.streams.items()) + + def stream(self, name: str, payload_type: type[T] | None = None) -> Stream[T]: + if payload_type is not None: + raise TypeError("Frozen memory streams cannot be created or retyped") + if name not in self.list_streams(): + raise KeyError(f"No stream {name!r}. Available: {self.list_streams()}") + if name not in self._streams: + store = self._source if name in self._source_names else self._derived + assert store is not None + self._streams[name] = store.stream(name).through(self.through_timestamp).as_read_only() + return cast("Stream[T]", self._streams[name]) + + def delete_stream(self, name: str) -> None: + raise PermissionError("Cannot delete streams from frozen memory") + + def stop(self) -> None: + for stream in self._streams.values(): + stream.stop() + super().stop() diff --git a/dimos/memory2/store/sqlite.py b/dimos/memory2/store/sqlite.py index 8afab3a714..2c5e3d592b 100644 --- a/dimos/memory2/store/sqlite.py +++ b/dimos/memory2/store/sqlite.py @@ -27,6 +27,7 @@ from dimos.memory2.observationstore.sqlite import SqliteObservationStore from dimos.memory2.registry import RegistryStore, deserialize_component, qual from dimos.memory2.store.base import Store, StoreConfig +from dimos.memory2.stream import Stream from dimos.memory2.utils.sqlite import open_disposable_sqlite_connection from dimos.memory2.utils.validation import validate_identifier from dimos.memory2.vectorstore.base import VectorStore @@ -41,6 +42,7 @@ class SqliteStoreConfig(StoreConfig): ] = "memory.db" page_size: int = 256 must_exist: bool = False + read_only: bool = False class SqliteStore(Store): @@ -54,16 +56,24 @@ def __init__(self, **kwargs: Any) -> None: raise FileNotFoundError( f"SQLite database not found: {os.path.abspath(self.config.path)}" ) - if not self.config.must_exist: + if self.config.read_only and not os.path.exists(self.config.path): + raise FileNotFoundError( + f"SQLite database not found: {os.path.abspath(self.config.path)}" + ) + if not self.config.must_exist and not self.config.read_only: parent = os.path.dirname(self.config.path) if parent: os.makedirs(parent, exist_ok=True) self._registry_conn = self._open_connection() - self._registry = RegistryStore(conn=self._registry_conn) + self._registry = RegistryStore( + conn=self._registry_conn, read_only=self.config.read_only + ) def _open_connection(self) -> sqlite3.Connection: """Open a new WAL-mode connection with sqlite-vec loaded.""" - disposable, connection = open_disposable_sqlite_connection(self.config.path) + disposable, connection = open_disposable_sqlite_connection( + self.config.path, read_only=self.config.read_only + ) self.register_disposable(disposable) return connection @@ -116,6 +126,7 @@ def _assemble_backend(self, name: str, stored: dict[str, Any]) -> Backend[Any]: codec=codec, blob_store_conn_match=blob_store_conn_match and eager_blobs, page_size=page_size, + read_only=self.config.read_only, ) backend: Backend[Any] = Backend( metadata_store=metadata_store, @@ -164,6 +175,9 @@ def _create_backend( ) return self._assemble_backend(name, stored) + if self.config.read_only: + raise KeyError(f"Stream {name!r} does not exist in read-only store") + # Create path: inject conn-shared defaults, then delegate to base if payload_type is None: raise TypeError(f"Stream {name!r} does not exist yet — payload_type is required") @@ -210,7 +224,15 @@ def list_streams(self) -> list[str]: db_names = set(self._registry.list_streams()) return sorted(db_names | set(self._streams.keys())) + def stream( + self, name: str, payload_type: type[Any] | None = None, **overrides: Any + ) -> Stream[Any]: + stream = super().stream(name, payload_type, **overrides) + return stream.as_read_only() if self.config.read_only else stream + def delete_stream(self, name: str) -> None: + if self.config.read_only: + raise PermissionError("Cannot delete streams from a read-only store") super().delete_stream(name) self._registry_conn.execute(f'DROP TABLE IF EXISTS "{name}"') self._registry_conn.execute(f'DROP TABLE IF EXISTS "{name}_blob"') diff --git a/dimos/memory2/store/test_frozen.py b/dimos/memory2/store/test_frozen.py new file mode 100644 index 0000000000..a4f3834a6b --- /dev/null +++ b/dimos/memory2/store/test_frozen.py @@ -0,0 +1,101 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from pathlib import Path +import sqlite3 + +import pytest + +from dimos.memory2.store.frozen import FrozenMemoryStore +from dimos.memory2.store.sqlite import SqliteStore + + +@pytest.fixture +def recorded_stores(tmp_path: Path): + source_path = tmp_path / "source.db" + derived_path = tmp_path / "derived.db" + with SqliteStore(path=str(source_path)) as source: + source.stream("camera", str).append("past", ts=10.0) + source.stream("camera", str).append("at-cutoff", ts=20.0) + source.stream("camera", str).append("future", ts=30.0) + with SqliteStore(path=str(derived_path)) as derived: + derived.stream("global_map", str).append("map-10", ts=10.0) + derived.stream("global_map", str).append("map-20", ts=20.0) + derived.stream("global_map", str).append("map-30", ts=30.0) + for path in (source_path, derived_path): + with sqlite3.connect(path) as connection: + connection.execute("PRAGMA wal_checkpoint(TRUNCATE)") + connection.execute("PRAGMA journal_mode=DELETE") + yield source_path, derived_path + + +def test_frozen_memory_merges_stores_and_includes_cutoff(recorded_stores) -> None: + source_path, derived_path = recorded_stores + with FrozenMemoryStore( + SqliteStore(path=str(source_path), must_exist=True, read_only=True), + derived=SqliteStore(path=str(derived_path), must_exist=True, read_only=True), + through_timestamp=20.0, + ) as memory: + assert memory.list_streams() == ["camera", "global_map"] + assert [obs.data for obs in memory.streams.camera] == ["past", "at-cutoff"] + assert memory.streams.global_map.last().data == "map-20" + + +def test_frozen_memory_rejects_mutation(recorded_stores) -> None: + source_path, derived_path = recorded_stores + with FrozenMemoryStore( + SqliteStore(path=str(source_path), must_exist=True, read_only=True), + derived=SqliteStore(path=str(derived_path), must_exist=True, read_only=True), + through_timestamp=20.0, + ) as memory: + with pytest.raises(PermissionError, match="read-only stream"): + memory.streams.camera.append("nope", ts=15.0) + with pytest.raises(PermissionError, match="frozen memory"): + memory.delete_stream("camera") + with pytest.raises(TypeError, match="cannot be created"): + memory.stream("new", str) + + +def test_read_only_sqlite_store_does_not_create_wal(recorded_stores) -> None: + source_path, _ = recorded_stores + wal_path = Path(f"{source_path}-wal") + shm_path = Path(f"{source_path}-shm") + original_bytes = source_path.read_bytes() + + with SqliteStore(path=str(source_path), must_exist=True, read_only=True) as source: + assert source.stream("camera").last().data == "future" + with pytest.raises(PermissionError, match="read-only stream"): + source.stream("camera").append("nope") + with pytest.raises(PermissionError, match="read-only store"): + source.delete_stream("camera") + with pytest.raises(KeyError, match="does not exist in read-only store"): + source.stream("new", str) + + assert source_path.read_bytes() == original_bytes + assert not wal_path.exists() + assert not shm_path.exists() + + +def test_frozen_memory_rejects_stream_collisions(recorded_stores) -> None: + source_path, _ = recorded_stores + source = SqliteStore(path=str(source_path), must_exist=True, read_only=True) + duplicate = SqliteStore(path=str(source_path), must_exist=True, read_only=True) + try: + with pytest.raises(ValueError, match="overlapping streams: camera"): + FrozenMemoryStore(source, derived=duplicate, through_timestamp=20.0) + finally: + source.stop() + duplicate.stop() diff --git a/dimos/memory2/stream.py b/dimos/memory2/stream.py index 1ed4651398..75d4dce477 100644 --- a/dimos/memory2/stream.py +++ b/dimos/memory2/stream.py @@ -38,6 +38,7 @@ PredicateFilter, StreamQuery, TagsFilter, + ThroughFilter, TimeRangeFilter, ) from dimos.memory2.type.observation import EmbeddedObservation, Observation @@ -127,6 +128,7 @@ def __init__( *, transform: Transformer[Any, T] | None = None, query: StreamQuery = StreamQuery(), + writable: bool = True, ) -> None: super().__init__() self._source = source @@ -134,6 +136,7 @@ def __init__( self.register_disposable(source) self._transform = transform self._query = query + self._writable = writable def stop(self) -> None: buf = self._query.live_buffer @@ -219,7 +222,21 @@ def _replace_query(self, **overrides: Any) -> Stream[T, O]: search_k=overrides.get("search_k", q.search_k), search_text=overrides.get("search_text", q.search_text), ) - return Stream(self._source, transform=self._transform, query=new_q) + return Stream( + self._source, + transform=self._transform, + query=new_q, + writable=self._writable, + ) + + def as_read_only(self) -> Stream[T, O]: + """Return a query-equivalent stream that rejects appends.""" + return Stream( + self._source, + transform=self._transform, + query=self._query, + writable=False, + ) def _with_filter(self, f: Filter) -> Stream[T, O]: return self._replace_query(filters=(*self._query.filters, f)) @@ -230,6 +247,10 @@ def after(self, t: float) -> Stream[T, O]: def before(self, t: float) -> Stream[T, O]: return self._with_filter(BeforeFilter(t)) + def through(self, t: float) -> Stream[T, O]: + """Keep observations at or before absolute timestamp ``t``.""" + return self._with_filter(ThroughFilter(t)) + def time_range(self, t1: float, t2: float) -> Stream[T, O]: return self._with_filter(TimeRangeFilter(t1, t2)) @@ -680,6 +701,8 @@ def append( Returns :class:`EmbeddedObservation` when *embedding* is provided, else a plain :class:`Observation`. """ + if not self._writable: + raise PermissionError("Cannot append to a read-only stream") if isinstance(self._source, Stream) or self._source is None: raise TypeError( "Cannot append to a transform/unbound stream. Append to the source stream." diff --git a/dimos/memory2/type/filter.py b/dimos/memory2/type/filter.py index 1250c4c31f..d64448b5db 100644 --- a/dimos/memory2/type/filter.py +++ b/dimos/memory2/type/filter.py @@ -57,6 +57,16 @@ def matches(self, obs: Observation[Any]) -> bool: return obs.ts < self.t +@dataclass(frozen=True) +class ThroughFilter(Filter): + """Include observations at or before an absolute timestamp.""" + + t: float + + def matches(self, obs: Observation[Any]) -> bool: + return obs.ts <= self.t + + @dataclass(frozen=True) class TimeRangeFilter(Filter): t1: float diff --git a/dimos/memory2/utils/sqlite.py b/dimos/memory2/utils/sqlite.py index 02a48f22b7..27475d4dfb 100644 --- a/dimos/memory2/utils/sqlite.py +++ b/dimos/memory2/utils/sqlite.py @@ -20,13 +20,18 @@ from reactivex.disposable import Disposable -def open_sqlite_connection(path: str | Path) -> sqlite3.Connection: +def open_sqlite_connection(path: str | Path, *, read_only: bool = False) -> sqlite3.Connection: """Open a WAL-mode SQLite connection with sqlite-vec loaded.""" import sqlite_vec - conn = sqlite3.connect(path, check_same_thread=False) - conn.execute("PRAGMA journal_mode=WAL") - conn.execute("PRAGMA synchronous=NORMAL") + if read_only: + uri = Path(path).resolve().as_uri() + "?mode=ro" + conn = sqlite3.connect(uri, uri=True, check_same_thread=False) + conn.execute("PRAGMA query_only=ON") + else: + conn = sqlite3.connect(path, check_same_thread=False) + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") conn.enable_load_extension(True) sqlite_vec.load(conn) conn.enable_load_extension(False) @@ -35,10 +40,12 @@ def open_sqlite_connection(path: str | Path) -> sqlite3.Connection: def open_disposable_sqlite_connection( path: str | Path, + *, + read_only: bool = False, ) -> tuple[Disposable, sqlite3.Connection]: """Open a WAL-mode SQLite connection and return (disposable, connection). The disposable closes the connection when disposed. """ - conn = open_sqlite_connection(path) + conn = open_sqlite_connection(path, read_only=read_only) return Disposable(lambda: conn.close()), conn diff --git a/docs/capabilities/agents/evaluation.md b/docs/capabilities/agents/evaluation.md new file mode 100644 index 0000000000..ee9b61c3bd --- /dev/null +++ b/docs/capabilities/agents/evaluation.md @@ -0,0 +1,94 @@ +--- +title: "Frozen recording evaluation" +--- + +`dimos eval run` asks one integer question about one immutable Memory2 recording. It +prepares a read-only map, gives a fresh Pi agent one `python_exec` tool, validates +the terminal answer against a private oracle, and writes durable attempt evidence. +It does not start a robot, simulation, replay blueprint, or live DimOS module. + +## Setup + +Install the Python agent dependencies and build the dedicated Node adapter from a +source checkout: + +```bash +uv sync --extra agents +npm ci --prefix packages/pi-code-policy-adapter +npm run build --prefix packages/pi-code-policy-adapter +``` + +The adapter requires Node 22.19.0 or newer. The Python command reports a preflight +error if `packages/pi-code-policy-adapter/dist/code-policy-main.js` is absent. + +## Run one case + +Run the Hong Kong office plumbing fixture with: + +```bash +uv run dimos eval run \ + dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/case.json \ + --output=/tmp/dimos-eval-smoke +``` + +The fixture's expected count is the synthetic sentinel `0`, not a reviewed room +count. A completed attempt may therefore report a semantic failure even when the +agent, map, and evidence pipeline work correctly. + +`eval run` accepts these options: + +| Option | Default | Purpose | +| --- | --- | --- | +| `--agent.backend` | `pi` | Select the pinned agent backend. | +| `--agent.model` | `gpt-5.6-luna` | Select the pinned model. | +| `--agent.thinking-level` | `medium` | Select the pinned thinking level. | +| `--agent.auth.mode` | inferred | Use `codex-oauth` or `openai-api-key`. | +| `--agent.auth.path` | `~/.pi/agent/auth.json` | Select a Codex OAuth file. | +| `--agent.auth.env` | `OPENAI_API_KEY` | Name the API-key environment variable. | +| `--output` | DimOS state directory | Set the append-only attempt root. | +| `--json` | off | Print one compact JSON result to stdout. | +| `--quiet` | off | Suppress live progress on stderr. | + +Do not pass credential values as command-line arguments. With no explicit mode, +an auth path selects OAuth, an auth environment name selects API-key auth, and a +set `OPENAI_API_KEY` selects API-key auth. Otherwise, the command uses Codex OAuth. +An explicit `--agent.auth.mode` takes precedence over environment inference. + +## Output and exit status + +The final human or JSON result goes to stdout. Live case, agent, and tool progress +goes to stderr, so scripts can parse `--json` output safely. `--quiet` suppresses +the progress stream without suppressing the final result. + +The command uses three exit codes: + +| Code | Meaning | +| --- | --- | +| `0` | The attempt completed. Its private score may be passed or failed. | +| `1` | The attempt started but infrastructure failed. | +| `2` | Preflight failed before an attempt was reserved. | + +Preflight verifies the case, private oracle digest, recording bundle, credential +binding, and built Node entrypoint. Once an attempt starts, its mode-`0700` +directory contains lifecycle events, public and private artifacts, content +descriptors, CodePolicy receipts, broker calls, and Pi evidence. Files are created +exclusively; reruns create new attempts instead of overwriting old evidence. + +## Privacy and trust boundary + +The agent receives the case's public projection: recording identity, cutoff, +question, and interaction contract. It does not receive the validator path, +oracle content, or credential value. Private oracle and score files remain private +attempt artifacts. Public prompts, progress, compact results, broker logs, Pi +evidence, and serialized runtime configuration contain no oracle or credential +material. + +CodePolicy is trusted, persistent, and unsandboxed Python execution. Its supplied +`memory` API is read-only and cutoff-limited, but Python code can still access the +host filesystem and processes. Run only trusted evaluation agents and code. Use an +OS sandbox or container for hostile code. + +Each attempt creates fresh Pi and CodePolicy processes. Normal completion, +failure, timeout, and interruption close those processes and release the output +lock. If a smoke run is interrupted externally, confirm no `code-policy` or +`pi-code-policy-adapter` child remains before retrying. diff --git a/docs/capabilities/agents/index.md b/docs/capabilities/agents/index.md index b168769505..4f5a023b43 100644 --- a/docs/capabilities/agents/index.md +++ b/docs/capabilities/agents/index.md @@ -3,6 +3,8 @@ title: "Agents" --- LLM agents run as native DimOS modules. They subscribe to camera, LiDAR, odometry, and spatial memory streams and they control the robot through skills. +For offline agent QA over immutable recordings, see [Frozen recording evaluation](/docs/capabilities/agents/evaluation.md). + ## Architecture ``` diff --git a/docs/development/testing.md b/docs/development/testing.md index 0de17adc77..9584721758 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -69,6 +69,48 @@ When writing or debugging a specific self-hosted test, override `-m` yourself to pytest -m self_hosted dimos/path/to/test_something.py ``` +### Frozen agent evaluation + +Install the optional Python dependencies, then build and test the dedicated Node +adapter: + +```bash +uv sync --extra agents +npm ci --prefix packages/pi-code-policy-adapter +npm run typecheck --prefix packages/pi-code-policy-adapter +npm run build --prefix packages/pi-code-policy-adapter +npm test --prefix packages/pi-code-policy-adapter +``` + +Run the focused Python suite with: + +```bash +uv run --extra agents pytest \ + dimos/memory2/store/test_frozen.py \ + dimos/agents/test_code_policy_core.py \ + dimos/agents/test_code_policy_server.py \ + dimos/agents/mcp/test_mcp_adapter.py \ + dimos/benchmark/agent_eval \ + dimos/benchmark/short_horizon_qa \ + dimos/cli/test_eval.py +``` + +The self-hosted Hong Kong mechanics gate requires the Git LFS +`go2_hongkong_office` recording. Run its marked test with the mapper on CPU: + +```bash +DIMOS_MAPPER_DEVICE=CPU:0 uv run --extra agents pytest \ + -m self_hosted \ + dimos/benchmark/short_horizon_qa/test_hongkong_eval.py +``` + +After the mechanics gate, follow the credentialed command in +[Frozen recording evaluation](/docs/capabilities/agents/evaluation.md). A failed +private score is expected against the synthetic `0` fixture oracle. Treat the +smoke as operationally successful only when the attempt completes, evidence +descriptors validate, no Pi or CodePolicy child remains, and a second run can +acquire the same output-root lock immediately. + ## Testing on a fresh Ubuntu install CI tests dimos with pre-built images and cached deps, so it can't catch gaps diff --git a/openspec/changes/extract-frozen-qa-eval/.openspec.yaml b/openspec/changes/extract-frozen-qa-eval/.openspec.yaml new file mode 100644 index 0000000000..46bad6b578 --- /dev/null +++ b/openspec/changes/extract-frozen-qa-eval/.openspec.yaml @@ -0,0 +1,2 @@ +schema: dimos-capability +created: 2026-08-05 diff --git a/openspec/changes/extract-frozen-qa-eval/README.md b/openspec/changes/extract-frozen-qa-eval/README.md new file mode 100644 index 0000000000..44a639aec6 --- /dev/null +++ b/openspec/changes/extract-frozen-qa-eval/README.md @@ -0,0 +1,3 @@ +# extract-frozen-qa-eval + +Extract the frozen short-horizon QA evaluation path from cc/frontier onto main as a focused, dependency-safe dimos eval run capability. diff --git a/openspec/changes/extract-frozen-qa-eval/design.md b/openspec/changes/extract-frozen-qa-eval/design.md new file mode 100644 index 0000000000..4df8f48c85 --- /dev/null +++ b/openspec/changes/extract-frozen-qa-eval/design.md @@ -0,0 +1,112 @@ +## Context + +The complete frozen QA path exists at reference commit `30e5f1c0e` on `cc/frontier`, primarily in commits `7cbf13845` and `10ca9bf15`. Those commits depend on earlier agent-evaluation and Pi adapter history and also contain accidental imports from live DimSim and spatial benchmark packages. The feature must therefore be ported file-by-file onto the recorded `origin/main` base rather than cherry-picked. + +The target is one source-checkout command that evaluates one frozen Memory2 recording synchronously. The implementation spans SQLite access, derived map preparation, trusted CodePolicy execution, a Node/Pi subprocess, private validation, evidence storage, and Typer registration. The base DimOS CLI must remain importable without the `agents` extra. + +## Goals / Non-Goals + +**Goals:** + +- Preserve the reference frozen-QA behavior behind `dimos eval run` with a focused dependency graph. +- Keep semantic cases immutable and independent of credentials, paths, ports, and output locations. +- Enforce private-oracle isolation and read-only, inclusive frozen-memory access. +- Run fresh CodePolicy and Pi processes per attempt with exactly `python_exec` exposed. +- Retain durable, non-overwriting evidence and always release processes and locks. +- Resolve all ported code against current `main` APIs and optional-dependency conventions. + +**Non-Goals:** + +- Dataset batching, retries, containers, scheduling, or distributed execution. +- Live DimSim, simulation episodes, replay blueprints, robot control, or hardware evaluation. +- Legacy smoke-runner configuration, backend abstraction for unrelated evaluators, or agentic blueprint integration. +- Treating read-only SQLite or trusted CodePolicy as a security sandbox. +- Shipping the Node adapter inside a Python wheel in this initial source-checkout slice. + +## DimOS Architecture + +The runtime flow is: + +```text +case.json + private oracle + | + v +single-case preflight -----> frozen bundle cache + | | + v v +AttemptStore lock -------> FrozenMemoryStore + | | + v v +Standalone CodePolicy MCP (`memory`, no `app`) + ^ + | python_exec MCP calls + v +dedicated Node/Pi process (exactly one tool) + | + v +prediction -> private validator -> terminal evidence -> cleanup/unlock +``` + +The focused Python package is split by dependency direction: + +- `agent_eval/base.py`: strict common Pydantic configuration. +- `agent_eval/json.py`: local UTF-8 canonical JSON using sorted keys and compact separators. +- `agent_eval/artifacts.py`: artifact references, typed IDs, and lifecycle records without benchmark-specific imports. +- `agent_eval/auth.py`: the runtime credential transport record without DimSim imports. +- `agent_eval/case.py`: only the supported frozen source, integer task, frozen interaction, exact validator, request, prediction, score, and outcome contracts. +- `agent_eval/interfaces.py`, `engine.py`, and `store.py`: adapter Protocols, lifecycle orchestration, durable evidence, and locking; no DimSim or spatial types. +- `agent_eval/pi*.py`: MCP binding, evidence log, Node subprocess, and progress transport. +- `agent_eval/single_case.py`: preflight, authentication resolution, adapter discovery, bundle preparation, and compact result projection. +- `short_horizon_qa/*`: frozen preparation, service binding, parser, validator, and interaction drivers. +- `agents/code_policy_core.py` and `code_policy_server.py`: persistent trusted Python execution and loopback-only standalone MCP hosting. +- `memory2/store/frozen.py` plus narrow existing Memory2 changes: read-only overlay and inclusive cutoff. + +The Node package `packages/pi-code-policy-adapter` contains only its line protocol, `python_exec` definition, pinned Pi session/auth setup, evidence retention, and tests. It does not depend on spatial tool definitions or adapter code. Python resolves its built `dist/code-policy-main.js` from the source checkout and reports an actionable preflight error when it is absent. + +No DimOS `Spec` Protocol, module stream, blueprint, or generated registry is added. Internal Python adapter Protocols define the source, interaction, validator, evidence, and Pi session seams. The only MCP-visible surface is the standalone `python_exec` tool created for the attempt. + +## Decisions + +1. **Port current file content, not commits.** Cherry-picking would import the full benchmark graph and unrelated lockfile changes. Each focused file is copied from `30e5f1c0e`, pruned, and reconciled with current `main`. + +2. **Use three small foundation modules instead of the reference generic `models.py` and `config.py`.** Canonical JSON, artifact records, and runtime credentials form a dependency floor that does not import DimSim or spatial packages. `case.py` omits live discriminated variants so unsupported inputs fail during schema validation. + +3. **Reserve attempts only after preflight.** Case/oracle validation, source preparation, credential resolution, and adapter discovery happen before attempt reservation and map to exit `2`. Once the store is reserved, normalized infrastructure failures map to exit `1`; semantic results map to exit `0`. + +4. **Make lock release structurally unconditional.** The attempt engine owns the store in an outer `try/finally`. Resource cleanup and terminal artifact publication may affect the outcome but cannot bypass store closure. Fault-injection tests cover fsync, manifest, event, and terminal-publication failures. + +5. **Treat private data as an information-flow boundary.** The public projection contains no validator. Oracle bytes are loaded only by the validator, and tests scan prompts, progress, compact results, Pi evidence, broker logs, and CodePolicy state for private material. + +6. **Implement real SQLite read-only mode.** Source and derived stores use SQLite `mode=ro` and `query_only`; writable connections alone configure WAL. Mutation methods and streams reject writes. A `ThroughFilter` implements `ts <= cutoff` and is applied to every overlaid stream. + +7. **Extract a dedicated one-tool Node package.** The session accepts a supplied `python_exec` definition, disables built-ins, and verifies the exact inventory. Node `>=22.19.0`, `@earendil-works/pi-ai` `0.80.10`, and `@earendil-works/pi-coding-agent` `0.80.10` remain pinned until upgraded deliberately. + +8. **Keep optional imports off the base CLI path.** `dimos.cli.eval` is a dependency-light Typer shell or imports the heavy implementation only inside the `run` callback. Jupyter, FastAPI, and Uvicorn are added to the `agents` extra, not base dependencies. + +9. **Use source-checkout adapter discovery initially.** Users build the dedicated package with npm before evaluation. Installed-wheel adapter distribution is deferred to a separately scoped packaging change. + +## Safety / Simulation / Replay + +This change never commands live hardware and does not start a robot, simulation, or replay blueprint. It reads a sealed recording and creates a derived map cache. The self-hosted mechanics gate uses `CPU:0`; normal first-time preparation may retain the reference mapper defaults and require CUDA. + +CodePolicy executes trusted Python persistently and without an OS sandbox. Read-only Memory2 prevents mutation through provided APIs but cannot prevent arbitrary filesystem or process access by hostile code. Documentation and runtime receipts must not imply stronger isolation. Untrusted execution requires a future container or OS sandbox. + +## Risks / Trade-offs + +- **Finalization can fail after useful work:** unconditional `finally` cleanup and prefix evidence reduce lock/process leakage; the terminal file may still be absent when storage itself fails. +- **Private data can leak through a new evidence path:** maintain distinct public/private models and add sentinel scans over every agent-visible channel. +- **Optional dependencies can leak into basic commands:** use callback-local imports and test in a subprocess that blocks agents-only modules. +- **Node/Python protocol drift:** share a version field, validate all frames, bound frames/stderr, correlate IDs, and run both sides' protocol tests. +- **Source-only adapter discovery limits installed users:** document the prerequisite and fail clearly; do not silently search ambiguous global locations. +- **Long or stuck agent turns complicate cleanup:** propagate abort, use bounded process waits, escalate terminate to kill, and record cleanup failures. +- **The smoke oracle can be mistaken for benchmark truth:** preserve warnings in the fixture, docs, and tests and describe semantic disagreement as expected plumbing behavior. + +## Migration / Rollout + +Create a fresh feature branch from `origin/main` SHA `e8a985d83a85c9827fa89ed7526e40a822eb1ae3`. Land in dependency order: Memory2 read-only support, CodePolicy runtime, generic evaluation foundation, Node adapter, frozen drivers, CLI, fixture, and docs. Update `pyproject.toml` narrowly and regenerate `uv.lock`; do not carry unrelated reference changes. + +Run Python and Node unit suites first, then the self-hosted Hong Kong mechanics gate, and finally the exact credentialed CLI smoke. No blueprint registry generation is required because no blueprint or module registry input changes. Rollback consists of removing the additive CLI registration and focused new packages; existing Memory2 read-only parameters remain backward-compatible defaults. + +## Open Questions + +None blocking this change. Distribution of the built Node adapter in Python wheels is explicitly deferred; this slice supports a source checkout with a documented adapter build step. diff --git a/openspec/changes/extract-frozen-qa-eval/docs.md b/openspec/changes/extract-frozen-qa-eval/docs.md new file mode 100644 index 0000000000..fec87b4899 --- /dev/null +++ b/openspec/changes/extract-frozen-qa-eval/docs.md @@ -0,0 +1,42 @@ +## User-Facing Docs + +- Add `docs/capabilities/agents/evaluation.md` covering: + - the exact `dimos eval run CASE` command and typed options; + - installation of the `agents` extra and the dedicated Node adapter build step; + - OAuth and API-key environment selection without secret CLI values; + - stdout/stderr behavior, `--json`, `--quiet`, and exit codes; + - attempt artifact locations, privacy boundaries, and cleanup expectations; + - the trusted, persistent, unsandboxed nature of CodePolicy; + - the difference between a completed semantic failure and infrastructure failure. +- Link the new guide from the existing agent capability index under `docs/capabilities/agents/`. +- Preserve the fixture-local `dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/README.md`, prominently stating that oracle `0` is a synthetic plumbing sentinel rather than benchmark truth. + +## Contributor Docs + +- Add a focused section to `docs/development/testing.md` for: + - building and testing `packages/pi-code-policy-adapter`; + - running the focused Python evaluation suite; + - running the `self_hosted` Hong Kong mechanics gate with its data prerequisite; + - performing the credentialed operational smoke and checking child-process and lock cleanup. +- No general module, blueprint, configuration, or hardware contributor documentation changes are required. + +## Coding-Agent Docs + +- Update `docs/coding-agents/index.md` only if it maintains a list of feature-specific validation surfaces; otherwise no coding-agent documentation change is needed. +- Do not modify `AGENTS.md`: the existing rules for optional dependencies, testing, imports, generated blueprints, and security are sufficient for this implementation. +- Retain `frozen-qa-main-extraction-handoff.md` as implementation context if the team wants the branch provenance in-repository; the OpenSpec artifacts become the normative implementation plan. + +## Doc Validation + +Run the repository-supported documentation checks after inspecting `docs/development/writing_docs.md` for the exact invocation: + +```bash +uv run doclinks +uv run md-babel-py run docs/capabilities/agents/evaluation.md +``` + +If the new page contains no executable Markdown blocks, record that `md-babel-py` has nothing to execute rather than adding artificial examples. No diagram generation is planned. + +## No Docs Needed + +Documentation is required because this change adds a public CLI, optional installation steps, credential handling, non-obvious exit semantics, and a trusted-unsandboxed execution boundary. diff --git a/openspec/changes/extract-frozen-qa-eval/proposal.md b/openspec/changes/extract-frozen-qa-eval/proposal.md new file mode 100644 index 0000000000..d45f040839 --- /dev/null +++ b/openspec/changes/extract-frozen-qa-eval/proposal.md @@ -0,0 +1,42 @@ +## Why + +DimOS has a working frozen short-horizon QA evaluation path on the long-running `cc/frontier` branch, but the implementation is entangled with unrelated DimSim, spatial benchmark, manipulation, runtime, and UI work. That makes the final feature commits unsafe to cherry-pick and prevents the focused `dimos eval run` workflow from landing on `main` with a reviewable dependency boundary. + +This change extracts the proven frozen-recording path as a single-case, synchronous evaluation capability. It preserves private-oracle isolation, immutable evidence, read-only Memory2 access, fresh agent sessions, and deterministic scoring while explicitly excluding live robot evaluation and the broader benchmark stack. + +## What Changes + +- Add the public `dimos eval run CASE` CLI for one immutable frozen-memory evaluation case. +- Add strict frozen source, integer-question, one-attempt interaction, and exact-integer validator contracts with deterministic case fingerprints. +- Add read-only Memory2 snapshots that combine source and derived streams through an inclusive timestamp cutoff. +- Add a standalone loopback CodePolicy MCP process and a dedicated Node/Pi adapter exposing exactly one tool, `python_exec`. +- Add append-only attempt storage, private/public evidence separation, progress streaming, exact terminal-answer parsing, and explicit exit semantics. +- Add the Hong Kong office plumbing fixture with a clearly non-authoritative synthetic oracle. +- Add only the optional Python and Node dependencies required by this focused path. +- No existing public API is removed or changed; this is an additive CLI capability. + +## Affected DimOS Surfaces + +- Modules/streams: Memory2 SQLite stores, stream filters, frozen source/derived overlays, standalone CodePolicy runtime, and generic agent-evaluation orchestration. +- Blueprints/CLI: new `dimos eval run` command; no blueprint composition or generated blueprint registry changes. +- Skills/MCP: loopback MCP exposure of one trusted `python_exec` tool; no robot `@skill` additions. +- Hardware/simulation/replay: consumes an existing recording and derived map only; no hardware control, live DimSim evaluation, simulation scheduling, or replay blueprint changes. +- Docs/generated registries: new agent-evaluation capability documentation and fixture warning; no `all_blueprints.py` regeneration. + +## Capabilities + +### New Capabilities + +- `frozen-agent-evaluation`: Single-case CLI execution, immutable case contracts, scoring, evidence, progress, privacy, and exit behavior. +- `frozen-memory-views`: Read-only source/derived Memory2 views bounded by an inclusive authored cutoff. +- `standalone-code-policy-runtime`: Fresh loopback CodePolicy and Pi processes with credential-safe setup, a one-tool inventory, bounded protocol handling, and reliable cleanup. + +### Modified Capabilities + +None. The affected behavior is not currently represented by an OpenSpec capability spec on `main`. + +## Impact + +Users gain a reproducible command for evaluating one frozen QA case and inspecting immutable attempt artifacts. The base CLI remains usable without the `agents` extra; evaluation requires the agents dependencies plus a built, pinned Node adapter. Credentials remain runtime-only and are never accepted as CLI secret values or serialized into evidence. + +The primary compatibility risks are optional-dependency import leakage, SQLite mutation through an allegedly frozen view, subprocess cleanup failures, private-oracle disclosure, and source-checkout discovery of the Node entrypoint. Validation therefore includes focused Python and Node suites, minimal-dependency CLI subprocess tests, failure-injection and lock-release tests, a self-hosted recording mechanics gate, and one credentialed end-to-end smoke run. diff --git a/openspec/changes/extract-frozen-qa-eval/specs/frozen-agent-evaluation/spec.md b/openspec/changes/extract-frozen-qa-eval/specs/frozen-agent-evaluation/spec.md new file mode 100644 index 0000000000..d719ff9779 --- /dev/null +++ b/openspec/changes/extract-frozen-qa-eval/specs/frozen-agent-evaluation/spec.md @@ -0,0 +1,107 @@ +## ADDED Requirements + +### Requirement: Single-case evaluation CLI +DimOS SHALL provide `dimos eval run CASE` as a synchronous command for exactly one immutable evaluation case. The command SHALL accept only the `pi` backend, `gpt-5.6-luna` model, and `medium` thinking level, and SHALL reject unsupported values or semantic overrides. + +#### Scenario: Run a supported case +- **GIVEN** a valid frozen-memory case, available recording, built adapter, and valid credentials +- **WHEN** the user runs `dimos eval run CASE` +- **THEN** DimOS executes one attempt to a terminal infrastructure outcome +- **AND** prints the final result even when live progress is suppressed + +#### Scenario: Request an unsupported agent condition +- **GIVEN** a valid case +- **WHEN** the user supplies an unsupported backend, model, or thinking level +- **THEN** the command rejects the request before starting an attempt + +### Requirement: Credential-safe authentication selection +The CLI SHALL support Codex OAuth and an OpenAI API-key environment binding without accepting secret values directly. Explicit authentication mode SHALL win; otherwise a nonempty `OPENAI_API_KEY` SHALL select API-key authentication and Codex OAuth SHALL be the fallback. Credential contents MUST NOT appear in command arguments, progress, results, or retained artifacts. + +#### Scenario: Infer API-key authentication +- **GIVEN** no explicit authentication mode and a nonempty `OPENAI_API_KEY` +- **WHEN** the command resolves runtime authentication +- **THEN** it selects API-key authentication using the environment binding +- **AND** does not serialize the key value + +#### Scenario: Resolve OAuth authentication +- **GIVEN** no API key and no explicit authentication mode +- **WHEN** the command resolves runtime authentication +- **THEN** it selects Codex OAuth using `--agent.auth.path`, `PI_SPATIAL_AUTH_PATH`, or `~/.pi/agent/auth.json` in precedence order +- **AND** fails preflight if the selected credential file is unavailable + +### Requirement: Immutable and private case contract +An evaluation case SHALL consist of a frozen recording source, integer question, one-attempt frozen interaction, and exact-integer validator reference. Unknown fields, unsafe validator paths, non-finite progress, fingerprint mismatches, and runtime-specific fields in the semantic case SHALL be rejected. Runtime paths, credentials, ports, and output locations SHALL remain outside the case fingerprint. + +#### Scenario: Validate a case before agent dispatch +- **GIVEN** a case containing a safe case-relative oracle path and expected SHA-256 +- **WHEN** the command performs preflight +- **THEN** it verifies the case fingerprint and exact oracle bytes before starting the agent +- **AND** rejects an escaped path or digest mismatch + +#### Scenario: Produce an agent-safe projection +- **GIVEN** a validated private case +- **WHEN** DimOS creates the public case projection and prompt +- **THEN** the projection omits the validator reference and all oracle content +- **AND** the private expected answer is not exposed to the agent runtime + +### Requirement: Exact terminal integer scoring +The response parser SHALL succeed only when final text contains exactly one `ANSWER:` marker and ends with `ANSWER: `. A malformed answer or validator mismatch SHALL be a completed semantic failure, not an infrastructure failure. + +#### Scenario: Parse a valid terminal answer +- **GIVEN** final agent text containing exactly one terminal `ANSWER: -3` +- **WHEN** DimOS parses and validates the response +- **THEN** it records integer prediction `-3` +- **AND** compares it privately with the exact-integer oracle + +#### Scenario: Reject malformed answer text +- **GIVEN** final text with multiple markers, a non-integer marker, or trailing content after the answer +- **WHEN** DimOS parses the response +- **THEN** it records an invalid prediction and a completed failed task +- **AND** returns process exit code `0` + +### Requirement: Immutable attempt evidence +Each started attempt SHALL reserve a fresh mode-`0700` `attempt_` directory beneath the output root while holding a nonblocking output-root lock. Artifacts SHALL use safe attempt-relative paths, exclusive creation, SHA-256 descriptors, fsync, and atomic terminal publication. Concurrent attempts targeting the same output root SHALL not interleave. + +#### Scenario: Retain a completed attempt +- **GIVEN** an attempt reaches scoring and cleanup succeeds +- **WHEN** DimOS publishes the terminal outcome +- **THEN** the attempt contains private and public case projections, source evidence, MCP and session evidence, tool-call and execution records, Pi evidence, prediction, private score, ordered lifecycle events, manifest, and terminal outcome +- **AND** existing artifacts are never overwritten + +#### Scenario: Reject a concurrent attempt +- **GIVEN** one attempt holds the lock for an output root +- **WHEN** another attempt targets the same root +- **THEN** the second attempt fails cleanly without creating an interleaved attempt + +### Requirement: Failure classification and lock release +Failures before attempt reservation SHALL exit `2`. Infrastructure, finalization, or cleanup failures normalized into a reserved attempt SHALL produce a failed attempt and exit `1`. Completed semantic pass or failure SHALL exit `0`. The output lock MUST be released after success, failure, interruption, partial startup, and artifact-publication failure. + +#### Scenario: Preflight failure +- **GIVEN** an invalid case, unavailable oracle, missing credentials, missing adapter, or source-preparation error before reservation +- **WHEN** the error escapes preflight +- **THEN** the command exits `2` +- **AND** no attempt is reported as completed + +#### Scenario: Attempt cleanup failure +- **GIVEN** an otherwise completed attempt whose process or resource cleanup fails +- **WHEN** DimOS finalizes the attempt +- **THEN** it reports a failed infrastructure attempt and exits `1` +- **AND** releases the output lock even if terminal artifact publication also fails + +### Requirement: Machine-readable output and private progress +With `--json`, stdout SHALL contain exactly one compact result while progress and tool-call rendering remain on stderr. `--quiet` SHALL suppress progress but not the final result. Neither channel SHALL expose private oracle material or credentials. + +#### Scenario: Consume compact JSON +- **GIVEN** a valid invocation using `--json` +- **WHEN** the attempt runs with progress enabled +- **THEN** stdout remains parseable as exactly one JSON result +- **AND** progress appears only on stderr + +### Requirement: Synthetic plumbing fixture +The shipped Hong Kong office room-count smoke case SHALL preserve its `case.json`, private oracle, and warning README. The expected value `0` MUST be described as a synthetic plumbing sentinel and MUST NOT be presented as the authoritative room count. + +#### Scenario: Agent disagrees with the sentinel +- **GIVEN** the fixture and an agent response other than `ANSWER: 0` +- **WHEN** infrastructure and scoring complete +- **THEN** the task may report semantic failure with exit code `0` +- **AND** documentation does not characterize that outcome as a mapping or agent regression diff --git a/openspec/changes/extract-frozen-qa-eval/specs/frozen-memory-views/spec.md b/openspec/changes/extract-frozen-qa-eval/specs/frozen-memory-views/spec.md new file mode 100644 index 0000000000..f9ac1f47b6 --- /dev/null +++ b/openspec/changes/extract-frozen-qa-eval/specs/frozen-memory-views/spec.md @@ -0,0 +1,53 @@ +## ADDED Requirements + +### Requirement: True read-only SQLite access +Frozen Memory2 source and derived databases SHALL be opened using SQLite read-only mode with query-only enforcement. All public mutation paths SHALL reject writes, and opening or reading a frozen view SHALL not create WAL or other database mutation sidecars. + +#### Scenario: Read a frozen recording +- **GIVEN** existing source and derived SQLite stores +- **WHEN** DimOS opens them as a frozen view and reads observations +- **THEN** the observations are available without modifying either database +- **AND** no WAL file is created by the frozen access + +#### Scenario: Attempt a mutation +- **GIVEN** an open frozen view or stream +- **WHEN** a caller appends an observation, deletes a stream, creates a stream, or invokes another mutation path +- **THEN** the operation fails with a read-only error +- **AND** source and derived bytes remain unchanged + +### Requirement: Inclusive authored cutoff +Every stream exposed through a frozen Memory2 view SHALL include observations whose timestamps are less than or equal to the authored cutoff and SHALL hide all observations after it. + +#### Scenario: Observe the exact cutoff boundary +- **GIVEN** observations immediately before, exactly at, and immediately after a cutoff +- **WHEN** the stream is queried through the frozen view +- **THEN** observations before and exactly at the cutoff are visible +- **AND** the observation after the cutoff is absent + +### Requirement: Source and derived overlay +A frozen view SHALL expose the union of source and derived stream names under the same cutoff. It SHALL reject ambiguous overlays in which both stores contain the same stream name and SHALL not allow callers to create or retype streams through the overlay. + +#### Scenario: Access a derived map with source observations +- **GIVEN** a source recording and a derived store containing a non-colliding `global_map` stream +- **WHEN** a caller lists and reads frozen streams +- **THEN** both source streams and `global_map` are available through one memory object +- **AND** the inclusive cutoff applies to all of them + +#### Scenario: Reject colliding streams +- **GIVEN** source and derived stores with the same stream name +- **WHEN** DimOS constructs the overlay +- **THEN** construction fails with the colliding names identified + +### Requirement: Deterministic frozen bundle preparation +DimOS SHALL resolve normalized progress over the sealed recording range, materialize or reuse a derived frozen bundle, and retain a manifest describing the selected cutoff and source/derived integrity. Progress `1.0` SHALL resolve to the recording end inclusively. + +#### Scenario: Prepare the final recording state +- **GIVEN** a named recording and normalized progress `1.0` +- **WHEN** DimOS prepares a frozen bundle +- **THEN** the selected cutoff equals the recording end +- **AND** the derived map and manifest describe only data available through that cutoff + +#### Scenario: Reject invalid progress +- **GIVEN** non-finite progress or a value outside `[0, 1]` +- **WHEN** bundle preparation validates the source +- **THEN** preparation fails before attempt execution diff --git a/openspec/changes/extract-frozen-qa-eval/specs/standalone-code-policy-runtime/spec.md b/openspec/changes/extract-frozen-qa-eval/specs/standalone-code-policy-runtime/spec.md new file mode 100644 index 0000000000..52be959c3a --- /dev/null +++ b/openspec/changes/extract-frozen-qa-eval/specs/standalone-code-policy-runtime/spec.md @@ -0,0 +1,73 @@ +## ADDED Requirements + +### Requirement: Fresh isolated runtime per attempt +Each attempt SHALL start one fresh standalone CodePolicy process on an ephemeral loopback port and one fresh Node/Pi process. The CodePolicy namespace SHALL preload frozen `memory`, SHALL NOT expose live DimOS `app`, and SHALL be disposed after the attempt. + +#### Scenario: Start a frozen policy session +- **GIVEN** a prepared frozen bundle +- **WHEN** an attempt starts its interaction +- **THEN** the policy can inspect the bounded `memory` view +- **AND** no live robot RPC object is present + +#### Scenario: Run consecutive attempts +- **GIVEN** two sequential evaluations +- **WHEN** each attempt starts +- **THEN** each receives distinct CodePolicy and Pi session identities and processes +- **AND** Python state does not leak between attempts + +### Requirement: Exactly one Pi tool +The Pi session SHALL disable built-in tools, extensions, skills, prompt templates, and context files, and SHALL expose exactly one custom tool named `python_exec`. The runtime SHALL fail closed if the activated inventory differs before or after session creation. + +#### Scenario: Validate tool inventory +- **GIVEN** a Pi session configured for frozen evaluation +- **WHEN** the session reports its active tools +- **THEN** the ordered inventory is exactly `["python_exec"]` +- **AND** any additional or missing tool causes infrastructure failure + +### Requirement: Pinned model and authentication runtime +The adapter SHALL run the pinned supported Pi libraries with model `gpt-5.6-luna` and medium thinking. It SHALL support Codex OAuth by credential-file path and OpenAI authentication by key supplied through the selected environment binding, without placing secret values in process arguments or evidence. + +#### Scenario: Launch with OAuth +- **GIVEN** a valid OAuth credential file +- **WHEN** the Pi process starts +- **THEN** it resolves the configured OAuth model runtime from the supplied path +- **AND** no credential bytes are emitted on the line protocol or retained in attempt evidence + +### Requirement: Bounded validated line protocol +Python and Node SHALL communicate through newline-delimited JSON with stdout reserved for protocol frames and diagnostics directed to bounded stderr evidence. Every inbound frame SHALL be size-bounded and schema-validated, and tool calls and replies SHALL be correlated by unique IDs. Unknown, duplicate, malformed, or oversized frames SHALL fail closed. + +#### Scenario: Broker a valid Python call +- **GIVEN** an idle Pi session and a valid `python_exec` request +- **WHEN** Node emits the correlated tool call and Python returns its result +- **THEN** the matching reply completes the pending call +- **AND** the call and bounded execution record are retained + +#### Scenario: Receive an invalid reply +- **GIVEN** no pending call with a supplied reply ID +- **WHEN** the adapter receives that reply +- **THEN** it reports a protocol error without applying the reply to another call + +### Requirement: Observable readiness and complete evidence +The standalone runtime SHALL retry both connection failures and read timeouts while waiting for MCP readiness. It SHALL retain the MCP inventory, CodePolicy session receipt and bounded execution records, broker call log, Pi prompt/session evidence, and bounded adapter stderr under the attempt directory. + +#### Scenario: Server becomes ready after transient failures +- **GIVEN** a starting loopback MCP process that initially refuses connections or times out reads +- **WHEN** readiness is polled within the configured deadline +- **THEN** polling continues until initialization succeeds or the deadline expires + +### Requirement: Reliable cancellation and cleanup +Abort, timeout, interrupt, normal completion, partial startup, and disposal SHALL terminate or kill remaining child processes within bounded cleanup periods. Cleanup failures SHALL be surfaced to the attempt result, and no child process SHALL remain after command termination. + +#### Scenario: Interrupt an active turn +- **GIVEN** an active Pi turn with an outstanding tool call +- **WHEN** the command is interrupted +- **THEN** pending broker calls are rejected, Pi is aborted and disposed, and CodePolicy is stopped +- **AND** the output lock is released + +### Requirement: Trusted unsandboxed execution disclosure +DimOS SHALL describe CodePolicy as trusted, persistent, unsandboxed Python. Read-only Memory2 SHALL be presented as protection against accidental API mutation, not as an operating-system security boundary. + +#### Scenario: Review the evaluation documentation +- **GIVEN** a user preparing to run a frozen evaluation +- **WHEN** they read the capability documentation +- **THEN** they are warned not to execute hostile policy code without an external container or OS sandbox diff --git a/openspec/changes/extract-frozen-qa-eval/tasks.md b/openspec/changes/extract-frozen-qa-eval/tasks.md new file mode 100644 index 0000000000..a5262376e1 --- /dev/null +++ b/openspec/changes/extract-frozen-qa-eval/tasks.md @@ -0,0 +1,75 @@ +## 1. Extraction Baseline and Dependencies + +- [x] 1.1 Ensure the implementation branch is based on `origin/main` SHA `e8a985d83a85c9827fa89ed7526e40a822eb1ae3`, record that base in the PR, and use `30e5f1c0e` only as the file-content reference. +- [x] 1.2 Add the focused `agent_eval` foundation modules for strict base models, canonical JSON, artifact/lifecycle records, and runtime credentials without importing live DimSim or spatial benchmark packages. +- [x] 1.3 Update `pyproject.toml` so the `agents` extra contains the Jupyter kernel/client, nbformat, pyzmq, FastAPI, and Uvicorn dependencies required by CodePolicy, then regenerate `uv.lock` without carrying unrelated reference changes. +- [x] 1.4 Add an import-boundary test that recursively checks the focused evaluation slice for forbidden live DimSim and spatial benchmark imports. + +## 2. Frozen Memory2 Views + +- [x] 2.1 Add a read-only option to Memory2 SQLite connection helpers using SQLite URI `mode=ro` and `PRAGMA query_only=ON`, while retaining WAL configuration only for writable connections. +- [x] 2.2 Propagate read-only mode through the Memory2 registry, observation store, SQLite store, and stream APIs, rejecting stream creation, append, deletion, and every other mutation path. +- [x] 2.3 Add inclusive through-time filtering (`observation.ts <= cutoff`) and ensure transformed read-only streams preserve the mutation boundary. +- [x] 2.4 Add the frozen source/derived overlay with deterministic stream listing, collision rejection, no stream creation/retyping, and the inclusive cutoff applied to every stream. +- [x] 2.5 Add Memory2 tests for exact-boundary visibility, source/derived union, collisions, mutation rejection, unchanged database bytes, and absence of WAL sidecars. + +## 3. Standalone CodePolicy Runtime + +- [x] 3.1 Port the module-independent trusted CodePolicy kernel runtime with lazy Jupyter imports and the actionable `uv sync --extra agents` error. +- [x] 3.2 Add frozen-memory environment setup that preloads read-only `memory`, omits live `app`, bounds execution output/records, and retains session receipts. +- [x] 3.3 Add the standalone FastAPI/Uvicorn MCP process on an ephemeral loopback port, including startup receipt, control endpoint, bounded shutdown, and terminate-to-kill escalation. +- [x] 3.4 Update MCP readiness polling to retry both connection failures and read timeouts until the configured deadline. +- [x] 3.5 Add CodePolicy and MCP tests for fresh sessions, namespace isolation, exactly one exposed tool, readiness retry, evidence retention, timeout/interruption behavior, partial startup, and child-process cleanup. + +## 4. Evaluation Contracts, Storage, and Engine + +- [x] 4.1 Add frozen-only source, integer-question, one-attempt interaction, exact-integer validator, request, prediction, private score, and terminal outcome contracts with strict unknown-field rejection and deterministic fingerprints. +- [x] 4.2 Add tests proving fingerprints exclude credentials, ports, output/host paths, reject semantic overrides and unsafe oracle paths, and produce validator-free public projections. +- [x] 4.3 Add the append-only attempt store with mode-`0700` attempt directories, a nonblocking output-root lock, safe relative paths, exclusive artifact creation, SHA-256 descriptors, fsync, monotonic lifecycle events, and atomic terminal publication. +- [x] 4.4 Add the generic source/interaction/validator/agent adapter Protocols and attempt engine with private/public evidence separation and resource cleanup in reverse dependency order. +- [x] 4.5 Structure attempt execution so store closure and lock release occur in an outer `finally`, regardless of event, manifest, fsync, terminal-publication, cleanup, interruption, or partial-startup failures. +- [x] 4.6 Add fault-injection tests for events, artifact fsync, directory fsync, manifest, terminal-link publication, cleanup, and interruption failures, asserting retained prefixes, correct status, no live children, and immediate lock reacquisition. + +## 5. Dedicated Node/Pi Adapter + +- [x] 5.1 Create `packages/pi-code-policy-adapter` with Node `>=22.19.0`, pinned Pi dependencies `0.80.10`, compatible TypeBox/TypeScript dependencies, build/typecheck/test configuration, lockfile, and a source-checkout README. +- [x] 5.2 Extract and simplify the newline-delimited code-policy protocol with bounded frames, strict inbound validation, unique call/reply correlation, protocol-only stdout, and bounded diagnostic stderr. +- [x] 5.3 Implement the dedicated `python_exec` definition and Pi session setup without spatial tools, disabling built-ins/extensions/skills/templates/context files and asserting the exact one-tool inventory before and after activation. +- [x] 5.4 Implement OAuth and API-key runtime setup, pinned model/thinking validation, fresh session creation, prompt/session evidence, abort/dispose propagation, and no secret process arguments or evidence fields. +- [x] 5.5 Add Node tests for entrypoint behavior, valid calls, malformed/unknown/duplicate/oversized frames, tool-inventory drift, authentication configuration, prompt/session evidence, abort, disposal, and stdout/stderr separation. +- [x] 5.6 Add the adapter's npm test command to the appropriate required CI workflow without adding generated `dist` or `node_modules` content. + +## 6. Frozen QA Preparation and Execution + +- [x] 6.1 Port normalized-progress validation, recording resolution, derived `global_map` preparation, cached bundle manifests, cutoff receipts, and source/derived integrity descriptors using current `main` mapping and Memory2 APIs. +- [x] 6.2 Add the frozen source driver, exact-integer oracle loader and SHA-256 verification, terminal `ANSWER: ` parser, private validator, and frozen CodePolicy interaction driver. +- [x] 6.3 Add the Python/Pi broker and process wrapper with bounded line frames, bounded progress/stderr, correlated tool replies, evidence references, startup/turn timeouts, abort, disposal, and terminate-to-kill cleanup. +- [x] 6.4 Add single-case preflight and execution orchestration with source-checkout discovery of `packages/pi-code-policy-adapter/dist/code-policy-main.js` and an actionable missing-build error. +- [x] 6.5 Add tests for bundle reuse and integrity, progress `0`/`1` boundaries, exact answer parsing, validator mismatch, malformed semantic failure, fresh process/session identities, cleanup, and complete attempt evidence. +- [x] 6.6 Add privacy tests that seed unique oracle and credential sentinels and prove they do not appear in the public projection, prompt, progress, compact result, CodePolicy namespace, Pi transcript/evidence, broker log, or serialized runtime configuration. + +## 7. CLI and Fixture + +- [x] 7.1 Add the dependency-light `dimos eval` Typer shell and callback-local heavy imports so ordinary base CLI commands do not require the `agents` extra. +- [x] 7.2 Implement the documented options, auth inference/precedence, default output root, compact/human results, stderr progress, `--quiet`, and exit codes `0`, `1`, and `2` at the preflight/attempt boundary. +- [x] 7.3 Add subprocess CLI tests for typed help, unsupported values, auth combinations, stdout/stderr separation, quiet mode, semantic failure, infrastructure failure, preflight failure, missing agents dependencies, and missing Node build. +- [x] 7.4 Port exactly the Hong Kong smoke fixture's `case.json`, private oracle, and warning README, preserving oracle SHA-256 and the explicit synthetic-`0` warning. +- [x] 7.5 Confirm no blueprint, module registry input, or `all_blueprints.py` output changes are introduced; no blueprint-regeneration task is required. + +## 8. Documentation + +- [x] 8.1 Add `docs/capabilities/agents/evaluation.md` with setup, adapter build, CLI options, authentication, output/exit behavior, evidence/privacy, cleanup, and trusted-unsandboxed warnings. +- [x] 8.2 Link the evaluation guide from the agent capability index and update `docs/development/testing.md` with Python, Node, self-hosted, and credentialed smoke procedures. +- [x] 8.3 Inspect `docs/coding-agents/index.md` and update it only if it enumerates feature-specific validation surfaces; leave `AGENTS.md` unchanged. +- [x] 8.4 Retain the fixture warning and decide in the PR whether `frozen-qa-main-extraction-handoff.md` remains as provenance or the OpenSpec artifacts supersede it. + +## 9. Verification and Manual QA + +- [x] 9.1 Run `openspec validate extract-frozen-qa-eval` and resolve all proposal/spec/design/docs/task validation errors. +- [x] 9.2 Run `npm ci --prefix packages/pi-code-policy-adapter`, its typecheck/build/tests, and verify no unexpected generated files are tracked. +- [x] 9.3 Run the focused Python CLI, agent-evaluation, frozen-QA, CodePolicy, MCP adapter, and frozen-Memory2 pytest targets listed in the handoff. +- [x] 9.4 Run the minimal-dependency base CLI subprocess regression, relevant lint/format checks, and `uv run mypy` for the added focused packages. +- [x] 9.5 Run `uv run doclinks` and the repository-supported executable-Markdown check for `docs/capabilities/agents/evaluation.md` when applicable. +- [x] 9.6 On a host with the LFS recording, run the `self_hosted` Hong Kong mechanics gate using `CPU:0` and verify its expected cutoff/map evidence. +- [ ] 9.7 Build the Node adapter and run the exact credentialed `uv run dimos eval run ... --output=/tmp/dimos-eval-smoke` command, accepting semantic failure against the synthetic oracle only when infrastructure and evidence complete. +- [x] 9.8 After the operational smoke, verify Pi and CodePolicy processes are gone, stdout/stderr obey the contract, credentials/oracle material are absent from public evidence, artifacts pass their descriptors, and the output lock can be immediately reacquired. diff --git a/packages/pi-code-policy-adapter/.gitignore b/packages/pi-code-policy-adapter/.gitignore new file mode 100644 index 0000000000..e9488b3e35 --- /dev/null +++ b/packages/pi-code-policy-adapter/.gitignore @@ -0,0 +1,6 @@ +# The repository-wide ignore rules treat package manifests as local tooling files. +!package.json +!package-lock.json +dist/ +dist-test/ +node_modules/ diff --git a/packages/pi-code-policy-adapter/README.md b/packages/pi-code-policy-adapter/README.md new file mode 100644 index 0000000000..4ac491748c --- /dev/null +++ b/packages/pi-code-policy-adapter/README.md @@ -0,0 +1,16 @@ +# Pi code-policy adapter + +This source-checkout package runs the pinned Pi session used by `dimos eval run`. +It disables Pi built-in tools and exposes exactly one host-brokered tool, +`python_exec`. + +```bash +npm ci --prefix packages/pi-code-policy-adapter +npm test --prefix packages/pi-code-policy-adapter +npm run build --prefix packages/pi-code-policy-adapter +``` + +The compiled entrypoint is `dist/code-policy-main.js`. Standard output is +reserved for newline-delimited protocol frames; diagnostics use standard error. +Credentials are supplied by the Python parent process through the supported +environment binding and must never be placed in command-line arguments. diff --git a/packages/pi-code-policy-adapter/package-lock.json b/packages/pi-code-policy-adapter/package-lock.json new file mode 100644 index 0000000000..cb5d4cd96a --- /dev/null +++ b/packages/pi-code-policy-adapter/package-lock.json @@ -0,0 +1,1825 @@ +{ + "name": "@dimos/pi-code-policy-adapter", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@dimos/pi-code-policy-adapter", + "version": "0.1.0", + "dependencies": { + "@earendil-works/pi-ai": "0.80.10", + "@earendil-works/pi-coding-agent": "0.80.10", + "typebox": "^1.3.6" + }, + "devDependencies": { + "@types/node": "^22.15.0", + "typescript": "^5.8.3" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.91.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.91.1.tgz", + "integrity": "sha512-LAmu761tSN9r66ixvmciswUj/ZC+1Q4iAfpedTfSVLeswRwnY3n2Nb6Tsk+cLPP28aLOPWeMgIuTuCcMC6W/iw==", + "license": "MIT", + "dependencies": { + "json-schema-to-ts": "^3.1.1" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-bedrock-runtime": { + "version": "3.1048.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-runtime/-/client-bedrock-runtime-3.1048.0.tgz", + "integrity": "sha512-u+NT61JZEkRFtpL0CAw1N1dwxnaLgwVXQl/zjJxTGgLyS/jTIdg2SdoEoCTHxgDyCnqa1HEi9QOoE9/pYRNpOQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/credential-provider-node": "^3.972.42", + "@aws-sdk/eventstream-handler-node": "^3.972.16", + "@aws-sdk/middleware-eventstream": "^3.972.12", + "@aws-sdk/middleware-websocket": "^3.972.19", + "@aws-sdk/token-providers": "3.1048.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/fetch-http-handler": "^5.4.2", + "@smithy/node-http-handler": "^4.7.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.975.3", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.975.3.tgz", + "integrity": "sha512-7ur3kCKuvPLqlsZ2XlvnNBVQ7KkpSu6Y6dOTwSPHLrFpTEfZM8isLBJc4cgv96WB7GifeVM436mpycwxBd2vEA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@aws-sdk/xml-builder": "^3.972.36", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.29.4", + "@smithy/signature-v4": "^5.6.5", + "@smithy/types": "^4.16.1", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.59", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.59.tgz", + "integrity": "sha512-Ny5e4Mfh3QPmiAc0AiUe+cbTXDlxkU3Rc+EpWOfyWeWEy6yp7Fa1KmfNeCc+1a8by9zQ9gtohmiQUkMPScF3ng==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.61", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.61.tgz", + "integrity": "sha512-8jAjgStl5Ytq4+HF3X/9f+EmRinaRbGRRtQGktlPfBRVx73H+R1y48vIeXerQtYGFaUqkEp3fT6jP854rVO2yQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/node-http-handler": "^4.9.6", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http/node_modules/@smithy/node-http-handler": { + "version": "4.9.7", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.7.tgz", + "integrity": "sha512-wCU8HCLjAtAVqxxe0j2xff9LcEPw3yjBbg5IdQDIYFnxnPxbxcSLc7rgex7kqm9L/WYOnJEgaWQlfDkZleozMA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.973.4", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.4.tgz", + "integrity": "sha512-e6ZvVsj90aRALf1kHP+J4iqC1496ZpVgqI/+u0LJ5HL7q7ATauGy4gdDvRCP13L1pN/fMiZLah162PGIYkbUVQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/credential-provider-env": "^3.972.59", + "@aws-sdk/credential-provider-http": "^3.972.61", + "@aws-sdk/credential-provider-login": "^3.972.66", + "@aws-sdk/credential-provider-process": "^3.972.59", + "@aws-sdk/credential-provider-sso": "^3.973.3", + "@aws-sdk/credential-provider-web-identity": "^3.972.65", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/credential-provider-imds": "^4.4.9", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.66", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.66.tgz", + "integrity": "sha512-g2fsqm87r/nKthLZ0VkkDBElkGg0PvSa8d97HQ6EilMbJTZ6hxa8FxkSZyJfgPfFdZn0TTmkOffQmTSUcAHIng==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.70", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.70.tgz", + "integrity": "sha512-3xzvkGdykBunxqh8WudmUpSyLWvIhfI6aBQo1b5rb3mDO5mNLadK+0hiI0qBQBMVynJbfLO+Ajy9dztMwy9O8w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.59", + "@aws-sdk/credential-provider-http": "^3.972.61", + "@aws-sdk/credential-provider-ini": "^3.973.4", + "@aws-sdk/credential-provider-process": "^3.972.59", + "@aws-sdk/credential-provider-sso": "^3.973.3", + "@aws-sdk/credential-provider-web-identity": "^3.972.65", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/credential-provider-imds": "^4.4.9", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.59", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.59.tgz", + "integrity": "sha512-DlZF2/MhLlatDdlrIy3CUCpfdbLrKx+3SMjVo+WyHnPpwzkc/M3vwAHw4OVJf7DMvO+4vfRqSCMc/E9I1auN0g==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.973.3", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.3.tgz", + "integrity": "sha512-hmdDHoy2G5Es2e8IgelNMYUuSQI6uCIAKZMJ2u2PdKDhxvbk1uWD/g4+R7R5c/tJfKEB1+KjjWiaoCr/S+ZTiQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/token-providers": "3.1088.0", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso/node_modules/@aws-sdk/token-providers": { + "version": "3.1088.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1088.0.tgz", + "integrity": "sha512-4ObatWt2qpJg5FBk4LOOKrTQYzaqeewAtdO3r9ZO8lH9YqLtpTzLyIdy0mJ+nVdfYOnqISkKNfmzP22bNDhwyw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.65", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.65.tgz", + "integrity": "sha512-gHQb/Kt0chjk/JQDa/GJDqmAvEuVn8n7z10wK2h0LFM9TUDRkohgOO4aEF+s2sBLM0br7Cl5W6P7phgjrrJvLQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/eventstream-handler-node": { + "version": "3.972.29", + "resolved": "https://registry.npmjs.org/@aws-sdk/eventstream-handler-node/-/eventstream-handler-node-3.972.29.tgz", + "integrity": "sha512-t3tKQRTVXsI2QNPE3CaNjHl0wRO9Xi3acZkAyti2RQsiFmZ9Gi0kArX2ighlRJ1BtDVuul413gThAgzyTfgmWA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-eventstream": { + "version": "3.972.24", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-eventstream/-/middleware-eventstream-3.972.24.tgz", + "integrity": "sha512-oykin4mDWxNOuYQ7SF1cHzgYeuFEkF4cdRwgvjFFbIklkx09qIFBiOgsORafG9sXZFO3TayMmQuAQYgADXhI8w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-websocket": { + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-websocket/-/middleware-websocket-3.972.41.tgz", + "integrity": "sha512-LSbGvvYmjc4Br9BPYI2dTLnIclmrSiQbahkP4D6nRGVEv4qsCZ8csVuKBPVEEFCVD+EEngGh8ROls6XpumtwMg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/signature-v4": "^5.6.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.33", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.33.tgz", + "integrity": "sha512-dVZOroI/r3/ENvqNGgjMPul+jjlz9GddfVusgTXlVjfZj5isibOxecLkGQbRPp8XOuX+RAfjXLFgPkD1JS5xrw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/signature-v4-multi-region": "^3.996.41", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/node-http-handler": "^4.9.6", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients/node_modules/@smithy/node-http-handler": { + "version": "4.9.7", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.7.tgz", + "integrity": "sha512-wCU8HCLjAtAVqxxe0j2xff9LcEPw3yjBbg5IdQDIYFnxnPxbxcSLc7rgex7kqm9L/WYOnJEgaWQlfDkZleozMA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.41.tgz", + "integrity": "sha512-QMUytg+FQMGouc8gHS00KoYih3+N6cqmVI/pQGOIo7Nr7OpQaiXjSYOuL+vsPZ1tymY4LAQ8MYcHJmws5LRxng==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@smithy/signature-v4": "^5.6.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1048.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1048.0.tgz", + "integrity": "sha512-k0y/GcuesuSfWyUM0WamrGyeZmltRYaPbHO82UDA6mZ/doB+FOHKutikPAtSXMn/hDz970cF+iRuuiYO9VEbAA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.2", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.2.tgz", + "integrity": "sha512-3W6IUtSxFbH6X7Wb7DzGCV5QiFQsd0g8bOfntpmDxQlzBoKWUMBu/JPQR0DwkE+Hpnxd6db1tXbOwdeHddG6cA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/util-locate-window": { + "version": "3.965.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.965.8.tgz", + "integrity": "sha512-uUbMs1cBZPafD0ohUj6EwNf0fPZ534NvBxHox4hjX+0Rxq5paSYUem7+hi833pYrzrcnBATKIYpR02MDXT5M9g==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.36", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.36.tgz", + "integrity": "sha512-RdGmS1GLrtaTOLE1ElSluMldNrpk9Emq6uYs8SS8iHlu5xTAmM9rRkM91o48+rIRryBtyO9t+uLYCoMG6jVMVA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@earendil-works/pi-agent-core": { + "version": "0.80.10", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.80.10.tgz", + "integrity": "sha512-nwnOR3SuLYGRFfyQm8ri4Nj5VGVAvAM9GuqQd3u7BUQj0d6hmD2F8w7OHAAjThE3CuySIdM+v8E22QJG6/RfCg==", + "license": "MIT", + "dependencies": { + "@earendil-works/pi-ai": "^0.80.10", + "ignore": "7.0.5", + "typebox": "1.1.38", + "yaml": "2.9.0" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-agent-core/node_modules/typebox": { + "version": "1.1.38", + "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.1.38.tgz", + "integrity": "sha512-pZ0aQPmMmXoUvSbeuWf/Hzsc+avNw/Zd6VeE8CFgkVGWyuHPJvqeJJDeJqLve+K70LvjYIoleGcoJHPT17cWoA==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-ai": { + "version": "0.80.10", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.80.10.tgz", + "integrity": "sha512-Moe/H8c87yacDGK9dPbWphZNjVsrb3nTrIHycOQJAkFEnY9PYxOOd74+ny44kATfPU9Dm7aTHefar3pZF+UKUA==", + "license": "MIT", + "dependencies": { + "@anthropic-ai/sdk": "0.91.1", + "@aws-sdk/client-bedrock-runtime": "3.1048.0", + "@google/genai": "1.52.0", + "@mistralai/mistralai": "2.2.6", + "@opentelemetry/api": "1.9.0", + "@smithy/node-http-handler": "4.7.3", + "http-proxy-agent": "7.0.2", + "https-proxy-agent": "7.0.6", + "openai": "6.26.0", + "partial-json": "0.1.7", + "typebox": "1.1.38" + }, + "bin": { + "pi-ai": "dist/cli.js" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-ai/node_modules/typebox": { + "version": "1.1.38", + "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.1.38.tgz", + "integrity": "sha512-pZ0aQPmMmXoUvSbeuWf/Hzsc+avNw/Zd6VeE8CFgkVGWyuHPJvqeJJDeJqLve+K70LvjYIoleGcoJHPT17cWoA==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent": { + "version": "0.80.10", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-coding-agent/-/pi-coding-agent-0.80.10.tgz", + "integrity": "sha512-aL4apbupCHiVLSXASXvRzH4Q2vmtfrDa+0s909CJuVu/GgGylbDzr7oyF1mPmip5E+VxYYxKWmph4hV04wUcQg==", + "license": "MIT", + "dependencies": { + "@earendil-works/pi-agent-core": "^0.80.10", + "@earendil-works/pi-ai": "^0.80.10", + "@earendil-works/pi-tui": "^0.80.10", + "@silvia-odwyer/photon-node": "0.3.4", + "chalk": "5.6.2", + "cross-spawn": "7.0.6", + "diff": "8.0.4", + "glob": "13.0.6", + "highlight.js": "10.7.3", + "hosted-git-info": "9.0.3", + "ignore": "7.0.5", + "jiti": "2.7.0", + "minimatch": "10.2.5", + "proper-lockfile": "4.1.2", + "semver": "7.8.0", + "typebox": "1.1.38", + "undici": "8.5.0", + "yaml": "2.9.0" + }, + "bin": { + "pi": "dist/cli.js" + }, + "engines": { + "node": ">=22.19.0" + }, + "optionalDependencies": { + "@mariozechner/clipboard": "0.3.9" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/typebox": { + "version": "1.1.38", + "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.1.38.tgz", + "integrity": "sha512-pZ0aQPmMmXoUvSbeuWf/Hzsc+avNw/Zd6VeE8CFgkVGWyuHPJvqeJJDeJqLve+K70LvjYIoleGcoJHPT17cWoA==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-tui": { + "version": "0.80.10", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.80.10.tgz", + "integrity": "sha512-c2JO29PbhKPEQ6fgHQKAl0WhwuFqzWfzspMmP+8B5tpDuP+0mvarRbKKg8gq4b+pQx/QX+6aVS4ko7deoyjQjg==", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "1.6.0", + "marked": "18.0.5" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@google/genai": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.52.0.tgz", + "integrity": "sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "google-auth-library": "^10.3.0", + "p-retry": "^4.6.2", + "protobufjs": "^7.5.4", + "ws": "^8.18.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@modelcontextprotocol/sdk": "^1.25.2" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/sdk": { + "optional": true + } + } + }, + "node_modules/@mariozechner/clipboard": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard/-/clipboard-0.3.9.tgz", + "integrity": "sha512-ABnA53mdfkGZwOFUdZNv2S0CWGO/EIuPj8Vv9xmBFmSYg/qFc7ihO6q5FcQjvoE67kZpWkEc4AhD6B/os04yuA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@mariozechner/clipboard-darwin-arm64": "0.3.9", + "@mariozechner/clipboard-darwin-universal": "0.3.9", + "@mariozechner/clipboard-darwin-x64": "0.3.9", + "@mariozechner/clipboard-linux-arm64-gnu": "0.3.9", + "@mariozechner/clipboard-linux-arm64-musl": "0.3.9", + "@mariozechner/clipboard-linux-riscv64-gnu": "0.3.9", + "@mariozechner/clipboard-linux-x64-gnu": "0.3.9", + "@mariozechner/clipboard-linux-x64-musl": "0.3.9", + "@mariozechner/clipboard-win32-arm64-msvc": "0.3.9", + "@mariozechner/clipboard-win32-x64-msvc": "0.3.9" + } + }, + "node_modules/@mariozechner/clipboard-darwin-arm64": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-arm64/-/clipboard-darwin-arm64-0.3.9.tgz", + "integrity": "sha512-BfgV7vCEWZwJwZJw03r6bP5+tf0iI/ANuQYCxi9RNn7FrWB3yzGuMKCrNLRl6V761vXRdL8+OqZ0wd4TqlsNOQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-darwin-universal": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-universal/-/clipboard-darwin-universal-0.3.9.tgz", + "integrity": "sha512-BGGR4iA9Z2shAjI65eI5xtyb3LYNlDW9X3gxKxDbqtbnREohsrqznov6zpKoIrsRWpzlYVEdKphS7ksJ0/ndSQ==", + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-darwin-x64": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-x64/-/clipboard-darwin-x64-0.3.9.tgz", + "integrity": "sha512-4kURmCbS6nt8uYhtmWpUcJWyPHfmAr5dTpXD1nO3pIfa+TSQ9DbrGOYCKH+aEFW47XhQ4Vp8ZTszie+wfFvDKg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-linux-arm64-gnu": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-arm64-gnu/-/clipboard-linux-arm64-gnu-0.3.9.tgz", + "integrity": "sha512-g59OkUGP2DDfCOIKypHeYgv2M55u/cKvXa5dSxFbEJ34XvIQMdcVmpKCkGUro3ZgefXiGVdwguvTMQGpHWzIXw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-linux-arm64-musl": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-arm64-musl/-/clipboard-linux-arm64-musl-0.3.9.tgz", + "integrity": "sha512-AGuJdgKsmJdm4Pych7kv3sqe591ERRaAHW3xjLooiFzn8J+PxUyof++7YZrB5Y5tpnTO+K18Og3taj2NpluCRQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-linux-riscv64-gnu": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-riscv64-gnu/-/clipboard-linux-riscv64-gnu-0.3.9.tgz", + "integrity": "sha512-DXBEAiuMpk7dhS1a9NzNxVAFi1vaKoPu7rQNgY8LIDLGrK3lnIp3nT10DUum+PKVJoJppIP+NAA8IZe4DMNDPw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-linux-x64-gnu": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-x64-gnu/-/clipboard-linux-x64-gnu-0.3.9.tgz", + "integrity": "sha512-WORrMLd6EpElEME7JRKfSaY34nW1P5LbdgK5YNCS1ncG2LqmITsSMEJ8nh2mpvxb3TxqbOOKgY7k9eMJYlW9Mw==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-linux-x64-musl": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-x64-musl/-/clipboard-linux-x64-musl-0.3.9.tgz", + "integrity": "sha512-/DHn+1DrfL6oRaPPWXaOKvonFFrni666fxd+zFqiQEfvBH0tsHVWjq9iqBk0oDp0qaPA72lIMy5BptxISBEhZQ==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-win32-arm64-msvc": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-win32-arm64-msvc/-/clipboard-win32-arm64-msvc-0.3.9.tgz", + "integrity": "sha512-O5FHD3ErkMwMhNzAfu3ggy0ug4z7btZuoQgwwxlzPrwV2bxlD6WDpqBY4NCgICAgZdDKdp+loUEKVAVt8aYnhQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-win32-x64-msvc": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-win32-x64-msvc/-/clipboard-win32-x64-msvc-0.3.9.tgz", + "integrity": "sha512-ihQC3EufqEY81vhXBgVBtK4prL+wc62zJsSvxrgz7K1hsdt6OObz6v9p3Rn1OG3GJksTTKMJF0u/guMISHPhSA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mistralai/mistralai": { + "version": "2.2.6", + "resolved": "https://registry.npmjs.org/@mistralai/mistralai/-/mistralai-2.2.6.tgz", + "integrity": "sha512-W8pX7zHxjJvMIpw8JMxeJEleapXX0Q9NPszdNzqkM3MIEoIGPObdodujj+WHteXEvGfaP/AMwlNyRfEzSY6dQQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.40.0", + "ws": "^8.18.0", + "zod": "^3.25.0 || ^4.0.0", + "zod-to-json-schema": "^3.25.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + } + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.43.0.tgz", + "integrity": "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, + "node_modules/@silvia-odwyer/photon-node": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@silvia-odwyer/photon-node/-/photon-node-0.3.4.tgz", + "integrity": "sha512-bnly4BKB3KDTFxrUIcgCLbaeVVS8lrAkri1pEzskpmxu9MdfGQTy8b8EgcD83ywD3RPMsIulY8xJH5Awa+t9fA==", + "license": "Apache-2.0" + }, + "node_modules/@smithy/core": { + "version": "3.29.5", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.29.5.tgz", + "integrity": "sha512-i0dk2t5B+CwV/dcJdUHILYkOQF5lof8f44dFCfDWToGCxjT9YQ+CgHqTAvJxzc3+zqQwm2QtVoJ5IqiNar/CnQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.4.10", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.10.tgz", + "integrity": "sha512-MJenAe4OKRZUo1LdYYFDCsSHxaHvInIU/z52GsheO9vl1/VSySVCr0zkyKD6TFiGkSUaWGxvKZ/70OvgUZR5HQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.6.7", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.7.tgz", + "integrity": "sha512-3zpg8yqqyXzoK2TsRDdkqVOj2RDBFfLXwCczOZ5c7TWB4eiaebfSCsbMjDPYB3PJ9ihV62QaeadZ+wLadZtNGA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.7.3", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.7.3.tgz", + "integrity": "sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.24.3", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.6.6", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.6.6.tgz", + "integrity": "sha512-efP6DN3UTFrzIsGO42/xcabv8jU7+9nwEdphFUH7yL0k010ERyAWaO41KFQIDLcFZLZ8xzIQr4wplFxNzslSGQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.16.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.16.1.tgz", + "integrity": "sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@types/node": { + "version": "22.20.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.1.tgz", + "integrity": "sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "license": "MIT" + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", + "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, + "node_modules/chalk": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/diff": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.4.tgz", + "integrity": "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/gaxios": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.2.0.tgz", + "integrity": "sha512-CUVb4wcYe+771XevyH6HtGmXFAGGKkIC3kswAP8Z1JCe0j80JMaTPZH930DWFrvo0atjh18Arc0pEyUCWa5bfg==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gcp-metadata": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^7.0.0", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz", + "integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/google-auth-library": { + "version": "10.9.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.9.0.tgz", + "integrity": "sha512-xtvUqvINPhTaBm7nXqlYPcrMHJPm1lCNdSovxnKKhTm+4JsvQ+KGVYJViLoH9Yxu8w+T0Qv5HubzYT9BLrppJg==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "8.1.2", + "google-logging-utils": "1.1.3", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/highlight.js": { + "version": "10.7.3", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", + "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==", + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, + "node_modules/hosted-git-info": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-9.0.3.tgz", + "integrity": "sha512-Hc+ghLoSt6QaYZUv0WBiIvmMDZuZZ7oaDvdH8MbfOO4lOsxdXLEvuC6ePoGs9H1X9oCLyq6+NVN0MKqD+ydxyg==", + "license": "ISC", + "dependencies": { + "lru-cache": "^11.1.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jiti": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.7.0.tgz", + "integrity": "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==", + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/lru-cache": { + "version": "11.5.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", + "integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/marked": { + "version": "18.0.5", + "resolved": "https://registry.npmjs.org/marked/-/marked-18.0.5.tgz", + "integrity": "sha512-S6GcvALHg6K4ohtu4E7x0a1AqhAjp6cV8KhLSyN9qVapnzJkusVBxZRcIU9AeYsbe6P1hKDusSbEOzGyyuce6w==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/openai": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.26.0.tgz", + "integrity": "sha512-zd23dbWTjiJ6sSAX6s0HrCZi41JwTA1bQVs0wLQPZ2/5o2gxOJA5wh7yOAUgwYybfhDXyhwlpeQf7Mlgx8EOCA==", + "license": "Apache-2.0", + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/partial-json": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/partial-json/-/partial-json-0.1.7.tgz", + "integrity": "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA==", + "license": "MIT" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/proper-lockfile/node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/protobufjs": { + "version": "7.6.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", + "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.0.tgz", + "integrity": "sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "license": "MIT" + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/typebox": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.3.6.tgz", + "integrity": "sha512-Sc8RA0NCMEFmApHNU9ZMzqcpQj46She44J8ffpLM/bdhLNUZKq7DJumcLcsFx1gRmDfQPgCgOmFFJ7rcnfWNyA==", + "license": "MIT" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici": { + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.5.0.tgz", + "integrity": "sha512-xamtWoB1EshgjpmlXd7GGm2VfdDtw1+rD8uhry8pSNW3If6S8E0m2T2+orSKeZXEn/aPJMviCpDBA65WJt8zhg==", + "license": "MIT", + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/ws": { + "version": "8.21.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.1.tgz", + "integrity": "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + } + } +} diff --git a/packages/pi-code-policy-adapter/package.json b/packages/pi-code-policy-adapter/package.json new file mode 100644 index 0000000000..345a094939 --- /dev/null +++ b/packages/pi-code-policy-adapter/package.json @@ -0,0 +1,24 @@ +{ + "name": "@dimos/pi-code-policy-adapter", + "version": "0.1.0", + "private": true, + "type": "module", + "engines": { + "node": ">=22.19.0" + }, + "scripts": { + "clean": "rm -rf dist dist-test", + "build": "npm run clean && tsc -p tsconfig.build.json", + "typecheck": "tsc -p tsconfig.json", + "test": "npm run clean && tsc -p tsconfig.build.json && tsc -p tsconfig.test.json && node --test dist-test/test/*.test.js" + }, + "dependencies": { + "@earendil-works/pi-ai": "0.80.10", + "@earendil-works/pi-coding-agent": "0.80.10", + "typebox": "^1.3.6" + }, + "devDependencies": { + "@types/node": "^22.15.0", + "typescript": "^5.8.3" + } +} diff --git a/packages/pi-code-policy-adapter/src/code-policy-main.ts b/packages/pi-code-policy-adapter/src/code-policy-main.ts new file mode 100644 index 0000000000..a5cc49e89a --- /dev/null +++ b/packages/pi-code-policy-adapter/src/code-policy-main.ts @@ -0,0 +1,241 @@ +import { createInterface } from "node:readline"; +import { stdin, stdout, stderr } from "node:process"; +import { + encodeCodePolicyFrame, + parseCodePolicyFrame, + type CodePolicyOutbound, +} from "./code-policy-protocol.js"; +import { + CODE_POLICY_TOOL_NAME, + createFreshCodePolicySession, + type CodePolicyBroker, +} from "./code-policy-session.js"; +import type { + SessionAdapterHandle, + SessionEvidenceMetadata, + StoredAuthOptions, +} from "./session.js"; + +export type CodePolicySessionFactory = ( + broker: CodePolicyBroker, + options: StoredAuthOptions, + config: { thinkingLevel: "medium" }, + initialPrompt: string, +) => Promise; + +function authOptionsFromEnvironment(env: NodeJS.ProcessEnv): StoredAuthOptions { + const mode = env.PI_SPATIAL_AUTH_MODE ?? "codex-oauth"; + if (mode === "codex-oauth" && env.PI_SPATIAL_AUTH_PATH) { + return { + authMode: mode, + authPath: env.PI_SPATIAL_AUTH_PATH, + modelsPath: env.PI_SPATIAL_MODELS_PATH, + }; + } + if (mode === "openai-api-key" && env.OPENAI_API_KEY) { + return { + authMode: mode, + apiKey: env.OPENAI_API_KEY, + modelsPath: env.PI_SPATIAL_MODELS_PATH, + }; + } + throw new Error("Pi authentication environment is incomplete"); +} + +class HostBroker implements CodePolicyBroker { + private sequence = 0; + private readonly pending = new Map< + string, + { resolve: (value: string) => void; reject: (error: Error) => void } + >(); + + constructor(private readonly emit: (frame: CodePolicyOutbound) => void) {} + + request( + tool: typeof CODE_POLICY_TOOL_NAME, + params: { code: string; timeout_s?: number }, + ): Promise { + const id = `tool-${++this.sequence}`; + return new Promise((resolve, reject) => { + this.pending.set(id, { resolve, reject }); + this.emit({ version: 1, type: "tool_call", id, tool, params }); + }); + } + + reply(id: string, ok: boolean, result?: string, error?: string): void { + const pending = this.pending.get(id); + if (!pending) throw new Error("unknown or duplicate code-policy tool reply"); + this.pending.delete(id); + if (ok && result !== undefined) pending.resolve(result); + else pending.reject(new Error(error ?? "host code-policy tool failed")); + } + + count(): number { + return this.sequence; + } + + close(reason: string): void { + for (const pending of this.pending.values()) pending.reject(new Error(reason)); + this.pending.clear(); + } +} + +function record(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function progressFrame(event: unknown): CodePolicyOutbound | undefined { + if (!record(event)) return undefined; + if (event.type === "agent_start" || event.type === "turn_start" || event.type === "agent_end") { + return { version: 1, type: "transcript", event: event.type }; + } + if (event.type !== "message_update" || !record(event.assistantMessageEvent)) { + return undefined; + } + const update = event.assistantMessageEvent; + if (update.type !== "text_delta" || typeof update.delta !== "string" || update.delta.length === 0) { + return undefined; + } + return { + version: 1, + type: "transcript", + event: "assistant_text_delta", + delta: update.delta, + }; +} + +function evidenceFrame(evidence: SessionEvidenceMetadata) { + return { + state: evidence.state, + persisted: evidence.persisted, + ...(evidence.relativePath ? { relative_path: evidence.relativePath } : {}), + ...(evidence.systemPrompt + ? { + system_prompt: { + relative_path: evidence.systemPrompt.relativePath, + byte_count: evidence.systemPrompt.byteCount, + sha256: evidence.systemPrompt.sha256, + }, + } + : {}), + ...(evidence.initialPrompt + ? { + initial_prompt: { + relative_path: evidence.initialPrompt.relativePath, + byte_count: evidence.initialPrompt.byteCount, + sha256: evidence.initialPrompt.sha256, + }, + } + : {}), + }; +} + +export async function runCodePolicyAdapter( + input: NodeJS.ReadableStream = stdin, + output: NodeJS.WritableStream = stdout, + diagnostics: NodeJS.WritableStream = stderr, + sessionFactory: CodePolicySessionFactory = createFreshCodePolicySession, +): Promise { + const lines = createInterface({ input, crlfDelay: Infinity }); + const emit = (frame: CodePolicyOutbound): void => { + output.write(encodeCodePolicyFrame(frame)); + }; + let session: SessionAdapterHandle | undefined; + let broker: HostBroker | undefined; + let sessionId = ""; + let activeTurn: Promise | undefined; + let activeVisibleText = ""; + let closed = false; + try { + for await (const line of lines) { + const frame = parseCodePolicyFrame(line); + if (frame.type === "session_start") { + if (session || broker) throw new Error("duplicate session_start"); + sessionId = frame.id; + broker = new HostBroker(emit); + session = await sessionFactory( + broker, + authOptionsFromEnvironment(process.env), + { thinkingLevel: frame.thinking_level }, + frame.initial_prompt, + ); + session.subscribe((event) => { + if (record(event) && event.type === "turn_start") { + activeVisibleText = ""; + } else if ( + record(event) && + event.type === "message_update" && + record(event.assistantMessageEvent) && + event.assistantMessageEvent.type === "text_delta" && + typeof event.assistantMessageEvent.delta === "string" + ) { + activeVisibleText = (activeVisibleText + event.assistantMessageEvent.delta).slice( + 0, + 16_384, + ); + } + const frame = progressFrame(event); + if (frame) emit(frame); + }); + emit({ version: 1, type: "session_started", id: sessionId, tools: ["python_exec"] }); + } else if (frame.type === "prompt") { + if (!session || !broker || activeTurn) throw new Error("prompt outside idle session"); + const before = broker.count(); + activeVisibleText = ""; + activeTurn = session + .prompt(frame.text) + .then((result: unknown) => { + const returnedText = + typeof result === "string" + ? result + : typeof result === "object" && result !== null + ? JSON.stringify(result).slice(0, 16_384) + : ""; + const finalText = returnedText || activeVisibleText; + emit({ + version: 1, + type: "turn_complete", + id: frame.id, + policy_call_count: broker?.count() ?? before, + final_text: finalText, + }); + }) + .finally(() => { + activeTurn = undefined; + }); + } else if (frame.type === "tool_reply") { + if (!broker) throw new Error("tool reply before session"); + broker.reply(frame.id, frame.ok, frame.result, frame.error); + } else if (frame.type === "abort") { + await session?.abort(); + } else { + if (!session || !broker) throw new Error("dispose before session"); + await session.abort().catch(() => undefined); + await activeTurn?.catch(() => undefined); + session.dispose(); + broker.close("session disposed"); + emit({ + version: 1, + type: "session_closed", + id: sessionId, + evidence: evidenceFrame(session.sessionEvidence(true)), + }); + closed = true; + lines.close(); + } + } + } catch (error) { + const message = error instanceof Error ? error.message : "code-policy adapter failure"; + diagnostics.write(`${message.replace(/[\r\n]+/g, " ").slice(0, 1024)}\n`); + emit({ version: 1, type: "protocol_error", error: message.slice(0, 1024) }); + } finally { + if (!closed) { + broker?.close("adapter input closed"); + session?.dispose(); + } + } +} + +if (process.argv[1]?.endsWith("code-policy-main.js")) { + void runCodePolicyAdapter(); +} diff --git a/packages/pi-code-policy-adapter/src/code-policy-protocol.ts b/packages/pi-code-policy-adapter/src/code-policy-protocol.ts new file mode 100644 index 0000000000..874793bcbd --- /dev/null +++ b/packages/pi-code-policy-adapter/src/code-policy-protocol.ts @@ -0,0 +1,122 @@ +export const CODE_POLICY_PROTOCOL_VERSION = 1; +export const CODE_POLICY_MAX_LINE_BYTES = 64 * 1024; + +export type CodePolicyInbound = + | { + version: 1; + type: "session_start"; + id: string; + initial_prompt: string; + thinking_level: "medium"; + } + | { version: 1; type: "prompt"; id: string; text: string } + | { + version: 1; + type: "tool_reply"; + id: string; + ok: boolean; + result?: string; + error?: string; + } + | { version: 1; type: "abort" } + | { version: 1; type: "dispose" }; + +export type CodePolicyOutbound = + | { version: 1; type: "session_started"; id: string; tools: ["python_exec"] } + | { + version: 1; + type: "tool_call"; + id: string; + tool: "python_exec"; + params: { code: string; timeout_s?: number }; + } + | { + version: 1; + type: "transcript"; + event: "agent_start" | "turn_start" | "agent_end"; + } + | { + version: 1; + type: "transcript"; + event: "assistant_text_delta"; + delta: string; + } + | { + version: 1; + type: "turn_complete"; + id: string; + policy_call_count: number; + final_text: string; + } + | { + version: 1; + type: "session_closed"; + id: string; + evidence: { + state: "complete" | "partial" | "unavailable"; + persisted: boolean; + relative_path?: string; + system_prompt?: { relative_path: string; byte_count: number; sha256: string }; + initial_prompt?: { relative_path: string; byte_count: number; sha256: string }; + }; + } + | { version: 1; type: "protocol_error"; error: string }; + +function record(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function parseCodePolicyFrame(line: string): CodePolicyInbound { + if (Buffer.byteLength(line, "utf8") > CODE_POLICY_MAX_LINE_BYTES) { + throw new Error("code-policy frame exceeds limit"); + } + let value: unknown; + try { + value = JSON.parse(line); + } catch { + throw new Error("invalid code-policy JSON frame"); + } + if (!record(value) || value.version !== CODE_POLICY_PROTOCOL_VERSION) { + throw new Error("invalid code-policy protocol version"); + } + if ( + value.type === "session_start" && + typeof value.id === "string" && + value.id.length > 0 && + typeof value.initial_prompt === "string" && + value.initial_prompt.length > 0 && + value.thinking_level === "medium" + ) { + return value as CodePolicyInbound; + } + if ( + value.type === "prompt" && + typeof value.id === "string" && + value.id.length > 0 && + typeof value.text === "string" && + value.text.length > 0 + ) { + return value as CodePolicyInbound; + } + if ( + value.type === "tool_reply" && + typeof value.id === "string" && + typeof value.ok === "boolean" && + (value.result === undefined || typeof value.result === "string") && + (value.error === undefined || typeof value.error === "string") + ) { + return value as CodePolicyInbound; + } + if (value.type === "abort" || value.type === "dispose") { + return value as CodePolicyInbound; + } + throw new Error("invalid code-policy frame"); +} + +export function encodeCodePolicyFrame(frame: CodePolicyOutbound): string { + const encoded = JSON.stringify(frame); + if (Buffer.byteLength(encoded, "utf8") > CODE_POLICY_MAX_LINE_BYTES) { + throw new Error("outbound code-policy frame exceeds limit"); + } + return `${encoded}\n`; +} diff --git a/packages/pi-code-policy-adapter/src/code-policy-session.ts b/packages/pi-code-policy-adapter/src/code-policy-session.ts new file mode 100644 index 0000000000..30ab0df70f --- /dev/null +++ b/packages/pi-code-policy-adapter/src/code-policy-session.ts @@ -0,0 +1,74 @@ +import { Type } from "typebox"; +import type { ToolDefinition } from "@earendil-works/pi-coding-agent"; +import { + createFreshSessionWithTools, + type SessionAdapterHandle, + type SessionConfig, + type StoredAuthOptions, +} from "./session.js"; + +export const CODE_POLICY_TOOL_NAME = "python_exec" as const; +export const CODE_POLICY_TOOL_NAMES = [CODE_POLICY_TOOL_NAME] as const; + +export function assertCodePolicyToolInventory(names: readonly string[]): void { + if (names.length !== 1 || names[0] !== CODE_POLICY_TOOL_NAME) { + throw new Error("Pi code-policy session did not activate exactly python_exec"); + } +} + +export interface CodePolicyBroker { + request( + tool: typeof CODE_POLICY_TOOL_NAME, + params: { code: string; timeout_s?: number }, + ): Promise; +} + +export function codePolicyToolDefinition(broker: CodePolicyBroker): ToolDefinition { + return { + name: CODE_POLICY_TOOL_NAME, + label: "Execute Python", + description: + "Execute one synchronous Python program in the persistent trusted, unsandboxed DimOS policy session. The session preloads app for deployed DimOS RPCs and memory for observations.", + parameters: Type.Object( + { + code: Type.String({ minLength: 1 }), + timeout_s: Type.Optional(Type.Number({ exclusiveMinimum: 0, maximum: 110 })), + }, + { additionalProperties: false }, + ), + execute: async (_id, params) => ({ + content: [ + { + type: "text", + text: await broker.request( + CODE_POLICY_TOOL_NAME, + params as { code: string; timeout_s?: number }, + ), + }, + ], + details: {}, + }), + }; +} + +export async function createFreshCodePolicySession( + broker: CodePolicyBroker, + options: StoredAuthOptions, + config: SessionConfig, + initialPrompt: string, +): Promise { + const result = await createFreshSessionWithTools( + [codePolicyToolDefinition(broker)], + options, + config, + initialPrompt, + CODE_POLICY_TOOL_NAMES, + ); + try { + assertCodePolicyToolInventory(result.activeToolNames); + } catch (error) { + result.handle.dispose(); + throw error; + } + return result.handle; +} diff --git a/packages/pi-code-policy-adapter/src/session.ts b/packages/pi-code-policy-adapter/src/session.ts new file mode 100644 index 0000000000..2fb55dece0 --- /dev/null +++ b/packages/pi-code-policy-adapter/src/session.ts @@ -0,0 +1,352 @@ +import { ModelRegistry, ModelRuntime, SessionManager, createAgentSession, readStoredCredential } from "@earendil-works/pi-coding-agent"; +import { InMemoryCredentialStore, type Model } from "@earendil-works/pi-ai"; +import type { ToolDefinition } from "@earendil-works/pi-coding-agent"; +import { chmodSync, closeSync, constants, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, unlinkSync, writeSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { basename, dirname, relative, resolve, sep } from "node:path"; +import { fileURLToPath } from "node:url"; + +export const MODEL_PROVIDER = "openai-codex"; +export const API_KEY_MODEL_PROVIDER = "openai"; +export const MODEL_ID = "gpt-5.6-luna"; +export const THINKING_LEVEL = "medium" as const; +export const REQUIRED_API = "openai-codex-responses"; +export const API_KEY_REQUIRED_API = "openai-responses"; +export const SESSION_DIR_ENV = "PI_SPATIAL_SESSION_DIR"; +export const AGENT_CWD = process.env.PI_SPATIAL_AGENT_CWD ?? "/work"; +export const PINNED_PI_VERSION = "0.80.10"; + +// The adapter is a dedicated process. Set this before Pi can create a session file. +process.umask(0o077); + +export type SessionEvidenceState = "complete" | "partial" | "unavailable"; + +export interface SystemPromptEvidenceMetadata { + readonly relativePath: "pi-prompt/system.txt"; + readonly byteCount: number; + readonly sha256: string; +} +export interface InitialPromptEvidenceMetadata { + readonly relativePath: "pi-prompt/initial.txt"; + readonly byteCount: number; + readonly sha256: string; +} + +export interface SessionEvidenceMetadata { + /** Safe path relative to the attempt cwd; never an absolute host path. */ + readonly relativePath?: string; + readonly persisted: boolean; + readonly state: SessionEvidenceState; + readonly systemPrompt?: SystemPromptEvidenceMetadata; + readonly initialPrompt?: InitialPromptEvidenceMetadata; +} + +export interface SessionAdapterHandle { + prompt(prompt: string): Promise; + subscribe(listener: (event: unknown) => void): void; + abort: () => Promise; + dispose: () => void; + sessionEvidence: (completed: boolean) => SessionEvidenceMetadata; +} + +interface SessionDirectory { + readonly name: string; + readonly path: string; +} + +function configuredSessionDirectory(): SessionDirectory { + const value = process.env[SESSION_DIR_ENV]; + if (value === undefined || !/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value)) { + throw new Error(`${SESSION_DIR_ENV} must be provided as one simple relative directory name`); + } + const path = resolve(process.cwd(), value); + if (dirname(path) !== process.cwd()) throw new Error(`${SESSION_DIR_ENV} must stay beneath process.cwd()`); + return { name: value, path }; +} + +function ensurePrivateSessionDirectory(directory: SessionDirectory): void { + try { + const stat = lstatSync(directory.path); + if (stat.isSymbolicLink() || !stat.isDirectory()) throw new Error(`${SESSION_DIR_ENV} must be a real directory`); + chmodSync(directory.path, 0o700); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + try { + mkdirSync(directory.path, { mode: 0o700 }); + } catch (mkdirError) { + if ((mkdirError as NodeJS.ErrnoException).code !== "EEXIST") throw mkdirError; + } + const stat = lstatSync(directory.path); + if (stat.isSymbolicLink() || !stat.isDirectory()) throw new Error(`${SESSION_DIR_ENV} must be a real directory`); + chmodSync(directory.path, 0o700); + } +} + +function ensurePrivatePromptDirectory(): string { + const path = resolve(process.cwd(), "pi-prompt"); + try { + const stat = lstatSync(path); + if (stat.isSymbolicLink() || !stat.isDirectory()) throw new Error("pi-prompt must be a real directory"); + chmodSync(path, 0o700); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + try { + mkdirSync(path, { mode: 0o700 }); + } catch (mkdirError) { + if ((mkdirError as NodeJS.ErrnoException).code !== "EEXIST") throw mkdirError; + } + const stat = lstatSync(path); + if (stat.isSymbolicLink() || !stat.isDirectory()) throw new Error("pi-prompt must be a real directory"); + chmodSync(path, 0o700); + } + return path; +} + +export function retainSystemPromptEvidence(systemPrompt: string): SystemPromptEvidenceMetadata { + const promptDirectory = ensurePrivatePromptDirectory(); + const path = resolve(promptDirectory, "system.txt"); + const bytes = Buffer.from(systemPrompt, "utf8"); + const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); + try { + const written = writeSync(fd, bytes); + if (written !== bytes.length) throw new Error("system prompt sidecar write was incomplete"); + fsyncSync(fd); + } catch (error) { + closeSync(fd); + try { unlinkSync(path); } catch { /* preserve the original setup failure */ } + throw error; + } + closeSync(fd); + try { + const directoryFd = openSync(promptDirectory, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + try { fsyncSync(directoryFd); } finally { closeSync(directoryFd); } + } catch { + // Directory fsync is best effort; the file itself was fsynced above. + } + return { + relativePath: "pi-prompt/system.txt", + byteCount: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + }; +} + +export function retainInitialPromptEvidence(initialPrompt: string): InitialPromptEvidenceMetadata { + const promptDirectory = ensurePrivatePromptDirectory(); + const path = resolve(promptDirectory, "initial.txt"); + const bytes = Buffer.from(initialPrompt, "utf8"); + const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); + try { + if (writeSync(fd, bytes) !== bytes.length) throw new Error("initial prompt sidecar write was incomplete"); + fsyncSync(fd); + } catch (error) { + closeSync(fd); + try { unlinkSync(path); } catch { /* preserve original setup failure */ } + throw error; + } + closeSync(fd); + return { relativePath: "pi-prompt/initial.txt", byteCount: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") }; +} + +export function createSessionManager(cwd: string = AGENT_CWD): SessionManager { + const directory = configuredSessionDirectory(); + ensurePrivateSessionDirectory(directory); + return SessionManager.create(cwd, directory.path); +} + +function safeRelativeSessionFile(manager: SessionManager): string | undefined { + const file = manager.getSessionFile(); + if (!file) return undefined; + const sessionDirectory = resolve(manager.getSessionDir()); + const relativeToSessionDirectory = relative(sessionDirectory, resolve(file)); + const relativeSessionDirectory = relative(process.cwd(), sessionDirectory); + if (!relativeToSessionDirectory || relativeToSessionDirectory.includes(sep) || relativeToSessionDirectory.startsWith("..") || !/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(relativeSessionDirectory) || !basename(relativeToSessionDirectory).endsWith(".jsonl")) { + return undefined; + } + let stat: ReturnType; + try { + stat = lstatSync(file); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } + if (!stat.isFile()) return undefined; + return `${relativeSessionDirectory}/${basename(relativeToSessionDirectory)}`; +} + +export function sessionEvidenceForManager(manager: SessionManager, completed: boolean): SessionEvidenceMetadata { + const relativePath = safeRelativeSessionFile(manager); + const persisted = manager.isPersisted(); + return { + ...(relativePath ? { relativePath } : {}), + persisted: persisted && relativePath !== undefined, + state: !persisted || relativePath === undefined ? "unavailable" : completed ? "complete" : "partial", + }; +} + +export function resolvePinnedPiCli(): string { + const packageName = "@earendil-works/pi-coding-agent"; + let directory = dirname(fileURLToPath(import.meta.url)); + while (true) { + const packageDirectory = resolve(directory, "node_modules", packageName); + const packageJson = resolve(packageDirectory, "package.json"); + if (existsSync(packageJson)) { + const metadata = JSON.parse(readFileSync(packageJson, "utf8")) as { version?: unknown; bin?: unknown }; + if (metadata.version !== PINNED_PI_VERSION) throw new Error(`expected pinned Pi ${PINNED_PI_VERSION}`); + const bin = metadata.bin; + const binPath = typeof bin === "object" && bin !== null && "pi" in bin && typeof bin.pi === "string" ? bin.pi : undefined; + if (binPath !== "dist/cli.js") throw new Error("pinned Pi package has an unexpected pi bin"); + const cli = resolve(packageDirectory, binPath); + if (!existsSync(cli)) throw new Error("pinned Pi CLI entrypoint is missing"); + return cli; + } + const parent = dirname(directory); + if (parent === directory) break; + directory = parent; + } + throw new Error("pinned Pi package cannot be resolved"); +} + +export interface PinnedPiExportCommand { + readonly executable: string; + readonly args: readonly [string, "--export", string, string]; + readonly packageVersion: typeof PINNED_PI_VERSION; +} + +export function resolvePinnedPiExportCommand(input: string, output: string): PinnedPiExportCommand { + return { + executable: process.execPath, + args: [resolvePinnedPiCli(), "--export", input, output], + packageVersion: PINNED_PI_VERSION, + }; +} + +export function modelProviderForAuthMode(authMode: AuthMode): typeof MODEL_PROVIDER | typeof API_KEY_MODEL_PROVIDER { + return authMode === "codex-oauth" ? MODEL_PROVIDER : API_KEY_MODEL_PROVIDER; +} + +export function requiredApiForAuthMode(authMode: AuthMode): typeof REQUIRED_API | typeof API_KEY_REQUIRED_API { + return authMode === "codex-oauth" ? REQUIRED_API : API_KEY_REQUIRED_API; +} + +export function resolveConfiguredModel( + registry: ModelRegistry, + authMode: AuthMode = "codex-oauth", +): Model<"openai-codex-responses" | "openai-responses"> { + const provider = modelProviderForAuthMode(authMode); + const requiredApi = requiredApiForAuthMode(authMode); + const model = registry.find(provider, MODEL_ID); + if (!model || model.api !== requiredApi || !model.input.includes("image") || !model.reasoning) { + throw new Error("configured model is missing, has the wrong API, does not accept images, or does not support thinking"); + } + return model as Model<"openai-codex-responses" | "openai-responses">; +} + +export type AuthMode = "codex-oauth" | "openai-api-key"; + +export interface StoredAuthOptions { + authMode: AuthMode; + authPath?: string; + apiKey?: string; + modelsPath?: string; +} + +export interface SessionConfig { + thinkingLevel: typeof THINKING_LEVEL; +} + +export function validateSessionConfig(config: SessionConfig): void { + if (config.thinkingLevel !== THINKING_LEVEL) throw new Error("unsupported thinking level"); +} + +export async function createFreshSessionWithTools( + tools: readonly ToolDefinition[], + options: StoredAuthOptions, + config: SessionConfig, + initialPrompt: string, + expectedToolNames: readonly string[], +): Promise<{ handle: SessionAdapterHandle; activeToolNames: readonly string[] }> { + validateSessionConfig(config); + if ( + expectedToolNames.length === 0 || + new Set(expectedToolNames).size !== expectedToolNames.length || + tools.length !== expectedToolNames.length || + tools.some((tool, index) => tool.name !== expectedToolNames[index]) + ) { + throw new Error("custom tools do not match the expected ordered inventory"); + } + const manager = createSessionManager(); + const initialPromptEvidence = retainInitialPromptEvidence(initialPrompt); + let runtime: ModelRuntime; + if (options.authMode === "codex-oauth") { + if (!options.authPath) throw new Error("Codex OAuth auth path is required"); + const credential = readStoredCredential(MODEL_PROVIDER, options.authPath); + if (!credential || credential.type !== "oauth") throw new Error("Codex OAuth credentials are not stored"); + runtime = await ModelRuntime.create({ authPath: options.authPath, modelsPath: options.modelsPath }); + } else { + if (!options.apiKey) throw new Error("OpenAI API key is required"); + const credentials = new InMemoryCredentialStore(); + await credentials.modify(API_KEY_MODEL_PROVIDER, async () => ({ + type: "api_key", + key: options.apiKey, + })); + runtime = await ModelRuntime.create({ + credentials, + modelsPath: options.modelsPath, + }); + } + const registry = new ModelRegistry(runtime); + await registry.refresh(); + const model = resolveConfiguredModel(registry, options.authMode); + if (options.authMode === "codex-oauth") { + if (!registry.isUsingOAuth(model)) throw new Error("configured model is not using Codex OAuth"); + } else { + const status = registry.getProviderAuthStatus(API_KEY_MODEL_PROVIDER); + if (registry.isUsingOAuth(model) || !status.configured || status.source !== "stored") { + throw new Error("configured model is not using an OpenAI API key"); + } + } + const custom = [...tools]; + const available = custom.map((tool) => tool.name); + const result = await createAgentSession({ + cwd: AGENT_CWD, + model, + thinkingLevel: config.thinkingLevel, + modelRuntime: runtime, + sessionManager: manager, + noTools: "builtin", + tools: available, + customTools: custom, + }); + const active = result.session.getActiveToolNames(); + if ( + active.length !== expectedToolNames.length || + active.some((name, index) => name !== expectedToolNames[index]) + ) { + result.session.dispose(); + throw new Error("Pi activated an unexpected tool inventory"); + } + let systemPrompt: SystemPromptEvidenceMetadata; + try { + systemPrompt = retainSystemPromptEvidence(result.session.systemPrompt); + } catch (error) { + result.session.dispose(); + throw error; + } + let disposed = false; + const handle = { + prompt: (prompt: string) => result.session.prompt(prompt), + subscribe: (listener: (event: unknown) => void) => { result.session.subscribe((event) => listener(event)); }, + abort: () => result.session.abort(), + dispose: () => { + if (!disposed) { + disposed = true; + result.session.dispose(); + } + }, + sessionEvidence: (completed: boolean): SessionEvidenceMetadata => { + const evidence = sessionEvidenceForManager(manager, completed); + return evidence.state === "unavailable" ? evidence : { ...evidence, systemPrompt, initialPrompt: initialPromptEvidence }; + }, + } satisfies SessionAdapterHandle; + return { handle, activeToolNames: active }; +} diff --git a/packages/pi-code-policy-adapter/test/code-policy-main.test.ts b/packages/pi-code-policy-adapter/test/code-policy-main.test.ts new file mode 100644 index 0000000000..27b36b47e5 --- /dev/null +++ b/packages/pi-code-policy-adapter/test/code-policy-main.test.ts @@ -0,0 +1,123 @@ +import assert from "node:assert/strict"; +import { Readable, Writable } from "node:stream"; +import test from "node:test"; +import { + progressFrame, + runCodePolicyAdapter, + type CodePolicySessionFactory, +} from "../src/code-policy-main.js"; + +function sink(): { stream: Writable; frames: () => Array> } { + const chunks: string[] = []; + return { + stream: new Writable({ + write(chunk, _encoding, callback) { + chunks.push(String(chunk)); + callback(); + }, + }), + frames: () => + chunks + .join("") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as Record), + }; +} + +test("normalizes visible progress and discards thinking and raw events", () => { + assert.deepEqual(progressFrame({ type: "agent_start" }), { + version: 1, + type: "transcript", + event: "agent_start", + }); + assert.deepEqual( + progressFrame({ + type: "message_update", + assistantMessageEvent: { type: "text_delta", delta: "Looking around" }, + }), + { + version: 1, + type: "transcript", + event: "assistant_text_delta", + delta: "Looking around", + }, + ); + assert.equal( + progressFrame({ + type: "message_update", + assistantMessageEvent: { type: "thinking_delta", delta: "private reasoning" }, + }), + undefined, + ); + assert.equal(progressFrame({ type: "before_provider_request", payload: "private" }), undefined); +}); + +test("streams concise progress during a code-policy turn", async () => { + const output = sink(); + const previousMode = process.env.PI_SPATIAL_AUTH_MODE; + const previousKey = process.env.OPENAI_API_KEY; + process.env.PI_SPATIAL_AUTH_MODE = "openai-api-key"; + process.env.OPENAI_API_KEY = "test-key"; + let listener: ((event: unknown) => void) | undefined; + const factory: CodePolicySessionFactory = async () => ({ + subscribe: (next) => { + listener = next; + }, + prompt: async () => { + listener?.({ type: "agent_start" }); + listener?.({ type: "turn_start" }); + listener?.({ + type: "message_update", + assistantMessageEvent: { type: "thinking_delta", delta: "do not emit" }, + }); + listener?.({ + type: "message_update", + assistantMessageEvent: { type: "text_delta", delta: "Visible text" }, + }); + listener?.({ + type: "message_update", + assistantMessageEvent: { type: "text_delta", delta: "\nANSWER: 2" }, + }); + listener?.({ type: "agent_end" }); + return undefined; + }, + abort: async () => undefined, + dispose: () => undefined, + sessionEvidence: () => ({ state: "complete", persisted: false }), + }); + const input = Readable.from( + [ + { + version: 1, + type: "session_start", + id: "session-1", + initial_prompt: "Count rooms", + thinking_level: "medium", + }, + { version: 1, type: "prompt", id: "turn-1", text: "Count rooms" }, + { version: 1, type: "dispose" }, + ].map((frame) => `${JSON.stringify(frame)}\n`), + ); + + try { + await runCodePolicyAdapter(input, output.stream, new Writable({ + write(_chunk, _encoding, callback) { + callback(); + }, + }), factory); + } finally { + if (previousMode === undefined) delete process.env.PI_SPATIAL_AUTH_MODE; + else process.env.PI_SPATIAL_AUTH_MODE = previousMode; + if (previousKey === undefined) delete process.env.OPENAI_API_KEY; + else process.env.OPENAI_API_KEY = previousKey; + } + + const frames = output.frames(); + assert.equal(frames.some((frame) => frame.event === "assistant_text_delta"), true); + assert.equal(JSON.stringify(frames).includes("Visible text"), true); + assert.equal(JSON.stringify(frames).includes("do not emit"), false); + const complete = frames.find((frame) => frame.type === "turn_complete"); + assert.equal(complete?.final_text, "Visible text\nANSWER: 2"); +}); diff --git a/packages/pi-code-policy-adapter/test/code-policy-protocol.test.ts b/packages/pi-code-policy-adapter/test/code-policy-protocol.test.ts new file mode 100644 index 0000000000..6cc4573dcf --- /dev/null +++ b/packages/pi-code-policy-adapter/test/code-policy-protocol.test.ts @@ -0,0 +1,31 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + CODE_POLICY_MAX_LINE_BYTES, + encodeCodePolicyFrame, + parseCodePolicyFrame, +} from "../src/code-policy-protocol.js"; + +test("accepts a valid prompt and emits one newline-delimited frame", () => { + assert.deepEqual( + parseCodePolicyFrame(JSON.stringify({ version: 1, type: "prompt", id: "turn-1", text: "Count" })), + { version: 1, type: "prompt", id: "turn-1", text: "Count" }, + ); + assert.equal( + encodeCodePolicyFrame({ version: 1, type: "session_started", id: "session-1", tools: ["python_exec"] }), + '{"version":1,"type":"session_started","id":"session-1","tools":["python_exec"]}\n', + ); +}); + +test("rejects malformed, unknown, and oversized inbound frames", () => { + assert.throws(() => parseCodePolicyFrame("{"), /invalid code-policy JSON/); + assert.throws( + () => parseCodePolicyFrame(JSON.stringify({ version: 1, type: "unknown" })), + /invalid code-policy frame/, + ); + assert.throws( + () => parseCodePolicyFrame("x".repeat(CODE_POLICY_MAX_LINE_BYTES + 1)), + /exceeds limit/, + ); +}); diff --git a/packages/pi-code-policy-adapter/test/code-policy-session.test.ts b/packages/pi-code-policy-adapter/test/code-policy-session.test.ts new file mode 100644 index 0000000000..37e019b2ca --- /dev/null +++ b/packages/pi-code-policy-adapter/test/code-policy-session.test.ts @@ -0,0 +1,40 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + CODE_POLICY_TOOL_NAMES, + assertCodePolicyToolInventory, + codePolicyToolDefinition, +} from "../src/code-policy-session.js"; + +test("code-policy facade registers exactly python_exec", async () => { + const calls: Array<{ tool: string; params: unknown }> = []; + const tool = codePolicyToolDefinition({ + request: async (name, params) => { + calls.push({ tool: name, params }); + return "ok"; + }, + }); + + assert.deepEqual(CODE_POLICY_TOOL_NAMES, ["python_exec"]); + assert.equal(tool.name, "python_exec"); + const result = await tool.execute( + "call-1", + { code: "1 + 1" }, + undefined, + undefined, + undefined as never, + ); + assert.deepEqual(calls, [ + { tool: "python_exec", params: { code: "1 + 1" } }, + ]); + assert.deepEqual(result.content, [{ type: "text", text: "ok" }]); +}); + +test("code-policy facade rejects any activated tool inventory drift", () => { + assert.doesNotThrow(() => assertCodePolicyToolInventory(["python_exec"])); + assert.throws(() => assertCodePolicyToolInventory([]), /exactly python_exec/); + assert.throws( + () => assertCodePolicyToolInventory(["python_exec", "read"]), + /exactly python_exec/, + ); +}); diff --git a/packages/pi-code-policy-adapter/test/session.test.ts b/packages/pi-code-policy-adapter/test/session.test.ts new file mode 100644 index 0000000000..027044d587 --- /dev/null +++ b/packages/pi-code-policy-adapter/test/session.test.ts @@ -0,0 +1,229 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, mkdirSync, readFileSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { execFileSync } from "node:child_process"; +import type { Message } from "@earendil-works/pi-ai"; +import { SessionManager } from "@earendil-works/pi-coding-agent"; +import { + PINNED_PI_VERSION, + SESSION_DIR_ENV, + createSessionManager, + modelProviderForAuthMode, + requiredApiForAuthMode, + retainSystemPromptEvidence, + resolvePinnedPiExportCommand, + sessionEvidenceForManager, +} from "../src/session.js"; + +test("auth mode selects the matching Pi provider and request API", () => { + assert.equal(modelProviderForAuthMode("codex-oauth"), "openai-codex"); + assert.equal(requiredApiForAuthMode("codex-oauth"), "openai-codex-responses"); + assert.equal(modelProviderForAuthMode("openai-api-key"), "openai"); + assert.equal(requiredApiForAuthMode("openai-api-key"), "openai-responses"); +}); + +function withSessionDirectory(name: string, callback: (directory: string) => T): T { + const previous = process.env[SESSION_DIR_ENV]; + process.env[SESSION_DIR_ENV] = name; + const directory = join(process.cwd(), name); + try { + return callback(directory); + } finally { + if (previous === undefined) delete process.env[SESSION_DIR_ENV]; + else process.env[SESSION_DIR_ENV] = previous; + rmSync(directory, { recursive: true, force: true }); + rmSync(join(process.cwd(), "pi-prompt"), { recursive: true, force: true }); + } +} + +test("file-backed session directory accepts only a simple relative name", () => { + for (const value of ["", ".", "..", "/tmp/pi", "../pi", "nested/pi", "pi\\session"]) { + process.env[SESSION_DIR_ENV] = value; + assert.throws(() => createSessionManager(), /PI_SPATIAL_SESSION_DIR/); + } + delete process.env[SESSION_DIR_ENV]; + assert.throws(() => createSessionManager(), /PI_SPATIAL_SESSION_DIR/); +}); + +test("precreated symlink and non-directory session children are rejected", () => { + const symlinkName = `pi-session-link-${process.pid}`; + const fileName = `pi-session-file-${process.pid}`; + const symlink = join(process.cwd(), symlinkName); + const file = join(process.cwd(), fileName); + rmSync(symlink, { recursive: true, force: true }); + rmSync(file, { recursive: true, force: true }); + mkdirSync(join(process.cwd(), `pi-session-target-${process.pid}`), { mode: 0o700 }); + try { + const previous = process.env[SESSION_DIR_ENV]; + process.env[SESSION_DIR_ENV] = symlinkName; + // The symlink is created outside the adapter so the real lstat admission path is tested. + symlinkSync(`pi-session-target-${process.pid}`, symlink); + assert.throws(() => createSessionManager(), /real directory/); + writeFileSync(file, "not a directory"); + process.env[SESSION_DIR_ENV] = fileName; + assert.throws(() => createSessionManager(), /real directory/); + if (previous === undefined) delete process.env[SESSION_DIR_ENV]; + else process.env[SESSION_DIR_ENV] = previous; + } finally { + rmSync(symlink, { recursive: true, force: true }); + rmSync(file, { recursive: true, force: true }); + rmSync(join(process.cwd(), `pi-session-target-${process.pid}`), { recursive: true, force: true }); + delete process.env[SESSION_DIR_ENV]; + } +}); + +test("fresh sessions are persisted, distinct, and discoverable through public APIs", () => { + withSessionDirectory(`pi-session-test-${process.pid}`, (directory) => { + const first = createSessionManager(); + const second = createSessionManager(); + assert.equal(first.isPersisted(), true); + assert.equal(second.isPersisted(), true); + assert.ok(first.getSessionFile()); + assert.ok(second.getSessionFile()); + assert.notEqual(first.getSessionFile(), second.getSessionFile()); + assert.equal(first.getSessionDir(), directory); + assert.equal(second.getSessionDir(), directory); + assert.equal(existsSync(first.getSessionFile() ?? ""), false); + assert.equal(existsSync(second.getSessionFile() ?? ""), false); + }); +}); + +test("a persisted manager with a delayed nonexistent file is unavailable", () => { + withSessionDirectory(`pi-session-delayed-${process.pid}`, () => { + const manager = createSessionManager(); + assert.equal(manager.isPersisted(), true); + assert.ok(manager.getSessionFile()); + assert.equal(existsSync(manager.getSessionFile() ?? ""), false); + assert.deepEqual(sessionEvidenceForManager(manager, true), { state: "unavailable", persisted: false }); + }); +}); + +test("system prompt evidence preserves exact unicode bytes with bounded metadata", () => { + withSessionDirectory(`pi-session-prompt-${process.pid}`, () => { + const prompt = "system π\n用户—✅"; + const metadata = retainSystemPromptEvidence(prompt); + const path = join(process.cwd(), metadata.relativePath); + assert.equal(metadata.relativePath, "pi-prompt/system.txt"); + assert.deepEqual(readFileSync(path), Buffer.from(prompt, "utf8")); + assert.equal(metadata.byteCount, Buffer.byteLength(prompt, "utf8")); + assert.match(metadata.sha256, /^[a-f0-9]{64}$/); + assert.equal(statSync(path).mode & 0o777, 0o600); + assert.equal(statSync(join(process.cwd(), "pi-prompt")).mode & 0o777, 0o700); + assert.throws(() => retainSystemPromptEvidence("replacement"), /EEXIST/); + assert.deepEqual(readFileSync(path), Buffer.from(prompt, "utf8")); + }); +}); + +test("native JSONL remains after a model-independent public-API append", () => { + withSessionDirectory(`pi-session-survival-${process.pid}`, (directory) => { + const manager = createSessionManager(); + const user: Message = { role: "user", content: [{ type: "text", text: "hello" }], timestamp: Date.now() }; + const assistant: Message = { + role: "assistant", + content: [{ type: "text", text: "done" }], + api: "openai-responses", + provider: "openai-codex", + model: "gpt-5.6-luna", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + }; + manager.appendMessage(user); + manager.appendMessage(assistant); + const file = manager.getSessionFile(); + assert.ok(file); + assert.equal(existsSync(file), true); + assert.equal(manager.isPersisted(), true); + assert.equal(manager.getEntries().length, 2); + assert.equal(existsSync(file), true); + assert.equal(statSync(file).mode & 0o777, 0o600); + assert.equal(manager.getSessionDir(), directory); + }); +}); + +test("pinned SessionManager.open reopens an unchanged native v3 session", () => { + withSessionDirectory(`pi-session-reopen-${process.pid}`, (directory) => { + const original = createSessionManager(); + const timestamp = 1_700_000_000_000; + const userId = original.appendMessage({ role: "user", content: [{ type: "text", text: "native user" }], timestamp }); + const assistantId = original.appendMessage({ + role: "assistant", + content: [{ type: "text", text: "native assistant" }], + api: "openai-responses", + provider: "openai-codex", + model: "gpt-5.6-luna", + usage: { + input: 1, + output: 2, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 3, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: timestamp + 1, + }); + const thinkingId = original.appendThinkingLevelChange("medium"); + const modelId = original.appendModelChange("openai-codex", "gpt-5.6-luna"); + const customId = original.appendCustomEntry("adapter-test", { stable: true }); + const file = original.getSessionFile(); + assert.ok(file); + const sourceBytes = readFileSync(file); + + // SessionManager writes synchronously and has no separate close operation; + // opening the generated file is the documented handoff lifecycle. + const reopened = SessionManager.open(file, directory); + assert.deepEqual(readFileSync(file), sourceBytes); + assert.equal(reopened.getSessionFile(), file); + assert.equal(reopened.getSessionDir(), directory); + assert.equal(reopened.getHeader()?.type, "session"); + assert.equal(reopened.getHeader()?.version, 3); + assert.equal(reopened.getHeader()?.id, original.getSessionId()); + assert.equal(reopened.getSessionId(), original.getSessionId()); + assert.deepEqual(reopened.getEntries().map((entry) => entry.id), [userId, assistantId, thinkingId, modelId, customId]); + assert.equal(reopened.getEntry(assistantId)?.parentId, userId); + assert.equal(reopened.getLeafId(), customId); + assert.equal(reopened.getLeafEntry()?.id, customId); + assert.equal(reopened.getTree().length, 1); + assert.equal(reopened.getTree()[0]?.children.length, 1); + assert.deepEqual(reopened.getBranch(), reopened.getEntries()); + assert.deepEqual(readFileSync(file), sourceBytes); + }); +}); + +test("pinned Pi CLI exports a synthetic native session without auth or rewriting JSONL", () => { + withSessionDirectory(`pi-session-export-${process.pid}`, (directory) => { + const manager = createSessionManager(); + manager.appendMessage({ role: "user", content: [{ type: "text", text: "hello" }], timestamp: Date.now() }); + manager.appendMessage({ + role: "assistant", content: [{ type: "text", text: "done" }], api: "openai-responses", provider: "openai-codex", model: "gpt-5.6-luna", + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, stopReason: "stop", timestamp: Date.now(), + }); + const file = manager.getSessionFile(); + assert.ok(file); + const before = readFileSync(file); + const html = join(directory, "export.html"); + const command = resolvePinnedPiExportCommand(file, html); + assert.equal(command.executable, process.execPath); + assert.deepEqual(command.args.slice(1), ["--export", file, html]); + assert.equal(command.packageVersion, PINNED_PI_VERSION); + execFileSync(command.executable, command.args, { stdio: "pipe" }); + assert.ok(readFileSync(html).length > 0); + assert.deepEqual(readFileSync(file), before); + }); +}); + +test("pinned package command validates executable and bin metadata", () => { + const command = resolvePinnedPiExportCommand("input.jsonl", "output.html"); + assert.equal(command.executable, process.execPath); + assert.match(command.args[0], /node_modules[\\/]@earendil-works[\\/]pi-coding-agent[\\/]dist[\\/]cli\.js$/); + assert.equal(PINNED_PI_VERSION, "0.80.10"); +}); diff --git a/packages/pi-code-policy-adapter/tsconfig.build.json b/packages/pi-code-policy-adapter/tsconfig.build.json new file mode 100644 index 0000000000..7b4b891a32 --- /dev/null +++ b/packages/pi-code-policy-adapter/tsconfig.build.json @@ -0,0 +1,13 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "sourceMap": true + }, + "include": [ + "src" + ] +} diff --git a/packages/pi-code-policy-adapter/tsconfig.json b/packages/pi-code-policy-adapter/tsconfig.json new file mode 100644 index 0000000000..fd64273095 --- /dev/null +++ b/packages/pi-code-policy-adapter/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "verbatimModuleSyntax": true, + "types": [ + "node" + ] + }, + "include": [ + "src", + "test" + ] +} diff --git a/packages/pi-code-policy-adapter/tsconfig.test.json b/packages/pi-code-policy-adapter/tsconfig.test.json new file mode 100644 index 0000000000..b2c0cfcf0c --- /dev/null +++ b/packages/pi-code-policy-adapter/tsconfig.test.json @@ -0,0 +1,14 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "outDir": "dist-test", + "rootDir": ".", + "declaration": false, + "sourceMap": false + }, + "include": [ + "src", + "test" + ] +} diff --git a/pyproject.toml b/pyproject.toml index 37bc540ae5..18c65e52a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -220,6 +220,14 @@ learning = [ ] agents = [ + # Persistent trusted Python runtime used by standalone CodePolicy evaluation. + "ipykernel>=7.2.0", + "jupyter-client>=8.8.0", + "nbformat>=5.10.4", + "pyzmq>=27.1.0", + # Loopback-only MCP host for standalone CodePolicy evaluation. + "fastapi>=0.115.6", + "uvicorn>=0.34.0", "langchain>=1.2.3,<2", "langchain-core>=1.2.22,<2", "langchain-openai>=1,<2", diff --git a/uv.lock b/uv.lock index 2292f5837e..9b1c614eff 100644 --- a/uv.lock +++ b/uv.lock @@ -1589,15 +1589,21 @@ dependencies = [ [package.optional-dependencies] agents = [ + { name = "fastapi" }, { name = "faster-whisper" }, + { name = "ipykernel" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-huggingface" }, { name = "langchain-ollama" }, { name = "langchain-openai" }, + { name = "nbformat" }, { name = "ollama" }, { name = "openai" }, + { name = "pyzmq" }, { name = "sounddevice" }, + { name = "uvicorn" }, ] all = [ { name = "a750-control", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, @@ -1621,6 +1627,7 @@ all = [ { name = "hydra-core" }, { name = "ipykernel" }, { name = "jinja2" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-huggingface" }, @@ -1631,6 +1638,7 @@ all = [ { name = "matplotlib" }, { name = "moondream" }, { name = "mujoco" }, + { name = "nbformat" }, { name = "ollama" }, { name = "omegaconf" }, { name = "onnxruntime" }, @@ -1647,6 +1655,7 @@ all = [ { name = "pyrealsense2-extended", marker = "sys_platform != 'darwin'" }, { name = "python-multipart" }, { name = "pyyaml" }, + { name = "pyzmq" }, { name = "reportlab" }, { name = "rerun-sdk" }, { name = "roboplan" }, @@ -1681,7 +1690,9 @@ base = [ { name = "faster-whisper" }, { name = "ffmpeg-python" }, { name = "hydra-core" }, + { name = "ipykernel" }, { name = "jinja2" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-huggingface" }, @@ -1689,10 +1700,12 @@ base = [ { name = "langchain-openai" }, { name = "lap" }, { name = "moondream" }, + { name = "nbformat" }, { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, { name = "pillow" }, + { name = "pyzmq" }, { name = "rerun-sdk" }, { name = "sounddevice" }, { name = "soundfile" }, @@ -1784,7 +1797,9 @@ unitree = [ { name = "ffmpeg-python" }, { name = "gtsam-extended" }, { name = "hydra-core" }, + { name = "ipykernel" }, { name = "jinja2" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-huggingface" }, @@ -1792,10 +1807,12 @@ unitree = [ { name = "langchain-openai" }, { name = "lap" }, { name = "moondream" }, + { name = "nbformat" }, { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, { name = "pillow" }, + { name = "pyzmq" }, { name = "rerun-sdk" }, { name = "sounddevice" }, { name = "soundfile" }, @@ -1817,7 +1834,9 @@ unitree-dds = [ { name = "ffmpeg-python" }, { name = "gtsam-extended" }, { name = "hydra-core" }, + { name = "ipykernel" }, { name = "jinja2" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-huggingface" }, @@ -1826,10 +1845,12 @@ unitree-dds = [ { name = "lap" }, { name = "mcap" }, { name = "moondream" }, + { name = "nbformat" }, { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, { name = "pillow" }, + { name = "pyzmq" }, { name = "rerun-sdk" }, { name = "sounddevice" }, { name = "soundfile" }, @@ -2054,6 +2075,7 @@ requires-dist = [ { name = "eclipse-zenoh", specifier = ">=1.9.0,<2.0" }, { name = "edgetam-dimos", marker = "extra == 'misc'" }, { name = "einops", marker = "extra == 'perception'", specifier = ">=0.8.1" }, + { name = "fastapi", marker = "extra == 'agents'", specifier = ">=0.115.6" }, { name = "fastapi", marker = "extra == 'web'", specifier = ">=0.115.6" }, { name = "faster-whisper", marker = "extra == 'agents'", specifier = ">=1.0.0" }, { name = "ffmpeg-python", marker = "extra == 'web'" }, @@ -2064,9 +2086,11 @@ requires-dist = [ { name = "h5py", marker = "extra == 'learning'" }, { name = "hydra-core", marker = "extra == 'perception'", specifier = ">=1.3.0" }, { name = "imagecodecs", specifier = ">=2024.6.1" }, + { name = "ipykernel", marker = "extra == 'agents'", specifier = ">=7.2.0" }, { name = "ipykernel", marker = "extra == 'misc'" }, { name = "ipython" }, { name = "jinja2", marker = "extra == 'web'", specifier = ">=3.1.6" }, + { name = "jupyter-client", marker = "extra == 'agents'", specifier = ">=8.8.0" }, { name = "langchain", marker = "extra == 'agents'", specifier = ">=1.2.3,<2" }, { name = "langchain-core", marker = "extra == 'agents'", specifier = ">=1.2.22,<2" }, { name = "langchain-huggingface", marker = "extra == 'agents'", specifier = ">=1,<2" }, @@ -2081,6 +2105,7 @@ requires-dist = [ { name = "mcap", marker = "extra == 'unitree-dds'", specifier = ">=1.2.0" }, { name = "moondream", marker = "extra == 'perception'" }, { name = "mujoco", marker = "extra == 'sim'", specifier = ">=3.3.4" }, + { name = "nbformat", marker = "extra == 'agents'", specifier = ">=5.10.4" }, { name = "numba", specifier = ">=0.60.0" }, { name = "numpy", specifier = ">=1.26.4" }, { name = "ollama", marker = "extra == 'agents'", specifier = ">=0.6.0" }, @@ -2115,6 +2140,7 @@ requires-dist = [ { name = "python-multipart", marker = "extra == 'misc'", specifier = ">=0.0.27" }, { name = "pyturbojpeg", specifier = "==1.8.2" }, { name = "pyyaml", marker = "extra == 'manipulation'", specifier = ">=6.0" }, + { name = "pyzmq", marker = "extra == 'agents'", specifier = ">=27.1.0" }, { name = "qpsolvers", extras = ["proxqp"], specifier = ">=4.12.0" }, { name = "reactivex" }, { name = "reportlab", marker = "extra == 'apriltag'", specifier = ">=4.5.0" }, @@ -2145,6 +2171,7 @@ requires-dist = [ { name = "unitree-sdk2py-dimos", marker = "extra == 'unitree-dds'", specifier = ">=1.0.2" }, { name = "unitree-webrtc-connect", marker = "extra == 'unitree'", specifier = ">=2.1.2" }, { name = "usd-core", marker = "extra == 'scene'", specifier = ">=23.11" }, + { name = "uvicorn", marker = "extra == 'agents'", specifier = ">=0.34.0" }, { name = "uvicorn", marker = "extra == 'web'", specifier = ">=0.34.0" }, { name = "viser", extras = ["urdf"], marker = "extra == 'manipulation'", specifier = ">=1.0.29" }, { name = "websocket-client", specifier = ">=1.8" }, From a7b0c0fe2c36feba6ed2f851bdf5059025c56517 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:12:59 +0000 Subject: [PATCH 05/15] [autofix.ci] apply automated fixes --- dimos/memory2/store/sqlite.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dimos/memory2/store/sqlite.py b/dimos/memory2/store/sqlite.py index 2c5e3d592b..181533b187 100644 --- a/dimos/memory2/store/sqlite.py +++ b/dimos/memory2/store/sqlite.py @@ -65,9 +65,7 @@ def __init__(self, **kwargs: Any) -> None: if parent: os.makedirs(parent, exist_ok=True) self._registry_conn = self._open_connection() - self._registry = RegistryStore( - conn=self._registry_conn, read_only=self.config.read_only - ) + self._registry = RegistryStore(conn=self._registry_conn, read_only=self.config.read_only) def _open_connection(self) -> sqlite3.Connection: """Open a new WAL-mode connection with sqlite-vec loaded.""" From 405d250eba09a5fc455c2a610c71913fa6823c60 Mon Sep 17 00:00:00 2001 From: cc Date: Wed, 5 Aug 2026 15:46:41 -0700 Subject: [PATCH 06/15] refactor: simplify frozen agent evaluation --- .github/workflows/ci.yml | 13 +- dimos/agents/code_policy_core.py | 653 ++++-------------- dimos/agents/code_policy_server.py | 307 ++------ dimos/agents/test_code_policy_core.py | 75 +- dimos/agents/test_code_policy_server.py | 83 +-- dimos/benchmark/agent_eval/artifacts.py | 78 --- dimos/benchmark/agent_eval/auth.py | 25 - dimos/benchmark/agent_eval/case.py | 232 ------- dimos/benchmark/agent_eval/engine.py | 262 ------- dimos/benchmark/agent_eval/interfaces.py | 106 --- dimos/benchmark/agent_eval/json.py | 29 - dimos/benchmark/agent_eval/models.py | 107 +++ dimos/benchmark/agent_eval/pi.py | 56 -- dimos/benchmark/agent_eval/pi_adapter.py | 302 -------- dimos/benchmark/agent_eval/pi_process.py | 523 ++++---------- dimos/benchmark/agent_eval/single_case.py | 342 ++++----- dimos/benchmark/agent_eval/store.py | 285 -------- dimos/benchmark/agent_eval/test_case.py | 148 ---- dimos/benchmark/agent_eval/test_engine.py | 281 -------- .../agent_eval/test_import_boundaries.py | 43 -- dimos/benchmark/agent_eval/test_json.py | 26 - dimos/benchmark/agent_eval/test_pi_adapter.py | 198 ------ dimos/benchmark/agent_eval/test_pi_process.py | 287 +++----- .../benchmark/agent_eval/test_single_case.py | 166 ++--- dimos/benchmark/agent_eval/test_store.py | 160 ----- .../README.md | 7 +- .../case.json | 22 + .../private/oracle.json | 0 .../case.json | 31 - dimos/benchmark/short_horizon_qa/eval.py | 387 +---------- dimos/benchmark/short_horizon_qa/models.py | 4 +- dimos/benchmark/short_horizon_qa/prepare.py | 30 +- dimos/benchmark/short_horizon_qa/service.py | 33 +- dimos/benchmark/short_horizon_qa/test_eval.py | 194 +----- .../short_horizon_qa/test_hongkong_eval.py | 78 +-- .../short_horizon_qa/test_prepare.py | 15 +- dimos/cli/eval.py | 69 +- dimos/cli/test_eval.py | 215 ++---- dimos/memory2/observationstore/sqlite.py | 5 +- dimos/memory2/store/frozen.py | 3 +- dimos/memory2/store/sqlite.py | 7 - dimos/memory2/store/test_frozen.py | 4 +- dimos/memory2/stream.py | 25 +- dimos/memory2/type/filter.py | 10 - docs/capabilities/agents/evaluation.md | 119 ++-- docs/development/testing.md | 13 +- .../changes/extract-frozen-qa-eval/design.md | 142 ++-- .../changes/extract-frozen-qa-eval/docs.md | 49 +- .../extract-frozen-qa-eval/proposal.md | 41 +- .../specs/frozen-agent-evaluation/spec.md | 170 ++--- .../specs/frozen-memory-views/spec.md | 85 +-- .../standalone-code-policy-runtime/spec.md | 125 ++-- .../changes/extract-frozen-qa-eval/tasks.md | 95 +-- packages/pi-code-policy-adapter/README.md | 16 - .../src/code-policy-main.ts | 241 ------- .../src/code-policy-protocol.ts | 122 ---- .../src/code-policy-session.ts | 74 -- .../pi-code-policy-adapter/src/session.ts | 352 ---------- .../test/code-policy-main.test.ts | 123 ---- .../test/code-policy-protocol.test.ts | 31 - .../test/code-policy-session.test.ts | 40 -- .../test/session.test.ts | 229 ------ .../.gitignore | 0 packages/pi-code-policy-extension/README.md | 7 + .../package-lock.json | 93 ++- .../package.json | 10 +- .../src/python-exec.ts | 83 +++ .../test/python-exec.test.ts | 43 ++ .../tsconfig.build.json | 0 .../tsconfig.json | 0 .../tsconfig.test.json | 0 pyproject.toml | 7 +- uv.lock | 139 +++- 73 files changed, 1712 insertions(+), 6663 deletions(-) delete mode 100644 dimos/benchmark/agent_eval/artifacts.py delete mode 100644 dimos/benchmark/agent_eval/auth.py delete mode 100644 dimos/benchmark/agent_eval/case.py delete mode 100644 dimos/benchmark/agent_eval/engine.py delete mode 100644 dimos/benchmark/agent_eval/interfaces.py delete mode 100644 dimos/benchmark/agent_eval/json.py create mode 100644 dimos/benchmark/agent_eval/models.py delete mode 100644 dimos/benchmark/agent_eval/pi.py delete mode 100644 dimos/benchmark/agent_eval/pi_adapter.py delete mode 100644 dimos/benchmark/agent_eval/store.py delete mode 100644 dimos/benchmark/agent_eval/test_case.py delete mode 100644 dimos/benchmark/agent_eval/test_engine.py delete mode 100644 dimos/benchmark/agent_eval/test_import_boundaries.py delete mode 100644 dimos/benchmark/agent_eval/test_json.py delete mode 100644 dimos/benchmark/agent_eval/test_pi_adapter.py delete mode 100644 dimos/benchmark/agent_eval/test_store.py rename dimos/benchmark/short_horizon_qa/cases/{go2_hongkong_office-room-count-smoke => demo_go2_hongkong_office-room-count-smoke}/README.md (59%) create mode 100644 dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/case.json rename dimos/benchmark/short_horizon_qa/cases/{go2_hongkong_office-room-count-smoke => demo_go2_hongkong_office-room-count-smoke}/private/oracle.json (100%) delete mode 100644 dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/case.json delete mode 100644 packages/pi-code-policy-adapter/README.md delete mode 100644 packages/pi-code-policy-adapter/src/code-policy-main.ts delete mode 100644 packages/pi-code-policy-adapter/src/code-policy-protocol.ts delete mode 100644 packages/pi-code-policy-adapter/src/code-policy-session.ts delete mode 100644 packages/pi-code-policy-adapter/src/session.ts delete mode 100644 packages/pi-code-policy-adapter/test/code-policy-main.test.ts delete mode 100644 packages/pi-code-policy-adapter/test/code-policy-protocol.test.ts delete mode 100644 packages/pi-code-policy-adapter/test/code-policy-session.test.ts delete mode 100644 packages/pi-code-policy-adapter/test/session.test.ts rename packages/{pi-code-policy-adapter => pi-code-policy-extension}/.gitignore (100%) create mode 100644 packages/pi-code-policy-extension/README.md rename packages/{pi-code-policy-adapter => pi-code-policy-extension}/package-lock.json (95%) rename packages/{pi-code-policy-adapter => pi-code-policy-extension}/package.json (74%) create mode 100644 packages/pi-code-policy-extension/src/python-exec.ts create mode 100644 packages/pi-code-policy-extension/test/python-exec.test.ts rename packages/{pi-code-policy-adapter => pi-code-policy-extension}/tsconfig.build.json (100%) rename packages/{pi-code-policy-adapter => pi-code-policy-extension}/tsconfig.json (100%) rename packages/{pi-code-policy-adapter => pi-code-policy-extension}/tsconfig.test.json (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aeeb558ca3..d32eb0cfa2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,7 +70,7 @@ jobs: - name: Run pre-commit uses: pre-commit/action@v3.0.1 - pi-code-policy-adapter: + pi-code-policy-extension: timeout-minutes: 10 runs-on: ubuntu-latest permissions: @@ -82,11 +82,11 @@ jobs: with: node-version: '22.19.0' cache: npm - cache-dependency-path: packages/pi-code-policy-adapter/package-lock.json - - name: Install adapter dependencies - run: npm ci --prefix packages/pi-code-policy-adapter - - name: Test adapter - run: npm test --prefix packages/pi-code-policy-adapter + cache-dependency-path: packages/pi-code-policy-extension/package-lock.json + - name: Install extension dependencies + run: npm ci --prefix packages/pi-code-policy-extension + - name: Test extension + run: npm test --prefix packages/pi-code-policy-extension rust: timeout-minutes: 20 @@ -995,6 +995,7 @@ jobs: needs: - lint + - pi-code-policy-extension - rust - cpp - md-babel diff --git a/dimos/agents/code_policy_core.py b/dimos/agents/code_policy_core.py index 4265e579f9..2ef5dccb9f 100644 --- a/dimos/agents/code_policy_core.py +++ b/dimos/agents/code_policy_core.py @@ -12,63 +12,47 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Module-independent persistent Python policy session.""" +"""Persistent, module-independent Python session for trusted CodePolicy agents.""" from __future__ import annotations -import base64 -from datetime import UTC, datetime import os +import queue import re import threading import time -from typing import Annotated, Any, Literal -from uuid import uuid4 +from typing import Any, Literal -from pydantic import BaseModel, ConfigDict, Field, model_validator - -from dimos.utils.logging_config import setup_logger - -logger = setup_logger() +from pydantic import BaseModel, ConfigDict, Field MAX_EXECUTION_TIMEOUT_S = 110.0 -DEFAULT_STARTUP_TIMEOUT_S = 10.0 -DEFAULT_INTERRUPT_GRACE_S = 2.0 DEFAULT_OUTPUT_LIMIT = 32_000 _RECORDING_PATH_ENV = "DIMOS_CODE_POLICY_RECORDING_PATH" _DERIVED_RECORDING_PATH_ENV = "DIMOS_CODE_POLICY_DERIVED_RECORDING_PATH" _MEMORY_CUTOFF_ENV = "DIMOS_CODE_POLICY_MEMORY_CUTOFF" _CONNECT_APP_ENV = "DIMOS_CODE_POLICY_CONNECT_APP" -_TRUNCATION_MARKER = "\n... [output truncated]" _ANSI_ESCAPE_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") +_TRUNCATION_MARKER = "\n... [output truncated]" +_CREDENTIAL_NAME_RE = re.compile( + r"(?:API_?KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|AUTH|OPENAI|ANTHROPIC|AWS_|AZURE_)", + re.IGNORECASE, +) -SessionId = Annotated[str, Field(pattern=r"^code_policy_session_[0-9a-f]{32}$")] -ExecutionId = Annotated[str, Field(pattern=r"^code_policy_call_[0-9a-f]{32}$")] -ExecutionStatus = Literal[ - "busy", - "completed", - "execution-failed", - "invalid-request", - "kernel-start-failed", - "module-stopped", - "python-error", - "timed-out", -] -ObserverAvailability = Literal["ready", "replaced", "stopped", "unavailable"] - - -class _EvidenceModel(BaseModel): - model_config = ConfigDict(extra="forbid", frozen=True, strict=True) +class FrozenMemoryEnvironment(BaseModel): + """A source and derived Memory2 recording pinned at an inclusive cutoff.""" -class FrozenMemoryEnvironment(_EvidenceModel): + model_config = ConfigDict(extra="forbid", frozen=True) kind: Literal["frozen_memory"] = "frozen_memory" recording_path: str = Field(min_length=1) derived_recording_path: str = Field(min_length=1) memory_cutoff_timestamp: float -class LiveDimosEnvironment(_EvidenceModel): +class LiveDimosEnvironment(BaseModel): + """A live DimOS environment with read-only memory and an attached app.""" + + model_config = ConfigDict(extra="forbid", frozen=True) kind: Literal["live_dimos"] = "live_dimos" recording_path: str = Field(min_length=1) @@ -78,111 +62,48 @@ class LiveDimosEnvironment(_EvidenceModel): class CodePolicySessionConfig(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) - environment: CodePolicyEnvironment - output_limit: int = DEFAULT_OUTPUT_LIMIT - startup_timeout_s: float = DEFAULT_STARTUP_TIMEOUT_S - interrupt_grace_s: float = DEFAULT_INTERRUPT_GRACE_S - - @model_validator(mode="after") - def limits_are_valid(self) -> CodePolicySessionConfig: - if self.output_limit < 0: - raise ValueError("output_limit must be non-negative") - if self.startup_timeout_s <= 0 or self.interrupt_grace_s <= 0: - raise ValueError("CodePolicy timeouts must be positive") - return self - - -class CodePolicySessionReceipt(_EvidenceModel): - session_id: SessionId - reset_at: datetime - previous_session_id: SessionId | None - - -class CodePolicyExecutionRecord(_EvidenceModel): - execution_id: ExecutionId - session_id: SessionId - source: str - requested_timeout_s: float - started_at: datetime - finished_at: datetime - monotonic_duration_s: Annotated[float, Field(ge=0)] - status: ExecutionStatus - jupyter_message_id: str | None - jupyter_execution_count: int | None - output: str - transcript: str - interrupt_attempted: bool - interrupt_recovered: bool - kernel_restarted: bool - namespace_preserved: bool - remote_work_may_continue: bool - - -class CodePolicyObserverDescriptor(_EvidenceModel): - transport: Literal["tcp", "ipc"] - ip: str - iopub_port: Annotated[int, Field(gt=0)] - signature_scheme: str - key_base64: str - code_policy_session_id: SessionId - jupyter_client_session_id: str - kernel_generation: Annotated[int, Field(gt=0)] - - -class CodePolicyObserverState(_EvidenceModel): - availability: ObserverAvailability - code_policy_session_id: SessionId - kernel_generation: Annotated[int, Field(ge=0)] - descriptor: CodePolicyObserverDescriptor | None - - -class CodePolicyObserverProbeReceipt(_EvidenceModel): - message_id: str - code_policy_session_id: SessionId - kernel_generation: Annotated[int, Field(gt=0)] - - -class _BoundedTextOutput: + output_limit: int = Field(default=DEFAULT_OUTPUT_LIMIT, ge=0) + startup_timeout_s: float = Field(default=10.0, gt=0) + interrupt_grace_s: float = Field(default=2.0, gt=0) + + +class _BoundedOutput: def __init__(self, limit: int) -> None: - self._limit = max(0, limit) - self._parts: list[str] = [] - self._length = 0 - self._truncated = False + self.limit = limit + self.parts: list[str] = [] + self.length = 0 + self.truncated = False def __call__(self, message: dict[str, Any]) -> None: message_type = message.get("header", {}).get("msg_type") content = message.get("content", {}) + value = "" if message_type == "stream": - self._append(str(content.get("text", ""))) + value = str(content.get("text", "")) elif message_type in {"execute_result", "display_data"}: - text = content.get("data", {}).get("text/plain") - if text is not None: - self._append(str(text)) + value = str(content.get("data", {}).get("text/plain", "")) elif message_type == "error": - traceback = content.get("traceback") - if isinstance(traceback, list): - self._append("\n".join(str(line) for line in traceback)) - else: - self._append(f"{content.get('ename', 'Error')}: {content.get('evalue', '')}") - - def text(self) -> str: - return "".join(self._parts) + traceback = content.get("traceback", []) + value = "\n".join(str(line) for line in traceback) + self._append(_ANSI_ESCAPE_RE.sub("", value)) def _append(self, value: str) -> None: - if not value or self._truncated: + if not value or self.truncated: return - value = _ANSI_ESCAPE_RE.sub("", value) - remaining = self._limit - self._length + remaining = self.limit - self.length if len(value) <= remaining: - self._parts.append(value) - self._length += len(value) + self.parts.append(value) + self.length += len(value) return marker = _TRUNCATION_MARKER[:remaining] content_limit = max(0, remaining - len(marker)) - self._parts.append(value[:content_limit] + marker) - self._length = self._limit - self._truncated = True + self.parts.append(value[:content_limit] + marker) + self.length = self.limit + self.truncated = True + + def text(self) -> str: + return "".join(self.parts) def _load_kernel_manager() -> type[Any]: @@ -190,7 +111,7 @@ def _load_kernel_manager() -> type[Any]: from jupyter_client.manager import KernelManager except ImportError as exc: raise RuntimeError( - "Code-policy execution requires the agents extra: uv sync --extra agents" + "CodePolicy requires ipykernel and jupyter-client; install the agents extra" ) from exc return KernelManager @@ -205,10 +126,8 @@ def _bootstrap_source() -> str: memory = _SqliteStore( path=_os.environ[{_RECORDING_PATH_ENV!r}], must_exist=True, read_only=True ) - memory.start() else: from dimos.memory2.store.frozen import FrozenMemoryStore as _FrozenMemoryStore - _source = _SqliteStore( path=_os.environ[{_RECORDING_PATH_ENV!r}], must_exist=True, read_only=True ) @@ -218,12 +137,11 @@ def _bootstrap_source() -> str: memory = _FrozenMemoryStore( source=_source, derived=_derived, through_timestamp=float(_cutoff) ) - memory.start() del _FrozenMemoryStore, _source, _derived -if _os.environ.get({_CONNECT_APP_ENV!r}, "1") == "1": +memory.start() +if _os.environ.get({_CONNECT_APP_ENV!r}) == "1": from dimos.porcelain.dimos import Dimos as _Dimos - app = _Dimos.connect() del _Dimos @@ -231,81 +149,56 @@ def _bootstrap_source() -> str: """ +def _kernel_environment(environment: CodePolicyEnvironment) -> dict[str, str]: + """Build a useful kernel environment without forwarding host credentials.""" + result = { + name: value for name, value in os.environ.items() if not _CREDENTIAL_NAME_RE.search(name) + } + result[_RECORDING_PATH_ENV] = environment.recording_path + if isinstance(environment, FrozenMemoryEnvironment): + result[_CONNECT_APP_ENV] = "0" + result[_DERIVED_RECORDING_PATH_ENV] = environment.derived_recording_path + result[_MEMORY_CUTOFF_ENV] = str(environment.memory_cutoff_timestamp) + else: + result[_CONNECT_APP_ENV] = "1" + result.pop(_DERIVED_RECORDING_PATH_ENV, None) + result.pop(_MEMORY_CUTOFF_ENV, None) + return result + + class CodePolicySession: - """Execute trusted agent-authored Python in one persistent kernel.""" + """Execute trusted Python serially in one persistent Jupyter kernel.""" def __init__(self, config: CodePolicySessionConfig) -> None: self.config = config + self.execution_count = 0 + self.execution_duration_s = 0.0 self._execution_lock = threading.Lock() - self._records_lock = threading.Lock() self._kernel_lock = threading.RLock() - self._kernel_manager: Any = None - self._kernel_client: Any = None - self._kernel_generation = 0 - self._session_id: str = _new_session_id() - self._session_reset_at = _utc_now() + self._manager: Any = None + self._client: Any = None self._stopped = True - self._execution_records: list[CodePolicyExecutionRecord] = [] def start(self) -> None: self._stopped = False def python_exec(self, code: str, timeout_s: float = MAX_EXECUTION_TIMEOUT_S) -> str: - started_at = _utc_now() - started_monotonic = time.monotonic() if self._stopped: - transcript = "Code Policy Module stopped" - self._record( - code, - timeout_s, - started_at, - started_monotonic, - status="module-stopped", - transcript=transcript, - ) - return transcript + return "CodePolicy session is stopped" + if not code: + return "python_exec code must be non-empty" if not 0 < timeout_s <= MAX_EXECUTION_TIMEOUT_S: - transcript = ( - f"Invalid timeout_s={timeout_s!r}; expected a value in " - f"(0, {MAX_EXECUTION_TIMEOUT_S:g}]" - ) - self._record( - code, - timeout_s, - started_at, - started_monotonic, - status="invalid-request", - transcript=transcript, - ) - return transcript + return f"timeout_s must be in (0, {MAX_EXECUTION_TIMEOUT_S:g}]" if not self._execution_lock.acquire(blocking=False): - transcript = "Code Policy Module busy: another python_exec call is active" - self._record( - code, - timeout_s, - started_at, - started_monotonic, - status="busy", - transcript=transcript, - ) - return transcript - logger.info("Code policy execution started", source=code, timeout_s=timeout_s) + return "CodePolicy session is busy" + started = time.monotonic() + self.execution_count += 1 try: try: client = self._ensure_kernel() except Exception as exc: - logger.exception("Code policy kernel failed to start") - transcript = f"Code policy kernel failed to start: {type(exc).__name__}: {exc}" - self._record( - code, - timeout_s, - started_at, - started_monotonic, - status="kernel-start-failed", - transcript=transcript, - ) - return transcript - output = _BoundedTextOutput(self.config.output_limit) + return f"CodePolicy kernel failed to start: {type(exc).__name__}: {exc}" + output = _BoundedOutput(self.config.output_limit) try: reply = client.execute_interactive( code, @@ -314,353 +207,105 @@ def python_exec(self, code: str, timeout_s: float = MAX_EXECUTION_TIMEOUT_S) -> store_history=True, timeout=timeout_s, ) - except TimeoutError: - interrupt_recovered, kernel_restarted = self._recover_from_timeout() - if interrupt_recovered: - transcript = ( - f"Execution timed out after {timeout_s:.1f}s and was interrupted. " - "The Python namespace was preserved. Remote RPC work may still be " - "running; it was not cancelled." - ) - else: - transcript = ( - f"Execution timed out after {timeout_s:.1f}s. The kernel did not " - "recover from interruption and was restarted; the Python namespace " - "was reset. Remote RPC work may still be running; it was not cancelled." - ) - self._record( - code, - timeout_s, - started_at, - started_monotonic, - status="timed-out", - output=output.text(), - transcript=transcript, - interrupt_attempted=True, - interrupt_recovered=interrupt_recovered, - kernel_restarted=kernel_restarted, - namespace_preserved=interrupt_recovered, - remote_work_may_continue=True, + except (TimeoutError, queue.Empty): + if self._interrupt_and_recover(): + return f"Execution timed out after {timeout_s:.1f}s and was interrupted" + return ( + f"Execution timed out after {timeout_s:.1f}s; " + "the kernel was restarted and its namespace was reset" ) - return transcript except Exception as exc: - self._shutdown_kernel(reason=type(exc).__name__) - logger.exception("Code policy execution failed") - transcript = ( - f"Code policy execution failed: {type(exc).__name__}: {exc}. " - "The Python namespace was reset." - ) - self._record( - code, - timeout_s, - started_at, - started_monotonic, - status="execution-failed", - output=output.text(), - transcript=transcript, - kernel_restarted=True, - ) - return transcript - duration_s = time.monotonic() - started_monotonic - transcript = _format_reply(reply, output.text(), duration_s) + self._shutdown_kernel() + return f"CodePolicy execution failed: {type(exc).__name__}: {exc}" content = reply.get("content", {}) - status: ExecutionStatus = ( - "completed" if content.get("status") == "ok" else "python-error" - ) - self._record( - code, - timeout_s, - started_at, - started_monotonic, - status=status, - output=output.text(), - transcript=transcript, - jupyter_message_id=reply.get("parent_header", {}).get("msg_id"), - jupyter_execution_count=content.get("execution_count"), - ) - return transcript - finally: - self._execution_lock.release() - - def reset_session(self) -> CodePolicySessionReceipt: - if not self._execution_lock.acquire(blocking=False): - raise RuntimeError("cannot reset code policy while python_exec is active") - try: - previous_session_id = self._session_id - self._shutdown_kernel(reason="session reset") - self._session_id = _new_session_id() - self._session_reset_at = _utc_now() - return CodePolicySessionReceipt( - session_id=self._session_id, - reset_at=self._session_reset_at, - previous_session_id=previous_session_id, - ) - finally: - self._execution_lock.release() - - def get_session_receipt(self) -> CodePolicySessionReceipt: - return CodePolicySessionReceipt( - session_id=self._session_id, - reset_at=self._session_reset_at, - previous_session_id=None, - ) - - def get_execution_records( - self, session_id: str | None = None - ) -> tuple[CodePolicyExecutionRecord, ...]: - with self._records_lock: - records = tuple(self._execution_records) - if session_id is None: - return records - return tuple(record for record in records if record.session_id == session_id) - - def prepare_observer(self) -> CodePolicyObserverState: - if self._stopped: - return self._observer_state("stopped") - self._ensure_kernel() - return self._observer_state("ready") - - def get_observer_state(self, known_generation: int | None = None) -> CodePolicyObserverState: - if self._stopped: - return self._observer_state("stopped") - with self._kernel_lock: - manager = self._kernel_manager - client = self._kernel_client - generation = self._kernel_generation - is_ready = manager is not None and client is not None and bool(manager.is_alive()) - if not is_ready: - return self._observer_state("unavailable") - availability: ObserverAvailability = ( - "replaced" - if known_generation is not None and known_generation != generation - else "ready" - ) - return self._observer_state(availability) - - def issue_observer_probe(self, kernel_generation: int) -> CodePolicyObserverProbeReceipt: - if self._stopped: - raise RuntimeError("code policy session is stopped") - if not self._execution_lock.acquire(blocking=False): - raise RuntimeError("cannot probe while python_exec is active") - try: - client = self._ensure_kernel() - with self._kernel_lock: - if kernel_generation != self._kernel_generation: - raise RuntimeError( - "code policy kernel generation changed before readiness probe" - ) - message_id = client.execute( - "None", silent=True, store_history=False, allow_stdin=False - ) - return CodePolicyObserverProbeReceipt( - message_id=message_id, - code_policy_session_id=self._session_id, - kernel_generation=self._kernel_generation, - ) + body = output.text().rstrip() + if not body and content.get("status") != "ok": + body = f"{content.get('ename', 'Error')}: {content.get('evalue', '')}" + if not body: + body = "(completed)" + state = "completed" if content.get("status") == "ok" else "failed" + return f"In [{content.get('execution_count', '?')}] {state}\n\n{body}" finally: + self.execution_duration_s += time.monotonic() - started self._execution_lock.release() - def interrupt_active(self) -> bool: - if self._execution_lock.acquire(blocking=False): - self._execution_lock.release() - return False - manager = self._kernel_manager - if manager is None or not manager.is_alive(): - return False - manager.interrupt_kernel() - return True - def stop(self) -> None: self._stopped = True - self._shutdown_kernel(reason="session stop") - - def _record( - self, - source: str, - timeout_s: float, - started_at: datetime, - started_monotonic: float, - *, - status: ExecutionStatus, - output: str = "", - transcript: str, - jupyter_message_id: str | None = None, - jupyter_execution_count: int | None = None, - interrupt_attempted: bool = False, - interrupt_recovered: bool = False, - kernel_restarted: bool = False, - namespace_preserved: bool = True, - remote_work_may_continue: bool = False, - ) -> None: - record = CodePolicyExecutionRecord( - execution_id=f"code_policy_call_{uuid4().hex}", - session_id=self._session_id, - source=source, - requested_timeout_s=timeout_s, - started_at=started_at, - finished_at=_utc_now(), - monotonic_duration_s=max(0.0, time.monotonic() - started_monotonic), - status=status, - jupyter_message_id=jupyter_message_id, - jupyter_execution_count=jupyter_execution_count, - output=output, - transcript=transcript, - interrupt_attempted=interrupt_attempted, - interrupt_recovered=interrupt_recovered, - kernel_restarted=kernel_restarted, - namespace_preserved=namespace_preserved, - remote_work_may_continue=remote_work_may_continue, - ) - with self._records_lock: - self._execution_records.append(record) + self._shutdown_kernel() def _ensure_kernel(self) -> Any: with self._kernel_lock: - manager = self._kernel_manager - client = self._kernel_client - if manager is not None and client is not None and manager.is_alive(): - return client - self._shutdown_kernel(reason="kernel unavailable") - manager_type = _load_kernel_manager() - manager = manager_type(kernel_name="python3") + if self._manager is not None and self._client is not None and self._manager.is_alive(): + return self._client + self._shutdown_kernel() + manager = _load_kernel_manager()(kernel_name="python3") client = None try: - env = os.environ.copy() - environment = self.config.environment - env[_RECORDING_PATH_ENV] = environment.recording_path - if isinstance(environment, FrozenMemoryEnvironment): - env[_CONNECT_APP_ENV] = "0" - env[_DERIVED_RECORDING_PATH_ENV] = environment.derived_recording_path - env[_MEMORY_CUTOFF_ENV] = str(environment.memory_cutoff_timestamp) - else: - env[_CONNECT_APP_ENV] = "1" - env.pop(_DERIVED_RECORDING_PATH_ENV, None) - env.pop(_MEMORY_CUTOFF_ENV, None) - manager.start_kernel(env=env) + manager.start_kernel(env=_kernel_environment(self.config.environment)) client = manager.client() client.start_channels() client.wait_for_ready(timeout=self.config.startup_timeout_s) - self._bootstrap(client) + reply = client.execute_interactive( + _bootstrap_source(), + allow_stdin=False, + output_hook=lambda _message: None, + silent=True, + store_history=False, + timeout=self.config.startup_timeout_s, + ) + if reply.get("content", {}).get("status") != "ok": + content = reply.get("content", {}) + raise RuntimeError( + f"{content.get('ename', 'KernelBootstrapError')}: " + f"{content.get('evalue', 'bootstrap failed')}" + ) except Exception: if client is not None: client.stop_channels() try: - manager.shutdown_kernel(now=True) + manager.shutdown_kernel(now=False) except Exception: - logger.exception("Failed to stop an uninitialized code policy kernel") + pass raise - self._kernel_manager = manager - self._kernel_client = client - self._kernel_generation += 1 - logger.info("Code policy kernel started", environment=self.config.environment.kind) - return client - - def _bootstrap(self, client: Any) -> None: - reply = client.execute_interactive( - _bootstrap_source(), - allow_stdin=False, - output_hook=lambda _message: None, - silent=True, - store_history=False, - timeout=self.config.startup_timeout_s, - ) - content = reply.get("content", {}) - if content.get("status") != "ok": - name = content.get("ename", "KernelBootstrapError") - value = content.get("evalue", "unknown bootstrap failure") - raise RuntimeError(f"{name}: {value}") - - def _recover_from_timeout(self) -> tuple[bool, bool]: - manager = self._kernel_manager - client = self._kernel_client + self._manager = manager + self._client = client + return client + + def _interrupt_and_recover(self) -> bool: + manager, client = self._manager, self._client if manager is None or client is None: - return False, False + return False try: manager.interrupt_kernel() client.wait_for_ready(timeout=self.config.interrupt_grace_s) - logger.info("Code policy kernel recovered after interrupt") - return True, False - except Exception: - logger.warning("Code policy kernel did not recover after interrupt") - try: - manager.restart_kernel(now=True) - client.wait_for_ready(timeout=self.config.startup_timeout_s) - self._bootstrap(client) - with self._kernel_lock: - self._kernel_generation += 1 - logger.info("Code policy kernel restarted after failed interrupt") - return False, True + return True except Exception: - logger.exception("Code policy kernel failed to restart") - self._shutdown_kernel(reason="restart failure") - return False, True - - def _shutdown_kernel(self, *, reason: str) -> None: - with self._kernel_lock: - manager = self._kernel_manager - client = self._kernel_client - self._kernel_manager = None - self._kernel_client = None - if manager is None and client is None: - return - try: - if manager is not None: - manager.shutdown_kernel(now=True) - except Exception: - logger.exception("Failed to stop code policy kernel") - finally: - if client is not None: - client.stop_channels() - logger.info("Code policy kernel stopped", reason=reason) + try: + manager.restart_kernel(now=True) + client.wait_for_ready(timeout=self.config.startup_timeout_s) + reply = client.execute_interactive( + _bootstrap_source(), + allow_stdin=False, + output_hook=lambda _message: None, + silent=True, + store_history=False, + timeout=self.config.startup_timeout_s, + ) + if reply.get("content", {}).get("status") != "ok": + raise RuntimeError("bootstrap failed after kernel restart") + except Exception: + self._shutdown_kernel() + return False - def _observer_state(self, availability: ObserverAvailability) -> CodePolicyObserverState: - descriptor: CodePolicyObserverDescriptor | None = None + def _shutdown_kernel(self) -> None: with self._kernel_lock: - generation = self._kernel_generation - manager = self._kernel_manager - client = self._kernel_client - if availability in {"ready", "replaced"}: - if manager is None or client is None or not manager.is_alive(): - availability = "unavailable" - else: - connection = manager.get_connection_info() - key = connection["key"] - if isinstance(key, str): - key = key.encode() - descriptor = CodePolicyObserverDescriptor( - transport=connection["transport"], - ip=connection["ip"], - iopub_port=connection["iopub_port"], - signature_scheme=connection["signature_scheme"], - key_base64=base64.b64encode(key).decode("ascii"), - code_policy_session_id=self._session_id, - jupyter_client_session_id=client.session.session, - kernel_generation=generation, - ) - return CodePolicyObserverState( - availability=availability, - code_policy_session_id=self._session_id, - kernel_generation=generation, - descriptor=descriptor, - ) - - -def _new_session_id() -> str: - return f"code_policy_session_{uuid4().hex}" - - -def _utc_now() -> datetime: - return datetime.now(UTC) - - -def _format_reply(reply: dict[str, Any], output: str, duration_s: float) -> str: - content = reply.get("content", {}) - status = content.get("status", "unknown") - execution_count = content.get("execution_count", "?") - state = "completed" if status == "ok" else "failed" - body = output.rstrip() - if not body and status != "ok": - body = f"{content.get('ename', 'Error')}: {content.get('evalue', '')}".rstrip() - if not body: - body = "(completed)" - return f"In [{execution_count}] {state} in {duration_s:.2f}s\n\n{body}" + manager, client = self._manager, self._client + self._manager = None + self._client = None + if manager is not None: + try: + manager.shutdown_kernel(now=False) + except Exception: + pass + if client is not None: + client.stop_channels() diff --git a/dimos/agents/code_policy_server.py b/dimos/agents/code_policy_server.py index 0a3db174c4..5cf0704e7d 100644 --- a/dimos/agents/code_policy_server.py +++ b/dimos/agents/code_policy_server.py @@ -12,117 +12,102 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Standalone one-tool MCP host for :class:`CodePolicySession`.""" +"""In-process, one-tool MCP server for a :class:`CodePolicySession`.""" from __future__ import annotations -import argparse import asyncio import socket -import subprocess -import sys import threading -from typing import Any +import time -from fastapi import FastAPI -from fastapi.responses import JSONResponse -import requests -from starlette.requests import Request +from mcp.server.mcpserver import MCPServer import uvicorn from dimos.agents.code_policy_core import ( MAX_EXECUTION_TIMEOUT_S, CodePolicySession, CodePolicySessionConfig, - FrozenMemoryEnvironment, - LiveDimosEnvironment, ) -from dimos.agents.mcp.mcp_adapter import McpAdapter -PYTHON_EXEC_DESCRIPTION = """Execute one synchronous Python program in the persistent policy session. +PYTHON_EXEC_DESCRIPTION = """Execute Python in a persistent trusted, unsandboxed session. -The trusted, unsandboxed session preloads `memory` for observations and, in a live -environment, `app` for deployed DimOS RPCs. Imports, functions, variables, and -mutations persist until the host resets the session. +The frozen evaluation session exposes read-only `memory`. Imports, functions, and +variables persist between calls. Use this tool to inspect the recording and compute +the answer; do not guess from the prompt. """ -PYTHON_EXEC_TOOL = { - "name": "python_exec", - "description": PYTHON_EXEC_DESCRIPTION, - "inputSchema": { - "type": "object", - "properties": { - "code": {"type": "string"}, - "timeout_s": {"type": "number", "default": MAX_EXECUTION_TIMEOUT_S}, - }, - "required": ["code"], - "additionalProperties": False, - }, -} - -class StandaloneCodePolicyServer: - """Own a CodePolicy session and serve it directly over MCP.""" +class CodePolicyMcpServer: + """Own the CodePolicy session and an official MCP HTTP server in one process.""" def __init__( self, config: CodePolicySessionConfig, *, host: str = "127.0.0.1", - port: int = 0, ) -> None: - self.config = config self.host = host - self.port = port + self.port = 0 self.session = CodePolicySession(config) - self.app = FastAPI() + self.mcp = MCPServer(name="dimos-code-policy", version="1.0.0") + + @self.mcp.tool( + name="python_exec", + description=PYTHON_EXEC_DESCRIPTION, + structured_output=False, + ) + async def python_exec(code: str, timeout_s: float = MAX_EXECUTION_TIMEOUT_S) -> str: + return await asyncio.to_thread(self.session.python_exec, code, timeout_s) + + self.app = self.mcp.streamable_http_app( + streamable_http_path="/mcp", + json_response=True, + stateless_http=True, + host=host, + ) self._server: uvicorn.Server | None = None self._thread: threading.Thread | None = None self._socket: socket.socket | None = None - self.shutdown_requested = threading.Event() - self._install_routes() @property def mcp_url(self) -> str: - if self.port <= 0: - raise RuntimeError("standalone CodePolicy server has not started") + if self.port == 0: + raise RuntimeError("CodePolicy MCP server is not running") return f"http://{self.host}:{self.port}/mcp" - @property - def control_url(self) -> str: - if self.port <= 0: - raise RuntimeError("standalone CodePolicy server has not started") - return f"http://{self.host}:{self.port}/control" - def start(self) -> None: if self._thread is not None: - raise RuntimeError("standalone CodePolicy server already started") - self.session.start() + raise RuntimeError("CodePolicy MCP server is already running") sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((self.host, self.port)) - sock.listen(2048) + sock.bind((self.host, 0)) + sock.listen(128) self.port = int(sock.getsockname()[1]) self._socket = sock + self.session.start() server = uvicorn.Server(uvicorn.Config(self.app, log_level="warning", access_log=False)) self._server = server def serve() -> None: asyncio.run(server.serve(sockets=[sock])) - self._thread = threading.Thread( + thread = threading.Thread( target=serve, name=f"code-policy-mcp-{self.port}", daemon=True, ) - self._thread.start() - if not McpAdapter(self.mcp_url, timeout=2).wait_for_ready(timeout=10, interval=0.05): + self._thread = thread + thread.start() + deadline = time.monotonic() + 10 + while not server.started and thread.is_alive() and time.monotonic() < deadline: + time.sleep(0.01) + if not server.started: self.stop() - raise TimeoutError("standalone CodePolicy MCP server did not become ready") + raise TimeoutError("CodePolicy MCP server did not start") def stop(self) -> None: - server = self._server - thread = self._thread + server, thread = self._server, self._thread self._server = None self._thread = None if server is not None: @@ -133,213 +118,11 @@ def stop(self) -> None: self._socket.close() self._socket = None self.session.stop() + self.port = 0 - def run_forever(self) -> None: - self.start() - try: - while not self.shutdown_requested.wait(0.2): - thread = self._thread - if thread is None or not thread.is_alive(): - raise RuntimeError("standalone CodePolicy MCP server stopped unexpectedly") - except KeyboardInterrupt: - pass - finally: - self.stop() - - def _install_routes(self) -> None: - @self.app.post("/mcp") - async def mcp_endpoint(request: Request) -> JSONResponse: - try: - body = await request.json() - except Exception: - return JSONResponse(_error(None, -32700, "Parse error"), status_code=400) - return JSONResponse(await self._handle_mcp(body)) - - @self.app.post("/control/{operation}") - async def control_endpoint(operation: str, request: Request) -> JSONResponse: - body: dict[str, Any] = {} - if request.headers.get("content-length") not in {None, "0"}: - body = await request.json() - if operation == "receipt": - value: Any = self.session.get_session_receipt().model_dump(mode="json") - elif operation == "reset": - value = self.session.reset_session().model_dump(mode="json") - elif operation == "interrupt": - value = {"interrupted": self.session.interrupt_active()} - elif operation == "records": - records = self.session.get_execution_records(body.get("session_id")) - value = [record.model_dump(mode="json") for record in records] - elif operation == "shutdown": - self.shutdown_requested.set() - value = {"accepted": True} - else: - return JSONResponse({"error": f"unknown control operation: {operation}"}, 404) - return JSONResponse(value) - - async def _handle_mcp(self, body: Any) -> dict[str, Any]: - if not isinstance(body, dict): - return _error(None, -32600, "Invalid request") - request_id = body.get("id") - method = body.get("method") - if method == "initialize": - return _result( - request_id, - { - "protocolVersion": "2025-11-25", - "capabilities": {"tools": {}}, - "serverInfo": {"name": "dimos-code-policy", "version": "1.0.0"}, - }, - ) - if method == "tools/list": - return _result(request_id, {"tools": [PYTHON_EXEC_TOOL]}) - if method != "tools/call": - return _error(request_id, -32601, f"Unknown: {method}") - params = body.get("params") or {} - if params.get("name") != "python_exec": - return _result(request_id, _text(f"Tool not found: {params.get('name', '')}")) - arguments = params.get("arguments") or {} - if not isinstance(arguments, dict) or set(arguments) - {"code", "timeout_s"}: - return _result(request_id, _text("Invalid python_exec arguments")) - code = arguments.get("code") - timeout_s = arguments.get("timeout_s", MAX_EXECUTION_TIMEOUT_S) - if not isinstance(code, str) or not code: - return _result(request_id, _text("python_exec code must be a non-empty string")) - if isinstance(timeout_s, bool) or not isinstance(timeout_s, (int, float)): - return _result(request_id, _text("python_exec timeout_s must be numeric")) - transcript = await asyncio.to_thread(self.session.python_exec, code, float(timeout_s)) - return _result(request_id, _text(transcript)) - - -class StandaloneCodePolicyProcess: - """Runner-owned standalone process plus private control client.""" - - def __init__(self, config: CodePolicySessionConfig) -> None: - self.config = config - self.port = _available_port() - self.mcp_url = f"http://127.0.0.1:{self.port}/mcp" - self.control_url = f"http://127.0.0.1:{self.port}/control" - self.process: subprocess.Popen[str] | None = None - - def start(self, timeout_s: float = 10.0) -> None: - if self.process is not None: - raise RuntimeError("standalone CodePolicy process already started") - self.process = subprocess.Popen( - ( - sys.executable, - "-m", - "dimos.agents.code_policy_server", - "--config-json", - self.config.model_dump_json(), - "--port", - str(self.port), - ), - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - if not McpAdapter(self.mcp_url, timeout=2).wait_for_ready(timeout=timeout_s, interval=0.05): - self.close() - raise TimeoutError("standalone CodePolicy process did not become ready") - - def receipt(self) -> dict[str, Any]: - value = self._control("receipt") - if not isinstance(value, dict): - raise TypeError("CodePolicy receipt response is not an object") - return value - - def reset(self) -> dict[str, Any]: - value = self._control("reset") - if not isinstance(value, dict): - raise TypeError("CodePolicy reset response is not an object") - return value - - def records(self, session_id: str | None = None) -> list[dict[str, Any]]: - value = self._control("records", {"session_id": session_id}) - if not isinstance(value, list): - raise TypeError("CodePolicy records response is not a list") - return value - - def interrupt(self) -> bool: - return bool(self._control("interrupt").get("interrupted")) - - def close(self) -> None: - process = self.process - self.process = None - if process is None: - return - if process.poll() is None: - try: - self._control("shutdown") - process.wait(timeout=5) - except Exception: - process.terminate() - try: - process.wait(timeout=2) - except subprocess.TimeoutExpired: - process.kill() - process.wait(timeout=2) - - def _control(self, operation: str, body: dict[str, Any] | None = None) -> Any: - response = requests.post(f"{self.control_url}/{operation}", json=body or {}, timeout=5) - response.raise_for_status() - return response.json() - - def __enter__(self) -> StandaloneCodePolicyProcess: + def __enter__(self) -> CodePolicyMcpServer: self.start() return self - def __exit__(self, *_args: Any) -> None: - self.close() - - -def _result(request_id: Any, result: Any) -> dict[str, Any]: - return {"jsonrpc": "2.0", "id": request_id, "result": result} - - -def _error(request_id: Any, code: int, message: str) -> dict[str, Any]: - return {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}} - - -def _text(value: str) -> dict[str, Any]: - return {"content": [{"type": "text", "text": value}]} - - -def _available_port() -> int: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.bind(("127.0.0.1", 0)) - return int(sock.getsockname()[1]) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m dimos.agents.code_policy_server") - parser.add_argument("--config-json") - parser.add_argument("--port", type=int, default=0) - environment = parser.add_mutually_exclusive_group() - environment.add_argument("--live-memory") - environment.add_argument("--frozen-source") - parser.add_argument("--derived-memory") - parser.add_argument("--cutoff-timestamp", type=float) - args = parser.parse_args(argv) - if args.config_json: - config = CodePolicySessionConfig.model_validate_json(args.config_json) - elif args.live_memory: - config = CodePolicySessionConfig( - environment=LiveDimosEnvironment(recording_path=args.live_memory) - ) - elif args.frozen_source and args.derived_memory and args.cutoff_timestamp is not None: - config = CodePolicySessionConfig( - environment=FrozenMemoryEnvironment( - recording_path=args.frozen_source, - derived_recording_path=args.derived_memory, - memory_cutoff_timestamp=args.cutoff_timestamp, - ) - ) - else: - parser.error("provide --config-json, --live-memory, or all frozen source arguments") - StandaloneCodePolicyServer(config, port=args.port).run_forever() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) + def __exit__(self, *_args: object) -> None: + self.stop() diff --git a/dimos/agents/test_code_policy_core.py b/dimos/agents/test_code_policy_core.py index 17c542dc2f..5f84d94c61 100644 --- a/dimos/agents/test_code_policy_core.py +++ b/dimos/agents/test_code_policy_core.py @@ -16,18 +16,17 @@ from pathlib import Path -import pytest - from dimos.agents.code_policy_core import ( CodePolicySession, CodePolicySessionConfig, FrozenMemoryEnvironment, LiveDimosEnvironment, + _kernel_environment, ) from dimos.memory2.store.sqlite import SqliteStore -def test_plain_session_persists_and_resets_without_module(mocker, tmp_path: Path) -> None: +def test_session_persists_python_namespace(mocker, tmp_path: Path) -> None: mocker.patch("dimos.agents.code_policy_core._bootstrap_source", return_value="pass") session = CodePolicySession( CodePolicySessionConfig( @@ -38,15 +37,32 @@ def test_plain_session_persists_and_resets_without_module(mocker, tmp_path: Path try: assert "[1]" in session.python_exec("items = [1]\nitems") assert "[1, 2]" in session.python_exec("items.append(2)\nitems") - first = session.get_session_receipt() - second = session.reset_session() - assert second.previous_session_id == first.session_id - assert "NameError" in session.python_exec("items") + assert session.execution_count == 2 finally: session.stop() -def test_frozen_session_bootstrap_exposes_memory_without_app(tmp_path: Path) -> None: +def test_new_session_starts_with_a_fresh_namespace(mocker, tmp_path: Path) -> None: + mocker.patch("dimos.agents.code_policy_core._bootstrap_source", return_value="pass") + config = CodePolicySessionConfig( + environment=LiveDimosEnvironment(recording_path=str(tmp_path / "unused.db")) + ) + first = CodePolicySession(config) + first.start() + try: + first.python_exec("session_only = 1") + finally: + first.stop() + + second = CodePolicySession(config) + second.start() + try: + assert "False" in second.python_exec("'session_only' in globals()") + finally: + second.stop() + + +def test_frozen_session_exposes_bounded_memory_without_app(tmp_path: Path) -> None: source_path = tmp_path / "source.db" derived_path = tmp_path / "derived.db" with SqliteStore(path=str(source_path)) as source: @@ -71,38 +87,33 @@ def test_frozen_session_bootstrap_exposes_memory_without_app(tmp_path: Path) -> "memory.streams.global_map.last().data, 'app' in globals())" ) assert "(['before'], 'map', False)" in result + assert "PermissionError" in session.python_exec("memory.streams.messages.append('blocked')") finally: session.stop() -def test_live_read_only_memory_observes_committed_writer_data(mocker, tmp_path: Path) -> None: - path = tmp_path / "live.db" - with SqliteStore(path=str(path)) as writer: - writer.stream("events", int).append(1, ts=1.0) +def test_kernel_environment_does_not_forward_credentials(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "secret") + monkeypatch.setenv("SOME_AUTH_TOKEN", "secret") + monkeypatch.setenv("CODE_POLICY_TEST_VALUE", "safe") + environment = LiveDimosEnvironment(recording_path=str(tmp_path / "memory.db")) + result = _kernel_environment(environment) + assert "OPENAI_API_KEY" not in result + assert "SOME_AUTH_TOKEN" not in result + assert result["CODE_POLICY_TEST_VALUE"] == "safe" - bootstrap = f""" -from dimos.memory2.store.sqlite import SqliteStore -memory = SqliteStore(path={str(path)!r}, must_exist=True, read_only=True) -memory.start() -""" - mocker.patch("dimos.agents.code_policy_core._bootstrap_source", return_value=bootstrap) + +def test_timeout_interrupts_kernel_and_keeps_session_usable(mocker, tmp_path: Path) -> None: + mocker.patch("dimos.agents.code_policy_core._bootstrap_source", return_value="pass") session = CodePolicySession( - CodePolicySessionConfig(environment=LiveDimosEnvironment(recording_path=str(path))) + CodePolicySessionConfig( + environment=LiveDimosEnvironment(recording_path=str(tmp_path / "unused.db")), + interrupt_grace_s=2, + ) ) session.start() try: - assert "\n\n1" in session.python_exec("memory.streams.events.count()") - with SqliteStore(path=str(path)) as writer: - writer.stream("events", int).append(2, ts=2.0) - assert "\n\n2" in session.python_exec("memory.streams.events.count()") - mutation = session.python_exec("memory.streams.events.append(3)") - assert "PermissionError" in mutation + assert "timed out" in session.python_exec("while True: pass", timeout_s=0.1) + assert "2" in session.python_exec("1 + 1") finally: session.stop() - - -def test_frozen_environment_requires_all_fields() -> None: - with pytest.raises(ValueError): - FrozenMemoryEnvironment.model_validate( - {"kind": "frozen_memory", "recording_path": "source.db"} - ) diff --git a/dimos/agents/test_code_policy_server.py b/dimos/agents/test_code_policy_server.py index afceb93b99..4f58bd885d 100644 --- a/dimos/agents/test_code_policy_server.py +++ b/dimos/agents/test_code_policy_server.py @@ -14,21 +14,14 @@ from __future__ import annotations +import asyncio from pathlib import Path -import requests +from mcp import Client +import pytest -from dimos.agents.code_policy_core import ( - CodePolicySessionConfig, - FrozenMemoryEnvironment, -) -from dimos.agents.code_policy_server import ( - StandaloneCodePolicyProcess, - StandaloneCodePolicyServer, -) -from dimos.agents.mcp.mcp_adapter import McpAdapter -from dimos.benchmark.agent_eval.pi_adapter import inspect_python_exec_inventory -from dimos.core.module import Module +from dimos.agents.code_policy_core import CodePolicySessionConfig, FrozenMemoryEnvironment +from dimos.agents.code_policy_server import CodePolicyMcpServer from dimos.memory2.store.sqlite import SqliteStore @@ -48,47 +41,25 @@ def _config(tmp_path: Path) -> CodePolicySessionConfig: ) -def test_standalone_server_has_exact_direct_mcp_surface(tmp_path: Path) -> None: - server = StandaloneCodePolicyServer(_config(tmp_path)) - assert not isinstance(server, Module) +@pytest.mark.asyncio +async def test_server_exposes_exactly_one_persistent_python_tool(tmp_path: Path) -> None: + server = CodePolicyMcpServer(_config(tmp_path)) server.start() - adapter = McpAdapter(server.mcp_url, timeout=5) try: - tools = adapter.list_tools() - assert [tool["name"] for tool in tools] == ["python_exec"] - inspect_python_exec_inventory(server.mcp_url, tools) - first = adapter.call_tool_text("python_exec", {"code": "items = [1]\nitems"}) - second = adapter.call_tool_text("python_exec", {"code": "items.append(2)\nitems"}) - assert "[1]" in first - assert "[1, 2]" in second - receipt = server.session.get_session_receipt() - assert len(server.session.get_execution_records(receipt.session_id)) == 2 + async with Client(server.mcp_url) as client: + tools = await client.list_tools() + assert [tool.name for tool in tools.tools] == ["python_exec"] + first = await client.call_tool("python_exec", {"code": "items = [1]\nitems"}) + second = await client.call_tool("python_exec", {"code": "items.append(2)\nitems"}) + assert "[1]" in first.content[0].text + assert "[1, 2]" in second.content[0].text + assert server.session.execution_count == 2 finally: - server.stop() - assert adapter.wait_for_down(timeout=2, interval=0.05) + await asyncio.to_thread(server.stop) -def test_standalone_process_control_resets_and_collects_records(tmp_path: Path) -> None: - process = StandaloneCodePolicyProcess(_config(tmp_path)) - process.start() - adapter = McpAdapter(process.mcp_url, timeout=5) - try: - receipt = process.receipt() - result = adapter.call_tool_text( - "python_exec", {"code": "memory.streams.messages.last().data"} - ) - assert "visible" in result - assert "app" not in adapter.call_tool_text("python_exec", {"code": "sorted(globals())"}) - assert len(process.records(receipt["session_id"])) == 2 - reset = process.reset() - assert reset["previous_session_id"] == receipt["session_id"] - assert reset["session_id"] != receipt["session_id"] - finally: - process.close() - assert process.process is None - - -def test_server_stops_after_execution_start_failure(tmp_path: Path) -> None: +@pytest.mark.asyncio +async def test_kernel_start_failure_is_returned_as_tool_text(tmp_path: Path) -> None: config = CodePolicySessionConfig( environment=FrozenMemoryEnvironment( recording_path=str(tmp_path / "missing.db"), @@ -96,15 +67,11 @@ def test_server_stops_after_execution_start_failure(tmp_path: Path) -> None: memory_cutoff_timestamp=1.0, ) ) - server = StandaloneCodePolicyServer(config) + server = CodePolicyMcpServer(config) server.start() - adapter = McpAdapter(server.mcp_url, timeout=5) - result = adapter.call_tool_text("python_exec", {"code": "1 + 1"}) - assert "failed to start" in result - server.stop() try: - requests.post(server.mcp_url, timeout=0.2) - except (requests.ConnectionError, requests.ReadTimeout): - pass - else: - raise AssertionError("standalone service remained reachable after stop") + async with Client(server.mcp_url) as client: + result = await client.call_tool("python_exec", {"code": "1 + 1"}) + assert "failed to start" in result.content[0].text + finally: + await asyncio.to_thread(server.stop) diff --git a/dimos/benchmark/agent_eval/artifacts.py b/dimos/benchmark/agent_eval/artifacts.py deleted file mode 100644 index a33c2d4d19..0000000000 --- a/dimos/benchmark/agent_eval/artifacts.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Generic records for immutable evaluation evidence.""" - -from __future__ import annotations - -from datetime import datetime -from typing import Annotated, Literal - -from pydantic import Field, JsonValue, model_validator - -from dimos.benchmark.agent_eval.base import BaseEvalModel - -NonEmpty = Annotated[str, Field(min_length=1)] -Sha256 = Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")] -AttemptId = Annotated[str, Field(pattern=r"^attempt_[0-9a-f]{32}$")] -OperationId = Annotated[str, Field(pattern=r"^operation_[0-9a-f]{32}$")] -CodePolicySessionId = Annotated[str, Field(pattern=r"^code_policy_session_[0-9a-f]{32}$")] - - -class ArtifactReference(BaseEvalModel): - record_type: Literal["artifact-reference"] = "artifact-reference" - path: NonEmpty - sha256: Sha256 - size_bytes: Annotated[int, Field(ge=0)] - - @model_validator(mode="after") - def path_is_relative(self) -> ArtifactReference: - if self.path.startswith("/") or ".." in self.path.split("/"): - raise ValueError("artifact path must be attempt-relative") - return self - - -class LifecycleEvent(BaseEvalModel): - record_type: Literal["agent-eval-lifecycle-event"] = "agent-eval-lifecycle-event" - sequence: Annotated[int, Field(ge=1)] - attempt_id: AttemptId - operation_id: OperationId | None = None - occurred_at: datetime - monotonic_offset_s: Annotated[float, Field(ge=0)] - kind: NonEmpty - payload: dict[str, JsonValue] = Field(default_factory=dict) - - -class NormalizedOutcome(BaseEvalModel): - """Generic terminal record retained for attempt-store callers.""" - - record_type: Literal["agent-eval-outcome"] = "agent-eval-outcome" - attempt_id: AttemptId - attempt_status: Literal["completed", "failed"] - task_result: Literal["passed", "failed", "not_evaluated"] - terminal_stage: NonEmpty - reason: NonEmpty - required_artifacts_complete: bool - finished_at: datetime - duration_s: Annotated[float, Field(ge=0)] - - @model_validator(mode="after") - def infrastructure_and_task_states_are_consistent(self) -> NormalizedOutcome: - if self.attempt_status == "failed" and self.task_result != "not_evaluated": - raise ValueError("failed infrastructure cannot report a task result") - if self.attempt_status == "completed" and self.task_result == "not_evaluated": - raise ValueError("completed evaluation must report pass or fail") - if self.attempt_status == "completed" and not self.required_artifacts_complete: - raise ValueError("completed evaluation requires complete artifacts") - return self diff --git a/dimos/benchmark/agent_eval/auth.py b/dimos/benchmark/agent_eval/auth.py deleted file mode 100644 index 9b29d17211..0000000000 --- a/dimos/benchmark/agent_eval/auth.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Runtime-only Pi credential transport without benchmark dependencies.""" - -from dataclasses import dataclass -from typing import Literal - - -@dataclass(frozen=True) -class RuntimeCredential: - auth_mode: Literal["subscription", "environment"] - binding_name: str - value: str | None diff --git a/dimos/benchmark/agent_eval/case.py b/dimos/benchmark/agent_eval/case.py deleted file mode 100644 index 4aec9021b0..0000000000 --- a/dimos/benchmark/agent_eval/case.py +++ /dev/null @@ -1,232 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Canonical source/task/interaction/validator contracts for agent evaluation.""" - -from __future__ import annotations - -import hashlib -import math -from pathlib import PurePosixPath -from typing import Annotated, Literal - -from pydantic import Field, JsonValue, model_validator - -from dimos.benchmark.agent_eval.base import BaseEvalModel -from dimos.benchmark.agent_eval.json import canonical_json - -NonEmpty = Annotated[str, Field(min_length=1)] -Sha256 = Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")] -NormalizedProgress = Annotated[float, Field(ge=0.0, le=1.0, allow_inf_nan=False)] - - -class FrozenRecordingSource(BaseEvalModel): - kind: Literal["frozen_memory"] = "frozen_memory" - recording: NonEmpty - progress: NormalizedProgress - bundle_manifest_sha256: Sha256 | None = None - - @model_validator(mode="after") - def progress_is_finite(self) -> FrozenRecordingSource: - if not math.isfinite(self.progress): - raise ValueError("recording progress must be finite") - return self - - -SourceSpec = FrozenRecordingSource - - -class IntegerQuestionTask(BaseEvalModel): - kind: Literal["integer_question"] = "integer_question" - prompt: NonEmpty - answer_marker: Literal["ANSWER:"] = "ANSWER:" - - -TaskSpec = IntegerQuestionTask - - -class FrozenCodePolicyInteraction(BaseEvalModel): - kind: Literal["frozen_code_policy"] = "frozen_code_policy" - driver_revision: NonEmpty - session_lifetime: Literal["one_attempt"] = "one_attempt" - - -InteractionSpec = FrozenCodePolicyInteraction - - -class ExactIntegerValidatorRef(BaseEvalModel): - kind: Literal["exact_integer"] = "exact_integer" - revision: NonEmpty - private_path: NonEmpty - private_sha256: Sha256 - - @model_validator(mode="after") - def private_path_is_relative(self) -> ExactIntegerValidatorRef: - path = PurePosixPath(self.private_path) - if path.is_absolute() or not path.parts or ".." in path.parts: - raise ValueError("validator private_path must be a safe relative path") - return self - - -ValidatorRef = ExactIntegerValidatorRef - - -class PublicEvalCase(BaseEvalModel): - """Agent-safe case projection; it intentionally cannot carry a validator.""" - - case_id: NonEmpty - source: SourceSpec - task: TaskSpec - interaction: InteractionSpec - - -class EvalCase(BaseEvalModel): - """Compiled private case binding all four semantic contracts.""" - - case_id: NonEmpty - source: SourceSpec - task: TaskSpec - interaction: InteractionSpec - validator: ValidatorRef - fingerprint: Sha256 - - @classmethod - def compile( - cls, - *, - case_id: str, - source: SourceSpec, - task: TaskSpec, - interaction: InteractionSpec, - validator: ValidatorRef, - ) -> EvalCase: - payload = _case_payload(case_id, source, task, interaction, validator) - return cls( - case_id=case_id, - source=source, - task=task, - interaction=interaction, - validator=validator, - fingerprint=hashlib.sha256(canonical_json(payload)).hexdigest(), - ) - - @model_validator(mode="after") - def fingerprint_matches_payload(self) -> EvalCase: - payload = _case_payload( - self.case_id, - self.source, - self.task, - self.interaction, - self.validator, - ) - expected = hashlib.sha256(canonical_json(payload)).hexdigest() - if self.fingerprint != expected: - raise ValueError("evaluation case fingerprint does not match its contracts") - return self - - def public_projection(self) -> PublicEvalCase: - return PublicEvalCase( - case_id=self.case_id, - source=self.source, - task=self.task, - interaction=self.interaction, - ) - - -class AgentCondition(BaseEvalModel): - agent_id: NonEmpty - adapter: NonEmpty - model: NonEmpty - thinking_level: NonEmpty - - -class RuntimeBinding(BaseEvalModel): - runtime_id: NonEmpty - parameters: dict[str, JsonValue] = Field(default_factory=dict) - - -class AttemptRequest(BaseEvalModel): - case: EvalCase - agent: AgentCondition - runtime: RuntimeBinding - seed: int | None = None - - -class AgentOutcome(BaseEvalModel): - final_text: str - tool_call_count: int = Field(ge=0) - terminal_reason: NonEmpty - agent_session_id: NonEmpty | None = None - interaction_session_id: NonEmpty | None = None - - -class Prediction(BaseEvalModel): - case_id: NonEmpty - attempt_id: NonEmpty - agent_session_id: NonEmpty - interaction_session_id: NonEmpty - parser_revision: NonEmpty - final_text: str - status: Literal["parsed", "invalid"] - integer_answer: int | None = None - diagnostic: NonEmpty | None = None - - @model_validator(mode="after") - def answer_matches_status(self) -> Prediction: - if self.status == "parsed" and (self.integer_answer is None or self.diagnostic is not None): - raise ValueError("parsed prediction requires only integer_answer") - if self.status == "invalid" and ( - self.integer_answer is not None or self.diagnostic is None - ): - raise ValueError("invalid prediction requires only diagnostic") - return self - - -class PrivateScore(BaseEvalModel): - case_id: NonEmpty - attempt_id: NonEmpty - validator_revision: NonEmpty - passed: bool - prediction_status: Literal["parsed", "invalid"] - - -class EvalOutcome(BaseEvalModel): - attempt_id: NonEmpty - attempt_status: Literal["completed", "failed"] - task_result: Literal["passed", "failed", "not_evaluated"] - reason: NonEmpty - - @model_validator(mode="after") - def states_are_consistent(self) -> EvalOutcome: - if self.attempt_status == "failed" and self.task_result != "not_evaluated": - raise ValueError("failed infrastructure cannot claim a task result") - if self.attempt_status == "completed" and self.task_result == "not_evaluated": - raise ValueError("completed evaluation must report passed or failed") - return self - - -def _case_payload( - case_id: str, - source: SourceSpec, - task: TaskSpec, - interaction: InteractionSpec, - validator: ValidatorRef, -) -> dict[str, JsonValue]: - return { - "case_id": case_id, - "source": source.model_dump(mode="json"), - "task": task.model_dump(mode="json"), - "interaction": interaction.model_dump(mode="json"), - "validator": validator.model_dump(mode="json"), - } diff --git a/dimos/benchmark/agent_eval/engine.py b/dimos/benchmark/agent_eval/engine.py deleted file mode 100644 index f03aa5b763..0000000000 --- a/dimos/benchmark/agent_eval/engine.py +++ /dev/null @@ -1,262 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Shared single-attempt engine for canonical agent-evaluation cases.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from pydantic import BaseModel, ConfigDict, JsonValue - -from dimos.benchmark.agent_eval.artifacts import ArtifactReference -from dimos.benchmark.agent_eval.case import AttemptRequest, EvalOutcome, PrivateScore -from dimos.benchmark.agent_eval.interfaces import ( - AgentAdapter, - AttemptContext, - EvidenceSink, - InteractionDriver, - SourceDriver, - ValidatorDriver, - ValidatorSession, -) -from dimos.benchmark.agent_eval.store import AttemptStore - - -class EngineResult(BaseModel): - model_config = ConfigDict(extra="forbid", frozen=True, strict=True) - - attempt_path: Path - outcome: EvalOutcome - artifacts: tuple[ArtifactReference, ...] - - -class AttemptEvidence(EvidenceSink): - def __init__(self, store: AttemptStore) -> None: - self.store = store - self.artifacts: list[ArtifactReference] = [] - - def event(self, kind: str, payload: dict[str, JsonValue] | None = None) -> None: - self.store.append_event(kind, payload=payload) - - def artifact(self, relative_path: str, value: BaseModel | JsonValue | bytes | str) -> None: - self.artifacts.append(self.store.write_artifact(relative_path, value)) - - def reference(self, relative_path: str) -> None: - self.artifacts.append(_reference(self.store.path, relative_path)) - - -class AttemptEngine: - """Coordinate source, private validator, interaction, evidence, and cleanup.""" - - def __init__( - self, - *, - request: AttemptRequest, - output_root: Path, - source: SourceDriver, - interaction: InteractionDriver, - validator: ValidatorDriver, - agent: AgentAdapter, - ) -> None: - self.request = request - self.output_root = output_root - self.source = source - self.interaction = interaction - self.validator = validator - self.agent = agent - - def run(self) -> EngineResult: - store = AttemptStore(self.output_root) - result: EngineResult | None = None - close_error: Exception | None = None - try: - result = self._run_reserved(store) - finally: - try: - store.close() - except Exception as exc: - close_error = exc - if close_error is not None: - return EngineResult( - attempt_path=store.path, - outcome=EvalOutcome( - attempt_id=store.attempt_id, - attempt_status="failed", - task_result="not_evaluated", - reason=f"attempt store cleanup failed: {type(close_error).__name__}: {close_error}"[ - :1024 - ], - ), - artifacts=result.artifacts if result is not None else (), - ) - assert result is not None - return result - - def _run_reserved(self, store: AttemptStore) -> EngineResult: - evidence = AttemptEvidence(store) - context = AttemptContext( - attempt_id=store.attempt_id, - path=store.path, - request=self.request, - ) - validator_session: ValidatorSession | None = None - agent_outcome = None - outcome: EvalOutcome - completed = False - passed = False - reason = "infrastructure failure" - try: - evidence.event("attempt-created") - evidence.artifact("case.private.v1.json", self.request.case) - evidence.artifact("case.public.v1.json", self.request.case.public_projection()) - prepared = self.source.prepare( - source=self.request.case.source, - context=context, - evidence=evidence, - ) - evidence.event("source-prepared") - validator_session = self.validator.prepare( - case=self.request.case, - prepared_source=prepared, - context=context, - evidence=evidence, - ) - evidence.event("validator-prepared") - agent_outcome = self.interaction.run( - case=self.request.case, - prepared_source=prepared, - agent=self.agent, - context=context, - evidence=evidence, - ) - evidence.artifact("agent-outcome.v1.json", agent_outcome) - evidence.event("interaction-completed") - score = validator_session.evaluate(agent_outcome) - _validate_score(score, store.attempt_id, self.request.case.case_id) - evidence.artifact("score.private.v1.json", score) - evidence.event("validation-completed", {"passed": score.passed}) - completed = True - passed = score.passed - reason = "validator passed" if passed else "validator failed" - except KeyboardInterrupt: - reason = "user interrupted" - _safe_event(evidence, "attempt-interrupted") - except Exception as exc: - reason = f"{type(exc).__name__}: {exc}"[:1024] - _safe_event(evidence, "infrastructure-failure", {"diagnostic": reason}) - cleanup_errors = self._cleanup(validator_session) - if cleanup_errors: - _safe_event(evidence, "cleanup-failure", {"diagnostic": "; ".join(cleanup_errors)}) - completed = False - passed = False - reason = "; ".join(cleanup_errors) - try: - evidence.reference("events.jsonl") - evidence.artifact( - "attempt-manifest.v1.json", - { - "schema_version": "1.0", - "attempt_id": store.attempt_id, - "case_id": self.request.case.case_id, - "case_fingerprint": self.request.case.fingerprint, - "agent": self.request.agent.model_dump(mode="json"), - "runtime": self.request.runtime.model_dump(mode="json"), - "agent_session_id": ( - agent_outcome.agent_session_id if agent_outcome is not None else None - ), - "interaction_session_id": ( - agent_outcome.interaction_session_id if agent_outcome is not None else None - ), - "artifacts": [ - artifact.model_dump(mode="json") for artifact in evidence.artifacts - ], - }, - ) - except Exception as exc: - completed = False - passed = False - reason = f"attempt finalization failed: {type(exc).__name__}: {exc}"[:1024] - _safe_event(evidence, "finalization-failure", {"diagnostic": reason}) - outcome = EvalOutcome( - attempt_id=store.attempt_id, - attempt_status="completed" if completed else "failed", - task_result=("passed" if passed else "failed") if completed else "not_evaluated", - reason=reason, - ) - try: - evidence.artifacts.append(store.write_eval_outcome(outcome)) - except Exception as exc: - outcome = EvalOutcome( - attempt_id=store.attempt_id, - attempt_status="failed", - task_result="not_evaluated", - reason=f"terminal publication failed: {type(exc).__name__}: {exc}"[:1024], - ) - _safe_event( - evidence, - "terminal-publication-failure", - {"diagnostic": outcome.reason}, - ) - return EngineResult( - attempt_path=store.path, - outcome=outcome, - artifacts=tuple(evidence.artifacts), - ) - - def _cleanup(self, validator_session: ValidatorSession | None) -> list[str]: - errors: list[str] = [] - resources: tuple[tuple[str, Any], ...] = ( - ("agent", self.agent), - ("interaction", self.interaction), - ("validator", validator_session), - ("source", self.source), - ) - for name, resource in resources: - if resource is None: - continue - try: - resource.close() - except Exception as exc: - errors.append(f"{name}: {type(exc).__name__}: {exc}"[:1024]) - return errors - - -def _validate_score(score: PrivateScore, attempt_id: str, case_id: str) -> None: - if score.attempt_id != attempt_id or score.case_id != case_id: - raise ValueError("validator score identity mismatch") - - -def _reference(root: Path, relative_path: str) -> ArtifactReference: - import hashlib - - data = (root / relative_path).read_bytes() - return ArtifactReference( - path=relative_path, - sha256=hashlib.sha256(data).hexdigest(), - size_bytes=len(data), - ) - - -def _safe_event( - evidence: AttemptEvidence, - kind: str, - payload: dict[str, JsonValue] | None = None, -) -> None: - try: - evidence.event(kind, payload) - except Exception: - # Evidence storage is already failing; cleanup and lock release remain mandatory. - return diff --git a/dimos/benchmark/agent_eval/interfaces.py b/dimos/benchmark/agent_eval/interfaces.py deleted file mode 100644 index 35b98982cd..0000000000 --- a/dimos/benchmark/agent_eval/interfaces.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Backend-neutral interfaces used by canonical agent-evaluation attempts.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any, Protocol - -from pydantic import BaseModel, ConfigDict, JsonValue - -from dimos.benchmark.agent_eval.case import ( - AgentOutcome, - AttemptRequest, - EvalCase, - PrivateScore, - SourceSpec, - TaskSpec, -) - - -class InterfaceModel(BaseModel): - model_config = ConfigDict(extra="forbid", frozen=True, strict=True) - - -class AttemptContext(InterfaceModel): - attempt_id: str - path: Path - request: AttemptRequest - - -class PreparedSource(InterfaceModel): - public: dict[str, JsonValue] - receipt: dict[str, JsonValue] - private_handle: Any = None - - -class AgentAdapter(Protocol): - def run( - self, *, task: TaskSpec, context: AttemptContext, interface: Any = None - ) -> AgentOutcome: ... - - def close(self) -> None: ... - - -class EvidenceSink(Protocol): - def event(self, kind: str, payload: dict[str, JsonValue] | None = None) -> None: ... - - def artifact(self, relative_path: str, value: BaseModel | JsonValue | bytes | str) -> None: ... - - def reference(self, relative_path: str) -> None: ... - - -class SourceDriver(Protocol): - def prepare( - self, - *, - source: SourceSpec, - context: AttemptContext, - evidence: EvidenceSink, - ) -> PreparedSource: ... - - def close(self) -> None: ... - - -class InteractionDriver(Protocol): - def run( - self, - *, - case: EvalCase, - prepared_source: PreparedSource, - agent: AgentAdapter, - context: AttemptContext, - evidence: EvidenceSink, - ) -> AgentOutcome: ... - - def close(self) -> None: ... - - -class ValidatorSession(Protocol): - def evaluate(self, outcome: AgentOutcome) -> PrivateScore: ... - - def close(self) -> None: ... - - -class ValidatorDriver(Protocol): - def prepare( - self, - *, - case: EvalCase, - prepared_source: PreparedSource, - context: AttemptContext, - evidence: EvidenceSink, - ) -> ValidatorSession: ... diff --git a/dimos/benchmark/agent_eval/json.py b/dimos/benchmark/agent_eval/json.py deleted file mode 100644 index 527804362d..0000000000 --- a/dimos/benchmark/agent_eval/json.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Deterministic JSON encoding for evaluation fingerprints and artifacts.""" - -import json -from typing import Any - - -def canonical_json(value: Any) -> bytes: - """Encode JSON with stable ordering, compact separators, and UTF-8 bytes.""" - return json.dumps( - value, - allow_nan=False, - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") diff --git a/dimos/benchmark/agent_eval/models.py b/dimos/benchmark/agent_eval/models.py new file mode 100644 index 0000000000..7787854381 --- /dev/null +++ b/dimos/benchmark/agent_eval/models.py @@ -0,0 +1,107 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Small tagged contracts for one frozen agent-evaluation case.""" + +from __future__ import annotations + +import math +from pathlib import PurePosixPath +from typing import Annotated, Literal + +from pydantic import Field, model_validator + +from dimos.benchmark.agent_eval.base import BaseEvalModel + +NonEmpty = Annotated[str, Field(min_length=1)] + + +class FrozenRecordingSource(BaseEvalModel): + kind: Literal["frozen_memory"] = "frozen_memory" + recording: NonEmpty + progress: float = Field(ge=0, le=1, allow_inf_nan=False) + + @model_validator(mode="after") + def finite_progress(self) -> FrozenRecordingSource: + if not math.isfinite(self.progress): + raise ValueError("recording progress must be finite") + return self + + +class IntegerQuestionTask(BaseEvalModel): + kind: Literal["integer_question"] = "integer_question" + prompt: NonEmpty + answer_marker: Literal["ANSWER:"] = "ANSWER:" + + +class ExactIntegerValidatorRef(BaseEvalModel): + kind: Literal["exact_integer"] = "exact_integer" + revision: NonEmpty + private_path: NonEmpty + + @model_validator(mode="after") + def safe_relative_path(self) -> ExactIntegerValidatorRef: + path = PurePosixPath(self.private_path) + if path.is_absolute() or not path.parts or ".." in path.parts: + raise ValueError("validator private_path must be a safe relative path") + return self + + +SourceSpec = Annotated[FrozenRecordingSource, Field(discriminator="kind")] +TaskSpec = Annotated[IntegerQuestionTask, Field(discriminator="kind")] +ValidatorRef = Annotated[ExactIntegerValidatorRef, Field(discriminator="kind")] + + +class EvalCase(BaseEvalModel): + case_id: NonEmpty + source: SourceSpec + task: TaskSpec + validator: ValidatorRef + + +class PiAgentConfig(BaseEvalModel): + backend: Literal["pi"] = "pi" + model: Literal["gpt-5.6-luna"] = "gpt-5.6-luna" + thinking_level: Literal["medium"] = "medium" + api_key_env: str = Field(default="OPENAI_API_KEY", min_length=1) + + +class EvalRunConfig(BaseEvalModel): + agent: PiAgentConfig = Field(default_factory=PiAgentConfig) + + +class CompactEvalResult(BaseEvalModel): + case_id: str + recording: str + progress: float + model: str + thinking_level: str + final_response: str = "" + prediction_status: Literal["parsed", "invalid", "not_evaluated"] + integer_answer: int | None = None + passed: bool | None = None + validator_revision: str + tool_call_count: int = Field(ge=0) + duration_seconds: float = Field(ge=0) + infra_error: str | None = None + + @property + def attempt_status(self) -> Literal["completed", "failed"]: + return "failed" if self.infra_error is not None else "completed" + + @property + def task_result(self) -> Literal["passed", "failed", "not_evaluated"]: + if self.passed is None: + return "not_evaluated" + return "passed" if self.passed else "failed" diff --git a/dimos/benchmark/agent_eval/pi.py b/dimos/benchmark/agent_eval/pi.py deleted file mode 100644 index 75aea78c21..0000000000 --- a/dimos/benchmark/agent_eval/pi.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Backend-neutral Pi code-policy session contracts.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Protocol - -from pydantic import BaseModel, ConfigDict, Field - -from dimos.benchmark.agent_eval.artifacts import ArtifactReference -from dimos.benchmark.agent_eval.pi_adapter import CodePolicyCallLog, McpBinding - - -class PiTurn(BaseModel): - model_config = ConfigDict(extra="forbid", frozen=True, strict=True) - - final_text: str = "" - policy_call_count: int = Field(ge=0) - - -class PiSession(Protocol): - session_id: str - - def prompt(self, prompt: str, timeout_s: float) -> PiTurn: ... - - def abort(self, timeout_s: float) -> None: ... - - def dispose(self) -> None: ... - - def artifact_references(self) -> tuple[ArtifactReference, ...]: ... - - -class PiSessionFactory(Protocol): - def create( - self, - *, - attempt_path: Path, - public_prompt: str, - code_policy_session_id: str, - call_log: CodePolicyCallLog, - mcp: McpBinding, - ) -> PiSession: ... diff --git a/dimos/benchmark/agent_eval/pi_adapter.py b/dimos/benchmark/agent_eval/pi_adapter.py deleted file mode 100644 index b4ebd816cd..0000000000 --- a/dimos/benchmark/agent_eval/pi_adapter.py +++ /dev/null @@ -1,302 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""One-tool Pi facade over an attached DimOS MCP server. - -The attached server is intentionally allowed to expose the normal robot skill -inventory. This module records that inventory but admits only the exact -``python_exec`` schema into the Pi model-facing session. -""" - -from __future__ import annotations - -from collections.abc import Callable, Mapping, Sequence -from datetime import UTC, datetime -import hashlib -import json -import os -from pathlib import Path -import threading -import time -from typing import Any, Protocol -from uuid import uuid4 - -from pydantic import BaseModel, ConfigDict, Field, JsonValue - -from dimos.agents.code_policy_core import MAX_EXECUTION_TIMEOUT_S -from dimos.benchmark.agent_eval.artifacts import ( - AttemptId, - CodePolicySessionId, - NonEmpty, -) -from dimos.benchmark.agent_eval.json import canonical_json - -PYTHON_EXEC_TOOL_NAME = "python_exec" -PI_TOOL_NAMES = (PYTHON_EXEC_TOOL_NAME,) -_EXPECTED_DESCRIPTION_PREFIX = ( - "Execute one synchronous Python program in the persistent policy session." -) - - -class McpBinding(Protocol): - """Minimum MCP behavior used by the one-tool facade.""" - - def wait_for_ready(self, timeout: float) -> bool: ... - - def list_tools(self) -> list[dict[str, Any]]: ... - - def call_tool(self, name: str, arguments: dict[str, Any] | None = None) -> dict[str, Any]: ... - - -class PiAdapterModel(BaseModel): - model_config = ConfigDict(extra="forbid", frozen=True, strict=True) - schema_version: str = "1.0" - - -class McpInventoryReceipt(PiAdapterModel): - record_type: str = "mcp-inventory-receipt" - endpoint: NonEmpty - observed_tools: tuple[dict[str, JsonValue], ...] - python_exec_schema_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") - - -class CodePolicyCallRecord(PiAdapterModel): - record_type: str = "code-policy-call" - call_id: NonEmpty - attempt_id: AttemptId - pi_session_id: NonEmpty - code_policy_session_id: CodePolicySessionId - tool_name: str - arguments: dict[str, JsonValue] - requested_at: datetime - completed_at: datetime - monotonic_duration_s: float = Field(ge=0) - ok: bool - result: dict[str, JsonValue] | None = None - error: NonEmpty | None = None - - -class ToolInventoryError(ValueError): - """The attached MCP inventory cannot safely back the Pi facade.""" - - -def inspect_python_exec_inventory( - endpoint: str, - tools: Sequence[Mapping[str, Any]], -) -> McpInventoryReceipt: - """Validate one exact code-policy tool while retaining the full inventory.""" - observed = tuple(_json_tool(tool) for tool in tools) - matches = [tool for tool in observed if tool.get("name") == PYTHON_EXEC_TOOL_NAME] - if len(matches) != 1: - raise ToolInventoryError("MCP inventory must contain exactly one python_exec tool") - schema = matches[0].get("inputSchema") - description = matches[0].get("description") - if not isinstance(schema, dict) or not _is_python_exec_schema(schema): - raise ToolInventoryError("python_exec input schema is incompatible") - if ( - not isinstance(description, str) - or not description.startswith(_EXPECTED_DESCRIPTION_PREFIX) - or "trusted, unsandboxed" not in description - ): - raise ToolInventoryError("python_exec description is incompatible") - return McpInventoryReceipt( - endpoint=endpoint, - observed_tools=observed, - python_exec_schema_sha256=hashlib.sha256(canonical_json(schema)).hexdigest(), - ) - - -def wait_for_python_exec( - endpoint: str, - mcp: McpBinding, - timeout_s: float, -) -> McpInventoryReceipt: - if timeout_s <= 0: - raise ValueError("MCP readiness timeout must be positive") - if not mcp.wait_for_ready(timeout_s): - raise TimeoutError(f"MCP server did not become ready at {endpoint}") - return inspect_python_exec_inventory(endpoint, mcp.list_tools()) - - -class CodePolicyCallLog: - """Append-only durable evidence for calls forwarded on Pi's behalf.""" - - def __init__(self, path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - self.path = path - self._descriptor = os.open( - path, - os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, - 0o600, - ) - self._lock = threading.Lock() - self._closed = False - - def append(self, record: CodePolicyCallRecord) -> None: - encoded = canonical_json(record.model_dump(mode="json")) + b"\n" - with self._lock: - if self._closed: - raise RuntimeError("code-policy call log is closed") - view = memoryview(encoded) - while view: - view = view[os.write(self._descriptor, view) :] - os.fsync(self._descriptor) - - def close(self) -> None: - with self._lock: - if self._closed: - return - self._closed = True - os.close(self._descriptor) - - def __enter__(self) -> CodePolicyCallLog: - return self - - def __exit__(self, *_args: Any) -> None: - self.close() - - -class PythonExecBroker: - """Forward the sole Pi tool and bind evidence to both session identities.""" - - def __init__( - self, - *, - attempt_id: str, - pi_session_id: str, - code_policy_session_id: str, - mcp: McpBinding, - call_log: CodePolicyCallLog, - clock: Callable[[], float] = time.monotonic, - ) -> None: - self.attempt_id = attempt_id - self.pi_session_id = pi_session_id - self.code_policy_session_id = code_policy_session_id - self.mcp = mcp - self.call_log = call_log - self.clock = clock - self.call_count = 0 - - def request(self, tool_name: str, arguments: Mapping[str, Any]) -> dict[str, Any]: - if tool_name != PYTHON_EXEC_TOOL_NAME: - raise PermissionError(f"Pi tool {tool_name!r} is not permitted") - safe_arguments = _validate_arguments(arguments) - requested_at = datetime.now(UTC) - started = self.clock() - self.call_count += 1 - result: dict[str, Any] | None = None - error: str | None = None - try: - result = self.mcp.call_tool(PYTHON_EXEC_TOOL_NAME, safe_arguments) - if not isinstance(result, dict): - raise TypeError("MCP tool result must be an object") - return result - except Exception as exc: - error = _bounded_diagnostic(exc) - raise - finally: - safe_result = _json_object(result) if result is not None else None - self.call_log.append( - CodePolicyCallRecord( - call_id=f"pi_tool_call_{uuid4().hex}", - attempt_id=self.attempt_id, - pi_session_id=self.pi_session_id, - code_policy_session_id=self.code_policy_session_id, - tool_name=PYTHON_EXEC_TOOL_NAME, - arguments=safe_arguments, - requested_at=requested_at, - completed_at=datetime.now(UTC), - monotonic_duration_s=max(0.0, self.clock() - started), - ok=error is None, - result=safe_result, - error=error, - ) - ) - - -def credential_binding_sha256( - auth_mode: str, - binding_name: str, - credential: str | bytes | None = None, -) -> str: - """Return a domain-separated binding digest without retaining the secret.""" - if not auth_mode or not binding_name: - raise ValueError("authentication mode and binding name are required") - digest = hashlib.sha256() - digest.update(b"dimos-agent-eval-credential-binding-v1\0") - digest.update(auth_mode.encode()) - digest.update(b"\0") - digest.update(binding_name.encode()) - if credential is not None: - digest.update(b"\0") - digest.update( - hashlib.sha256( - credential.encode() if isinstance(credential, str) else credential - ).digest() - ) - return digest.hexdigest() - - -def _is_python_exec_schema(schema: Mapping[str, Any]) -> bool: - properties = schema.get("properties") - if ( - schema.get("type") != "object" - or schema.get("required") != ["code"] - or not isinstance(properties, dict) - or set(properties) != {"code", "timeout_s"} - ): - return False - code = properties["code"] - timeout = properties["timeout_s"] - return ( - isinstance(code, dict) - and code.get("type") == "string" - and isinstance(timeout, dict) - and timeout.get("type") == "number" - and timeout.get("default") == MAX_EXECUTION_TIMEOUT_S - ) - - -def _validate_arguments(arguments: Mapping[str, Any]) -> dict[str, JsonValue]: - if set(arguments) - {"code", "timeout_s"}: - raise ValueError("python_exec arguments contain unknown fields") - code = arguments.get("code") - timeout = arguments.get("timeout_s", MAX_EXECUTION_TIMEOUT_S) - if not isinstance(code, str) or not code: - raise ValueError("python_exec code must be a non-empty string") - if isinstance(timeout, bool) or not isinstance(timeout, (float, int)): - raise ValueError("python_exec timeout_s must be numeric") - if not 0 < float(timeout) <= MAX_EXECUTION_TIMEOUT_S: - raise ValueError("python_exec timeout_s is outside the supported range") - return {"code": code, "timeout_s": float(timeout)} - - -def _json_tool(tool: Mapping[str, Any]) -> dict[str, JsonValue]: - return _json_object(dict(tool)) - - -def _json_object(value: Mapping[str, Any]) -> dict[str, JsonValue]: - try: - encoded = json.dumps(value, allow_nan=False) - decoded = json.loads(encoded) - except (TypeError, ValueError) as exc: - raise ValueError("value is not strict JSON") from exc - if not isinstance(decoded, dict): - raise ValueError("value must be a JSON object") - return decoded - - -def _bounded_diagnostic(exc: Exception) -> str: - message = f"{type(exc).__name__}: {exc}".replace("\r", " ").replace("\n", " ") - return message[:1024] or type(exc).__name__ diff --git a/dimos/benchmark/agent_eval/pi_process.py b/dimos/benchmark/agent_eval/pi_process.py index f1a8f8f356..121583be18 100644 --- a/dimos/benchmark/agent_eval/pi_process.py +++ b/dimos/benchmark/agent_eval/pi_process.py @@ -12,418 +12,185 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Process binding for the interactive one-tool Pi SDK host.""" +"""Launch the pinned stock Pi CLI and parse its native JSON event stream.""" from __future__ import annotations -import hashlib +from dataclasses import dataclass import json import os from pathlib import Path -import queue import subprocess -import threading import time -from typing import IO, Any -from uuid import uuid4 -from dimos.benchmark.agent_eval.artifacts import ArtifactReference -from dimos.benchmark.agent_eval.auth import RuntimeCredential -from dimos.benchmark.agent_eval.pi import PiTurn -from dimos.benchmark.agent_eval.pi_adapter import ( - CodePolicyCallLog, - McpBinding, - PythonExecBroker, -) -from dimos.benchmark.agent_eval.progress import ( - AssistantTextProgress, - FinalResponseProgress, - ProgressSink, - StatusProgress, - ToolEndProgress, - ToolStartProgress, - emit_progress, -) +PI_VERSION = "0.80.10" +MAX_STDERR_BYTES = 64 * 1024 -_PROTOCOL_VERSION = 1 -_MAX_FRAME_BYTES = 64 * 1024 -_MAX_STDERR_BYTES = 64 * 1024 -_MAX_PROGRESS_BYTES = 4 * 1024 +@dataclass(frozen=True) +class PiRunResult: + final_text: str + tool_call_count: int + duration_seconds: float + transcript_path: Path | None + stderr: str + + +class PiRunError(RuntimeError): + def __init__(self, message: str, *, stderr: str = "") -> None: + super().__init__(message) + self.stderr = stderr + + +class PiCliRunner: + """Thin stock-CLI binding; the extension owns only the `python_exec` tool.""" -class NodePiSessionFactory: def __init__( self, *, - command: tuple[str, ...], - credential: RuntimeCredential, + cli: Path, + extension: Path, model: str, thinking_level: str, - startup_timeout_s: float, - progress: ProgressSink | None = None, + timeout_s: float, ) -> None: - if not command or startup_timeout_s <= 0: - raise ValueError("Pi adapter command and startup timeout are required") - if model != "gpt-5.6-luna" or thinking_level != "medium": - raise ValueError("the pinned Pi adapter supports only gpt-5.6-luna/medium") - self.command = command - self.credential = credential - self.startup_timeout_s = startup_timeout_s - self.progress = progress + if not cli.is_file(): + raise FileNotFoundError(f"Pi {PI_VERSION} CLI is not installed: {cli}") + if not extension.is_file(): + raise FileNotFoundError( + f"Pi CodePolicy extension is not built: {extension}; " + "run `npm run build --prefix packages/pi-code-policy-extension`" + ) + self.cli = cli + self.extension = extension + self.model = model + self.thinking_level = thinking_level + self.timeout_s = timeout_s - def create( + def run( self, *, - attempt_path: Path, - public_prompt: str, - code_policy_session_id: str, - call_log: CodePolicyCallLog, - mcp: McpBinding, - ) -> NodePiSession: - session_id = f"pi_session_{uuid4().hex}" - broker = PythonExecBroker( - attempt_id=attempt_path.name, - pi_session_id=session_id, - code_policy_session_id=code_policy_session_id, - mcp=mcp, - call_log=call_log, - ) - return NodePiSession( - command=self.command, - credential=self.credential, - attempt_path=attempt_path, - session_id=session_id, - initial_prompt=public_prompt, - broker=broker, - startup_timeout_s=self.startup_timeout_s, - progress=self.progress, + prompt: str, + system_prompt: str, + mcp_url: str, + api_key: str, + run_dir: Path, + ) -> PiRunResult: + session_dir = run_dir / "pi-session" + agent_dir = run_dir / ".pi-agent" + system_prompt_path = run_dir / "system-prompt.txt" + system_prompt_path.write_text(system_prompt, encoding="utf-8") + command = ( + "node", + str(self.cli), + "--mode", + "json", + "--model", + f"openai/{self.model}", + "--thinking", + self.thinking_level, + "--session-dir", + str(session_dir), + "--name", + "dimos-frozen-eval", + "--no-builtin-tools", + "--tools", + "python_exec", + "--no-extensions", + "--extension", + str(self.extension), + "--no-skills", + "--no-prompt-templates", + "--no-themes", + "--no-context-files", + "--no-approve", + "--system-prompt", + str(system_prompt_path), + prompt, ) - - -class NodePiSession: - def __init__( - self, - *, - command: tuple[str, ...], - credential: RuntimeCredential, - attempt_path: Path, - session_id: str, - initial_prompt: str, - broker: PythonExecBroker, - startup_timeout_s: float, - progress: ProgressSink | None, - ) -> None: - self.session_id = session_id - self.policy_call_count = 0 - self._attempt_path = attempt_path - self._broker = broker - self._frames: queue.Queue[dict[str, Any] | BaseException] = queue.Queue() - self._write_lock = threading.Lock() - self._closed_evidence: dict[str, Any] | None = None - self._disposed = False - self._progress = progress - self._stderr_path = attempt_path / "pi-adapter.stderr.log" - self._stderr = bytearray() - self._process = subprocess.Popen( + env = { + "PATH": os.environ.get("PATH", ""), + "OPENAI_API_KEY": api_key, + "DIMOS_CODE_POLICY_MCP_URL": mcp_url, + "PI_CODING_AGENT_DIR": str(agent_dir), + "PI_SKIP_VERSION_CHECK": "1", + "PI_TELEMETRY": "0", + } + started = time.monotonic() + process = subprocess.Popen( command, - stdin=subprocess.PIPE, + cwd=run_dir, + env=env, + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - cwd=attempt_path, - env=_adapter_environment(credential, attempt_path), - ) - assert ( - self._process.stdin is not None - and self._process.stdout is not None - and self._process.stderr is not None - ) - self._stdin: IO[str] = self._process.stdin - self._reader = threading.Thread( - target=self._read_frames, - args=(self._process.stdout,), - name=f"pi-adapter-reader-{session_id}", - daemon=True, - ) - self._stderr_reader = threading.Thread( - target=self._read_stderr, - args=(self._process.stderr,), - name=f"pi-adapter-stderr-{session_id}", - daemon=True, - ) - self._reader.start() - self._stderr_reader.start() - emit_progress(self._progress, StatusProgress(channel="pi", message="session starting")) - self._send( - { - "version": _PROTOCOL_VERSION, - "type": "session_start", - "id": session_id, - "initial_prompt": initial_prompt, - "thinking_level": "medium", - } ) try: - started = self._await("session_started", session_id, startup_timeout_s) - except BaseException: - self._terminate_process() - raise - if started.get("tools") != ["python_exec"]: - self.dispose() - raise RuntimeError("Pi adapter activated an unexpected tool inventory") - emit_progress(self._progress, StatusProgress(channel="pi", message="session started")) - - def prompt(self, prompt: str, timeout_s: float) -> PiTurn: - if self._disposed: - raise RuntimeError("Pi session is disposed") - turn_id = f"turn_{uuid4().hex}" - self._send( - { - "version": _PROTOCOL_VERSION, - "type": "prompt", - "id": turn_id, - "text": prompt, - } - ) - frame = self._await("turn_complete", turn_id, timeout_s) - count = frame.get("policy_call_count") - if not isinstance(count, int) or count < self.policy_call_count: - raise RuntimeError("Pi adapter returned an invalid policy-call count") - self.policy_call_count = count - final_text = frame.get("final_text") - text = final_text if isinstance(final_text, str) else "" - emit_progress(self._progress, FinalResponseProgress(text=_bounded_progress(text))) - return PiTurn( - final_text=text, - policy_call_count=count, - ) - - def abort(self, timeout_s: float) -> None: - del timeout_s - if not self._disposed and self._process.poll() is None: - self._send({"version": _PROTOCOL_VERSION, "type": "abort"}) - - def dispose(self) -> None: - if self._disposed: - return - self._disposed = True - try: - if self._process.poll() is None: - self._send({"version": _PROTOCOL_VERSION, "type": "dispose"}) - try: - frame = self._await("session_closed", self.session_id, 5.0) - evidence = frame.get("evidence") - if isinstance(evidence, dict): - self._closed_evidence = evidence - except (RuntimeError, TimeoutError): - self._process.terminate() - self._process.wait(timeout=2.0) - except subprocess.TimeoutExpired: - self._terminate_process() - finally: - self._reader.join(timeout=2.0) - self._stderr_reader.join(timeout=2.0) - self._stderr_path.write_bytes(bytes(self._stderr)) - - def _terminate_process(self) -> None: - self._disposed = True - if self._process.poll() is None: - self._process.terminate() + stdout, stderr = process.communicate(timeout=self.timeout_s) + except subprocess.TimeoutExpired as exc: + process.terminate() try: - self._process.wait(timeout=2.0) + stdout, stderr = process.communicate(timeout=5) except subprocess.TimeoutExpired: - self._process.kill() - self._process.wait(timeout=2.0) - self._reader.join(timeout=2.0) - self._stderr_reader.join(timeout=2.0) - self._stderr_path.write_bytes(bytes(self._stderr)) - - def artifact_references(self) -> tuple[ArtifactReference, ...]: - if not self._disposed: - raise RuntimeError("Pi session evidence is available only after disposal") - relative_paths = ["pi-adapter.stderr.log"] - evidence = self._closed_evidence or {} - session_path = evidence.get("relative_path") - if evidence.get("persisted") is True and isinstance(session_path, str): - relative_paths.append(session_path) - for key in ("system_prompt", "initial_prompt"): - prompt = evidence.get(key) - if isinstance(prompt, dict) and isinstance(prompt.get("relative_path"), str): - relative_paths.append(prompt["relative_path"]) - return tuple(_artifact(self._attempt_path, path) for path in relative_paths) - - def _read_frames(self, output: IO[str]) -> None: - try: - for line in output: - if len(line.encode()) > _MAX_FRAME_BYTES: - raise RuntimeError("Pi adapter frame exceeds limit") - frame = json.loads(line) - if not isinstance(frame, dict) or frame.get("version") != _PROTOCOL_VERSION: - raise RuntimeError("invalid Pi adapter frame") - if frame.get("type") == "tool_call": - self._handle_tool_call(frame) - elif frame.get("type") == "transcript": - self._handle_transcript(frame) - else: - self._frames.put(frame) - except BaseException as exc: - self._frames.put(exc) - - def _handle_tool_call(self, frame: dict[str, Any]) -> None: - call_id = frame.get("id") - tool = frame.get("tool") - params = frame.get("params") - if ( - not isinstance(call_id, str) - or not isinstance(tool, str) - or not isinstance(params, dict) - ): - raise RuntimeError("malformed Pi tool call") - code = params.get("code") - if isinstance(code, str) and code: - emit_progress( - self._progress, - ToolStartProgress(code=_bounded_progress(code)), - ) - started = time.monotonic() - try: - result = self._broker.request(tool, params) - text = _mcp_text(result) - emit_progress( - self._progress, - ToolEndProgress( - ok=True, - result=_bounded_progress(text), - duration_seconds=max(0.0, time.monotonic() - started), - ), - ) - reply = { - "version": _PROTOCOL_VERSION, - "type": "tool_reply", - "id": call_id, - "ok": True, - "result": text, - } - except Exception as exc: - diagnostic = f"{type(exc).__name__}: {exc}" - emit_progress( - self._progress, - ToolEndProgress( - ok=False, - result=_bounded_progress(diagnostic), - duration_seconds=max(0.0, time.monotonic() - started), - ), - ) - reply = { - "version": _PROTOCOL_VERSION, - "type": "tool_reply", - "id": call_id, - "ok": False, - "error": diagnostic[:1024], - } - self._send(reply) - - def _handle_transcript(self, frame: dict[str, Any]) -> None: - event = frame.get("event") - if event == "assistant_text_delta": - delta = frame.get("delta") - if isinstance(delta, str) and delta: - emit_progress( - self._progress, - AssistantTextProgress(delta=_bounded_progress(delta)), - ) - elif event == "agent_start": - emit_progress(self._progress, StatusProgress(channel="pi", message="agent started")) - elif event == "turn_start": - emit_progress(self._progress, StatusProgress(channel="pi", message="turn started")) - elif event == "agent_end": - emit_progress(self._progress, StatusProgress(channel="pi", message="agent finished")) - - def _read_stderr(self, stderr: IO[str]) -> None: - for chunk in iter(lambda: stderr.read(4096), ""): - remaining = _MAX_STDERR_BYTES - len(self._stderr) - if remaining > 0: - self._stderr.extend(chunk.encode()[:remaining]) + process.kill() + stdout, stderr = process.communicate() + raise PiRunError( + f"Pi timed out after {self.timeout_s:g}s", + stderr=_bounded_stderr(stderr), + ) from exc + duration = time.monotonic() - started + stderr = _bounded_stderr(stderr) + final_text, tool_count, stop_error = parse_pi_events(stdout) + if process.returncode != 0: + raise PiRunError(f"Pi exited with status {process.returncode}", stderr=stderr) + if stop_error is not None: + raise PiRunError(stop_error, stderr=stderr) + if final_text is None: + raise PiRunError("Pi produced no final assistant response", stderr=stderr) + transcripts = sorted(session_dir.rglob("*.jsonl")) if session_dir.exists() else [] + return PiRunResult( + final_text=final_text, + tool_call_count=tool_count, + duration_seconds=duration, + transcript_path=transcripts[-1] if transcripts else None, + stderr=stderr, + ) - def _send(self, frame: dict[str, Any]) -> None: - encoded = json.dumps(frame, allow_nan=False, separators=(",", ":")) - if len(encoded.encode()) > _MAX_FRAME_BYTES: - raise ValueError("outbound Pi adapter frame exceeds limit") - with self._write_lock: - self._stdin.write(encoded + "\n") - self._stdin.flush() - def _await(self, frame_type: str, frame_id: str, timeout_s: float) -> dict[str, Any]: - deadline = time.monotonic() + timeout_s - deferred: list[dict[str, Any]] = [] +def parse_pi_events(stream: str) -> tuple[str | None, int, str | None]: + """Return the final assistant text, tool count, and terminal error.""" + final_text: str | None = None + tool_count = 0 + stop_error: str | None = None + for line in stream.splitlines(): try: - while True: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise TimeoutError(f"Pi adapter timed out waiting for {frame_type}") - try: - item = self._frames.get(timeout=remaining) - except queue.Empty as exc: - raise TimeoutError(f"Pi adapter timed out waiting for {frame_type}") from exc - if isinstance(item, BaseException): - raise RuntimeError(f"Pi adapter reader failed: {item}") from item - if item.get("type") == "protocol_error": - raise RuntimeError(str(item.get("error", "Pi adapter protocol error"))) - if item.get("type") == frame_type and item.get("id") == frame_id: - return item - deferred.append(item) - finally: - for item in deferred: - self._frames.put(item) - - -def _adapter_environment( - credential: RuntimeCredential, - attempt_path: Path, -) -> dict[str, str]: - env = { - "PATH": os.environ.get("PATH", ""), - "PI_SPATIAL_AGENT_CWD": str(attempt_path), - "PI_SPATIAL_SESSION_DIR": "pi-session", - } - if credential.auth_mode == "subscription": - env["PI_SPATIAL_AUTH_MODE"] = "codex-oauth" - env["PI_SPATIAL_AUTH_PATH"] = credential.binding_name - elif credential.auth_mode == "environment" and credential.value: - env["PI_SPATIAL_AUTH_MODE"] = "openai-api-key" - env["OPENAI_API_KEY"] = credential.value - else: - raise ValueError("unsupported or incomplete Pi credential binding") - return env - - -def _mcp_text(result: dict[str, Any]) -> str: - content = result.get("content") - if not isinstance(content, list) or not content: - return "" - first = content[0] - if isinstance(first, dict): - text = first.get("text") - if isinstance(text, str): - return text - return json.dumps(first, allow_nan=False, separators=(",", ":"))[:32_000] + event = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(event, dict): + continue + if event.get("type") == "tool_execution_start": + tool_count += 1 + if event.get("type") != "message_end": + continue + message = event.get("message") + if not isinstance(message, dict) or message.get("role") != "assistant": + continue + text = "".join( + str(item.get("text", "")) + for item in message.get("content", []) + if isinstance(item, dict) and item.get("type") == "text" + ) + final_text = text + stop_reason = message.get("stopReason") + if stop_reason in {"error", "aborted"}: + stop_error = str(message.get("errorMessage") or f"Pi request {stop_reason}") + return final_text, tool_count, stop_error -def _bounded_progress(value: str) -> str: +def _bounded_stderr(value: str) -> str: encoded = value.encode() - if len(encoded) <= _MAX_PROGRESS_BYTES: + if len(encoded) <= MAX_STDERR_BYTES: return value - marker = "\n… [truncated]" - keep = _MAX_PROGRESS_BYTES - len(marker.encode()) - return encoded[:keep].decode(errors="ignore") + marker - - -def _artifact(root: Path, relative_path: str) -> ArtifactReference: - if not relative_path or relative_path.startswith("/") or ".." in relative_path.split("/"): - raise ValueError("Pi evidence path is not attempt-relative") - data = (root / relative_path).read_bytes() - return ArtifactReference( - path=relative_path, - sha256=hashlib.sha256(data).hexdigest(), - size_bytes=len(data), - ) + return encoded[:MAX_STDERR_BYTES].decode(errors="ignore") diff --git a/dimos/benchmark/agent_eval/single_case.py b/dimos/benchmark/agent_eval/single_case.py index 52b7a1867b..287c8e825a 100644 --- a/dimos/benchmark/agent_eval/single_case.py +++ b/dimos/benchmark/agent_eval/single_case.py @@ -12,234 +12,194 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""One immutable case bound to typed local agent execution configuration.""" +"""Direct runner for one frozen-memory Pi evaluation case.""" from __future__ import annotations -import hashlib +import json import os from pathlib import Path +import re +import shutil +import tempfile import time -from typing import Annotated, Literal, TypeVar - -from pydantic import Field - -from dimos.benchmark.agent_eval.auth import RuntimeCredential -from dimos.benchmark.agent_eval.base import BaseEvalModel -from dimos.benchmark.agent_eval.case import ( - AgentCondition, - AgentOutcome, - EvalCase, - FrozenRecordingSource, - Prediction, - RuntimeBinding, -) -from dimos.benchmark.agent_eval.pi_adapter import credential_binding_sha256 -from dimos.benchmark.agent_eval.pi_process import NodePiSessionFactory -from dimos.benchmark.agent_eval.progress import ( - CaseHeaderProgress, - ProgressSink, - StatusProgress, - emit_progress, -) + +from dimos.agents.code_policy_core import CodePolicySessionConfig, FrozenMemoryEnvironment +from dimos.agents.code_policy_server import CodePolicyMcpServer +from dimos.benchmark.agent_eval.models import CompactEvalResult, EvalCase, EvalRunConfig +from dimos.benchmark.agent_eval.pi_process import PiCliRunner, PiRunError +from dimos.benchmark.agent_eval.progress import ProgressSink, StatusProgress, emit_progress from dimos.benchmark.short_horizon_qa.eval import ( load_exact_integer_oracle, - run_frozen_case, + parse_integer_prediction, ) +from dimos.benchmark.short_horizon_qa.models import MapperSettings from dimos.benchmark.short_horizon_qa.prepare import prepare_bundle -from dimos.constants import CACHE_DIR, STATE_DIR - -DEFAULT_MODEL: Literal["gpt-5.6-luna"] = "gpt-5.6-luna" -DEFAULT_OUTPUT_ROOT = STATE_DIR / "evals" -DEFAULT_CODEX_AUTH_PATH = Path.home() / ".pi" / "agent" / "auth.json" -DEFAULT_OPENAI_API_KEY_ENV = "OPENAI_API_KEY" -SINGLE_CASE_TURN_TIMEOUT_SECONDS = 600.0 -EvalModelT = TypeVar("EvalModelT", bound=BaseEvalModel) - - -class CodexOAuthConfig(BaseEvalModel): - mode: Literal["codex-oauth"] = "codex-oauth" - path: Path | None = None - - -class OpenAIApiKeyConfig(BaseEvalModel): - mode: Literal["openai-api-key"] = "openai-api-key" - env: str = Field(default=DEFAULT_OPENAI_API_KEY_ENV, min_length=1) - - -AgentAuthConfig = Annotated[ - CodexOAuthConfig | OpenAIApiKeyConfig, - Field(discriminator="mode"), -] - +from dimos.benchmark.short_horizon_qa.service import load_bundle +from dimos.constants import CACHE_DIR +from dimos.memory2.cli.dataset import resolve_dataset -class PiAgentConfig(BaseEvalModel): - backend: Literal["pi"] = "pi" - model: Literal["gpt-5.6-luna"] = DEFAULT_MODEL - thinking_level: Literal["medium"] = "medium" - auth: AgentAuthConfig = Field(default_factory=CodexOAuthConfig) +TURN_TIMEOUT_SECONDS = 600.0 +SYSTEM_PROMPT = """You are answering a question about a frozen robot recording. -class EvalRunConfig(BaseEvalModel): - agent: PiAgentConfig = Field(default_factory=PiAgentConfig) - - -class CompactEvalResult(BaseEvalModel): - attempt_id: str - case_id: str - source: str - progress: float | None - question: str - attempt_status: Literal["completed", "failed"] - task_result: Literal["passed", "failed", "not_evaluated"] - reason: str - prediction_status: Literal["parsed", "invalid"] | None = None - integer_answer: int | None = None - agent: AgentCondition - tool_call_count: int = Field(ge=0) - duration_seconds: float = Field(ge=0) - artifact_path: Path +You have exactly one tool, `python_exec`. It runs trusted, unsandboxed Python in a +persistent Jupyter kernel with a read-only `memory` object. Inspect Memory2 streams +and compute the answer from the recording. Do not guess. End with exactly one line: +ANSWER: +""" def execute_single_case( case_path: Path, *, config: EvalRunConfig, - output_root: Path = DEFAULT_OUTPUT_ROOT, + output: Path, progress: ProgressSink | None = None, ) -> CompactEvalResult: - """Preflight and execute exactly one static frozen-memory case.""" + """Preflight, run, and atomically publish exactly one result directory.""" path = case_path.expanduser().resolve() + output = output.expanduser().resolve() + _validate_output(output) emit_progress(progress, StatusProgress(channel="eval", message="loading case")) case = EvalCase.model_validate_json(path.read_bytes()) - if not isinstance(case.source, FrozenRecordingSource): - raise ValueError("single-case CLI currently supports frozen-memory cases only") - task = case.task - emit_progress( - progress, - CaseHeaderProgress( - case_id=case.case_id, - source=case.source.recording, - progress=case.source.progress, - question=getattr(task, "prompt", ""), - ), - ) - - # Resolve all private case material before starting Pi. - emit_progress(progress, StatusProgress(channel="eval", message="verifying validator")) - load_exact_integer_oracle(case, path.parent) - emit_progress(progress, StatusProgress(channel="eval", message="preparing frozen memory")) - bundle = _materialize_frozen_memory(case) - emit_progress(progress, StatusProgress(channel="eval", message="frozen memory ready")) - credential, binding_digest = _resolve_credential(config.agent.auth) - adapter = _adapter_entrypoint() - condition = AgentCondition( - agent_id="pi-code-policy", - adapter="pi-node", - model=config.agent.model, - thinking_level=config.agent.thinking_level, - ) - runtime = RuntimeBinding( - runtime_id="local-standalone-code-policy", - parameters={ - "auth_mode": config.agent.auth.mode, - "credential_binding_sha256": binding_digest, - "turn_timeout_seconds": SINGLE_CASE_TURN_TIMEOUT_SECONDS, - }, - ) - factory = NodePiSessionFactory( - command=("node", str(adapter)), - credential=credential, + oracle = load_exact_integer_oracle(case, path.parent) + api_key = os.environ.get(config.agent.api_key_env) + if not api_key: + raise ValueError(f"API key environment variable {config.agent.api_key_env!r} is unset") + bundle = _materialize_frozen_memory(case, progress) + _, cutoff, source_path, derived_path = load_bundle(bundle, progress=case.source.progress) + cli, extension = _pi_paths() + runner = PiCliRunner( + cli=cli, + extension=extension, model=config.agent.model, thinking_level=config.agent.thinking_level, - startup_timeout_s=180.0, - progress=progress, + timeout_s=TURN_TIMEOUT_SECONDS, ) - emit_progress(progress, StatusProgress(channel="eval", message="starting attempt")) - started = time.monotonic() - engine_result = run_frozen_case( - case=case, - bundle=bundle, - private_root=path.parent, - output_root=output_root.expanduser(), - pi_factory=factory, - agent_condition=condition, - runtime_binding=runtime, - turn_timeout_s=SINGLE_CASE_TURN_TIMEOUT_SECONDS, - ) - duration = time.monotonic() - started - emit_progress(progress, StatusProgress(channel="eval", message="attempt finished")) - prediction = _optional_model(engine_result.attempt_path / "prediction.v1.json", Prediction) - agent_outcome = _optional_model( - engine_result.attempt_path / "agent-outcome.v1.json", AgentOutcome - ) - return CompactEvalResult( - attempt_id=engine_result.outcome.attempt_id, - case_id=case.case_id, - source=case.source.recording, - progress=case.source.progress, - question=getattr(task, "prompt", ""), - attempt_status=engine_result.outcome.attempt_status, - task_result=engine_result.outcome.task_result, - reason=engine_result.outcome.reason, - prediction_status=prediction.status if prediction is not None else None, - integer_answer=prediction.integer_answer if prediction is not None else None, - agent=condition, - tool_call_count=agent_outcome.tool_call_count if agent_outcome is not None else 0, - duration_seconds=duration, - artifact_path=engine_result.attempt_path, - ) - -def _resolve_credential(auth: AgentAuthConfig) -> tuple[RuntimeCredential, str]: - if isinstance(auth, CodexOAuthConfig): - configured = auth.path or Path( - os.environ.get("PI_SPATIAL_AUTH_PATH", DEFAULT_CODEX_AUTH_PATH) + output.parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix=f".{output.name}-", dir=output.parent)) + runtime_dir = temporary / "runtime" + runtime_dir.mkdir() + started = time.monotonic() + stderr = "" + server: CodePolicyMcpServer | None = None + try: + emit_progress(progress, StatusProgress(channel="eval", message="starting agent")) + server = CodePolicyMcpServer( + CodePolicySessionConfig( + environment=FrozenMemoryEnvironment( + recording_path=str(source_path), + derived_recording_path=str(derived_path), + memory_cutoff_timestamp=cutoff.cutoff_timestamp, + ) + ) ) - path = configured.expanduser().resolve() - if not path.is_file(): - raise FileNotFoundError( - f"Codex OAuth credential not found at {path}; use --agent.auth.path" + try: + server.start() + pi_result = runner.run( + prompt=_agent_prompt(case), + system_prompt=SYSTEM_PROMPT, + mcp_url=server.mcp_url, + api_key=api_key, + run_dir=runtime_dir, + ) + stderr = pi_result.stderr + if pi_result.transcript_path is not None: + shutil.copy2(pi_result.transcript_path, temporary / "pi-transcript.jsonl") + prediction = parse_integer_prediction(pi_result.final_text) + passed = ( + prediction.status == "parsed" and prediction.integer_answer == oracle.expected_count ) - material = path.read_bytes() - return ( - RuntimeCredential(auth_mode="subscription", binding_name=str(path), value=None), - credential_binding_sha256("subscription", str(path), material), + result = CompactEvalResult( + case_id=case.case_id, + recording=case.source.recording, + progress=case.source.progress, + model=config.agent.model, + thinking_level=config.agent.thinking_level, + final_response=pi_result.final_text, + prediction_status=prediction.status, + integer_answer=prediction.integer_answer, + passed=passed, + validator_revision=case.validator.revision, + tool_call_count=pi_result.tool_call_count, + duration_seconds=time.monotonic() - started, + ) + finally: + server.stop() + except Exception as exc: + if isinstance(exc, PiRunError): + stderr = exc.stderr + result = CompactEvalResult( + case_id=case.case_id, + recording=case.source.recording, + progress=case.source.progress, + model=config.agent.model, + thinking_level=config.agent.thinking_level, + prediction_status="not_evaluated", + passed=None, + validator_revision=case.validator.revision, + tool_call_count=server.session.execution_count if server is not None else 0, + duration_seconds=time.monotonic() - started, + infra_error=f"{type(exc).__name__}: {exc}", ) - value = os.environ.get(auth.env) - if not value: - raise ValueError(f"credential environment variable {auth.env!r} is unset") - return ( - RuntimeCredential(auth_mode="environment", binding_name=auth.env, value=value), - credential_binding_sha256("environment", auth.env, value), + finally: + shutil.rmtree(runtime_dir, ignore_errors=True) + + if stderr: + (temporary / "stderr.log").write_text(stderr, encoding="utf-8") + (temporary / "result.json").write_text( + json.dumps(result.model_dump(mode="json"), indent=2, sort_keys=True) + "\n", + encoding="utf-8", ) - - -def _materialize_frozen_memory(case: EvalCase) -> Path: - source = case.source - assert isinstance(source, FrozenRecordingSource) - key = hashlib.sha256(source.model_dump_json().encode()).hexdigest()[:24] + if output.exists(): + output.rmdir() + os.replace(temporary, output) + emit_progress(progress, StatusProgress(channel="eval", message="result published")) + return result + + +def _validate_output(output: Path) -> None: + if output.exists() and (not output.is_dir() or any(output.iterdir())): + raise FileExistsError(f"Output must be absent or an empty directory: {output}") + + +def _materialize_frozen_memory(case: EvalCase, progress: ProgressSink | None) -> Path: + source_path = resolve_dataset(case.source.recording).resolve() + stat = source_path.stat() + stem = re.sub(r"[^A-Za-z0-9_.-]+", "-", source_path.stem)[:64] + mapper = MapperSettings() + raw_key = ( + f"{stem}-{stat.st_size}-{stat.st_mtime_ns}-p{case.source.progress:.9f}-" + f"v{mapper.voxel_size_m}-b{mapper.block_count}-d{mapper.device}-" + f"c{int(mapper.carve_columns)}-f{mapper.frame_id}-e{mapper.emit_every}" + ) + key = re.sub(r"[^A-Za-z0-9_.-]+", "-", raw_key) bundle = CACHE_DIR / "agent_eval" / "frozen_memory" / key if not (bundle / "manifest.v1.json").is_file(): + emit_progress(progress, StatusProgress(channel="eval", message="preparing memory")) bundle.parent.mkdir(parents=True, exist_ok=True) - prepare_bundle(source.recording, [], bundle, progress=[source.progress]) + prepare_bundle( + case.source.recording, + [], + bundle, + progress=[case.source.progress], + mapper=mapper, + ) return bundle -def _adapter_entrypoint() -> Path: - path = ( - Path(__file__).resolve().parents[3] - / "packages" - / "pi-code-policy-adapter" - / "dist" - / "code-policy-main.js" - ) - if not path.is_file(): - raise FileNotFoundError( - "Pi adapter is not built; run npm run build in packages/pi-code-policy-adapter" - ) - return path +def _pi_paths() -> tuple[Path, Path]: + package = Path(__file__).resolve().parents[3] / "packages" / "pi-code-policy-extension" + cli = package / "node_modules" / "@earendil-works" / "pi-coding-agent" / "dist" / "cli.js" + extension = package / "dist" / "python-exec.js" + return cli, extension -def _optional_model(path: Path, model: type[EvalModelT]) -> EvalModelT | None: - return model.model_validate_json(path.read_bytes()) if path.is_file() else None +def _agent_prompt(case: EvalCase) -> str: + return ( + f"{case.task.prompt}\n\n" + "Use python_exec to inspect the read-only recording. " + f"End with `{case.task.answer_marker} `." + ) diff --git a/dimos/benchmark/agent_eval/store.py b/dimos/benchmark/agent_eval/store.py deleted file mode 100644 index 8b87cd6285..0000000000 --- a/dimos/benchmark/agent_eval/store.py +++ /dev/null @@ -1,285 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Exclusive, append-only, non-overwriting storage for one local attempt.""" - -from __future__ import annotations - -from datetime import UTC, datetime -import fcntl -import hashlib -import os -from pathlib import Path -import time -from typing import Any -from uuid import uuid4 - -from pydantic import BaseModel, JsonValue - -from dimos.benchmark.agent_eval.artifacts import ( - ArtifactReference, - AttemptId, - LifecycleEvent, - NormalizedOutcome, - OperationId, -) -from dimos.benchmark.agent_eval.case import EvalOutcome -from dimos.benchmark.agent_eval.json import canonical_json - - -class AttemptAlreadyActiveError(RuntimeError): - pass - - -class AttemptStore: - """Own a target-wide lock and one fresh immutable attempt directory.""" - - def __init__(self, output_root: Path, attempt_id: str | None = None) -> None: - self.output_root = output_root.resolve() - self.attempt_id: AttemptId = attempt_id or f"attempt_{uuid4().hex}" - self.path = self.output_root / self.attempt_id - self._lock_fd = -1 - self._events_fd = -1 - self._started_monotonic = time.monotonic() - self._event_sequence = 0 - self._last_event_offset = 0.0 - self._closed = False - self._reserve() - - def _reserve(self) -> None: - self.output_root.mkdir(parents=True, exist_ok=True) - lock_path = self.output_root / ".attached-target.lock" - self._lock_fd = os.open(lock_path, os.O_RDWR | os.O_CREAT | os.O_CLOEXEC, 0o600) - try: - fcntl.flock(self._lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError as exc: - os.close(self._lock_fd) - self._lock_fd = -1 - raise AttemptAlreadyActiveError( - f"another attempt is active for {self.output_root}" - ) from exc - try: - self.path.mkdir(mode=0o700) - except BaseException: - self.close() - raise - try: - self._events_fd = os.open( - self.path / "events.jsonl", - os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, - 0o600, - ) - except BaseException: - self.close() - raise - - def append_event( - self, - kind: str, - *, - operation_id: str | None = None, - payload: dict[str, JsonValue] | None = None, - ) -> LifecycleEvent: - self._require_open() - offset = max( - self._last_event_offset, - time.monotonic() - self._started_monotonic, - ) - self._event_sequence += 1 - event = LifecycleEvent( - sequence=self._event_sequence, - attempt_id=self.attempt_id, - operation_id=operation_id, - occurred_at=datetime.now(UTC), - monotonic_offset_s=offset, - kind=kind, - payload=payload or {}, - ) - encoded = canonical_json(event.model_dump(mode="json")) + b"\n" - _write_all(self._events_fd, encoded) - os.fsync(self._events_fd) - self._last_event_offset = offset - return event - - def write_artifact( - self, relative_path: str, value: BaseModel | JsonValue | bytes | str - ) -> ArtifactReference: - self._require_open() - path = self._resolve_relative(relative_path) - path.parent.mkdir(parents=True, exist_ok=True) - if isinstance(value, BaseModel): - data = canonical_json(value.model_dump(mode="json")) + b"\n" - elif isinstance(value, bytes): - data = value - elif isinstance(value, str): - data = value.encode("utf-8") - else: - data = canonical_json(value) + b"\n" - descriptor = os.open( - path, - os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, - 0o600, - ) - try: - _write_all(descriptor, data) - os.fsync(descriptor) - except BaseException: - os.close(descriptor) - path.unlink(missing_ok=True) - raise - else: - os.close(descriptor) - _fsync_directory(path.parent) - return ArtifactReference( - path=relative_path, - sha256=hashlib.sha256(data).hexdigest(), - size_bytes=len(data), - ) - - def write_outcome(self, outcome: NormalizedOutcome) -> ArtifactReference: - self._require_open() - if outcome.attempt_id != self.attempt_id: - raise ValueError("outcome attempt identity mismatch") - relative_path = "outcome.v1.json" - final_path = self.path / relative_path - temp_path = self.path / f".outcome.v1.json.tmp-{uuid4().hex}" - data = canonical_json(outcome.model_dump(mode="json")) + b"\n" - descriptor = os.open( - temp_path, - os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, - 0o600, - ) - try: - _write_all(descriptor, data) - os.fsync(descriptor) - os.close(descriptor) - descriptor = -1 - os.link(temp_path, final_path) - _fsync_directory(self.path) - finally: - if descriptor >= 0: - os.close(descriptor) - temp_path.unlink(missing_ok=True) - return ArtifactReference( - path=relative_path, - sha256=hashlib.sha256(data).hexdigest(), - size_bytes=len(data), - ) - - def write_eval_outcome(self, outcome: EvalOutcome) -> ArtifactReference: - """Atomically retain the backend-neutral terminal outcome.""" - if outcome.attempt_id != self.attempt_id: - raise ValueError("outcome attempt identity mismatch") - return self._write_terminal("outcome.v1.json", outcome) - - def _write_terminal(self, relative_path: str, value: BaseModel) -> ArtifactReference: - self._require_open() - final_path = self.path / relative_path - temp_path = self.path / f".{relative_path}.tmp-{uuid4().hex}" - data = canonical_json(value.model_dump(mode="json")) + b"\n" - descriptor = os.open( - temp_path, - os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, - 0o600, - ) - try: - _write_all(descriptor, data) - os.fsync(descriptor) - os.close(descriptor) - descriptor = -1 - os.link(temp_path, final_path) - _fsync_directory(self.path) - finally: - if descriptor >= 0: - os.close(descriptor) - temp_path.unlink(missing_ok=True) - return ArtifactReference( - path=relative_path, - sha256=hashlib.sha256(data).hexdigest(), - size_bytes=len(data), - ) - - def verify_artifacts(self, artifacts: tuple[ArtifactReference, ...]) -> bool: - """Return whether every admitted reference still matches retained bytes.""" - for artifact in artifacts: - try: - path = self._resolve_relative(artifact.path) - data = path.read_bytes() - except (OSError, ValueError): - return False - if ( - len(data) != artifact.size_bytes - or hashlib.sha256(data).hexdigest() != artifact.sha256 - ): - return False - return True - - def _resolve_relative(self, relative_path: str) -> Path: - if not relative_path or relative_path.startswith("/") or ".." in relative_path.split("/"): - raise ValueError("artifact path must be attempt-relative") - result = self.path / relative_path - if result.resolve().parent != self.path and self.path not in result.resolve().parents: - raise ValueError("artifact path escapes attempt directory") - return result - - def _require_open(self) -> None: - if self._closed: - raise RuntimeError("attempt store is closed") - - def close(self) -> None: - if self._closed: - return - self._closed = True - first_error: OSError | None = None - if self._events_fd >= 0: - try: - os.close(self._events_fd) - except OSError as exc: - first_error = exc - finally: - self._events_fd = -1 - if self._lock_fd >= 0: - try: - fcntl.flock(self._lock_fd, fcntl.LOCK_UN) - os.close(self._lock_fd) - except OSError as exc: - first_error = first_error or exc - finally: - self._lock_fd = -1 - if first_error is not None: - raise first_error - - def __enter__(self) -> AttemptStore: - return self - - def __exit__(self, *_args: Any) -> None: - self.close() - - -def new_operation_id() -> OperationId: - return f"operation_{uuid4().hex}" - - -def _write_all(descriptor: int, data: bytes) -> None: - view = memoryview(data) - while view: - view = view[os.write(descriptor, view) :] - - -def _fsync_directory(path: Path) -> None: - descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC) - try: - os.fsync(descriptor) - finally: - os.close(descriptor) diff --git a/dimos/benchmark/agent_eval/test_case.py b/dimos/benchmark/agent_eval/test_case.py deleted file mode 100644 index 6fe35212b6..0000000000 --- a/dimos/benchmark/agent_eval/test_case.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import json - -from pydantic import ValidationError -import pytest - -from dimos.benchmark.agent_eval.case import ( - AgentCondition, - AttemptRequest, - EvalCase, - EvalOutcome, - ExactIntegerValidatorRef, - FrozenCodePolicyInteraction, - FrozenRecordingSource, - IntegerQuestionTask, - Prediction, - RuntimeBinding, -) - - -def _case(prompt: str = "How many rooms?") -> EvalCase: - return EvalCase.compile( - case_id="office-room-count", - source=FrozenRecordingSource(recording="office", progress=1.0), - task=IntegerQuestionTask(prompt=prompt), - interaction=FrozenCodePolicyInteraction(driver_revision="v1"), - validator=ExactIntegerValidatorRef( - revision="exact-v1", - private_path="private/oracle.json", - private_sha256="a" * 64, - ), - ) - - -def test_compiled_case_has_stable_fingerprint_and_public_projection() -> None: - first = _case() - second = _case() - - assert first.fingerprint == second.fingerprint - public = first.public_projection().model_dump(mode="json") - assert "validator" not in public - assert "private/oracle.json" not in json.dumps(public) - assert "a" * 64 not in json.dumps(public) - - -def test_case_fingerprint_binds_task_and_private_validator() -> None: - assert _case("How many rooms?").fingerprint != _case("Count the rooms.").fingerprint - encoded = _case().model_dump(mode="json") - encoded["fingerprint"] = "0" * 64 - with pytest.raises(ValidationError, match="fingerprint"): - EvalCase.model_validate(encoded) - - -def test_case_rejects_missing_contract_and_unknown_discriminator() -> None: - encoded = _case().model_dump(mode="json") - del encoded["validator"] - with pytest.raises(ValidationError): - EvalCase.model_validate(encoded) - - encoded = _case().model_dump(mode="json") - encoded["interaction"]["kind"] = "prompt_dump" - with pytest.raises(ValidationError, match="frozen_code_policy"): - EvalCase.model_validate(encoded) - - -@pytest.mark.parametrize("progress", [-0.1, 1.1, float("inf"), float("nan")]) -def test_frozen_source_rejects_invalid_progress(progress: float) -> None: - with pytest.raises(ValidationError): - FrozenRecordingSource(recording="office", progress=progress) - - -def test_one_source_can_back_independent_tasks() -> None: - first = _case("Question one") - second = _case("Question two") - assert first.source == second.source - assert first.task != second.task - assert first.fingerprint != second.fingerprint - - -def test_attempt_request_keeps_runtime_out_of_case_identity() -> None: - case = _case() - request = AttemptRequest( - case=case, - agent=AgentCondition( - agent_id="pi", - adapter="pi-node", - model="gpt-5.6-luna", - thinking_level="medium", - ), - runtime=RuntimeBinding(runtime_id="local", parameters={"port": 10090}), - ) - assert request.case.fingerprint == case.fingerprint - - -def test_prediction_status_is_strict() -> None: - common = { - "case_id": "case", - "attempt_id": "attempt", - "agent_session_id": "pi", - "interaction_session_id": "code-policy", - "parser_revision": "v1", - "final_text": "ANSWER: 4", - } - Prediction(**common, status="parsed", integer_answer=4) - Prediction(**common, status="invalid", diagnostic="missing marker") - with pytest.raises(ValidationError): - Prediction(**common, status="parsed", diagnostic="bad") - - -@pytest.mark.parametrize( - ("attempt_status", "task_result", "valid"), - [ - ("completed", "passed", True), - ("completed", "failed", True), - ("completed", "not_evaluated", False), - ("failed", "not_evaluated", True), - ("failed", "failed", False), - ], -) -def test_outcome_separates_operation_from_task( - attempt_status: str, task_result: str, valid: bool -) -> None: - values = { - "attempt_id": "attempt", - "attempt_status": attempt_status, - "task_result": task_result, - "reason": "test", - } - if valid: - EvalOutcome.model_validate(values) - else: - with pytest.raises(ValidationError): - EvalOutcome.model_validate(values) diff --git a/dimos/benchmark/agent_eval/test_engine.py b/dimos/benchmark/agent_eval/test_engine.py deleted file mode 100644 index 897f6e270d..0000000000 --- a/dimos/benchmark/agent_eval/test_engine.py +++ /dev/null @@ -1,281 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -import pytest - -from dimos.benchmark.agent_eval.case import ( - AgentCondition, - AgentOutcome, - AttemptRequest, - EvalCase, - ExactIntegerValidatorRef, - FrozenCodePolicyInteraction, - FrozenRecordingSource, - IntegerQuestionTask, - PrivateScore, - RuntimeBinding, -) -from dimos.benchmark.agent_eval.engine import AttemptEngine -from dimos.benchmark.agent_eval.interfaces import AttemptContext, PreparedSource -from dimos.benchmark.agent_eval.store import AttemptStore - - -def _request() -> AttemptRequest: - case = EvalCase.compile( - case_id="case", - source=FrozenRecordingSource(recording="recording", progress=1.0), - task=IntegerQuestionTask(prompt="Question"), - interaction=FrozenCodePolicyInteraction(driver_revision="v1"), - validator=ExactIntegerValidatorRef( - revision="v1", - private_path="private/oracle.json", - private_sha256="a" * 64, - ), - ) - return AttemptRequest( - case=case, - agent=AgentCondition( - agent_id="agent", - adapter="fake", - model="fake", - thinking_level="off", - ), - runtime=RuntimeBinding(runtime_id="local"), - ) - - -class FakeAgent: - def __init__(self, fail: bool = False, cleanup_fail: bool = False) -> None: - self.fail = fail - self.cleanup_fail = cleanup_fail - self.closed = False - - def run(self, *, task: Any, context: AttemptContext, interface: Any = None) -> AgentOutcome: - del task, context, interface - if self.fail: - raise RuntimeError("agent failed") - return AgentOutcome( - final_text="ANSWER: 4", tool_call_count=1, terminal_reason="agent completed" - ) - - def close(self) -> None: - self.closed = True - if self.cleanup_fail: - raise RuntimeError("agent cleanup failed") - - -class FakeSource: - def __init__(self, fail: bool = False) -> None: - self.fail = fail - self.closed = False - - def prepare(self, *, source: Any, context: AttemptContext, evidence: Any) -> PreparedSource: - del source, context - evidence.artifact("source-receipt.v1.json", {"ready": True}) - if self.fail: - raise RuntimeError("source failed") - return PreparedSource(public={"ready": True}, receipt={"source": "fake"}) - - def close(self) -> None: - self.closed = True - - -class FakeInteraction: - def __init__(self, fail: bool = False) -> None: - self.fail = fail - self.closed = False - - def run( - self, - *, - case: EvalCase, - prepared_source: PreparedSource, - agent: FakeAgent, - context: AttemptContext, - evidence: Any, - ) -> AgentOutcome: - del case, prepared_source, evidence - if self.fail: - raise RuntimeError("interaction failed") - return agent.run(task=context.request.case.task, context=context) - - def close(self) -> None: - self.closed = True - - -class FakeValidatorSession: - def __init__( - self, context: AttemptContext, *, passed: bool, evaluate_fail: bool = False - ) -> None: - self.context = context - self.passed = passed - self.evaluate_fail = evaluate_fail - self.closed = False - - def evaluate(self, outcome: AgentOutcome) -> PrivateScore: - del outcome - if self.evaluate_fail: - raise RuntimeError("validation failed") - return PrivateScore( - case_id=self.context.request.case.case_id, - attempt_id=self.context.attempt_id, - validator_revision="v1", - passed=self.passed, - prediction_status="parsed" if self.passed else "invalid", - ) - - def close(self) -> None: - self.closed = True - - -class FakeValidator: - def __init__( - self, *, passed: bool = True, prepare_fail: bool = False, evaluate_fail: bool = False - ) -> None: - self.passed = passed - self.prepare_fail = prepare_fail - self.evaluate_fail = evaluate_fail - - def prepare( - self, - *, - case: EvalCase, - prepared_source: PreparedSource, - context: AttemptContext, - evidence: Any, - ) -> FakeValidatorSession: - del case, prepared_source, evidence - if self.prepare_fail: - raise RuntimeError("validator prepare failed") - return FakeValidatorSession(context, passed=self.passed, evaluate_fail=self.evaluate_fail) - - -def _run(tmp_path: Path, **kwargs: Any): - source = FakeSource(fail=kwargs.get("source_fail", False)) - interaction = FakeInteraction(fail=kwargs.get("interaction_fail", False)) - agent = FakeAgent( - fail=kwargs.get("agent_fail", False), - cleanup_fail=kwargs.get("cleanup_fail", False), - ) - validator = FakeValidator( - passed=kwargs.get("passed", True), - prepare_fail=kwargs.get("validator_prepare_fail", False), - evaluate_fail=kwargs.get("validator_evaluate_fail", False), - ) - result = AttemptEngine( - request=_request(), - output_root=tmp_path, - source=source, - interaction=interaction, - validator=validator, - agent=agent, - ).run() - return result, source, interaction, agent - - -@pytest.mark.parametrize("passed", [True, False]) -def test_engine_completes_pass_and_wrong_answer(tmp_path: Path, passed: bool) -> None: - result, source, interaction, agent = _run(tmp_path, passed=passed) - assert result.outcome.attempt_status == "completed" - assert result.outcome.task_result == ("passed" if passed else "failed") - assert (result.attempt_path / "score.private.v1.json").is_file() - assert source.closed and interaction.closed and agent.closed - - -@pytest.mark.parametrize( - "failure", - [ - "source_fail", - "interaction_fail", - "agent_fail", - "validator_prepare_fail", - "validator_evaluate_fail", - ], -) -def test_engine_retains_failed_prefix_and_not_evaluated(tmp_path: Path, failure: str) -> None: - result, source, interaction, agent = _run(tmp_path, **{failure: True}) - assert result.outcome.attempt_status == "failed" - assert result.outcome.task_result == "not_evaluated" - assert (result.attempt_path / "outcome.v1.json").is_file() - assert (result.attempt_path / "events.jsonl").is_file() - assert source.closed and interaction.closed and agent.closed - - -def test_cleanup_failure_invalidates_completed_attempt(tmp_path: Path) -> None: - result, _, _, _ = _run(tmp_path, cleanup_fail=True) - assert result.outcome.attempt_status == "failed" - assert result.outcome.task_result == "not_evaluated" - assert "cleanup" in result.outcome.reason - - -def test_engine_writes_exactly_one_terminal_outcome(tmp_path: Path) -> None: - result, _, _, _ = _run(tmp_path) - assert [path.name for path in result.attempt_path.glob("outcome*")] == ["outcome.v1.json"] - - -def test_manifest_write_failure_returns_failed_attempt_and_releases_lock( - tmp_path: Path, mocker -) -> None: - original = AttemptStore.write_artifact - - def fail_manifest(store, relative_path, value): - if relative_path == "attempt-manifest.v1.json": - raise OSError("manifest disk failure") - return original(store, relative_path, value) - - mocker.patch.object(AttemptStore, "write_artifact", autospec=True, side_effect=fail_manifest) - - result, _, _, _ = _run(tmp_path) - - assert result.outcome.attempt_status == "failed" - assert "finalization" in result.outcome.reason - with AttemptStore(tmp_path) as subsequent: - assert subsequent.path != result.attempt_path - - -def test_terminal_publication_failure_returns_failed_attempt_and_releases_lock( - tmp_path: Path, mocker -) -> None: - mocker.patch.object( - AttemptStore, - "write_eval_outcome", - autospec=True, - side_effect=OSError("terminal link failure"), - ) - - result, _, _, _ = _run(tmp_path) - - assert result.outcome.attempt_status == "failed" - assert "terminal publication" in result.outcome.reason - assert not (result.attempt_path / "outcome.v1.json").exists() - with AttemptStore(tmp_path) as subsequent: - assert subsequent.path != result.attempt_path - - -def test_event_fsync_failure_still_cleans_resources_and_releases_lock( - tmp_path: Path, mocker -) -> None: - mocker.patch("dimos.benchmark.agent_eval.store.os.fsync", side_effect=OSError("fsync failed")) - - result, source, interaction, agent = _run(tmp_path) - - assert result.outcome.attempt_status == "failed" - assert source.closed and interaction.closed and agent.closed - with AttemptStore(tmp_path) as subsequent: - assert subsequent.path != result.attempt_path diff --git a/dimos/benchmark/agent_eval/test_import_boundaries.py b/dimos/benchmark/agent_eval/test_import_boundaries.py deleted file mode 100644 index b367c668e2..0000000000 --- a/dimos/benchmark/agent_eval/test_import_boundaries.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import ast -from pathlib import Path - -FORBIDDEN_PREFIXES = ( - "dimos.benchmark.dimsim", - "dimos.benchmark.spatial", -) - - -def test_focused_evaluation_slice_has_no_live_benchmark_imports() -> None: - package = Path(__file__).parent - violations: list[str] = [] - - for path in sorted(package.glob("*.py")): - if path.name.startswith("test_"): - continue - tree = ast.parse(path.read_text(), filename=str(path)) - for node in ast.walk(tree): - if isinstance(node, ast.Import): - imports = [alias.name for alias in node.names] - elif isinstance(node, ast.ImportFrom) and node.module is not None: - imports = [node.module] - else: - continue - for imported in imports: - if imported.startswith(FORBIDDEN_PREFIXES): - violations.append(f"{path.name}: {imported}") - - assert violations == [] diff --git a/dimos/benchmark/agent_eval/test_json.py b/dimos/benchmark/agent_eval/test_json.py deleted file mode 100644 index 73d3739492..0000000000 --- a/dimos/benchmark/agent_eval/test_json.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest - -from dimos.benchmark.agent_eval.json import canonical_json - - -def test_canonical_json_is_sorted_compact_utf8() -> None: - assert canonical_json({"z": "café", "a": [2, 1]}) == (b'{"a":[2,1],"z":"caf\xc3\xa9"}') - - -def test_canonical_json_rejects_nonfinite_numbers() -> None: - with pytest.raises(ValueError, match="JSON compliant"): - canonical_json({"value": float("nan")}) diff --git a/dimos/benchmark/agent_eval/test_pi_adapter.py b/dimos/benchmark/agent_eval/test_pi_adapter.py deleted file mode 100644 index 7db3b0a124..0000000000 --- a/dimos/benchmark/agent_eval/test_pi_adapter.py +++ /dev/null @@ -1,198 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import json - -import pytest - -from dimos.agents.mcp.mcp_server import _handle_tools_list -from dimos.benchmark.agent_eval.pi_adapter import ( - PI_TOOL_NAMES, - CodePolicyCallLog, - PythonExecBroker, - ToolInventoryError, - credential_binding_sha256, - inspect_python_exec_inventory, - wait_for_python_exec, -) -from dimos.core.module import SkillInfo - -ATTEMPT_ID = "attempt_" + "a" * 32 -PI_SESSION_ID = "pi_session_" + "b" * 32 -POLICY_SESSION_ID = "code_policy_session_" + "c" * 32 - - -def _python_exec_tool() -> dict[str, object]: - return { - "name": "python_exec", - "description": ( - "Execute one synchronous Python program in the persistent policy session.\n\n" - "The trusted, unsandboxed session preloads `app` for deployed DimOS RPCs." - ), - "inputSchema": { - "type": "object", - "properties": { - "code": {"title": "Code", "type": "string"}, - "timeout_s": { - "default": 110.0, - "title": "Timeout S", - "type": "number", - }, - }, - "required": ["code"], - }, - } - - -class FakeMcp: - def __init__( - self, - tools: list[dict[str, object]] | None = None, - result: dict[str, object] | None = None, - ) -> None: - self.tools = tools or [_python_exec_tool()] - self.result = result or {"content": [{"type": "text", "text": "done"}]} - self.calls: list[tuple[str, dict[str, object]]] = [] - self.ready = True - - def wait_for_ready(self, timeout: float) -> bool: - assert timeout > 0 - return self.ready - - def list_tools(self) -> list[dict[str, object]]: - return self.tools - - def call_tool(self, name: str, arguments: dict[str, object] | None = None) -> dict[str, object]: - self.calls.append((name, arguments or {})) - return self.result - - -def test_inventory_retains_additional_tools_but_admits_only_python_exec() -> None: - tools = [ - _python_exec_tool(), - { - "name": "move", - "description": "Direct robot motion", - "inputSchema": {"type": "object", "properties": {}}, - }, - ] - - receipt = inspect_python_exec_inventory("http://localhost/mcp", tools) - - assert [tool["name"] for tool in receipt.observed_tools] == ["python_exec", "move"] - assert PI_TOOL_NAMES == ("python_exec",) - - -@pytest.mark.parametrize( - "tools", - [ - [], - [_python_exec_tool(), _python_exec_tool()], - [{**_python_exec_tool(), "description": "changed"}], - [ - { - **_python_exec_tool(), - "inputSchema": { - "type": "object", - "properties": {"code": {"type": "string"}}, - "required": ["code"], - }, - } - ], - ], -) -def test_inventory_rejects_missing_duplicate_or_changed_tool( - tools: list[dict[str, object]], -) -> None: - with pytest.raises(ToolInventoryError): - inspect_python_exec_inventory("http://localhost/mcp", tools) - - -def test_readiness_timeout_is_infrastructure_failure() -> None: - mcp = FakeMcp() - mcp.ready = False - - with pytest.raises(TimeoutError): - wait_for_python_exec("http://localhost/mcp", mcp, 0.1) - - -def test_broker_forwards_one_tool_and_records_both_sessions(tmp_path) -> None: - mcp = FakeMcp() - path = tmp_path / "code-policy-calls.jsonl" - with CodePolicyCallLog(path) as call_log: - broker = PythonExecBroker( - attempt_id=ATTEMPT_ID, - pi_session_id=PI_SESSION_ID, - code_policy_session_id=POLICY_SESSION_ID, - mcp=mcp, - call_log=call_log, - ) - result = broker.request("python_exec", {"code": "print('hello')"}) - - record = json.loads(path.read_text()) - assert result == mcp.result - assert mcp.calls == [("python_exec", {"code": "print('hello')", "timeout_s": 110.0})] - assert record["attempt_id"] == ATTEMPT_ID - assert record["pi_session_id"] == PI_SESSION_ID - assert record["code_policy_session_id"] == POLICY_SESSION_ID - assert record["ok"] is True - - -def test_broker_rejects_every_other_tool_without_forwarding(tmp_path) -> None: - mcp = FakeMcp() - with CodePolicyCallLog(tmp_path / "calls.jsonl") as call_log: - broker = PythonExecBroker( - attempt_id=ATTEMPT_ID, - pi_session_id=PI_SESSION_ID, - code_policy_session_id=POLICY_SESSION_ID, - mcp=mcp, - call_log=call_log, - ) - with pytest.raises(PermissionError): - broker.request("move", {"x": 1}) - assert mcp.calls == [] - - -def test_credentials_do_not_enter_records_or_diagnostics(tmp_path) -> None: - secret = "sk-super-secret-value" - digest = credential_binding_sha256("environment", "OPENAI_API_KEY", secret) - mcp = FakeMcp(result={"content": [{"type": "text", "text": "safe"}]}) - path = tmp_path / "calls.jsonl" - with CodePolicyCallLog(path) as call_log: - broker = PythonExecBroker( - attempt_id=ATTEMPT_ID, - pi_session_id=PI_SESSION_ID, - code_policy_session_id=POLICY_SESSION_ID, - mcp=mcp, - call_log=call_log, - ) - broker.request("python_exec", {"code": "1 + 1"}) - - retained = path.read_text() + digest - assert secret not in retained - assert digest == credential_binding_sha256("environment", "OPENAI_API_KEY", secret) - - -def test_inventory_rejects_non_json_tool() -> None: - skill = SkillInfo( - class_name="Bad", - func_name="python_exec", - args_schema=json.dumps({"type": "object"}), - ) - tool = _handle_tools_list(1, [skill])["result"]["tools"][0] - tool["not_json"] = object() - with pytest.raises(ValueError, match="strict JSON"): - inspect_python_exec_inventory("http://localhost/mcp", [tool]) diff --git a/dimos/benchmark/agent_eval/test_pi_process.py b/dimos/benchmark/agent_eval/test_pi_process.py index 7bd58ddc31..bc8c8df892 100644 --- a/dimos/benchmark/agent_eval/test_pi_process.py +++ b/dimos/benchmark/agent_eval/test_pi_process.py @@ -12,218 +12,107 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import annotations - -import hashlib -import sys +import json +from pathlib import Path +import subprocess import pytest -from dimos.benchmark.agent_eval.auth import RuntimeCredential -from dimos.benchmark.agent_eval.pi_adapter import CodePolicyCallLog -from dimos.benchmark.agent_eval.pi_process import NodePiSessionFactory -from dimos.benchmark.agent_eval.progress import ( - AssistantTextProgress, - FinalResponseProgress, - ToolEndProgress, - ToolStartProgress, -) - -_HOST = r""" -import json, pathlib, sys -def send(value): - print(json.dumps(value, separators=(",", ":")), flush=True) -start = json.loads(sys.stdin.readline()) -send({"version":1,"type":"session_started","id":start["id"],"tools":["python_exec"]}) -for line in sys.stdin: - frame = json.loads(line) - if frame["type"] == "prompt": - send({"version":1,"type":"transcript","event":"agent_start"}) - send({"version":1,"type":"transcript","event":"assistant_text_delta","delta":"Checking memory"}) - send({"version":1,"type":"transcript","event":"thinking_delta","delta":"private reasoning"}) - send({"version":1,"type":"tool_call","id":"tool-1","tool":"python_exec","params":{"code":"1 + 1"}}) - reply = json.loads(sys.stdin.readline()) - assert reply["type"] == "tool_reply" and reply["ok"] - send({"version":1,"type":"turn_complete","id":frame["id"],"policy_call_count":1,"final_text":"done"}) - elif frame["type"] == "dispose": - pathlib.Path("pi-session").mkdir() - pathlib.Path("pi-prompt").mkdir() - pathlib.Path("pi-session/native.jsonl").write_text('{"type":"session"}\n') - pathlib.Path("pi-prompt/system.txt").write_text("system") - pathlib.Path("pi-prompt/initial.txt").write_text(start["initial_prompt"]) - send({"version":1,"type":"session_closed","id":start["id"],"evidence":{ - "state":"complete","persisted":True,"relative_path":"pi-session/native.jsonl", - "system_prompt":{"relative_path":"pi-prompt/system.txt","byte_count":6,"sha256":"0"*64}, - "initial_prompt":{"relative_path":"pi-prompt/initial.txt","byte_count":len(start["initial_prompt"]),"sha256":"1"*64} - }}) - break -""" - - -class _Mcp: - def __init__(self, result: str = "2") -> None: - self.result = result - - def wait_for_ready(self, timeout: float) -> bool: - return True - - def list_tools(self): - return [] - - def call_tool(self, name, arguments=None): - assert name == "python_exec" - return {"content": [{"type": "text", "text": self.result}]} - - -def test_node_pi_process_roundtrip_and_native_evidence(tmp_path) -> None: - attempt = tmp_path / ("attempt_" + "a" * 32) - attempt.mkdir() - calls = CodePolicyCallLog(attempt / "code-policy-calls.jsonl") - progress = [] - factory = NodePiSessionFactory( - command=(sys.executable, "-c", _HOST), - credential=RuntimeCredential( - auth_mode="environment", - binding_name="OPENAI_API_KEY", - value="secret", - ), - model="gpt-5.6-luna", - thinking_level="medium", - startup_timeout_s=2.0, - progress=progress.append, - ) - - session = factory.create( - attempt_path=attempt, - public_prompt="Navigate to the bathtub.", - code_policy_session_id="code_policy_session_" + "b" * 32, - call_log=calls, - mcp=_Mcp(), - ) - turn = session.prompt("Navigate to the bathtub.", 2.0) - session.dispose() - calls.close() - - assert turn.policy_call_count == 1 - assert any( - isinstance(event, AssistantTextProgress) and event.delta == "Checking memory" - for event in progress - ) - assert any(isinstance(event, ToolStartProgress) and event.code == "1 + 1" for event in progress) - assert any( - isinstance(event, ToolEndProgress) and event.ok and event.result == "2" - for event in progress - ) - assert any( - isinstance(event, FinalResponseProgress) and event.text == "done" for event in progress - ) - assert "private reasoning" not in repr(progress) - references = session.artifact_references() - assert {item.path for item in references} == { - "pi-adapter.stderr.log", - "pi-session/native.jsonl", - "pi-prompt/system.txt", - "pi-prompt/initial.txt", - } - retained = (attempt / "code-policy-calls.jsonl").read_text() - assert "secret" not in retained - assert hashlib.sha256((attempt / "pi-session/native.jsonl").read_bytes()).hexdigest() - - -def test_progress_observer_failure_does_not_fail_turn(tmp_path) -> None: - attempt = tmp_path / ("attempt_" + "e" * 32) - attempt.mkdir() - calls = CodePolicyCallLog(attempt / "code-policy-calls.jsonl") - - def broken_progress(_event) -> None: - raise RuntimeError("presentation failed") - - factory = NodePiSessionFactory( - command=(sys.executable, "-c", _HOST), - credential=RuntimeCredential( - auth_mode="environment", - binding_name="OPENAI_API_KEY", - value="secret", +from dimos.benchmark.agent_eval.pi_process import PiCliRunner, PiRunError, parse_pi_events + + +def test_parse_stock_pi_events_uses_final_message_and_counts_tools() -> None: + events = [ + {"type": "tool_execution_start", "toolName": "python_exec"}, + { + "type": "message_end", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": "Reasoning\nANSWER: 8"}], + "stopReason": "stop", + }, + }, + ] + text, count, error = parse_pi_events("\n".join(json.dumps(item) for item in events)) + assert text == "Reasoning\nANSWER: 8" + assert count == 1 + assert error is None + + +def test_stock_cli_receives_only_api_key_and_evaluator_binding(mocker, tmp_path: Path) -> None: + cli = tmp_path / "cli.js" + extension = tmp_path / "extension.js" + cli.touch() + extension.touch() + process = mocker.Mock(returncode=0) + process.communicate.return_value = ( + json.dumps( + { + "type": "message_end", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": "ANSWER: 2"}], + "stopReason": "stop", + }, + } ), - model="gpt-5.6-luna", - thinking_level="medium", - startup_timeout_s=2.0, - progress=broken_progress, + "", ) - session = factory.create( - attempt_path=attempt, - public_prompt="Count rooms.", - code_policy_session_id="code_policy_session_" + "f" * 32, - call_log=calls, - mcp=_Mcp(), + popen = mocker.patch( + "dimos.benchmark.agent_eval.pi_process.subprocess.Popen", return_value=process ) - try: - turn = session.prompt("Count rooms.", 2.0) - assert turn.final_text == "done" - finally: - session.dispose() - calls.close() - - -def test_tool_result_progress_is_bounded(tmp_path) -> None: - attempt = tmp_path / ("attempt_" + "1" * 32) - attempt.mkdir() - calls = CodePolicyCallLog(attempt / "code-policy-calls.jsonl") - progress = [] - factory = NodePiSessionFactory( - command=(sys.executable, "-c", _HOST), - credential=RuntimeCredential( - auth_mode="environment", - binding_name="OPENAI_API_KEY", - value="secret", - ), + runner = PiCliRunner( + cli=cli, + extension=extension, model="gpt-5.6-luna", thinking_level="medium", - startup_timeout_s=2.0, - progress=progress.append, + timeout_s=10, ) - session = factory.create( - attempt_path=attempt, - public_prompt="Count rooms.", - code_policy_session_id="code_policy_session_" + "2" * 32, - call_log=calls, - mcp=_Mcp("x" * 10_000), + result = runner.run( + prompt="Count", + system_prompt="Use memory", + mcp_url="http://127.0.0.1:1234/mcp", + api_key="secret", + run_dir=tmp_path, ) - try: - session.prompt("Count rooms.", 2.0) - finally: - session.dispose() - calls.close() - - result = next(event.result for event in progress if isinstance(event, ToolEndProgress)) - assert len(result.encode()) <= 4 * 1024 - assert result.endswith("… [truncated]") - - -def test_node_pi_process_reaps_child_when_startup_times_out(tmp_path) -> None: - attempt = tmp_path / ("attempt_" + "c" * 32) - attempt.mkdir() - calls = CodePolicyCallLog(attempt / "code-policy-calls.jsonl") - factory = NodePiSessionFactory( - command=(sys.executable, "-c", "import time; time.sleep(60)"), - credential=RuntimeCredential( - auth_mode="environment", - binding_name="OPENAI_API_KEY", - value="secret", - ), + command = popen.call_args.args[0] + env = popen.call_args.kwargs["env"] + assert command[command.index("--mode") + 1] == "json" + assert "--no-builtin-tools" in command + assert command[command.index("--tools") + 1] == "python_exec" + assert env["OPENAI_API_KEY"] == "secret" + assert env["DIMOS_CODE_POLICY_MCP_URL"].endswith("/mcp") + assert "secret" not in command + assert result.final_text == "ANSWER: 2" + + +def test_stock_cli_timeout_terminates_the_child(mocker, tmp_path: Path) -> None: + cli = tmp_path / "cli.js" + extension = tmp_path / "extension.js" + cli.touch() + extension.touch() + process = mocker.Mock() + process.communicate.side_effect = [ + subprocess.TimeoutExpired("pi", 0.01), + ("", "stopped"), + ] + mocker.patch("dimos.benchmark.agent_eval.pi_process.subprocess.Popen", return_value=process) + runner = PiCliRunner( + cli=cli, + extension=extension, model="gpt-5.6-luna", thinking_level="medium", - startup_timeout_s=0.05, + timeout_s=0.01, ) - with pytest.raises(TimeoutError, match="session_started"): - factory.create( - attempt_path=attempt, - public_prompt="Navigate to the bathtub.", - code_policy_session_id="code_policy_session_" + "d" * 32, - call_log=calls, - mcp=_Mcp(), + with pytest.raises(PiRunError, match="timed out"): + runner.run( + prompt="Count", + system_prompt="Use memory", + mcp_url="http://127.0.0.1:1234/mcp", + api_key="secret", + run_dir=tmp_path, ) - calls.close() - assert (attempt / "pi-adapter.stderr.log").read_bytes() == b"" + process.terminate.assert_called_once_with() + process.kill.assert_not_called() diff --git a/dimos/benchmark/agent_eval/test_single_case.py b/dimos/benchmark/agent_eval/test_single_case.py index 5af3d4aad6..f4c895b669 100644 --- a/dimos/benchmark/agent_eval/test_single_case.py +++ b/dimos/benchmark/agent_eval/test_single_case.py @@ -12,113 +12,99 @@ # See the License for the specific language governing permissions and # limitations under the License. -import hashlib -import json +from pathlib import Path +from types import SimpleNamespace import pytest -from dimos.benchmark.agent_eval.case import ( +from dimos.benchmark.agent_eval.models import ( EvalCase, + EvalRunConfig, ExactIntegerValidatorRef, - FrozenCodePolicyInteraction, FrozenRecordingSource, IntegerQuestionTask, ) +from dimos.benchmark.agent_eval.pi_process import PiRunResult import dimos.benchmark.agent_eval.single_case as single_case -from dimos.benchmark.agent_eval.single_case import ( - EvalRunConfig, - OpenAIApiKeyConfig, - _resolve_credential, - execute_single_case, -) -from dimos.benchmark.short_horizon_qa.eval import load_exact_integer_oracle - - -def test_run_config_round_trips_through_pydantic() -> None: - configured = EvalRunConfig() - - decoded = EvalRunConfig.model_validate_json(configured.model_dump_json()) - - assert decoded == configured - assert decoded.agent.backend == "pi" - assert decoded.agent.model == "gpt-5.6-luna" - assert decoded.agent.auth.mode == "codex-oauth" - - -def test_api_key_auth_uses_named_environment_without_serializing_secret(monkeypatch) -> None: - monkeypatch.setenv("EVAL_TEST_KEY", "private-value") - auth = OpenAIApiKeyConfig(env="EVAL_TEST_KEY") - - credential, digest = _resolve_credential(auth) +from dimos.benchmark.agent_eval.single_case import execute_single_case - assert credential.value == "private-value" - assert len(digest) == 64 - assert "private-value" not in auth.model_dump_json() - assert "private-value" not in digest - -def test_private_validator_resolves_relative_to_case_directory(tmp_path) -> None: +def _case(tmp_path: Path) -> Path: private = tmp_path / "private" private.mkdir() - oracle = private / "oracle.json" - oracle.write_text( - json.dumps( - { - "schema_version": "1.0", - "expected_count": 4, - "counting_policy": "Count enclosed rooms.", - "rooms": [], - "reviewed_by": ["reviewer"], - } - ) + (private / "oracle.json").write_text( + '{"schema_version":"1.0","expected_count":2,' + '"counting_policy":"count rooms","rooms":[],' + '"reviewed_by":["reviewer"]}' ) - case = EvalCase.compile( - case_id="case", + case = EvalCase( + case_id="demo", source=FrozenRecordingSource(recording="recording", progress=1.0), task=IntegerQuestionTask(prompt="How many rooms?"), - interaction=FrozenCodePolicyInteraction(driver_revision="v1"), - validator=ExactIntegerValidatorRef( - revision="v1", - private_path="private/oracle.json", - private_sha256=hashlib.sha256(oracle.read_bytes()).hexdigest(), - ), + validator=ExactIntegerValidatorRef(revision="v1", private_path="private/oracle.json"), ) - - loaded = load_exact_integer_oracle(case, tmp_path) - - assert loaded.expected_count == 4 - - -def test_single_case_emits_public_question_before_private_preflight(tmp_path, monkeypatch) -> None: - case = EvalCase.compile( - case_id="case", - source=FrozenRecordingSource(recording="recording", progress=1.0), - task=IntegerQuestionTask(prompt="How many rooms?"), - interaction=FrozenCodePolicyInteraction(driver_revision="v1"), - validator=ExactIntegerValidatorRef( - revision="v1", - private_path="private/oracle.json", - private_sha256="0" * 64, + path = tmp_path / "case.json" + path.write_text(case.model_dump_json()) + return path + + +def test_direct_run_publishes_only_compact_result_and_native_transcript( + monkeypatch, tmp_path: Path +) -> None: + case_path = _case(tmp_path) + bundle = tmp_path / "bundle" + bundle.mkdir() + monkeypatch.setenv("OPENAI_API_KEY", "secret") + monkeypatch.setattr(single_case, "_materialize_frozen_memory", lambda *_args: bundle) + monkeypatch.setattr( + single_case, + "load_bundle", + lambda *_args, **_kwargs: ( + object(), + SimpleNamespace(cutoff_timestamp=10.0), + tmp_path / "source.db", + tmp_path / "derived.db", ), ) - case_path = tmp_path / "case.json" - case_path.write_text(case.model_dump_json()) - events = [] - - def stop_at_private_preflight(*args, **kwargs): - raise RuntimeError("stop after public header") - - monkeypatch.setattr(single_case, "load_exact_integer_oracle", stop_at_private_preflight) - - with pytest.raises(RuntimeError, match="stop after public header"): - execute_single_case(case_path, config=EvalRunConfig(), progress=events.append) - - assert [event.kind for event in events] == ["status", "case_header", "status"] - header = events[1] - assert header.model_dump() == { - "kind": "case_header", - "case_id": "case", - "source": "recording", - "progress": 1.0, - "question": "How many rooms?", + monkeypatch.setattr(single_case, "_pi_paths", lambda: (case_path, case_path)) + + class Server: + mcp_url = "http://127.0.0.1:1234/mcp" + session = SimpleNamespace(execution_count=3) + + def __init__(self, _config): + pass + + def start(self): + pass + + def stop(self): + pass + + class Runner: + def __init__(self, **_kwargs): + pass + + def run(self, *, run_dir, **_kwargs): + transcript = run_dir / "native.jsonl" + transcript.write_text('{"type":"session"}\n') + return PiRunResult("Checked\nANSWER: 2", 3, 1.0, transcript, "") + + monkeypatch.setattr(single_case, "CodePolicyMcpServer", Server) + monkeypatch.setattr(single_case, "PiCliRunner", Runner) + output = tmp_path / "output" + result = execute_single_case(case_path, config=EvalRunConfig(), output=output) + assert result.passed is True + assert {path.name for path in output.iterdir()} == { + "result.json", + "pi-transcript.jsonl", } + + +def test_nonempty_output_is_rejected_before_execution(tmp_path: Path) -> None: + output = tmp_path / "output" + output.mkdir() + (output / "keep").write_text("user data") + with pytest.raises(FileExistsError, match="absent or an empty"): + execute_single_case(_case(tmp_path), config=EvalRunConfig(), output=output) + assert (output / "keep").read_text() == "user data" diff --git a/dimos/benchmark/agent_eval/test_store.py b/dimos/benchmark/agent_eval/test_store.py deleted file mode 100644 index c39b679d7c..0000000000 --- a/dimos/benchmark/agent_eval/test_store.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from datetime import UTC, datetime -import json -from pathlib import Path - -import pytest - -from dimos.benchmark.agent_eval.artifacts import NormalizedOutcome -from dimos.benchmark.agent_eval.store import ( - AttemptAlreadyActiveError, - AttemptStore, - new_operation_id, -) - - -def _outcome( - attempt_id: str, - *, - attempt_status: str = "completed", - task_result: str = "passed", - complete: bool = True, - reason: str = "native evaluator terminal result", -) -> NormalizedOutcome: - return NormalizedOutcome( - attempt_id=attempt_id, - attempt_status=attempt_status, - task_result=task_result, - terminal_stage="terminal", - reason=reason, - required_artifacts_complete=complete, - finished_at=datetime.now(UTC), - duration_s=1.0, - ) - - -@pytest.mark.parametrize("task_result", ["passed", "failed"]) -def test_completed_pass_and_fail_write_atomic_outcome(tmp_path: Path, task_result: str) -> None: - with AttemptStore(tmp_path) as store: - reference = store.write_outcome(_outcome(store.attempt_id, task_result=task_result)) - - assert reference.path == "outcome.v1.json" - assert store.verify_artifacts((reference,)) - payload = json.loads((store.path / reference.path).read_text()) - assert payload["attempt_status"] == "completed" - assert payload["task_result"] == task_result - - -def test_infrastructure_failure_is_not_evaluated_and_retains_partial_evidence( - tmp_path: Path, -) -> None: - with AttemptStore(tmp_path) as store: - event = store.append_event( - "reset-failed", - operation_id=new_operation_id(), - payload={"stage": "reset"}, - ) - diagnostic = store.write_artifact("diagnostics/reset.txt", "reset timed out") - outcome = store.write_outcome( - _outcome( - store.attempt_id, - attempt_status="failed", - task_result="not_evaluated", - complete=False, - reason="reset timed out", - ) - ) - - assert event.sequence == 1 - assert store.verify_artifacts((diagnostic, outcome)) - - -def test_interrupted_store_retains_events_and_releases_target_lock( - tmp_path: Path, -) -> None: - first = AttemptStore(tmp_path) - first.append_event("interrupted") - first_path = first.path - first.close() - - with AttemptStore(tmp_path) as second: - assert second.path != first_path - assert (first_path / "events.jsonl").read_text() - - -def test_missing_or_changed_artifact_is_detected(tmp_path: Path) -> None: - with AttemptStore(tmp_path) as store: - missing = store.write_artifact("partial.json", {"retained": True}) - (store.path / missing.path).unlink() - - assert not store.verify_artifacts((missing,)) - - -def test_existing_attempt_directory_is_never_reused(tmp_path: Path) -> None: - attempt_id = "attempt_" + "1" * 32 - (tmp_path / attempt_id).mkdir(parents=True) - - with pytest.raises(FileExistsError): - AttemptStore(tmp_path, attempt_id) - - -def test_concurrent_attempt_against_same_target_is_rejected(tmp_path: Path) -> None: - first = AttemptStore(tmp_path) - try: - with pytest.raises(AttemptAlreadyActiveError): - AttemptStore(tmp_path) - finally: - first.close() - - -def test_events_are_append_only_correlated_and_monotonic(tmp_path: Path) -> None: - with AttemptStore(tmp_path) as store: - operation_id = new_operation_id() - first = store.append_event("reset-started", operation_id=operation_id) - second = store.append_event("reset-finished", operation_id=operation_id) - - records = [json.loads(line) for line in (store.path / "events.jsonl").read_text().splitlines()] - assert [record["sequence"] for record in records] == [1, 2] - assert first.operation_id == second.operation_id == operation_id - assert first.monotonic_offset_s <= second.monotonic_offset_s - assert all(record["attempt_id"] == store.attempt_id for record in records) - - -def test_outcome_is_non_overwriting(tmp_path: Path) -> None: - with AttemptStore(tmp_path) as store: - store.write_outcome(_outcome(store.attempt_id)) - - with pytest.raises(FileExistsError): - store.write_outcome( - _outcome(store.attempt_id, task_result="failed", reason="different") - ) - - -def test_outcome_write_failure_keeps_partial_evidence_and_no_outcome( - tmp_path: Path, monkeypatch -) -> None: - with AttemptStore(tmp_path) as store: - evidence = store.write_artifact("task.v1.json", {"task": "public"}) - - def fail_link(_source: Path, _destination: Path) -> None: - raise OSError("simulated link failure") - - monkeypatch.setattr("dimos.benchmark.agent_eval.store.os.link", fail_link) - with pytest.raises(OSError, match="simulated link failure"): - store.write_outcome(_outcome(store.attempt_id)) - - assert store.verify_artifacts((evidence,)) - assert not (store.path / "outcome.v1.json").exists() diff --git a/dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/README.md b/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/README.md similarity index 59% rename from dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/README.md rename to dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/README.md index a6a132141a..823d39aa3c 100644 --- a/dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/README.md +++ b/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/README.md @@ -10,8 +10,5 @@ failed task score as an agent or mapping regression. The authoritative case remains incomplete until a human-authored room inventory, counting policy, and independent review establish the expected count. -The credentialed API-key smoke was exercised on 2026-08-04 after adding live -progress. The final operational attempt completed in 312.9 seconds with 40 -successful `python_exec` broker calls and a parsed `ANSWER: 8`. Its task score -was expectedly failed because this fixture's synthetic oracle is `0`; the result -must not be used as the authoritative room count. +Use this case to exercise the direct stock-Pi CLI path. Any observed answer is +experimental until the oracle is replaced with a reviewed room inventory. diff --git a/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/case.json b/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/case.json new file mode 100644 index 0000000000..7e2cf52fcb --- /dev/null +++ b/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/case.json @@ -0,0 +1,22 @@ +{ + "schema_version": "1.0", + "case_id": "demo-go2-hongkong-office-room-count-smoke", + "source": { + "schema_version": "1.0", + "kind": "frozen_memory", + "recording": "go2_hongkong_office", + "progress": 1.0 + }, + "task": { + "schema_version": "1.0", + "kind": "integer_question", + "prompt": "How many rooms in total?", + "answer_marker": "ANSWER:" + }, + "validator": { + "schema_version": "1.0", + "kind": "exact_integer", + "revision": "synthetic-cli-smoke-v1", + "private_path": "private/oracle.json" + } +} diff --git a/dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/private/oracle.json b/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/private/oracle.json similarity index 100% rename from dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/private/oracle.json rename to dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/private/oracle.json diff --git a/dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/case.json b/dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/case.json deleted file mode 100644 index 69fcc00ef1..0000000000 --- a/dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/case.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "schema_version": "1.0", - "case_id": "go2-hongkong-office-room-count-smoke", - "source": { - "schema_version": "1.0", - "kind": "frozen_memory", - "recording": "go2_hongkong_office", - "progress": 1.0, - "bundle_manifest_sha256": null - }, - "task": { - "schema_version": "1.0", - "kind": "integer_question", - "prompt": "How many rooms in total?", - "answer_marker": "ANSWER:" - }, - "interaction": { - "schema_version": "1.0", - "kind": "frozen_code_policy", - "driver_revision": "standalone-frozen-v1", - "session_lifetime": "one_attempt" - }, - "validator": { - "schema_version": "1.0", - "kind": "exact_integer", - "revision": "synthetic-cli-smoke-v1", - "private_path": "private/oracle.json", - "private_sha256": "e60d4c73c7b3d75a85358f209a15e5fc4e5a4191395bf9d799be42db5fa1f196" - }, - "fingerprint": "5a55ee32257d7acb1f9918f327cfe9253a632619fad40db7f9ad164b69568e8e" -} diff --git a/dimos/benchmark/short_horizon_qa/eval.py b/dimos/benchmark/short_horizon_qa/eval.py index f09eb278be..245ad42916 100644 --- a/dimos/benchmark/short_horizon_qa/eval.py +++ b/dimos/benchmark/short_horizon_qa/eval.py @@ -12,401 +12,48 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Frozen Memory2 QA drivers for the canonical agent-evaluation engine.""" +"""Exact-integer validation for short-horizon frozen-memory questions.""" from __future__ import annotations -from dataclasses import dataclass -import hashlib from pathlib import Path import re -from typing import Any, cast +from typing import Any, Literal -from pydantic import Field, JsonValue +from pydantic import Field -from dimos.agents.code_policy_core import ( - CodePolicySessionConfig, - FrozenMemoryEnvironment, -) -from dimos.agents.code_policy_server import StandaloneCodePolicyProcess -from dimos.agents.mcp.mcp_adapter import McpAdapter from dimos.benchmark.agent_eval.base import BaseEvalModel -from dimos.benchmark.agent_eval.case import ( - AgentCondition, - AgentOutcome, - AttemptRequest, - EvalCase, - ExactIntegerValidatorRef, - FrozenCodePolicyInteraction, - FrozenRecordingSource, - IntegerQuestionTask, - Prediction, - PrivateScore, - RuntimeBinding, - SourceSpec, -) -from dimos.benchmark.agent_eval.engine import AttemptEngine, EngineResult -from dimos.benchmark.agent_eval.interfaces import ( - AgentAdapter, - AttemptContext, - EvidenceSink, - PreparedSource, - ValidatorSession, -) -from dimos.benchmark.agent_eval.pi import PiSession, PiSessionFactory -from dimos.benchmark.agent_eval.pi_adapter import ( - CodePolicyCallLog, - wait_for_python_exec, -) -from dimos.benchmark.short_horizon_qa.service import load_bundle +from dimos.benchmark.agent_eval.models import EvalCase, ExactIntegerValidatorRef _ANSWER_LINE = re.compile(r"(?m)^ANSWER:\s*") _TERMINAL_INTEGER = re.compile(r"(?:^|\n)ANSWER:\s*(-?\d+)\s*\Z") -class RoomOracleEntry(BaseEvalModel): - label: str = Field(min_length=1) - evidence: tuple[str, ...] = Field(min_length=1) - - class ExactIntegerOracle(BaseEvalModel): expected_count: int = Field(ge=0) counting_policy: str = Field(min_length=1) - rooms: tuple[RoomOracleEntry, ...] + rooms: tuple[dict[str, Any], ...] = () reviewed_by: tuple[str, ...] = Field(min_length=1) -class FrozenMemorySourceDriver: - def __init__(self, bundle: Path) -> None: - self.bundle = bundle - - def prepare( - self, - *, - source: SourceSpec, - context: AttemptContext, - evidence: EvidenceSink, - ) -> PreparedSource: - del context - if not isinstance(source, FrozenRecordingSource): - raise TypeError("frozen source driver requires FrozenRecordingSource") - manifest, cutoff, source_path, derived_path = load_bundle( - self.bundle, progress=source.progress - ) - if source_path.stem != source.recording: - raise ValueError("prepared recording does not match the authored source") - if ( - source.bundle_manifest_sha256 is not None - and source.bundle_manifest_sha256 - != hashlib.sha256((self.bundle / "manifest.v1.json").read_bytes()).hexdigest() - ): - raise ValueError("prepared manifest digest does not match the case") - evidence.artifact("source-manifest.v1.json", manifest) - receipt: dict[str, JsonValue] = { - "recording": source.recording, - "progress": source.progress, - "cutoff_seconds": cutoff.cutoff_seconds, - "cutoff_timestamp": cutoff.cutoff_timestamp, - "source_sha256": manifest.source_sha256, - "derived_sha256": manifest.derived_sha256, - } - return PreparedSource( - public={"recording": source.recording, "progress": source.progress}, - receipt=receipt, - private_handle={ - "source_path": str(source_path), - "derived_path": str(derived_path), - "cutoff_timestamp": cutoff.cutoff_timestamp, - }, - ) - - def close(self) -> None: - return None - - -@dataclass(frozen=True) -class CodePolicyAgentInterface: - mcp: McpAdapter - session_id: str - call_log: CodePolicyCallLog - evidence: EvidenceSink - - -class PiCodePolicyAgent(AgentAdapter): - def __init__(self, factory: PiSessionFactory, *, turn_timeout_s: float = 180.0) -> None: - self.factory = factory - self.turn_timeout_s = turn_timeout_s - self._session: PiSession | None = None - self._interface: CodePolicyAgentInterface | None = None - - def run( - self, - *, - task: Any, - context: AttemptContext, - interface: Any = None, - ) -> AgentOutcome: - if not isinstance(task, IntegerQuestionTask): - raise TypeError("frozen Pi agent requires an integer question task") - if not isinstance(interface, CodePolicyAgentInterface): - raise TypeError("frozen Pi agent requires a CodePolicy interface") - prompt = ( - f"{task.prompt}\n\n" - "Use the provided python_exec tool and the read-only `memory` API to " - "answer from the recording. End your final response with exactly " - f"`{task.answer_marker} `." - ) - session = self.factory.create( - attempt_path=context.path, - public_prompt=prompt, - code_policy_session_id=interface.session_id, - call_log=interface.call_log, - mcp=interface.mcp, - ) - self._session = session - self._interface = interface - turn = session.prompt(prompt, self.turn_timeout_s) - return AgentOutcome( - final_text=turn.final_text, - tool_call_count=turn.policy_call_count, - terminal_reason="pi turn completed", - agent_session_id=session.session_id, - interaction_session_id=interface.session_id, - ) - - def close(self) -> None: - session = self._session - interface = self._interface - self._session = None - self._interface = None - if session is None: - return - session.dispose() - if interface is not None: - for artifact in session.artifact_references(): - interface.evidence.reference(artifact.path) - - -class FrozenCodePolicyInteractionDriver: - def __init__(self, *, readiness_timeout_s: float = 10.0) -> None: - self.readiness_timeout_s = readiness_timeout_s - self._process: StandaloneCodePolicyProcess | None = None - self._call_log: CodePolicyCallLog | None = None - self._evidence: EvidenceSink | None = None - self._session_id: str | None = None +class IntegerPrediction(BaseEvalModel): + status: Literal["parsed", "invalid"] + integer_answer: int | None = None - def run( - self, - *, - case: EvalCase, - prepared_source: PreparedSource, - agent: AgentAdapter, - context: AttemptContext, - evidence: EvidenceSink, - ) -> AgentOutcome: - if not isinstance(case.interaction, FrozenCodePolicyInteraction): - raise TypeError("frozen interaction driver received an incompatible case") - handle = prepared_source.private_handle - if not isinstance(handle, dict): - raise TypeError("frozen source did not provide a private binding") - config = CodePolicySessionConfig( - environment=FrozenMemoryEnvironment( - recording_path=str(handle["source_path"]), - derived_recording_path=str(handle["derived_path"]), - memory_cutoff_timestamp=float(handle["cutoff_timestamp"]), - ) - ) - process = StandaloneCodePolicyProcess(config) - process.start(self.readiness_timeout_s) - self._process = process - self._evidence = evidence - adapter = McpAdapter(process.mcp_url, timeout=120) - inventory = wait_for_python_exec(process.mcp_url, adapter, self.readiness_timeout_s) - evidence.artifact("mcp-inventory.v1.json", inventory) - receipt = process.receipt() - session_id = str(receipt["session_id"]) - self._session_id = session_id - evidence.artifact("code-policy-session.v1.json", receipt) - call_log = CodePolicyCallLog(context.path / "code-policy-calls.jsonl") - self._call_log = call_log - return agent.run( - task=case.task, - context=context, - interface=CodePolicyAgentInterface( - mcp=adapter, - session_id=session_id, - call_log=call_log, - evidence=evidence, - ), - ) - def close(self) -> None: - process = self._process - call_log = self._call_log - evidence = self._evidence - session_id = self._session_id - self._process = None - self._call_log = None - self._evidence = None - self._session_id = None - if call_log is not None: - call_log.close() - if evidence is not None: - evidence.reference("code-policy-calls.jsonl") - if process is not None: - try: - if evidence is not None: - evidence.artifact( - "code-policy-records.v1.json", - cast("JsonValue", process.records(session_id)), - ) - finally: - process.close() - - -class ExactIntegerValidatorDriver: - def __init__(self, private_root: Path) -> None: - self.private_root = private_root.resolve() - - def prepare( - self, - *, - case: EvalCase, - prepared_source: PreparedSource, - context: AttemptContext, - evidence: EvidenceSink, - ) -> ValidatorSession: - del prepared_source - oracle = load_exact_integer_oracle(case, self.private_root) - reference = case.validator - evidence.artifact("oracle.private.v1.json", oracle) - return _ExactIntegerValidatorSession( - case=case, - context=context, - evidence=evidence, - oracle=oracle, - revision=reference.revision, - ) - - -class _ExactIntegerValidatorSession: - def __init__( - self, - *, - case: EvalCase, - context: AttemptContext, - evidence: EvidenceSink, - oracle: ExactIntegerOracle, - revision: str, - ) -> None: - self.case = case - self.context = context - self.evidence = evidence - self.oracle = oracle - self.revision = revision - - def evaluate(self, outcome: AgentOutcome) -> PrivateScore: - if outcome.agent_session_id is None or outcome.interaction_session_id is None: - raise ValueError("agent outcome is missing session identities") - prediction = parse_integer_prediction( - case_id=self.case.case_id, - attempt_id=self.context.attempt_id, - agent_session_id=outcome.agent_session_id, - interaction_session_id=outcome.interaction_session_id, - final_text=outcome.final_text, - ) - self.evidence.artifact("prediction.v1.json", prediction) - passed = ( - prediction.status == "parsed" - and prediction.integer_answer == self.oracle.expected_count - ) - return PrivateScore( - case_id=self.case.case_id, - attempt_id=self.context.attempt_id, - validator_revision=self.revision, - passed=passed, - prediction_status=prediction.status, - ) - - def close(self) -> None: - return None - - -def load_exact_integer_oracle(case: EvalCase, private_root: Path) -> ExactIntegerOracle: - """Resolve and verify a case-relative private oracle before agent dispatch.""" +def load_exact_integer_oracle(case: EvalCase, case_root: Path) -> ExactIntegerOracle: reference = case.validator if not isinstance(reference, ExactIntegerValidatorRef): - raise TypeError("exact integer validator requires ExactIntegerValidatorRef") - root = private_root.resolve() + raise TypeError("case does not use exact-integer validation") + root = case_root.resolve() path = (root / reference.private_path).resolve() if root not in path.parents: - raise ValueError("private oracle path escapes its case directory") - data = path.read_bytes() - if hashlib.sha256(data).hexdigest() != reference.private_sha256: - raise ValueError("private oracle digest does not match the case") - return ExactIntegerOracle.model_validate_json(data) + raise ValueError("private oracle path escapes the case directory") + return ExactIntegerOracle.model_validate_json(path.read_bytes()) -def parse_integer_prediction( - *, - case_id: str, - attempt_id: str, - agent_session_id: str, - interaction_session_id: str, - final_text: str, -) -> Prediction: - markers = _ANSWER_LINE.findall(final_text) +def parse_integer_prediction(final_text: str) -> IntegerPrediction: match = _TERMINAL_INTEGER.search(final_text) - if len(markers) != 1 or match is None: - return Prediction( - case_id=case_id, - attempt_id=attempt_id, - agent_session_id=agent_session_id, - interaction_session_id=interaction_session_id, - parser_revision="marked-integer-v1", - final_text=final_text, - status="invalid", - diagnostic="expected exactly one terminal ANSWER: marker", - ) - return Prediction( - case_id=case_id, - attempt_id=attempt_id, - agent_session_id=agent_session_id, - interaction_session_id=interaction_session_id, - parser_revision="marked-integer-v1", - final_text=final_text, - status="parsed", - integer_answer=int(match.group(1)), - ) - - -def run_frozen_case( - *, - case: EvalCase, - bundle: Path, - private_root: Path, - output_root: Path, - pi_factory: PiSessionFactory, - agent_condition: AgentCondition | None = None, - runtime_binding: RuntimeBinding | None = None, - turn_timeout_s: float = 180.0, -) -> EngineResult: - request = AttemptRequest( - case=case, - agent=agent_condition - or AgentCondition( - agent_id="pi-code-policy", - adapter="pi-node", - model="gpt-5.6-luna", - thinking_level="medium", - ), - runtime=runtime_binding or RuntimeBinding(runtime_id="local-standalone-code-policy"), - ) - return AttemptEngine( - request=request, - output_root=output_root, - source=FrozenMemorySourceDriver(bundle), - interaction=FrozenCodePolicyInteractionDriver(), - validator=ExactIntegerValidatorDriver(private_root), - agent=PiCodePolicyAgent(pi_factory, turn_timeout_s=turn_timeout_s), - ).run() + if len(_ANSWER_LINE.findall(final_text)) != 1 or match is None: + return IntegerPrediction(status="invalid") + return IntegerPrediction(status="parsed", integer_answer=int(match.group(1))) diff --git a/dimos/benchmark/short_horizon_qa/models.py b/dimos/benchmark/short_horizon_qa/models.py index 47703b8533..85b6d042a1 100644 --- a/dimos/benchmark/short_horizon_qa/models.py +++ b/dimos/benchmark/short_horizon_qa/models.py @@ -61,12 +61,12 @@ def progress_is_finite(self) -> CutoffRecord: class FrozenMemoryManifest(FrozenQaModel): record_type: Literal["frozen-memory-bundle"] = "frozen-memory-bundle" schema_version: Literal["1.0"] = "1.0" + source_identity: str = Field(min_length=1) source_path: str = Field(min_length=1) - source_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") source_size_bytes: int = Field(gt=0) + source_mtime_ns: int = Field(gt=0) recording_start_timestamp: float recording_end_timestamp: float derived_path: Literal["derived.db"] = "derived.db" - derived_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") mapper: MapperSettings cutoffs: tuple[CutoffRecord, ...] = Field(min_length=1) diff --git a/dimos/benchmark/short_horizon_qa/prepare.py b/dimos/benchmark/short_horizon_qa/prepare.py index 0a3b8ced74..32d53e160f 100644 --- a/dimos/benchmark/short_horizon_qa/prepare.py +++ b/dimos/benchmark/short_horizon_qa/prepare.py @@ -16,7 +16,6 @@ from __future__ import annotations -import hashlib import json import math import os @@ -34,20 +33,13 @@ from dimos.mapping.voxels.module import VoxelMapTransformer from dimos.memory2.cli.dataset import resolve_dataset from dimos.memory2.store.sqlite import SqliteStore +from dimos.memory2.stream import Stream from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 MANIFEST_NAME = "manifest.v1.json" DERIVED_NAME = "derived.db" -def file_sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - def prepare_bundle( recording: str | Path, cutoff_seconds: list[float] | None, @@ -74,13 +66,21 @@ def prepare_bundle( output.parent.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(prefix=f".{output.name}-", dir=output.parent) as temporary: temporary_path = Path(temporary) - manifest = _prepare_into(source_path, cutoffs, progresses, temporary_path, mapper) + manifest = _prepare_into( + source_path, + str(recording), + cutoffs, + progresses, + temporary_path, + mapper, + ) os.replace(temporary_path, output) return manifest def _prepare_into( source_path: Path, + source_identity: str, cutoffs: list[float], progresses: list[float], output: Path, @@ -127,12 +127,12 @@ def _prepare_into( _seal_sqlite(derived_path) manifest = FrozenMemoryManifest( + source_identity=source_identity, source_path=str(source_path), - source_sha256=file_sha256(source_path), source_size_bytes=source_path.stat().st_size, + source_mtime_ns=source_path.stat().st_mtime_ns, recording_start_timestamp=recording_start, recording_end_timestamp=recording_end, - derived_sha256=file_sha256(derived_path), mapper=mapper, cutoffs=records, ) @@ -176,7 +176,7 @@ def _validate_progress(values: list[float]) -> list[float]: def _stream_ranges(source: SqliteStore) -> dict[str, tuple[int, float, float]]: result: dict[str, tuple[int, float, float]] = {} for name in source.list_streams(): - stream = source.stream(name) + stream: Stream[Any] = source.stream(name) count = stream.count() if count: start, end = stream.get_time_range() @@ -187,7 +187,7 @@ def _stream_ranges(source: SqliteStore) -> dict[str, tuple[int, float, float]]: def _stream_boundaries(source: SqliteStore, cutoff: float) -> tuple[StreamBoundary, ...]: boundaries: list[StreamBoundary] = [] for name in source.list_streams(): - bounded = source.stream(name).through(cutoff) + bounded: Stream[Any] = source.stream(name).time_range(-math.inf, cutoff) count = bounded.count() last = bounded.last() if count else None boundaries.append( @@ -209,7 +209,7 @@ def _write_maps( ) -> list[Any]: if "lidar" not in source.list_streams(): raise ValueError("Source recording has no lidar stream") - lidar = source.stream("lidar", PointCloud2).as_read_only() + lidar = source.stream("lidar", PointCloud2) first = next(iter(lidar), None) if first is None: raise ValueError("No lidar observations exist before the final cutoff") diff --git a/dimos/benchmark/short_horizon_qa/service.py b/dimos/benchmark/short_horizon_qa/service.py index b372aedeea..b35e49920d 100644 --- a/dimos/benchmark/short_horizon_qa/service.py +++ b/dimos/benchmark/short_horizon_qa/service.py @@ -18,17 +18,17 @@ import math from pathlib import Path +import time from dimos.agents.code_policy_core import ( CodePolicySessionConfig, FrozenMemoryEnvironment, ) -from dimos.agents.code_policy_server import StandaloneCodePolicyServer +from dimos.agents.code_policy_server import CodePolicyMcpServer from dimos.benchmark.short_horizon_qa.models import CutoffRecord, FrozenMemoryManifest from dimos.benchmark.short_horizon_qa.prepare import ( DERIVED_NAME, MANIFEST_NAME, - file_sha256, ) @@ -37,7 +37,6 @@ def load_bundle( cutoff_seconds: float | None = None, *, progress: float | None = None, - verify_integrity: bool = True, ) -> tuple[FrozenMemoryManifest, CutoffRecord, Path, Path]: """Validate a prepared bundle and resolve one exact configured cutoff.""" if (cutoff_seconds is None) == (progress is None): @@ -81,13 +80,12 @@ def load_bundle( f"Requested {requested} is not unique in the bundle. Available: {available}" ) - if verify_integrity: - if source_path.stat().st_size != manifest.source_size_bytes: - raise ValueError(f"Source recording size changed: {source_path}") - if file_sha256(source_path) != manifest.source_sha256: - raise ValueError(f"Source recording hash changed: {source_path}") - if file_sha256(derived_path) != manifest.derived_sha256: - raise ValueError(f"Derived recording hash changed: {derived_path}") + source_stat = source_path.stat() + if ( + source_stat.st_size != manifest.source_size_bytes + or source_stat.st_mtime_ns != manifest.source_mtime_ns + ): + raise ValueError(f"Source recording identity changed: {source_path}") return manifest, matches[0], source_path, derived_path @@ -111,12 +109,15 @@ def serve_bundle( cutoff_seconds: float | None = None, *, progress: float | None = None, - mcp_port: int = 9990, ) -> None: """Run the frozen QA MCP endpoint until interrupted.""" _, cutoff, source_path, derived_path = load_bundle(bundle, cutoff_seconds, progress=progress) - server = StandaloneCodePolicyServer( - frozen_qa_config(source_path, derived_path, cutoff), - port=mcp_port, - ) - server.run_forever() + server = CodePolicyMcpServer(frozen_qa_config(source_path, derived_path, cutoff)) + server.start() + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + pass + finally: + server.stop() diff --git a/dimos/benchmark/short_horizon_qa/test_eval.py b/dimos/benchmark/short_horizon_qa/test_eval.py index 13e5f785b7..87d4adba4f 100644 --- a/dimos/benchmark/short_horizon_qa/test_eval.py +++ b/dimos/benchmark/short_horizon_qa/test_eval.py @@ -12,98 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import annotations - -import hashlib -import json -from pathlib import Path - -import numpy as np -import open3d as o3d import pytest -from dimos.benchmark.agent_eval.case import ( - EvalCase, - ExactIntegerValidatorRef, - FrozenCodePolicyInteraction, - FrozenRecordingSource, - IntegerQuestionTask, - RuntimeBinding, -) -from dimos.benchmark.agent_eval.pi import PiTurn -from dimos.benchmark.agent_eval.pi_adapter import ( - PythonExecBroker, - credential_binding_sha256, -) -from dimos.benchmark.short_horizon_qa.eval import ( - parse_integer_prediction, - run_frozen_case, -) -from dimos.benchmark.short_horizon_qa.models import MapperSettings -from dimos.benchmark.short_horizon_qa.prepare import prepare_bundle -from dimos.memory2.store.sqlite import SqliteStore -from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 - - -def _cloud(x: float, ts: float) -> PointCloud2: - cloud = o3d.geometry.PointCloud() - cloud.points = o3d.utility.Vector3dVector(np.asarray([[x, 0.0, 0.5]])) - return PointCloud2(cloud, frame_id="world", ts=ts) - - -class ScriptedPiSession: - def __init__(self, broker: PythonExecBroker, final_text: str) -> None: - self.session_id = "pi_session_scripted" - self.broker = broker - self.final_text = final_text - - def prompt(self, prompt: str, timeout_s: float) -> PiTurn: - del prompt, timeout_s - self.broker.request( - "python_exec", - { - "code": "(memory.streams.lidar.count(), " - "memory.streams.global_map.last().tags['frame_count'], " - "'app' in globals())" - }, - ) - return PiTurn(final_text=self.final_text, policy_call_count=1) - - def abort(self, timeout_s: float) -> None: - del timeout_s - - def dispose(self) -> None: - return None - - def artifact_references(self): - return () - - -class ScriptedPiFactory: - def __init__(self, final_text: str) -> None: - self.final_text = final_text - self.public_prompts: list[str] = [] - - def create( - self, - *, - attempt_path: Path, - public_prompt: str, - code_policy_session_id: str, - call_log, - mcp, - ) -> ScriptedPiSession: - self.public_prompts.append(public_prompt) - return ScriptedPiSession( - PythonExecBroker( - attempt_id=attempt_path.name, - pi_session_id="pi_session_scripted", - code_policy_session_id=code_policy_session_id, - mcp=mcp, - call_log=call_log, - ), - self.final_text, - ) +from dimos.benchmark.short_horizon_qa.eval import parse_integer_prediction @pytest.mark.parametrize( @@ -117,107 +28,6 @@ def create( ], ) def test_marked_integer_parser(text: str, status: str, answer: int | None) -> None: - prediction = parse_integer_prediction( - case_id="case", - attempt_id="attempt", - agent_session_id="pi", - interaction_session_id="policy", - final_text=text, - ) + prediction = parse_integer_prediction(text) assert prediction.status == status assert prediction.integer_answer == answer - - -@pytest.mark.parametrize( - ("final_text", "task_result"), - [("Used memory.\nANSWER: 2", "passed"), ("ANSWER: 3", "failed"), ("2", "failed")], -) -def test_real_standalone_frozen_attempt_scores_scripted_pi( - tmp_path: Path, final_text: str, task_result: str -) -> None: - recording = tmp_path / "recording.db" - with SqliteStore(path=str(recording)) as store: - lidar = store.stream("lidar", PointCloud2) - for index in range(5): - ts = 100.0 + index - lidar.append(_cloud(float(index), ts), ts=ts) - bundle = tmp_path / "bundle" - prepare_bundle( - recording, - [], - bundle, - progress=[1.0], - mapper=MapperSettings(device="CPU:0"), - ) - private_root = tmp_path / "validators" - oracle_path = private_root / "private" / "oracle.json" - oracle_path.parent.mkdir(parents=True) - oracle_sentinel = "ORACLE_PRIVATE_SENTINEL_7f9d" - credential_sentinel = "CREDENTIAL_PRIVATE_SENTINEL_a13c" - oracle = { - "schema_version": "1.0", - "expected_count": 2, - "counting_policy": oracle_sentinel, - "rooms": [ - {"schema_version": "1.0", "label": "one", "evidence": ["test"]}, - {"schema_version": "1.0", "label": "two", "evidence": ["test"]}, - ], - "reviewed_by": ["test-reviewer"], - } - oracle_path.write_text(json.dumps(oracle)) - digest = hashlib.sha256(oracle_path.read_bytes()).hexdigest() - case = EvalCase.compile( - case_id="recording-room-count", - source=FrozenRecordingSource(recording="recording", progress=1.0), - task=IntegerQuestionTask(prompt="How many rooms in total?"), - interaction=FrozenCodePolicyInteraction(driver_revision="v1"), - validator=ExactIntegerValidatorRef( - revision="exact-v1", - private_path="private/oracle.json", - private_sha256=digest, - ), - ) - - factory = ScriptedPiFactory(final_text) - result = run_frozen_case( - case=case, - bundle=bundle, - private_root=private_root, - output_root=tmp_path / "attempts", - pi_factory=factory, - runtime_binding=RuntimeBinding( - runtime_id="local-standalone-code-policy", - parameters={ - "credential_binding_sha256": credential_binding_sha256( - "environment", "TEST_API_KEY", credential_sentinel - ) - }, - ), - ) - - assert result.outcome.attempt_status == "completed" - assert result.outcome.task_result == task_result - assert (result.attempt_path / "prediction.v1.json").is_file() - assert (result.attempt_path / "score.private.v1.json").is_file() - assert (result.attempt_path / "attempt-manifest.v1.json").is_file() - calls = (result.attempt_path / "code-policy-calls.jsonl").read_text().splitlines() - assert len(calls) == 1 - records = json.loads((result.attempt_path / "code-policy-records.v1.json").read_text()) - assert "False" in records[0]["output"] - - # The private oracle is intentionally retained in its private artifact. Every - # model-facing or public/runtime surface must remain sentinel-free. - public_surfaces = [ - case.public_projection().model_dump_json(), - *factory.public_prompts, - result.outcome.model_dump_json(), - ] - for path in result.attempt_path.rglob("*"): - if path.is_file() and path.name not in { - "oracle.private.v1.json", - "score.private.v1.json", - }: - public_surfaces.append(path.read_text(errors="replace")) - serialized = "\n".join(public_surfaces) - assert oracle_sentinel not in serialized - assert credential_sentinel not in serialized diff --git a/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py b/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py index bbe24b1e38..a3139897cc 100644 --- a/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py +++ b/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py @@ -12,87 +12,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Self-hosted mechanics gate over the real Hong Kong office recording. +"""Self-hosted preparation gate over the real Hong Kong office recording.""" -The zero-valued oracle in this test is deliberately synthetic and must never be -used as the north-star room-count oracle. It validates plumbing only. -""" - -from __future__ import annotations - -import hashlib -import json from pathlib import Path import pytest -from dimos.benchmark.agent_eval.case import ( - EvalCase, - ExactIntegerValidatorRef, - FrozenCodePolicyInteraction, - FrozenRecordingSource, - IntegerQuestionTask, -) -from dimos.benchmark.short_horizon_qa.eval import run_frozen_case +from dimos.benchmark.agent_eval.models import EvalCase from dimos.benchmark.short_horizon_qa.models import MapperSettings from dimos.benchmark.short_horizon_qa.prepare import prepare_bundle -from dimos.benchmark.short_horizon_qa.test_eval import ScriptedPiFactory from dimos.utils.data import get_data @pytest.mark.self_hosted -def test_real_hongkong_recording_standalone_scripted_mechanics(tmp_path: Path) -> None: - recording = get_data("go2_hongkong_office.db") - bundle = tmp_path / "bundle" +def test_real_hongkong_recording_prepares_direct_demo_case(tmp_path: Path) -> None: + case_path = ( + Path(__file__).parent / "cases" / "demo_go2_hongkong_office-room-count-smoke" / "case.json" + ) + case = EvalCase.model_validate_json(case_path.read_bytes()) manifest = prepare_bundle( - recording, + get_data("go2_hongkong_office.db"), [], - bundle, - progress=[1.0], + tmp_path / "bundle", + progress=[case.source.progress], mapper=MapperSettings(device="CPU:0"), ) - private_root = tmp_path / "validators" - oracle_path = private_root / "private" / "mechanics-only.json" - oracle_path.parent.mkdir(parents=True) - oracle_path.write_text( - json.dumps( - { - "schema_version": "1.0", - "expected_count": 0, - "counting_policy": "Synthetic mechanics value; not a room oracle.", - "rooms": [], - "reviewed_by": ["self-hosted-mechanics-test"], - } - ) - ) - case = EvalCase.compile( - case_id="hongkong-office-mechanics-only", - source=FrozenRecordingSource( - recording="go2_hongkong_office", - progress=1.0, - bundle_manifest_sha256=hashlib.sha256( - (bundle / "manifest.v1.json").read_bytes() - ).hexdigest(), - ), - task=IntegerQuestionTask(prompt="How many rooms in total?"), - interaction=FrozenCodePolicyInteraction(driver_revision="v1"), - validator=ExactIntegerValidatorRef( - revision="mechanics-only-v1", - private_path="private/mechanics-only.json", - private_sha256=hashlib.sha256(oracle_path.read_bytes()).hexdigest(), - ), - ) - - result = run_frozen_case( - case=case, - bundle=bundle, - private_root=private_root, - output_root=tmp_path / "attempts", - pi_factory=ScriptedPiFactory("ANSWER: 0"), - ) - - assert result.outcome.task_result == "passed" + assert case.case_id == "demo-go2-hongkong-office-room-count-smoke" assert manifest.cutoffs[0].normalized_progress == 1.0 assert manifest.cutoffs[0].map_frame_count == 4235 - records = json.loads((result.attempt_path / "code-policy-records.v1.json").read_text()) - assert "False" in records[0]["output"] diff --git a/dimos/benchmark/short_horizon_qa/test_prepare.py b/dimos/benchmark/short_horizon_qa/test_prepare.py index 93c8b073e0..dca25bbf68 100644 --- a/dimos/benchmark/short_horizon_qa/test_prepare.py +++ b/dimos/benchmark/short_horizon_qa/test_prepare.py @@ -24,7 +24,6 @@ from dimos.agents.code_policy_core import FrozenMemoryEnvironment from dimos.benchmark.short_horizon_qa.models import MapperSettings from dimos.benchmark.short_horizon_qa.prepare import ( - file_sha256, prepare_bundle, resolve_progress, ) @@ -57,7 +56,7 @@ def test_prepare_builds_reusable_runtime_maps_without_changing_source( recording: Path, tmp_path: Path ) -> None: output = tmp_path / "bundle" - before = file_sha256(recording) + before = recording.read_bytes() manifest = prepare_bundle( recording, @@ -66,12 +65,14 @@ def test_prepare_builds_reusable_runtime_maps_without_changing_source( mapper=MapperSettings(device="CPU:0"), ) - assert file_sha256(recording) == before + assert recording.read_bytes() == before assert [item.map_frame_count for item in manifest.cutoffs] == [5, 10] assert [item.map_timestamp for item in manifest.cutoffs] == [104.0, 109.0] assert (output / "derived.db").is_file() encoded = json.loads((output / "manifest.v1.json").read_text()) - assert encoded["source_sha256"] == before + assert encoded["source_size_bytes"] == recording.stat().st_size + assert encoded["source_mtime_ns"] == recording.stat().st_mtime_ns + assert "source_sha256" not in encoded with FrozenMemoryStore( SqliteStore(path=str(recording), must_exist=True, read_only=True), @@ -202,7 +203,7 @@ def test_bundle_loads_into_standalone_code_policy_config(recording: Path, tmp_pa assert config.environment.memory_cutoff_timestamp == 104.0 -def test_bundle_integrity_check_rejects_changed_derived_recording( +def test_bundle_identity_check_rejects_changed_source_recording( recording: Path, tmp_path: Path ) -> None: output = tmp_path / "bundle" @@ -212,8 +213,8 @@ def test_bundle_integrity_check_rejects_changed_derived_recording( output, mapper=MapperSettings(device="CPU:0"), ) - with (output / "derived.db").open("ab") as stream: + with recording.open("ab") as stream: stream.write(b"tampered") - with pytest.raises(ValueError, match="Derived recording hash changed"): + with pytest.raises(ValueError, match="Source recording identity changed"): load_bundle(output, 4.0) diff --git a/dimos/cli/eval.py b/dimos/cli/eval.py index 478b1c6dfe..1286089673 100644 --- a/dimos/cli/eval.py +++ b/dimos/cli/eval.py @@ -16,7 +16,6 @@ from __future__ import annotations -import os from pathlib import Path import threading from typing import Any, Literal @@ -44,65 +43,32 @@ def run( agent_backend: Literal["pi"] = typer.Option("pi", "--agent.backend"), agent_model: Literal["gpt-5.6-luna"] = typer.Option("gpt-5.6-luna", "--agent.model"), thinking_level: Literal["medium"] = typer.Option("medium", "--agent.thinking-level"), - auth_mode: Literal["codex-oauth", "openai-api-key"] | None = typer.Option( - None, - "--agent.auth.mode", - help="Auth mode; inferred from auth options or OPENAI_API_KEY when omitted", - ), - auth_path: Path | None = typer.Option(None, "--agent.auth.path"), - auth_env: str | None = typer.Option(None, "--agent.auth.env"), - output: Path | None = typer.Option(None, "--output"), + api_key_env: str = typer.Option("OPENAI_API_KEY", "--agent.api-key-env"), + output: Path = typer.Option(..., "--output"), json_output: bool = typer.Option(False, "--json", help="Print compact JSON"), quiet: bool = typer.Option(False, "--quiet", help="Suppress live evaluation progress"), ) -> None: """Run one static evaluation case synchronously.""" - from dimos.benchmark.agent_eval.single_case import ( - DEFAULT_OPENAI_API_KEY_ENV, - CodexOAuthConfig, + from dimos.benchmark.agent_eval.models import ( EvalRunConfig, - OpenAIApiKeyConfig, PiAgentConfig, ) - if auth_mode is None: - if auth_path is not None and auth_env is not None: - raise typer.BadParameter( - "--agent.auth.path and --agent.auth.env select different auth modes" - ) - if auth_path is not None: - auth_mode = "codex-oauth" - elif auth_env is not None or os.environ.get(DEFAULT_OPENAI_API_KEY_ENV): - auth_mode = "openai-api-key" - else: - auth_mode = "codex-oauth" - auth: CodexOAuthConfig | OpenAIApiKeyConfig - if auth_mode == "codex-oauth": - if auth_env is not None: - raise typer.BadParameter("--agent.auth.env requires --agent.auth.mode=openai-api-key") - auth = CodexOAuthConfig(path=auth_path) - else: - if auth_path is not None: - raise typer.BadParameter("--agent.auth.path requires --agent.auth.mode=codex-oauth") - auth = OpenAIApiKeyConfig(env=auth_env or DEFAULT_OPENAI_API_KEY_ENV) config = EvalRunConfig( agent=PiAgentConfig( backend=agent_backend, model=agent_model, thinking_level=thinking_level, - auth=auth, + api_key_env=api_key_env, ) ) renderer = None if quiet else ProgressRenderer() try: - result = ( - execute_single_case(case, config=config, progress=renderer) - if output is None - else execute_single_case( - case, - config=config, - output_root=output, - progress=renderer, - ) + result = execute_single_case( + case, + config=config, + output=output, + progress=renderer, ) except Exception as exc: if renderer is not None: @@ -111,12 +77,12 @@ def run( raise typer.Exit(2) from exc if renderer is not None: renderer.finish() - typer.echo(result.model_dump_json() if json_output else format_result(result)) + typer.echo(result.model_dump_json() if json_output else format_result(result, output)) if result.attempt_status == "failed": raise typer.Exit(1) -def format_result(result: Any) -> str: +def format_result(result: Any, output: Path | None = None) -> str: """Render the compact typed result without exposing private oracle material.""" if result.attempt_status == "failed": heading = "! Evaluation not evaluated" @@ -124,24 +90,17 @@ def format_result(result: Any) -> str: heading = "✓ Evaluation passed" else: heading = "✗ Evaluation failed" - source = result.source - if result.progress is not None: - source += f" @ {result.progress * 100:g}%" + source = f"{result.recording} @ {result.progress * 100:g}%" answer = str(result.integer_answer) if result.integer_answer is not None else "—" rows = ( ("Case", result.case_id), ("Source", source), - ("Question", result.question), ("Answer", answer), ("Result", result.task_result), - ( - "Agent", - f"{result.agent.agent_id} · {result.agent.model} · {result.agent.thinking_level}", - ), + ("Agent", f"Pi · {result.model} · {result.thinking_level}"), ("Tool calls", str(result.tool_call_count)), ("Duration", f"{result.duration_seconds:.1f}s"), - ("Attempt", result.attempt_id), - ("Artifacts", str(result.artifact_path)), + ("Output", str((output / "result.json") if output is not None else "result.json")), ) body = "\n".join(f" {label:<10} {value}" for label, value in rows) return f"{heading}\n\n{body}" diff --git a/dimos/cli/test_eval.py b/dimos/cli/test_eval.py index d75b9ee901..78e6fc415f 100644 --- a/dimos/cli/test_eval.py +++ b/dimos/cli/test_eval.py @@ -22,42 +22,27 @@ import pytest from typer.testing import CliRunner -from dimos.benchmark.agent_eval.case import AgentCondition -from dimos.benchmark.agent_eval.progress import ( - AssistantTextProgress, - CaseHeaderProgress, - StatusProgress, - ToolEndProgress, - ToolStartProgress, -) -from dimos.benchmark.agent_eval.single_case import CompactEvalResult +from dimos.benchmark.agent_eval.models import CompactEvalResult +from dimos.benchmark.agent_eval.progress import StatusProgress from dimos.cli.dimos import main import dimos.cli.eval as eval_cli -def _result( - tmp_path: Path, *, status: str = "completed", task: str = "passed" -) -> CompactEvalResult: +def _result(*, passed: bool | None = True) -> CompactEvalResult: return CompactEvalResult( - attempt_id="attempt_" + "a" * 32, - case_id="hongkong-room-count", - source="go2_hongkong_office", + case_id="demo-room-count", + recording="go2_hongkong_office", progress=1.0, - question="How many rooms in total?", - attempt_status=status, - task_result=task, - reason="validator passed" if status == "completed" else "infrastructure failed", - prediction_status="parsed" if status == "completed" else None, - integer_answer=4 if status == "completed" else None, - agent=AgentCondition( - agent_id="pi-code-policy", - adapter="pi-node", - model="gpt-5.6-luna", - thinking_level="medium", - ), + model="gpt-5.6-luna", + thinking_level="medium", + final_response="ANSWER: 4" if passed is not None else "", + prediction_status="parsed" if passed is not None else "not_evaluated", + integer_answer=4 if passed is not None else None, + passed=passed, + validator_revision="v1", tool_call_count=7, duration_seconds=42.75, - artifact_path=tmp_path / "attempt", + infra_error="Pi failed" if passed is None else None, ) @@ -67,201 +52,115 @@ def _case(tmp_path: Path) -> Path: return path -def test_eval_run_uses_typed_defaults_and_separates_progress(tmp_path, monkeypatch) -> None: - monkeypatch.delenv("OPENAI_API_KEY", raising=False) +def test_eval_run_uses_api_key_default_and_separates_progress(tmp_path, monkeypatch) -> None: captured = {} - def execute(path, *, config, progress, **kwargs): - captured.update(path=path, config=config, progress=progress, kwargs=kwargs) + def execute(path, *, config, progress, output): + captured.update(path=path, config=config, progress=progress, output=output) progress(StatusProgress(channel="eval", message="loading case")) - progress( - CaseHeaderProgress( - case_id="hongkong-room-count", - source="go2_hongkong_office", - progress=1.0, - question="How many rooms in total?", - ) - ) - progress(AssistantTextProgress(delta="Inspecting memory")) - progress(ToolStartProgress(code="memory.streams()")) - progress(ToolEndProgress(ok=True, result="['lidar']", duration_seconds=0.25)) - return _result(tmp_path) + return _result() monkeypatch.setattr(eval_cli, "execute_single_case", execute) - result = CliRunner().invoke(main, ["eval", "run", str(_case(tmp_path))]) - + output = tmp_path / "run" + result = CliRunner().invoke(main, ["eval", "run", str(_case(tmp_path)), f"--output={output}"]) assert result.exit_code == 0, result.output - assert captured["config"].agent.auth.mode == "codex-oauth" + assert captured["config"].agent.api_key_env == "OPENAI_API_KEY" + assert captured["output"] == output assert "✓ Evaluation passed" in result.stdout - assert "go2_hongkong_office @ 100%" in result.stdout assert "[eval] loading case" in result.stderr - assert "[pi] Inspecting memory" in result.stderr - assert "[python_exec] ok (0.2s)" in result.stderr - - -def test_eval_run_auth_inference_and_explicit_precedence(tmp_path, monkeypatch) -> None: - monkeypatch.setenv("OPENAI_API_KEY", "credential-sentinel") - captured = [] - - def execute(*args, **kwargs): - captured.append(kwargs["config"]) - return _result(tmp_path) - - monkeypatch.setattr(eval_cli, "execute_single_case", execute) - runner = CliRunner() - automatic = runner.invoke(main, ["eval", "run", str(_case(tmp_path))]) - explicit = runner.invoke( - main, - ["eval", "run", str(_case(tmp_path)), "--agent.auth.mode=codex-oauth"], - ) - - assert automatic.exit_code == explicit.exit_code == 0 - assert captured[0].agent.auth.mode == "openai-api-key" - assert captured[1].agent.auth.mode == "codex-oauth" - assert "credential-sentinel" not in automatic.output + explicit.output -def test_eval_run_accepts_dotted_options_and_json(tmp_path, monkeypatch) -> None: +def test_eval_run_accepts_named_api_key_env_and_json(tmp_path, monkeypatch) -> None: captured = {} def execute(*args, **kwargs): captured.update(kwargs) - return _result(tmp_path) + return _result() monkeypatch.setattr(eval_cli, "execute_single_case", execute) - output = tmp_path / "results" + output = tmp_path / "run" result = CliRunner().invoke( main, [ "eval", "run", str(_case(tmp_path)), - "--agent.backend=pi", - "--agent.model=gpt-5.6-luna", - "--agent.auth.mode=openai-api-key", - "--agent.auth.env=MY_OPENAI_KEY", + "--agent.api-key-env=MY_OPENAI_KEY", f"--output={output}", "--json", ], ) - assert result.exit_code == 0, result.output - assert captured["config"].agent.auth.env == "MY_OPENAI_KEY" - assert captured["output_root"] == output - assert json.loads(result.stdout)["task_result"] == "passed" - assert "private" not in result.stdout - - -def test_eval_run_quiet_and_exit_codes(tmp_path, monkeypatch) -> None: - observed = [] + assert captured["config"].agent.api_key_env == "MY_OPENAI_KEY" + assert json.loads(result.stdout)["passed"] is True - def failed_attempt(*args, **kwargs): - observed.append(kwargs["progress"]) - return _result(tmp_path, status="failed", task="not_evaluated") - monkeypatch.setattr(eval_cli, "execute_single_case", failed_attempt) - failed = CliRunner().invoke(main, ["eval", "run", str(_case(tmp_path)), "--quiet"]) +def test_eval_exit_codes_distinguish_infra_semantic_and_preflight(tmp_path, monkeypatch) -> None: + output = tmp_path / "run" + monkeypatch.setattr(eval_cli, "execute_single_case", lambda *a, **k: _result(passed=None)) + infra = CliRunner().invoke( + main, ["eval", "run", str(_case(tmp_path)), f"--output={output}", "--quiet"] + ) + monkeypatch.setattr(eval_cli, "execute_single_case", lambda *a, **k: _result(passed=False)) + semantic = CliRunner().invoke( + main, ["eval", "run", str(_case(tmp_path)), f"--output={output}", "--quiet"] + ) - def preflight(*args, **kwargs): - raise FileNotFoundError("adapter build missing") + def preflight(*_args, **_kwargs): + raise FileNotFoundError("extension build missing") monkeypatch.setattr(eval_cli, "execute_single_case", preflight) - preflight_result = CliRunner().invoke(main, ["eval", "run", str(_case(tmp_path))]) - - assert failed.exit_code == 1 - assert observed == [None] - assert failed.stderr == "" - assert preflight_result.exit_code == 2 - assert "Evaluation preflight failed: FileNotFoundError" in preflight_result.stderr - - -def test_eval_rejects_invalid_auth_combinations_and_semantic_override(tmp_path) -> None: - runner = CliRunner() - case = _case(tmp_path) - conflicting = runner.invoke( - main, - ["eval", "run", str(case), "--agent.auth.path=x", "--agent.auth.env=Y"], + preflight_result = CliRunner().invoke( + main, ["eval", "run", str(_case(tmp_path)), f"--output={output}"] ) - semantic = runner.invoke(main, ["eval", "run", str(case), "--source.recording=other"]) - - assert conflicting.exit_code == 2 - assert "select different auth" in conflicting.stderr - assert "modes" in conflicting.stderr - assert semantic.exit_code == 2 - assert "No such option" in semantic.stderr + assert infra.exit_code == 1 + assert semantic.exit_code == 0 + assert preflight_result.exit_code == 2 -def test_eval_help_is_typed_and_rejects_unsupported_model(tmp_path) -> None: +def test_eval_help_is_typed_and_output_is_required(tmp_path) -> None: runner = CliRunner() help_result = runner.invoke(main, ["eval", "run", "--help"]) - unsupported = runner.invoke( - main, - ["eval", "run", str(_case(tmp_path)), "--agent.model=unreviewed-model"], - ) - + missing_output = runner.invoke(main, ["eval", "run", str(_case(tmp_path))]) assert help_result.exit_code == 0 - assert "--agent.model" in help_result.stdout - assert "gpt-5.6-luna" in help_result.stdout - assert "--agent.thinking-level" in help_result.stdout - assert unsupported.exit_code == 2 - assert "unreviewed-model" in unsupported.stderr + assert "--agent.api-key-env" in help_result.stdout + assert "--output" in help_result.stdout + assert missing_output.exit_code == 2 -def test_eval_semantic_failure_is_a_successful_attempt(tmp_path, monkeypatch) -> None: - monkeypatch.setattr( - eval_cli, - "execute_single_case", - lambda *args, **kwargs: _result(tmp_path, status="completed", task="failed"), - ) - - result = CliRunner().invoke(main, ["eval", "run", str(_case(tmp_path))]) - - assert result.exit_code == 0 - assert "Evaluation failed" in result.stdout - - -def test_lazy_runtime_import_has_actionable_missing_agents_error(monkeypatch) -> None: +def test_lazy_runtime_import_has_actionable_error(monkeypatch) -> None: original_import = builtins.__import__ def fail_single_case(name, *args, **kwargs): if name == "dimos.benchmark.agent_eval.single_case": - raise ModuleNotFoundError("No module named 'fastapi'") + raise ModuleNotFoundError("No module named 'mcp'") return original_import(name, *args, **kwargs) monkeypatch.setattr(builtins, "__import__", fail_single_case) - with pytest.raises(RuntimeError, match="uv sync --extra agents"): eval_cli.execute_single_case(Path("case.json"), config=None) -def test_base_cli_help_imports_without_agents_only_modules() -> None: +def test_base_cli_help_imports_without_eval_runtime() -> None: script = textwrap.dedent( """ import sys - class BlockAgentsImports: + class BlockEvalImports: def find_spec(self, fullname, path=None, target=None): - if fullname.split('.')[0] in { - 'fastapi', 'ipykernel', 'jupyter_client', 'uvicorn' - }: - raise RuntimeError(f'agents-only import attempted: {fullname}') + if fullname.split('.')[0] in {'mcp', 'ipykernel', 'jupyter_client', 'uvicorn'}: + raise RuntimeError(f'eval runtime import attempted: {fullname}') return None - sys.meta_path.insert(0, BlockAgentsImports()) + sys.meta_path.insert(0, BlockEvalImports()) from typer.testing import CliRunner from dimos.cli.dimos import main - result = CliRunner().invoke(main, ['--help']) assert result.exit_code == 0, result.output assert 'eval' in result.stdout """ ) - completed = subprocess.run( - [sys.executable, "-c", script], - check=False, - capture_output=True, - text=True, + [sys.executable, "-c", script], capture_output=True, text=True, check=False ) - assert completed.returncode == 0, completed.stdout + completed.stderr diff --git a/dimos/memory2/observationstore/sqlite.py b/dimos/memory2/observationstore/sqlite.py index ef7e4a1774..0dedf0ad9f 100644 --- a/dimos/memory2/observationstore/sqlite.py +++ b/dimos/memory2/observationstore/sqlite.py @@ -30,7 +30,6 @@ BeforeFilter, NearFilter, TagsFilter, - ThroughFilter, TimeRangeFilter, ) from dimos.memory2.type.observation import _UNLOADED, Observation, PoseTuple @@ -72,8 +71,6 @@ def _compile_filter(f: Filter, stream: str, prefix: str = "") -> tuple[str, list return (f"{prefix}ts > ?", [f.t]) if isinstance(f, BeforeFilter): return (f"{prefix}ts < ?", [f.t]) - if isinstance(f, ThroughFilter): - return (f"{prefix}ts <= ?", [f.t]) if isinstance(f, TimeRangeFilter): return (f"{prefix}ts >= ? AND {prefix}ts <= ?", [f.t1, f.t2]) if isinstance(f, AtFilter): @@ -344,6 +341,8 @@ def _ensure_tag_indexes(self, tags: dict[str, Any]) -> None: self._tag_indexes.add(key) def insert(self, obs: Observation[T]) -> int: + if self.config.read_only: + raise PermissionError("Cannot append to a read-only SQLite store") pose = obs.pose_tuple tags_json = json.dumps(obs.tags) if obs.tags else "{}" value = obs._data if isinstance(obs._data, (int, float)) else None diff --git a/dimos/memory2/store/frozen.py b/dimos/memory2/store/frozen.py index 27158a3b28..bf3555d835 100644 --- a/dimos/memory2/store/frozen.py +++ b/dimos/memory2/store/frozen.py @@ -16,6 +16,7 @@ from __future__ import annotations +import math from typing import Any, TypeVar, cast from dimos.core.resource import CompositeResource @@ -68,7 +69,7 @@ def stream(self, name: str, payload_type: type[T] | None = None) -> Stream[T]: if name not in self._streams: store = self._source if name in self._source_names else self._derived assert store is not None - self._streams[name] = store.stream(name).through(self.through_timestamp).as_read_only() + self._streams[name] = store.stream(name).time_range(-math.inf, self.through_timestamp) return cast("Stream[T]", self._streams[name]) def delete_stream(self, name: str) -> None: diff --git a/dimos/memory2/store/sqlite.py b/dimos/memory2/store/sqlite.py index 181533b187..b547e90eba 100644 --- a/dimos/memory2/store/sqlite.py +++ b/dimos/memory2/store/sqlite.py @@ -27,7 +27,6 @@ from dimos.memory2.observationstore.sqlite import SqliteObservationStore from dimos.memory2.registry import RegistryStore, deserialize_component, qual from dimos.memory2.store.base import Store, StoreConfig -from dimos.memory2.stream import Stream from dimos.memory2.utils.sqlite import open_disposable_sqlite_connection from dimos.memory2.utils.validation import validate_identifier from dimos.memory2.vectorstore.base import VectorStore @@ -222,12 +221,6 @@ def list_streams(self) -> list[str]: db_names = set(self._registry.list_streams()) return sorted(db_names | set(self._streams.keys())) - def stream( - self, name: str, payload_type: type[Any] | None = None, **overrides: Any - ) -> Stream[Any]: - stream = super().stream(name, payload_type, **overrides) - return stream.as_read_only() if self.config.read_only else stream - def delete_stream(self, name: str) -> None: if self.config.read_only: raise PermissionError("Cannot delete streams from a read-only store") diff --git a/dimos/memory2/store/test_frozen.py b/dimos/memory2/store/test_frozen.py index a4f3834a6b..74d2ace3d0 100644 --- a/dimos/memory2/store/test_frozen.py +++ b/dimos/memory2/store/test_frozen.py @@ -61,7 +61,7 @@ def test_frozen_memory_rejects_mutation(recorded_stores) -> None: derived=SqliteStore(path=str(derived_path), must_exist=True, read_only=True), through_timestamp=20.0, ) as memory: - with pytest.raises(PermissionError, match="read-only stream"): + with pytest.raises(PermissionError, match="read-only SQLite store"): memory.streams.camera.append("nope", ts=15.0) with pytest.raises(PermissionError, match="frozen memory"): memory.delete_stream("camera") @@ -77,7 +77,7 @@ def test_read_only_sqlite_store_does_not_create_wal(recorded_stores) -> None: with SqliteStore(path=str(source_path), must_exist=True, read_only=True) as source: assert source.stream("camera").last().data == "future" - with pytest.raises(PermissionError, match="read-only stream"): + with pytest.raises(PermissionError, match="read-only SQLite store"): source.stream("camera").append("nope") with pytest.raises(PermissionError, match="read-only store"): source.delete_stream("camera") diff --git a/dimos/memory2/stream.py b/dimos/memory2/stream.py index 75d4dce477..1ed4651398 100644 --- a/dimos/memory2/stream.py +++ b/dimos/memory2/stream.py @@ -38,7 +38,6 @@ PredicateFilter, StreamQuery, TagsFilter, - ThroughFilter, TimeRangeFilter, ) from dimos.memory2.type.observation import EmbeddedObservation, Observation @@ -128,7 +127,6 @@ def __init__( *, transform: Transformer[Any, T] | None = None, query: StreamQuery = StreamQuery(), - writable: bool = True, ) -> None: super().__init__() self._source = source @@ -136,7 +134,6 @@ def __init__( self.register_disposable(source) self._transform = transform self._query = query - self._writable = writable def stop(self) -> None: buf = self._query.live_buffer @@ -222,21 +219,7 @@ def _replace_query(self, **overrides: Any) -> Stream[T, O]: search_k=overrides.get("search_k", q.search_k), search_text=overrides.get("search_text", q.search_text), ) - return Stream( - self._source, - transform=self._transform, - query=new_q, - writable=self._writable, - ) - - def as_read_only(self) -> Stream[T, O]: - """Return a query-equivalent stream that rejects appends.""" - return Stream( - self._source, - transform=self._transform, - query=self._query, - writable=False, - ) + return Stream(self._source, transform=self._transform, query=new_q) def _with_filter(self, f: Filter) -> Stream[T, O]: return self._replace_query(filters=(*self._query.filters, f)) @@ -247,10 +230,6 @@ def after(self, t: float) -> Stream[T, O]: def before(self, t: float) -> Stream[T, O]: return self._with_filter(BeforeFilter(t)) - def through(self, t: float) -> Stream[T, O]: - """Keep observations at or before absolute timestamp ``t``.""" - return self._with_filter(ThroughFilter(t)) - def time_range(self, t1: float, t2: float) -> Stream[T, O]: return self._with_filter(TimeRangeFilter(t1, t2)) @@ -701,8 +680,6 @@ def append( Returns :class:`EmbeddedObservation` when *embedding* is provided, else a plain :class:`Observation`. """ - if not self._writable: - raise PermissionError("Cannot append to a read-only stream") if isinstance(self._source, Stream) or self._source is None: raise TypeError( "Cannot append to a transform/unbound stream. Append to the source stream." diff --git a/dimos/memory2/type/filter.py b/dimos/memory2/type/filter.py index d64448b5db..1250c4c31f 100644 --- a/dimos/memory2/type/filter.py +++ b/dimos/memory2/type/filter.py @@ -57,16 +57,6 @@ def matches(self, obs: Observation[Any]) -> bool: return obs.ts < self.t -@dataclass(frozen=True) -class ThroughFilter(Filter): - """Include observations at or before an absolute timestamp.""" - - t: float - - def matches(self, obs: Observation[Any]) -> bool: - return obs.ts <= self.t - - @dataclass(frozen=True) class TimeRangeFilter(Filter): t1: float diff --git a/docs/capabilities/agents/evaluation.md b/docs/capabilities/agents/evaluation.md index ee9b61c3bd..211c26375a 100644 --- a/docs/capabilities/agents/evaluation.md +++ b/docs/capabilities/agents/evaluation.md @@ -2,93 +2,78 @@ title: "Frozen recording evaluation" --- -`dimos eval run` asks one integer question about one immutable Memory2 recording. It -prepares a read-only map, gives a fresh Pi agent one `python_exec` tool, validates -the terminal answer against a private oracle, and writes durable attempt evidence. -It does not start a robot, simulation, replay blueprint, or live DimOS module. +`dimos eval run` asks Pi one integer question about a frozen Memory2 recording. +The evaluator prepares the runtime map, exposes read-only `memory` through one +`python_exec` MCP tool, and checks the final `ANSWER: ` line against a +private oracle. It does not start a robot, simulation, replay blueprint, or live +DimOS module. ## Setup -Install the Python agent dependencies and build the dedicated Node adapter from a -source checkout: +From a source checkout, install the lightweight Python runtime and build the Pi +extension: ```bash uv sync --extra agents -npm ci --prefix packages/pi-code-policy-adapter -npm run build --prefix packages/pi-code-policy-adapter +npm ci --prefix packages/pi-code-policy-extension +npm run build --prefix packages/pi-code-policy-extension ``` -The adapter requires Node 22.19.0 or newer. The Python command reports a preflight -error if `packages/pi-code-policy-adapter/dist/code-policy-main.js` is absent. +The package pins Pi `0.80.10` and requires Node 22.19.0 or newer. Set the API key +before running a case: -## Run one case +```bash +export OPENAI_API_KEY=... +``` -Run the Hong Kong office plumbing fixture with: +## Run the direct demo case ```bash uv run dimos eval run \ - dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/case.json \ + dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/case.json \ --output=/tmp/dimos-eval-smoke ``` -The fixture's expected count is the synthetic sentinel `0`, not a reviewed room -count. A completed attempt may therefore report a semantic failure even when the -agent, map, and evidence pipeline work correctly. +The demo fixture uses the synthetic sentinel `0`, not a reviewed Hong Kong office +room count. A semantic failure can therefore mean the agent and runtime worked but +the response did not match that plumbing sentinel. -`eval run` accepts these options: +The supported options are deliberately small: | Option | Default | Purpose | | --- | --- | --- | -| `--agent.backend` | `pi` | Select the pinned agent backend. | -| `--agent.model` | `gpt-5.6-luna` | Select the pinned model. | -| `--agent.thinking-level` | `medium` | Select the pinned thinking level. | -| `--agent.auth.mode` | inferred | Use `codex-oauth` or `openai-api-key`. | -| `--agent.auth.path` | `~/.pi/agent/auth.json` | Select a Codex OAuth file. | -| `--agent.auth.env` | `OPENAI_API_KEY` | Name the API-key environment variable. | -| `--output` | DimOS state directory | Set the append-only attempt root. | -| `--json` | off | Print one compact JSON result to stdout. | -| `--quiet` | off | Suppress live progress on stderr. | - -Do not pass credential values as command-line arguments. With no explicit mode, -an auth path selects OAuth, an auth environment name selects API-key auth, and a -set `OPENAI_API_KEY` selects API-key auth. Otherwise, the command uses Codex OAuth. -An explicit `--agent.auth.mode` takes precedence over environment inference. +| `--agent.backend` | `pi` | Use the pinned Pi backend. | +| `--agent.model` | `gpt-5.6-luna` | Use the pinned model. | +| `--agent.thinking-level` | `medium` | Use the pinned thinking level. | +| `--agent.api-key-env` | `OPENAI_API_KEY` | Select the environment variable containing the API key. | +| `--output` | required | Publish this run to the exact directory. | +| `--json` | off | Print the compact result as JSON. | +| `--quiet` | off | Suppress status messages on stderr. | + +The API key is passed only to the Pi subprocess. It is not placed in arguments, +results, or the Jupyter kernel environment. ## Output and exit status -The final human or JSON result goes to stdout. Live case, agent, and tool progress -goes to stderr, so scripts can parse `--json` output safely. `--quiet` suppresses -the progress stream without suppressing the final result. - -The command uses three exit codes: - -| Code | Meaning | -| --- | --- | -| `0` | The attempt completed. Its private score may be passed or failed. | -| `1` | The attempt started but infrastructure failed. | -| `2` | Preflight failed before an attempt was reserved. | - -Preflight verifies the case, private oracle digest, recording bundle, credential -binding, and built Node entrypoint. Once an attempt starts, its mode-`0700` -directory contains lifecycle events, public and private artifacts, content -descriptors, CodePolicy receipts, broker calls, and Pi evidence. Files are created -exclusively; reruns create new attempts instead of overwriting old evidence. - -## Privacy and trust boundary - -The agent receives the case's public projection: recording identity, cutoff, -question, and interaction contract. It does not receive the validator path, -oracle content, or credential value. Private oracle and score files remain private -attempt artifacts. Public prompts, progress, compact results, broker logs, Pi -evidence, and serialized runtime configuration contain no oracle or credential -material. - -CodePolicy is trusted, persistent, and unsandboxed Python execution. Its supplied -`memory` API is read-only and cutoff-limited, but Python code can still access the -host filesystem and processes. Run only trusted evaluation agents and code. Use an -OS sandbox or container for hostile code. - -Each attempt creates fresh Pi and CodePolicy processes. Normal completion, -failure, timeout, and interruption close those processes and release the output -lock. If a smoke run is interrupted externally, confirm no `code-policy` or -`pi-code-policy-adapter` child remains before retrying. +`--output` must name an absent or empty directory. The evaluator builds the run in +a temporary sibling and atomically publishes it on completion. It never merges +with or overwrites a nonempty directory. + +The directory contains only: + +- `result.json`; +- `pi-transcript.jsonl`, when Pi wrote a native transcript; +- `stderr.log`, only when nonempty diagnostics are available. + +Exit code `0` means evaluation completed, whether the semantic score passed or +failed. Exit code `1` means a caught runtime or agent infrastructure failure; the +published `result.json` includes `infra_error`. Exit code `2` means preflight +failed before a run started. + +## Trust boundary + +CodePolicy executes agent-authored Python in a persistent Jupyter kernel. It is +trusted and **unsandboxed**. The `memory` object is cutoff-limited and its SQLite +connections are truly read-only, but Python can still access other host files and +processes. Run only trusted evaluation agents, or place the whole command in an OS +sandbox or container. diff --git a/docs/development/testing.md b/docs/development/testing.md index 9584721758..8932ad1481 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -71,15 +71,15 @@ pytest -m self_hosted dimos/path/to/test_something.py ### Frozen agent evaluation -Install the optional Python dependencies, then build and test the dedicated Node -adapter: +Install the optional Python dependencies, then build and test the small Pi +extension: ```bash uv sync --extra agents -npm ci --prefix packages/pi-code-policy-adapter -npm run typecheck --prefix packages/pi-code-policy-adapter -npm run build --prefix packages/pi-code-policy-adapter -npm test --prefix packages/pi-code-policy-adapter +npm ci --prefix packages/pi-code-policy-extension +npm run typecheck --prefix packages/pi-code-policy-extension +npm run build --prefix packages/pi-code-policy-extension +npm test --prefix packages/pi-code-policy-extension ``` Run the focused Python suite with: @@ -89,7 +89,6 @@ uv run --extra agents pytest \ dimos/memory2/store/test_frozen.py \ dimos/agents/test_code_policy_core.py \ dimos/agents/test_code_policy_server.py \ - dimos/agents/mcp/test_mcp_adapter.py \ dimos/benchmark/agent_eval \ dimos/benchmark/short_horizon_qa \ dimos/cli/test_eval.py diff --git a/openspec/changes/extract-frozen-qa-eval/design.md b/openspec/changes/extract-frozen-qa-eval/design.md index 4df8f48c85..44e42566b8 100644 --- a/openspec/changes/extract-frozen-qa-eval/design.md +++ b/openspec/changes/extract-frozen-qa-eval/design.md @@ -1,112 +1,102 @@ ## Context -The complete frozen QA path exists at reference commit `30e5f1c0e` on `cc/frontier`, primarily in commits `7cbf13845` and `10ca9bf15`. Those commits depend on earlier agent-evaluation and Pi adapter history and also contain accidental imports from live DimSim and spatial benchmark packages. The feature must therefore be ported file-by-file onto the recorded `origin/main` base rather than cherry-picked. +The target is one source-checkout command: -The target is one source-checkout command that evaluates one frozen Memory2 recording synchronously. The implementation spans SQLite access, derived map preparation, trusted CodePolicy execution, a Node/Pi subprocess, private validation, evidence storage, and Typer registration. The base DimOS CLI must remain importable without the `agents` extra. - -## Goals / Non-Goals +```bash +uv run dimos eval run \ + dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/case.json \ + --output=/tmp/dimos-eval-smoke +``` -**Goals:** +The first implementation on PR #3378 was intentionally defensive, but it produced 17 attempt files and three copies of the agent trajectory. Review established that Memory2 already owns environmental observations and Pi already owns its native transcript. The evaluator needs only a compact result beside that transcript. -- Preserve the reference frozen-QA behavior behind `dimos eval run` with a focused dependency graph. -- Keep semantic cases immutable and independent of credentials, paths, ports, and output locations. -- Enforce private-oracle isolation and read-only, inclusive frozen-memory access. -- Run fresh CodePolicy and Pi processes per attempt with exactly `python_exec` exposed. -- Retain durable, non-overwriting evidence and always release processes and locks. -- Resolve all ported code against current `main` APIs and optional-dependency conventions. +## Goals -**Non-Goals:** +- Keep the direct command small and understandable. +- Preserve extension seams only for source, task, validator, and the shared CodePolicy session. +- Prevent writes to source and derived Memory2 databases. +- Keep the private oracle and API key outside the model-facing namespace. +- Use official MCP transports and Pi's stock process lifecycle. +- Leave no child process after normal completion, caught failure, timeout, or interruption. +- Pass the repository's existing CI with focused Python and Node unit tests. -- Dataset batching, retries, containers, scheduling, or distributed execution. -- Live DimSim, simulation episodes, replay blueprints, robot control, or hardware evaluation. -- Legacy smoke-runner configuration, backend abstraction for unrelated evaluators, or agentic blueprint integration. -- Treating read-only SQLite or trusted CodePolicy as a security sandbox. -- Shipping the Node adapter inside a Python wheel in this initial source-checkout slice. +## Non-Goals -## DimOS Architecture +- Generic attempt engines, adapter registries, dataset scheduling, retries, or distributed execution. +- Crash-forensic event logs, artifact descriptors, cryptographic result attestations, or replay guarantees. +- OAuth, live robot RPCs, simulation, DimSim, blueprints, or agentic module integration. +- Migrating the existing repository-wide DimOS MCP implementation. +- Treating Jupyter or read-only Memory2 as a hostile-code sandbox. -The runtime flow is: +## Architecture ```text case.json + private oracle | v -single-case preflight -----> frozen bundle cache - | | - v v -AttemptStore lock -------> FrozenMemoryStore - | | - v v -Standalone CodePolicy MCP (`memory`, no `app`) - ^ - | python_exec MCP calls - v -dedicated Node/Pi process (exactly one tool) +frozen bundle cache -----> FrozenMemoryStore + | + v +evaluator process -----> CodePolicySession -----> Jupyter kernel child + | ^ + | official MCP server | python_exec + v | +stock Pi CLI child ---- tiny official-MCP extension | v -prediction -> private validator -> terminal evidence -> cleanup/unlock +native JSON events + transcript -> parse -> private score -> result.json ``` -The focused Python package is split by dependency direction: - -- `agent_eval/base.py`: strict common Pydantic configuration. -- `agent_eval/json.py`: local UTF-8 canonical JSON using sorted keys and compact separators. -- `agent_eval/artifacts.py`: artifact references, typed IDs, and lifecycle records without benchmark-specific imports. -- `agent_eval/auth.py`: the runtime credential transport record without DimSim imports. -- `agent_eval/case.py`: only the supported frozen source, integer task, frozen interaction, exact validator, request, prediction, score, and outcome contracts. -- `agent_eval/interfaces.py`, `engine.py`, and `store.py`: adapter Protocols, lifecycle orchestration, durable evidence, and locking; no DimSim or spatial types. -- `agent_eval/pi*.py`: MCP binding, evidence log, Node subprocess, and progress transport. -- `agent_eval/single_case.py`: preflight, authentication resolution, adapter discovery, bundle preparation, and compact result projection. -- `short_horizon_qa/*`: frozen preparation, service binding, parser, validator, and interaction drivers. -- `agents/code_policy_core.py` and `code_policy_server.py`: persistent trusted Python execution and loopback-only standalone MCP hosting. -- `memory2/store/frozen.py` plus narrow existing Memory2 changes: read-only overlay and inclusive cutoff. - -The Node package `packages/pi-code-policy-adapter` contains only its line protocol, `python_exec` definition, pinned Pi session/auth setup, evidence retention, and tests. It does not depend on spatial tool definitions or adapter code. Python resolves its built `dist/code-policy-main.js` from the source checkout and reports an actionable preflight error when it is absent. +### Python modules -No DimOS `Spec` Protocol, module stream, blueprint, or generated registry is added. Internal Python adapter Protocols define the source, interaction, validator, evidence, and Pi session seams. The only MCP-visible surface is the standalone `python_exec` tool created for the attempt. +- `dimos/agents/code_policy.py`: the reusable Jupyter session, environment bootstrap, timeout recovery, and credential-scrubbed kernel launch. It knows nothing about MCP, Pi, or evaluation. +- `dimos/memory2/store/frozen.py`: source/derived overlay and inclusive cutoff. +- `dimos/benchmark/agent_eval/models.py`: strict tagged case and compact result contracts. +- `dimos/benchmark/short_horizon_qa/prepare.py`: recording resolution and derived map cache. +- `dimos/benchmark/short_horizon_qa/eval.py`: in-process MCP host, stock Pi launcher/event parser, answer parser, private scorer, cleanup, and output publication. +- `dimos/cli/eval.py`: dependency-light Typer shell with callback-local runtime imports. -## Decisions - -1. **Port current file content, not commits.** Cherry-picking would import the full benchmark graph and unrelated lockfile changes. Each focused file is copied from `30e5f1c0e`, pruned, and reconciled with current `main`. - -2. **Use three small foundation modules instead of the reference generic `models.py` and `config.py`.** Canonical JSON, artifact records, and runtime credentials form a dependency floor that does not import DimSim or spatial packages. `case.py` omits live discriminated variants so unsupported inputs fail during schema validation. +Small helpers may remain separate only when they own a concrete lifecycle boundary; generic artifact, engine, broker, and Protocol layers are removed. -3. **Reserve attempts only after preflight.** Case/oracle validation, source preparation, credential resolution, and adapter discovery happen before attempt reservation and map to exit `2`. Once the store is reserved, normalized infrastructure failures map to exit `1`; semantic results map to exit `0`. +### Official MCP -4. **Make lock release structurally unconditional.** The attempt engine owns the store in an outer `try/finally`. Resource cleanup and terminal artifact publication may affect the outcome but cannot bypass store closure. Fault-injection tests cover fsync, manifest, event, and terminal-publication failures. +Pin the official Python `mcp==2.0.0` SDK and register one `python_exec` tool. The evaluator pre-binds a loopback socket to port `0`, supplies it to Uvicorn, and runs the SDK's ASGI application on a server thread. The evaluator directly owns server shutdown and the `CodePolicySession`; no `/control` HTTP surface or extra Python process exists. -5. **Treat private data as an information-flow boundary.** The public projection contains no validator. Oracle bytes are loaded only by the validator, and tests scan prompts, progress, compact results, Pi evidence, broker logs, and CodePolicy state for private material. +Pin `@modelcontextprotocol/client==2.0.0` in the Node extension. The extension validates the one-tool inventory, calls `python_exec`, uses a timeout longer than the kernel execution timeout, and closes the MCP session during Pi shutdown. -6. **Implement real SQLite read-only mode.** Source and derived stores use SQLite `mode=ro` and `query_only`; writable connections alone configure WAL. Mutation methods and streams reject writes. A `ThroughFilter` implements `ts <= cutoff` and is applied to every overlaid stream. +### Stock Pi process -7. **Extract a dedicated one-tool Node package.** The session accepts a supplied `python_exec` definition, disables built-ins, and verifies the exact inventory. Node `>=22.19.0`, `@earendil-works/pi-ai` `0.80.10`, and `@earendil-works/pi-coding-agent` `0.80.10` remain pinned until upgraded deliberately. +Launch pinned Pi `0.80.10` with `--mode json`, built-in tools disabled, and one explicit extension. Python consumes Pi's official JSON event stream and inspects the final assistant stop reason rather than trusting the process exit code alone. The Pi-native session JSONL is the sole trajectory record. -8. **Keep optional imports off the base CLI path.** `dimos.cli.eval` is a dependency-light Typer shell or imports the heavy implementation only inside the `run` callback. Jupyter, FastAPI, and Uvicorn are added to the `agents` extra, not base dependencies. +API-key material is passed only in the Pi subprocess environment. It is absent from argv and removed from the environment supplied to the Jupyter kernel. -9. **Use source-checkout adapter discovery initially.** Users build the dedicated package with npm before evaluation. Installed-wheel adapter distribution is deferred to a separately scoped packaging change. +### Output -## Safety / Simulation / Replay +`--output` is the exact directory for one run. A non-empty target fails preflight. Work occurs in a sibling temporary directory and is atomically renamed on completion or caught failure. -This change never commands live hardware and does not start a robot, simulation, or replay blueprint. It reads a sealed recording and creates a derived map cache. The self-hosted mechanics gate uses `CPU:0`; normal first-time preparation may retain the reference mapper defaults and require CUDA. +Published files are: -CodePolicy executes trusted Python persistently and without an OS sandbox. Read-only Memory2 prevents mutation through provided APIs but cannot prevent arbitrary filesystem or process access by hostile code. Documentation and runtime receipts must not imply stronger isolation. Untrusted execution requires a future container or OS sandbox. +- `result.json`: case/source/model, response, prediction, score, timing, tool count, and optional infrastructure error. +- `transcript.jsonl`: Pi's native session when available. +- `stderr.log`: bounded Node diagnostics only when nonempty. -## Risks / Trade-offs +There are no nested attempt IDs, locks, manifests, copied cases/oracles/cache manifests, hashes, lifecycle logs, or duplicate call records. -- **Finalization can fail after useful work:** unconditional `finally` cleanup and prefix evidence reduce lock/process leakage; the terminal file may still be absent when storage itself fails. -- **Private data can leak through a new evidence path:** maintain distinct public/private models and add sentinel scans over every agent-visible channel. -- **Optional dependencies can leak into basic commands:** use callback-local imports and test in a subprocess that blocks agents-only modules. -- **Node/Python protocol drift:** share a version field, validate all frames, bound frames/stderr, correlate IDs, and run both sides' protocol tests. -- **Source-only adapter discovery limits installed users:** document the prerequisite and fail clearly; do not silently search ambiguous global locations. -- **Long or stuck agent turns complicate cleanup:** propagate abort, use bounded process waits, escalate terminate to kill, and record cleanup failures. -- **The smoke oracle can be mistaken for benchmark truth:** preserve warnings in the fixture, docs, and tests and describe semantic disagreement as expected plumbing behavior. - -## Migration / Rollout +## Decisions -Create a fresh feature branch from `origin/main` SHA `e8a985d83a85c9827fa89ed7526e40a822eb1ae3`. Land in dependency order: Memory2 read-only support, CodePolicy runtime, generic evaluation foundation, Node adapter, frozen drivers, CLI, fixture, and docs. Update `pyproject.toml` narrowly and regenerate `uv.lock`; do not carry unrelated reference changes. +1. **Lean feature-specific runner.** A second evaluator can justify a generic engine later. +2. **Real shared CodePolicy core.** This PR owns the production Jupyter session; experimental PR #3259 can later wrap it as a DimOS module. +3. **In-process MCP, Jupyter child.** Jupyter already supplies the execution process boundary, interrupt, restart, and shutdown. +4. **Official SDKs on both sides.** The new standalone boundary does not reuse or expand DimOS's legacy hand-written MCP transport. +5. **Stock Pi JSON mode.** A tiny extension replaces the custom adapter and broker protocol. +6. **Minimal durable output.** Memory2 owns observations; Pi owns trajectory; `result.json` owns scoring. +7. **Minimal Memory2 edits.** Read-only connection propagation is required because existing constructors configure WAL and create tables. Existing time-range filtering supplies the inclusive cutoff; general stream/filter APIs are not expanded. +8. **No cryptographic framework.** Strict parsing, safe paths, and cache metadata are sufficient for this local demo stage. +9. **API key only.** Default to `OPENAI_API_KEY`, with a named environment override for later extension. +10. **Existing CI plus required Node tests.** Python test/lint groups receive the minimal runtime dependencies; the small Node job gates the aggregate check. -Run Python and Node unit suites first, then the self-hosted Hong Kong mechanics gate, and finally the exact credentialed CLI smoke. No blueprint registry generation is required because no blueprint or module registry input changes. Rollback consists of removing the additive CLI registration and focused new packages; existing Memory2 read-only parameters remain backward-compatible defaults. +## Safety and cleanup -## Open Questions +The evaluator owns cleanup in one `finally`: stop Pi, stop the MCP server thread, and shut down Jupyter. Timeouts escalate Pi terminate to kill. The kernel environment is scrubbed of common credential variables. Private oracle values never enter prompts, MCP metadata, transcripts, results, or stderr. -None blocking this change. Distribution of the built Node adapter in Python wheels is explicitly deferred; this slice supports a source checkout with a documented adapter build step. +CodePolicy remains trusted unsandboxed execution because kernel code may access the host filesystem and start processes. Documentation must state this directly. diff --git a/openspec/changes/extract-frozen-qa-eval/docs.md b/openspec/changes/extract-frozen-qa-eval/docs.md index fec87b4899..0ba3a41811 100644 --- a/openspec/changes/extract-frozen-qa-eval/docs.md +++ b/openspec/changes/extract-frozen-qa-eval/docs.md @@ -1,42 +1,15 @@ -## User-Facing Docs +## User-facing docs -- Add `docs/capabilities/agents/evaluation.md` covering: - - the exact `dimos eval run CASE` command and typed options; - - installation of the `agents` extra and the dedicated Node adapter build step; - - OAuth and API-key environment selection without secret CLI values; - - stdout/stderr behavior, `--json`, `--quiet`, and exit codes; - - attempt artifact locations, privacy boundaries, and cleanup expectations; - - the trusted, persistent, unsandboxed nature of CodePolicy; - - the difference between a completed semantic failure and infrastructure failure. -- Link the new guide from the existing agent capability index under `docs/capabilities/agents/`. -- Preserve the fixture-local `dimos/benchmark/short_horizon_qa/cases/go2_hongkong_office-room-count-smoke/README.md`, prominently stating that oracle `0` is a synthetic plumbing sentinel rather than benchmark truth. +- Document the direct `dimos eval run CASE --output=DIR` command. +- Document the pinned Pi extension build and API-key environment selection. +- Explain the three exit codes and the compact atomic output directory. +- State prominently that CodePolicy runs trusted, unsandboxed Python. +- Preserve the demo fixture warning that oracle `0` is a plumbing sentinel. -## Contributor Docs +## Contributor docs -- Add a focused section to `docs/development/testing.md` for: - - building and testing `packages/pi-code-policy-adapter`; - - running the focused Python evaluation suite; - - running the `self_hosted` Hong Kong mechanics gate with its data prerequisite; - - performing the credentialed operational smoke and checking child-process and lock cleanup. -- No general module, blueprint, configuration, or hardware contributor documentation changes are required. +- Document focused Python tests and the one-file Node extension test. +- Document the self-hosted Hong Kong preparation gate. -## Coding-Agent Docs - -- Update `docs/coding-agents/index.md` only if it maintains a list of feature-specific validation surfaces; otherwise no coding-agent documentation change is needed. -- Do not modify `AGENTS.md`: the existing rules for optional dependencies, testing, imports, generated blueprints, and security are sufficient for this implementation. -- Retain `frozen-qa-main-extraction-handoff.md` as implementation context if the team wants the branch provenance in-repository; the OpenSpec artifacts become the normative implementation plan. - -## Doc Validation - -Run the repository-supported documentation checks after inspecting `docs/development/writing_docs.md` for the exact invocation: - -```bash -uv run doclinks -uv run md-babel-py run docs/capabilities/agents/evaluation.md -``` - -If the new page contains no executable Markdown blocks, record that `md-babel-py` has nothing to execute rather than adding artificial examples. No diagram generation is planned. - -## No Docs Needed - -Documentation is required because this change adds a public CLI, optional installation steps, credential handling, non-obvious exit semantics, and a trusted-unsandboxed execution boundary. +No blueprint, generated registry, hardware, or `AGENTS.md` documentation changes +are required. diff --git a/openspec/changes/extract-frozen-qa-eval/proposal.md b/openspec/changes/extract-frozen-qa-eval/proposal.md index d45f040839..c89d291189 100644 --- a/openspec/changes/extract-frozen-qa-eval/proposal.md +++ b/openspec/changes/extract-frozen-qa-eval/proposal.md @@ -1,42 +1,35 @@ ## Why -DimOS has a working frozen short-horizon QA evaluation path on the long-running `cc/frontier` branch, but the implementation is entangled with unrelated DimSim, spatial benchmark, manipulation, runtime, and UI work. That makes the final feature commits unsafe to cherry-pick and prevents the focused `dimos eval run` workflow from landing on `main` with a reviewable dependency boundary. +DimOS needs one direct command that asks an agent an integer question about a frozen Memory2 recording. The first extraction proved the path, but review showed that it introduced a generic evaluation framework, a custom Node protocol, duplicated evidence, and integrity machinery before those abstractions had a second use. -This change extracts the proven frozen-recording path as a single-case, synchronous evaluation capability. It preserves private-oracle isolation, immutable evidence, read-only Memory2 access, fresh agent sessions, and deterministic scoring while explicitly excluding live robot evaluation and the broader benchmark stack. +This revision keeps the production seams that matter: true read-only Memory2, a reusable Jupyter-backed CodePolicy session, official MCP libraries, a stock Pi process, strict case/result models, and private scoring. It removes the custom orchestration and audit framework. ## What Changes -- Add the public `dimos eval run CASE` CLI for one immutable frozen-memory evaluation case. -- Add strict frozen source, integer-question, one-attempt interaction, and exact-integer validator contracts with deterministic case fingerprints. -- Add read-only Memory2 snapshots that combine source and derived streams through an inclusive timestamp cutoff. -- Add a standalone loopback CodePolicy MCP process and a dedicated Node/Pi adapter exposing exactly one tool, `python_exec`. -- Add append-only attempt storage, private/public evidence separation, progress streaming, exact terminal-answer parsing, and explicit exit semantics. -- Add the Hong Kong office plumbing fixture with a clearly non-authoritative synthetic oracle. -- Add only the optional Python and Node dependencies required by this focused path. -- No existing public API is removed or changed; this is an additive CLI capability. - -## Affected DimOS Surfaces - -- Modules/streams: Memory2 SQLite stores, stream filters, frozen source/derived overlays, standalone CodePolicy runtime, and generic agent-evaluation orchestration. -- Blueprints/CLI: new `dimos eval run` command; no blueprint composition or generated blueprint registry changes. -- Skills/MCP: loopback MCP exposure of one trusted `python_exec` tool; no robot `@skill` additions. -- Hardware/simulation/replay: consumes an existing recording and derived map only; no hardware control, live DimSim evaluation, simulation scheduling, or replay blueprint changes. -- Docs/generated registries: new agent-evaluation capability documentation and fixture warning; no `all_blueprints.py` regeneration. +- Add `dimos eval run CASE --output=DIR` for one synchronous frozen QA run. +- Keep a compact tagged case model for source, task, and validator kinds. +- Add true read-only source/derived Memory2 views with an inclusive cutoff. +- Add a module-independent Jupyter `CodePolicySession` and expose its sole `python_exec` operation through the official Python MCP SDK in the evaluator process. +- Launch the stock Pi CLI in one-shot JSON mode with one small TypeScript extension using the official MCP client. +- Support API-key authentication through a named environment variable only. +- Publish only `result.json`, the native Pi transcript, and failure-only stderr in one non-overwriting output directory. +- Rename the Hong Kong fixture and case ID with `demo_`/`demo-` to make its synthetic `0` oracle unmistakable. +- Remove fingerprints, artifact manifests, lifecycle logs, broker logs, duplicated execution records, and the custom Python/Node protocol. ## Capabilities ### New Capabilities -- `frozen-agent-evaluation`: Single-case CLI execution, immutable case contracts, scoring, evidence, progress, privacy, and exit behavior. -- `frozen-memory-views`: Read-only source/derived Memory2 views bounded by an inclusive authored cutoff. -- `standalone-code-policy-runtime`: Fresh loopback CodePolicy and Pi processes with credential-safe setup, a one-tool inventory, bounded protocol handling, and reliable cleanup. +- `frozen-agent-evaluation`: one-case CLI execution, private integer scoring, compact output, and exit behavior. +- `frozen-memory-views`: read-only source/derived Memory2 access through an inclusive authored cutoff. +- `standalone-code-policy-runtime`: reusable Jupyter session plus an in-process official MCP adapter and stock Pi CLI integration. ### Modified Capabilities -None. The affected behavior is not currently represented by an OpenSpec capability spec on `main`. +None. ## Impact -Users gain a reproducible command for evaluating one frozen QA case and inspecting immutable attempt artifacts. The base CLI remains usable without the `agents` extra; evaluation requires the agents dependencies plus a built, pinned Node adapter. Credentials remain runtime-only and are never accepted as CLI secret values or serialized into evidence. +The change affects Memory2 SQLite opening, agent optional dependencies, the DimOS CLI, one focused benchmark package, a small Node extension package, CI dependencies, and agent documentation. It adds no blueprint, robot skill, live evaluation path, simulator, or generated blueprint entry. -The primary compatibility risks are optional-dependency import leakage, SQLite mutation through an allegedly frozen view, subprocess cleanup failures, private-oracle disclosure, and source-checkout discovery of the Node entrypoint. Validation therefore includes focused Python and Node suites, minimal-dependency CLI subprocess tests, failure-injection and lock-release tests, a self-hosted recording mechanics gate, and one credentialed end-to-end smoke run. +CodePolicy remains trusted unsandboxed Python. The Jupyter kernel receives a scrubbed environment without API credentials, but it is not an operating-system sandbox. diff --git a/openspec/changes/extract-frozen-qa-eval/specs/frozen-agent-evaluation/spec.md b/openspec/changes/extract-frozen-qa-eval/specs/frozen-agent-evaluation/spec.md index d719ff9779..b11cad93f8 100644 --- a/openspec/changes/extract-frozen-qa-eval/specs/frozen-agent-evaluation/spec.md +++ b/openspec/changes/extract-frozen-qa-eval/specs/frozen-agent-evaluation/spec.md @@ -1,107 +1,71 @@ ## ADDED Requirements -### Requirement: Single-case evaluation CLI -DimOS SHALL provide `dimos eval run CASE` as a synchronous command for exactly one immutable evaluation case. The command SHALL accept only the `pi` backend, `gpt-5.6-luna` model, and `medium` thinking level, and SHALL reject unsupported values or semantic overrides. +### Requirement: One-case frozen evaluation command +DimOS SHALL provide `dimos eval run CASE --output=DIR` for one synchronous frozen-memory integer-question case. The case SHALL use strict tagged source, task, and validator models and SHALL reject unknown fields, unsafe oracle paths, non-finite progress, and unsupported kinds. #### Scenario: Run a supported case -- **GIVEN** a valid frozen-memory case, available recording, built adapter, and valid credentials -- **WHEN** the user runs `dimos eval run CASE` -- **THEN** DimOS executes one attempt to a terminal infrastructure outcome -- **AND** prints the final result even when live progress is suppressed - -#### Scenario: Request an unsupported agent condition -- **GIVEN** a valid case -- **WHEN** the user supplies an unsupported backend, model, or thinking level -- **THEN** the command rejects the request before starting an attempt - -### Requirement: Credential-safe authentication selection -The CLI SHALL support Codex OAuth and an OpenAI API-key environment binding without accepting secret values directly. Explicit authentication mode SHALL win; otherwise a nonempty `OPENAI_API_KEY` SHALL select API-key authentication and Codex OAuth SHALL be the fallback. Credential contents MUST NOT appear in command arguments, progress, results, or retained artifacts. - -#### Scenario: Infer API-key authentication -- **GIVEN** no explicit authentication mode and a nonempty `OPENAI_API_KEY` -- **WHEN** the command resolves runtime authentication -- **THEN** it selects API-key authentication using the environment binding -- **AND** does not serialize the key value - -#### Scenario: Resolve OAuth authentication -- **GIVEN** no API key and no explicit authentication mode -- **WHEN** the command resolves runtime authentication -- **THEN** it selects Codex OAuth using `--agent.auth.path`, `PI_SPATIAL_AUTH_PATH`, or `~/.pi/agent/auth.json` in precedence order -- **AND** fails preflight if the selected credential file is unavailable - -### Requirement: Immutable and private case contract -An evaluation case SHALL consist of a frozen recording source, integer question, one-attempt frozen interaction, and exact-integer validator reference. Unknown fields, unsafe validator paths, non-finite progress, fingerprint mismatches, and runtime-specific fields in the semantic case SHALL be rejected. Runtime paths, credentials, ports, and output locations SHALL remain outside the case fingerprint. - -#### Scenario: Validate a case before agent dispatch -- **GIVEN** a case containing a safe case-relative oracle path and expected SHA-256 -- **WHEN** the command performs preflight -- **THEN** it verifies the case fingerprint and exact oracle bytes before starting the agent -- **AND** rejects an escaped path or digest mismatch - -#### Scenario: Produce an agent-safe projection -- **GIVEN** a validated private case -- **WHEN** DimOS creates the public case projection and prompt -- **THEN** the projection omits the validator reference and all oracle content -- **AND** the private expected answer is not exposed to the agent runtime - -### Requirement: Exact terminal integer scoring -The response parser SHALL succeed only when final text contains exactly one `ANSWER:` marker and ends with `ANSWER: `. A malformed answer or validator mismatch SHALL be a completed semantic failure, not an infrastructure failure. - -#### Scenario: Parse a valid terminal answer -- **GIVEN** final agent text containing exactly one terminal `ANSWER: -3` -- **WHEN** DimOS parses and validates the response -- **THEN** it records integer prediction `-3` -- **AND** compares it privately with the exact-integer oracle - -#### Scenario: Reject malformed answer text -- **GIVEN** final text with multiple markers, a non-integer marker, or trailing content after the answer -- **WHEN** DimOS parses the response -- **THEN** it records an invalid prediction and a completed failed task -- **AND** returns process exit code `0` - -### Requirement: Immutable attempt evidence -Each started attempt SHALL reserve a fresh mode-`0700` `attempt_` directory beneath the output root while holding a nonblocking output-root lock. Artifacts SHALL use safe attempt-relative paths, exclusive creation, SHA-256 descriptors, fsync, and atomic terminal publication. Concurrent attempts targeting the same output root SHALL not interleave. - -#### Scenario: Retain a completed attempt -- **GIVEN** an attempt reaches scoring and cleanup succeeds -- **WHEN** DimOS publishes the terminal outcome -- **THEN** the attempt contains private and public case projections, source evidence, MCP and session evidence, tool-call and execution records, Pi evidence, prediction, private score, ordered lifecycle events, manifest, and terminal outcome -- **AND** existing artifacts are never overwritten - -#### Scenario: Reject a concurrent attempt -- **GIVEN** one attempt holds the lock for an output root -- **WHEN** another attempt targets the same root -- **THEN** the second attempt fails cleanly without creating an interleaved attempt - -### Requirement: Failure classification and lock release -Failures before attempt reservation SHALL exit `2`. Infrastructure, finalization, or cleanup failures normalized into a reserved attempt SHALL produce a failed attempt and exit `1`. Completed semantic pass or failure SHALL exit `0`. The output lock MUST be released after success, failure, interruption, partial startup, and artifact-publication failure. - -#### Scenario: Preflight failure -- **GIVEN** an invalid case, unavailable oracle, missing credentials, missing adapter, or source-preparation error before reservation -- **WHEN** the error escapes preflight -- **THEN** the command exits `2` -- **AND** no attempt is reported as completed - -#### Scenario: Attempt cleanup failure -- **GIVEN** an otherwise completed attempt whose process or resource cleanup fails -- **WHEN** DimOS finalizes the attempt -- **THEN** it reports a failed infrastructure attempt and exits `1` -- **AND** releases the output lock even if terminal artifact publication also fails - -### Requirement: Machine-readable output and private progress -With `--json`, stdout SHALL contain exactly one compact result while progress and tool-call rendering remain on stderr. `--quiet` SHALL suppress progress but not the final result. Neither channel SHALL expose private oracle material or credentials. - -#### Scenario: Consume compact JSON -- **GIVEN** a valid invocation using `--json` -- **WHEN** the attempt runs with progress enabled -- **THEN** stdout remains parseable as exactly one JSON result -- **AND** progress appears only on stderr - -### Requirement: Synthetic plumbing fixture -The shipped Hong Kong office room-count smoke case SHALL preserve its `case.json`, private oracle, and warning README. The expected value `0` MUST be described as a synthetic plumbing sentinel and MUST NOT be presented as the authoritative room count. - -#### Scenario: Agent disagrees with the sentinel -- **GIVEN** the fixture and an agent response other than `ANSWER: 0` -- **WHEN** infrastructure and scoring complete -- **THEN** the task may report semantic failure with exit code `0` -- **AND** documentation does not characterize that outcome as a mapping or agent regression +- **GIVEN** a valid case, prepared recording, built Pi extension, and API-key environment variable +- **WHEN** the user runs `dimos eval run CASE --output=DIR` +- **THEN** DimOS runs one fresh agent turn and privately scores its terminal integer answer +- **AND** publishes a compact result even when the semantic score fails + +#### Scenario: Reject an unsafe case +- **GIVEN** a case with an escaped oracle path, invalid progress, unknown field, or unsupported kind +- **WHEN** preflight parses the case +- **THEN** the command exits `2` before starting CodePolicy or Pi + +### Requirement: API-key-only authentication +The evaluator SHALL read an OpenAI API key from `OPENAI_API_KEY` or a user-selected environment variable. The key MUST be passed only to the Pi subprocess environment and MUST NOT appear in argv, the Jupyter kernel environment, MCP data, results, transcripts, stderr, or cache metadata. + +#### Scenario: Run with the default key environment +- **GIVEN** a nonempty `OPENAI_API_KEY` +- **WHEN** the evaluator launches Pi +- **THEN** Pi receives the value through its environment +- **AND** CodePolicy cannot read that value from the Jupyter kernel environment + +### Requirement: Exact private integer scoring +The evaluator SHALL accept only final text with exactly one marker ending in `ANSWER: `. The private oracle SHALL be loaded only by the scorer and SHALL never enter the model-facing prompt or runtime. + +#### Scenario: Score a valid answer +- **GIVEN** final text ending in exactly one `ANSWER: 4` +- **WHEN** the private oracle expects `4` +- **THEN** the compact result records parsed integer `4` and task result `passed` + +#### Scenario: Score malformed or mismatched output +- **GIVEN** a missing, repeated, non-integer, trailing, or mismatched answer +- **WHEN** scoring completes +- **THEN** the run is a completed semantic failure +- **AND** the command exits `0` + +### Requirement: Compact non-overwriting output +The `--output` value SHALL be the exact directory for one run. A nonempty target SHALL fail preflight. The evaluator SHALL atomically publish `result.json`, the Pi-native transcript when available, and bounded nonempty Node stderr when present. It SHALL NOT copy source databases, cache manifests, case files, oracle files, prompts, MCP inventories, kernel records, or duplicate call logs. + +#### Scenario: Publish a completed run +- **GIVEN** an unused output path and a completed agent turn +- **WHEN** scoring finishes +- **THEN** `result.json` contains case/source/model, final response, prediction, score, duration, and tool count +- **AND** the native Pi transcript is the sole tool/assistant trajectory + +#### Scenario: Publish a caught infrastructure failure +- **GIVEN** Pi or CodePolicy fails after preflight +- **WHEN** cleanup completes +- **THEN** `result.json` records an infrastructure error and the command exits `1` +- **AND** any available native transcript or nonempty bounded stderr is retained + +### Requirement: Output channel contract +The final human or JSON result SHALL go to stdout. Coarse runtime status SHALL go to stderr and `--quiet` SHALL suppress it. Credentials and oracle contents MUST NOT appear on either channel. + +#### Scenario: Consume JSON output +- **GIVEN** `--json` with progress enabled +- **WHEN** a run finishes +- **THEN** stdout contains exactly one JSON value +- **AND** coarse status appears only on stderr + +### Requirement: Demo fixture identity +The shipped fixture directory SHALL start with `demo_`, its case ID SHALL start with `demo-`, and its README SHALL state that oracle value `0` tests plumbing rather than authoritative room-count accuracy. + +#### Scenario: Agent disagrees with the demo oracle +- **GIVEN** a completed answer other than `0` +- **WHEN** the demo case scores the answer +- **THEN** it reports semantic failure with exit `0` +- **AND** documentation does not characterize the result as an agent or mapping regression diff --git a/openspec/changes/extract-frozen-qa-eval/specs/frozen-memory-views/spec.md b/openspec/changes/extract-frozen-qa-eval/specs/frozen-memory-views/spec.md index f9ac1f47b6..646b1a1e02 100644 --- a/openspec/changes/extract-frozen-qa-eval/specs/frozen-memory-views/spec.md +++ b/openspec/changes/extract-frozen-qa-eval/specs/frozen-memory-views/spec.md @@ -1,53 +1,42 @@ ## ADDED Requirements ### Requirement: True read-only SQLite access -Frozen Memory2 source and derived databases SHALL be opened using SQLite read-only mode with query-only enforcement. All public mutation paths SHALL reject writes, and opening or reading a frozen view SHALL not create WAL or other database mutation sidecars. +Frozen Memory2 source and derived databases SHALL use SQLite URI read-only mode and `PRAGMA query_only=ON`. Read-only initialization SHALL skip WAL configuration and table creation. Public frozen-store mutation operations SHALL fail, and reading SHALL not create database sidecars. -#### Scenario: Read a frozen recording +#### Scenario: Read without mutation - **GIVEN** existing source and derived SQLite stores -- **WHEN** DimOS opens them as a frozen view and reads observations -- **THEN** the observations are available without modifying either database -- **AND** no WAL file is created by the frozen access - -#### Scenario: Attempt a mutation -- **GIVEN** an open frozen view or stream -- **WHEN** a caller appends an observation, deletes a stream, creates a stream, or invokes another mutation path -- **THEN** the operation fails with a read-only error -- **AND** source and derived bytes remain unchanged - -### Requirement: Inclusive authored cutoff -Every stream exposed through a frozen Memory2 view SHALL include observations whose timestamps are less than or equal to the authored cutoff and SHALL hide all observations after it. - -#### Scenario: Observe the exact cutoff boundary -- **GIVEN** observations immediately before, exactly at, and immediately after a cutoff -- **WHEN** the stream is queried through the frozen view -- **THEN** observations before and exactly at the cutoff are visible -- **AND** the observation after the cutoff is absent - -### Requirement: Source and derived overlay -A frozen view SHALL expose the union of source and derived stream names under the same cutoff. It SHALL reject ambiguous overlays in which both stores contain the same stream name and SHALL not allow callers to create or retype streams through the overlay. - -#### Scenario: Access a derived map with source observations -- **GIVEN** a source recording and a derived store containing a non-colliding `global_map` stream -- **WHEN** a caller lists and reads frozen streams -- **THEN** both source streams and `global_map` are available through one memory object -- **AND** the inclusive cutoff applies to all of them - -#### Scenario: Reject colliding streams -- **GIVEN** source and derived stores with the same stream name -- **WHEN** DimOS constructs the overlay -- **THEN** construction fails with the colliding names identified - -### Requirement: Deterministic frozen bundle preparation -DimOS SHALL resolve normalized progress over the sealed recording range, materialize or reuse a derived frozen bundle, and retain a manifest describing the selected cutoff and source/derived integrity. Progress `1.0` SHALL resolve to the recording end inclusively. - -#### Scenario: Prepare the final recording state -- **GIVEN** a named recording and normalized progress `1.0` -- **WHEN** DimOS prepares a frozen bundle -- **THEN** the selected cutoff equals the recording end -- **AND** the derived map and manifest describe only data available through that cutoff - -#### Scenario: Reject invalid progress -- **GIVEN** non-finite progress or a value outside `[0, 1]` -- **WHEN** bundle preparation validates the source -- **THEN** preparation fails before attempt execution +- **WHEN** a frozen view opens and reads them +- **THEN** the databases remain byte-identical +- **AND** no WAL or SHM sidecar is created + +#### Scenario: Reject mutation +- **GIVEN** an open frozen view +- **WHEN** a caller creates, retypes, deletes, or appends to a stream +- **THEN** the operation fails as read-only + +### Requirement: Inclusive cutoff overlay +A frozen view SHALL expose the deterministic union of non-colliding source and derived stream names. Every returned stream SHALL use existing time-range filtering to include observations with timestamps `<= cutoff` and hide later observations. + +#### Scenario: Read the cutoff boundary +- **GIVEN** observations before, exactly at, and after the cutoff +- **WHEN** the frozen stream is read +- **THEN** the first two observations are visible +- **AND** the later observation is hidden + +#### Scenario: Reject collision +- **GIVEN** source and derived stores containing the same stream name +- **WHEN** the overlay is created +- **THEN** construction fails and identifies the collision + +### Requirement: Metadata-based frozen bundle cache +Bundle preparation SHALL resolve progress over the recording range and cache a derived `global_map`. Cache reuse SHALL compare recording identity, source file size/mtime, mapper settings, progress, and cutoff metadata. It SHALL not hash full database files. + +#### Scenario: Reuse an unchanged bundle +- **GIVEN** matching source metadata, mapper settings, and normalized progress +- **WHEN** preparation runs again +- **THEN** it reuses the derived bundle without remapping or hashing the databases + +#### Scenario: Rebuild stale metadata +- **GIVEN** changed source metadata, mapper settings, or cutoff inputs +- **WHEN** preparation runs +- **THEN** it rebuilds the derived bundle before evaluation diff --git a/openspec/changes/extract-frozen-qa-eval/specs/standalone-code-policy-runtime/spec.md b/openspec/changes/extract-frozen-qa-eval/specs/standalone-code-policy-runtime/spec.md index 52be959c3a..0636483da1 100644 --- a/openspec/changes/extract-frozen-qa-eval/specs/standalone-code-policy-runtime/spec.md +++ b/openspec/changes/extract-frozen-qa-eval/specs/standalone-code-policy-runtime/spec.md @@ -1,73 +1,56 @@ ## ADDED Requirements -### Requirement: Fresh isolated runtime per attempt -Each attempt SHALL start one fresh standalone CodePolicy process on an ephemeral loopback port and one fresh Node/Pi process. The CodePolicy namespace SHALL preload frozen `memory`, SHALL NOT expose live DimOS `app`, and SHALL be disposed after the attempt. - -#### Scenario: Start a frozen policy session -- **GIVEN** a prepared frozen bundle -- **WHEN** an attempt starts its interaction -- **THEN** the policy can inspect the bounded `memory` view -- **AND** no live robot RPC object is present - -#### Scenario: Run consecutive attempts -- **GIVEN** two sequential evaluations -- **WHEN** each attempt starts -- **THEN** each receives distinct CodePolicy and Pi session identities and processes -- **AND** Python state does not leak between attempts - -### Requirement: Exactly one Pi tool -The Pi session SHALL disable built-in tools, extensions, skills, prompt templates, and context files, and SHALL expose exactly one custom tool named `python_exec`. The runtime SHALL fail closed if the activated inventory differs before or after session creation. - -#### Scenario: Validate tool inventory -- **GIVEN** a Pi session configured for frozen evaluation -- **WHEN** the session reports its active tools -- **THEN** the ordered inventory is exactly `["python_exec"]` -- **AND** any additional or missing tool causes infrastructure failure - -### Requirement: Pinned model and authentication runtime -The adapter SHALL run the pinned supported Pi libraries with model `gpt-5.6-luna` and medium thinking. It SHALL support Codex OAuth by credential-file path and OpenAI authentication by key supplied through the selected environment binding, without placing secret values in process arguments or evidence. - -#### Scenario: Launch with OAuth -- **GIVEN** a valid OAuth credential file -- **WHEN** the Pi process starts -- **THEN** it resolves the configured OAuth model runtime from the supplied path -- **AND** no credential bytes are emitted on the line protocol or retained in attempt evidence - -### Requirement: Bounded validated line protocol -Python and Node SHALL communicate through newline-delimited JSON with stdout reserved for protocol frames and diagnostics directed to bounded stderr evidence. Every inbound frame SHALL be size-bounded and schema-validated, and tool calls and replies SHALL be correlated by unique IDs. Unknown, duplicate, malformed, or oversized frames SHALL fail closed. - -#### Scenario: Broker a valid Python call -- **GIVEN** an idle Pi session and a valid `python_exec` request -- **WHEN** Node emits the correlated tool call and Python returns its result -- **THEN** the matching reply completes the pending call -- **AND** the call and bounded execution record are retained - -#### Scenario: Receive an invalid reply -- **GIVEN** no pending call with a supplied reply ID -- **WHEN** the adapter receives that reply -- **THEN** it reports a protocol error without applying the reply to another call - -### Requirement: Observable readiness and complete evidence -The standalone runtime SHALL retry both connection failures and read timeouts while waiting for MCP readiness. It SHALL retain the MCP inventory, CodePolicy session receipt and bounded execution records, broker call log, Pi prompt/session evidence, and bounded adapter stderr under the attempt directory. - -#### Scenario: Server becomes ready after transient failures -- **GIVEN** a starting loopback MCP process that initially refuses connections or times out reads -- **WHEN** readiness is polled within the configured deadline -- **THEN** polling continues until initialization succeeds or the deadline expires - -### Requirement: Reliable cancellation and cleanup -Abort, timeout, interrupt, normal completion, partial startup, and disposal SHALL terminate or kill remaining child processes within bounded cleanup periods. Cleanup failures SHALL be surfaced to the attempt result, and no child process SHALL remain after command termination. - -#### Scenario: Interrupt an active turn -- **GIVEN** an active Pi turn with an outstanding tool call -- **WHEN** the command is interrupted -- **THEN** pending broker calls are rejected, Pi is aborted and disposed, and CodePolicy is stopped -- **AND** the output lock is released - -### Requirement: Trusted unsandboxed execution disclosure -DimOS SHALL describe CodePolicy as trusted, persistent, unsandboxed Python. Read-only Memory2 SHALL be presented as protection against accidental API mutation, not as an operating-system security boundary. - -#### Scenario: Review the evaluation documentation -- **GIVEN** a user preparing to run a frozen evaluation -- **WHEN** they read the capability documentation -- **THEN** they are warned not to execute hostile policy code without an external container or OS sandbox +### Requirement: Shared Jupyter CodePolicy session +DimOS SHALL provide a module-independent `CodePolicySession` that owns one Jupyter kernel, persistent namespace, serialized execution, timeout interruption, restart recovery, and bounded shutdown. Frozen bootstrap SHALL expose read-only `memory`, omit live `app`, and scrub credentials from the kernel environment. + +#### Scenario: Start a frozen session +- **GIVEN** prepared source and derived Memory2 paths +- **WHEN** a frozen CodePolicy session starts +- **THEN** `memory` is available in the persistent namespace +- **AND** `app` and API-key environment variables are absent + +#### Scenario: Recover from timeout +- **GIVEN** an execution exceeds its timeout +- **WHEN** CodePolicy interrupts it +- **THEN** the session restarts and re-applies the frozen bootstrap +- **AND** later calls execute in a usable clean kernel + +### Requirement: In-process official MCP server +The evaluator SHALL expose CodePolicy through the official Python MCP SDK running in the evaluator process. It SHALL register exactly one tool named `python_exec`, bind a pre-created loopback port-`0` socket, and stop through direct evaluator ownership. It SHALL expose no HTTP control API. + +#### Scenario: Start without a port race +- **GIVEN** a new evaluator run +- **WHEN** it starts MCP +- **THEN** the evaluator binds the socket before serving and passes the actual URL to Pi +- **AND** the client observes exactly one tool named `python_exec` + +### Requirement: Stock Pi CLI and official MCP extension +The evaluator SHALL launch pinned Pi `0.80.10` in one-shot JSON mode with built-in tools, implicit extensions, skills, prompt templates, themes, and context files disabled. One explicit extension using `@modelcontextprotocol/client==2.0.0` SHALL register `python_exec` and call the official Python MCP server directly. + +#### Scenario: Complete one Pi turn +- **GIVEN** an initialized one-tool MCP server +- **WHEN** the stock Pi CLI receives the authored question +- **THEN** it emits official JSON events and persists its native session transcript +- **AND** Python derives final text, stop reason, and tool count without a custom protocol + +#### Scenario: Reject tool drift +- **GIVEN** the extension observes an MCP inventory other than exactly `python_exec` +- **WHEN** Pi initializes the extension +- **THEN** startup fails before the model turn + +### Requirement: Bounded cleanup +The evaluator SHALL stop Pi before stopping MCP and Jupyter. Pi timeout SHALL escalate terminate to kill. Normal completion, caught failure, interruption, and partial startup SHALL leave no live Pi process, MCP server thread, or Jupyter kernel. + +#### Scenario: Pi fails during startup +- **GIVEN** MCP and Jupyter have started but Pi fails +- **WHEN** the evaluator handles the error +- **THEN** it stops the server thread and kernel within bounded deadlines +- **AND** publishes a compact infrastructure failure result + +### Requirement: Trusted execution disclosure +Documentation SHALL describe CodePolicy as trusted persistent unsandboxed Python. Read-only Memory2 and a scrubbed environment SHALL not be presented as an operating-system sandbox. + +#### Scenario: Read the operator guide +- **GIVEN** a user preparing a frozen evaluation +- **WHEN** they read its safety section +- **THEN** they are warned to use an external OS sandbox or container for hostile code diff --git a/openspec/changes/extract-frozen-qa-eval/tasks.md b/openspec/changes/extract-frozen-qa-eval/tasks.md index a5262376e1..1db8f788e8 100644 --- a/openspec/changes/extract-frozen-qa-eval/tasks.md +++ b/openspec/changes/extract-frozen-qa-eval/tasks.md @@ -1,75 +1,46 @@ -## 1. Extraction Baseline and Dependencies +## 1. Rebaseline -- [x] 1.1 Ensure the implementation branch is based on `origin/main` SHA `e8a985d83a85c9827fa89ed7526e40a822eb1ae3`, record that base in the PR, and use `30e5f1c0e` only as the file-content reference. -- [x] 1.2 Add the focused `agent_eval` foundation modules for strict base models, canonical JSON, artifact/lifecycle records, and runtime credentials without importing live DimSim or spatial benchmark packages. -- [x] 1.3 Update `pyproject.toml` so the `agents` extra contains the Jupyter kernel/client, nbformat, pyzmq, FastAPI, and Uvicorn dependencies required by CodePolicy, then regenerate `uv.lock` without carrying unrelated reference changes. -- [x] 1.4 Add an import-boundary test that recursively checks the focused evaluation slice for forbidden live DimSim and spatial benchmark imports. +- [x] 1.1 Rewrite the OpenSpec proposal, design, capability specs, and tasks for the reviewer-approved lean architecture. +- [x] 1.2 Replace optional dependencies with official MCP SDK pins and the minimal existing-CI dependency set; regenerate Python and Node locks. -## 2. Frozen Memory2 Views +## 2. Frozen Memory2 -- [x] 2.1 Add a read-only option to Memory2 SQLite connection helpers using SQLite URI `mode=ro` and `PRAGMA query_only=ON`, while retaining WAL configuration only for writable connections. -- [x] 2.2 Propagate read-only mode through the Memory2 registry, observation store, SQLite store, and stream APIs, rejecting stream creation, append, deletion, and every other mutation path. -- [x] 2.3 Add inclusive through-time filtering (`observation.ts <= cutoff`) and ensure transformed read-only streams preserve the mutation boundary. -- [x] 2.4 Add the frozen source/derived overlay with deterministic stream listing, collision rejection, no stream creation/retyping, and the inclusive cutoff applied to every stream. -- [x] 2.5 Add Memory2 tests for exact-boundary visibility, source/derived union, collisions, mutation rejection, unchanged database bytes, and absence of WAL sidecars. +- [x] 2.1 Retain read-only propagation only through the SQLite helper, registry, observation store, and SQLite store. +- [x] 2.2 Remove the new general `ThroughFilter` and stream writability APIs; implement the inclusive cutoff inside the frozen facade with existing filters. +- [x] 2.3 Keep focused tests for no-WAL reads, source/derived overlay, exact cutoff, collisions, and rejected mutations. -## 3. Standalone CodePolicy Runtime +## 3. Production CodePolicy and MCP -- [x] 3.1 Port the module-independent trusted CodePolicy kernel runtime with lazy Jupyter imports and the actionable `uv sync --extra agents` error. -- [x] 3.2 Add frozen-memory environment setup that preloads read-only `memory`, omits live `app`, bounds execution output/records, and retains session receipts. -- [x] 3.3 Add the standalone FastAPI/Uvicorn MCP process on an ephemeral loopback port, including startup receipt, control endpoint, bounded shutdown, and terminate-to-kill escalation. -- [x] 3.4 Update MCP readiness polling to retry both connection failures and read timeouts until the configured deadline. -- [x] 3.5 Add CodePolicy and MCP tests for fresh sessions, namespace isolation, exactly one exposed tool, readiness retry, evidence retention, timeout/interruption behavior, partial startup, and child-process cleanup. +- [x] 3.1 Collapse the Jupyter implementation into a reusable module-independent `CodePolicySession` with frozen bootstrap, timeout recovery, shutdown, and scrubbed kernel credentials. +- [x] 3.2 Replace the hand-written standalone process/server/control API with an in-process official MCP server exposing exactly `python_exec` on a pre-bound loopback socket. +- [x] 3.3 Add hermetic tests for fresh namespaces, no `app`, no credentials, interrupt/restart, exact tool inventory, race-free startup, and bounded shutdown. -## 4. Evaluation Contracts, Storage, and Engine +## 4. Stock Pi CLI Extension -- [x] 4.1 Add frozen-only source, integer-question, one-attempt interaction, exact-integer validator, request, prediction, private score, and terminal outcome contracts with strict unknown-field rejection and deterministic fingerprints. -- [x] 4.2 Add tests proving fingerprints exclude credentials, ports, output/host paths, reject semantic overrides and unsafe oracle paths, and produce validator-free public projections. -- [x] 4.3 Add the append-only attempt store with mode-`0700` attempt directories, a nonblocking output-root lock, safe relative paths, exclusive artifact creation, SHA-256 descriptors, fsync, monotonic lifecycle events, and atomic terminal publication. -- [x] 4.4 Add the generic source/interaction/validator/agent adapter Protocols and attempt engine with private/public evidence separation and resource cleanup in reverse dependency order. -- [x] 4.5 Structure attempt execution so store closure and lock release occur in an outer `finally`, regardless of event, manifest, fsync, terminal-publication, cleanup, interruption, or partial-startup failures. -- [x] 4.6 Add fault-injection tests for events, artifact fsync, directory fsync, manifest, terminal-link publication, cleanup, and interruption failures, asserting retained prefixes, correct status, no live children, and immediate lock reacquisition. +- [x] 4.1 Replace `pi-code-policy-adapter` with the minimal `pi-code-policy-extension` package using pinned Pi `0.80.10` and official MCP client `2.0.0`. +- [x] 4.2 Register exactly `python_exec`, validate the MCP inventory, forward calls directly, and close the client on Pi shutdown. +- [x] 4.3 Launch stock Pi `--mode json`; parse official events for final text, stop reason, tool count, and native transcript without a custom protocol or Python broker. +- [x] 4.4 Add focused Node extension tests and hermetic Python event-parser/process-cleanup tests. -## 5. Dedicated Node/Pi Adapter +## 5. Lean Evaluation and CLI -- [x] 5.1 Create `packages/pi-code-policy-adapter` with Node `>=22.19.0`, pinned Pi dependencies `0.80.10`, compatible TypeBox/TypeScript dependencies, build/typecheck/test configuration, lockfile, and a source-checkout README. -- [x] 5.2 Extract and simplify the newline-delimited code-policy protocol with bounded frames, strict inbound validation, unique call/reply correlation, protocol-only stdout, and bounded diagnostic stderr. -- [x] 5.3 Implement the dedicated `python_exec` definition and Pi session setup without spatial tools, disabling built-ins/extensions/skills/templates/context files and asserting the exact one-tool inventory before and after activation. -- [x] 5.4 Implement OAuth and API-key runtime setup, pinned model/thinking validation, fresh session creation, prompt/session evidence, abort/dispose propagation, and no secret process arguments or evidence fields. -- [x] 5.5 Add Node tests for entrypoint behavior, valid calls, malformed/unknown/duplicate/oversized frames, tool-inventory drift, authentication configuration, prompt/session evidence, abort, disposal, and stdout/stderr separation. -- [x] 5.6 Add the adapter's npm test command to the appropriate required CI workflow without adding generated `dist` or `node_modules` content. +- [x] 5.1 Consolidate strict source/task/validator/result contracts in `agent_eval/models.py`; remove interaction/runtime/agent/fingerprint/artifact models and generic adapter Protocols. +- [x] 5.2 Simplify frozen bundle caching to recording metadata, mapper settings, and cutoffs without cryptographic descriptors. +- [x] 5.3 Replace the attempt engine/store with one runner that privately loads the oracle, runs CodePolicy and Pi, parses `ANSWER: `, and publishes compact output atomically. +- [x] 5.4 Persist only `result.json`, the native transcript when available, and nonempty bounded stderr; refuse a non-empty output directory. +- [x] 5.5 Support API-key environment authentication only and preserve exit codes `0` completed, `1` caught infrastructure failure, and `2` preflight failure. +- [x] 5.6 Rename the fixture directory and case ID with `demo_`/`demo-`, retaining the synthetic-`0` warning. +- [x] 5.7 Keep the base CLI dependency-light and add behavior-focused tests for case validation, privacy, scoring, output, cleanup, and CLI stdout/stderr. -## 6. Frozen QA Preparation and Execution +## 6. Documentation and CI -- [x] 6.1 Port normalized-progress validation, recording resolution, derived `global_map` preparation, cached bundle manifests, cutoff receipts, and source/derived integrity descriptors using current `main` mapping and Memory2 APIs. -- [x] 6.2 Add the frozen source driver, exact-integer oracle loader and SHA-256 verification, terminal `ANSWER: ` parser, private validator, and frozen CodePolicy interaction driver. -- [x] 6.3 Add the Python/Pi broker and process wrapper with bounded line frames, bounded progress/stderr, correlated tool replies, evidence references, startup/turn timeouts, abort, disposal, and terminate-to-kill cleanup. -- [x] 6.4 Add single-case preflight and execution orchestration with source-checkout discovery of `packages/pi-code-policy-adapter/dist/code-policy-main.js` and an actionable missing-build error. -- [x] 6.5 Add tests for bundle reuse and integrity, progress `0`/`1` boundaries, exact answer parsing, validator mismatch, malformed semantic failure, fresh process/session identities, cleanup, and complete attempt evidence. -- [x] 6.6 Add privacy tests that seed unique oracle and credential sentinels and prove they do not appear in the public projection, prompt, progress, compact result, CodePolicy namespace, Pi transcript/evidence, broker log, or serialized runtime configuration. +- [x] 6.1 Update the evaluation guide, fixture README, agent index, and testing guide for the stock Pi extension, API-key-only auth, compact output, and trusted-unsandboxed boundary. +- [x] 6.2 Make existing Python test/lint environments install the minimal evaluation dependencies and make the Node extension job gate aggregate CI. +- [x] 6.3 Remove documentation and tests for obsolete fingerprints, evidence stores, protocols, OAuth, and attempt locks. -## 7. CLI and Fixture +## 7. Verification and Review -- [x] 7.1 Add the dependency-light `dimos eval` Typer shell and callback-local heavy imports so ordinary base CLI commands do not require the `agents` extra. -- [x] 7.2 Implement the documented options, auth inference/precedence, default output root, compact/human results, stderr progress, `--quiet`, and exit codes `0`, `1`, and `2` at the preflight/attempt boundary. -- [x] 7.3 Add subprocess CLI tests for typed help, unsupported values, auth combinations, stdout/stderr separation, quiet mode, semantic failure, infrastructure failure, preflight failure, missing agents dependencies, and missing Node build. -- [x] 7.4 Port exactly the Hong Kong smoke fixture's `case.json`, private oracle, and warning README, preserving oracle SHA-256 and the explicit synthetic-`0` warning. -- [x] 7.5 Confirm no blueprint, module registry input, or `all_blueprints.py` output changes are introduced; no blueprint-regeneration task is required. - -## 8. Documentation - -- [x] 8.1 Add `docs/capabilities/agents/evaluation.md` with setup, adapter build, CLI options, authentication, output/exit behavior, evidence/privacy, cleanup, and trusted-unsandboxed warnings. -- [x] 8.2 Link the evaluation guide from the agent capability index and update `docs/development/testing.md` with Python, Node, self-hosted, and credentialed smoke procedures. -- [x] 8.3 Inspect `docs/coding-agents/index.md` and update it only if it enumerates feature-specific validation surfaces; leave `AGENTS.md` unchanged. -- [x] 8.4 Retain the fixture warning and decide in the PR whether `frozen-qa-main-extraction-handoff.md` remains as provenance or the OpenSpec artifacts supersede it. - -## 9. Verification and Manual QA - -- [x] 9.1 Run `openspec validate extract-frozen-qa-eval` and resolve all proposal/spec/design/docs/task validation errors. -- [x] 9.2 Run `npm ci --prefix packages/pi-code-policy-adapter`, its typecheck/build/tests, and verify no unexpected generated files are tracked. -- [x] 9.3 Run the focused Python CLI, agent-evaluation, frozen-QA, CodePolicy, MCP adapter, and frozen-Memory2 pytest targets listed in the handoff. -- [x] 9.4 Run the minimal-dependency base CLI subprocess regression, relevant lint/format checks, and `uv run mypy` for the added focused packages. -- [x] 9.5 Run `uv run doclinks` and the repository-supported executable-Markdown check for `docs/capabilities/agents/evaluation.md` when applicable. -- [x] 9.6 On a host with the LFS recording, run the `self_hosted` Hong Kong mechanics gate using `CPU:0` and verify its expected cutoff/map evidence. -- [ ] 9.7 Build the Node adapter and run the exact credentialed `uv run dimos eval run ... --output=/tmp/dimos-eval-smoke` command, accepting semantic failure against the synthetic oracle only when infrastructure and evidence complete. -- [x] 9.8 After the operational smoke, verify Pi and CodePolicy processes are gone, stdout/stderr obey the contract, credentials/oracle material are absent from public evidence, artifacts pass their descriptors, and the output lock can be immediately reacquired. +- [x] 7.1 Validate OpenSpec and run focused Python, Node, Ruff, mypy, doclinks, and executable-Markdown checks. +- [x] 7.2 Run the real Hong Kong self-hosted CPU mechanics gate. +- [ ] 7.3 Build the extension and run the exact API-key smoke command when credentials are available; verify compact output and child cleanup. (The extension and preflight path are verified; this workspace has no `OPENAI_API_KEY` for the live model call.) +- [x] 7.4 Commit and push the redesign, update draft PR #3378, and reply to all review threads with the agreed resolutions. diff --git a/packages/pi-code-policy-adapter/README.md b/packages/pi-code-policy-adapter/README.md deleted file mode 100644 index 4ac491748c..0000000000 --- a/packages/pi-code-policy-adapter/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Pi code-policy adapter - -This source-checkout package runs the pinned Pi session used by `dimos eval run`. -It disables Pi built-in tools and exposes exactly one host-brokered tool, -`python_exec`. - -```bash -npm ci --prefix packages/pi-code-policy-adapter -npm test --prefix packages/pi-code-policy-adapter -npm run build --prefix packages/pi-code-policy-adapter -``` - -The compiled entrypoint is `dist/code-policy-main.js`. Standard output is -reserved for newline-delimited protocol frames; diagnostics use standard error. -Credentials are supplied by the Python parent process through the supported -environment binding and must never be placed in command-line arguments. diff --git a/packages/pi-code-policy-adapter/src/code-policy-main.ts b/packages/pi-code-policy-adapter/src/code-policy-main.ts deleted file mode 100644 index a5cc49e89a..0000000000 --- a/packages/pi-code-policy-adapter/src/code-policy-main.ts +++ /dev/null @@ -1,241 +0,0 @@ -import { createInterface } from "node:readline"; -import { stdin, stdout, stderr } from "node:process"; -import { - encodeCodePolicyFrame, - parseCodePolicyFrame, - type CodePolicyOutbound, -} from "./code-policy-protocol.js"; -import { - CODE_POLICY_TOOL_NAME, - createFreshCodePolicySession, - type CodePolicyBroker, -} from "./code-policy-session.js"; -import type { - SessionAdapterHandle, - SessionEvidenceMetadata, - StoredAuthOptions, -} from "./session.js"; - -export type CodePolicySessionFactory = ( - broker: CodePolicyBroker, - options: StoredAuthOptions, - config: { thinkingLevel: "medium" }, - initialPrompt: string, -) => Promise; - -function authOptionsFromEnvironment(env: NodeJS.ProcessEnv): StoredAuthOptions { - const mode = env.PI_SPATIAL_AUTH_MODE ?? "codex-oauth"; - if (mode === "codex-oauth" && env.PI_SPATIAL_AUTH_PATH) { - return { - authMode: mode, - authPath: env.PI_SPATIAL_AUTH_PATH, - modelsPath: env.PI_SPATIAL_MODELS_PATH, - }; - } - if (mode === "openai-api-key" && env.OPENAI_API_KEY) { - return { - authMode: mode, - apiKey: env.OPENAI_API_KEY, - modelsPath: env.PI_SPATIAL_MODELS_PATH, - }; - } - throw new Error("Pi authentication environment is incomplete"); -} - -class HostBroker implements CodePolicyBroker { - private sequence = 0; - private readonly pending = new Map< - string, - { resolve: (value: string) => void; reject: (error: Error) => void } - >(); - - constructor(private readonly emit: (frame: CodePolicyOutbound) => void) {} - - request( - tool: typeof CODE_POLICY_TOOL_NAME, - params: { code: string; timeout_s?: number }, - ): Promise { - const id = `tool-${++this.sequence}`; - return new Promise((resolve, reject) => { - this.pending.set(id, { resolve, reject }); - this.emit({ version: 1, type: "tool_call", id, tool, params }); - }); - } - - reply(id: string, ok: boolean, result?: string, error?: string): void { - const pending = this.pending.get(id); - if (!pending) throw new Error("unknown or duplicate code-policy tool reply"); - this.pending.delete(id); - if (ok && result !== undefined) pending.resolve(result); - else pending.reject(new Error(error ?? "host code-policy tool failed")); - } - - count(): number { - return this.sequence; - } - - close(reason: string): void { - for (const pending of this.pending.values()) pending.reject(new Error(reason)); - this.pending.clear(); - } -} - -function record(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -export function progressFrame(event: unknown): CodePolicyOutbound | undefined { - if (!record(event)) return undefined; - if (event.type === "agent_start" || event.type === "turn_start" || event.type === "agent_end") { - return { version: 1, type: "transcript", event: event.type }; - } - if (event.type !== "message_update" || !record(event.assistantMessageEvent)) { - return undefined; - } - const update = event.assistantMessageEvent; - if (update.type !== "text_delta" || typeof update.delta !== "string" || update.delta.length === 0) { - return undefined; - } - return { - version: 1, - type: "transcript", - event: "assistant_text_delta", - delta: update.delta, - }; -} - -function evidenceFrame(evidence: SessionEvidenceMetadata) { - return { - state: evidence.state, - persisted: evidence.persisted, - ...(evidence.relativePath ? { relative_path: evidence.relativePath } : {}), - ...(evidence.systemPrompt - ? { - system_prompt: { - relative_path: evidence.systemPrompt.relativePath, - byte_count: evidence.systemPrompt.byteCount, - sha256: evidence.systemPrompt.sha256, - }, - } - : {}), - ...(evidence.initialPrompt - ? { - initial_prompt: { - relative_path: evidence.initialPrompt.relativePath, - byte_count: evidence.initialPrompt.byteCount, - sha256: evidence.initialPrompt.sha256, - }, - } - : {}), - }; -} - -export async function runCodePolicyAdapter( - input: NodeJS.ReadableStream = stdin, - output: NodeJS.WritableStream = stdout, - diagnostics: NodeJS.WritableStream = stderr, - sessionFactory: CodePolicySessionFactory = createFreshCodePolicySession, -): Promise { - const lines = createInterface({ input, crlfDelay: Infinity }); - const emit = (frame: CodePolicyOutbound): void => { - output.write(encodeCodePolicyFrame(frame)); - }; - let session: SessionAdapterHandle | undefined; - let broker: HostBroker | undefined; - let sessionId = ""; - let activeTurn: Promise | undefined; - let activeVisibleText = ""; - let closed = false; - try { - for await (const line of lines) { - const frame = parseCodePolicyFrame(line); - if (frame.type === "session_start") { - if (session || broker) throw new Error("duplicate session_start"); - sessionId = frame.id; - broker = new HostBroker(emit); - session = await sessionFactory( - broker, - authOptionsFromEnvironment(process.env), - { thinkingLevel: frame.thinking_level }, - frame.initial_prompt, - ); - session.subscribe((event) => { - if (record(event) && event.type === "turn_start") { - activeVisibleText = ""; - } else if ( - record(event) && - event.type === "message_update" && - record(event.assistantMessageEvent) && - event.assistantMessageEvent.type === "text_delta" && - typeof event.assistantMessageEvent.delta === "string" - ) { - activeVisibleText = (activeVisibleText + event.assistantMessageEvent.delta).slice( - 0, - 16_384, - ); - } - const frame = progressFrame(event); - if (frame) emit(frame); - }); - emit({ version: 1, type: "session_started", id: sessionId, tools: ["python_exec"] }); - } else if (frame.type === "prompt") { - if (!session || !broker || activeTurn) throw new Error("prompt outside idle session"); - const before = broker.count(); - activeVisibleText = ""; - activeTurn = session - .prompt(frame.text) - .then((result: unknown) => { - const returnedText = - typeof result === "string" - ? result - : typeof result === "object" && result !== null - ? JSON.stringify(result).slice(0, 16_384) - : ""; - const finalText = returnedText || activeVisibleText; - emit({ - version: 1, - type: "turn_complete", - id: frame.id, - policy_call_count: broker?.count() ?? before, - final_text: finalText, - }); - }) - .finally(() => { - activeTurn = undefined; - }); - } else if (frame.type === "tool_reply") { - if (!broker) throw new Error("tool reply before session"); - broker.reply(frame.id, frame.ok, frame.result, frame.error); - } else if (frame.type === "abort") { - await session?.abort(); - } else { - if (!session || !broker) throw new Error("dispose before session"); - await session.abort().catch(() => undefined); - await activeTurn?.catch(() => undefined); - session.dispose(); - broker.close("session disposed"); - emit({ - version: 1, - type: "session_closed", - id: sessionId, - evidence: evidenceFrame(session.sessionEvidence(true)), - }); - closed = true; - lines.close(); - } - } - } catch (error) { - const message = error instanceof Error ? error.message : "code-policy adapter failure"; - diagnostics.write(`${message.replace(/[\r\n]+/g, " ").slice(0, 1024)}\n`); - emit({ version: 1, type: "protocol_error", error: message.slice(0, 1024) }); - } finally { - if (!closed) { - broker?.close("adapter input closed"); - session?.dispose(); - } - } -} - -if (process.argv[1]?.endsWith("code-policy-main.js")) { - void runCodePolicyAdapter(); -} diff --git a/packages/pi-code-policy-adapter/src/code-policy-protocol.ts b/packages/pi-code-policy-adapter/src/code-policy-protocol.ts deleted file mode 100644 index 874793bcbd..0000000000 --- a/packages/pi-code-policy-adapter/src/code-policy-protocol.ts +++ /dev/null @@ -1,122 +0,0 @@ -export const CODE_POLICY_PROTOCOL_VERSION = 1; -export const CODE_POLICY_MAX_LINE_BYTES = 64 * 1024; - -export type CodePolicyInbound = - | { - version: 1; - type: "session_start"; - id: string; - initial_prompt: string; - thinking_level: "medium"; - } - | { version: 1; type: "prompt"; id: string; text: string } - | { - version: 1; - type: "tool_reply"; - id: string; - ok: boolean; - result?: string; - error?: string; - } - | { version: 1; type: "abort" } - | { version: 1; type: "dispose" }; - -export type CodePolicyOutbound = - | { version: 1; type: "session_started"; id: string; tools: ["python_exec"] } - | { - version: 1; - type: "tool_call"; - id: string; - tool: "python_exec"; - params: { code: string; timeout_s?: number }; - } - | { - version: 1; - type: "transcript"; - event: "agent_start" | "turn_start" | "agent_end"; - } - | { - version: 1; - type: "transcript"; - event: "assistant_text_delta"; - delta: string; - } - | { - version: 1; - type: "turn_complete"; - id: string; - policy_call_count: number; - final_text: string; - } - | { - version: 1; - type: "session_closed"; - id: string; - evidence: { - state: "complete" | "partial" | "unavailable"; - persisted: boolean; - relative_path?: string; - system_prompt?: { relative_path: string; byte_count: number; sha256: string }; - initial_prompt?: { relative_path: string; byte_count: number; sha256: string }; - }; - } - | { version: 1; type: "protocol_error"; error: string }; - -function record(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -export function parseCodePolicyFrame(line: string): CodePolicyInbound { - if (Buffer.byteLength(line, "utf8") > CODE_POLICY_MAX_LINE_BYTES) { - throw new Error("code-policy frame exceeds limit"); - } - let value: unknown; - try { - value = JSON.parse(line); - } catch { - throw new Error("invalid code-policy JSON frame"); - } - if (!record(value) || value.version !== CODE_POLICY_PROTOCOL_VERSION) { - throw new Error("invalid code-policy protocol version"); - } - if ( - value.type === "session_start" && - typeof value.id === "string" && - value.id.length > 0 && - typeof value.initial_prompt === "string" && - value.initial_prompt.length > 0 && - value.thinking_level === "medium" - ) { - return value as CodePolicyInbound; - } - if ( - value.type === "prompt" && - typeof value.id === "string" && - value.id.length > 0 && - typeof value.text === "string" && - value.text.length > 0 - ) { - return value as CodePolicyInbound; - } - if ( - value.type === "tool_reply" && - typeof value.id === "string" && - typeof value.ok === "boolean" && - (value.result === undefined || typeof value.result === "string") && - (value.error === undefined || typeof value.error === "string") - ) { - return value as CodePolicyInbound; - } - if (value.type === "abort" || value.type === "dispose") { - return value as CodePolicyInbound; - } - throw new Error("invalid code-policy frame"); -} - -export function encodeCodePolicyFrame(frame: CodePolicyOutbound): string { - const encoded = JSON.stringify(frame); - if (Buffer.byteLength(encoded, "utf8") > CODE_POLICY_MAX_LINE_BYTES) { - throw new Error("outbound code-policy frame exceeds limit"); - } - return `${encoded}\n`; -} diff --git a/packages/pi-code-policy-adapter/src/code-policy-session.ts b/packages/pi-code-policy-adapter/src/code-policy-session.ts deleted file mode 100644 index 30ab0df70f..0000000000 --- a/packages/pi-code-policy-adapter/src/code-policy-session.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { Type } from "typebox"; -import type { ToolDefinition } from "@earendil-works/pi-coding-agent"; -import { - createFreshSessionWithTools, - type SessionAdapterHandle, - type SessionConfig, - type StoredAuthOptions, -} from "./session.js"; - -export const CODE_POLICY_TOOL_NAME = "python_exec" as const; -export const CODE_POLICY_TOOL_NAMES = [CODE_POLICY_TOOL_NAME] as const; - -export function assertCodePolicyToolInventory(names: readonly string[]): void { - if (names.length !== 1 || names[0] !== CODE_POLICY_TOOL_NAME) { - throw new Error("Pi code-policy session did not activate exactly python_exec"); - } -} - -export interface CodePolicyBroker { - request( - tool: typeof CODE_POLICY_TOOL_NAME, - params: { code: string; timeout_s?: number }, - ): Promise; -} - -export function codePolicyToolDefinition(broker: CodePolicyBroker): ToolDefinition { - return { - name: CODE_POLICY_TOOL_NAME, - label: "Execute Python", - description: - "Execute one synchronous Python program in the persistent trusted, unsandboxed DimOS policy session. The session preloads app for deployed DimOS RPCs and memory for observations.", - parameters: Type.Object( - { - code: Type.String({ minLength: 1 }), - timeout_s: Type.Optional(Type.Number({ exclusiveMinimum: 0, maximum: 110 })), - }, - { additionalProperties: false }, - ), - execute: async (_id, params) => ({ - content: [ - { - type: "text", - text: await broker.request( - CODE_POLICY_TOOL_NAME, - params as { code: string; timeout_s?: number }, - ), - }, - ], - details: {}, - }), - }; -} - -export async function createFreshCodePolicySession( - broker: CodePolicyBroker, - options: StoredAuthOptions, - config: SessionConfig, - initialPrompt: string, -): Promise { - const result = await createFreshSessionWithTools( - [codePolicyToolDefinition(broker)], - options, - config, - initialPrompt, - CODE_POLICY_TOOL_NAMES, - ); - try { - assertCodePolicyToolInventory(result.activeToolNames); - } catch (error) { - result.handle.dispose(); - throw error; - } - return result.handle; -} diff --git a/packages/pi-code-policy-adapter/src/session.ts b/packages/pi-code-policy-adapter/src/session.ts deleted file mode 100644 index 2fb55dece0..0000000000 --- a/packages/pi-code-policy-adapter/src/session.ts +++ /dev/null @@ -1,352 +0,0 @@ -import { ModelRegistry, ModelRuntime, SessionManager, createAgentSession, readStoredCredential } from "@earendil-works/pi-coding-agent"; -import { InMemoryCredentialStore, type Model } from "@earendil-works/pi-ai"; -import type { ToolDefinition } from "@earendil-works/pi-coding-agent"; -import { chmodSync, closeSync, constants, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, unlinkSync, writeSync } from "node:fs"; -import { createHash } from "node:crypto"; -import { basename, dirname, relative, resolve, sep } from "node:path"; -import { fileURLToPath } from "node:url"; - -export const MODEL_PROVIDER = "openai-codex"; -export const API_KEY_MODEL_PROVIDER = "openai"; -export const MODEL_ID = "gpt-5.6-luna"; -export const THINKING_LEVEL = "medium" as const; -export const REQUIRED_API = "openai-codex-responses"; -export const API_KEY_REQUIRED_API = "openai-responses"; -export const SESSION_DIR_ENV = "PI_SPATIAL_SESSION_DIR"; -export const AGENT_CWD = process.env.PI_SPATIAL_AGENT_CWD ?? "/work"; -export const PINNED_PI_VERSION = "0.80.10"; - -// The adapter is a dedicated process. Set this before Pi can create a session file. -process.umask(0o077); - -export type SessionEvidenceState = "complete" | "partial" | "unavailable"; - -export interface SystemPromptEvidenceMetadata { - readonly relativePath: "pi-prompt/system.txt"; - readonly byteCount: number; - readonly sha256: string; -} -export interface InitialPromptEvidenceMetadata { - readonly relativePath: "pi-prompt/initial.txt"; - readonly byteCount: number; - readonly sha256: string; -} - -export interface SessionEvidenceMetadata { - /** Safe path relative to the attempt cwd; never an absolute host path. */ - readonly relativePath?: string; - readonly persisted: boolean; - readonly state: SessionEvidenceState; - readonly systemPrompt?: SystemPromptEvidenceMetadata; - readonly initialPrompt?: InitialPromptEvidenceMetadata; -} - -export interface SessionAdapterHandle { - prompt(prompt: string): Promise; - subscribe(listener: (event: unknown) => void): void; - abort: () => Promise; - dispose: () => void; - sessionEvidence: (completed: boolean) => SessionEvidenceMetadata; -} - -interface SessionDirectory { - readonly name: string; - readonly path: string; -} - -function configuredSessionDirectory(): SessionDirectory { - const value = process.env[SESSION_DIR_ENV]; - if (value === undefined || !/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value)) { - throw new Error(`${SESSION_DIR_ENV} must be provided as one simple relative directory name`); - } - const path = resolve(process.cwd(), value); - if (dirname(path) !== process.cwd()) throw new Error(`${SESSION_DIR_ENV} must stay beneath process.cwd()`); - return { name: value, path }; -} - -function ensurePrivateSessionDirectory(directory: SessionDirectory): void { - try { - const stat = lstatSync(directory.path); - if (stat.isSymbolicLink() || !stat.isDirectory()) throw new Error(`${SESSION_DIR_ENV} must be a real directory`); - chmodSync(directory.path, 0o700); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - try { - mkdirSync(directory.path, { mode: 0o700 }); - } catch (mkdirError) { - if ((mkdirError as NodeJS.ErrnoException).code !== "EEXIST") throw mkdirError; - } - const stat = lstatSync(directory.path); - if (stat.isSymbolicLink() || !stat.isDirectory()) throw new Error(`${SESSION_DIR_ENV} must be a real directory`); - chmodSync(directory.path, 0o700); - } -} - -function ensurePrivatePromptDirectory(): string { - const path = resolve(process.cwd(), "pi-prompt"); - try { - const stat = lstatSync(path); - if (stat.isSymbolicLink() || !stat.isDirectory()) throw new Error("pi-prompt must be a real directory"); - chmodSync(path, 0o700); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - try { - mkdirSync(path, { mode: 0o700 }); - } catch (mkdirError) { - if ((mkdirError as NodeJS.ErrnoException).code !== "EEXIST") throw mkdirError; - } - const stat = lstatSync(path); - if (stat.isSymbolicLink() || !stat.isDirectory()) throw new Error("pi-prompt must be a real directory"); - chmodSync(path, 0o700); - } - return path; -} - -export function retainSystemPromptEvidence(systemPrompt: string): SystemPromptEvidenceMetadata { - const promptDirectory = ensurePrivatePromptDirectory(); - const path = resolve(promptDirectory, "system.txt"); - const bytes = Buffer.from(systemPrompt, "utf8"); - const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); - try { - const written = writeSync(fd, bytes); - if (written !== bytes.length) throw new Error("system prompt sidecar write was incomplete"); - fsyncSync(fd); - } catch (error) { - closeSync(fd); - try { unlinkSync(path); } catch { /* preserve the original setup failure */ } - throw error; - } - closeSync(fd); - try { - const directoryFd = openSync(promptDirectory, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); - try { fsyncSync(directoryFd); } finally { closeSync(directoryFd); } - } catch { - // Directory fsync is best effort; the file itself was fsynced above. - } - return { - relativePath: "pi-prompt/system.txt", - byteCount: bytes.length, - sha256: createHash("sha256").update(bytes).digest("hex"), - }; -} - -export function retainInitialPromptEvidence(initialPrompt: string): InitialPromptEvidenceMetadata { - const promptDirectory = ensurePrivatePromptDirectory(); - const path = resolve(promptDirectory, "initial.txt"); - const bytes = Buffer.from(initialPrompt, "utf8"); - const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); - try { - if (writeSync(fd, bytes) !== bytes.length) throw new Error("initial prompt sidecar write was incomplete"); - fsyncSync(fd); - } catch (error) { - closeSync(fd); - try { unlinkSync(path); } catch { /* preserve original setup failure */ } - throw error; - } - closeSync(fd); - return { relativePath: "pi-prompt/initial.txt", byteCount: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") }; -} - -export function createSessionManager(cwd: string = AGENT_CWD): SessionManager { - const directory = configuredSessionDirectory(); - ensurePrivateSessionDirectory(directory); - return SessionManager.create(cwd, directory.path); -} - -function safeRelativeSessionFile(manager: SessionManager): string | undefined { - const file = manager.getSessionFile(); - if (!file) return undefined; - const sessionDirectory = resolve(manager.getSessionDir()); - const relativeToSessionDirectory = relative(sessionDirectory, resolve(file)); - const relativeSessionDirectory = relative(process.cwd(), sessionDirectory); - if (!relativeToSessionDirectory || relativeToSessionDirectory.includes(sep) || relativeToSessionDirectory.startsWith("..") || !/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(relativeSessionDirectory) || !basename(relativeToSessionDirectory).endsWith(".jsonl")) { - return undefined; - } - let stat: ReturnType; - try { - stat = lstatSync(file); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; - throw error; - } - if (!stat.isFile()) return undefined; - return `${relativeSessionDirectory}/${basename(relativeToSessionDirectory)}`; -} - -export function sessionEvidenceForManager(manager: SessionManager, completed: boolean): SessionEvidenceMetadata { - const relativePath = safeRelativeSessionFile(manager); - const persisted = manager.isPersisted(); - return { - ...(relativePath ? { relativePath } : {}), - persisted: persisted && relativePath !== undefined, - state: !persisted || relativePath === undefined ? "unavailable" : completed ? "complete" : "partial", - }; -} - -export function resolvePinnedPiCli(): string { - const packageName = "@earendil-works/pi-coding-agent"; - let directory = dirname(fileURLToPath(import.meta.url)); - while (true) { - const packageDirectory = resolve(directory, "node_modules", packageName); - const packageJson = resolve(packageDirectory, "package.json"); - if (existsSync(packageJson)) { - const metadata = JSON.parse(readFileSync(packageJson, "utf8")) as { version?: unknown; bin?: unknown }; - if (metadata.version !== PINNED_PI_VERSION) throw new Error(`expected pinned Pi ${PINNED_PI_VERSION}`); - const bin = metadata.bin; - const binPath = typeof bin === "object" && bin !== null && "pi" in bin && typeof bin.pi === "string" ? bin.pi : undefined; - if (binPath !== "dist/cli.js") throw new Error("pinned Pi package has an unexpected pi bin"); - const cli = resolve(packageDirectory, binPath); - if (!existsSync(cli)) throw new Error("pinned Pi CLI entrypoint is missing"); - return cli; - } - const parent = dirname(directory); - if (parent === directory) break; - directory = parent; - } - throw new Error("pinned Pi package cannot be resolved"); -} - -export interface PinnedPiExportCommand { - readonly executable: string; - readonly args: readonly [string, "--export", string, string]; - readonly packageVersion: typeof PINNED_PI_VERSION; -} - -export function resolvePinnedPiExportCommand(input: string, output: string): PinnedPiExportCommand { - return { - executable: process.execPath, - args: [resolvePinnedPiCli(), "--export", input, output], - packageVersion: PINNED_PI_VERSION, - }; -} - -export function modelProviderForAuthMode(authMode: AuthMode): typeof MODEL_PROVIDER | typeof API_KEY_MODEL_PROVIDER { - return authMode === "codex-oauth" ? MODEL_PROVIDER : API_KEY_MODEL_PROVIDER; -} - -export function requiredApiForAuthMode(authMode: AuthMode): typeof REQUIRED_API | typeof API_KEY_REQUIRED_API { - return authMode === "codex-oauth" ? REQUIRED_API : API_KEY_REQUIRED_API; -} - -export function resolveConfiguredModel( - registry: ModelRegistry, - authMode: AuthMode = "codex-oauth", -): Model<"openai-codex-responses" | "openai-responses"> { - const provider = modelProviderForAuthMode(authMode); - const requiredApi = requiredApiForAuthMode(authMode); - const model = registry.find(provider, MODEL_ID); - if (!model || model.api !== requiredApi || !model.input.includes("image") || !model.reasoning) { - throw new Error("configured model is missing, has the wrong API, does not accept images, or does not support thinking"); - } - return model as Model<"openai-codex-responses" | "openai-responses">; -} - -export type AuthMode = "codex-oauth" | "openai-api-key"; - -export interface StoredAuthOptions { - authMode: AuthMode; - authPath?: string; - apiKey?: string; - modelsPath?: string; -} - -export interface SessionConfig { - thinkingLevel: typeof THINKING_LEVEL; -} - -export function validateSessionConfig(config: SessionConfig): void { - if (config.thinkingLevel !== THINKING_LEVEL) throw new Error("unsupported thinking level"); -} - -export async function createFreshSessionWithTools( - tools: readonly ToolDefinition[], - options: StoredAuthOptions, - config: SessionConfig, - initialPrompt: string, - expectedToolNames: readonly string[], -): Promise<{ handle: SessionAdapterHandle; activeToolNames: readonly string[] }> { - validateSessionConfig(config); - if ( - expectedToolNames.length === 0 || - new Set(expectedToolNames).size !== expectedToolNames.length || - tools.length !== expectedToolNames.length || - tools.some((tool, index) => tool.name !== expectedToolNames[index]) - ) { - throw new Error("custom tools do not match the expected ordered inventory"); - } - const manager = createSessionManager(); - const initialPromptEvidence = retainInitialPromptEvidence(initialPrompt); - let runtime: ModelRuntime; - if (options.authMode === "codex-oauth") { - if (!options.authPath) throw new Error("Codex OAuth auth path is required"); - const credential = readStoredCredential(MODEL_PROVIDER, options.authPath); - if (!credential || credential.type !== "oauth") throw new Error("Codex OAuth credentials are not stored"); - runtime = await ModelRuntime.create({ authPath: options.authPath, modelsPath: options.modelsPath }); - } else { - if (!options.apiKey) throw new Error("OpenAI API key is required"); - const credentials = new InMemoryCredentialStore(); - await credentials.modify(API_KEY_MODEL_PROVIDER, async () => ({ - type: "api_key", - key: options.apiKey, - })); - runtime = await ModelRuntime.create({ - credentials, - modelsPath: options.modelsPath, - }); - } - const registry = new ModelRegistry(runtime); - await registry.refresh(); - const model = resolveConfiguredModel(registry, options.authMode); - if (options.authMode === "codex-oauth") { - if (!registry.isUsingOAuth(model)) throw new Error("configured model is not using Codex OAuth"); - } else { - const status = registry.getProviderAuthStatus(API_KEY_MODEL_PROVIDER); - if (registry.isUsingOAuth(model) || !status.configured || status.source !== "stored") { - throw new Error("configured model is not using an OpenAI API key"); - } - } - const custom = [...tools]; - const available = custom.map((tool) => tool.name); - const result = await createAgentSession({ - cwd: AGENT_CWD, - model, - thinkingLevel: config.thinkingLevel, - modelRuntime: runtime, - sessionManager: manager, - noTools: "builtin", - tools: available, - customTools: custom, - }); - const active = result.session.getActiveToolNames(); - if ( - active.length !== expectedToolNames.length || - active.some((name, index) => name !== expectedToolNames[index]) - ) { - result.session.dispose(); - throw new Error("Pi activated an unexpected tool inventory"); - } - let systemPrompt: SystemPromptEvidenceMetadata; - try { - systemPrompt = retainSystemPromptEvidence(result.session.systemPrompt); - } catch (error) { - result.session.dispose(); - throw error; - } - let disposed = false; - const handle = { - prompt: (prompt: string) => result.session.prompt(prompt), - subscribe: (listener: (event: unknown) => void) => { result.session.subscribe((event) => listener(event)); }, - abort: () => result.session.abort(), - dispose: () => { - if (!disposed) { - disposed = true; - result.session.dispose(); - } - }, - sessionEvidence: (completed: boolean): SessionEvidenceMetadata => { - const evidence = sessionEvidenceForManager(manager, completed); - return evidence.state === "unavailable" ? evidence : { ...evidence, systemPrompt, initialPrompt: initialPromptEvidence }; - }, - } satisfies SessionAdapterHandle; - return { handle, activeToolNames: active }; -} diff --git a/packages/pi-code-policy-adapter/test/code-policy-main.test.ts b/packages/pi-code-policy-adapter/test/code-policy-main.test.ts deleted file mode 100644 index 27b36b47e5..0000000000 --- a/packages/pi-code-policy-adapter/test/code-policy-main.test.ts +++ /dev/null @@ -1,123 +0,0 @@ -import assert from "node:assert/strict"; -import { Readable, Writable } from "node:stream"; -import test from "node:test"; -import { - progressFrame, - runCodePolicyAdapter, - type CodePolicySessionFactory, -} from "../src/code-policy-main.js"; - -function sink(): { stream: Writable; frames: () => Array> } { - const chunks: string[] = []; - return { - stream: new Writable({ - write(chunk, _encoding, callback) { - chunks.push(String(chunk)); - callback(); - }, - }), - frames: () => - chunks - .join("") - .trim() - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line) as Record), - }; -} - -test("normalizes visible progress and discards thinking and raw events", () => { - assert.deepEqual(progressFrame({ type: "agent_start" }), { - version: 1, - type: "transcript", - event: "agent_start", - }); - assert.deepEqual( - progressFrame({ - type: "message_update", - assistantMessageEvent: { type: "text_delta", delta: "Looking around" }, - }), - { - version: 1, - type: "transcript", - event: "assistant_text_delta", - delta: "Looking around", - }, - ); - assert.equal( - progressFrame({ - type: "message_update", - assistantMessageEvent: { type: "thinking_delta", delta: "private reasoning" }, - }), - undefined, - ); - assert.equal(progressFrame({ type: "before_provider_request", payload: "private" }), undefined); -}); - -test("streams concise progress during a code-policy turn", async () => { - const output = sink(); - const previousMode = process.env.PI_SPATIAL_AUTH_MODE; - const previousKey = process.env.OPENAI_API_KEY; - process.env.PI_SPATIAL_AUTH_MODE = "openai-api-key"; - process.env.OPENAI_API_KEY = "test-key"; - let listener: ((event: unknown) => void) | undefined; - const factory: CodePolicySessionFactory = async () => ({ - subscribe: (next) => { - listener = next; - }, - prompt: async () => { - listener?.({ type: "agent_start" }); - listener?.({ type: "turn_start" }); - listener?.({ - type: "message_update", - assistantMessageEvent: { type: "thinking_delta", delta: "do not emit" }, - }); - listener?.({ - type: "message_update", - assistantMessageEvent: { type: "text_delta", delta: "Visible text" }, - }); - listener?.({ - type: "message_update", - assistantMessageEvent: { type: "text_delta", delta: "\nANSWER: 2" }, - }); - listener?.({ type: "agent_end" }); - return undefined; - }, - abort: async () => undefined, - dispose: () => undefined, - sessionEvidence: () => ({ state: "complete", persisted: false }), - }); - const input = Readable.from( - [ - { - version: 1, - type: "session_start", - id: "session-1", - initial_prompt: "Count rooms", - thinking_level: "medium", - }, - { version: 1, type: "prompt", id: "turn-1", text: "Count rooms" }, - { version: 1, type: "dispose" }, - ].map((frame) => `${JSON.stringify(frame)}\n`), - ); - - try { - await runCodePolicyAdapter(input, output.stream, new Writable({ - write(_chunk, _encoding, callback) { - callback(); - }, - }), factory); - } finally { - if (previousMode === undefined) delete process.env.PI_SPATIAL_AUTH_MODE; - else process.env.PI_SPATIAL_AUTH_MODE = previousMode; - if (previousKey === undefined) delete process.env.OPENAI_API_KEY; - else process.env.OPENAI_API_KEY = previousKey; - } - - const frames = output.frames(); - assert.equal(frames.some((frame) => frame.event === "assistant_text_delta"), true); - assert.equal(JSON.stringify(frames).includes("Visible text"), true); - assert.equal(JSON.stringify(frames).includes("do not emit"), false); - const complete = frames.find((frame) => frame.type === "turn_complete"); - assert.equal(complete?.final_text, "Visible text\nANSWER: 2"); -}); diff --git a/packages/pi-code-policy-adapter/test/code-policy-protocol.test.ts b/packages/pi-code-policy-adapter/test/code-policy-protocol.test.ts deleted file mode 100644 index 6cc4573dcf..0000000000 --- a/packages/pi-code-policy-adapter/test/code-policy-protocol.test.ts +++ /dev/null @@ -1,31 +0,0 @@ -import assert from "node:assert/strict"; -import test from "node:test"; - -import { - CODE_POLICY_MAX_LINE_BYTES, - encodeCodePolicyFrame, - parseCodePolicyFrame, -} from "../src/code-policy-protocol.js"; - -test("accepts a valid prompt and emits one newline-delimited frame", () => { - assert.deepEqual( - parseCodePolicyFrame(JSON.stringify({ version: 1, type: "prompt", id: "turn-1", text: "Count" })), - { version: 1, type: "prompt", id: "turn-1", text: "Count" }, - ); - assert.equal( - encodeCodePolicyFrame({ version: 1, type: "session_started", id: "session-1", tools: ["python_exec"] }), - '{"version":1,"type":"session_started","id":"session-1","tools":["python_exec"]}\n', - ); -}); - -test("rejects malformed, unknown, and oversized inbound frames", () => { - assert.throws(() => parseCodePolicyFrame("{"), /invalid code-policy JSON/); - assert.throws( - () => parseCodePolicyFrame(JSON.stringify({ version: 1, type: "unknown" })), - /invalid code-policy frame/, - ); - assert.throws( - () => parseCodePolicyFrame("x".repeat(CODE_POLICY_MAX_LINE_BYTES + 1)), - /exceeds limit/, - ); -}); diff --git a/packages/pi-code-policy-adapter/test/code-policy-session.test.ts b/packages/pi-code-policy-adapter/test/code-policy-session.test.ts deleted file mode 100644 index 37e019b2ca..0000000000 --- a/packages/pi-code-policy-adapter/test/code-policy-session.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import assert from "node:assert/strict"; -import test from "node:test"; -import { - CODE_POLICY_TOOL_NAMES, - assertCodePolicyToolInventory, - codePolicyToolDefinition, -} from "../src/code-policy-session.js"; - -test("code-policy facade registers exactly python_exec", async () => { - const calls: Array<{ tool: string; params: unknown }> = []; - const tool = codePolicyToolDefinition({ - request: async (name, params) => { - calls.push({ tool: name, params }); - return "ok"; - }, - }); - - assert.deepEqual(CODE_POLICY_TOOL_NAMES, ["python_exec"]); - assert.equal(tool.name, "python_exec"); - const result = await tool.execute( - "call-1", - { code: "1 + 1" }, - undefined, - undefined, - undefined as never, - ); - assert.deepEqual(calls, [ - { tool: "python_exec", params: { code: "1 + 1" } }, - ]); - assert.deepEqual(result.content, [{ type: "text", text: "ok" }]); -}); - -test("code-policy facade rejects any activated tool inventory drift", () => { - assert.doesNotThrow(() => assertCodePolicyToolInventory(["python_exec"])); - assert.throws(() => assertCodePolicyToolInventory([]), /exactly python_exec/); - assert.throws( - () => assertCodePolicyToolInventory(["python_exec", "read"]), - /exactly python_exec/, - ); -}); diff --git a/packages/pi-code-policy-adapter/test/session.test.ts b/packages/pi-code-policy-adapter/test/session.test.ts deleted file mode 100644 index 027044d587..0000000000 --- a/packages/pi-code-policy-adapter/test/session.test.ts +++ /dev/null @@ -1,229 +0,0 @@ -import test from "node:test"; -import assert from "node:assert/strict"; -import { existsSync, mkdirSync, readFileSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; -import { execFileSync } from "node:child_process"; -import type { Message } from "@earendil-works/pi-ai"; -import { SessionManager } from "@earendil-works/pi-coding-agent"; -import { - PINNED_PI_VERSION, - SESSION_DIR_ENV, - createSessionManager, - modelProviderForAuthMode, - requiredApiForAuthMode, - retainSystemPromptEvidence, - resolvePinnedPiExportCommand, - sessionEvidenceForManager, -} from "../src/session.js"; - -test("auth mode selects the matching Pi provider and request API", () => { - assert.equal(modelProviderForAuthMode("codex-oauth"), "openai-codex"); - assert.equal(requiredApiForAuthMode("codex-oauth"), "openai-codex-responses"); - assert.equal(modelProviderForAuthMode("openai-api-key"), "openai"); - assert.equal(requiredApiForAuthMode("openai-api-key"), "openai-responses"); -}); - -function withSessionDirectory(name: string, callback: (directory: string) => T): T { - const previous = process.env[SESSION_DIR_ENV]; - process.env[SESSION_DIR_ENV] = name; - const directory = join(process.cwd(), name); - try { - return callback(directory); - } finally { - if (previous === undefined) delete process.env[SESSION_DIR_ENV]; - else process.env[SESSION_DIR_ENV] = previous; - rmSync(directory, { recursive: true, force: true }); - rmSync(join(process.cwd(), "pi-prompt"), { recursive: true, force: true }); - } -} - -test("file-backed session directory accepts only a simple relative name", () => { - for (const value of ["", ".", "..", "/tmp/pi", "../pi", "nested/pi", "pi\\session"]) { - process.env[SESSION_DIR_ENV] = value; - assert.throws(() => createSessionManager(), /PI_SPATIAL_SESSION_DIR/); - } - delete process.env[SESSION_DIR_ENV]; - assert.throws(() => createSessionManager(), /PI_SPATIAL_SESSION_DIR/); -}); - -test("precreated symlink and non-directory session children are rejected", () => { - const symlinkName = `pi-session-link-${process.pid}`; - const fileName = `pi-session-file-${process.pid}`; - const symlink = join(process.cwd(), symlinkName); - const file = join(process.cwd(), fileName); - rmSync(symlink, { recursive: true, force: true }); - rmSync(file, { recursive: true, force: true }); - mkdirSync(join(process.cwd(), `pi-session-target-${process.pid}`), { mode: 0o700 }); - try { - const previous = process.env[SESSION_DIR_ENV]; - process.env[SESSION_DIR_ENV] = symlinkName; - // The symlink is created outside the adapter so the real lstat admission path is tested. - symlinkSync(`pi-session-target-${process.pid}`, symlink); - assert.throws(() => createSessionManager(), /real directory/); - writeFileSync(file, "not a directory"); - process.env[SESSION_DIR_ENV] = fileName; - assert.throws(() => createSessionManager(), /real directory/); - if (previous === undefined) delete process.env[SESSION_DIR_ENV]; - else process.env[SESSION_DIR_ENV] = previous; - } finally { - rmSync(symlink, { recursive: true, force: true }); - rmSync(file, { recursive: true, force: true }); - rmSync(join(process.cwd(), `pi-session-target-${process.pid}`), { recursive: true, force: true }); - delete process.env[SESSION_DIR_ENV]; - } -}); - -test("fresh sessions are persisted, distinct, and discoverable through public APIs", () => { - withSessionDirectory(`pi-session-test-${process.pid}`, (directory) => { - const first = createSessionManager(); - const second = createSessionManager(); - assert.equal(first.isPersisted(), true); - assert.equal(second.isPersisted(), true); - assert.ok(first.getSessionFile()); - assert.ok(second.getSessionFile()); - assert.notEqual(first.getSessionFile(), second.getSessionFile()); - assert.equal(first.getSessionDir(), directory); - assert.equal(second.getSessionDir(), directory); - assert.equal(existsSync(first.getSessionFile() ?? ""), false); - assert.equal(existsSync(second.getSessionFile() ?? ""), false); - }); -}); - -test("a persisted manager with a delayed nonexistent file is unavailable", () => { - withSessionDirectory(`pi-session-delayed-${process.pid}`, () => { - const manager = createSessionManager(); - assert.equal(manager.isPersisted(), true); - assert.ok(manager.getSessionFile()); - assert.equal(existsSync(manager.getSessionFile() ?? ""), false); - assert.deepEqual(sessionEvidenceForManager(manager, true), { state: "unavailable", persisted: false }); - }); -}); - -test("system prompt evidence preserves exact unicode bytes with bounded metadata", () => { - withSessionDirectory(`pi-session-prompt-${process.pid}`, () => { - const prompt = "system π\n用户—✅"; - const metadata = retainSystemPromptEvidence(prompt); - const path = join(process.cwd(), metadata.relativePath); - assert.equal(metadata.relativePath, "pi-prompt/system.txt"); - assert.deepEqual(readFileSync(path), Buffer.from(prompt, "utf8")); - assert.equal(metadata.byteCount, Buffer.byteLength(prompt, "utf8")); - assert.match(metadata.sha256, /^[a-f0-9]{64}$/); - assert.equal(statSync(path).mode & 0o777, 0o600); - assert.equal(statSync(join(process.cwd(), "pi-prompt")).mode & 0o777, 0o700); - assert.throws(() => retainSystemPromptEvidence("replacement"), /EEXIST/); - assert.deepEqual(readFileSync(path), Buffer.from(prompt, "utf8")); - }); -}); - -test("native JSONL remains after a model-independent public-API append", () => { - withSessionDirectory(`pi-session-survival-${process.pid}`, (directory) => { - const manager = createSessionManager(); - const user: Message = { role: "user", content: [{ type: "text", text: "hello" }], timestamp: Date.now() }; - const assistant: Message = { - role: "assistant", - content: [{ type: "text", text: "done" }], - api: "openai-responses", - provider: "openai-codex", - model: "gpt-5.6-luna", - usage: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 0, - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, - }, - stopReason: "stop", - timestamp: Date.now(), - }; - manager.appendMessage(user); - manager.appendMessage(assistant); - const file = manager.getSessionFile(); - assert.ok(file); - assert.equal(existsSync(file), true); - assert.equal(manager.isPersisted(), true); - assert.equal(manager.getEntries().length, 2); - assert.equal(existsSync(file), true); - assert.equal(statSync(file).mode & 0o777, 0o600); - assert.equal(manager.getSessionDir(), directory); - }); -}); - -test("pinned SessionManager.open reopens an unchanged native v3 session", () => { - withSessionDirectory(`pi-session-reopen-${process.pid}`, (directory) => { - const original = createSessionManager(); - const timestamp = 1_700_000_000_000; - const userId = original.appendMessage({ role: "user", content: [{ type: "text", text: "native user" }], timestamp }); - const assistantId = original.appendMessage({ - role: "assistant", - content: [{ type: "text", text: "native assistant" }], - api: "openai-responses", - provider: "openai-codex", - model: "gpt-5.6-luna", - usage: { - input: 1, - output: 2, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 3, - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, - }, - stopReason: "stop", - timestamp: timestamp + 1, - }); - const thinkingId = original.appendThinkingLevelChange("medium"); - const modelId = original.appendModelChange("openai-codex", "gpt-5.6-luna"); - const customId = original.appendCustomEntry("adapter-test", { stable: true }); - const file = original.getSessionFile(); - assert.ok(file); - const sourceBytes = readFileSync(file); - - // SessionManager writes synchronously and has no separate close operation; - // opening the generated file is the documented handoff lifecycle. - const reopened = SessionManager.open(file, directory); - assert.deepEqual(readFileSync(file), sourceBytes); - assert.equal(reopened.getSessionFile(), file); - assert.equal(reopened.getSessionDir(), directory); - assert.equal(reopened.getHeader()?.type, "session"); - assert.equal(reopened.getHeader()?.version, 3); - assert.equal(reopened.getHeader()?.id, original.getSessionId()); - assert.equal(reopened.getSessionId(), original.getSessionId()); - assert.deepEqual(reopened.getEntries().map((entry) => entry.id), [userId, assistantId, thinkingId, modelId, customId]); - assert.equal(reopened.getEntry(assistantId)?.parentId, userId); - assert.equal(reopened.getLeafId(), customId); - assert.equal(reopened.getLeafEntry()?.id, customId); - assert.equal(reopened.getTree().length, 1); - assert.equal(reopened.getTree()[0]?.children.length, 1); - assert.deepEqual(reopened.getBranch(), reopened.getEntries()); - assert.deepEqual(readFileSync(file), sourceBytes); - }); -}); - -test("pinned Pi CLI exports a synthetic native session without auth or rewriting JSONL", () => { - withSessionDirectory(`pi-session-export-${process.pid}`, (directory) => { - const manager = createSessionManager(); - manager.appendMessage({ role: "user", content: [{ type: "text", text: "hello" }], timestamp: Date.now() }); - manager.appendMessage({ - role: "assistant", content: [{ type: "text", text: "done" }], api: "openai-responses", provider: "openai-codex", model: "gpt-5.6-luna", - usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, stopReason: "stop", timestamp: Date.now(), - }); - const file = manager.getSessionFile(); - assert.ok(file); - const before = readFileSync(file); - const html = join(directory, "export.html"); - const command = resolvePinnedPiExportCommand(file, html); - assert.equal(command.executable, process.execPath); - assert.deepEqual(command.args.slice(1), ["--export", file, html]); - assert.equal(command.packageVersion, PINNED_PI_VERSION); - execFileSync(command.executable, command.args, { stdio: "pipe" }); - assert.ok(readFileSync(html).length > 0); - assert.deepEqual(readFileSync(file), before); - }); -}); - -test("pinned package command validates executable and bin metadata", () => { - const command = resolvePinnedPiExportCommand("input.jsonl", "output.html"); - assert.equal(command.executable, process.execPath); - assert.match(command.args[0], /node_modules[\\/]@earendil-works[\\/]pi-coding-agent[\\/]dist[\\/]cli\.js$/); - assert.equal(PINNED_PI_VERSION, "0.80.10"); -}); diff --git a/packages/pi-code-policy-adapter/.gitignore b/packages/pi-code-policy-extension/.gitignore similarity index 100% rename from packages/pi-code-policy-adapter/.gitignore rename to packages/pi-code-policy-extension/.gitignore diff --git a/packages/pi-code-policy-extension/README.md b/packages/pi-code-policy-extension/README.md new file mode 100644 index 0000000000..d070df0668 --- /dev/null +++ b/packages/pi-code-policy-extension/README.md @@ -0,0 +1,7 @@ +# Pi CodePolicy extension + +This package adds one `python_exec` tool to the stock Pi CLI. The tool connects +directly to the evaluator-owned MCP server named by `DIMOS_CODE_POLICY_MCP_URL`. + +The Python evaluator launches Pi with all built-in tools and extension discovery +disabled, then loads only `dist/python-exec.js`. diff --git a/packages/pi-code-policy-adapter/package-lock.json b/packages/pi-code-policy-extension/package-lock.json similarity index 95% rename from packages/pi-code-policy-adapter/package-lock.json rename to packages/pi-code-policy-extension/package-lock.json index cb5d4cd96a..dea1ad1adf 100644 --- a/packages/pi-code-policy-adapter/package-lock.json +++ b/packages/pi-code-policy-extension/package-lock.json @@ -1,20 +1,20 @@ { - "name": "@dimos/pi-code-policy-adapter", + "name": "@dimos/pi-code-policy-extension", "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "@dimos/pi-code-policy-adapter", + "name": "@dimos/pi-code-policy-extension", "version": "0.1.0", "dependencies": { - "@earendil-works/pi-ai": "0.80.10", "@earendil-works/pi-coding-agent": "0.80.10", - "typebox": "^1.3.6" + "@modelcontextprotocol/client": "2.0.0", + "typebox": "1.3.6" }, "devDependencies": { - "@types/node": "^22.15.0", - "typescript": "^5.8.3" + "@types/node": "22.15.0", + "typescript": "5.8.3" }, "engines": { "node": ">=22.19.0" @@ -825,6 +825,36 @@ } } }, + "node_modules/@modelcontextprotocol/client": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/client/-/client-2.0.0.tgz", + "integrity": "sha512-8f1OghQ2rjzIOfqgUCP+8GiUWqRs89njoWLNqAe8kWmDePv3s1fZXseej+QXemssEuuOvLLmLO/kqM3IQHtISw==", + "license": "MIT", + "dependencies": { + "@modelcontextprotocol/core": "2.0.0", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "jose": "^6.1.3", + "pkce-challenge": "^5.0.0", + "zod": "^4.2.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@modelcontextprotocol/core": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/core/-/core-2.0.0.tgz", + "integrity": "sha512-pJCEwGG7Lfr/+PQp9ZTwKXNeO5wzbfKL7H3MYpCorM4oFBoQrdjnBgEoqG+RjhsvS1FKrDbKux+M1HhlnGWqcA==", + "license": "MIT", + "dependencies": { + "zod": "^4.2.0" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@opentelemetry/api": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", @@ -1026,9 +1056,9 @@ } }, "node_modules/@types/node": { - "version": "22.20.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.1.tgz", - "integrity": "sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==", + "version": "22.15.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.0.tgz", + "integrity": "sha512-99S8dWD2DkeE6PBaEDw+In3aar7hdoBvjyJMR6vaKBTzpvR0P00ClzJMOoVrj9D2+Sy/YCwACYHnBTpMhg1UCA==", "license": "MIT", "dependencies": { "undici-types": "~6.21.0" @@ -1181,6 +1211,27 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.0.tgz", + "integrity": "sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -1382,6 +1433,15 @@ "jiti": "lib/jiti-cli.mjs" } }, + "node_modules/jose": { + "version": "6.2.8", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.8.tgz", + "integrity": "sha512-Bsdjwm3Qsd/P0jR+BHDe3LytDfY7WBq2HmCCLIwuVRHMuEC9ae7/R474GIUdF1NgCyZjzVo/A9DOiOBtXq8ZoQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -1585,6 +1645,15 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, "node_modules/proper-lockfile": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", @@ -1715,9 +1784,9 @@ "license": "MIT" }, "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "version": "5.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", "dev": true, "license": "Apache-2.0", "bin": { diff --git a/packages/pi-code-policy-adapter/package.json b/packages/pi-code-policy-extension/package.json similarity index 74% rename from packages/pi-code-policy-adapter/package.json rename to packages/pi-code-policy-extension/package.json index 345a094939..018375f2e4 100644 --- a/packages/pi-code-policy-adapter/package.json +++ b/packages/pi-code-policy-extension/package.json @@ -1,5 +1,5 @@ { - "name": "@dimos/pi-code-policy-adapter", + "name": "@dimos/pi-code-policy-extension", "version": "0.1.0", "private": true, "type": "module", @@ -13,12 +13,12 @@ "test": "npm run clean && tsc -p tsconfig.build.json && tsc -p tsconfig.test.json && node --test dist-test/test/*.test.js" }, "dependencies": { - "@earendil-works/pi-ai": "0.80.10", "@earendil-works/pi-coding-agent": "0.80.10", - "typebox": "^1.3.6" + "@modelcontextprotocol/client": "2.0.0", + "typebox": "1.3.6" }, "devDependencies": { - "@types/node": "^22.15.0", - "typescript": "^5.8.3" + "@types/node": "22.15.0", + "typescript": "5.8.3" } } diff --git a/packages/pi-code-policy-extension/src/python-exec.ts b/packages/pi-code-policy-extension/src/python-exec.ts new file mode 100644 index 0000000000..5a74266436 --- /dev/null +++ b/packages/pi-code-policy-extension/src/python-exec.ts @@ -0,0 +1,83 @@ +import { + Client, + StreamableHTTPClientTransport, + type CallToolResult, +} from "@modelcontextprotocol/client"; +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { Type } from "typebox"; + +const TOOL_NAME = "python_exec"; +const DEFAULT_TIMEOUT_SECONDS = 110; + +interface McpClient { + listTools(): Promise<{ tools: Array<{ name: string }> }>; + callTool( + params: { name: string; arguments: Record }, + options?: { timeout?: number }, + ): Promise; + close(): Promise; +} + +export function textFromResult(result: CallToolResult): string { + return result.content + .filter((item): item is Extract => item.type === "text") + .map((item) => item.text) + .join("\n"); +} + +async function connect(url: string): Promise { + const client = new Client({ name: "dimos-pi-code-policy", version: "1.0.0" }); + await client.connect(new StreamableHTTPClientTransport(new URL(url))); + return client; +} + +export async function installPythonExec( + pi: ExtensionAPI, + mcpUrl: string, + connectClient: (url: string) => Promise = connect, +): Promise { + const client = await connectClient(mcpUrl); + const inventory = await client.listTools(); + if (inventory.tools.length !== 1 || inventory.tools[0]?.name !== TOOL_NAME) { + await client.close(); + throw new Error("CodePolicy MCP server must expose exactly python_exec"); + } + + pi.registerTool({ + name: TOOL_NAME, + label: "Execute Python", + description: + "Execute Python in a persistent trusted, unsandboxed session with read-only memory.", + parameters: Type.Object( + { + code: Type.String({ minLength: 1 }), + timeout_s: Type.Optional( + Type.Number({ exclusiveMinimum: 0, maximum: DEFAULT_TIMEOUT_SECONDS }), + ), + }, + { additionalProperties: false }, + ), + executionMode: "sequential", + execute: async (_id, params) => { + const timeoutSeconds = params.timeout_s ?? DEFAULT_TIMEOUT_SECONDS; + const result = await client.callTool( + { + name: TOOL_NAME, + arguments: { code: params.code, timeout_s: timeoutSeconds }, + }, + { timeout: (timeoutSeconds + 10) * 1000 }, + ); + return { content: [{ type: "text", text: textFromResult(result) }], details: {} }; + }, + }); + + pi.on("session_shutdown", async () => { + await client.close(); + }); +} + +export default async function pythonExecExtension(pi: ExtensionAPI): Promise { + const mcpUrl = process.env.DIMOS_CODE_POLICY_MCP_URL; + if (!mcpUrl) throw new Error("DIMOS_CODE_POLICY_MCP_URL is required"); + await installPythonExec(pi, mcpUrl); +} diff --git a/packages/pi-code-policy-extension/test/python-exec.test.ts b/packages/pi-code-policy-extension/test/python-exec.test.ts new file mode 100644 index 0000000000..9f351564c7 --- /dev/null +++ b/packages/pi-code-policy-extension/test/python-exec.test.ts @@ -0,0 +1,43 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import type { CallToolResult } from "@modelcontextprotocol/client"; +import type { ExtensionAPI, ToolDefinition } from "@earendil-works/pi-coding-agent"; + +import { installPythonExec } from "../src/python-exec.js"; + +test("registers one tool that calls MCP directly", async () => { + let tool: ToolDefinition | undefined; + let shutdown: (() => Promise) | undefined; + let closed = false; + const pi = { + registerTool(value: ToolDefinition) { + tool = value; + }, + on(event: string, handler: () => Promise) { + if (event === "session_shutdown") shutdown = handler; + }, + } as ExtensionAPI; + const client = { + async listTools() { + return { tools: [{ name: "python_exec" }] }; + }, + async callTool(params: { name: string; arguments: Record }) { + assert.deepEqual(params, { + name: "python_exec", + arguments: { code: "1 + 1", timeout_s: 3 }, + }); + return { content: [{ type: "text", text: "2" }] } as CallToolResult; + }, + async close() { + closed = true; + }, + }; + + await installPythonExec(pi, "http://127.0.0.1:1/mcp", async () => client); + assert.equal(tool?.name, "python_exec"); + const result = await tool!.execute("call-1", { code: "1 + 1", timeout_s: 3 }, undefined, undefined, {} as never); + assert.deepEqual(result.content, [{ type: "text", text: "2" }]); + await shutdown!(); + assert.equal(closed, true); +}); diff --git a/packages/pi-code-policy-adapter/tsconfig.build.json b/packages/pi-code-policy-extension/tsconfig.build.json similarity index 100% rename from packages/pi-code-policy-adapter/tsconfig.build.json rename to packages/pi-code-policy-extension/tsconfig.build.json diff --git a/packages/pi-code-policy-adapter/tsconfig.json b/packages/pi-code-policy-extension/tsconfig.json similarity index 100% rename from packages/pi-code-policy-adapter/tsconfig.json rename to packages/pi-code-policy-extension/tsconfig.json diff --git a/packages/pi-code-policy-adapter/tsconfig.test.json b/packages/pi-code-policy-extension/tsconfig.test.json similarity index 100% rename from packages/pi-code-policy-adapter/tsconfig.test.json rename to packages/pi-code-policy-extension/tsconfig.test.json diff --git a/pyproject.toml b/pyproject.toml index 18c65e52a3..3249ac5735 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -226,7 +226,7 @@ agents = [ "nbformat>=5.10.4", "pyzmq>=27.1.0", # Loopback-only MCP host for standalone CodePolicy evaluation. - "fastapi>=0.115.6", + "mcp==2.0.0", "uvicorn>=0.34.0", "langchain>=1.2.3,<2", "langchain-core>=1.2.22,<2", @@ -403,6 +403,11 @@ project-deps = [ "lap>=0.5.12", "langchain-openai>=1,<2", "ollama>=0.6.0", + # Lightweight frozen-agent evaluation runtime used by tests and mypy. + "ipykernel>=7.2.0", + "jupyter-client>=8.8.0", + "mcp==2.0.0", + "uvicorn>=0.34.0", ] tests = [ diff --git a/uv.lock b/uv.lock index 9b1c614eff..9e6869f184 100644 --- a/uv.lock +++ b/uv.lock @@ -1589,7 +1589,6 @@ dependencies = [ [package.optional-dependencies] agents = [ - { name = "fastapi" }, { name = "faster-whisper" }, { name = "ipykernel" }, { name = "jupyter-client" }, @@ -1598,6 +1597,7 @@ agents = [ { name = "langchain-huggingface" }, { name = "langchain-ollama" }, { name = "langchain-openai" }, + { name = "mcp" }, { name = "nbformat" }, { name = "ollama" }, { name = "openai" }, @@ -1636,6 +1636,7 @@ all = [ { name = "lap" }, { name = "manifold3d" }, { name = "matplotlib" }, + { name = "mcp" }, { name = "moondream" }, { name = "mujoco" }, { name = "nbformat" }, @@ -1699,6 +1700,7 @@ base = [ { name = "langchain-ollama" }, { name = "langchain-openai" }, { name = "lap" }, + { name = "mcp" }, { name = "moondream" }, { name = "nbformat" }, { name = "ollama" }, @@ -1806,6 +1808,7 @@ unitree = [ { name = "langchain-ollama" }, { name = "langchain-openai" }, { name = "lap" }, + { name = "mcp" }, { name = "moondream" }, { name = "nbformat" }, { name = "ollama" }, @@ -1844,6 +1847,7 @@ unitree-dds = [ { name = "langchain-openai" }, { name = "lap" }, { name = "mcap" }, + { name = "mcp" }, { name = "moondream" }, { name = "nbformat" }, { name = "ollama" }, @@ -1896,12 +1900,15 @@ lint = [ { name = "gdown" }, { name = "googlemaps" }, { name = "hydra-core" }, + { name = "ipykernel" }, { name = "ipython", version = "8.38.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "ipython", version = "9.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-openai" }, { name = "lap" }, + { name = "mcp" }, { name = "moondream" }, { name = "mypy" }, { name = "ollama" }, @@ -1925,6 +1932,7 @@ lint = [ { name = "types-reportlab" }, { name = "types-requests" }, { name = "ultralytics" }, + { name = "uvicorn" }, { name = "watchdog" }, { name = "xacro" }, ] @@ -1935,10 +1943,13 @@ project-deps = [ { name = "gdown" }, { name = "googlemaps" }, { name = "hydra-core" }, + { name = "ipykernel" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-openai" }, { name = "lap" }, + { name = "mcp" }, { name = "moondream" }, { name = "ollama" }, { name = "open-clip-torch" }, @@ -1948,6 +1959,7 @@ project-deps = [ { name = "torchreid" }, { name = "transformers", extra = ["torch"] }, { name = "ultralytics" }, + { name = "uvicorn" }, { name = "xacro" }, ] tests = [ @@ -1959,11 +1971,14 @@ tests = [ { name = "gdown" }, { name = "googlemaps" }, { name = "hydra-core" }, + { name = "ipykernel" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-openai" }, { name = "lap" }, { name = "maturin" }, + { name = "mcp" }, { name = "md-babel-py" }, { name = "moondream" }, { name = "mujoco" }, @@ -1993,6 +2008,7 @@ tests = [ { name = "trimesh" }, { name = "ultralytics" }, { name = "unitree-webrtc-connect" }, + { name = "uvicorn" }, { name = "viser", extra = ["urdf"], marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, { name = "watchdog" }, { name = "xacro" }, @@ -2006,12 +2022,15 @@ tests-self-hosted = [ { name = "gdown" }, { name = "googlemaps" }, { name = "hydra-core" }, + { name = "ipykernel" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-openai" }, { name = "lap" }, { name = "maturin" }, { name = "mcap" }, + { name = "mcp" }, { name = "md-babel-py" }, { name = "moondream" }, { name = "mujoco" }, @@ -2042,6 +2061,7 @@ tests-self-hosted = [ { name = "trimesh" }, { name = "ultralytics" }, { name = "unitree-webrtc-connect" }, + { name = "uvicorn" }, { name = "viser", extra = ["urdf"], marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, { name = "watchdog" }, { name = "xacro" }, @@ -2075,7 +2095,6 @@ requires-dist = [ { name = "eclipse-zenoh", specifier = ">=1.9.0,<2.0" }, { name = "edgetam-dimos", marker = "extra == 'misc'" }, { name = "einops", marker = "extra == 'perception'", specifier = ">=0.8.1" }, - { name = "fastapi", marker = "extra == 'agents'", specifier = ">=0.115.6" }, { name = "fastapi", marker = "extra == 'web'", specifier = ">=0.115.6" }, { name = "faster-whisper", marker = "extra == 'agents'", specifier = ">=1.0.0" }, { name = "ffmpeg-python", marker = "extra == 'web'" }, @@ -2103,6 +2122,7 @@ requires-dist = [ { name = "manifold3d", marker = "extra == 'apriltag'", specifier = ">=2.5.0" }, { name = "matplotlib", marker = "extra == 'manipulation'", specifier = ">=3.7.1" }, { name = "mcap", marker = "extra == 'unitree-dds'", specifier = ">=1.2.0" }, + { name = "mcp", marker = "extra == 'agents'", specifier = "==2.0.0" }, { name = "moondream", marker = "extra == 'perception'" }, { name = "mujoco", marker = "extra == 'sim'", specifier = ">=3.3.4" }, { name = "nbformat", marker = "extra == 'agents'", specifier = ">=5.10.4" }, @@ -2194,11 +2214,14 @@ lint = [ { name = "gdown", specifier = "==6.0.0" }, { name = "googlemaps", specifier = ">=4.10.0" }, { name = "hydra-core", specifier = ">=1.3.0" }, + { name = "ipykernel", specifier = ">=7.2.0" }, { name = "ipython" }, + { name = "jupyter-client", specifier = ">=8.8.0" }, { name = "langchain", specifier = "==1.2.3" }, { name = "langchain-core", specifier = "==1.3.3" }, { name = "langchain-openai", specifier = ">=1,<2" }, { name = "lap", specifier = ">=0.5.12" }, + { name = "mcp", specifier = "==2.0.0" }, { name = "moondream" }, { name = "mypy", specifier = "==1.19.0" }, { name = "ollama", specifier = ">=0.6.0" }, @@ -2222,6 +2245,7 @@ lint = [ { name = "types-reportlab", specifier = ">=4.5.0" }, { name = "types-requests", specifier = ">=2.32.4.20260107,<3" }, { name = "ultralytics", specifier = ">=8.3.70" }, + { name = "uvicorn", specifier = ">=0.34.0" }, { name = "watchdog", specifier = ">=3.0.0" }, { name = "xacro" }, ] @@ -2232,10 +2256,13 @@ project-deps = [ { name = "gdown", specifier = "==6.0.0" }, { name = "googlemaps", specifier = ">=4.10.0" }, { name = "hydra-core", specifier = ">=1.3.0" }, + { name = "ipykernel", specifier = ">=7.2.0" }, + { name = "jupyter-client", specifier = ">=8.8.0" }, { name = "langchain", specifier = "==1.2.3" }, { name = "langchain-core", specifier = "==1.3.3" }, { name = "langchain-openai", specifier = ">=1,<2" }, { name = "lap", specifier = ">=0.5.12" }, + { name = "mcp", specifier = "==2.0.0" }, { name = "moondream" }, { name = "ollama", specifier = ">=0.6.0" }, { name = "open-clip-torch", specifier = "==3.2.0" }, @@ -2245,6 +2272,7 @@ project-deps = [ { name = "torchreid", specifier = "==0.2.5" }, { name = "transformers", extras = ["torch"], specifier = "==4.53.3" }, { name = "ultralytics", specifier = ">=8.3.70" }, + { name = "uvicorn", specifier = ">=0.34.0" }, { name = "xacro" }, ] tests = [ @@ -2257,11 +2285,14 @@ tests = [ { name = "gdown", specifier = "==6.0.0" }, { name = "googlemaps", specifier = ">=4.10.0" }, { name = "hydra-core", specifier = ">=1.3.0" }, + { name = "ipykernel", specifier = ">=7.2.0" }, + { name = "jupyter-client", specifier = ">=8.8.0" }, { name = "langchain", specifier = "==1.2.3" }, { name = "langchain-core", specifier = "==1.3.3" }, { name = "langchain-openai", specifier = ">=1,<2" }, { name = "lap", specifier = ">=0.5.12" }, { name = "maturin", specifier = ">=1.7" }, + { name = "mcp", specifier = "==2.0.0" }, { name = "md-babel-py", specifier = ">=1.4.0" }, { name = "moondream" }, { name = "mujoco", specifier = ">=3.3.4" }, @@ -2291,6 +2322,7 @@ tests = [ { name = "trimesh", specifier = ">=4.0.0" }, { name = "ultralytics", specifier = ">=8.3.70" }, { name = "unitree-webrtc-connect", specifier = ">=2.1.2" }, + { name = "uvicorn", specifier = ">=0.34.0" }, { name = "viser", extras = ["urdf"], marker = "platform_machine != 'aarch64' or sys_platform != 'linux'", specifier = ">=1.0.29" }, { name = "watchdog", specifier = ">=3.0.0" }, { name = "xacro" }, @@ -2306,12 +2338,15 @@ tests-self-hosted = [ { name = "gdown", specifier = "==6.0.0" }, { name = "googlemaps", specifier = ">=4.10.0" }, { name = "hydra-core", specifier = ">=1.3.0" }, + { name = "ipykernel", specifier = ">=7.2.0" }, + { name = "jupyter-client", specifier = ">=8.8.0" }, { name = "langchain", specifier = "==1.2.3" }, { name = "langchain-core", specifier = "==1.3.3" }, { name = "langchain-openai", specifier = ">=1,<2" }, { name = "lap", specifier = ">=0.5.12" }, { name = "maturin", specifier = ">=1.7" }, { name = "mcap", specifier = ">=1.2.0" }, + { name = "mcp", specifier = "==2.0.0" }, { name = "md-babel-py", specifier = ">=1.4.0" }, { name = "moondream" }, { name = "mujoco", specifier = ">=3.3.4" }, @@ -2342,6 +2377,7 @@ tests-self-hosted = [ { name = "trimesh", specifier = ">=4.0.0" }, { name = "ultralytics", specifier = ">=8.3.70" }, { name = "unitree-webrtc-connect", specifier = ">=2.1.2" }, + { name = "uvicorn", specifier = ">=0.34.0" }, { name = "viser", extras = ["urdf"], marker = "platform_machine != 'aarch64' or sys_platform != 'linux'", specifier = ">=1.0.29" }, { name = "watchdog", specifier = ">=3.0.0" }, { name = "xacro" }, @@ -3293,6 +3329,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/a8/20ed1ed79cbc2ecdf5301c0968ab7c85547212e2a7bd126ddd2d986e206e/httpcore2-2.9.1.tar.gz", hash = "sha256:4d8acbf8b306f48c9d6046591fd5ba4037d1b1b1000d140fc2c3eab1e9a0c0e2", size = 67089, upload-time = "2026-07-24T09:21:03.867Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/fb/46c52b781975c335a2bcf1072c7bbc007cbdc8d674217f5ee1daba2c848b/httpcore2-2.9.1-py3-none-any.whl", hash = "sha256:6182472379e855fe4221246a2bb7ecede403bc61c6798062ae1787d051ccde26", size = 82809, upload-time = "2026-07-24T09:21:01.178Z" }, +] + [[package]] name = "httptools" version = "0.8.0" @@ -3337,6 +3386,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx2" +version = "2.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpcore2" }, + { name = "idna" }, + { name = "truststore" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/14/38128fbafd7e0ed41d874df6c9a653d47c2d111cfe59e2b4ac95161b4abd/httpx2-2.9.1.tar.gz", hash = "sha256:1932a768737e3666291582833da748cc4e563c337cf96706fccc04fa6e58764a", size = 95458, upload-time = "2026-07-24T09:21:04.972Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/b8/cfd91c4ab9134d386d48f0b6ac662ff3d4be6efdee59ee1c67ebc3c0487c/httpx2-2.9.1-py3-none-any.whl", hash = "sha256:1820fe14a9ab1107bfeff39259987429450b070ec0ff38cc87eb0d8c97fdc71a", size = 91191, upload-time = "2026-07-24T09:21:02.6Z" }, +] + [[package]] name = "huggingface-hub" version = "0.36.2" @@ -3390,11 +3455,11 @@ wheels = [ [[package]] name = "idna" -version = "3.11" +version = "3.18" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] [[package]] @@ -4798,6 +4863,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, ] +[[package]] +name = "mcp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx2" }, + { name = "jsonschema" }, + { name = "mcp-types" }, + { name = "opentelemetry-api" }, + { name = "pydantic" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "python-multipart" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "sse-starlette" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, + { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/33/32d4dff2c95bb5d897c3ef4c83649a08996b17b58f0a326d2495d4c81179/mcp-2.0.0.tar.gz", hash = "sha256:0f440e735c13ece8bb19bc62cf0b86f4313448432fbb77d35e14034f4e050728", size = 1662284, upload-time = "2026-07-28T13:45:32.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/72/7d7897418912c1d12e87556630dfb7bf0eac71160e9bef8b447960804ee3/mcp-2.0.0-py3-none-any.whl", hash = "sha256:1cb4c75d2d2c7b8c1d756355e5d82a39f2822cc7f13e22a2051d7ca3592349d6", size = 349980, upload-time = "2026-07-28T13:45:28.853Z" }, +] + +[[package]] +name = "mcp-types" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/56/9b8e1c152f61f6c6b07c4b5896c88c7d0ae90bac6ee6306f852fcc5c1eb0/mcp_types-2.0.0.tar.gz", hash = "sha256:d7d939b9285c9961ae8866ba75ef85da34d12bafe276efbf4eb6a131786d8379", size = 66632, upload-time = "2026-07-28T13:45:33.804Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/4c/c78d78c3d52b0ac594ad7cc8ef5972adfe070e3597a8a4c6ce0cd39196ea/mcp_types-2.0.0-py3-none-any.whl", hash = "sha256:6b2de797ca2797f568b79529e1b25948e34de511bcc0bd82fef1039a6d1b8eb0", size = 69649, upload-time = "2026-07-28T13:45:30.713Z" }, +] + [[package]] name = "md-babel-py" version = "1.4.0" @@ -7046,6 +7149,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] +[[package]] +name = "pyjwt" +version = "2.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + [[package]] name = "pylibsrtp" version = "1.0.0" @@ -9083,6 +9203,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ab/a8/cdf8b3e4c98132f965f88c2313a4b493266832ad47fb52f23d14d4f86bb5/triton-3.6.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74caf5e34b66d9f3a429af689c1c7128daba1d8208df60e81106b115c00d6fca", size = 188266850, upload-time = "2026-01-20T16:00:43.041Z" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "typer" version = "0.23.1" From ca270d4f6b9c626f5574415a683e1528dcf13305 Mon Sep 17 00:00:00 2001 From: cc Date: Wed, 5 Aug 2026 15:50:22 -0700 Subject: [PATCH 07/15] refactor: consolidate evaluation models --- dimos/benchmark/agent_eval/base.py | 26 ------------------------ dimos/benchmark/agent_eval/models.py | 9 ++++++-- dimos/benchmark/short_horizon_qa/eval.py | 7 +++++-- 3 files changed, 12 insertions(+), 30 deletions(-) delete mode 100644 dimos/benchmark/agent_eval/base.py diff --git a/dimos/benchmark/agent_eval/base.py b/dimos/benchmark/agent_eval/base.py deleted file mode 100644 index 4fe7a15771..0000000000 --- a/dimos/benchmark/agent_eval/base.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Shared Pydantic policy for serialized evaluation contracts.""" - -from typing import Literal - -from pydantic import BaseModel, ConfigDict - - -class BaseEvalModel(BaseModel): - """Strict immutable base for serialized evaluation contracts.""" - - model_config = ConfigDict(extra="forbid", frozen=True, strict=True) - schema_version: Literal["1.0"] = "1.0" diff --git a/dimos/benchmark/agent_eval/models.py b/dimos/benchmark/agent_eval/models.py index 7787854381..0f7af3746c 100644 --- a/dimos/benchmark/agent_eval/models.py +++ b/dimos/benchmark/agent_eval/models.py @@ -20,9 +20,14 @@ from pathlib import PurePosixPath from typing import Annotated, Literal -from pydantic import Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, model_validator -from dimos.benchmark.agent_eval.base import BaseEvalModel + +class BaseEvalModel(BaseModel): + """Strict immutable base for the compact evaluation contracts.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + schema_version: Literal["1.0"] = "1.0" NonEmpty = Annotated[str, Field(min_length=1)] diff --git a/dimos/benchmark/short_horizon_qa/eval.py b/dimos/benchmark/short_horizon_qa/eval.py index 245ad42916..72916e9615 100644 --- a/dimos/benchmark/short_horizon_qa/eval.py +++ b/dimos/benchmark/short_horizon_qa/eval.py @@ -22,8 +22,11 @@ from pydantic import Field -from dimos.benchmark.agent_eval.base import BaseEvalModel -from dimos.benchmark.agent_eval.models import EvalCase, ExactIntegerValidatorRef +from dimos.benchmark.agent_eval.models import ( + BaseEvalModel, + EvalCase, + ExactIntegerValidatorRef, +) _ANSWER_LINE = re.compile(r"(?m)^ANSWER:\s*") _TERMINAL_INTEGER = re.compile(r"(?:^|\n)ANSWER:\s*(-?\d+)\s*\Z") From 7430fb7e3f77e44b10c6971218854d27584bedbc Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 22:51:17 +0000 Subject: [PATCH 08/15] [autofix.ci] apply automated fixes --- dimos/benchmark/agent_eval/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dimos/benchmark/agent_eval/models.py b/dimos/benchmark/agent_eval/models.py index 0f7af3746c..7991d89b59 100644 --- a/dimos/benchmark/agent_eval/models.py +++ b/dimos/benchmark/agent_eval/models.py @@ -29,6 +29,7 @@ class BaseEvalModel(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, strict=True) schema_version: Literal["1.0"] = "1.0" + NonEmpty = Annotated[str, Field(min_length=1)] From a11488d2b6f9ea5690ddfbb67e039f8754686d74 Mon Sep 17 00:00:00 2001 From: cc Date: Wed, 5 Aug 2026 16:07:24 -0700 Subject: [PATCH 09/15] feat: stream evaluation progress live --- dimos/benchmark/agent_eval/pi_process.py | 167 +++++++++++++++--- dimos/benchmark/agent_eval/single_case.py | 6 + dimos/benchmark/agent_eval/test_pi_process.py | 105 ++++++++++- dimos/benchmark/short_horizon_qa/prepare.py | 32 +++- .../short_horizon_qa/test_hongkong_eval.py | 3 + .../short_horizon_qa/test_prepare.py | 3 + 6 files changed, 280 insertions(+), 36 deletions(-) diff --git a/dimos/benchmark/agent_eval/pi_process.py b/dimos/benchmark/agent_eval/pi_process.py index 121583be18..ecd04336e0 100644 --- a/dimos/benchmark/agent_eval/pi_process.py +++ b/dimos/benchmark/agent_eval/pi_process.py @@ -21,7 +21,19 @@ import os from pathlib import Path import subprocess +import threading import time +from typing import Any + +from dimos.benchmark.agent_eval.progress import ( + AssistantTextProgress, + FinalResponseProgress, + ProgressSink, + StatusProgress, + ToolEndProgress, + ToolStartProgress, + emit_progress, +) PI_VERSION = "0.80.10" MAX_STDERR_BYTES = 64 * 1024 @@ -53,6 +65,7 @@ def __init__( model: str, thinking_level: str, timeout_s: float, + progress: ProgressSink | None = None, ) -> None: if not cli.is_file(): raise FileNotFoundError(f"Pi {PI_VERSION} CLI is not installed: {cli}") @@ -66,6 +79,7 @@ def __init__( self.model = model self.thinking_level = thinking_level self.timeout_s = timeout_s + self.progress = progress def run( self, @@ -125,33 +139,82 @@ def run( stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + bufsize=1, ) + stdout = process.stdout + stderr_stream = process.stderr + assert stdout is not None + assert stderr_stream is not None + events = _PiEventAccumulator(self.progress) + stderr_parts: list[str] = [] + stderr_bytes = 0 + + def read_stdout() -> None: + for line in stdout: + events.feed(line) + + def read_stderr() -> None: + nonlocal stderr_bytes + for line in stderr_stream: + line = line.replace(api_key, "[REDACTED]") + encoded = line.encode() + remaining = MAX_STDERR_BYTES - stderr_bytes + if remaining > 0: + retained = encoded[:remaining].decode(errors="ignore") + stderr_parts.append(retained) + stderr_bytes += len(retained.encode()) + message = line.strip() + if message: + emit_progress( + self.progress, + StatusProgress(channel="pi", message=_bounded_stderr(message)), + ) + + readers = ( + threading.Thread(target=read_stdout, name="pi-stdout", daemon=True), + threading.Thread(target=read_stderr, name="pi-stderr", daemon=True), + ) + for reader in readers: + reader.start() + timed_out = False try: - stdout, stderr = process.communicate(timeout=self.timeout_s) - except subprocess.TimeoutExpired as exc: + process.wait(timeout=self.timeout_s) + except subprocess.TimeoutExpired: + timed_out = True process.terminate() try: - stdout, stderr = process.communicate(timeout=5) + process.wait(timeout=5) except subprocess.TimeoutExpired: process.kill() - stdout, stderr = process.communicate() + process.wait() + finally: + for reader, stream in zip( + readers, + (stdout, stderr_stream), + strict=True, + ): + reader.join(timeout=5) + if reader.is_alive(): + stream.close() + reader.join(timeout=1) + + stderr = "".join(stderr_parts) + if timed_out: raise PiRunError( f"Pi timed out after {self.timeout_s:g}s", - stderr=_bounded_stderr(stderr), - ) from exc + stderr=stderr, + ) duration = time.monotonic() - started - stderr = _bounded_stderr(stderr) - final_text, tool_count, stop_error = parse_pi_events(stdout) if process.returncode != 0: raise PiRunError(f"Pi exited with status {process.returncode}", stderr=stderr) - if stop_error is not None: - raise PiRunError(stop_error, stderr=stderr) - if final_text is None: + if events.stop_error is not None: + raise PiRunError(events.stop_error, stderr=stderr) + if events.final_text is None: raise PiRunError("Pi produced no final assistant response", stderr=stderr) transcripts = sorted(session_dir.rglob("*.jsonl")) if session_dir.exists() else [] return PiRunResult( - final_text=final_text, - tool_call_count=tool_count, + final_text=events.final_text, + tool_call_count=events.tool_count, duration_seconds=duration, transcript_path=transcripts[-1] if transcripts else None, stderr=stderr, @@ -160,33 +223,83 @@ def run( def parse_pi_events(stream: str) -> tuple[str | None, int, str | None]: """Return the final assistant text, tool count, and terminal error.""" - final_text: str | None = None - tool_count = 0 - stop_error: str | None = None + events = _PiEventAccumulator() for line in stream.splitlines(): + events.feed(line) + return events.final_text, events.tool_count, events.stop_error + + +class _PiEventAccumulator: + def __init__(self, progress: ProgressSink | None = None) -> None: + self.progress = progress + self.final_text: str | None = None + self.tool_count = 0 + self.stop_error: str | None = None + self._tool_started: dict[str, float] = {} + + def feed(self, line: str) -> None: try: event = json.loads(line) except json.JSONDecodeError: - continue + return if not isinstance(event, dict): - continue - if event.get("type") == "tool_execution_start": - tool_count += 1 - if event.get("type") != "message_end": - continue + return + event_type = event.get("type") + if event_type == "message_update": + update = event.get("assistantMessageEvent") + if isinstance(update, dict) and update.get("type") == "text_delta": + delta = update.get("delta") + if isinstance(delta, str) and delta: + emit_progress(self.progress, AssistantTextProgress(delta=delta)) + return + if event_type == "tool_execution_start": + self.tool_count += 1 + call_id = str(event.get("toolCallId", "")) + self._tool_started[call_id] = time.monotonic() + args = event.get("args") + code = args.get("code") if isinstance(args, dict) else None + if event.get("toolName") == "python_exec" and isinstance(code, str) and code: + emit_progress(self.progress, ToolStartProgress(code=code)) + return + if event_type == "tool_execution_end": + call_id = str(event.get("toolCallId", "")) + started = self._tool_started.pop(call_id, time.monotonic()) + if event.get("toolName") == "python_exec": + emit_progress( + self.progress, + ToolEndProgress( + ok=not bool(event.get("isError")), + result=_tool_result_text(event.get("result")), + duration_seconds=max(0.0, time.monotonic() - started), + ), + ) + return + if event_type != "message_end": + return message = event.get("message") if not isinstance(message, dict) or message.get("role") != "assistant": - continue - text = "".join( + return + self.final_text = "".join( str(item.get("text", "")) for item in message.get("content", []) if isinstance(item, dict) and item.get("type") == "text" ) - final_text = text + emit_progress(self.progress, FinalResponseProgress(text=self.final_text)) stop_reason = message.get("stopReason") if stop_reason in {"error", "aborted"}: - stop_error = str(message.get("errorMessage") or f"Pi request {stop_reason}") - return final_text, tool_count, stop_error + self.stop_error = str(message.get("errorMessage") or f"Pi request {stop_reason}") + + +def _tool_result_text(result: Any) -> str: + if isinstance(result, dict): + content = result.get("content") + if isinstance(content, list): + return "\n".join( + str(item.get("text", "")) + for item in content + if isinstance(item, dict) and item.get("type") == "text" + ) + return str(result) def _bounded_stderr(value: str) -> str: diff --git a/dimos/benchmark/agent_eval/single_case.py b/dimos/benchmark/agent_eval/single_case.py index 287c8e825a..14dd052aea 100644 --- a/dimos/benchmark/agent_eval/single_case.py +++ b/dimos/benchmark/agent_eval/single_case.py @@ -69,6 +69,7 @@ def execute_single_case( raise ValueError(f"API key environment variable {config.agent.api_key_env!r} is unset") bundle = _materialize_frozen_memory(case, progress) _, cutoff, source_path, derived_path = load_bundle(bundle, progress=case.source.progress) + emit_progress(progress, StatusProgress(channel="eval", message="memory ready")) cli, extension = _pi_paths() runner = PiCliRunner( cli=cli, @@ -76,6 +77,7 @@ def execute_single_case( model=config.agent.model, thinking_level=config.agent.thinking_level, timeout_s=TURN_TIMEOUT_SECONDS, + progress=progress, ) output.parent.mkdir(parents=True, exist_ok=True) @@ -186,6 +188,10 @@ def _materialize_frozen_memory(case: EvalCase, progress: ProgressSink | None) -> bundle, progress=[case.source.progress], mapper=mapper, + map_progress=lambda current, total: emit_progress( + progress, + StatusProgress(channel="eval", message=f"mapping {current}/{total} frames"), + ), ) return bundle diff --git a/dimos/benchmark/agent_eval/test_pi_process.py b/dimos/benchmark/agent_eval/test_pi_process.py index bc8c8df892..b7de377df9 100644 --- a/dimos/benchmark/agent_eval/test_pi_process.py +++ b/dimos/benchmark/agent_eval/test_pi_process.py @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from io import StringIO import json from pathlib import Path import subprocess +import threading import pytest @@ -39,13 +41,103 @@ def test_parse_stock_pi_events_uses_final_message_and_counts_tools() -> None: assert error is None +def test_stock_cli_streams_assistant_tools_and_stderr_while_running(mocker, tmp_path: Path) -> None: + cli = tmp_path / "cli.js" + extension = tmp_path / "extension.js" + cli.touch() + extension.touch() + process = mocker.Mock(returncode=0) + process.stdout = StringIO( + "\n".join( + json.dumps(event) + for event in ( + { + "type": "message_update", + "assistantMessageEvent": {"type": "text_delta", "delta": "Checking"}, + }, + { + "type": "tool_execution_start", + "toolCallId": "call-1", + "toolName": "python_exec", + "args": {"code": "memory.list_streams()"}, + }, + { + "type": "tool_execution_end", + "toolCallId": "call-1", + "toolName": "python_exec", + "result": {"content": [{"type": "text", "text": "['lidar']"}]}, + "isError": False, + }, + { + "type": "message_end", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": "ANSWER: 2"}], + "stopReason": "stop", + }, + }, + ) + ) + ) + process.stderr = StringIO("provider secret connected\n") + mocker.patch("dimos.benchmark.agent_eval.pi_process.subprocess.Popen", return_value=process) + progress = [] + assistant_seen = threading.Event() + + def observe(event) -> None: + progress.append(event) + if event.kind == "assistant_text": + assistant_seen.set() + + def wait_for_process(*, timeout) -> int: + assert timeout == 10 + assert assistant_seen.wait(timeout=1) + return 0 + + process.wait.side_effect = wait_for_process + runner = PiCliRunner( + cli=cli, + extension=extension, + model="gpt-5.6-luna", + thinking_level="medium", + timeout_s=10, + progress=observe, + ) + + result = runner.run( + prompt="Count", + system_prompt="Use memory", + mcp_url="http://127.0.0.1:1234/mcp", + api_key="secret", + run_dir=tmp_path, + ) + + assert [(event.kind, getattr(event, "delta", None)) for event in progress[:1]] == [ + ("assistant_text", "Checking") + ] + structured = [event for event in progress if event.kind != "status"] + assert [event.kind for event in structured] == [ + "assistant_text", + "tool_start", + "tool_end", + "final_response", + ] + assert structured[1].code == "memory.list_streams()" + assert structured[2].result == "['lidar']" + assert [event.message for event in progress if event.kind == "status"] == [ + "provider [REDACTED] connected" + ] + assert result.stderr == "provider [REDACTED] connected\n" + assert result.final_text == "ANSWER: 2" + + def test_stock_cli_receives_only_api_key_and_evaluator_binding(mocker, tmp_path: Path) -> None: cli = tmp_path / "cli.js" extension = tmp_path / "extension.js" cli.touch() extension.touch() process = mocker.Mock(returncode=0) - process.communicate.return_value = ( + process.stdout = StringIO( json.dumps( { "type": "message_end", @@ -55,9 +147,10 @@ def test_stock_cli_receives_only_api_key_and_evaluator_binding(mocker, tmp_path: "stopReason": "stop", }, } - ), - "", + ) ) + process.stderr = StringIO() + process.wait.return_value = 0 popen = mocker.patch( "dimos.benchmark.agent_eval.pi_process.subprocess.Popen", return_value=process ) @@ -92,9 +185,11 @@ def test_stock_cli_timeout_terminates_the_child(mocker, tmp_path: Path) -> None: cli.touch() extension.touch() process = mocker.Mock() - process.communicate.side_effect = [ + process.stdout = StringIO() + process.stderr = StringIO("stopped") + process.wait.side_effect = [ subprocess.TimeoutExpired("pi", 0.01), - ("", "stopped"), + 0, ] mocker.patch("dimos.benchmark.agent_eval.pi_process.subprocess.Popen", return_value=process) runner = PiCliRunner( diff --git a/dimos/benchmark/short_horizon_qa/prepare.py b/dimos/benchmark/short_horizon_qa/prepare.py index 32d53e160f..90351501c8 100644 --- a/dimos/benchmark/short_horizon_qa/prepare.py +++ b/dimos/benchmark/short_horizon_qa/prepare.py @@ -16,6 +16,7 @@ from __future__ import annotations +from collections.abc import Callable import json import math import os @@ -47,6 +48,7 @@ def prepare_bundle( *, progress: list[float] | None = None, mapper: MapperSettings = MapperSettings(), + map_progress: Callable[[int, int], None] | None = None, ) -> FrozenMemoryManifest: """Build one derived map sidecar without copying the source recording.""" cutoffs = _validate_seconds(cutoff_seconds or []) @@ -73,6 +75,7 @@ def prepare_bundle( progresses, temporary_path, mapper, + map_progress, ) os.replace(temporary_path, output) return manifest @@ -85,6 +88,7 @@ def _prepare_into( progresses: list[float], output: Path, mapper: MapperSettings, + map_progress: Callable[[int, int], None] | None, ) -> FrozenMemoryManifest: derived_path = output / DERIVED_NAME with SqliteStore(path=str(source_path), must_exist=True, read_only=True) as source: @@ -109,7 +113,13 @@ def _prepare_into( raise ValueError( f"Cutoff {selections[-1].seconds}s exceeds recording duration {duration:.3f}s" ) - cutoff_maps = _write_maps(source, derived_path, absolute_cutoffs, mapper) + cutoff_maps = _write_maps( + source, + derived_path, + absolute_cutoffs, + mapper, + map_progress, + ) records = tuple( CutoffRecord( cutoff_seconds=selection.seconds, @@ -206,10 +216,12 @@ def _write_maps( derived_path: Path, cutoffs: list[float], mapper: MapperSettings, + map_progress: Callable[[int, int], None] | None, ) -> list[Any]: if "lidar" not in source.list_streams(): raise ValueError("Source recording has no lidar stream") lidar = source.stream("lidar", PointCloud2) + total_frames = lidar.count() first = next(iter(lidar), None) if first is None: raise ValueError("No lidar observations exist before the final cutoff") @@ -227,8 +239,20 @@ def _write_maps( show_startup_log=False, ) emissions = iter(lidar.transform(transformer)) + last_reported = 0 + + def next_emission() -> Any: + nonlocal last_reported + observation = next(emissions, None) + if observation is not None and map_progress is not None: + frame_count = int(observation.tags["frame_count"]) + if frame_count == total_frames or frame_count - last_reported >= 250: + map_progress(frame_count, total_frames) + last_reported = frame_count + return observation + try: - latest = next(emissions, None) + latest = next_emission() if latest is None: raise ValueError("Mapper produced no global map") @@ -236,11 +260,11 @@ def _write_maps( with SqliteStore(path=str(derived_path)) as derived: target = derived.stream("global_map", PointCloud2) stored_by_source_id: dict[int, Any] = {} - following = next(emissions, None) + following = next_emission() for cutoff in cutoffs: while following is not None and following.ts <= cutoff: latest = following - following = next(emissions, None) + following = next_emission() if latest.ts > cutoff: raise ValueError(f"No runtime map was emitted by cutoff {cutoff}") source_key = latest.id diff --git a/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py b/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py index a3139897cc..0c7def1bfa 100644 --- a/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py +++ b/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py @@ -30,13 +30,16 @@ def test_real_hongkong_recording_prepares_direct_demo_case(tmp_path: Path) -> No Path(__file__).parent / "cases" / "demo_go2_hongkong_office-room-count-smoke" / "case.json" ) case = EvalCase.model_validate_json(case_path.read_bytes()) + map_progress: list[tuple[int, int]] = [] manifest = prepare_bundle( get_data("go2_hongkong_office.db"), [], tmp_path / "bundle", progress=[case.source.progress], mapper=MapperSettings(device="CPU:0"), + map_progress=lambda current, total: map_progress.append((current, total)), ) assert case.case_id == "demo-go2-hongkong-office-room-count-smoke" assert manifest.cutoffs[0].normalized_progress == 1.0 assert manifest.cutoffs[0].map_frame_count == 4235 + assert map_progress[-1] == (4235, 4235) diff --git a/dimos/benchmark/short_horizon_qa/test_prepare.py b/dimos/benchmark/short_horizon_qa/test_prepare.py index dca25bbf68..3a26330b33 100644 --- a/dimos/benchmark/short_horizon_qa/test_prepare.py +++ b/dimos/benchmark/short_horizon_qa/test_prepare.py @@ -57,17 +57,20 @@ def test_prepare_builds_reusable_runtime_maps_without_changing_source( ) -> None: output = tmp_path / "bundle" before = recording.read_bytes() + progress: list[tuple[int, int]] = [] manifest = prepare_bundle( recording, [4.0, 9.0], output, mapper=MapperSettings(device="CPU:0"), + map_progress=lambda current, total: progress.append((current, total)), ) assert recording.read_bytes() == before assert [item.map_frame_count for item in manifest.cutoffs] == [5, 10] assert [item.map_timestamp for item in manifest.cutoffs] == [104.0, 109.0] + assert progress == [(10, 10)] assert (output / "derived.db").is_file() encoded = json.loads((output / "manifest.v1.json").read_text()) assert encoded["source_size_bytes"] == recording.stat().st_size From d963021bdd026ef7c1f1cb36aefb67e82f456910 Mon Sep 17 00:00:00 2001 From: cc Date: Wed, 5 Aug 2026 16:11:54 -0700 Subject: [PATCH 10/15] fix: reduce evaluation log noise --- dimos/agents/code_policy_server.py | 8 +++++++ dimos/agents/test_code_policy_server.py | 32 +++++++++++++++---------- dimos/cli/eval.py | 13 +++++++++- dimos/cli/test_eval.py | 25 ++++++++++++++++++- 4 files changed, 63 insertions(+), 15 deletions(-) diff --git a/dimos/agents/code_policy_server.py b/dimos/agents/code_policy_server.py index 5cf0704e7d..70b89de7dd 100644 --- a/dimos/agents/code_policy_server.py +++ b/dimos/agents/code_policy_server.py @@ -17,6 +17,7 @@ from __future__ import annotations import asyncio +import logging import socket import threading import time @@ -37,6 +38,11 @@ the answer; do not guess from the prompt. """ +_NOISY_MCP_TRANSPORT_LOGGERS = ( + "mcp.server.streamable_http", + "mcp.server.streamable_http_manager", +) + class CodePolicyMcpServer: """Own the CodePolicy session and an official MCP HTTP server in one process.""" @@ -79,6 +85,8 @@ def mcp_url(self) -> str: def start(self) -> None: if self._thread is not None: raise RuntimeError("CodePolicy MCP server is already running") + for logger_name in _NOISY_MCP_TRANSPORT_LOGGERS: + logging.getLogger(logger_name).setLevel(logging.WARNING) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind((self.host, 0)) diff --git a/dimos/agents/test_code_policy_server.py b/dimos/agents/test_code_policy_server.py index 4f58bd885d..66964aff21 100644 --- a/dimos/agents/test_code_policy_server.py +++ b/dimos/agents/test_code_policy_server.py @@ -15,6 +15,7 @@ from __future__ import annotations import asyncio +import logging from pathlib import Path from mcp import Client @@ -42,20 +43,25 @@ def _config(tmp_path: Path) -> CodePolicySessionConfig: @pytest.mark.asyncio -async def test_server_exposes_exactly_one_persistent_python_tool(tmp_path: Path) -> None: +async def test_server_exposes_exactly_one_persistent_python_tool( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: server = CodePolicyMcpServer(_config(tmp_path)) - server.start() - try: - async with Client(server.mcp_url) as client: - tools = await client.list_tools() - assert [tool.name for tool in tools.tools] == ["python_exec"] - first = await client.call_tool("python_exec", {"code": "items = [1]\nitems"}) - second = await client.call_tool("python_exec", {"code": "items.append(2)\nitems"}) - assert "[1]" in first.content[0].text - assert "[1, 2]" in second.content[0].text - assert server.session.execution_count == 2 - finally: - await asyncio.to_thread(server.stop) + with caplog.at_level(logging.INFO): + server.start() + try: + async with Client(server.mcp_url) as client: + tools = await client.list_tools() + assert [tool.name for tool in tools.tools] == ["python_exec"] + first = await client.call_tool("python_exec", {"code": "items = [1]\nitems"}) + second = await client.call_tool("python_exec", {"code": "items.append(2)\nitems"}) + assert "[1]" in first.content[0].text + assert "[1, 2]" in second.content[0].text + assert server.session.execution_count == 2 + finally: + await asyncio.to_thread(server.stop) + assert "StreamableHTTP session manager started" not in caplog.text + assert "Terminating session" not in caplog.text @pytest.mark.asyncio diff --git a/dimos/cli/eval.py b/dimos/cli/eval.py index 1286089673..d3517a5c2a 100644 --- a/dimos/cli/eval.py +++ b/dimos/cli/eval.py @@ -24,6 +24,8 @@ app = typer.Typer(help="Run immutable agent evaluation cases", no_args_is_help=True) +MAX_RENDERED_TOOL_RESULT_CHARS = 2_000 + def execute_single_case(*args: Any, **kwargs: Any) -> Any: """Import and dispatch the evaluation runtime only when ``eval run`` executes.""" @@ -117,6 +119,8 @@ def __init__(self) -> None: def __call__(self, event: Any) -> None: with self._lock: if event.kind == "assistant_text": + if not self._assistant_open and not event.delta.strip(): + return if not self._assistant_open: typer.echo("[pi] ", err=True, nl=False) self._assistant_open = True @@ -142,7 +146,7 @@ def __call__(self, event: Any) -> None: status = "ok" if event.ok else "error" typer.echo(f"[python_exec] {status} ({event.duration_seconds:.1f}s)", err=True) if event.result: - typer.echo(_indent(event.result), err=True) + typer.echo(_indent(_truncate_tool_result(event.result)), err=True) elif event.kind == "final_response" and not self._saw_assistant_text: typer.echo(f"[pi] {event.text}", err=True) @@ -158,3 +162,10 @@ def _end_assistant_line(self) -> None: def _indent(value: str) -> str: return "\n".join(f" {line}" for line in value.splitlines()) + + +def _truncate_tool_result(value: str) -> str: + if len(value) <= MAX_RENDERED_TOOL_RESULT_CHARS: + return value + visible = value[:MAX_RENDERED_TOOL_RESULT_CHARS].rstrip() + return f"{visible}\n... [terminal output truncated; full result retained]" diff --git a/dimos/cli/test_eval.py b/dimos/cli/test_eval.py index 78e6fc415f..e7fc36c801 100644 --- a/dimos/cli/test_eval.py +++ b/dimos/cli/test_eval.py @@ -23,7 +23,11 @@ from typer.testing import CliRunner from dimos.benchmark.agent_eval.models import CompactEvalResult -from dimos.benchmark.agent_eval.progress import StatusProgress +from dimos.benchmark.agent_eval.progress import ( + AssistantTextProgress, + StatusProgress, + ToolEndProgress, +) from dimos.cli.dimos import main import dimos.cli.eval as eval_cli @@ -164,3 +168,22 @@ def find_spec(self, fullname, path=None, target=None): [sys.executable, "-c", script], capture_output=True, text=True, check=False ) assert completed.returncode == 0, completed.stdout + completed.stderr + + +def test_progress_renderer_ignores_leading_assistant_whitespace(capsys) -> None: + renderer = eval_cli.ProgressRenderer() + + renderer(AssistantTextProgress(delta="\n")) + + assert capsys.readouterr().err == "" + + +def test_progress_renderer_truncates_large_tool_results(capsys) -> None: + renderer = eval_cli.ProgressRenderer() + result = "x" * 10_000 + + renderer(ToolEndProgress(ok=True, result=result, duration_seconds=0.1)) + + rendered = capsys.readouterr().err + assert len(rendered) < len(result) + assert "terminal output truncated" in rendered From 537c9f4b0134b55d186fef0842e805884cbfbb16 Mon Sep 17 00:00:00 2001 From: cc Date: Wed, 5 Aug 2026 16:17:13 -0700 Subject: [PATCH 11/15] spec: remove --- .../extract-frozen-qa-eval/.openspec.yaml | 2 - .../changes/extract-frozen-qa-eval/README.md | 3 - .../changes/extract-frozen-qa-eval/design.md | 102 ------------------ .../changes/extract-frozen-qa-eval/docs.md | 15 --- .../extract-frozen-qa-eval/proposal.md | 35 ------ .../specs/frozen-agent-evaluation/spec.md | 71 ------------ .../specs/frozen-memory-views/spec.md | 42 -------- .../standalone-code-policy-runtime/spec.md | 56 ---------- .../changes/extract-frozen-qa-eval/tasks.md | 46 -------- 9 files changed, 372 deletions(-) delete mode 100644 openspec/changes/extract-frozen-qa-eval/.openspec.yaml delete mode 100644 openspec/changes/extract-frozen-qa-eval/README.md delete mode 100644 openspec/changes/extract-frozen-qa-eval/design.md delete mode 100644 openspec/changes/extract-frozen-qa-eval/docs.md delete mode 100644 openspec/changes/extract-frozen-qa-eval/proposal.md delete mode 100644 openspec/changes/extract-frozen-qa-eval/specs/frozen-agent-evaluation/spec.md delete mode 100644 openspec/changes/extract-frozen-qa-eval/specs/frozen-memory-views/spec.md delete mode 100644 openspec/changes/extract-frozen-qa-eval/specs/standalone-code-policy-runtime/spec.md delete mode 100644 openspec/changes/extract-frozen-qa-eval/tasks.md diff --git a/openspec/changes/extract-frozen-qa-eval/.openspec.yaml b/openspec/changes/extract-frozen-qa-eval/.openspec.yaml deleted file mode 100644 index 46bad6b578..0000000000 --- a/openspec/changes/extract-frozen-qa-eval/.openspec.yaml +++ /dev/null @@ -1,2 +0,0 @@ -schema: dimos-capability -created: 2026-08-05 diff --git a/openspec/changes/extract-frozen-qa-eval/README.md b/openspec/changes/extract-frozen-qa-eval/README.md deleted file mode 100644 index 44a639aec6..0000000000 --- a/openspec/changes/extract-frozen-qa-eval/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# extract-frozen-qa-eval - -Extract the frozen short-horizon QA evaluation path from cc/frontier onto main as a focused, dependency-safe dimos eval run capability. diff --git a/openspec/changes/extract-frozen-qa-eval/design.md b/openspec/changes/extract-frozen-qa-eval/design.md deleted file mode 100644 index 44e42566b8..0000000000 --- a/openspec/changes/extract-frozen-qa-eval/design.md +++ /dev/null @@ -1,102 +0,0 @@ -## Context - -The target is one source-checkout command: - -```bash -uv run dimos eval run \ - dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/case.json \ - --output=/tmp/dimos-eval-smoke -``` - -The first implementation on PR #3378 was intentionally defensive, but it produced 17 attempt files and three copies of the agent trajectory. Review established that Memory2 already owns environmental observations and Pi already owns its native transcript. The evaluator needs only a compact result beside that transcript. - -## Goals - -- Keep the direct command small and understandable. -- Preserve extension seams only for source, task, validator, and the shared CodePolicy session. -- Prevent writes to source and derived Memory2 databases. -- Keep the private oracle and API key outside the model-facing namespace. -- Use official MCP transports and Pi's stock process lifecycle. -- Leave no child process after normal completion, caught failure, timeout, or interruption. -- Pass the repository's existing CI with focused Python and Node unit tests. - -## Non-Goals - -- Generic attempt engines, adapter registries, dataset scheduling, retries, or distributed execution. -- Crash-forensic event logs, artifact descriptors, cryptographic result attestations, or replay guarantees. -- OAuth, live robot RPCs, simulation, DimSim, blueprints, or agentic module integration. -- Migrating the existing repository-wide DimOS MCP implementation. -- Treating Jupyter or read-only Memory2 as a hostile-code sandbox. - -## Architecture - -```text -case.json + private oracle - | - v -frozen bundle cache -----> FrozenMemoryStore - | - v -evaluator process -----> CodePolicySession -----> Jupyter kernel child - | ^ - | official MCP server | python_exec - v | -stock Pi CLI child ---- tiny official-MCP extension - | - v -native JSON events + transcript -> parse -> private score -> result.json -``` - -### Python modules - -- `dimos/agents/code_policy.py`: the reusable Jupyter session, environment bootstrap, timeout recovery, and credential-scrubbed kernel launch. It knows nothing about MCP, Pi, or evaluation. -- `dimos/memory2/store/frozen.py`: source/derived overlay and inclusive cutoff. -- `dimos/benchmark/agent_eval/models.py`: strict tagged case and compact result contracts. -- `dimos/benchmark/short_horizon_qa/prepare.py`: recording resolution and derived map cache. -- `dimos/benchmark/short_horizon_qa/eval.py`: in-process MCP host, stock Pi launcher/event parser, answer parser, private scorer, cleanup, and output publication. -- `dimos/cli/eval.py`: dependency-light Typer shell with callback-local runtime imports. - -Small helpers may remain separate only when they own a concrete lifecycle boundary; generic artifact, engine, broker, and Protocol layers are removed. - -### Official MCP - -Pin the official Python `mcp==2.0.0` SDK and register one `python_exec` tool. The evaluator pre-binds a loopback socket to port `0`, supplies it to Uvicorn, and runs the SDK's ASGI application on a server thread. The evaluator directly owns server shutdown and the `CodePolicySession`; no `/control` HTTP surface or extra Python process exists. - -Pin `@modelcontextprotocol/client==2.0.0` in the Node extension. The extension validates the one-tool inventory, calls `python_exec`, uses a timeout longer than the kernel execution timeout, and closes the MCP session during Pi shutdown. - -### Stock Pi process - -Launch pinned Pi `0.80.10` with `--mode json`, built-in tools disabled, and one explicit extension. Python consumes Pi's official JSON event stream and inspects the final assistant stop reason rather than trusting the process exit code alone. The Pi-native session JSONL is the sole trajectory record. - -API-key material is passed only in the Pi subprocess environment. It is absent from argv and removed from the environment supplied to the Jupyter kernel. - -### Output - -`--output` is the exact directory for one run. A non-empty target fails preflight. Work occurs in a sibling temporary directory and is atomically renamed on completion or caught failure. - -Published files are: - -- `result.json`: case/source/model, response, prediction, score, timing, tool count, and optional infrastructure error. -- `transcript.jsonl`: Pi's native session when available. -- `stderr.log`: bounded Node diagnostics only when nonempty. - -There are no nested attempt IDs, locks, manifests, copied cases/oracles/cache manifests, hashes, lifecycle logs, or duplicate call records. - -## Decisions - -1. **Lean feature-specific runner.** A second evaluator can justify a generic engine later. -2. **Real shared CodePolicy core.** This PR owns the production Jupyter session; experimental PR #3259 can later wrap it as a DimOS module. -3. **In-process MCP, Jupyter child.** Jupyter already supplies the execution process boundary, interrupt, restart, and shutdown. -4. **Official SDKs on both sides.** The new standalone boundary does not reuse or expand DimOS's legacy hand-written MCP transport. -5. **Stock Pi JSON mode.** A tiny extension replaces the custom adapter and broker protocol. -6. **Minimal durable output.** Memory2 owns observations; Pi owns trajectory; `result.json` owns scoring. -7. **Minimal Memory2 edits.** Read-only connection propagation is required because existing constructors configure WAL and create tables. Existing time-range filtering supplies the inclusive cutoff; general stream/filter APIs are not expanded. -8. **No cryptographic framework.** Strict parsing, safe paths, and cache metadata are sufficient for this local demo stage. -9. **API key only.** Default to `OPENAI_API_KEY`, with a named environment override for later extension. -10. **Existing CI plus required Node tests.** Python test/lint groups receive the minimal runtime dependencies; the small Node job gates the aggregate check. - -## Safety and cleanup - -The evaluator owns cleanup in one `finally`: stop Pi, stop the MCP server thread, and shut down Jupyter. Timeouts escalate Pi terminate to kill. The kernel environment is scrubbed of common credential variables. Private oracle values never enter prompts, MCP metadata, transcripts, results, or stderr. - -CodePolicy remains trusted unsandboxed execution because kernel code may access the host filesystem and start processes. Documentation must state this directly. diff --git a/openspec/changes/extract-frozen-qa-eval/docs.md b/openspec/changes/extract-frozen-qa-eval/docs.md deleted file mode 100644 index 0ba3a41811..0000000000 --- a/openspec/changes/extract-frozen-qa-eval/docs.md +++ /dev/null @@ -1,15 +0,0 @@ -## User-facing docs - -- Document the direct `dimos eval run CASE --output=DIR` command. -- Document the pinned Pi extension build and API-key environment selection. -- Explain the three exit codes and the compact atomic output directory. -- State prominently that CodePolicy runs trusted, unsandboxed Python. -- Preserve the demo fixture warning that oracle `0` is a plumbing sentinel. - -## Contributor docs - -- Document focused Python tests and the one-file Node extension test. -- Document the self-hosted Hong Kong preparation gate. - -No blueprint, generated registry, hardware, or `AGENTS.md` documentation changes -are required. diff --git a/openspec/changes/extract-frozen-qa-eval/proposal.md b/openspec/changes/extract-frozen-qa-eval/proposal.md deleted file mode 100644 index c89d291189..0000000000 --- a/openspec/changes/extract-frozen-qa-eval/proposal.md +++ /dev/null @@ -1,35 +0,0 @@ -## Why - -DimOS needs one direct command that asks an agent an integer question about a frozen Memory2 recording. The first extraction proved the path, but review showed that it introduced a generic evaluation framework, a custom Node protocol, duplicated evidence, and integrity machinery before those abstractions had a second use. - -This revision keeps the production seams that matter: true read-only Memory2, a reusable Jupyter-backed CodePolicy session, official MCP libraries, a stock Pi process, strict case/result models, and private scoring. It removes the custom orchestration and audit framework. - -## What Changes - -- Add `dimos eval run CASE --output=DIR` for one synchronous frozen QA run. -- Keep a compact tagged case model for source, task, and validator kinds. -- Add true read-only source/derived Memory2 views with an inclusive cutoff. -- Add a module-independent Jupyter `CodePolicySession` and expose its sole `python_exec` operation through the official Python MCP SDK in the evaluator process. -- Launch the stock Pi CLI in one-shot JSON mode with one small TypeScript extension using the official MCP client. -- Support API-key authentication through a named environment variable only. -- Publish only `result.json`, the native Pi transcript, and failure-only stderr in one non-overwriting output directory. -- Rename the Hong Kong fixture and case ID with `demo_`/`demo-` to make its synthetic `0` oracle unmistakable. -- Remove fingerprints, artifact manifests, lifecycle logs, broker logs, duplicated execution records, and the custom Python/Node protocol. - -## Capabilities - -### New Capabilities - -- `frozen-agent-evaluation`: one-case CLI execution, private integer scoring, compact output, and exit behavior. -- `frozen-memory-views`: read-only source/derived Memory2 access through an inclusive authored cutoff. -- `standalone-code-policy-runtime`: reusable Jupyter session plus an in-process official MCP adapter and stock Pi CLI integration. - -### Modified Capabilities - -None. - -## Impact - -The change affects Memory2 SQLite opening, agent optional dependencies, the DimOS CLI, one focused benchmark package, a small Node extension package, CI dependencies, and agent documentation. It adds no blueprint, robot skill, live evaluation path, simulator, or generated blueprint entry. - -CodePolicy remains trusted unsandboxed Python. The Jupyter kernel receives a scrubbed environment without API credentials, but it is not an operating-system sandbox. diff --git a/openspec/changes/extract-frozen-qa-eval/specs/frozen-agent-evaluation/spec.md b/openspec/changes/extract-frozen-qa-eval/specs/frozen-agent-evaluation/spec.md deleted file mode 100644 index b11cad93f8..0000000000 --- a/openspec/changes/extract-frozen-qa-eval/specs/frozen-agent-evaluation/spec.md +++ /dev/null @@ -1,71 +0,0 @@ -## ADDED Requirements - -### Requirement: One-case frozen evaluation command -DimOS SHALL provide `dimos eval run CASE --output=DIR` for one synchronous frozen-memory integer-question case. The case SHALL use strict tagged source, task, and validator models and SHALL reject unknown fields, unsafe oracle paths, non-finite progress, and unsupported kinds. - -#### Scenario: Run a supported case -- **GIVEN** a valid case, prepared recording, built Pi extension, and API-key environment variable -- **WHEN** the user runs `dimos eval run CASE --output=DIR` -- **THEN** DimOS runs one fresh agent turn and privately scores its terminal integer answer -- **AND** publishes a compact result even when the semantic score fails - -#### Scenario: Reject an unsafe case -- **GIVEN** a case with an escaped oracle path, invalid progress, unknown field, or unsupported kind -- **WHEN** preflight parses the case -- **THEN** the command exits `2` before starting CodePolicy or Pi - -### Requirement: API-key-only authentication -The evaluator SHALL read an OpenAI API key from `OPENAI_API_KEY` or a user-selected environment variable. The key MUST be passed only to the Pi subprocess environment and MUST NOT appear in argv, the Jupyter kernel environment, MCP data, results, transcripts, stderr, or cache metadata. - -#### Scenario: Run with the default key environment -- **GIVEN** a nonempty `OPENAI_API_KEY` -- **WHEN** the evaluator launches Pi -- **THEN** Pi receives the value through its environment -- **AND** CodePolicy cannot read that value from the Jupyter kernel environment - -### Requirement: Exact private integer scoring -The evaluator SHALL accept only final text with exactly one marker ending in `ANSWER: `. The private oracle SHALL be loaded only by the scorer and SHALL never enter the model-facing prompt or runtime. - -#### Scenario: Score a valid answer -- **GIVEN** final text ending in exactly one `ANSWER: 4` -- **WHEN** the private oracle expects `4` -- **THEN** the compact result records parsed integer `4` and task result `passed` - -#### Scenario: Score malformed or mismatched output -- **GIVEN** a missing, repeated, non-integer, trailing, or mismatched answer -- **WHEN** scoring completes -- **THEN** the run is a completed semantic failure -- **AND** the command exits `0` - -### Requirement: Compact non-overwriting output -The `--output` value SHALL be the exact directory for one run. A nonempty target SHALL fail preflight. The evaluator SHALL atomically publish `result.json`, the Pi-native transcript when available, and bounded nonempty Node stderr when present. It SHALL NOT copy source databases, cache manifests, case files, oracle files, prompts, MCP inventories, kernel records, or duplicate call logs. - -#### Scenario: Publish a completed run -- **GIVEN** an unused output path and a completed agent turn -- **WHEN** scoring finishes -- **THEN** `result.json` contains case/source/model, final response, prediction, score, duration, and tool count -- **AND** the native Pi transcript is the sole tool/assistant trajectory - -#### Scenario: Publish a caught infrastructure failure -- **GIVEN** Pi or CodePolicy fails after preflight -- **WHEN** cleanup completes -- **THEN** `result.json` records an infrastructure error and the command exits `1` -- **AND** any available native transcript or nonempty bounded stderr is retained - -### Requirement: Output channel contract -The final human or JSON result SHALL go to stdout. Coarse runtime status SHALL go to stderr and `--quiet` SHALL suppress it. Credentials and oracle contents MUST NOT appear on either channel. - -#### Scenario: Consume JSON output -- **GIVEN** `--json` with progress enabled -- **WHEN** a run finishes -- **THEN** stdout contains exactly one JSON value -- **AND** coarse status appears only on stderr - -### Requirement: Demo fixture identity -The shipped fixture directory SHALL start with `demo_`, its case ID SHALL start with `demo-`, and its README SHALL state that oracle value `0` tests plumbing rather than authoritative room-count accuracy. - -#### Scenario: Agent disagrees with the demo oracle -- **GIVEN** a completed answer other than `0` -- **WHEN** the demo case scores the answer -- **THEN** it reports semantic failure with exit `0` -- **AND** documentation does not characterize the result as an agent or mapping regression diff --git a/openspec/changes/extract-frozen-qa-eval/specs/frozen-memory-views/spec.md b/openspec/changes/extract-frozen-qa-eval/specs/frozen-memory-views/spec.md deleted file mode 100644 index 646b1a1e02..0000000000 --- a/openspec/changes/extract-frozen-qa-eval/specs/frozen-memory-views/spec.md +++ /dev/null @@ -1,42 +0,0 @@ -## ADDED Requirements - -### Requirement: True read-only SQLite access -Frozen Memory2 source and derived databases SHALL use SQLite URI read-only mode and `PRAGMA query_only=ON`. Read-only initialization SHALL skip WAL configuration and table creation. Public frozen-store mutation operations SHALL fail, and reading SHALL not create database sidecars. - -#### Scenario: Read without mutation -- **GIVEN** existing source and derived SQLite stores -- **WHEN** a frozen view opens and reads them -- **THEN** the databases remain byte-identical -- **AND** no WAL or SHM sidecar is created - -#### Scenario: Reject mutation -- **GIVEN** an open frozen view -- **WHEN** a caller creates, retypes, deletes, or appends to a stream -- **THEN** the operation fails as read-only - -### Requirement: Inclusive cutoff overlay -A frozen view SHALL expose the deterministic union of non-colliding source and derived stream names. Every returned stream SHALL use existing time-range filtering to include observations with timestamps `<= cutoff` and hide later observations. - -#### Scenario: Read the cutoff boundary -- **GIVEN** observations before, exactly at, and after the cutoff -- **WHEN** the frozen stream is read -- **THEN** the first two observations are visible -- **AND** the later observation is hidden - -#### Scenario: Reject collision -- **GIVEN** source and derived stores containing the same stream name -- **WHEN** the overlay is created -- **THEN** construction fails and identifies the collision - -### Requirement: Metadata-based frozen bundle cache -Bundle preparation SHALL resolve progress over the recording range and cache a derived `global_map`. Cache reuse SHALL compare recording identity, source file size/mtime, mapper settings, progress, and cutoff metadata. It SHALL not hash full database files. - -#### Scenario: Reuse an unchanged bundle -- **GIVEN** matching source metadata, mapper settings, and normalized progress -- **WHEN** preparation runs again -- **THEN** it reuses the derived bundle without remapping or hashing the databases - -#### Scenario: Rebuild stale metadata -- **GIVEN** changed source metadata, mapper settings, or cutoff inputs -- **WHEN** preparation runs -- **THEN** it rebuilds the derived bundle before evaluation diff --git a/openspec/changes/extract-frozen-qa-eval/specs/standalone-code-policy-runtime/spec.md b/openspec/changes/extract-frozen-qa-eval/specs/standalone-code-policy-runtime/spec.md deleted file mode 100644 index 0636483da1..0000000000 --- a/openspec/changes/extract-frozen-qa-eval/specs/standalone-code-policy-runtime/spec.md +++ /dev/null @@ -1,56 +0,0 @@ -## ADDED Requirements - -### Requirement: Shared Jupyter CodePolicy session -DimOS SHALL provide a module-independent `CodePolicySession` that owns one Jupyter kernel, persistent namespace, serialized execution, timeout interruption, restart recovery, and bounded shutdown. Frozen bootstrap SHALL expose read-only `memory`, omit live `app`, and scrub credentials from the kernel environment. - -#### Scenario: Start a frozen session -- **GIVEN** prepared source and derived Memory2 paths -- **WHEN** a frozen CodePolicy session starts -- **THEN** `memory` is available in the persistent namespace -- **AND** `app` and API-key environment variables are absent - -#### Scenario: Recover from timeout -- **GIVEN** an execution exceeds its timeout -- **WHEN** CodePolicy interrupts it -- **THEN** the session restarts and re-applies the frozen bootstrap -- **AND** later calls execute in a usable clean kernel - -### Requirement: In-process official MCP server -The evaluator SHALL expose CodePolicy through the official Python MCP SDK running in the evaluator process. It SHALL register exactly one tool named `python_exec`, bind a pre-created loopback port-`0` socket, and stop through direct evaluator ownership. It SHALL expose no HTTP control API. - -#### Scenario: Start without a port race -- **GIVEN** a new evaluator run -- **WHEN** it starts MCP -- **THEN** the evaluator binds the socket before serving and passes the actual URL to Pi -- **AND** the client observes exactly one tool named `python_exec` - -### Requirement: Stock Pi CLI and official MCP extension -The evaluator SHALL launch pinned Pi `0.80.10` in one-shot JSON mode with built-in tools, implicit extensions, skills, prompt templates, themes, and context files disabled. One explicit extension using `@modelcontextprotocol/client==2.0.0` SHALL register `python_exec` and call the official Python MCP server directly. - -#### Scenario: Complete one Pi turn -- **GIVEN** an initialized one-tool MCP server -- **WHEN** the stock Pi CLI receives the authored question -- **THEN** it emits official JSON events and persists its native session transcript -- **AND** Python derives final text, stop reason, and tool count without a custom protocol - -#### Scenario: Reject tool drift -- **GIVEN** the extension observes an MCP inventory other than exactly `python_exec` -- **WHEN** Pi initializes the extension -- **THEN** startup fails before the model turn - -### Requirement: Bounded cleanup -The evaluator SHALL stop Pi before stopping MCP and Jupyter. Pi timeout SHALL escalate terminate to kill. Normal completion, caught failure, interruption, and partial startup SHALL leave no live Pi process, MCP server thread, or Jupyter kernel. - -#### Scenario: Pi fails during startup -- **GIVEN** MCP and Jupyter have started but Pi fails -- **WHEN** the evaluator handles the error -- **THEN** it stops the server thread and kernel within bounded deadlines -- **AND** publishes a compact infrastructure failure result - -### Requirement: Trusted execution disclosure -Documentation SHALL describe CodePolicy as trusted persistent unsandboxed Python. Read-only Memory2 and a scrubbed environment SHALL not be presented as an operating-system sandbox. - -#### Scenario: Read the operator guide -- **GIVEN** a user preparing a frozen evaluation -- **WHEN** they read its safety section -- **THEN** they are warned to use an external OS sandbox or container for hostile code diff --git a/openspec/changes/extract-frozen-qa-eval/tasks.md b/openspec/changes/extract-frozen-qa-eval/tasks.md deleted file mode 100644 index 1db8f788e8..0000000000 --- a/openspec/changes/extract-frozen-qa-eval/tasks.md +++ /dev/null @@ -1,46 +0,0 @@ -## 1. Rebaseline - -- [x] 1.1 Rewrite the OpenSpec proposal, design, capability specs, and tasks for the reviewer-approved lean architecture. -- [x] 1.2 Replace optional dependencies with official MCP SDK pins and the minimal existing-CI dependency set; regenerate Python and Node locks. - -## 2. Frozen Memory2 - -- [x] 2.1 Retain read-only propagation only through the SQLite helper, registry, observation store, and SQLite store. -- [x] 2.2 Remove the new general `ThroughFilter` and stream writability APIs; implement the inclusive cutoff inside the frozen facade with existing filters. -- [x] 2.3 Keep focused tests for no-WAL reads, source/derived overlay, exact cutoff, collisions, and rejected mutations. - -## 3. Production CodePolicy and MCP - -- [x] 3.1 Collapse the Jupyter implementation into a reusable module-independent `CodePolicySession` with frozen bootstrap, timeout recovery, shutdown, and scrubbed kernel credentials. -- [x] 3.2 Replace the hand-written standalone process/server/control API with an in-process official MCP server exposing exactly `python_exec` on a pre-bound loopback socket. -- [x] 3.3 Add hermetic tests for fresh namespaces, no `app`, no credentials, interrupt/restart, exact tool inventory, race-free startup, and bounded shutdown. - -## 4. Stock Pi CLI Extension - -- [x] 4.1 Replace `pi-code-policy-adapter` with the minimal `pi-code-policy-extension` package using pinned Pi `0.80.10` and official MCP client `2.0.0`. -- [x] 4.2 Register exactly `python_exec`, validate the MCP inventory, forward calls directly, and close the client on Pi shutdown. -- [x] 4.3 Launch stock Pi `--mode json`; parse official events for final text, stop reason, tool count, and native transcript without a custom protocol or Python broker. -- [x] 4.4 Add focused Node extension tests and hermetic Python event-parser/process-cleanup tests. - -## 5. Lean Evaluation and CLI - -- [x] 5.1 Consolidate strict source/task/validator/result contracts in `agent_eval/models.py`; remove interaction/runtime/agent/fingerprint/artifact models and generic adapter Protocols. -- [x] 5.2 Simplify frozen bundle caching to recording metadata, mapper settings, and cutoffs without cryptographic descriptors. -- [x] 5.3 Replace the attempt engine/store with one runner that privately loads the oracle, runs CodePolicy and Pi, parses `ANSWER: `, and publishes compact output atomically. -- [x] 5.4 Persist only `result.json`, the native transcript when available, and nonempty bounded stderr; refuse a non-empty output directory. -- [x] 5.5 Support API-key environment authentication only and preserve exit codes `0` completed, `1` caught infrastructure failure, and `2` preflight failure. -- [x] 5.6 Rename the fixture directory and case ID with `demo_`/`demo-`, retaining the synthetic-`0` warning. -- [x] 5.7 Keep the base CLI dependency-light and add behavior-focused tests for case validation, privacy, scoring, output, cleanup, and CLI stdout/stderr. - -## 6. Documentation and CI - -- [x] 6.1 Update the evaluation guide, fixture README, agent index, and testing guide for the stock Pi extension, API-key-only auth, compact output, and trusted-unsandboxed boundary. -- [x] 6.2 Make existing Python test/lint environments install the minimal evaluation dependencies and make the Node extension job gate aggregate CI. -- [x] 6.3 Remove documentation and tests for obsolete fingerprints, evidence stores, protocols, OAuth, and attempt locks. - -## 7. Verification and Review - -- [x] 7.1 Validate OpenSpec and run focused Python, Node, Ruff, mypy, doclinks, and executable-Markdown checks. -- [x] 7.2 Run the real Hong Kong self-hosted CPU mechanics gate. -- [ ] 7.3 Build the extension and run the exact API-key smoke command when credentials are available; verify compact output and child cleanup. (The extension and preflight path are verified; this workspace has no `OPENAI_API_KEY` for the live model call.) -- [x] 7.4 Commit and push the redesign, update draft PR #3378, and reply to all review threads with the agreed resolutions. From 3fe0b2eb2b05516cab0990b10b0a341121556004 Mon Sep 17 00:00:00 2001 From: cc Date: Wed, 5 Aug 2026 16:20:07 -0700 Subject: [PATCH 12/15] ci: drop dedicated Pi extension job --- .github/workflows/ci.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d32eb0cfa2..d980c9d8ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,24 +70,6 @@ jobs: - name: Run pre-commit uses: pre-commit/action@v3.0.1 - pi-code-policy-extension: - timeout-minutes: 10 - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-node@v7 - with: - node-version: '22.19.0' - cache: npm - cache-dependency-path: packages/pi-code-policy-extension/package-lock.json - - name: Install extension dependencies - run: npm ci --prefix packages/pi-code-policy-extension - - name: Test extension - run: npm test --prefix packages/pi-code-policy-extension - rust: timeout-minutes: 20 runs-on: ubuntu-latest @@ -995,7 +977,6 @@ jobs: needs: - lint - - pi-code-policy-extension - rust - cpp - md-babel From 2805308503358f5713050307e5a09d6a5a65aa2c Mon Sep 17 00:00:00 2001 From: cc Date: Wed, 5 Aug 2026 16:42:02 -0700 Subject: [PATCH 13/15] fix: harden frozen eval CI and caching --- dimos/benchmark/agent_eval/single_case.py | 34 +++++++++++------- .../benchmark/agent_eval/test_single_case.py | 26 ++++++++++++++ dimos/cli/test_eval.py | 8 +++-- dimos/memory2/blobstore/sqlite.py | 5 ++- dimos/memory2/store/sqlite.py | 24 ++++++++----- dimos/memory2/store/test_frozen.py | 35 +++++++++++++++++++ dimos/memory2/vectorstore/sqlite.py | 5 ++- 7 files changed, 112 insertions(+), 25 deletions(-) diff --git a/dimos/benchmark/agent_eval/single_case.py b/dimos/benchmark/agent_eval/single_case.py index 14dd052aea..1c951d7825 100644 --- a/dimos/benchmark/agent_eval/single_case.py +++ b/dimos/benchmark/agent_eval/single_case.py @@ -16,6 +16,7 @@ from __future__ import annotations +import errno import json import os from pathlib import Path @@ -179,20 +180,29 @@ def _materialize_frozen_memory(case: EvalCase, progress: ProgressSink | None) -> ) key = re.sub(r"[^A-Za-z0-9_.-]+", "-", raw_key) bundle = CACHE_DIR / "agent_eval" / "frozen_memory" / key - if not (bundle / "manifest.v1.json").is_file(): + manifest = bundle / "manifest.v1.json" + if not manifest.is_file(): emit_progress(progress, StatusProgress(channel="eval", message="preparing memory")) bundle.parent.mkdir(parents=True, exist_ok=True) - prepare_bundle( - case.source.recording, - [], - bundle, - progress=[case.source.progress], - mapper=mapper, - map_progress=lambda current, total: emit_progress( - progress, - StatusProgress(channel="eval", message=f"mapping {current}/{total} frames"), - ), - ) + try: + prepare_bundle( + case.source.recording, + [], + bundle, + progress=[case.source.progress], + mapper=mapper, + map_progress=lambda current, total: emit_progress( + progress, + StatusProgress(channel="eval", message=f"mapping {current}/{total} frames"), + ), + ) + except OSError as exc: + concurrent_publish = isinstance(exc, FileExistsError) or exc.errno in { + errno.EEXIST, + errno.ENOTEMPTY, + } + if not concurrent_publish or not manifest.is_file(): + raise return bundle diff --git a/dimos/benchmark/agent_eval/test_single_case.py b/dimos/benchmark/agent_eval/test_single_case.py index f4c895b669..6b8deb758b 100644 --- a/dimos/benchmark/agent_eval/test_single_case.py +++ b/dimos/benchmark/agent_eval/test_single_case.py @@ -108,3 +108,29 @@ def test_nonempty_output_is_rejected_before_execution(tmp_path: Path) -> None: with pytest.raises(FileExistsError, match="absent or an empty"): execute_single_case(_case(tmp_path), config=EvalRunConfig(), output=output) assert (output / "keep").read_text() == "user data" + + +def test_materialize_accepts_bundle_published_by_concurrent_runner( + monkeypatch, tmp_path: Path +) -> None: + recording = tmp_path / "recording.db" + recording.touch() + case = EvalCase( + case_id="concurrent", + source=FrozenRecordingSource(recording=str(recording), progress=1.0), + task=IntegerQuestionTask(prompt="How many rooms?"), + validator=ExactIntegerValidatorRef(revision="v1", private_path="private/oracle.json"), + ) + monkeypatch.setattr(single_case, "CACHE_DIR", tmp_path / "cache") + monkeypatch.setattr(single_case, "resolve_dataset", lambda _recording: recording) + + def publish_first(_recording, _cutoffs, output: Path, **_kwargs) -> None: + output.mkdir() + (output / "manifest.v1.json").write_text("{}") + raise FileExistsError(output) + + monkeypatch.setattr(single_case, "prepare_bundle", publish_first) + + bundle = single_case._materialize_frozen_memory(case, None) + + assert bundle.joinpath("manifest.v1.json").is_file() diff --git a/dimos/cli/test_eval.py b/dimos/cli/test_eval.py index e7fc36c801..c94ded418a 100644 --- a/dimos/cli/test_eval.py +++ b/dimos/cli/test_eval.py @@ -19,6 +19,7 @@ import sys import textwrap +from click import unstyle import pytest from typer.testing import CliRunner @@ -124,11 +125,12 @@ def preflight(*_args, **_kwargs): def test_eval_help_is_typed_and_output_is_required(tmp_path) -> None: runner = CliRunner() - help_result = runner.invoke(main, ["eval", "run", "--help"]) + help_result = runner.invoke(main, ["eval", "run", "--help"], color=True) missing_output = runner.invoke(main, ["eval", "run", str(_case(tmp_path))]) assert help_result.exit_code == 0 - assert "--agent.api-key-env" in help_result.stdout - assert "--output" in help_result.stdout + help_text = unstyle(help_result.stdout) + assert "--agent.api-key-env" in help_text + assert "--output" in help_text assert missing_output.exit_code == 2 diff --git a/dimos/memory2/blobstore/sqlite.py b/dimos/memory2/blobstore/sqlite.py index 06021dcd9a..ddfa70923f 100644 --- a/dimos/memory2/blobstore/sqlite.py +++ b/dimos/memory2/blobstore/sqlite.py @@ -27,6 +27,7 @@ class SqliteBlobStoreConfig(BlobStoreConfig): conn: sqlite3.Connection | None = Field(default=None, exclude=True) path: str | None = None + read_only: bool = False @model_validator(mode="after") def _conn_xor_path(self) -> SqliteBlobStoreConfig: @@ -76,7 +77,9 @@ def _ensure_table(self, stream_name: str) -> None: def start(self) -> None: if self._conn is None: assert self._path is not None - disposable, self._conn = open_disposable_sqlite_connection(self._path) + disposable, self._conn = open_disposable_sqlite_connection( + self._path, read_only=self.config.read_only + ) self.register_disposable(disposable) def put(self, stream_name: str, key: int, data: bytes) -> None: diff --git a/dimos/memory2/store/sqlite.py b/dimos/memory2/store/sqlite.py index b547e90eba..edaa0ef0d9 100644 --- a/dimos/memory2/store/sqlite.py +++ b/dimos/memory2/store/sqlite.py @@ -89,23 +89,31 @@ def _assemble_backend(self, name: str, stored: dict[str, Any]) -> Backend[Any]: # Reconstruct components from serialized config bs_data = stored.get("blob_store") if bs_data is not None: - bs_cfg = bs_data.get("config", {}) - if bs_cfg.get("path") is None and bs_data["class"] == qual(SqliteBlobStore): - bs: Any = SqliteBlobStore(conn=backend_conn) + bs_cfg = dict(bs_data.get("config", {})) + if bs_data["class"] == qual(SqliteBlobStore): + if self.config.read_only: + bs_cfg["read_only"] = True + if bs_cfg.get("path") is None: + bs_cfg["conn"] = backend_conn + bs: Any = SqliteBlobStore(**bs_cfg) else: bs = deserialize_component(bs_data) else: - bs = SqliteBlobStore(conn=backend_conn) + bs = SqliteBlobStore(conn=backend_conn, read_only=self.config.read_only) vs_data = stored.get("vector_store") if vs_data is not None: - vs_cfg = vs_data.get("config", {}) - if vs_cfg.get("path") is None and vs_data["class"] == qual(SqliteVectorStore): - vs: Any = SqliteVectorStore(conn=backend_conn) + vs_cfg = dict(vs_data.get("config", {})) + if vs_data["class"] == qual(SqliteVectorStore): + if self.config.read_only: + vs_cfg["read_only"] = True + if vs_cfg.get("path") is None: + vs_cfg["conn"] = backend_conn + vs: Any = SqliteVectorStore(**vs_cfg) else: vs = deserialize_component(vs_data) else: - vs = SqliteVectorStore(conn=backend_conn) + vs = SqliteVectorStore(conn=backend_conn, read_only=self.config.read_only) notifier_data = stored.get("notifier") if notifier_data is not None: diff --git a/dimos/memory2/store/test_frozen.py b/dimos/memory2/store/test_frozen.py index 74d2ace3d0..f2fa24d348 100644 --- a/dimos/memory2/store/test_frozen.py +++ b/dimos/memory2/store/test_frozen.py @@ -19,8 +19,10 @@ import pytest +from dimos.memory2.blobstore.sqlite import SqliteBlobStore from dimos.memory2.store.frozen import FrozenMemoryStore from dimos.memory2.store.sqlite import SqliteStore +from dimos.memory2.vectorstore.sqlite import SqliteVectorStore @pytest.fixture @@ -89,6 +91,39 @@ def test_read_only_sqlite_store_does_not_create_wal(recorded_stores) -> None: assert not shm_path.exists() +def test_read_only_store_propagates_to_external_sqlite_components(tmp_path: Path) -> None: + source_path = tmp_path / "source.db" + blob_path = tmp_path / "blobs.db" + vector_path = tmp_path / "vectors.db" + blob_store = SqliteBlobStore(path=str(blob_path)) + vector_store = SqliteVectorStore(path=str(vector_path)) + with SqliteStore(path=str(source_path)) as source: + stream = source.stream( + "camera", + str, + blob_store=blob_store, + vector_store=vector_store, + ) + stream.append("frame", ts=1.0) + blob_store._conn.commit() + for path in (source_path, blob_path, vector_path): + with sqlite3.connect(path) as connection: + connection.execute("PRAGMA wal_checkpoint(TRUNCATE)") + connection.execute("PRAGMA journal_mode=DELETE") + + with SqliteStore(path=str(source_path), must_exist=True, read_only=True) as source: + stream = source.stream("camera") + assert stream.first().data == "frame" + backend = stream._source + assert backend is not None + assert backend.blob_store is not None + assert backend.vector_store is not None + assert backend.blob_store._conn.execute("PRAGMA query_only").fetchone() == (1,) + assert backend.vector_store._conn.execute("PRAGMA query_only").fetchone() == (1,) + assert not Path(f"{blob_path}-wal").exists() + assert not Path(f"{vector_path}-wal").exists() + + def test_frozen_memory_rejects_stream_collisions(recorded_stores) -> None: source_path, _ = recorded_stores source = SqliteStore(path=str(source_path), must_exist=True, read_only=True) diff --git a/dimos/memory2/vectorstore/sqlite.py b/dimos/memory2/vectorstore/sqlite.py index bb5e9d200e..9956548024 100644 --- a/dimos/memory2/vectorstore/sqlite.py +++ b/dimos/memory2/vectorstore/sqlite.py @@ -31,6 +31,7 @@ class SqliteVectorStoreConfig(VectorStoreConfig): conn: sqlite3.Connection | None = Field(default=None, exclude=True) path: str | None = None + read_only: bool = False @model_validator(mode="after") def _conn_xor_path(self) -> SqliteVectorStoreConfig: @@ -74,7 +75,9 @@ def _ensure_table(self, stream_name: str, dim: int) -> None: def start(self) -> None: if self._conn is None: assert self._path is not None - disposable, self._conn = open_disposable_sqlite_connection(self._path) + disposable, self._conn = open_disposable_sqlite_connection( + self._path, read_only=self.config.read_only + ) self.register_disposable(disposable) def put(self, stream_name: str, key: int, embedding: Embedding) -> None: From c7f7c8912b69e02f9764d8179f238ae2b162132d Mon Sep 17 00:00:00 2001 From: cc Date: Sat, 8 Aug 2026 01:25:46 -0700 Subject: [PATCH 14/15] feat: standardize agent evaluation framework --- CONTEXT-MAP.md | 10 + dimos/agents/code_policy_server.py | 5 +- dimos/benchmark/CONTEXT.md | 45 ++++ dimos/benchmark/agent_eval/models.py | 113 -------- dimos/benchmark/agent_eval/single_case.py | 221 --------------- .../benchmark/agent_eval/test_single_case.py | 136 ---------- dimos/benchmark/evaluation/models.py | 138 ++++++++++ .../{agent_eval => evaluation}/pi_process.py | 4 +- .../{agent_eval => evaluation}/progress.py | 0 dimos/benchmark/evaluation/protocol.py | 68 +++++ dimos/benchmark/evaluation/registry.py | 136 ++++++++++ dimos/benchmark/evaluation/runner.py | 142 ++++++++++ dimos/benchmark/evaluation/runtime.py | 254 ++++++++++++++++++ dimos/benchmark/evaluation/test_models.py | 64 +++++ .../test_pi_process.py | 8 +- dimos/benchmark/evaluation/test_registry.py | 98 +++++++ dimos/benchmark/evaluation/test_runner.py | 188 +++++++++++++ dimos/benchmark/evaluation/test_runtime.py | 117 ++++++++ .../README.md | 5 +- .../run.json | 14 + .../benchmark/short_horizon_qa/evaluation.py | 195 ++++++++++++++ .../{eval.py => integer_answer.py} | 29 +- dimos/benchmark/short_horizon_qa/models.py | 66 ++++- .../short_horizon_qa/test_evaluation.py | 169 ++++++++++++ .../short_horizon_qa/test_hongkong_eval.py | 5 +- .../{test_eval.py => test_integer_answer.py} | 2 +- dimos/cli/eval.py | 85 +++--- dimos/cli/test_eval.py | 178 ++++++++---- .../0001-evaluations-own-result-semantics.md | 3 + ...02-code-policy-runtime-responsibilities.md | 3 + docs/capabilities/agents/evaluation.md | 171 ++++++++---- docs/development/testing.md | 2 +- .../src/python-exec.ts | 5 +- .../test/python-exec.test.ts | 10 +- pyproject.toml | 2 + uv.lock | 29 ++ 36 files changed, 2057 insertions(+), 663 deletions(-) create mode 100644 CONTEXT-MAP.md create mode 100644 dimos/benchmark/CONTEXT.md delete mode 100644 dimos/benchmark/agent_eval/models.py delete mode 100644 dimos/benchmark/agent_eval/single_case.py delete mode 100644 dimos/benchmark/agent_eval/test_single_case.py create mode 100644 dimos/benchmark/evaluation/models.py rename dimos/benchmark/{agent_eval => evaluation}/pi_process.py (99%) rename dimos/benchmark/{agent_eval => evaluation}/progress.py (100%) create mode 100644 dimos/benchmark/evaluation/protocol.py create mode 100644 dimos/benchmark/evaluation/registry.py create mode 100644 dimos/benchmark/evaluation/runner.py create mode 100644 dimos/benchmark/evaluation/runtime.py create mode 100644 dimos/benchmark/evaluation/test_models.py rename dimos/benchmark/{agent_eval => evaluation}/test_pi_process.py (96%) create mode 100644 dimos/benchmark/evaluation/test_registry.py create mode 100644 dimos/benchmark/evaluation/test_runner.py create mode 100644 dimos/benchmark/evaluation/test_runtime.py create mode 100644 dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/run.json create mode 100644 dimos/benchmark/short_horizon_qa/evaluation.py rename dimos/benchmark/short_horizon_qa/{eval.py => integer_answer.py} (63%) create mode 100644 dimos/benchmark/short_horizon_qa/test_evaluation.py rename dimos/benchmark/short_horizon_qa/{test_eval.py => test_integer_answer.py} (93%) create mode 100644 docs/adr/0001-evaluations-own-result-semantics.md create mode 100644 docs/adr/0002-code-policy-runtime-responsibilities.md diff --git a/CONTEXT-MAP.md b/CONTEXT-MAP.md new file mode 100644 index 0000000000..f249c89365 --- /dev/null +++ b/CONTEXT-MAP.md @@ -0,0 +1,10 @@ +# Context Map + +## Contexts + +- [Manipulation Planning](./CONTEXT.md) — describes requests for planning robot motion through joint and Cartesian spaces. +- [Evaluation](./dimos/benchmark/CONTEXT.md) — describes executable evaluations and records of their execution. + +## Relationships + +- **Evaluation → Manipulation Planning**: An evaluation may exercise manipulation planning as part of the subject being evaluated, but does not define planning semantics. diff --git a/dimos/agents/code_policy_server.py b/dimos/agents/code_policy_server.py index 70b89de7dd..a16765382d 100644 --- a/dimos/agents/code_policy_server.py +++ b/dimos/agents/code_policy_server.py @@ -33,9 +33,8 @@ PYTHON_EXEC_DESCRIPTION = """Execute Python in a persistent trusted, unsandboxed session. -The frozen evaluation session exposes read-only `memory`. Imports, functions, and -variables persist between calls. Use this tool to inspect the recording and compute -the answer; do not guess from the prompt. +Imports, functions, and variables persist between calls. The runtime environment +determines which globals are available. """ _NOISY_MCP_TRANSPORT_LOGGERS = ( diff --git a/dimos/benchmark/CONTEXT.md b/dimos/benchmark/CONTEXT.md new file mode 100644 index 0000000000..933fd43539 --- /dev/null +++ b/dimos/benchmark/CONTEXT.md @@ -0,0 +1,45 @@ +# Evaluation + +This context describes executable evaluations, requests to run them, and immutable records of their execution. + +## Language + +**Evaluation**: +An executable definition that owns its protocol and result semantics. +_Avoid_: Evaluator, benchmark integration + +**Evaluation Run Specification**: +A user-authored request that binds an evaluation and its configuration to a CodePolicy agent configuration. Output paths, credentials, infrastructure timeouts, and concurrency are operational settings outside it. +_Avoid_: Run, case + +**Evaluation Run**: +An immutable record of one resolved execution of an evaluation run specification. +_Avoid_: Run configuration, mutable run + +**Evaluation Case**: +An optional atomic input owned by an evaluation. It is not part of the universal evaluation run contract. +_Avoid_: Evaluation target + +**Evaluation Attempt**: +One execution of an evaluation case or another evaluation-owned unit. +_Avoid_: Evaluation run + +**CodePolicy Agent Runtime**: +The required agent interaction contract for an evaluation: exactly one `python_exec` MCP tool backed by a persistent Python environment. +_Avoid_: Evaluation subject, generic agent runtime + +**CodePolicy Session**: +A lifecycle-bounded CodePolicy interaction whose Python namespace and agent conversation persist until the evaluation closes it. The evaluation owns session boundaries. +_Avoid_: Evaluation run, global agent session + +**CodePolicy Runtime Profile**: +A versioned definition of the runtime-owned system instructions, `python_exec` tool surface, and session behavior used by a CodePolicy agent runtime. Evaluations cannot override its prompt responsibilities. +_Avoid_: Evaluation prompt, benchmark prompt + +**Prompt Component**: +An immutable, separately recorded part of an agent prompt owned by either the runtime profile or the evaluation. Evaluation protocol and task input remain distinct components even when transported in one message. +_Avoid_: Prompt fragment, appended prompt + +**Prompt Assembly**: +The deterministic runtime-owned combination of prompt components for one CodePolicy session. +_Avoid_: Prompt concatenation, prompt settings diff --git a/dimos/benchmark/agent_eval/models.py b/dimos/benchmark/agent_eval/models.py deleted file mode 100644 index 7991d89b59..0000000000 --- a/dimos/benchmark/agent_eval/models.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Small tagged contracts for one frozen agent-evaluation case.""" - -from __future__ import annotations - -import math -from pathlib import PurePosixPath -from typing import Annotated, Literal - -from pydantic import BaseModel, ConfigDict, Field, model_validator - - -class BaseEvalModel(BaseModel): - """Strict immutable base for the compact evaluation contracts.""" - - model_config = ConfigDict(extra="forbid", frozen=True, strict=True) - schema_version: Literal["1.0"] = "1.0" - - -NonEmpty = Annotated[str, Field(min_length=1)] - - -class FrozenRecordingSource(BaseEvalModel): - kind: Literal["frozen_memory"] = "frozen_memory" - recording: NonEmpty - progress: float = Field(ge=0, le=1, allow_inf_nan=False) - - @model_validator(mode="after") - def finite_progress(self) -> FrozenRecordingSource: - if not math.isfinite(self.progress): - raise ValueError("recording progress must be finite") - return self - - -class IntegerQuestionTask(BaseEvalModel): - kind: Literal["integer_question"] = "integer_question" - prompt: NonEmpty - answer_marker: Literal["ANSWER:"] = "ANSWER:" - - -class ExactIntegerValidatorRef(BaseEvalModel): - kind: Literal["exact_integer"] = "exact_integer" - revision: NonEmpty - private_path: NonEmpty - - @model_validator(mode="after") - def safe_relative_path(self) -> ExactIntegerValidatorRef: - path = PurePosixPath(self.private_path) - if path.is_absolute() or not path.parts or ".." in path.parts: - raise ValueError("validator private_path must be a safe relative path") - return self - - -SourceSpec = Annotated[FrozenRecordingSource, Field(discriminator="kind")] -TaskSpec = Annotated[IntegerQuestionTask, Field(discriminator="kind")] -ValidatorRef = Annotated[ExactIntegerValidatorRef, Field(discriminator="kind")] - - -class EvalCase(BaseEvalModel): - case_id: NonEmpty - source: SourceSpec - task: TaskSpec - validator: ValidatorRef - - -class PiAgentConfig(BaseEvalModel): - backend: Literal["pi"] = "pi" - model: Literal["gpt-5.6-luna"] = "gpt-5.6-luna" - thinking_level: Literal["medium"] = "medium" - api_key_env: str = Field(default="OPENAI_API_KEY", min_length=1) - - -class EvalRunConfig(BaseEvalModel): - agent: PiAgentConfig = Field(default_factory=PiAgentConfig) - - -class CompactEvalResult(BaseEvalModel): - case_id: str - recording: str - progress: float - model: str - thinking_level: str - final_response: str = "" - prediction_status: Literal["parsed", "invalid", "not_evaluated"] - integer_answer: int | None = None - passed: bool | None = None - validator_revision: str - tool_call_count: int = Field(ge=0) - duration_seconds: float = Field(ge=0) - infra_error: str | None = None - - @property - def attempt_status(self) -> Literal["completed", "failed"]: - return "failed" if self.infra_error is not None else "completed" - - @property - def task_result(self) -> Literal["passed", "failed", "not_evaluated"]: - if self.passed is None: - return "not_evaluated" - return "passed" if self.passed else "failed" diff --git a/dimos/benchmark/agent_eval/single_case.py b/dimos/benchmark/agent_eval/single_case.py deleted file mode 100644 index 1c951d7825..0000000000 --- a/dimos/benchmark/agent_eval/single_case.py +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Direct runner for one frozen-memory Pi evaluation case.""" - -from __future__ import annotations - -import errno -import json -import os -from pathlib import Path -import re -import shutil -import tempfile -import time - -from dimos.agents.code_policy_core import CodePolicySessionConfig, FrozenMemoryEnvironment -from dimos.agents.code_policy_server import CodePolicyMcpServer -from dimos.benchmark.agent_eval.models import CompactEvalResult, EvalCase, EvalRunConfig -from dimos.benchmark.agent_eval.pi_process import PiCliRunner, PiRunError -from dimos.benchmark.agent_eval.progress import ProgressSink, StatusProgress, emit_progress -from dimos.benchmark.short_horizon_qa.eval import ( - load_exact_integer_oracle, - parse_integer_prediction, -) -from dimos.benchmark.short_horizon_qa.models import MapperSettings -from dimos.benchmark.short_horizon_qa.prepare import prepare_bundle -from dimos.benchmark.short_horizon_qa.service import load_bundle -from dimos.constants import CACHE_DIR -from dimos.memory2.cli.dataset import resolve_dataset - -TURN_TIMEOUT_SECONDS = 600.0 - -SYSTEM_PROMPT = """You are answering a question about a frozen robot recording. - -You have exactly one tool, `python_exec`. It runs trusted, unsandboxed Python in a -persistent Jupyter kernel with a read-only `memory` object. Inspect Memory2 streams -and compute the answer from the recording. Do not guess. End with exactly one line: -ANSWER: -""" - - -def execute_single_case( - case_path: Path, - *, - config: EvalRunConfig, - output: Path, - progress: ProgressSink | None = None, -) -> CompactEvalResult: - """Preflight, run, and atomically publish exactly one result directory.""" - path = case_path.expanduser().resolve() - output = output.expanduser().resolve() - _validate_output(output) - emit_progress(progress, StatusProgress(channel="eval", message="loading case")) - case = EvalCase.model_validate_json(path.read_bytes()) - oracle = load_exact_integer_oracle(case, path.parent) - api_key = os.environ.get(config.agent.api_key_env) - if not api_key: - raise ValueError(f"API key environment variable {config.agent.api_key_env!r} is unset") - bundle = _materialize_frozen_memory(case, progress) - _, cutoff, source_path, derived_path = load_bundle(bundle, progress=case.source.progress) - emit_progress(progress, StatusProgress(channel="eval", message="memory ready")) - cli, extension = _pi_paths() - runner = PiCliRunner( - cli=cli, - extension=extension, - model=config.agent.model, - thinking_level=config.agent.thinking_level, - timeout_s=TURN_TIMEOUT_SECONDS, - progress=progress, - ) - - output.parent.mkdir(parents=True, exist_ok=True) - temporary = Path(tempfile.mkdtemp(prefix=f".{output.name}-", dir=output.parent)) - runtime_dir = temporary / "runtime" - runtime_dir.mkdir() - started = time.monotonic() - stderr = "" - server: CodePolicyMcpServer | None = None - try: - emit_progress(progress, StatusProgress(channel="eval", message="starting agent")) - server = CodePolicyMcpServer( - CodePolicySessionConfig( - environment=FrozenMemoryEnvironment( - recording_path=str(source_path), - derived_recording_path=str(derived_path), - memory_cutoff_timestamp=cutoff.cutoff_timestamp, - ) - ) - ) - try: - server.start() - pi_result = runner.run( - prompt=_agent_prompt(case), - system_prompt=SYSTEM_PROMPT, - mcp_url=server.mcp_url, - api_key=api_key, - run_dir=runtime_dir, - ) - stderr = pi_result.stderr - if pi_result.transcript_path is not None: - shutil.copy2(pi_result.transcript_path, temporary / "pi-transcript.jsonl") - prediction = parse_integer_prediction(pi_result.final_text) - passed = ( - prediction.status == "parsed" and prediction.integer_answer == oracle.expected_count - ) - result = CompactEvalResult( - case_id=case.case_id, - recording=case.source.recording, - progress=case.source.progress, - model=config.agent.model, - thinking_level=config.agent.thinking_level, - final_response=pi_result.final_text, - prediction_status=prediction.status, - integer_answer=prediction.integer_answer, - passed=passed, - validator_revision=case.validator.revision, - tool_call_count=pi_result.tool_call_count, - duration_seconds=time.monotonic() - started, - ) - finally: - server.stop() - except Exception as exc: - if isinstance(exc, PiRunError): - stderr = exc.stderr - result = CompactEvalResult( - case_id=case.case_id, - recording=case.source.recording, - progress=case.source.progress, - model=config.agent.model, - thinking_level=config.agent.thinking_level, - prediction_status="not_evaluated", - passed=None, - validator_revision=case.validator.revision, - tool_call_count=server.session.execution_count if server is not None else 0, - duration_seconds=time.monotonic() - started, - infra_error=f"{type(exc).__name__}: {exc}", - ) - finally: - shutil.rmtree(runtime_dir, ignore_errors=True) - - if stderr: - (temporary / "stderr.log").write_text(stderr, encoding="utf-8") - (temporary / "result.json").write_text( - json.dumps(result.model_dump(mode="json"), indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - if output.exists(): - output.rmdir() - os.replace(temporary, output) - emit_progress(progress, StatusProgress(channel="eval", message="result published")) - return result - - -def _validate_output(output: Path) -> None: - if output.exists() and (not output.is_dir() or any(output.iterdir())): - raise FileExistsError(f"Output must be absent or an empty directory: {output}") - - -def _materialize_frozen_memory(case: EvalCase, progress: ProgressSink | None) -> Path: - source_path = resolve_dataset(case.source.recording).resolve() - stat = source_path.stat() - stem = re.sub(r"[^A-Za-z0-9_.-]+", "-", source_path.stem)[:64] - mapper = MapperSettings() - raw_key = ( - f"{stem}-{stat.st_size}-{stat.st_mtime_ns}-p{case.source.progress:.9f}-" - f"v{mapper.voxel_size_m}-b{mapper.block_count}-d{mapper.device}-" - f"c{int(mapper.carve_columns)}-f{mapper.frame_id}-e{mapper.emit_every}" - ) - key = re.sub(r"[^A-Za-z0-9_.-]+", "-", raw_key) - bundle = CACHE_DIR / "agent_eval" / "frozen_memory" / key - manifest = bundle / "manifest.v1.json" - if not manifest.is_file(): - emit_progress(progress, StatusProgress(channel="eval", message="preparing memory")) - bundle.parent.mkdir(parents=True, exist_ok=True) - try: - prepare_bundle( - case.source.recording, - [], - bundle, - progress=[case.source.progress], - mapper=mapper, - map_progress=lambda current, total: emit_progress( - progress, - StatusProgress(channel="eval", message=f"mapping {current}/{total} frames"), - ), - ) - except OSError as exc: - concurrent_publish = isinstance(exc, FileExistsError) or exc.errno in { - errno.EEXIST, - errno.ENOTEMPTY, - } - if not concurrent_publish or not manifest.is_file(): - raise - return bundle - - -def _pi_paths() -> tuple[Path, Path]: - package = Path(__file__).resolve().parents[3] / "packages" / "pi-code-policy-extension" - cli = package / "node_modules" / "@earendil-works" / "pi-coding-agent" / "dist" / "cli.js" - extension = package / "dist" / "python-exec.js" - return cli, extension - - -def _agent_prompt(case: EvalCase) -> str: - return ( - f"{case.task.prompt}\n\n" - "Use python_exec to inspect the read-only recording. " - f"End with `{case.task.answer_marker} `." - ) diff --git a/dimos/benchmark/agent_eval/test_single_case.py b/dimos/benchmark/agent_eval/test_single_case.py deleted file mode 100644 index 6b8deb758b..0000000000 --- a/dimos/benchmark/agent_eval/test_single_case.py +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pathlib import Path -from types import SimpleNamespace - -import pytest - -from dimos.benchmark.agent_eval.models import ( - EvalCase, - EvalRunConfig, - ExactIntegerValidatorRef, - FrozenRecordingSource, - IntegerQuestionTask, -) -from dimos.benchmark.agent_eval.pi_process import PiRunResult -import dimos.benchmark.agent_eval.single_case as single_case -from dimos.benchmark.agent_eval.single_case import execute_single_case - - -def _case(tmp_path: Path) -> Path: - private = tmp_path / "private" - private.mkdir() - (private / "oracle.json").write_text( - '{"schema_version":"1.0","expected_count":2,' - '"counting_policy":"count rooms","rooms":[],' - '"reviewed_by":["reviewer"]}' - ) - case = EvalCase( - case_id="demo", - source=FrozenRecordingSource(recording="recording", progress=1.0), - task=IntegerQuestionTask(prompt="How many rooms?"), - validator=ExactIntegerValidatorRef(revision="v1", private_path="private/oracle.json"), - ) - path = tmp_path / "case.json" - path.write_text(case.model_dump_json()) - return path - - -def test_direct_run_publishes_only_compact_result_and_native_transcript( - monkeypatch, tmp_path: Path -) -> None: - case_path = _case(tmp_path) - bundle = tmp_path / "bundle" - bundle.mkdir() - monkeypatch.setenv("OPENAI_API_KEY", "secret") - monkeypatch.setattr(single_case, "_materialize_frozen_memory", lambda *_args: bundle) - monkeypatch.setattr( - single_case, - "load_bundle", - lambda *_args, **_kwargs: ( - object(), - SimpleNamespace(cutoff_timestamp=10.0), - tmp_path / "source.db", - tmp_path / "derived.db", - ), - ) - monkeypatch.setattr(single_case, "_pi_paths", lambda: (case_path, case_path)) - - class Server: - mcp_url = "http://127.0.0.1:1234/mcp" - session = SimpleNamespace(execution_count=3) - - def __init__(self, _config): - pass - - def start(self): - pass - - def stop(self): - pass - - class Runner: - def __init__(self, **_kwargs): - pass - - def run(self, *, run_dir, **_kwargs): - transcript = run_dir / "native.jsonl" - transcript.write_text('{"type":"session"}\n') - return PiRunResult("Checked\nANSWER: 2", 3, 1.0, transcript, "") - - monkeypatch.setattr(single_case, "CodePolicyMcpServer", Server) - monkeypatch.setattr(single_case, "PiCliRunner", Runner) - output = tmp_path / "output" - result = execute_single_case(case_path, config=EvalRunConfig(), output=output) - assert result.passed is True - assert {path.name for path in output.iterdir()} == { - "result.json", - "pi-transcript.jsonl", - } - - -def test_nonempty_output_is_rejected_before_execution(tmp_path: Path) -> None: - output = tmp_path / "output" - output.mkdir() - (output / "keep").write_text("user data") - with pytest.raises(FileExistsError, match="absent or an empty"): - execute_single_case(_case(tmp_path), config=EvalRunConfig(), output=output) - assert (output / "keep").read_text() == "user data" - - -def test_materialize_accepts_bundle_published_by_concurrent_runner( - monkeypatch, tmp_path: Path -) -> None: - recording = tmp_path / "recording.db" - recording.touch() - case = EvalCase( - case_id="concurrent", - source=FrozenRecordingSource(recording=str(recording), progress=1.0), - task=IntegerQuestionTask(prompt="How many rooms?"), - validator=ExactIntegerValidatorRef(revision="v1", private_path="private/oracle.json"), - ) - monkeypatch.setattr(single_case, "CACHE_DIR", tmp_path / "cache") - monkeypatch.setattr(single_case, "resolve_dataset", lambda _recording: recording) - - def publish_first(_recording, _cutoffs, output: Path, **_kwargs) -> None: - output.mkdir() - (output / "manifest.v1.json").write_text("{}") - raise FileExistsError(output) - - monkeypatch.setattr(single_case, "prepare_bundle", publish_first) - - bundle = single_case._materialize_frozen_memory(case, None) - - assert bundle.joinpath("manifest.v1.json").is_file() diff --git a/dimos/benchmark/evaluation/models.py b/dimos/benchmark/evaluation/models.py new file mode 100644 index 0000000000..bd4839041b --- /dev/null +++ b/dimos/benchmark/evaluation/models.py @@ -0,0 +1,138 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Universal contracts for requesting and recording evaluation runs.""" + +from __future__ import annotations + +from datetime import datetime +from pathlib import PurePosixPath +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class EvaluationModel(BaseModel): + """Strict immutable base for persisted evaluation contracts.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class EvaluationReference(EvaluationModel): + name: str = Field(min_length=1) + config: dict[str, Any] = Field(default_factory=dict) + + +class CodePolicyAgentConfig(EvaluationModel): + profile: Literal["code-policy-v1"] = "code-policy-v1" + model: Literal["gpt-5.6-luna"] = "gpt-5.6-luna" + thinking_level: Literal["medium"] = "medium" + + +class EvaluationRunSpecification(EvaluationModel): + schema_version: Literal["1.0"] = "1.0" + evaluation: EvaluationReference + agent: CodePolicyAgentConfig = Field(default_factory=CodePolicyAgentConfig) + + +class ArtifactReference(EvaluationModel): + path: str = Field(min_length=1) + label: str = Field(min_length=1) + media_type: str | None = Field(default=None, min_length=1) + + @model_validator(mode="after") + def path_is_safe_and_relative(self) -> ArtifactReference: + path = PurePosixPath(self.path) + if path.is_absolute() or not path.parts or ".." in path.parts: + raise ValueError("artifact path must be a safe relative POSIX path") + return self + + +SummaryValue = str | int | float | bool | None + + +class SummaryItem(EvaluationModel): + key: str = Field(min_length=1, pattern=r"^[a-z][a-z0-9_]*$") + label: str = Field(min_length=1) + value: SummaryValue + + +class InlineNativeResult(EvaluationModel): + kind: Literal["inline"] = "inline" + value: Any + + +class ArtifactNativeResult(EvaluationModel): + kind: Literal["artifact"] = "artifact" + artifact: ArtifactReference + + +NativeResult = Annotated[ + InlineNativeResult | ArtifactNativeResult, + Field(discriminator="kind"), +] + + +class EvaluationReport(EvaluationModel): + summary: tuple[SummaryItem, ...] = () + native_result: NativeResult + artifacts: tuple[ArtifactReference, ...] = () + + +class EvaluationIdentity(EvaluationModel): + name: str = Field(min_length=1) + provider: str = Field(min_length=1) + version: str = Field(min_length=1) + + +class RuntimeIdentity(EvaluationModel): + profile: Literal["code-policy-v1"] = "code-policy-v1" + driver: Literal["pi"] = "pi" + driver_version: str = Field(min_length=1) + model: str = Field(min_length=1) + thinking_level: str = Field(min_length=1) + + +class EvaluationRunError(EvaluationModel): + stage: Literal["evaluation", "publication"] + error_type: str = Field(min_length=1) + message: str = Field(min_length=1) + + +class EvaluationRun(EvaluationModel): + schema_version: Literal["1.0"] = "1.0" + run_id: str = Field(min_length=1) + specification: EvaluationRunSpecification + evaluation: EvaluationIdentity + runtime: RuntimeIdentity + status: Literal["completed", "failed", "cancelled"] + started_at: datetime + finished_at: datetime + duration_seconds: float = Field(ge=0) + report: EvaluationReport | None = None + error: EvaluationRunError | None = None + runtime_artifacts: tuple[ArtifactReference, ...] = () + prompt_evidence: tuple[ArtifactReference, ...] = () + + @model_validator(mode="after") + def status_matches_payload(self) -> EvaluationRun: + if self.status == "completed" and self.report is None: + raise ValueError("completed evaluation runs require a report") + if self.status != "completed" and self.error is None: + raise ValueError("non-completed evaluation runs require an error") + if self.status == "completed" and self.error is not None: + raise ValueError("completed evaluation runs cannot contain an error") + if self.status != "completed" and self.report is not None: + raise ValueError("non-completed evaluation runs cannot contain a report") + return self diff --git a/dimos/benchmark/agent_eval/pi_process.py b/dimos/benchmark/evaluation/pi_process.py similarity index 99% rename from dimos/benchmark/agent_eval/pi_process.py rename to dimos/benchmark/evaluation/pi_process.py index ecd04336e0..f02fe926c1 100644 --- a/dimos/benchmark/agent_eval/pi_process.py +++ b/dimos/benchmark/evaluation/pi_process.py @@ -25,7 +25,7 @@ import time from typing import Any -from dimos.benchmark.agent_eval.progress import ( +from dimos.benchmark.evaluation.progress import ( AssistantTextProgress, FinalResponseProgress, ProgressSink, @@ -106,7 +106,7 @@ def run( "--session-dir", str(session_dir), "--name", - "dimos-frozen-eval", + "dimos-evaluation", "--no-builtin-tools", "--tools", "python_exec", diff --git a/dimos/benchmark/agent_eval/progress.py b/dimos/benchmark/evaluation/progress.py similarity index 100% rename from dimos/benchmark/agent_eval/progress.py rename to dimos/benchmark/evaluation/progress.py diff --git a/dimos/benchmark/evaluation/protocol.py b/dimos/benchmark/evaluation/protocol.py new file mode 100644 index 0000000000..1f6c61a224 --- /dev/null +++ b/dimos/benchmark/evaluation/protocol.py @@ -0,0 +1,68 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The complete Evaluation extension point.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Protocol, runtime_checkable + +from pydantic import BaseModel + +from dimos.benchmark.evaluation.models import EvaluationReport +from dimos.benchmark.evaluation.progress import ProgressSink + + +@runtime_checkable +class CodePolicyRuntime(Protocol): + """Factory supplied to evaluations for evaluation-owned agent sessions.""" + + def open_session(self, environment: BaseModel) -> CodePolicySessionHandle: ... + + +@runtime_checkable +class CodePolicySessionHandle(Protocol): + def __enter__(self) -> CodePolicySessionHandle: ... + + def __exit__(self, *args: object) -> None: ... + + def run(self, *, evaluation_protocol: str, task_input: str) -> AgentOutcome: ... + + +@dataclass(frozen=True) +class AgentOutcome: + final_text: str + tool_call_count: int + duration_seconds: float + + +@dataclass(frozen=True) +class EvaluationContext: + run_id: str + spec_dir: Path + workspace: Path + agent: CodePolicyRuntime + progress: ProgressSink | None + + +@runtime_checkable +class Evaluation(Protocol): + """A complete executable evaluation with native result semantics.""" + + name: str + config_model: type[BaseModel] + + def run(self, config: BaseModel, context: EvaluationContext) -> EvaluationReport: ... diff --git a/dimos/benchmark/evaluation/registry.py b/dimos/benchmark/evaluation/registry.py new file mode 100644 index 0000000000..a6608daf67 --- /dev/null +++ b/dimos/benchmark/evaluation/registry.py @@ -0,0 +1,136 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Lazy built-in and installed-package Evaluation discovery.""" + +from __future__ import annotations + +from dataclasses import dataclass +import importlib +import importlib.metadata as importlib_metadata +import re +from typing import Any + +from packaging.utils import canonicalize_name +from pydantic import BaseModel + +from dimos.benchmark.evaluation.protocol import Evaluation + +ENTRY_POINT_GROUP = "dimos.evaluations" +LOCAL_NAME_PATTERN = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") +BUILTIN_EVALUATIONS = { + "frozen-integer-qa": ("dimos.benchmark.short_horizon_qa.evaluation:frozen_integer_qa"), +} + + +class EvaluationRegistryError(ValueError): + """An Evaluation name or plugin could not be resolved.""" + + +@dataclass(frozen=True) +class ResolvedEvaluation: + name: str + provider: str + version: str + evaluation: Evaluation + + +def available_evaluations() -> list[str]: + return sorted([*BUILTIN_EVALUATIONS, *_external_entries()]) + + +def resolve_evaluation(name: str) -> ResolvedEvaluation: + if name in BUILTIN_EVALUATIONS: + target = _load_target(BUILTIN_EVALUATIONS[name], name) + return ResolvedEvaluation( + name=name, + provider="dimos", + version=_distribution_version("dimos"), + evaluation=_validate_target(name, target), + ) + + entries = _external_entries() + entry = entries.get(name) + if entry is None: + available = available_evaluations() + suffix = f" Available evaluations: {', '.join(available)}." if available else "" + raise EvaluationRegistryError(f"Unknown evaluation {name!r}.{suffix}") + try: + target = entry.load() + except Exception as exc: + raise EvaluationRegistryError( + f"Failed to load evaluation {name!r} from {entry.value!r}: {type(exc).__name__}: {exc}" + ) from exc + distribution = entry.dist + assert distribution is not None + distribution_name = distribution.metadata["Name"] + return ResolvedEvaluation( + name=name, + provider=distribution_name, + version=distribution.version, + evaluation=_validate_target(name, target), + ) + + +def _external_entries() -> dict[str, importlib_metadata.EntryPoint]: + result: dict[str, importlib_metadata.EntryPoint] = {} + for entry in importlib_metadata.entry_points(group=ENTRY_POINT_GROUP): + distribution = entry.dist + if distribution is None: + continue + distribution_name = distribution.metadata.get("Name") + if not distribution_name or LOCAL_NAME_PATTERN.fullmatch(entry.name) is None: + continue + namespace = str(canonicalize_name(distribution_name)) + qualified_name = f"{namespace}.{entry.name}" + if qualified_name in result: + raise EvaluationRegistryError( + f"Multiple installed entry points provide evaluation {qualified_name!r}" + ) + result[qualified_name] = entry + return result + + +def _load_target(path: str, name: str) -> Any: + module_name, separator, attribute = path.partition(":") + if not separator: + raise EvaluationRegistryError(f"Invalid built-in evaluation target for {name!r}: {path}") + try: + return getattr(importlib.import_module(module_name), attribute) + except Exception as exc: + raise EvaluationRegistryError( + f"Failed to load evaluation {name!r} from {path!r}: {type(exc).__name__}: {exc}" + ) from exc + + +def _validate_target(name: str, target: Any) -> Evaluation: + if not isinstance(target, Evaluation): + raise EvaluationRegistryError( + f"Evaluation {name!r} must expose name, config_model, and run()" + ) + config_model = target.config_model + if not isinstance(config_model, type) or not issubclass(config_model, BaseModel): + raise EvaluationRegistryError( + f"Evaluation {name!r} config_model must be a Pydantic BaseModel type" + ) + if target.name != name.rpartition(".")[2]: + raise EvaluationRegistryError(f"Evaluation {name!r} loaded a target named {target.name!r}") + return target + + +def _distribution_version(name: str) -> str: + try: + return importlib_metadata.version(name) + except importlib_metadata.PackageNotFoundError: + return "source" diff --git a/dimos/benchmark/evaluation/runner.py b/dimos/benchmark/evaluation/runner.py new file mode 100644 index 0000000000..3bef855ae2 --- /dev/null +++ b/dimos/benchmark/evaluation/runner.py @@ -0,0 +1,142 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Resolve, execute, and atomically publish one Evaluation Run.""" + +from __future__ import annotations + +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import shutil +import tempfile +import time +from typing import Literal +from uuid import uuid4 + +from dimos.benchmark.evaluation.models import ( + EvaluationIdentity, + EvaluationRun, + EvaluationRunError, + EvaluationRunSpecification, +) +from dimos.benchmark.evaluation.progress import ProgressSink, StatusProgress, emit_progress +from dimos.benchmark.evaluation.protocol import EvaluationContext +from dimos.benchmark.evaluation.registry import resolve_evaluation +from dimos.benchmark.evaluation.runtime import CodePolicyRuntimeFactory + + +def execute_evaluation( + specification_path: Path, + *, + output: Path, + api_key_env: str = "OPENAI_API_KEY", + progress: ProgressSink | None = None, +) -> EvaluationRun: + """Run one resolved Evaluation and publish its immutable record.""" + specification_path = specification_path.expanduser().resolve() + output = output.expanduser().resolve() + _validate_output(output) + specification = EvaluationRunSpecification.model_validate_json(specification_path.read_bytes()) + resolved = resolve_evaluation(specification.evaluation.name) + config = resolved.evaluation.config_model.model_validate_json( + json.dumps(specification.evaluation.config), + strict=True, + ) + api_key = os.environ.get(api_key_env) + if not api_key: + raise ValueError(f"API key environment variable {api_key_env!r} is unset") + + output.parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix=f".{output.name}-", dir=output.parent)) + run_id = str(uuid4()) + started_at = datetime.now(timezone.utc) + started = time.monotonic() + runtime = CodePolicyRuntimeFactory( + config=specification.agent, + api_key=api_key, + workspace=temporary, + progress=progress, + ) + context = EvaluationContext( + run_id=run_id, + spec_dir=specification_path.parent, + workspace=temporary, + agent=runtime, + progress=progress, + ) + emit_progress(progress, StatusProgress(channel="eval", message="evaluation started")) + try: + status: Literal["completed", "failed", "cancelled"] + try: + report = resolved.evaluation.run(config, context) + status = "completed" + error = None + except KeyboardInterrupt: + report = None + status = "cancelled" + error = EvaluationRunError( + stage="evaluation", + error_type="KeyboardInterrupt", + message="Evaluation cancelled by user", + ) + except Exception as exc: + report = None + status = "failed" + error = EvaluationRunError( + stage="evaluation", + error_type=type(exc).__name__, + message=_redact_error(str(exc) or type(exc).__name__, api_key), + ) + finished_at = datetime.now(timezone.utc) + run = EvaluationRun( + run_id=run_id, + specification=specification, + evaluation=EvaluationIdentity( + name=resolved.name, + provider=resolved.provider, + version=resolved.version, + ), + runtime=runtime.identity, + status=status, + started_at=started_at, + finished_at=finished_at, + duration_seconds=time.monotonic() - started, + report=report, + error=error, + runtime_artifacts=runtime.runtime_artifacts, + prompt_evidence=runtime.prompt_evidence, + ) + (temporary / "run.json").write_text( + run.model_dump_json(indent=2) + "\n", + encoding="utf-8", + ) + if output.exists(): + output.rmdir() + os.replace(temporary, output) + emit_progress(progress, StatusProgress(channel="eval", message="run published")) + return run + finally: + if temporary.exists(): + shutil.rmtree(temporary) + + +def _validate_output(output: Path) -> None: + if output.exists() and (not output.is_dir() or any(output.iterdir())): + raise FileExistsError(f"Output must be absent or an empty directory: {output}") + + +def _redact_error(message: str, api_key: str) -> str: + return message.replace(api_key, "[REDACTED]") diff --git a/dimos/benchmark/evaluation/runtime.py b/dimos/benchmark/evaluation/runtime.py new file mode 100644 index 0000000000..fc29befaff --- /dev/null +++ b/dimos/benchmark/evaluation/runtime.py @@ -0,0 +1,254 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The versioned Pi implementation of the CodePolicy agent runtime.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import shutil + +from pydantic import BaseModel + +from dimos.agents.code_policy_core import ( + CodePolicyEnvironment, + CodePolicySessionConfig, + FrozenMemoryEnvironment, + LiveDimosEnvironment, +) +from dimos.agents.code_policy_server import CodePolicyMcpServer +from dimos.benchmark.evaluation.models import ( + ArtifactReference, + CodePolicyAgentConfig, + RuntimeIdentity, +) +from dimos.benchmark.evaluation.pi_process import PI_VERSION, PiCliRunner, PiRunError +from dimos.benchmark.evaluation.progress import ProgressSink +from dimos.benchmark.evaluation.protocol import AgentOutcome + +CODE_POLICY_PROFILE = "code-policy-v1" +TURN_TIMEOUT_SECONDS = 600.0 +SYSTEM_INSTRUCTIONS = """You are a CodePolicy agent. + +Use the single `python_exec` tool to solve the supplied task. Python executes in a +persistent trusted, unsandboxed environment, so imports, variables, and functions +persist between tool calls. Follow the evaluation protocol exactly. +""" + + +class CodePolicyRuntimeFactory: + """Create evaluation-owned sessions with one fixed CodePolicy profile.""" + + def __init__( + self, + *, + config: CodePolicyAgentConfig, + api_key: str, + workspace: Path, + progress: ProgressSink | None = None, + ) -> None: + self.config = config + self.api_key = api_key + self.workspace = workspace + self.progress = progress + self._session_count = 0 + self._prompt_evidence: list[ArtifactReference] = [] + self._runtime_artifacts: list[ArtifactReference] = [] + + @property + def identity(self) -> RuntimeIdentity: + return RuntimeIdentity( + profile=self.config.profile, + driver_version=PI_VERSION, + model=self.config.model, + thinking_level=self.config.thinking_level, + ) + + @property + def prompt_evidence(self) -> tuple[ArtifactReference, ...]: + return tuple(self._prompt_evidence) + + @property + def runtime_artifacts(self) -> tuple[ArtifactReference, ...]: + return tuple(self._runtime_artifacts) + + def open_session(self, environment: BaseModel) -> CodePolicyRuntimeSession: + if not isinstance(environment, (FrozenMemoryEnvironment, LiveDimosEnvironment)): + raise TypeError(f"Unsupported CodePolicy environment: {type(environment).__name__}") + self._session_count += 1 + session_path = Path("runtime") / f"session-{self._session_count:04d}" + return CodePolicyRuntimeSession( + factory=self, + environment=environment, + relative_path=session_path, + ) + + def _record_prompt_evidence(self, references: list[ArtifactReference]) -> None: + self._prompt_evidence.extend(references) + self._runtime_artifacts.extend(references) + + def _record_runtime_artifact(self, reference: ArtifactReference) -> None: + self._runtime_artifacts.append(reference) + + +class CodePolicyRuntimeSession: + """One lifecycle-bounded, single-turn CodePolicy interaction.""" + + def __init__( + self, + *, + factory: CodePolicyRuntimeFactory, + environment: CodePolicyEnvironment, + relative_path: Path, + ) -> None: + self.factory = factory + self.environment = environment + self.relative_path = relative_path + self.path = factory.workspace / relative_path + self.server: CodePolicyMcpServer | None = None + self._ran = False + + def __enter__(self) -> CodePolicyRuntimeSession: + self.path.mkdir(parents=True) + self.server = CodePolicyMcpServer(CodePolicySessionConfig(environment=self.environment)) + self.server.start() + return self + + def __exit__(self, *_args: object) -> None: + if self.server is not None: + self.server.stop() + self.server = None + shutil.rmtree(self.path / "working", ignore_errors=True) + + def run(self, *, evaluation_protocol: str, task_input: str) -> AgentOutcome: + if self.server is None: + raise RuntimeError("CodePolicy session must be entered before run()") + if self._ran: + raise RuntimeError("code-policy-v1 sessions accept exactly one initial turn") + if not evaluation_protocol.strip() or not task_input.strip(): + raise ValueError("evaluation protocol and task input must be non-empty") + self._ran = True + + user_message = _assemble_user_message(evaluation_protocol, task_input) + evidence = self._write_prompt_evidence(evaluation_protocol, task_input, user_message) + self.factory._record_prompt_evidence(evidence) + working = self.path / "working" + working.mkdir() + cli, extension = _pi_paths() + runner = PiCliRunner( + cli=cli, + extension=extension, + model=self.factory.config.model, + thinking_level=self.factory.config.thinking_level, + timeout_s=TURN_TIMEOUT_SECONDS, + progress=self.factory.progress, + ) + try: + result = runner.run( + prompt=user_message, + system_prompt=SYSTEM_INSTRUCTIONS, + mcp_url=self.server.mcp_url, + api_key=self.factory.api_key, + run_dir=working, + ) + except PiRunError as exc: + self._record_stderr(exc.stderr) + raise + if result.transcript_path is not None: + target = self.path / "pi-transcript.jsonl" + shutil.copy2(result.transcript_path, target) + self.factory._record_runtime_artifact( + self._artifact(target, "Pi transcript", "application/x-ndjson") + ) + if result.stderr: + self._record_stderr(result.stderr) + return AgentOutcome( + final_text=result.final_text, + tool_call_count=result.tool_call_count, + duration_seconds=result.duration_seconds, + ) + + def _record_stderr(self, stderr: str) -> None: + if not stderr: + return + target = self.path / "stderr.log" + target.write_text(stderr, encoding="utf-8") + self.factory._record_runtime_artifact(self._artifact(target, "Pi stderr", "text/plain")) + + def _write_prompt_evidence( + self, + evaluation_protocol: str, + task_input: str, + user_message: str, + ) -> list[ArtifactReference]: + components = ( + ("runtime-system.txt", "runtime", SYSTEM_INSTRUCTIONS), + ("evaluation-protocol.txt", "evaluation", evaluation_protocol), + ("task-input.txt", "evaluation", task_input), + ("assembled-user-message.txt", "runtime", user_message), + ) + manifest_components: list[dict[str, str]] = [] + references: list[ArtifactReference] = [] + for filename, owner, text in components: + path = self.path / filename + path.write_text(text, encoding="utf-8") + manifest_components.append( + { + "path": path.relative_to(self.factory.workspace).as_posix(), + "owner": owner, + "sha256": hashlib.sha256(text.encode()).hexdigest(), + } + ) + references.append(self._artifact(path, filename, "text/plain")) + manifest = self.path / "prompt-assembly.json" + manifest.write_text( + json.dumps( + { + "schema_version": "1.0", + "runtime_profile": CODE_POLICY_PROFILE, + "components": manifest_components, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + references.append(self._artifact(manifest, "Prompt assembly", "application/json")) + return references + + def _artifact(self, path: Path, label: str, media_type: str) -> ArtifactReference: + return ArtifactReference( + path=path.relative_to(self.factory.workspace).as_posix(), + label=label, + media_type=media_type, + ) + + +def _assemble_user_message(evaluation_protocol: str, task_input: str) -> str: + return ( + "# Evaluation protocol\n\n" + f"{evaluation_protocol.strip()}\n\n" + "# Task input\n\n" + f"{task_input.strip()}\n" + ) + + +def _pi_paths() -> tuple[Path, Path]: + package = Path(__file__).resolve().parents[3] / "packages" / "pi-code-policy-extension" + cli = package / "node_modules" / "@earendil-works" / "pi-coding-agent" / "dist" / "cli.js" + extension = package / "dist" / "python-exec.js" + return cli, extension diff --git a/dimos/benchmark/evaluation/test_models.py b/dimos/benchmark/evaluation/test_models.py new file mode 100644 index 0000000000..447d4ab675 --- /dev/null +++ b/dimos/benchmark/evaluation/test_models.py @@ -0,0 +1,64 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime, timezone + +from pydantic import ValidationError +import pytest + +from dimos.benchmark.evaluation.models import ( + ArtifactReference, + EvaluationIdentity, + EvaluationReference, + EvaluationReport, + EvaluationRun, + EvaluationRunSpecification, + InlineNativeResult, + RuntimeIdentity, +) + + +def test_artifact_reference_rejects_paths_outside_run() -> None: + with pytest.raises(ValidationError, match="safe relative"): + ArtifactReference(path="../private.json", label="Private") + + +def test_completed_run_requires_evaluation_report() -> None: + now = datetime.now(timezone.utc) + + with pytest.raises(ValidationError, match="require a report"): + EvaluationRun( + run_id="run", + specification=EvaluationRunSpecification( + evaluation=EvaluationReference(name="fixture") + ), + evaluation=EvaluationIdentity(name="fixture", provider="tests", version="1"), + runtime=RuntimeIdentity( + driver_version="test", + model="gpt-5.6-luna", + thinking_level="medium", + ), + status="completed", + started_at=now, + finished_at=now, + duration_seconds=0, + ) + + +def test_native_result_preserves_nested_benchmark_payload() -> None: + payload = {"metrics": {"success_rate": 0.75}, "episodes": [True, False]} + + report = EvaluationReport(native_result=InlineNativeResult(value=payload)) + + assert report.model_dump(mode="json")["native_result"]["value"] == payload diff --git a/dimos/benchmark/agent_eval/test_pi_process.py b/dimos/benchmark/evaluation/test_pi_process.py similarity index 96% rename from dimos/benchmark/agent_eval/test_pi_process.py rename to dimos/benchmark/evaluation/test_pi_process.py index b7de377df9..2eaa9bedbf 100644 --- a/dimos/benchmark/agent_eval/test_pi_process.py +++ b/dimos/benchmark/evaluation/test_pi_process.py @@ -20,7 +20,7 @@ import pytest -from dimos.benchmark.agent_eval.pi_process import PiCliRunner, PiRunError, parse_pi_events +from dimos.benchmark.evaluation.pi_process import PiCliRunner, PiRunError, parse_pi_events def test_parse_stock_pi_events_uses_final_message_and_counts_tools() -> None: @@ -80,7 +80,7 @@ def test_stock_cli_streams_assistant_tools_and_stderr_while_running(mocker, tmp_ ) ) process.stderr = StringIO("provider secret connected\n") - mocker.patch("dimos.benchmark.agent_eval.pi_process.subprocess.Popen", return_value=process) + mocker.patch("dimos.benchmark.evaluation.pi_process.subprocess.Popen", return_value=process) progress = [] assistant_seen = threading.Event() @@ -152,7 +152,7 @@ def test_stock_cli_receives_only_api_key_and_evaluator_binding(mocker, tmp_path: process.stderr = StringIO() process.wait.return_value = 0 popen = mocker.patch( - "dimos.benchmark.agent_eval.pi_process.subprocess.Popen", return_value=process + "dimos.benchmark.evaluation.pi_process.subprocess.Popen", return_value=process ) runner = PiCliRunner( cli=cli, @@ -191,7 +191,7 @@ def test_stock_cli_timeout_terminates_the_child(mocker, tmp_path: Path) -> None: subprocess.TimeoutExpired("pi", 0.01), 0, ] - mocker.patch("dimos.benchmark.agent_eval.pi_process.subprocess.Popen", return_value=process) + mocker.patch("dimos.benchmark.evaluation.pi_process.subprocess.Popen", return_value=process) runner = PiCliRunner( cli=cli, extension=extension, diff --git a/dimos/benchmark/evaluation/test_registry.py b/dimos/benchmark/evaluation/test_registry.py new file mode 100644 index 0000000000..0c7b164d87 --- /dev/null +++ b/dimos/benchmark/evaluation/test_registry.py @@ -0,0 +1,98 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pydantic import BaseModel +import pytest + +from dimos.benchmark.evaluation.models import EvaluationReport, InlineNativeResult +from dimos.benchmark.evaluation.protocol import EvaluationContext +import dimos.benchmark.evaluation.registry as registry + + +class Config(BaseModel): + value: int + + +class PluginEvaluation: + name = "sample" + config_model: type[BaseModel] = Config + + def run(self, config: BaseModel, context: EvaluationContext) -> EvaluationReport: + return EvaluationReport(native_result=InlineNativeResult(value=None)) + + +class Distribution: + metadata = {"Name": "Acme_Evals"} + version = "2.0" + + +class EntryPoint: + name = "sample" + value = "acme.evals:sample" + dist = Distribution() + + def __init__(self, target) -> None: + self.target = target + + def load(self): + return self.target + + +def test_external_evaluation_uses_distribution_namespace(monkeypatch) -> None: + entry = EntryPoint(PluginEvaluation()) + monkeypatch.setattr( + registry.importlib_metadata, + "entry_points", + lambda **_kwargs: [entry], + ) + + resolved = registry.resolve_evaluation("acme-evals.sample") + + assert resolved.provider == "Acme_Evals" + assert resolved.version == "2.0" + assert resolved.evaluation is entry.target + + +def test_unknown_evaluation_lists_available_names(monkeypatch) -> None: + monkeypatch.setattr( + registry.importlib_metadata, + "entry_points", + lambda **_kwargs: [], + ) + + with pytest.raises(registry.EvaluationRegistryError, match="frozen-integer-qa"): + registry.resolve_evaluation("missing") + + +def test_external_target_must_implement_whole_evaluation(monkeypatch) -> None: + entry = EntryPoint(object()) + monkeypatch.setattr( + registry.importlib_metadata, + "entry_points", + lambda **_kwargs: [entry], + ) + + with pytest.raises(registry.EvaluationRegistryError, match="name, config_model, and run"): + registry.resolve_evaluation("acme-evals.sample") + + +def test_duplicate_external_evaluation_names_are_rejected(monkeypatch) -> None: + monkeypatch.setattr( + registry.importlib_metadata, + "entry_points", + lambda **_kwargs: [EntryPoint(PluginEvaluation()), EntryPoint(PluginEvaluation())], + ) + + with pytest.raises(registry.EvaluationRegistryError, match="Multiple installed"): + registry.available_evaluations() diff --git a/dimos/benchmark/evaluation/test_runner.py b/dimos/benchmark/evaluation/test_runner.py new file mode 100644 index 0000000000..d89931bda4 --- /dev/null +++ b/dimos/benchmark/evaluation/test_runner.py @@ -0,0 +1,188 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from pathlib import Path + +from pydantic import BaseModel, ConfigDict +import pytest + +from dimos.benchmark.evaluation.models import ( + CodePolicyAgentConfig, + EvaluationReport, + InlineNativeResult, + RuntimeIdentity, + SummaryItem, +) +from dimos.benchmark.evaluation.protocol import AgentOutcome, EvaluationContext +from dimos.benchmark.evaluation.registry import ResolvedEvaluation +import dimos.benchmark.evaluation.runner as runner + + +class HarnessConfig(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + samples: tuple[int, ...] + + +class Environment(BaseModel): + sample: int + + +class NativeHarnessEvaluation: + name = "native-harness" + config_model: type[BaseModel] = HarnessConfig + + def run(self, config: BaseModel, context: EvaluationContext) -> EvaluationReport: + assert isinstance(config, HarnessConfig) + predictions = [] + for sample in config.samples: + with context.agent.open_session(Environment(sample=sample)) as session: + outcome = session.run( + evaluation_protocol="Return the native benchmark answer.", + task_input=str(sample), + ) + predictions.append(int(outcome.final_text)) + native = { + "benchmark": "fixture", + "predictions": predictions, + "aggregate": {"sum": sum(predictions)}, + } + return EvaluationReport( + summary=(SummaryItem(key="native_sum", label="Native sum", value=sum(predictions)),), + native_result=InlineNativeResult(value=native), + ) + + +class FailingEvaluation: + name = "native-harness" + config_model: type[BaseModel] = HarnessConfig + + def run(self, config: BaseModel, context: EvaluationContext) -> EvaluationReport: + raise RuntimeError("credential=secret") + + +class FakeSession: + def __init__(self, sample: int) -> None: + self.sample = sample + + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def run(self, *, evaluation_protocol: str, task_input: str) -> AgentOutcome: + assert evaluation_protocol == "Return the native benchmark answer." + assert task_input == str(self.sample) + return AgentOutcome(str(self.sample * 2), 1, 0.01) + + +class FakeRuntime: + def __init__(self, *, config: CodePolicyAgentConfig, **_kwargs) -> None: + self.config = config + self.prompt_evidence = () + self.runtime_artifacts = () + + @property + def identity(self) -> RuntimeIdentity: + return RuntimeIdentity( + driver_version="test", + model=self.config.model, + thinking_level=self.config.thinking_level, + ) + + def open_session(self, environment: BaseModel) -> FakeSession: + assert isinstance(environment, Environment) + return FakeSession(environment.sample) + + +def _write_spec(tmp_path: Path) -> Path: + path = tmp_path / "spec.json" + path.write_text( + json.dumps( + { + "schema_version": "1.0", + "evaluation": { + "name": "native-harness", + "config": {"samples": [2, 3]}, + }, + "agent": {"profile": "code-policy-v1"}, + } + ) + ) + return path + + +def test_native_harness_owns_loop_scoring_and_aggregation(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "secret") + monkeypatch.setattr(runner, "CodePolicyRuntimeFactory", FakeRuntime) + monkeypatch.setattr( + runner, + "resolve_evaluation", + lambda _name: ResolvedEvaluation( + name="native-harness", + provider="fixture", + version="1", + evaluation=NativeHarnessEvaluation(), + ), + ) + output = tmp_path / "output" + + result = runner.execute_evaluation(_write_spec(tmp_path), output=output) + + assert result.status == "completed" + assert result.report is not None + assert result.report.native_result.value == { + "benchmark": "fixture", + "predictions": [4, 6], + "aggregate": {"sum": 10}, + } + assert json.loads((output / "run.json").read_text())["status"] == "completed" + + +def test_nonempty_output_is_rejected_before_execution(tmp_path: Path) -> None: + output = tmp_path / "output" + output.mkdir() + (output / "keep").write_text("user data") + + with pytest.raises(FileExistsError, match="absent or an empty"): + runner.execute_evaluation(_write_spec(tmp_path), output=output) + + assert (output / "keep").read_text() == "user data" + + +def test_started_failure_is_published_with_credentials_redacted( + monkeypatch, + tmp_path: Path, +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "secret") + monkeypatch.setattr(runner, "CodePolicyRuntimeFactory", FakeRuntime) + monkeypatch.setattr( + runner, + "resolve_evaluation", + lambda _name: ResolvedEvaluation( + name="native-harness", + provider="fixture", + version="1", + evaluation=FailingEvaluation(), + ), + ) + output = tmp_path / "output" + + result = runner.execute_evaluation(_write_spec(tmp_path), output=output) + + assert result.status == "failed" + assert result.error is not None + assert result.error.message == "credential=[REDACTED]" + assert "secret" not in (output / "run.json").read_text() diff --git a/dimos/benchmark/evaluation/test_runtime.py b/dimos/benchmark/evaluation/test_runtime.py new file mode 100644 index 0000000000..77c6493b8e --- /dev/null +++ b/dimos/benchmark/evaluation/test_runtime.py @@ -0,0 +1,117 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import json +from pathlib import Path + +import pytest + +from dimos.agents.code_policy_core import FrozenMemoryEnvironment +from dimos.benchmark.evaluation.models import CodePolicyAgentConfig +from dimos.benchmark.evaluation.pi_process import PiRunError, PiRunResult +import dimos.benchmark.evaluation.runtime as runtime + + +class FakeServer: + mcp_url = "http://127.0.0.1:1234/mcp" + + def __init__(self, config) -> None: + self.config = config + self.started = False + + def start(self) -> None: + self.started = True + + def stop(self) -> None: + self.started = False + + +class FakeRunner: + def __init__(self, **kwargs) -> None: + self.kwargs = kwargs + + def run(self, *, run_dir: Path, prompt: str, system_prompt: str, **_kwargs): + transcript = run_dir / "native.jsonl" + transcript.write_text('{"type":"session"}\n') + return PiRunResult("ANSWER: 2", 3, 1.0, transcript, "") + + +class FailingRunner(FakeRunner): + def run(self, **_kwargs): + raise PiRunError("Pi failed", stderr="diagnostic") + + +def test_runtime_records_separate_prompt_components(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setattr(runtime, "CodePolicyMcpServer", FakeServer) + monkeypatch.setattr(runtime, "PiCliRunner", FakeRunner) + marker = tmp_path / "exists" + marker.touch() + monkeypatch.setattr(runtime, "_pi_paths", lambda: (marker, marker)) + factory = runtime.CodePolicyRuntimeFactory( + config=CodePolicyAgentConfig(), + api_key="secret", + workspace=tmp_path, + ) + environment = FrozenMemoryEnvironment( + recording_path="source.db", + derived_recording_path="derived.db", + memory_cutoff_timestamp=1.0, + ) + + with factory.open_session(environment) as session: + outcome = session.run( + evaluation_protocol="End with ANSWER: .", + task_input="How many rooms?", + ) + + assert outcome.final_text == "ANSWER: 2" + session_path = tmp_path / "runtime" / "session-0001" + assert (session_path / "evaluation-protocol.txt").read_text() == ("End with ANSWER: .") + assert (session_path / "task-input.txt").read_text() == "How many rooms?" + assembly = json.loads((session_path / "prompt-assembly.json").read_text()) + task = next(item for item in assembly["components"] if item["path"].endswith("task-input.txt")) + assert task["owner"] == "evaluation" + assert task["sha256"] == hashlib.sha256(b"How many rooms?").hexdigest() + assert not (session_path / "working").exists() + assert {item.path for item in factory.prompt_evidence} >= { + "runtime/session-0001/runtime-system.txt", + "runtime/session-0001/evaluation-protocol.txt", + "runtime/session-0001/task-input.txt", + } + + +def test_runtime_retains_pi_stderr_on_failure(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setattr(runtime, "CodePolicyMcpServer", FakeServer) + monkeypatch.setattr(runtime, "PiCliRunner", FailingRunner) + marker = tmp_path / "exists" + marker.touch() + monkeypatch.setattr(runtime, "_pi_paths", lambda: (marker, marker)) + factory = runtime.CodePolicyRuntimeFactory( + config=CodePolicyAgentConfig(), + api_key="secret", + workspace=tmp_path, + ) + environment = FrozenMemoryEnvironment( + recording_path="source.db", + derived_recording_path="derived.db", + memory_cutoff_timestamp=1.0, + ) + + with pytest.raises(PiRunError, match="Pi failed"): + with factory.open_session(environment) as session: + session.run(evaluation_protocol="Use memory.", task_input="Question") + + assert (tmp_path / "runtime/session-0001/stderr.log").read_text() == "diagnostic" + assert factory.runtime_artifacts[-1].path == "runtime/session-0001/stderr.log" diff --git a/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/README.md b/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/README.md index 823d39aa3c..52ac5c7a0a 100644 --- a/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/README.md +++ b/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/README.md @@ -10,5 +10,6 @@ failed task score as an agent or mapping regression. The authoritative case remains incomplete until a human-authored room inventory, counting policy, and independent review establish the expected count. -Use this case to exercise the direct stock-Pi CLI path. Any observed answer is -experimental until the oracle is replaced with a reviewed room inventory. +Use `run.json` to exercise the direct stock-Pi CLI path; it references the +Evaluation-owned `case.json`. Any observed answer is experimental until the +oracle is replaced with a reviewed room inventory. diff --git a/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/run.json b/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/run.json new file mode 100644 index 0000000000..20d64fef70 --- /dev/null +++ b/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/run.json @@ -0,0 +1,14 @@ +{ + "schema_version": "1.0", + "evaluation": { + "name": "frozen-integer-qa", + "config": { + "case": "case.json" + } + }, + "agent": { + "profile": "code-policy-v1", + "model": "gpt-5.6-luna", + "thinking_level": "medium" + } +} diff --git a/dimos/benchmark/short_horizon_qa/evaluation.py b/dimos/benchmark/short_horizon_qa/evaluation.py new file mode 100644 index 0000000000..90f6369307 --- /dev/null +++ b/dimos/benchmark/short_horizon_qa/evaluation.py @@ -0,0 +1,195 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Frozen integer QA as one complete built-in Evaluation.""" + +from __future__ import annotations + +import errno +from pathlib import Path +import re +import time + +from openevals.exact import exact_match +from pydantic import BaseModel + +from dimos.agents.code_policy_core import FrozenMemoryEnvironment +from dimos.benchmark.evaluation.models import ( + EvaluationReport, + InlineNativeResult, + SummaryItem, +) +from dimos.benchmark.evaluation.progress import ( + CaseHeaderProgress, + StatusProgress, + emit_progress, +) +from dimos.benchmark.evaluation.protocol import EvaluationContext +from dimos.benchmark.short_horizon_qa.integer_answer import ( + load_exact_integer_oracle, + parse_integer_prediction, +) +from dimos.benchmark.short_horizon_qa.models import ( + FrozenIntegerQaCase, + FrozenIntegerQaConfig, + MapperSettings, +) +from dimos.benchmark.short_horizon_qa.prepare import prepare_bundle +from dimos.benchmark.short_horizon_qa.service import load_bundle +from dimos.constants import CACHE_DIR +from dimos.memory2.cli.dataset import resolve_dataset + +EVALUATION_PROTOCOL = """Use `python_exec` to inspect the read-only `memory` object +for the frozen robot recording. Compute the requested integer from the recording; +do not guess. End with exactly one line in this form: + +ANSWER: +""" + + +class FrozenIntegerQaEvaluation: + name = "frozen-integer-qa" + config_model: type[BaseModel] = FrozenIntegerQaConfig + + def run(self, config: BaseModel, context: EvaluationContext) -> EvaluationReport: + if not isinstance(config, FrozenIntegerQaConfig): + raise TypeError("frozen-integer-qa received the wrong configuration type") + case_path = Path(config.case).expanduser() + if not case_path.is_absolute(): + case_path = context.spec_dir / case_path + case_path = case_path.resolve() + case = FrozenIntegerQaCase.model_validate_json(case_path.read_bytes()) + oracle = load_exact_integer_oracle(case, case_path.parent) + emit_progress( + context.progress, + CaseHeaderProgress( + case_id=case.case_id, + source=case.source.recording, + progress=case.source.progress, + question=case.task.prompt, + ), + ) + bundle = _materialize_frozen_memory(case, context) + _, cutoff, source_path, derived_path = load_bundle( + bundle, + progress=case.source.progress, + ) + emit_progress( + context.progress, + StatusProgress(channel="eval", message="memory ready"), + ) + started = time.monotonic() + with context.agent.open_session( + FrozenMemoryEnvironment( + recording_path=str(source_path), + derived_recording_path=str(derived_path), + memory_cutoff_timestamp=cutoff.cutoff_timestamp, + ) + ) as session: + outcome = session.run( + evaluation_protocol=EVALUATION_PROTOCOL, + task_input=case.task.prompt, + ) + prediction = parse_integer_prediction(outcome.final_text) + native_result = exact_match( + outputs={ + "status": prediction.status, + "integer_answer": prediction.integer_answer, + }, + reference_outputs={ + "status": "parsed", + "integer_answer": oracle.expected_count, + }, + ) + return EvaluationReport( + summary=( + SummaryItem(key="case", label="Case", value=case.case_id), + SummaryItem( + key="recording", + label="Recording", + value=f"{case.source.recording} @ {case.source.progress * 100:g}%", + ), + SummaryItem( + key="answer", + label="Answer", + value=prediction.integer_answer, + ), + SummaryItem( + key="exact_match", + label="Exact match", + value=bool(native_result["score"]), + ), + SummaryItem( + key="tool_calls", + label="Tool calls", + value=outcome.tool_call_count, + ), + SummaryItem( + key="duration", + label="Duration", + value=f"{time.monotonic() - started:.1f}s", + ), + ), + native_result=InlineNativeResult(value=native_result), + ) + + +def _materialize_frozen_memory( + case: FrozenIntegerQaCase, + context: EvaluationContext, +) -> Path: + source_path = resolve_dataset(case.source.recording).resolve() + stat = source_path.stat() + stem = re.sub(r"[^A-Za-z0-9_.-]+", "-", source_path.stem)[:64] + mapper = MapperSettings() + raw_key = ( + f"{stem}-{stat.st_size}-{stat.st_mtime_ns}-p{case.source.progress:.9f}-" + f"v{mapper.voxel_size_m}-b{mapper.block_count}-d{mapper.device}-" + f"c{int(mapper.carve_columns)}-f{mapper.frame_id}-e{mapper.emit_every}" + ) + key = re.sub(r"[^A-Za-z0-9_.-]+", "-", raw_key) + bundle = CACHE_DIR / "evaluation" / "frozen_memory" / key + manifest = bundle / "manifest.v1.json" + if not manifest.is_file(): + emit_progress( + context.progress, + StatusProgress(channel="eval", message="preparing memory"), + ) + bundle.parent.mkdir(parents=True, exist_ok=True) + try: + prepare_bundle( + case.source.recording, + [], + bundle, + progress=[case.source.progress], + mapper=mapper, + map_progress=lambda current, total: emit_progress( + context.progress, + StatusProgress( + channel="eval", + message=f"mapping {current}/{total} frames", + ), + ), + ) + except OSError as exc: + concurrent_publish = isinstance(exc, FileExistsError) or exc.errno in { + errno.EEXIST, + errno.ENOTEMPTY, + } + if not concurrent_publish or not manifest.is_file(): + raise + return bundle + + +frozen_integer_qa = FrozenIntegerQaEvaluation() diff --git a/dimos/benchmark/short_horizon_qa/eval.py b/dimos/benchmark/short_horizon_qa/integer_answer.py similarity index 63% rename from dimos/benchmark/short_horizon_qa/eval.py rename to dimos/benchmark/short_horizon_qa/integer_answer.py index 72916e9615..174b56de07 100644 --- a/dimos/benchmark/short_horizon_qa/eval.py +++ b/dimos/benchmark/short_horizon_qa/integer_answer.py @@ -12,42 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Exact-integer validation for short-horizon frozen-memory questions.""" +"""Integer-answer decoding and private oracle loading for frozen-memory questions.""" from __future__ import annotations from pathlib import Path import re -from typing import Any, Literal -from pydantic import Field - -from dimos.benchmark.agent_eval.models import ( - BaseEvalModel, - EvalCase, - ExactIntegerValidatorRef, +from dimos.benchmark.short_horizon_qa.models import ( + ExactIntegerOracle, + FrozenIntegerQaCase, + IntegerPrediction, ) _ANSWER_LINE = re.compile(r"(?m)^ANSWER:\s*") _TERMINAL_INTEGER = re.compile(r"(?:^|\n)ANSWER:\s*(-?\d+)\s*\Z") -class ExactIntegerOracle(BaseEvalModel): - expected_count: int = Field(ge=0) - counting_policy: str = Field(min_length=1) - rooms: tuple[dict[str, Any], ...] = () - reviewed_by: tuple[str, ...] = Field(min_length=1) - - -class IntegerPrediction(BaseEvalModel): - status: Literal["parsed", "invalid"] - integer_answer: int | None = None - - -def load_exact_integer_oracle(case: EvalCase, case_root: Path) -> ExactIntegerOracle: +def load_exact_integer_oracle(case: FrozenIntegerQaCase, case_root: Path) -> ExactIntegerOracle: reference = case.validator - if not isinstance(reference, ExactIntegerValidatorRef): - raise TypeError("case does not use exact-integer validation") root = case_root.resolve() path = (root / reference.private_path).resolve() if root not in path.parents: diff --git a/dimos/benchmark/short_horizon_qa/models.py b/dimos/benchmark/short_horizon_qa/models.py index 85b6d042a1..eb26237509 100644 --- a/dimos/benchmark/short_horizon_qa/models.py +++ b/dimos/benchmark/short_horizon_qa/models.py @@ -17,7 +17,8 @@ from __future__ import annotations import math -from typing import Literal +from pathlib import PurePosixPath +from typing import Annotated, Any, Literal from pydantic import BaseModel, ConfigDict, Field, model_validator @@ -70,3 +71,66 @@ class FrozenMemoryManifest(FrozenQaModel): derived_path: Literal["derived.db"] = "derived.db" mapper: MapperSettings cutoffs: tuple[CutoffRecord, ...] = Field(min_length=1) + + +NonEmpty = Annotated[str, Field(min_length=1)] + + +class FrozenRecordingSource(FrozenQaModel): + schema_version: Literal["1.0"] = "1.0" + kind: Literal["frozen_memory"] = "frozen_memory" + recording: NonEmpty + progress: float = Field(ge=0, le=1, allow_inf_nan=False) + + @model_validator(mode="after") + def finite_progress(self) -> FrozenRecordingSource: + if not math.isfinite(self.progress): + raise ValueError("recording progress must be finite") + return self + + +class IntegerQuestionTask(FrozenQaModel): + schema_version: Literal["1.0"] = "1.0" + kind: Literal["integer_question"] = "integer_question" + prompt: NonEmpty + answer_marker: Literal["ANSWER:"] = "ANSWER:" + + +class ExactIntegerValidatorRef(FrozenQaModel): + schema_version: Literal["1.0"] = "1.0" + kind: Literal["exact_integer"] = "exact_integer" + revision: NonEmpty + private_path: NonEmpty + + @model_validator(mode="after") + def safe_relative_path(self) -> ExactIntegerValidatorRef: + path = PurePosixPath(self.private_path) + if path.is_absolute() or not path.parts or ".." in path.parts: + raise ValueError("validator private_path must be a safe relative path") + return self + + +class FrozenIntegerQaCase(FrozenQaModel): + schema_version: Literal["1.0"] = "1.0" + case_id: NonEmpty + source: FrozenRecordingSource + task: IntegerQuestionTask + validator: ExactIntegerValidatorRef + + +class FrozenIntegerQaConfig(FrozenQaModel): + case: NonEmpty + + +class ExactIntegerOracle(FrozenQaModel): + schema_version: Literal["1.0"] = "1.0" + expected_count: int = Field(ge=0) + counting_policy: str = Field(min_length=1) + rooms: tuple[dict[str, Any], ...] = () + reviewed_by: tuple[str, ...] = Field(min_length=1) + + +class IntegerPrediction(FrozenQaModel): + schema_version: Literal["1.0"] = "1.0" + status: Literal["parsed", "invalid"] + integer_answer: int | None = None diff --git a/dimos/benchmark/short_horizon_qa/test_evaluation.py b/dimos/benchmark/short_horizon_qa/test_evaluation.py new file mode 100644 index 0000000000..31a3172a88 --- /dev/null +++ b/dimos/benchmark/short_horizon_qa/test_evaluation.py @@ -0,0 +1,169 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +from types import SimpleNamespace + +from dimos.benchmark.evaluation.protocol import AgentOutcome, EvaluationContext +import dimos.benchmark.short_horizon_qa.evaluation as evaluation +from dimos.benchmark.short_horizon_qa.models import ( + ExactIntegerValidatorRef, + FrozenIntegerQaCase, + FrozenIntegerQaConfig, + FrozenRecordingSource, + IntegerQuestionTask, +) + + +class FakeSession: + def __init__(self, captured: dict) -> None: + self.captured = captured + + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def run(self, *, evaluation_protocol: str, task_input: str) -> AgentOutcome: + self.captured.update(protocol=evaluation_protocol, task=task_input) + return AgentOutcome("Checked\nANSWER: 2", 3, 1.0) + + +class FakeAgent: + def __init__(self, captured: dict) -> None: + self.captured = captured + + def open_session(self, environment): + self.captured["environment"] = environment + return FakeSession(self.captured) + + +def _case(tmp_path: Path) -> Path: + private = tmp_path / "private" + private.mkdir() + (private / "oracle.json").write_text( + '{"schema_version":"1.0","expected_count":2,' + '"counting_policy":"count rooms","rooms":[],' + '"reviewed_by":["reviewer"]}' + ) + case = FrozenIntegerQaCase( + case_id="demo", + source=FrozenRecordingSource(recording="recording", progress=1.0), + task=IntegerQuestionTask(prompt="How many rooms?"), + validator=ExactIntegerValidatorRef( + revision="v1", + private_path="private/oracle.json", + ), + ) + path = tmp_path / "case.json" + path.write_text(case.model_dump_json()) + return path + + +def test_frozen_evaluation_owns_protocol_decoder_and_openevals( + monkeypatch, + tmp_path: Path, +) -> None: + case_path = _case(tmp_path) + bundle = tmp_path / "bundle" + bundle.mkdir() + captured = {} + monkeypatch.setattr(evaluation, "_materialize_frozen_memory", lambda *_args: bundle) + monkeypatch.setattr( + evaluation, + "load_bundle", + lambda *_args, **_kwargs: ( + object(), + SimpleNamespace(cutoff_timestamp=10.0), + tmp_path / "source.db", + tmp_path / "derived.db", + ), + ) + + def exact_match(*, outputs, reference_outputs): + captured.update(outputs=outputs, reference_outputs=reference_outputs) + return {"key": "exact_match", "score": True, "comment": None} + + monkeypatch.setattr(evaluation, "exact_match", exact_match) + context = EvaluationContext( + run_id="run", + spec_dir=tmp_path, + workspace=tmp_path / "workspace", + agent=FakeAgent(captured), + progress=None, + ) + + report = evaluation.frozen_integer_qa.run( + FrozenIntegerQaConfig(case=case_path.name), + context, + ) + + assert captured["task"] == "How many rooms?" + assert "ANSWER: " in captured["protocol"] + assert captured["outputs"] == {"status": "parsed", "integer_answer": 2} + assert captured["reference_outputs"] == { + "status": "parsed", + "integer_answer": 2, + } + assert report.native_result.value == { + "key": "exact_match", + "score": True, + "comment": None, + } + assert [item.key for item in report.summary] == [ + "case", + "recording", + "answer", + "exact_match", + "tool_calls", + "duration", + ] + + +def test_materialize_accepts_bundle_published_by_concurrent_runner( + monkeypatch, + tmp_path: Path, +) -> None: + recording = tmp_path / "recording.db" + recording.touch() + case = FrozenIntegerQaCase( + case_id="concurrent", + source=FrozenRecordingSource(recording=str(recording), progress=1.0), + task=IntegerQuestionTask(prompt="How many rooms?"), + validator=ExactIntegerValidatorRef( + revision="v1", + private_path="private/oracle.json", + ), + ) + monkeypatch.setattr(evaluation, "CACHE_DIR", tmp_path / "cache") + monkeypatch.setattr(evaluation, "resolve_dataset", lambda _recording: recording) + + def publish_first(_recording, _cutoffs, output: Path, **_kwargs) -> None: + output.mkdir() + (output / "manifest.v1.json").write_text("{}") + raise FileExistsError(output) + + monkeypatch.setattr(evaluation, "prepare_bundle", publish_first) + context = EvaluationContext( + run_id="run", + spec_dir=tmp_path, + workspace=tmp_path, + agent=FakeAgent({}), + progress=None, + ) + + bundle = evaluation._materialize_frozen_memory(case, context) + + assert bundle.joinpath("manifest.v1.json").is_file() diff --git a/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py b/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py index 0c7def1bfa..7406146321 100644 --- a/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py +++ b/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py @@ -18,8 +18,7 @@ import pytest -from dimos.benchmark.agent_eval.models import EvalCase -from dimos.benchmark.short_horizon_qa.models import MapperSettings +from dimos.benchmark.short_horizon_qa.models import FrozenIntegerQaCase, MapperSettings from dimos.benchmark.short_horizon_qa.prepare import prepare_bundle from dimos.utils.data import get_data @@ -29,7 +28,7 @@ def test_real_hongkong_recording_prepares_direct_demo_case(tmp_path: Path) -> No case_path = ( Path(__file__).parent / "cases" / "demo_go2_hongkong_office-room-count-smoke" / "case.json" ) - case = EvalCase.model_validate_json(case_path.read_bytes()) + case = FrozenIntegerQaCase.model_validate_json(case_path.read_bytes()) map_progress: list[tuple[int, int]] = [] manifest = prepare_bundle( get_data("go2_hongkong_office.db"), diff --git a/dimos/benchmark/short_horizon_qa/test_eval.py b/dimos/benchmark/short_horizon_qa/test_integer_answer.py similarity index 93% rename from dimos/benchmark/short_horizon_qa/test_eval.py rename to dimos/benchmark/short_horizon_qa/test_integer_answer.py index 87d4adba4f..56475a4490 100644 --- a/dimos/benchmark/short_horizon_qa/test_eval.py +++ b/dimos/benchmark/short_horizon_qa/test_integer_answer.py @@ -14,7 +14,7 @@ import pytest -from dimos.benchmark.short_horizon_qa.eval import parse_integer_prediction +from dimos.benchmark.short_horizon_qa.integer_answer import parse_integer_prediction @pytest.mark.parametrize( diff --git a/dimos/cli/eval.py b/dimos/cli/eval.py index d3517a5c2a..781c8692fc 100644 --- a/dimos/cli/eval.py +++ b/dimos/cli/eval.py @@ -12,25 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Dependency-light shell for the immutable single-case evaluation CLI.""" +"""Dependency-light shell for the unified Evaluation CLI.""" from __future__ import annotations from pathlib import Path import threading -from typing import Any, Literal +from typing import Any import typer -app = typer.Typer(help="Run immutable agent evaluation cases", no_args_is_help=True) +app = typer.Typer(help="Run executable evaluations", no_args_is_help=True) MAX_RENDERED_TOOL_RESULT_CHARS = 2_000 -def execute_single_case(*args: Any, **kwargs: Any) -> Any: +def execute_evaluation(*args: Any, **kwargs: Any) -> Any: """Import and dispatch the evaluation runtime only when ``eval run`` executes.""" try: - from dimos.benchmark.agent_eval.single_case import execute_single_case as execute + from dimos.benchmark.evaluation.runner import execute_evaluation as execute except ModuleNotFoundError as exc: raise RuntimeError( "Evaluation dependencies are missing; run `uv sync --extra agents`" @@ -41,35 +41,19 @@ def execute_single_case(*args: Any, **kwargs: Any) -> Any: @app.command("run") def run( - case: Path = typer.Argument(..., exists=True, dir_okay=False, readable=True), - agent_backend: Literal["pi"] = typer.Option("pi", "--agent.backend"), - agent_model: Literal["gpt-5.6-luna"] = typer.Option("gpt-5.6-luna", "--agent.model"), - thinking_level: Literal["medium"] = typer.Option("medium", "--agent.thinking-level"), - api_key_env: str = typer.Option("OPENAI_API_KEY", "--agent.api-key-env"), + specification: Path = typer.Argument(..., exists=True, dir_okay=False, readable=True), + api_key_env: str = typer.Option("OPENAI_API_KEY", "--api-key-env"), output: Path = typer.Option(..., "--output"), json_output: bool = typer.Option(False, "--json", help="Print compact JSON"), quiet: bool = typer.Option(False, "--quiet", help="Suppress live evaluation progress"), ) -> None: - """Run one static evaluation case synchronously.""" - from dimos.benchmark.agent_eval.models import ( - EvalRunConfig, - PiAgentConfig, - ) - - config = EvalRunConfig( - agent=PiAgentConfig( - backend=agent_backend, - model=agent_model, - thinking_level=thinking_level, - api_key_env=api_key_env, - ) - ) + """Run one Evaluation Run Specification synchronously.""" renderer = None if quiet else ProgressRenderer() try: - result = execute_single_case( - case, - config=config, + result = execute_evaluation( + specification, output=output, + api_key_env=api_key_env, progress=renderer, ) except Exception as exc: @@ -80,31 +64,36 @@ def run( if renderer is not None: renderer.finish() typer.echo(result.model_dump_json() if json_output else format_result(result, output)) - if result.attempt_status == "failed": + if result.status == "cancelled": + raise typer.Exit(130) + if result.status == "failed": raise typer.Exit(1) def format_result(result: Any, output: Path | None = None) -> str: - """Render the compact typed result without exposing private oracle material.""" - if result.attempt_status == "failed": - heading = "! Evaluation not evaluated" - elif result.task_result == "passed": - heading = "✓ Evaluation passed" - else: - heading = "✗ Evaluation failed" - source = f"{result.recording} @ {result.progress * 100:g}%" - answer = str(result.integer_answer) if result.integer_answer is not None else "—" - rows = ( - ("Case", result.case_id), - ("Source", source), - ("Answer", answer), - ("Result", result.task_result), - ("Agent", f"Pi · {result.model} · {result.thinking_level}"), - ("Tool calls", str(result.tool_call_count)), - ("Duration", f"{result.duration_seconds:.1f}s"), - ("Output", str((output / "result.json") if output is not None else "result.json")), - ) - body = "\n".join(f" {label:<10} {value}" for label, value in rows) + """Render infrastructure status plus Evaluation-supplied summary rows.""" + heading = { + "completed": "✓ Evaluation completed", + "failed": "! Evaluation failed", + "cancelled": "! Evaluation cancelled", + }[result.status] + rows = [ + ("Evaluation", result.evaluation.name), + ( + "Agent", + f"Pi · {result.runtime.model} · {result.runtime.thinking_level}", + ), + ] + if result.report is not None: + rows.extend( + (item.label, "—" if item.value is None else str(item.value)) + for item in result.report.summary + ) + if result.error is not None: + rows.append(("Error", f"{result.error.error_type}: {result.error.message}")) + rows.append(("Output", str((output / "run.json") if output is not None else "run.json"))) + width = max(len(label) for label, _ in rows) + body = "\n".join(f" {label:<{width}} {value}" for label, value in rows) return f"{heading}\n\n{body}" diff --git a/dimos/cli/test_eval.py b/dimos/cli/test_eval.py index c94ded418a..47f1516a58 100644 --- a/dimos/cli/test_eval.py +++ b/dimos/cli/test_eval.py @@ -13,6 +13,7 @@ # limitations under the License. import builtins +from datetime import datetime, timezone import json from pathlib import Path import subprocess @@ -23,8 +24,19 @@ import pytest from typer.testing import CliRunner -from dimos.benchmark.agent_eval.models import CompactEvalResult -from dimos.benchmark.agent_eval.progress import ( +from dimos.benchmark.evaluation.models import ( + CodePolicyAgentConfig, + EvaluationIdentity, + EvaluationReference, + EvaluationReport, + EvaluationRun, + EvaluationRunError, + EvaluationRunSpecification, + InlineNativeResult, + RuntimeIdentity, + SummaryItem, +) +from dimos.benchmark.evaluation.progress import ( AssistantTextProgress, StatusProgress, ToolEndProgress, @@ -33,46 +45,81 @@ import dimos.cli.eval as eval_cli -def _result(*, passed: bool | None = True) -> CompactEvalResult: - return CompactEvalResult( - case_id="demo-room-count", - recording="go2_hongkong_office", - progress=1.0, - model="gpt-5.6-luna", - thinking_level="medium", - final_response="ANSWER: 4" if passed is not None else "", - prediction_status="parsed" if passed is not None else "not_evaluated", - integer_answer=4 if passed is not None else None, - passed=passed, - validator_revision="v1", - tool_call_count=7, - duration_seconds=42.75, - infra_error="Pi failed" if passed is None else None, +def _result(status: str = "completed") -> EvaluationRun: + now = datetime.now(timezone.utc) + completed = status == "completed" + return EvaluationRun( + run_id="run-1", + specification=EvaluationRunSpecification( + evaluation=EvaluationReference(name="fixture", config={}), + agent=CodePolicyAgentConfig(), + ), + evaluation=EvaluationIdentity(name="fixture", provider="tests", version="1"), + runtime=RuntimeIdentity( + driver_version="test", + model="gpt-5.6-luna", + thinking_level="medium", + ), + status=status, + started_at=now, + finished_at=now, + duration_seconds=1.0, + report=( + EvaluationReport( + summary=(SummaryItem(key="native_score", label="Native score", value=0.5),), + native_result=InlineNativeResult(value={"score": 0.5}), + ) + if completed + else None + ), + error=( + None + if completed + else EvaluationRunError( + stage="evaluation", + error_type="RuntimeError", + message="agent failed", + ) + ), ) -def _case(tmp_path: Path) -> Path: - path = tmp_path / "case.json" +def _spec(tmp_path: Path) -> Path: + path = tmp_path / "spec.json" path.write_text("{}") return path -def test_eval_run_uses_api_key_default_and_separates_progress(tmp_path, monkeypatch) -> None: +def test_eval_run_uses_operational_api_key_and_renders_native_summary( + tmp_path, + monkeypatch, +) -> None: captured = {} - def execute(path, *, config, progress, output): - captured.update(path=path, config=config, progress=progress, output=output) - progress(StatusProgress(channel="eval", message="loading case")) + def execute(path, *, api_key_env, progress, output): + captured.update( + path=path, + api_key_env=api_key_env, + progress=progress, + output=output, + ) + progress(StatusProgress(channel="eval", message="loading specification")) return _result() - monkeypatch.setattr(eval_cli, "execute_single_case", execute) + monkeypatch.setattr(eval_cli, "execute_evaluation", execute) output = tmp_path / "run" - result = CliRunner().invoke(main, ["eval", "run", str(_case(tmp_path)), f"--output={output}"]) + + result = CliRunner().invoke( + main, + ["eval", "run", str(_spec(tmp_path)), f"--output={output}"], + ) + assert result.exit_code == 0, result.output - assert captured["config"].agent.api_key_env == "OPENAI_API_KEY" + assert captured["api_key_env"] == "OPENAI_API_KEY" assert captured["output"] == output - assert "✓ Evaluation passed" in result.stdout - assert "[eval] loading case" in result.stderr + assert "✓ Evaluation completed" in result.stdout + assert "Native score" in result.stdout + assert "[eval] loading specification" in result.stderr def test_eval_run_accepts_named_api_key_env_and_json(tmp_path, monkeypatch) -> None: @@ -82,54 +129,66 @@ def execute(*args, **kwargs): captured.update(kwargs) return _result() - monkeypatch.setattr(eval_cli, "execute_single_case", execute) + monkeypatch.setattr(eval_cli, "execute_evaluation", execute) output = tmp_path / "run" result = CliRunner().invoke( main, [ "eval", "run", - str(_case(tmp_path)), - "--agent.api-key-env=MY_OPENAI_KEY", + str(_spec(tmp_path)), + "--api-key-env=MY_OPENAI_KEY", f"--output={output}", "--json", ], ) + assert result.exit_code == 0, result.output - assert captured["config"].agent.api_key_env == "MY_OPENAI_KEY" - assert json.loads(result.stdout)["passed"] is True + assert captured["api_key_env"] == "MY_OPENAI_KEY" + payload = json.loads(result.stdout) + assert payload["status"] == "completed" + assert payload["report"]["native_result"]["value"] == {"score": 0.5} -def test_eval_exit_codes_distinguish_infra_semantic_and_preflight(tmp_path, monkeypatch) -> None: - output = tmp_path / "run" - monkeypatch.setattr(eval_cli, "execute_single_case", lambda *a, **k: _result(passed=None)) - infra = CliRunner().invoke( - main, ["eval", "run", str(_case(tmp_path)), f"--output={output}", "--quiet"] - ) - monkeypatch.setattr(eval_cli, "execute_single_case", lambda *a, **k: _result(passed=False)) - semantic = CliRunner().invoke( - main, ["eval", "run", str(_case(tmp_path)), f"--output={output}", "--quiet"] +@pytest.mark.parametrize(("status", "exit_code"), [("failed", 1), ("cancelled", 130)]) +def test_eval_exit_codes_for_noncompleted_runs( + status, + exit_code, + tmp_path, + monkeypatch, +) -> None: + monkeypatch.setattr(eval_cli, "execute_evaluation", lambda *a, **k: _result(status)) + + result = CliRunner().invoke( + main, + ["eval", "run", str(_spec(tmp_path)), f"--output={tmp_path / 'run'}", "--quiet"], ) + assert result.exit_code == exit_code + + +def test_eval_preflight_failure_uses_exit_two(tmp_path, monkeypatch) -> None: def preflight(*_args, **_kwargs): - raise FileNotFoundError("extension build missing") + raise FileNotFoundError("evaluation plugin missing") - monkeypatch.setattr(eval_cli, "execute_single_case", preflight) - preflight_result = CliRunner().invoke( - main, ["eval", "run", str(_case(tmp_path)), f"--output={output}"] + monkeypatch.setattr(eval_cli, "execute_evaluation", preflight) + result = CliRunner().invoke( + main, + ["eval", "run", str(_spec(tmp_path)), f"--output={tmp_path / 'run'}"], ) - assert infra.exit_code == 1 - assert semantic.exit_code == 0 - assert preflight_result.exit_code == 2 + + assert result.exit_code == 2 -def test_eval_help_is_typed_and_output_is_required(tmp_path) -> None: +def test_eval_help_exposes_thin_operational_settings(tmp_path) -> None: runner = CliRunner() help_result = runner.invoke(main, ["eval", "run", "--help"], color=True) - missing_output = runner.invoke(main, ["eval", "run", str(_case(tmp_path))]) + missing_output = runner.invoke(main, ["eval", "run", str(_spec(tmp_path))]) + assert help_result.exit_code == 0 help_text = unstyle(help_result.stdout) - assert "--agent.api-key-env" in help_text + assert "--api-key-env" in help_text + assert "--agent.model" not in help_text assert "--output" in help_text assert missing_output.exit_code == 2 @@ -137,14 +196,15 @@ def test_eval_help_is_typed_and_output_is_required(tmp_path) -> None: def test_lazy_runtime_import_has_actionable_error(monkeypatch) -> None: original_import = builtins.__import__ - def fail_single_case(name, *args, **kwargs): - if name == "dimos.benchmark.agent_eval.single_case": + def fail_runtime(name, *args, **kwargs): + if name == "dimos.benchmark.evaluation.runner": raise ModuleNotFoundError("No module named 'mcp'") return original_import(name, *args, **kwargs) - monkeypatch.setattr(builtins, "__import__", fail_single_case) + monkeypatch.setattr(builtins, "__import__", fail_runtime) + with pytest.raises(RuntimeError, match="uv sync --extra agents"): - eval_cli.execute_single_case(Path("case.json"), config=None) + eval_cli.execute_evaluation(Path("spec.json"), output=Path("output")) def test_base_cli_help_imports_without_eval_runtime() -> None: @@ -167,8 +227,12 @@ def find_spec(self, fullname, path=None, target=None): """ ) completed = subprocess.run( - [sys.executable, "-c", script], capture_output=True, text=True, check=False + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, ) + assert completed.returncode == 0, completed.stdout + completed.stderr diff --git a/docs/adr/0001-evaluations-own-result-semantics.md b/docs/adr/0001-evaluations-own-result-semantics.md new file mode 100644 index 0000000000..547a526672 --- /dev/null +++ b/docs/adr/0001-evaluations-own-result-semantics.md @@ -0,0 +1,3 @@ +# Evaluations own their result semantics + +The public extension point is a complete Evaluation, not a scorer or case type. Every Evaluation owns its inputs, protocol, scoring, and aggregation; an in-house Evaluation may use OpenEvals internally, while a third-party Evaluation delegates to its native harness without routing native results through a DimOS evaluator. The universal run specification binds only the Evaluation and CodePolicy agent configuration, and the immutable Evaluation Run records infrastructure status, native results, and artifacts. diff --git a/docs/adr/0002-code-policy-runtime-responsibilities.md b/docs/adr/0002-code-policy-runtime-responsibilities.md new file mode 100644 index 0000000000..fed0f613eb --- /dev/null +++ b/docs/adr/0002-code-policy-runtime-responsibilities.md @@ -0,0 +1,3 @@ +# CodePolicy runtime responsibilities are fixed across evaluations + +The initial evaluation system requires the CodePolicy agent runtime: one `python_exec` MCP tool backed by a persistent Python environment. Evaluations own session boundaries and public evaluation/task prompt components; the versioned runtime profile owns system instructions, tool descriptions, session-control messages, and deterministic prompt assembly. This keeps evaluation-specific semantics visible while allowing both in-house and third-party Evaluations to use the same agent interaction contract. diff --git a/docs/capabilities/agents/evaluation.md b/docs/capabilities/agents/evaluation.md index 211c26375a..82f314ea27 100644 --- a/docs/capabilities/agents/evaluation.md +++ b/docs/capabilities/agents/evaluation.md @@ -1,79 +1,160 @@ --- -title: "Frozen recording evaluation" +title: "Agent evaluations" --- -`dimos eval run` asks Pi one integer question about a frozen Memory2 recording. -The evaluator prepares the runtime map, exposes read-only `memory` through one -`python_exec` MCP tool, and checks the final `ANSWER: ` line against a -private oracle. It does not start a robot, simulation, replay blueprint, or live -DimOS module. +DimOS runs complete **Evaluations**. An Evaluation owns its inputs, protocol, +scoring, aggregation, and native result semantics. The shared framework resolves +the Evaluation, supplies CodePolicy agent sessions, and records an immutable +Evaluation Run. + +```text +Evaluation Run Specification + | + v + Evaluation ------ dataset / cases / native harness + | + v + CodePolicy Runtime ----- Pi + one persistent python_exec tool + | + v + Evaluation Run ------- status / native result / artifacts +``` + +This is deliberately not a universal scorer. The built-in frozen integer QA +Evaluation uses OpenEvals internally. A third-party benchmark should instead call +its own harness and return that harness's native result without rescoring it. ## Setup -From a source checkout, install the lightweight Python runtime and build the Pi -extension: +From a source checkout, install the Python runtime and build the Pi extension: ```bash uv sync --extra agents npm ci --prefix packages/pi-code-policy-extension npm run build --prefix packages/pi-code-policy-extension +export OPENAI_API_KEY=... ``` -The package pins Pi `0.80.10` and requires Node 22.19.0 or newer. Set the API key -before running a case: - -```bash -export OPENAI_API_KEY=... +The initial runtime profile is `code-policy-v1`: Pi 0.80.10, model +`gpt-5.6-luna`, medium thinking, and exactly one `python_exec` MCP tool. Pi is the +profile's driver, not a user-selectable evaluation runtime. + +## Run specification and CLI + +An Evaluation Run Specification binds an Evaluation configuration to the agent +configuration: + +```json +{ + "schema_version": "1.0", + "evaluation": { + "name": "frozen-integer-qa", + "config": {"case": "case.json"} + }, + "agent": { + "profile": "code-policy-v1", + "model": "gpt-5.6-luna", + "thinking_level": "medium" + } +} ``` -## Run the direct demo case +Evaluation-owned relative paths are resolved from the specification directory. +Run the included smoke specification with: ```bash uv run dimos eval run \ - dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/case.json \ + dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/run.json \ --output=/tmp/dimos-eval-smoke ``` -The demo fixture uses the synthetic sentinel `0`, not a reviewed Hong Kong office -room count. A semantic failure can therefore mean the agent and runtime worked but -the response did not match that plumbing sentinel. - -The supported options are deliberately small: +The operational CLI settings stay thin: | Option | Default | Purpose | | --- | --- | --- | -| `--agent.backend` | `pi` | Use the pinned Pi backend. | -| `--agent.model` | `gpt-5.6-luna` | Use the pinned model. | -| `--agent.thinking-level` | `medium` | Use the pinned thinking level. | -| `--agent.api-key-env` | `OPENAI_API_KEY` | Select the environment variable containing the API key. | -| `--output` | required | Publish this run to the exact directory. | -| `--json` | off | Print the compact result as JSON. | -| `--quiet` | off | Suppress status messages on stderr. | +| `--api-key-env` | `OPENAI_API_KEY` | Name of the environment variable containing the API key. | +| `--output` | required | Atomically publish the run to this directory. | +| `--json` | off | Print the complete Evaluation Run as JSON. | +| `--quiet` | off | Suppress live progress on stderr. | + +The API key is passed only to Pi. It is not written to the specification, run +record, prompt evidence, subprocess arguments, or Python kernel environment. + +## Results, artifacts, and exit status + +`--output` must be absent or empty. DimOS builds the run in a temporary sibling +directory and publishes it atomically as: + +```text +run.json +runtime/session-0001/ + runtime-system.txt + evaluation-protocol.txt + task-input.txt + assembled-user-message.txt + prompt-assembly.json + pi-transcript.jsonl # when Pi emits one + stderr.log # when nonempty +``` -The API key is passed only to the Pi subprocess. It is not placed in arguments, -results, or the Jupyter kernel environment. +`run.json` contains only universal infrastructure status—`completed`, `failed`, +or `cancelled`—plus the Evaluation's summary, opaque native result or artifact +reference, and artifact metadata. A native score of `false` can still be a +successfully completed run. + +| Exit | Meaning | +| --- | --- | +| `0` | The Evaluation completed, regardless of native semantic score. | +| `1` | Evaluation or agent infrastructure failed after execution started. | +| `2` | Specification, discovery, configuration, credential, or output preflight failed. | +| `130` | The user cancelled the Evaluation. | + +## Implement an Evaluation + +An Evaluation is the only public semantic extension point: + +```python +class MyEvaluation: + name = "my-evaluation" + config_model = MyEvaluationConfig + + def run(self, config, context): + with context.agent.open_session(environment) as session: + outcome = session.run( + evaluation_protocol="Return one answer per benchmark rules.", + task_input=sample.question, + ) + native_result = my_existing_harness.score(outcome.final_text) + return EvaluationReport( + summary=(...), + native_result=InlineNativeResult(value=native_result), + ) +``` -## Output and exit status +Built-ins are registered lazily inside DimOS. External distributions expose an +Evaluation object through the `dimos.evaluations` entry-point group: -`--output` must name an absent or empty directory. The evaluator builds the run in -a temporary sibling and atomically publishes it on completion. It never merges -with or overwrites a nonempty directory. +```toml +[project.entry-points."dimos.evaluations"] +my-evaluation = "my_package.evaluation:my_evaluation" +``` -The directory contains only: +An installed external Evaluation is addressed as +`.my-evaluation`. Keep benchmark datasets, sample +loops, success checks, and aggregation in the Evaluation or native harness. Do +not translate them into a universal DimOS case or scorer. -- `result.json`; -- `pi-transcript.jsonl`, when Pi wrote a native transcript; -- `stderr.log`, only when nonempty diagnostics are available. +## Prompt ownership -Exit code `0` means evaluation completed, whether the semantic score passed or -failed. Exit code `1` means a caught runtime or agent infrastructure failure; the -published `result.json` includes `infra_error`. Exit code `2` means preflight -failed before a run started. +The versioned runtime profile owns system instructions, the `python_exec` tool +surface, Pi flags, and deterministic assembly. Every Evaluation supplies two +immutable strings: its Evaluation Protocol and its Task Input. Their owners and +SHA-256 hashes are recorded separately even though Pi receives them together in +one user message. There are no prompt-template settings. ## Trust boundary CodePolicy executes agent-authored Python in a persistent Jupyter kernel. It is -trusted and **unsandboxed**. The `memory` object is cutoff-limited and its SQLite -connections are truly read-only, but Python can still access other host files and -processes. Run only trusted evaluation agents, or place the whole command in an OS -sandbox or container. +trusted and **unsandboxed**. Frozen Memory2 SQLite connections are read-only, but +Python can still access other host files and processes. Run only trusted agents, +or place the entire evaluation command in an OS sandbox or container. diff --git a/docs/development/testing.md b/docs/development/testing.md index 8932ad1481..ac15ffa1bd 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -89,7 +89,7 @@ uv run --extra agents pytest \ dimos/memory2/store/test_frozen.py \ dimos/agents/test_code_policy_core.py \ dimos/agents/test_code_policy_server.py \ - dimos/benchmark/agent_eval \ + dimos/benchmark/evaluation \ dimos/benchmark/short_horizon_qa \ dimos/cli/test_eval.py ``` diff --git a/packages/pi-code-policy-extension/src/python-exec.ts b/packages/pi-code-policy-extension/src/python-exec.ts index 5a74266436..5d0c05bdd6 100644 --- a/packages/pi-code-policy-extension/src/python-exec.ts +++ b/packages/pi-code-policy-extension/src/python-exec.ts @@ -10,7 +10,7 @@ const TOOL_NAME = "python_exec"; const DEFAULT_TIMEOUT_SECONDS = 110; interface McpClient { - listTools(): Promise<{ tools: Array<{ name: string }> }>; + listTools(): Promise<{ tools: Array<{ name: string; description?: string }> }>; callTool( params: { name: string; arguments: Record }, options?: { timeout?: number }, @@ -47,7 +47,8 @@ export async function installPythonExec( name: TOOL_NAME, label: "Execute Python", description: - "Execute Python in a persistent trusted, unsandboxed session with read-only memory.", + inventory.tools[0].description ?? + "Execute Python in a persistent trusted, unsandboxed session.", parameters: Type.Object( { code: Type.String({ minLength: 1 }), diff --git a/packages/pi-code-policy-extension/test/python-exec.test.ts b/packages/pi-code-policy-extension/test/python-exec.test.ts index 9f351564c7..2fd25148f7 100644 --- a/packages/pi-code-policy-extension/test/python-exec.test.ts +++ b/packages/pi-code-policy-extension/test/python-exec.test.ts @@ -20,7 +20,14 @@ test("registers one tool that calls MCP directly", async () => { } as ExtensionAPI; const client = { async listTools() { - return { tools: [{ name: "python_exec" }] }; + return { + tools: [ + { + name: "python_exec", + description: "Canonical CodePolicy description", + }, + ], + }; }, async callTool(params: { name: string; arguments: Record }) { assert.deepEqual(params, { @@ -36,6 +43,7 @@ test("registers one tool that calls MCP directly", async () => { await installPythonExec(pi, "http://127.0.0.1:1/mcp", async () => client); assert.equal(tool?.name, "python_exec"); + assert.equal(tool?.description, "Canonical CodePolicy description"); const result = await tool!.execute("call-1", { code: "1 + 1", timeout_s: 3 }, undefined, undefined, {} as never); assert.deepEqual(result.content, [{ type: "text", text: "2" }]); await shutdown!(); diff --git a/pyproject.toml b/pyproject.toml index 3249ac5735..223f8998e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -234,6 +234,7 @@ agents = [ "langchain-huggingface>=1,<2", "langchain-ollama>=1,<2", "ollama>=0.6.0", + "openevals>=0.2,<0.3", # Audio "openai", @@ -408,6 +409,7 @@ project-deps = [ "jupyter-client>=8.8.0", "mcp==2.0.0", "uvicorn>=0.34.0", + "openevals>=0.2,<0.3", ] tests = [ diff --git a/uv.lock b/uv.lock index 9e6869f184..248c0c4484 100644 --- a/uv.lock +++ b/uv.lock @@ -1601,6 +1601,7 @@ agents = [ { name = "nbformat" }, { name = "ollama" }, { name = "openai" }, + { name = "openevals" }, { name = "pyzmq" }, { name = "sounddevice" }, { name = "uvicorn" }, @@ -1646,6 +1647,7 @@ all = [ { name = "onnxruntime-gpu", marker = "platform_machine == 'x86_64'" }, { name = "open-clip-torch" }, { name = "openai" }, + { name = "openevals" }, { name = "pillow" }, { name = "piper-sdk" }, { name = "playground" }, @@ -1706,6 +1708,7 @@ base = [ { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, + { name = "openevals" }, { name = "pillow" }, { name = "pyzmq" }, { name = "rerun-sdk" }, @@ -1814,6 +1817,7 @@ unitree = [ { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, + { name = "openevals" }, { name = "pillow" }, { name = "pyzmq" }, { name = "rerun-sdk" }, @@ -1853,6 +1857,7 @@ unitree-dds = [ { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, + { name = "openevals" }, { name = "pillow" }, { name = "pyzmq" }, { name = "rerun-sdk" }, @@ -1915,6 +1920,7 @@ lint = [ { name = "open-clip-torch" }, { name = "openai" }, { name = "openai-whisper" }, + { name = "openevals" }, { name = "pandas-stubs" }, { name = "pytest" }, { name = "python-can" }, @@ -1954,6 +1960,7 @@ project-deps = [ { name = "ollama" }, { name = "open-clip-torch" }, { name = "openai" }, + { name = "openevals" }, { name = "tensorboard" }, { name = "torch" }, { name = "torchreid" }, @@ -1985,6 +1992,7 @@ tests = [ { name = "ollama" }, { name = "open-clip-torch" }, { name = "openai" }, + { name = "openevals" }, { name = "pre-commit" }, { name = "py-spy" }, { name = "pygame" }, @@ -2037,6 +2045,7 @@ tests-self-hosted = [ { name = "ollama" }, { name = "open-clip-torch" }, { name = "openai" }, + { name = "openevals" }, { name = "pre-commit" }, { name = "py-spy" }, { name = "pybind11" }, @@ -2137,6 +2146,7 @@ requires-dist = [ { name = "open3d-unofficial-arm", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'", specifier = ">=0.19.0.post9" }, { name = "openai", marker = "extra == 'agents'" }, { name = "opencv-contrib-python", specifier = ">=4.8,<5" }, + { name = "openevals", marker = "extra == 'agents'", specifier = ">=0.2,<0.3" }, { name = "packaging", specifier = ">=24.0" }, { name = "pandas", marker = "extra == 'learning'" }, { name = "pillow", marker = "extra == 'perception'" }, @@ -2228,6 +2238,7 @@ lint = [ { name = "open-clip-torch", specifier = "==3.2.0" }, { name = "openai" }, { name = "openai-whisper" }, + { name = "openevals", specifier = ">=0.2,<0.3" }, { name = "pandas-stubs", specifier = ">=2.3.2.250926,<3" }, { name = "pytest", specifier = "==8.3.5" }, { name = "python-can", specifier = ">=4" }, @@ -2267,6 +2278,7 @@ project-deps = [ { name = "ollama", specifier = ">=0.6.0" }, { name = "open-clip-torch", specifier = "==3.2.0" }, { name = "openai" }, + { name = "openevals", specifier = ">=0.2,<0.3" }, { name = "tensorboard", specifier = "==2.20.0" }, { name = "torch" }, { name = "torchreid", specifier = "==0.2.5" }, @@ -2299,6 +2311,7 @@ tests = [ { name = "ollama", specifier = ">=0.6.0" }, { name = "open-clip-torch", specifier = "==3.2.0" }, { name = "openai" }, + { name = "openevals", specifier = ">=0.2,<0.3" }, { name = "pre-commit", specifier = "==4.2.0" }, { name = "py-spy" }, { name = "pygame", specifier = ">=2.6.1" }, @@ -2353,6 +2366,7 @@ tests-self-hosted = [ { name = "ollama", specifier = ">=0.6.0" }, { name = "open-clip-torch", specifier = "==3.2.0" }, { name = "openai" }, + { name = "openevals", specifier = ">=0.2,<0.3" }, { name = "pre-commit", specifier = "==4.2.0" }, { name = "py-spy" }, { name = "pybind11", specifier = ">=2.12" }, @@ -5903,6 +5917,21 @@ dependencies = [ { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')" }, ] +[[package]] +name = "openevals" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain" }, + { name = "langchain-openai" }, + { name = "langsmith" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/b1/028a05846136805b29b7a3afb58a940c2d213fa2c3d0a7d7003c7fbaa115/openevals-0.2.0.tar.gz", hash = "sha256:7e95fa64625be53eaa8c657d7f69b842a52bda10bdf3bb91781c7d09a385b069", size = 140711, upload-time = "2026-04-07T19:45:22.749Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/8b/00f402b7f3475e235c339a9bc82d2eaf46cc08b53ecd85d4a117850170d3/openevals-0.2.0-py3-none-any.whl", hash = "sha256:2bce5964be9d162e3d38c2dfd026739156e1ac521536ade6b8e2f0a89b632f2c", size = 106958, upload-time = "2026-04-07T19:45:21.575Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.42.1" From be3ada047c62bbaf47d2a91eb5f6e9ee6749776e Mon Sep 17 00:00:00 2001 From: cc Date: Sat, 8 Aug 2026 09:57:46 -0700 Subject: [PATCH 15/15] spec: remove --- CONTEXT-MAP.md | 10 -- dimos/benchmark/CONTEXT.md | 45 ------ .../0001-evaluations-own-result-semantics.md | 3 - ...02-code-policy-runtime-responsibilities.md | 3 - docs/agents/domain.md | 60 -------- docs/agents/issue-tracker.md | 62 --------- openspec/config.yaml | 45 ------ openspec/schemas/dimos-capability/schema.yaml | 128 ------------------ .../dimos-capability/templates/design.md | 35 ----- .../dimos-capability/templates/docs.md | 19 --- .../dimos-capability/templates/proposal.md | 32 ----- .../dimos-capability/templates/spec.md | 16 --- .../dimos-capability/templates/tasks.md | 15 -- 13 files changed, 473 deletions(-) delete mode 100644 CONTEXT-MAP.md delete mode 100644 dimos/benchmark/CONTEXT.md delete mode 100644 docs/adr/0001-evaluations-own-result-semantics.md delete mode 100644 docs/adr/0002-code-policy-runtime-responsibilities.md delete mode 100644 docs/agents/domain.md delete mode 100644 docs/agents/issue-tracker.md delete mode 100644 openspec/config.yaml delete mode 100644 openspec/schemas/dimos-capability/schema.yaml delete mode 100644 openspec/schemas/dimos-capability/templates/design.md delete mode 100644 openspec/schemas/dimos-capability/templates/docs.md delete mode 100644 openspec/schemas/dimos-capability/templates/proposal.md delete mode 100644 openspec/schemas/dimos-capability/templates/spec.md delete mode 100644 openspec/schemas/dimos-capability/templates/tasks.md diff --git a/CONTEXT-MAP.md b/CONTEXT-MAP.md deleted file mode 100644 index f249c89365..0000000000 --- a/CONTEXT-MAP.md +++ /dev/null @@ -1,10 +0,0 @@ -# Context Map - -## Contexts - -- [Manipulation Planning](./CONTEXT.md) — describes requests for planning robot motion through joint and Cartesian spaces. -- [Evaluation](./dimos/benchmark/CONTEXT.md) — describes executable evaluations and records of their execution. - -## Relationships - -- **Evaluation → Manipulation Planning**: An evaluation may exercise manipulation planning as part of the subject being evaluated, but does not define planning semantics. diff --git a/dimos/benchmark/CONTEXT.md b/dimos/benchmark/CONTEXT.md deleted file mode 100644 index 933fd43539..0000000000 --- a/dimos/benchmark/CONTEXT.md +++ /dev/null @@ -1,45 +0,0 @@ -# Evaluation - -This context describes executable evaluations, requests to run them, and immutable records of their execution. - -## Language - -**Evaluation**: -An executable definition that owns its protocol and result semantics. -_Avoid_: Evaluator, benchmark integration - -**Evaluation Run Specification**: -A user-authored request that binds an evaluation and its configuration to a CodePolicy agent configuration. Output paths, credentials, infrastructure timeouts, and concurrency are operational settings outside it. -_Avoid_: Run, case - -**Evaluation Run**: -An immutable record of one resolved execution of an evaluation run specification. -_Avoid_: Run configuration, mutable run - -**Evaluation Case**: -An optional atomic input owned by an evaluation. It is not part of the universal evaluation run contract. -_Avoid_: Evaluation target - -**Evaluation Attempt**: -One execution of an evaluation case or another evaluation-owned unit. -_Avoid_: Evaluation run - -**CodePolicy Agent Runtime**: -The required agent interaction contract for an evaluation: exactly one `python_exec` MCP tool backed by a persistent Python environment. -_Avoid_: Evaluation subject, generic agent runtime - -**CodePolicy Session**: -A lifecycle-bounded CodePolicy interaction whose Python namespace and agent conversation persist until the evaluation closes it. The evaluation owns session boundaries. -_Avoid_: Evaluation run, global agent session - -**CodePolicy Runtime Profile**: -A versioned definition of the runtime-owned system instructions, `python_exec` tool surface, and session behavior used by a CodePolicy agent runtime. Evaluations cannot override its prompt responsibilities. -_Avoid_: Evaluation prompt, benchmark prompt - -**Prompt Component**: -An immutable, separately recorded part of an agent prompt owned by either the runtime profile or the evaluation. Evaluation protocol and task input remain distinct components even when transported in one message. -_Avoid_: Prompt fragment, appended prompt - -**Prompt Assembly**: -The deterministic runtime-owned combination of prompt components for one CodePolicy session. -_Avoid_: Prompt concatenation, prompt settings diff --git a/docs/adr/0001-evaluations-own-result-semantics.md b/docs/adr/0001-evaluations-own-result-semantics.md deleted file mode 100644 index 547a526672..0000000000 --- a/docs/adr/0001-evaluations-own-result-semantics.md +++ /dev/null @@ -1,3 +0,0 @@ -# Evaluations own their result semantics - -The public extension point is a complete Evaluation, not a scorer or case type. Every Evaluation owns its inputs, protocol, scoring, and aggregation; an in-house Evaluation may use OpenEvals internally, while a third-party Evaluation delegates to its native harness without routing native results through a DimOS evaluator. The universal run specification binds only the Evaluation and CodePolicy agent configuration, and the immutable Evaluation Run records infrastructure status, native results, and artifacts. diff --git a/docs/adr/0002-code-policy-runtime-responsibilities.md b/docs/adr/0002-code-policy-runtime-responsibilities.md deleted file mode 100644 index fed0f613eb..0000000000 --- a/docs/adr/0002-code-policy-runtime-responsibilities.md +++ /dev/null @@ -1,3 +0,0 @@ -# CodePolicy runtime responsibilities are fixed across evaluations - -The initial evaluation system requires the CodePolicy agent runtime: one `python_exec` MCP tool backed by a persistent Python environment. Evaluations own session boundaries and public evaluation/task prompt components; the versioned runtime profile owns system instructions, tool descriptions, session-control messages, and deterministic prompt assembly. This keeps evaluation-specific semantics visible while allowing both in-house and third-party Evaluations to use the same agent interaction contract. diff --git a/docs/agents/domain.md b/docs/agents/domain.md deleted file mode 100644 index e1de27973a..0000000000 --- a/docs/agents/domain.md +++ /dev/null @@ -1,60 +0,0 @@ -# DimOS agent domain context - -## Context loading - -Before working on a change, load the repository context in this order: - -1. Read `AGENTS.md` and follow its applicable instructions. -2. Read `openspec/config.yaml` for the OpenSpec schema, terminology, and rules. -3. Read the relevant files under `openspec/specs/`. -4. Read the root `CONTEXT.md` if it exists. -5. Read relevant records under `docs/adr/` if that directory exists. - -`CONTEXT.md` and `docs/adr/` are optional. If either is absent, continue -silently; do not report the absence as an error. Select specs and ADRs based on -the affected behavior and implementation surface rather than reading -unrelated material. - -## Two meanings of “spec” - -Keep these terms separate: - -- An **OpenSpec spec** is a behavior specification under `openspec/specs/`. - It describes observable behavior, user or developer outcomes, public - interfaces, safety constraints, and testable scenarios. -- A **DimOS Python Spec Protocol** is a code-level interface contract, usually - a `Protocol` inheriting from `dimos.spec.utils.Spec`, often found in a - `*_spec.py` file. It describes module RPCs and injected interfaces. - -An OpenSpec spec is not a Python Protocol, and a Python Protocol does not -replace an OpenSpec behavioral requirement. Keep implementation details such as -class names, module wiring, stream types, generated registries, and rollout -steps in the OpenSpec change design or tasks unless they are externally -observable. - -## Work layout - -Organize work through this chain: - -```text -Linear issue -> OpenSpec change -> implementation tasks -> pull request -``` - -Linear provides intake and tracking. The OpenSpec change is the source of truth -for the behavioral change, design, and tasks. The pull request implements and -reviews those tasks. Keep the identifiers and links aligned across all three -artifacts; any Linear link edit requires user confirmation before it is made. - -When a task affects behavior, update the relevant OpenSpec change and, where -appropriate, the corresponding spec under `openspec/specs/`. Include concrete -scenarios for behavioral requirements. Call out DimOS Python Spec Protocols, -blueprint composition, streams, skills/MCP exposure, generated files, and -hardware, simulation, or replay assumptions in design and task material when -they are relevant. - -## Conflicting guidance - -Surface conflicts between an ADR and an OpenSpec spec explicitly. Do not -silently reconcile, overwrite, or guess which decision applies. Report the -conflict, identify the affected behavior or implementation, and ask for the -decision or update the authoritative document only when instructed. diff --git a/docs/agents/issue-tracker.md b/docs/agents/issue-tracker.md deleted file mode 100644 index c0db692d0f..0000000000 --- a/docs/agents/issue-tracker.md +++ /dev/null @@ -1,62 +0,0 @@ -# Issue tracking with Linear - -## Workspace - -DimOS work is tracked in the **DIM** team in Linear: - - - -Access Linear through the configured Linear MCP. Do not assume that a local -copy, an unconfigured client, or a direct API call is an alternative source of -truth. - -## Confirmation policy - -User confirmation is required immediately before **every** Linear edit. This -includes, without limitation: - -- creating an issue; -- changing any issue field, including title, description, assignee, priority, - project, or due date; -- adding, removing, or changing labels; -- posting comments; -- changing state or making any other state transition; and -- adding, removing, or changing links. - -Reading Linear is not an edit. Before an edit, state exactly what will change -and wait for explicit user confirmation. One confirmation does not authorize -later edits, even when they concern the same issue or change. - -## Linking convention - -Keep the work chain navigable: - -```text -Linear issue <-> openspec/changes/ <-> pull request -``` - -Use the OpenSpec change ID as the stable identifier in the relationship. Link -the Linear issue to the relevant OpenSpec change and link the pull request to -both when the tools support those links. If a link must be created or changed, -it is a Linear edit and requires confirmation under the policy above. - -## Source of truth and workflow - -Linear is the intake and tracking system. It records requests, ownership, -status, discussion, and delivery progress. OpenSpec is the source of truth for -the behavioral change, its design, and its implementation tasks. The pull -request is the review and delivery vehicle. - -Use this sequence: - -1. Capture or find the Linear issue in the DIM team. -2. Create or update `openspec/changes//` for the proposed behavior, - design, and tasks. -3. Implement the tasks and keep the OpenSpec change current. -4. Open the pull request and connect it to the issue and OpenSpec change. -5. Reflect progress in Linear only after confirming each requested edit. - -Do not use a Linear description, comment, or state as a substitute for an -OpenSpec requirement, design decision, or task. If Linear and OpenSpec -disagree about behavior, treat OpenSpec as authoritative and surface the -discrepancy to the user rather than silently choosing a version. diff --git a/openspec/config.yaml b/openspec/config.yaml deleted file mode 100644 index 62a72bba63..0000000000 --- a/openspec/config.yaml +++ /dev/null @@ -1,45 +0,0 @@ -schema: dimos-capability - -context: | - DimOS is a robotics operating system for generalist robots. Modules communicate - through typed streams (`In[T]`, `Out[T]`) over LCM, SHM, ROS, DDS, or other - transports. Blueprints compose modules into runnable robot stacks. Skills are - `@skill`-annotated RPC methods exposed to agents and MCP clients. - - Terminology boundary: - - "OpenSpec spec" means a behavior specification under `openspec/specs/`. - - "DimOS Spec" means a Python Protocol/RPC contract in `*_spec.py` files, - usually inheriting `dimos.spec.utils.Spec` and `typing.Protocol`. - Keep these separate. OpenSpec specs describe observable behavior; DimOS Specs - describe code-level module interfaces. - - OpenSpec specs should capture current behavior, user/developer-visible - outcomes, public CLI/API/tool surfaces, robot safety constraints, and testable - scenarios. Put implementation choices, class names, module wiring, generated - registry updates, and rollout details in `design.md` or `tasks.md`. - - Documentation lives in: - - `docs/usage/` for user-facing concepts and APIs. - - `docs/capabilities/` for capability and platform guides. - - `docs/development/` for contributor process. - - `docs/coding-agents/` and `AGENTS.md` for coding-agent guidance. - -rules: - proposal: - - "Identify affected DimOS surfaces: modules, streams, blueprints, CLI, skills/MCP, docs, hardware, simulation, replay, or generated registries." - - Use capability names that match behavior domains, not Python class names. - - Mark hardware safety or public API/CLI changes explicitly. - specs: - - Write behavior-first requirements; avoid implementation detail unless it is externally observable. - - Every requirement must include at least one `#### Scenario:` block with concrete observable outcomes. - - Use "OpenSpec capability spec" when prose might otherwise be confused with DimOS Python `Spec` Protocols. - design: - - Call out DimOS `Spec` Protocols, adapter Protocols, blueprint composition, stream names/types, and skill/MCP exposure when relevant. - - Mention generated files and required regeneration commands, especially `pytest dimos/robot/test_all_blueprints_generation.py` for blueprint registry changes. - - Include hardware/simulation/replay assumptions and safety constraints for robot-facing work. - docs: - - List user-facing docs, contributor docs, coding-agent docs, and AGENTS.md updates required by the change. - - Include documentation validation commands for changed docs, such as `doclinks` and `md-babel-py run ` where applicable. - tasks: - - Include verification tasks for OpenSpec validation, relevant pytest targets, type checks when needed, and manual QA through the user-facing surface. - - Add registry generation tasks when blueprint names, module classes, or generated registry inputs change. diff --git a/openspec/schemas/dimos-capability/schema.yaml b/openspec/schemas/dimos-capability/schema.yaml deleted file mode 100644 index fedb7964ee..0000000000 --- a/openspec/schemas/dimos-capability/schema.yaml +++ /dev/null @@ -1,128 +0,0 @@ -name: dimos-capability -version: 1 -description: DimOS capability workflow - proposal → specs/design/docs → tasks -artifacts: - - id: proposal - generates: proposal.md - description: DimOS change proposal covering intent, scope, capability impact, and affected robot/software surfaces - template: proposal.md - instruction: | - Create the proposal document that establishes WHY this change is needed and what DimOS behavior it affects. - - Sections: - - **Why**: 1-2 concise paragraphs on the problem or opportunity. Explain why the change matters now. - - **What Changes**: Bullet list of added, modified, or removed behavior. Mark public API/CLI or hardware-safety breaking changes with **BREAKING**. - - **Affected DimOS Surfaces**: Identify modules, streams, blueprints, CLI commands, skills/MCP tools, docs, hardware, simulation, replay, generated registries, or external protocols touched by the change. - - **Capabilities**: Identify which OpenSpec capability specs will be created or modified: - - **New Capabilities**: List behavior domains introduced by the change. Each becomes `specs//spec.md`. Use kebab-case names (for example, `agent-skills-mcp`, `blueprint-composition`, `manipulation-stack`). - - **Modified Capabilities**: List existing `openspec/specs//` entries whose requirements change. Only include spec-level behavior changes, not implementation-only refactors. - - **Impact**: Summarize user/developer impact, compatibility risks, dependency changes, documentation updates, and test/QA scope. - - Keep proposals concise. Do not include line-by-line implementation details; put architecture and rollout decisions in `design.md`. - requires: [] - - id: specs - generates: specs/**/*.md - description: Behavior-first OpenSpec capability delta specifications - template: spec.md - instruction: | - Create OpenSpec capability specs that define WHAT DimOS should do, not how it is implemented. - - Create one delta spec file per capability listed in proposal.md: - - New capabilities: use `specs//spec.md` with the exact kebab-case name from the proposal. - - Modified capabilities: use the existing folder from `openspec/specs//`. - - Use these delta sections as `##` headers: - - **ADDED Requirements**: New externally observable behavior. - - **MODIFIED Requirements**: Changed behavior. Include the full updated requirement block, not a partial patch. - - **REMOVED Requirements**: Deprecated behavior. Include **Reason** and **Migration**. - - **RENAMED Requirements**: Name-only changes. Use FROM:/TO: format. - - Requirement format: - - Use `### Requirement: `. - - Use SHALL/MUST for normative requirements. - - Include at least one `#### Scenario: ` per requirement. Scenario headings MUST use exactly four `#` characters. - - Prefer `- **GIVEN**`, `- **WHEN**`, `- **THEN**`, and `- **AND**` bullets. - - Cover happy path plus meaningful edge/error/safety cases. - - DimOS-specific guidance: - - Specify user/developer-visible behavior, robot outcomes, CLI behavior, skill/MCP tool behavior, stream contracts, safety constraints, and compatibility expectations. - - Avoid Python class names, private module internals, transport implementation choices, and generated-file details unless those details are observable API contracts. - - Use "OpenSpec capability spec" in prose when needed to avoid confusion with DimOS Python `Spec` Protocols. - - If the behavior only changes implementation and not observable requirements, do not create a spec delta. - requires: - - proposal - - id: design - generates: design.md - description: DimOS technical design and architecture decisions - template: design.md - instruction: | - Create the design document that explains HOW the change should be implemented in DimOS. - - Include design.md for cross-module changes, new robot/hardware integration, new public interfaces, new dependencies, safety-sensitive behavior, generated registry changes, or unclear architecture. - - Sections: - - **Context**: Current state, relevant modules/blueprints/docs, and constraints. - - **Goals / Non-Goals**: What the design achieves and explicitly excludes. - - **DimOS Architecture**: Modules, streams, transports, blueprints, RPC/module refs, DimOS `Spec` Protocols, adapter Protocols, skills/MCP exposure, CLI entry points, and generated registries involved. - - **Decisions**: Key choices with rationale and alternatives considered. - - **Safety / Simulation / Replay**: Hardware assumptions, sim/replay behavior, safety constraints, and manual QA surface. - - **Risks / Trade-offs**: Known risks and mitigations. - - **Migration / Rollout**: Compatibility, generated files, docs, and deployment steps. - - **Open Questions**: Outstanding decisions or unknowns. - - Reference proposal.md for intent and specs for behavior. Keep line-by-line work in tasks.md. - requires: - - proposal - - id: docs - generates: docs.md - description: Documentation impact plan for user, contributor, and coding-agent docs - template: docs.md - instruction: | - Create the documentation impact plan for the change. - - Sections: - - **User-Facing Docs**: Updates under `docs/usage/`, `docs/capabilities/`, `docs/platforms/`, or README files. - - **Contributor Docs**: Updates under `docs/development/`. - - **Coding-Agent Docs**: Updates under `docs/coding-agents/` or `AGENTS.md`. - - **Doc Validation**: Commands needed for changed docs, such as `doclinks`, `md-babel-py run `, and `bin/gen-diagrams`. - - **No Docs Needed**: If no docs are needed, explain why. - - Match `docs/development/writing_docs.md`: contributor-only docs belong in `docs/development`; user-facing behavior belongs in `docs/usage` or `docs/capabilities`. - requires: - - proposal - - id: tasks - generates: tasks.md - description: Implementation, validation, docs, and manual-QA checklist - template: tasks.md - instruction: | - Create the implementation checklist. The apply phase parses checkbox format, so every actionable task MUST use `- [ ]`. - - Guidelines: - - Group tasks under numbered `##` headings. - - Each task must be `- [ ] X.Y Task description`. - - Keep tasks small enough to complete in one focused session. - - Order tasks by dependency. - - Include docs and validation tasks from docs.md. - - Include generated registry tasks when blueprints or module registry inputs change. - - Include manual QA through the actual user surface: CLI, TUI, HTTP API, MCP tool, simulation/replay blueprint, hardware procedure, or library driver. - - Typical DimOS validation tasks: - - Run `openspec validate `. - - Run focused pytest targets for changed modules. - - Run `pytest dimos/robot/test_all_blueprints_generation.py` when blueprint registry output may change. - - Run docs validation commands for changed docs. - - Run lints/types when the touched area requires them. - - Reference specs for WHAT, design for HOW, and docs.md for documentation work. - requires: - - specs - - design - - docs -apply: - requires: - - tasks - tracks: tasks.md - instruction: | - Read proposal.md, specs, design.md, docs.md, and tasks.md before editing code. - Work through pending tasks, mark checkboxes complete as they finish, and keep artifacts current when implementation changes the plan. - Verify with OpenSpec validation, focused tests, docs checks, and manual QA through the relevant DimOS surface. diff --git a/openspec/schemas/dimos-capability/templates/design.md b/openspec/schemas/dimos-capability/templates/design.md deleted file mode 100644 index 25031ceb8b..0000000000 --- a/openspec/schemas/dimos-capability/templates/design.md +++ /dev/null @@ -1,35 +0,0 @@ -## Context - - - -## Goals / Non-Goals - -**Goals:** - - -**Non-Goals:** - - -## DimOS Architecture - - - -## Decisions - - - -## Safety / Simulation / Replay - - - -## Risks / Trade-offs - - - -## Migration / Rollout - - - -## Open Questions - - diff --git a/openspec/schemas/dimos-capability/templates/docs.md b/openspec/schemas/dimos-capability/templates/docs.md deleted file mode 100644 index d274aed653..0000000000 --- a/openspec/schemas/dimos-capability/templates/docs.md +++ /dev/null @@ -1,19 +0,0 @@ -## User-Facing Docs - - - -## Contributor Docs - - - -## Coding-Agent Docs - - - -## Doc Validation - - - -## No Docs Needed - - diff --git a/openspec/schemas/dimos-capability/templates/proposal.md b/openspec/schemas/dimos-capability/templates/proposal.md deleted file mode 100644 index 98d409e8de..0000000000 --- a/openspec/schemas/dimos-capability/templates/proposal.md +++ /dev/null @@ -1,32 +0,0 @@ -## Why - - - -## What Changes - - - -## Affected DimOS Surfaces - - -- Modules/streams: -- Blueprints/CLI: -- Skills/MCP: -- Hardware/simulation/replay: -- Docs/generated registries: - -## Capabilities - -### New Capabilities - -- ``: - -### Modified Capabilities - -- ``: - -## Impact - - diff --git a/openspec/schemas/dimos-capability/templates/spec.md b/openspec/schemas/dimos-capability/templates/spec.md deleted file mode 100644 index afc0c1ff58..0000000000 --- a/openspec/schemas/dimos-capability/templates/spec.md +++ /dev/null @@ -1,16 +0,0 @@ -## ADDED Requirements - -### Requirement: - - -#### Scenario: -- **GIVEN** -- **WHEN** -- **THEN** -- **AND** - - diff --git a/openspec/schemas/dimos-capability/templates/tasks.md b/openspec/schemas/dimos-capability/templates/tasks.md deleted file mode 100644 index b38fcdfabb..0000000000 --- a/openspec/schemas/dimos-capability/templates/tasks.md +++ /dev/null @@ -1,15 +0,0 @@ -## 1. Implementation - -- [ ] 1.1 -- [ ] 1.2 - -## 2. Documentation - -- [ ] 2.1 - -## 3. Verification - -- [ ] 3.1 Run `openspec validate ` -- [ ] 3.2 Run focused tests for changed code -- [ ] 3.3 Run docs validation commands for changed docs -- [ ] 3.4 Manually QA through the relevant DimOS surface (CLI, MCP, simulation/replay, hardware procedure, HTTP API, or library driver)