From ba3f489280fe3c77ad092ce0723a20746cf86067 Mon Sep 17 00:00:00 2001 From: cc Date: Mon, 10 Aug 2026 16:51:00 -0700 Subject: [PATCH] feat(evals): add code policy evaluation runtime Add a fixed Pi exploration stage with typed callable policy submission, fresh debug trials, inspectable TrialRun artifacts, and clean agent-free held-out execution.\n\nKeep benchmark lifecycle and privileged scoring evaluation-owned while exposing the complete DimOS runtime to authored policies. --- CONTEXT.md | 49 + dimos/agents/code_policy_core.py | 325 +++ dimos/agents/code_policy_server.py | 181 ++ dimos/agents/test_code_policy_core.py | 87 + dimos/benchmark/evaluation/models.py | 131 ++ dimos/benchmark/evaluation/pi_process.py | 309 +++ dimos/benchmark/evaluation/progress.py | 84 + dimos/benchmark/evaluation/protocol.py | 144 ++ dimos/benchmark/evaluation/registry.py | 134 ++ dimos/benchmark/evaluation/runner.py | 141 ++ dimos/benchmark/evaluation/runtime.py | 402 ++++ dimos/benchmark/evaluation/test_framework.py | 112 + dimos/benchmark/evaluation/test_models.py | 64 + dimos/benchmark/evaluation/test_pi_process.py | 213 ++ .../evaluation/test_policy_runtime.py | 168 ++ dimos/benchmark/evaluation/test_registry.py | 98 + dimos/cli/dimos.py | 2 + dimos/cli/eval.py | 160 ++ dimos/memory2/blobstore/sqlite.py | 5 +- dimos/memory2/observationstore/sqlite.py | 16 +- dimos/memory2/registry.py | 22 +- dimos/memory2/store/sqlite.py | 43 +- dimos/memory2/utils/sqlite.py | 17 +- dimos/memory2/vectorstore/sqlite.py | 5 +- docs/capabilities/agents/evaluation.md | 107 + docs/capabilities/agents/index.md | 3 + packages/pi-code-policy-extension/.gitignore | 6 + packages/pi-code-policy-extension/README.md | 8 + .../package-lock.json | 1894 +++++++++++++++++ .../pi-code-policy-extension/package.json | 24 + .../src/python-exec.ts | 84 + .../test/python-exec.test.ts | 51 + .../tsconfig.build.json | 13 + .../pi-code-policy-extension/tsconfig.json | 18 + .../tsconfig.test.json | 14 + pyproject.toml | 14 + uv.lock | 168 +- 37 files changed, 5285 insertions(+), 31 deletions(-) create mode 100644 CONTEXT.md create mode 100644 dimos/agents/code_policy_core.py create mode 100644 dimos/agents/code_policy_server.py create mode 100644 dimos/agents/test_code_policy_core.py create mode 100644 dimos/benchmark/evaluation/models.py create mode 100644 dimos/benchmark/evaluation/pi_process.py create mode 100644 dimos/benchmark/evaluation/progress.py create mode 100644 dimos/benchmark/evaluation/protocol.py create mode 100644 dimos/benchmark/evaluation/registry.py create mode 100644 dimos/benchmark/evaluation/runner.py create mode 100644 dimos/benchmark/evaluation/runtime.py create mode 100644 dimos/benchmark/evaluation/test_framework.py create mode 100644 dimos/benchmark/evaluation/test_models.py create mode 100644 dimos/benchmark/evaluation/test_pi_process.py create mode 100644 dimos/benchmark/evaluation/test_policy_runtime.py create mode 100644 dimos/benchmark/evaluation/test_registry.py create mode 100644 dimos/cli/eval.py create mode 100644 docs/capabilities/agents/evaluation.md create mode 100644 packages/pi-code-policy-extension/.gitignore create mode 100644 packages/pi-code-policy-extension/README.md create mode 100644 packages/pi-code-policy-extension/package-lock.json create mode 100644 packages/pi-code-policy-extension/package.json create mode 100644 packages/pi-code-policy-extension/src/python-exec.ts create mode 100644 packages/pi-code-policy-extension/test/python-exec.test.ts create mode 100644 packages/pi-code-policy-extension/tsconfig.build.json create mode 100644 packages/pi-code-policy-extension/tsconfig.json create mode 100644 packages/pi-code-policy-extension/tsconfig.test.json diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000000..8dcceec696 --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,49 @@ +# Evaluation + +The language used to describe how DimOS measures task-performing behavior across recorded, simulated, and live environments. + +## Language + +**Evaluation**: +A complete benchmark integration that owns its environment lifecycle, cases, scoring, aggregation, and native result semantics. Third-party evaluations retain their original harness rather than translating it into DimOS scoring primitives. +_Avoid_: Runtime, universal scorer + +**Evaluation Case**: +A fixed task, environment, budget, and scoring definition. It does not select the behavior being evaluated. +_Avoid_: Policy configuration, run + +**Evaluation Run**: +One attempt of an evaluation case by a selected policy. Different policies can attempt the same case without changing its definition. +_Avoid_: Case, suite + +**Runtime**: +The injected DimOS facility that attaches to a system launched by an evaluation and executes the selected policy. It does not own the benchmark environment or scoring. +_Avoid_: Evaluation, benchmark harness + +**Policy**: +The callable produced during exploration and replayed without an agent during evaluation. Its canonical signature is `policy(app: Dimos) -> None`. +_Avoid_: Agent, exploration transcript, execution mode + +**Policy Artifact**: +The serialized callable and human-readable source captured when `submit_policy(policy)` is called. One artifact is produced per benchmark task and reused across its held-out evaluation cases or seeds. REPL outputs and the agent transcript are separate exploration evidence. +_Avoid_: Exploration transcript, policy source + +**Exploration Stage**: +The unscored stage in which an agent uses a persistent Python REPL and calls `submit_policy(policy)` to run complete debug trials in fresh environments and blueprints. Model latency does not consume the evaluation horizon. +_Avoid_: Evaluation rollout, scoring + +**Evaluation Stage**: +The measured stage in which the Policy Artifact executes without an agent against a reset or held-out environment. The native benchmark owns its real-time or step horizon and privileged scoring. +_Avoid_: Agent session, policy generation + +**Policy Environment**: +The capabilities exposed by the fresh policy-only DimOS blueprint while a policy runs. Simulated, live, and replay-backed evaluations all pass the policy a connected `Dimos` application; completed trials additionally expose their Memory2 recording read-only through `TrialRun`. +_Avoid_: Agent tools, scorer context + +**Evaluation Oracle**: +The evaluator-only source of truth used to score a policy attempt. In simulation it contains privileged state, such as true poses and object identities, that the Policy Environment cannot access. +_Avoid_: Runtime memory, perception output + +**Agent**: +The model-backed participant in the Exploration Stage that issues code through the runtime's Python REPL and produces a Policy Artifact. It is absent from the Evaluation Stage and does not receive the evaluation oracle. +_Avoid_: Evaluator, runner diff --git a/dimos/agents/code_policy_core.py b/dimos/agents/code_policy_core.py new file mode 100644 index 0000000000..347dfbc346 --- /dev/null +++ b/dimos/agents/code_policy_core.py @@ -0,0 +1,325 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Persistent, module-independent Python session for trusted CodePolicy agents.""" + +from __future__ import annotations + +import base64 +import inspect +import os +import queue +import re +import threading +import time +from typing import TYPE_CHECKING, Any, get_type_hints + +from pydantic import BaseModel, ConfigDict, Field + +if TYPE_CHECKING: + from dimos.benchmark.evaluation.protocol import TrialRun + +MAX_EXECUTION_TIMEOUT_S = 600.0 +DEFAULT_OUTPUT_LIMIT = 32_000 +_SUBMISSION_URL_ENV = "DIMOS_CODE_POLICY_SUBMISSION_URL" +_SUBMISSION_TOKEN_ENV = "DIMOS_CODE_POLICY_SUBMISSION_TOKEN" +_ANSI_ESCAPE_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") +_TRUNCATION_MARKER = "\n... [output truncated]" +_CREDENTIAL_NAME_RE = re.compile( + r"(?:API_?KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|AUTH|OPENAI|ANTHROPIC|AWS_|AZURE_)", + re.IGNORECASE, +) + + +class CodePolicySessionConfig(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + submission_url: str = Field(min_length=1) + submission_token: str = Field(min_length=1) + output_limit: int = Field(default=DEFAULT_OUTPUT_LIMIT, ge=0) + startup_timeout_s: float = Field(default=10.0, gt=0) + interrupt_grace_s: float = Field(default=2.0, gt=0) + + +class _BoundedOutput: + def __init__(self, limit: int) -> None: + self.limit = limit + self.parts: list[str] = [] + self.length = 0 + self.truncated = False + + def __call__(self, message: dict[str, Any]) -> None: + message_type = message.get("header", {}).get("msg_type") + content = message.get("content", {}) + value = "" + if message_type == "stream": + value = str(content.get("text", "")) + elif message_type in {"execute_result", "display_data"}: + value = str(content.get("data", {}).get("text/plain", "")) + elif message_type == "error": + traceback = content.get("traceback", []) + value = "\n".join(str(line) for line in traceback) + self._append(_ANSI_ESCAPE_RE.sub("", value)) + + def _append(self, value: str) -> None: + if not value or self.truncated: + return + remaining = self.limit - self.length + if len(value) <= remaining: + self.parts.append(value) + self.length += len(value) + return + marker = _TRUNCATION_MARKER[:remaining] + content_limit = max(0, remaining - len(marker)) + self.parts.append(value[:content_limit] + marker) + self.length = self.limit + self.truncated = True + + def text(self) -> str: + return "".join(self.parts) + + +def _load_kernel_manager() -> type[Any]: + try: + from jupyter_client.manager import KernelManager + except ImportError as exc: + raise RuntimeError( + "CodePolicy requires ipykernel and jupyter-client; install the agents extra" + ) from exc + return KernelManager + + +def _bootstrap_source() -> str: + return """ +from dimos.agents.code_policy_core import submit_policy +from dimos.porcelain.dimos import Dimos +""" + + +def _kernel_environment(config: CodePolicySessionConfig) -> dict[str, str]: + """Build an exploration environment without forwarding host credentials.""" + result = { + name: value for name, value in os.environ.items() if not _CREDENTIAL_NAME_RE.search(name) + } + result[_SUBMISSION_URL_ENV] = config.submission_url + result[_SUBMISSION_TOKEN_ENV] = config.submission_token + return result + + +def submit_policy(policy: Any) -> TrialRun: + """Submit a typed callable from the exploration kernel for one fresh trial.""" + validate_policy_callable(policy) + try: + source = inspect.getsource(policy) + except (OSError, TypeError) as exc: + raise TypeError("policy source is unavailable; define it in the exploration REPL") from exc + try: + import cloudpickle # type: ignore[import-untyped] + + serialized = cloudpickle.dumps(policy) + except Exception as exc: + raise TypeError(f"policy is not serializable: {type(exc).__name__}: {exc}") from exc + + import requests + + response = requests.post( + os.environ[_SUBMISSION_URL_ENV], + headers={"Authorization": f"Bearer {os.environ[_SUBMISSION_TOKEN_ENV]}"}, + json={"source": source, "serialized": base64.b64encode(serialized).decode("ascii")}, + timeout=None, + ) + if response.status_code != 200: + detail = response.json().get("error", response.text) + raise RuntimeError(f"policy submission failed: {detail}") + from pathlib import Path + + from dimos.benchmark.evaluation.protocol import TrialOutcome, TrialRun + + payload = response.json() + return TrialRun( + run_id=payload["run_id"], + outcome=TrialOutcome(**payload["outcome"]), + artifacts=Path(payload["artifacts"]), + log_path=Path(payload["log_path"]), + memory_path=Path(payload["memory_path"]), + ) + + +def validate_policy_callable(policy: Any) -> None: + """Enforce the one canonical callable contract before a trial is launched.""" + from dimos.porcelain.dimos import Dimos + + if not inspect.isfunction(policy) or inspect.iscoroutinefunction(policy): + raise TypeError("policy must be a synchronous Python function") + if policy.__name__ != "policy": + raise TypeError("submitted function must be named 'policy'") + signature = inspect.signature(policy) + parameters = list(signature.parameters.values()) + if len(parameters) != 1 or parameters[0].kind not in { + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + }: + raise TypeError("policy must accept exactly one positional app parameter") + try: + hints = get_type_hints(policy) + except Exception as exc: + raise TypeError(f"policy annotations could not be resolved: {exc}") from exc + if hints.get(parameters[0].name) is not Dimos or hints.get("return") not in {None, type(None)}: + raise TypeError("policy signature must be policy(app: Dimos) -> None") + + +class CodePolicySession: + """Execute trusted Python serially in one persistent Jupyter kernel.""" + + def __init__(self, config: CodePolicySessionConfig) -> None: + self.config = config + self.execution_count = 0 + self.execution_duration_s = 0.0 + self._execution_lock = threading.Lock() + self._kernel_lock = threading.RLock() + self._manager: Any = None + self._client: Any = None + self._stopped = True + + def start(self) -> None: + self._stopped = False + + def python_exec(self, code: str, timeout_s: float = MAX_EXECUTION_TIMEOUT_S) -> str: + if self._stopped: + return "CodePolicy session is stopped" + if not code: + return "python_exec code must be non-empty" + if not 0 < timeout_s <= MAX_EXECUTION_TIMEOUT_S: + return f"timeout_s must be in (0, {MAX_EXECUTION_TIMEOUT_S:g}]" + if not self._execution_lock.acquire(blocking=False): + return "CodePolicy session is busy" + started = time.monotonic() + self.execution_count += 1 + try: + try: + client = self._ensure_kernel() + except Exception as exc: + return f"CodePolicy kernel failed to start: {type(exc).__name__}: {exc}" + output = _BoundedOutput(self.config.output_limit) + try: + reply = client.execute_interactive( + code, + allow_stdin=False, + output_hook=output, + store_history=True, + timeout=timeout_s, + ) + except (TimeoutError, queue.Empty): + if self._interrupt_and_recover(): + return f"Execution timed out after {timeout_s:.1f}s and was interrupted" + return ( + f"Execution timed out after {timeout_s:.1f}s; " + "the kernel was restarted and its namespace was reset" + ) + except Exception as exc: + self._shutdown_kernel() + return f"CodePolicy execution failed: {type(exc).__name__}: {exc}" + content = reply.get("content", {}) + body = output.text().rstrip() + if not body and content.get("status") != "ok": + body = f"{content.get('ename', 'Error')}: {content.get('evalue', '')}" + if not body: + body = "(completed)" + state = "completed" if content.get("status") == "ok" else "failed" + return f"In [{content.get('execution_count', '?')}] {state}\n\n{body}" + finally: + self.execution_duration_s += time.monotonic() - started + self._execution_lock.release() + + def stop(self) -> None: + self._stopped = True + self._shutdown_kernel() + + def _ensure_kernel(self) -> Any: + with self._kernel_lock: + if self._manager is not None and self._client is not None and self._manager.is_alive(): + return self._client + self._shutdown_kernel() + manager = _load_kernel_manager()(kernel_name="python3") + client = None + try: + manager.start_kernel(env=_kernel_environment(self.config)) + client = manager.client() + client.start_channels() + client.wait_for_ready(timeout=self.config.startup_timeout_s) + reply = client.execute_interactive( + _bootstrap_source(), + allow_stdin=False, + output_hook=lambda _message: None, + silent=True, + store_history=False, + timeout=self.config.startup_timeout_s, + ) + if reply.get("content", {}).get("status") != "ok": + content = reply.get("content", {}) + raise RuntimeError( + f"{content.get('ename', 'KernelBootstrapError')}: " + f"{content.get('evalue', 'bootstrap failed')}" + ) + except Exception: + if client is not None: + client.stop_channels() + try: + manager.shutdown_kernel(now=True) + manager.cleanup_resources() + except Exception: + pass + raise + self._manager = manager + self._client = client + return client + + def _interrupt_and_recover(self) -> bool: + manager, client = self._manager, self._client + if manager is None or client is None: + return False + try: + manager.interrupt_kernel() + client.wait_for_ready(timeout=self.config.interrupt_grace_s) + return True + except Exception: + try: + manager.restart_kernel(now=True) + client.wait_for_ready(timeout=self.config.startup_timeout_s) + reply = client.execute_interactive( + _bootstrap_source(), + allow_stdin=False, + output_hook=lambda _message: None, + silent=True, + store_history=False, + timeout=self.config.startup_timeout_s, + ) + if reply.get("content", {}).get("status") != "ok": + raise RuntimeError("bootstrap failed after kernel restart") + except Exception: + self._shutdown_kernel() + return False + + def _shutdown_kernel(self) -> None: + with self._kernel_lock: + manager, client = self._manager, self._client + self._manager = None + self._client = None + if client is not None: + client.stop_channels() + if manager is not None: + try: + manager.shutdown_kernel(now=True) + manager.cleanup_resources() + except Exception: + pass diff --git a/dimos/agents/code_policy_server.py b/dimos/agents/code_policy_server.py new file mode 100644 index 0000000000..a757423ba5 --- /dev/null +++ b/dimos/agents/code_policy_server.py @@ -0,0 +1,181 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""One-tool MCP server plus a private callback for policy submissions.""" + +from __future__ import annotations + +import asyncio +import base64 +from collections.abc import Callable +import logging +import secrets +import socket +import threading +import time + +from mcp.server.mcpserver import MCPServer +from starlette.requests import Request +from starlette.responses import JSONResponse +from starlette.routing import Route +import uvicorn + +from dimos.agents.code_policy_core import ( + MAX_EXECUTION_TIMEOUT_S, + CodePolicySession, + CodePolicySessionConfig, +) +from dimos.benchmark.evaluation.protocol import TrialRun + +PYTHON_EXEC_DESCRIPTION = """Execute Python in a persistent trusted, unsandboxed session. + +Imports, functions, and variables persist between calls. Define a typed +`policy(app: Dimos) -> None` and call `submit_policy(policy)` to run a fresh trial. +""" + +SubmissionHandler = Callable[[str, bytes], TrialRun] + +_NOISY_MCP_TRANSPORT_LOGGERS = ( + "mcp.server.streamable_http", + "mcp.server.streamable_http_manager", +) + + +class CodePolicyMcpServer: + """Own the exploration kernel and its evaluator-owned submission callback.""" + + def __init__(self, submission_handler: SubmissionHandler, *, host: str = "127.0.0.1") -> None: + self.host = host + self.port = 0 + self.submission_handler = submission_handler + self.submission_token = secrets.token_urlsafe(32) + self.session: CodePolicySession | None = None + self.mcp = MCPServer(name="dimos-code-policy", version="1.0.0") + + @self.mcp.tool( + name="python_exec", + description=PYTHON_EXEC_DESCRIPTION, + structured_output=False, + ) + async def python_exec(code: str, timeout_s: float = MAX_EXECUTION_TIMEOUT_S) -> str: + if self.session is None: + return "CodePolicy session is stopped" + return await asyncio.to_thread(self.session.python_exec, code, timeout_s) + + self.app = self.mcp.streamable_http_app( + streamable_http_path="/mcp", + json_response=True, + stateless_http=True, + host=host, + ) + self.app.routes.append(Route("/submit-policy", self._submit_policy, methods=["POST"])) + self._server: uvicorn.Server | None = None + self._thread: threading.Thread | None = None + self._socket: socket.socket | None = None + + @property + def mcp_url(self) -> str: + if self.port == 0: + raise RuntimeError("CodePolicy MCP server is not running") + return f"http://{self.host}:{self.port}/mcp" + + async def _submit_policy(self, request: Request) -> JSONResponse: + if request.headers.get("authorization") != f"Bearer {self.submission_token}": + return JSONResponse({"error": "unauthorized"}, status_code=401) + try: + body = await request.json() + source = str(body["source"]) + serialized = base64.b64decode(str(body["serialized"]), validate=True) + trial = await asyncio.to_thread(self.submission_handler, source, serialized) + except Exception as exc: + return JSONResponse( + {"error": f"{type(exc).__name__}: {exc}"}, + status_code=400, + ) + return JSONResponse( + { + "run_id": trial.run_id, + "outcome": { + "success": trial.outcome.success, + "reward": trial.outcome.reward, + "status": trial.outcome.status, + "error": trial.outcome.error, + "duration_seconds": trial.outcome.duration_seconds, + }, + "artifacts": str(trial.artifacts), + "log_path": str(trial.log_path), + "memory_path": str(trial.memory_path), + } + ) + + def start(self) -> None: + if self._thread is not None: + raise RuntimeError("CodePolicy MCP server is already running") + for logger_name in _NOISY_MCP_TRANSPORT_LOGGERS: + logging.getLogger(logger_name).setLevel(logging.WARNING) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((self.host, 0)) + sock.listen(128) + self.port = int(sock.getsockname()[1]) + self._socket = sock + self.session = CodePolicySession( + CodePolicySessionConfig( + submission_url=f"http://{self.host}:{self.port}/submit-policy", + submission_token=self.submission_token, + ) + ) + self.session.start() + server = uvicorn.Server(uvicorn.Config(self.app, log_level="warning", access_log=False)) + self._server = server + + def serve() -> None: + asyncio.run(server.serve(sockets=[sock])) + + thread = threading.Thread( + target=serve, + name=f"code-policy-mcp-{self.port}", + daemon=True, + ) + self._thread = thread + thread.start() + deadline = time.monotonic() + 10 + while not server.started and thread.is_alive() and time.monotonic() < deadline: + time.sleep(0.01) + if not server.started: + self.stop() + raise TimeoutError("CodePolicy MCP server did not start") + + def stop(self) -> None: + server, thread = self._server, self._thread + self._server = None + self._thread = None + if server is not None: + server.should_exit = True + if thread is not None: + thread.join(timeout=5) + if self._socket is not None: + self._socket.close() + self._socket = None + if self.session is not None: + self.session.stop() + self.session = None + self.port = 0 + + def __enter__(self) -> CodePolicyMcpServer: + self.start() + return self + + def __exit__(self, *_args: object) -> None: + self.stop() diff --git a/dimos/agents/test_code_policy_core.py b/dimos/agents/test_code_policy_core.py new file mode 100644 index 0000000000..26e1a48fb0 --- /dev/null +++ b/dimos/agents/test_code_policy_core.py @@ -0,0 +1,87 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from pathlib import Path + +import cloudpickle +import pytest + +from dimos.agents.code_policy_core import validate_policy_callable +from dimos.agents.code_policy_server import CodePolicyMcpServer +from dimos.benchmark.evaluation.protocol import TrialOutcome, TrialRun +from dimos.memory2.store.sqlite import SqliteStore +from dimos.porcelain.dimos import Dimos + + +def policy(app: Dimos) -> None: + del app + + +def test_validate_policy_callable_accepts_canonical_signature() -> None: + validate_policy_callable(policy) + + +@pytest.mark.parametrize( + ("candidate", "message"), + [ + (lambda: None, "named 'policy'"), + (lambda app: None, "named 'policy'"), + ], +) +def test_validate_policy_callable_rejects_noncanonical_functions(candidate, message: str) -> None: + with pytest.raises(TypeError, match=message): + validate_policy_callable(candidate) + + +def test_exploration_repl_submits_callable_and_receives_trial(tmp_path: Path) -> None: + artifacts = tmp_path / "trial" + artifacts.mkdir() + log_path = artifacts / "main.jsonl" + log_path.write_text(json.dumps({"module": "Planner", "event": "failed"}) + "\n") + memory_path = artifacts / "recording.db" + with SqliteStore(path=str(memory_path)) as memory: + memory.stream("events", str).append("attempted") + submitted: list[object] = [] + + def handle(source: str, serialized: bytes) -> TrialRun: + submitted.append(cloudpickle.loads(serialized)) + assert "def policy" in source + return TrialRun( + run_id="debug-1", + outcome=TrialOutcome( + success=False, + reward=0.0, + status="completed", + error=None, + duration_seconds=1.0, + ), + artifacts=artifacts, + log_path=log_path, + memory_path=memory_path, + ) + + with CodePolicyMcpServer(handle) as server: + assert server.session is not None + result = server.session.python_exec( + "def policy(app: Dimos) -> None:\n" + " app.list_modules()\n\n" + "trial = submit_policy(policy)\n" + "(trial.run_id, trial.outcome.success)" + ) + + assert "('debug-1', False)" in result + assert len(submitted) == 1 diff --git a/dimos/benchmark/evaluation/models.py b/dimos/benchmark/evaluation/models.py new file mode 100644 index 0000000000..6f58ccc0fe --- /dev/null +++ b/dimos/benchmark/evaluation/models.py @@ -0,0 +1,131 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Universal contracts for requesting and recording evaluation runs.""" + +from __future__ import annotations + +from datetime import datetime +from pathlib import PurePosixPath +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class EvaluationModel(BaseModel): + """Strict immutable base for persisted evaluation contracts.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class EvaluationReference(EvaluationModel): + name: str = Field(min_length=1) + config: dict[str, Any] = Field(default_factory=dict) + + +class EvaluationRunSpecification(EvaluationModel): + schema_version: Literal["1.0"] = "1.0" + evaluation: EvaluationReference + + +class ArtifactReference(EvaluationModel): + path: str = Field(min_length=1) + label: str = Field(min_length=1) + media_type: str | None = Field(default=None, min_length=1) + + @model_validator(mode="after") + def path_is_safe_and_relative(self) -> ArtifactReference: + path = PurePosixPath(self.path) + if path.is_absolute() or not path.parts or ".." in path.parts: + raise ValueError("artifact path must be a safe relative POSIX path") + return self + + +SummaryValue = str | int | float | bool | None + + +class SummaryItem(EvaluationModel): + key: str = Field(min_length=1, pattern=r"^[a-z][a-z0-9_]*$") + label: str = Field(min_length=1) + value: SummaryValue + + +class InlineNativeResult(EvaluationModel): + kind: Literal["inline"] = "inline" + value: Any + + +class ArtifactNativeResult(EvaluationModel): + kind: Literal["artifact"] = "artifact" + artifact: ArtifactReference + + +NativeResult = Annotated[ + InlineNativeResult | ArtifactNativeResult, + Field(discriminator="kind"), +] + + +class EvaluationReport(EvaluationModel): + summary: tuple[SummaryItem, ...] = () + native_result: NativeResult + artifacts: tuple[ArtifactReference, ...] = () + + +class EvaluationIdentity(EvaluationModel): + name: str = Field(min_length=1) + provider: str = Field(min_length=1) + version: str = Field(min_length=1) + + +class RuntimeIdentity(EvaluationModel): + profile: Literal["code-policy-v1"] = "code-policy-v1" + driver: Literal["pi"] = "pi" + driver_version: str = Field(min_length=1) + model: str = Field(min_length=1) + thinking_level: str = Field(min_length=1) + + +class EvaluationRunError(EvaluationModel): + stage: Literal["evaluation", "publication"] + error_type: str = Field(min_length=1) + message: str = Field(min_length=1) + + +class EvaluationRun(EvaluationModel): + schema_version: Literal["1.0"] = "1.0" + run_id: str = Field(min_length=1) + specification: EvaluationRunSpecification + evaluation: EvaluationIdentity + runtime: RuntimeIdentity + status: Literal["completed", "failed", "cancelled"] + started_at: datetime + finished_at: datetime + duration_seconds: float = Field(ge=0) + report: EvaluationReport | None = None + error: EvaluationRunError | None = None + runtime_artifacts: tuple[ArtifactReference, ...] = () + prompt_evidence: tuple[ArtifactReference, ...] = () + + @model_validator(mode="after") + def status_matches_payload(self) -> EvaluationRun: + if self.status == "completed" and self.report is None: + raise ValueError("completed evaluation runs require a report") + if self.status != "completed" and self.error is None: + raise ValueError("non-completed evaluation runs require an error") + if self.status == "completed" and self.error is not None: + raise ValueError("completed evaluation runs cannot contain an error") + if self.status != "completed" and self.report is not None: + raise ValueError("non-completed evaluation runs cannot contain a report") + return self diff --git a/dimos/benchmark/evaluation/pi_process.py b/dimos/benchmark/evaluation/pi_process.py new file mode 100644 index 0000000000..f02fe926c1 --- /dev/null +++ b/dimos/benchmark/evaluation/pi_process.py @@ -0,0 +1,309 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Launch the pinned stock Pi CLI and parse its native JSON event stream.""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +import os +from pathlib import Path +import subprocess +import threading +import time +from typing import Any + +from dimos.benchmark.evaluation.progress import ( + AssistantTextProgress, + FinalResponseProgress, + ProgressSink, + StatusProgress, + ToolEndProgress, + ToolStartProgress, + emit_progress, +) + +PI_VERSION = "0.80.10" +MAX_STDERR_BYTES = 64 * 1024 + + +@dataclass(frozen=True) +class PiRunResult: + final_text: str + tool_call_count: int + duration_seconds: float + transcript_path: Path | None + stderr: str + + +class PiRunError(RuntimeError): + def __init__(self, message: str, *, stderr: str = "") -> None: + super().__init__(message) + self.stderr = stderr + + +class PiCliRunner: + """Thin stock-CLI binding; the extension owns only the `python_exec` tool.""" + + def __init__( + self, + *, + cli: Path, + extension: Path, + model: str, + thinking_level: str, + timeout_s: float, + progress: ProgressSink | None = None, + ) -> None: + if not cli.is_file(): + raise FileNotFoundError(f"Pi {PI_VERSION} CLI is not installed: {cli}") + if not extension.is_file(): + raise FileNotFoundError( + f"Pi CodePolicy extension is not built: {extension}; " + "run `npm run build --prefix packages/pi-code-policy-extension`" + ) + self.cli = cli + self.extension = extension + self.model = model + self.thinking_level = thinking_level + self.timeout_s = timeout_s + self.progress = progress + + def run( + self, + *, + prompt: str, + system_prompt: str, + mcp_url: str, + api_key: str, + run_dir: Path, + ) -> PiRunResult: + session_dir = run_dir / "pi-session" + agent_dir = run_dir / ".pi-agent" + system_prompt_path = run_dir / "system-prompt.txt" + system_prompt_path.write_text(system_prompt, encoding="utf-8") + command = ( + "node", + str(self.cli), + "--mode", + "json", + "--model", + f"openai/{self.model}", + "--thinking", + self.thinking_level, + "--session-dir", + str(session_dir), + "--name", + "dimos-evaluation", + "--no-builtin-tools", + "--tools", + "python_exec", + "--no-extensions", + "--extension", + str(self.extension), + "--no-skills", + "--no-prompt-templates", + "--no-themes", + "--no-context-files", + "--no-approve", + "--system-prompt", + str(system_prompt_path), + prompt, + ) + env = { + "PATH": os.environ.get("PATH", ""), + "OPENAI_API_KEY": api_key, + "DIMOS_CODE_POLICY_MCP_URL": mcp_url, + "PI_CODING_AGENT_DIR": str(agent_dir), + "PI_SKIP_VERSION_CHECK": "1", + "PI_TELEMETRY": "0", + } + started = time.monotonic() + process = subprocess.Popen( + command, + cwd=run_dir, + env=env, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + stdout = process.stdout + stderr_stream = process.stderr + assert stdout is not None + assert stderr_stream is not None + events = _PiEventAccumulator(self.progress) + stderr_parts: list[str] = [] + stderr_bytes = 0 + + def read_stdout() -> None: + for line in stdout: + events.feed(line) + + def read_stderr() -> None: + nonlocal stderr_bytes + for line in stderr_stream: + line = line.replace(api_key, "[REDACTED]") + encoded = line.encode() + remaining = MAX_STDERR_BYTES - stderr_bytes + if remaining > 0: + retained = encoded[:remaining].decode(errors="ignore") + stderr_parts.append(retained) + stderr_bytes += len(retained.encode()) + message = line.strip() + if message: + emit_progress( + self.progress, + StatusProgress(channel="pi", message=_bounded_stderr(message)), + ) + + readers = ( + threading.Thread(target=read_stdout, name="pi-stdout", daemon=True), + threading.Thread(target=read_stderr, name="pi-stderr", daemon=True), + ) + for reader in readers: + reader.start() + timed_out = False + try: + process.wait(timeout=self.timeout_s) + except subprocess.TimeoutExpired: + timed_out = True + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + finally: + for reader, stream in zip( + readers, + (stdout, stderr_stream), + strict=True, + ): + reader.join(timeout=5) + if reader.is_alive(): + stream.close() + reader.join(timeout=1) + + stderr = "".join(stderr_parts) + if timed_out: + raise PiRunError( + f"Pi timed out after {self.timeout_s:g}s", + stderr=stderr, + ) + duration = time.monotonic() - started + if process.returncode != 0: + raise PiRunError(f"Pi exited with status {process.returncode}", stderr=stderr) + if events.stop_error is not None: + raise PiRunError(events.stop_error, stderr=stderr) + if events.final_text is None: + raise PiRunError("Pi produced no final assistant response", stderr=stderr) + transcripts = sorted(session_dir.rglob("*.jsonl")) if session_dir.exists() else [] + return PiRunResult( + final_text=events.final_text, + tool_call_count=events.tool_count, + duration_seconds=duration, + transcript_path=transcripts[-1] if transcripts else None, + stderr=stderr, + ) + + +def parse_pi_events(stream: str) -> tuple[str | None, int, str | None]: + """Return the final assistant text, tool count, and terminal error.""" + events = _PiEventAccumulator() + for line in stream.splitlines(): + events.feed(line) + return events.final_text, events.tool_count, events.stop_error + + +class _PiEventAccumulator: + def __init__(self, progress: ProgressSink | None = None) -> None: + self.progress = progress + self.final_text: str | None = None + self.tool_count = 0 + self.stop_error: str | None = None + self._tool_started: dict[str, float] = {} + + def feed(self, line: str) -> None: + try: + event = json.loads(line) + except json.JSONDecodeError: + return + if not isinstance(event, dict): + return + event_type = event.get("type") + if event_type == "message_update": + update = event.get("assistantMessageEvent") + if isinstance(update, dict) and update.get("type") == "text_delta": + delta = update.get("delta") + if isinstance(delta, str) and delta: + emit_progress(self.progress, AssistantTextProgress(delta=delta)) + return + if event_type == "tool_execution_start": + self.tool_count += 1 + call_id = str(event.get("toolCallId", "")) + self._tool_started[call_id] = time.monotonic() + args = event.get("args") + code = args.get("code") if isinstance(args, dict) else None + if event.get("toolName") == "python_exec" and isinstance(code, str) and code: + emit_progress(self.progress, ToolStartProgress(code=code)) + return + if event_type == "tool_execution_end": + call_id = str(event.get("toolCallId", "")) + started = self._tool_started.pop(call_id, time.monotonic()) + if event.get("toolName") == "python_exec": + emit_progress( + self.progress, + ToolEndProgress( + ok=not bool(event.get("isError")), + result=_tool_result_text(event.get("result")), + duration_seconds=max(0.0, time.monotonic() - started), + ), + ) + return + if event_type != "message_end": + return + message = event.get("message") + if not isinstance(message, dict) or message.get("role") != "assistant": + return + self.final_text = "".join( + str(item.get("text", "")) + for item in message.get("content", []) + if isinstance(item, dict) and item.get("type") == "text" + ) + emit_progress(self.progress, FinalResponseProgress(text=self.final_text)) + stop_reason = message.get("stopReason") + if stop_reason in {"error", "aborted"}: + self.stop_error = str(message.get("errorMessage") or f"Pi request {stop_reason}") + + +def _tool_result_text(result: Any) -> str: + if isinstance(result, dict): + content = result.get("content") + if isinstance(content, list): + return "\n".join( + str(item.get("text", "")) + for item in content + if isinstance(item, dict) and item.get("type") == "text" + ) + return str(result) + + +def _bounded_stderr(value: str) -> str: + encoded = value.encode() + if len(encoded) <= MAX_STDERR_BYTES: + return value + return encoded[:MAX_STDERR_BYTES].decode(errors="ignore") diff --git a/dimos/benchmark/evaluation/progress.py b/dimos/benchmark/evaluation/progress.py new file mode 100644 index 0000000000..f8e58ad149 --- /dev/null +++ b/dimos/benchmark/evaluation/progress.py @@ -0,0 +1,84 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Presentation-only progress contracts for interactive evaluation runs.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class ProgressModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class StatusProgress(ProgressModel): + kind: Literal["status"] = "status" + channel: Literal["eval", "pi"] + message: str = Field(min_length=1) + + +class CaseHeaderProgress(ProgressModel): + kind: Literal["case_header"] = "case_header" + case_id: str = Field(min_length=1) + source: str = Field(min_length=1) + progress: float | None = Field(default=None, ge=0, le=1) + question: str = Field(min_length=1) + + +class AssistantTextProgress(ProgressModel): + kind: Literal["assistant_text"] = "assistant_text" + delta: str = Field(min_length=1) + + +class ToolStartProgress(ProgressModel): + kind: Literal["tool_start"] = "tool_start" + code: str = Field(min_length=1) + + +class ToolEndProgress(ProgressModel): + kind: Literal["tool_end"] = "tool_end" + ok: bool + result: str + duration_seconds: float = Field(ge=0) + + +class FinalResponseProgress(ProgressModel): + kind: Literal["final_response"] = "final_response" + text: str + + +EvalProgress = Annotated[ + StatusProgress + | CaseHeaderProgress + | AssistantTextProgress + | ToolStartProgress + | ToolEndProgress + | FinalResponseProgress, + Field(discriminator="kind"), +] +ProgressSink = Callable[[EvalProgress], None] + + +def emit_progress(sink: ProgressSink | None, event: EvalProgress) -> None: + """Notify a presentation observer without allowing it to affect evaluation.""" + if sink is None: + return + try: + sink(event) + except Exception: + return diff --git a/dimos/benchmark/evaluation/protocol.py b/dimos/benchmark/evaluation/protocol.py new file mode 100644 index 0000000000..fe67dac0c4 --- /dev/null +++ b/dimos/benchmark/evaluation/protocol.py @@ -0,0 +1,144 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public contracts for native evaluations and CodePolicy execution.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +import json +from pathlib import Path +from typing import TYPE_CHECKING, Literal, Protocol, runtime_checkable + +from pydantic import BaseModel + +from dimos.benchmark.evaluation.models import EvaluationReport +from dimos.benchmark.evaluation.progress import ProgressSink + +if TYPE_CHECKING: + from dimos.memory2.store.base import Store + + +@dataclass(frozen=True) +class PolicyArtifact: + """A task-level callable captured during exploration.""" + + source_path: Path + serialized_path: Path + sha256: str + + +@dataclass(frozen=True) +class PolicyExecution: + status: Literal["completed", "policy_error", "timed_out", "infrastructure_error"] + duration_seconds: float + error: str | None = None + + +@dataclass(frozen=True) +class TrialOutcome: + success: bool + reward: float | None + status: Literal["completed", "policy_error", "timed_out", "infrastructure_error"] + error: str | None + duration_seconds: float + + +@dataclass(frozen=True) +class TrialRun: + """Read-only postmortem handle for one fully stopped policy trial.""" + + run_id: str + outcome: TrialOutcome + artifacts: Path + log_path: Path + memory_path: Path + + def read_logs( + self, + *, + module: str | None = None, + tail: int | None = None, + ) -> tuple[dict[str, object], ...]: + from dimos.core.log_viewer import read_log + + records = [json.loads(line) for line in read_log(self.log_path, count=None)] + if module is not None: + records = [record for record in records if record.get("module") == module] + if tail is not None: + if tail < 0: + raise ValueError("tail must be non-negative") + records = records[-tail:] if tail else [] + return tuple(records) + + def open_memory(self) -> Store: + """Open the completed trial's Memory2 recording read-only.""" + from dimos.memory2.store.sqlite import SqliteStore + + return SqliteStore(path=str(self.memory_path), must_exist=True, read_only=True) + + +DebugTrialSubmitter = Callable[[PolicyArtifact, int, Path], TrialRun] + + +@dataclass(frozen=True) +class ExplorationOutcome: + status: Literal["valid", "invalid"] + policy: PolicyArtifact | None + trials: tuple[TrialRun, ...] + final_text: str + tool_call_count: int + duration_seconds: float + error: str | None = None + + +@runtime_checkable +class CodePolicyRuntime(Protocol): + """Fixed Pi runtime injected into complete native evaluations.""" + + def explore( + self, + *, + evaluation_protocol: str, + task_input: str, + submit_debug_trial: DebugTrialSubmitter, + max_submissions: int = 5, + ) -> ExplorationOutcome: ... + + def execute( + self, + policy: PolicyArtifact, + *, + timeout_s: float, + ) -> PolicyExecution: ... + + +@dataclass(frozen=True) +class EvaluationContext: + run_id: str + spec_dir: Path + workspace: Path + runtime: CodePolicyRuntime + progress: ProgressSink | None + + +@runtime_checkable +class Evaluation(Protocol): + """A complete benchmark integration with native result semantics.""" + + name: str + config_model: type[BaseModel] + + def run(self, config: BaseModel, context: EvaluationContext) -> EvaluationReport: ... diff --git a/dimos/benchmark/evaluation/registry.py b/dimos/benchmark/evaluation/registry.py new file mode 100644 index 0000000000..66f8cb5373 --- /dev/null +++ b/dimos/benchmark/evaluation/registry.py @@ -0,0 +1,134 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Lazy built-in and installed-package Evaluation discovery.""" + +from __future__ import annotations + +from dataclasses import dataclass +import importlib +import importlib.metadata as importlib_metadata +import re +from typing import Any + +from packaging.utils import canonicalize_name +from pydantic import BaseModel + +from dimos.benchmark.evaluation.protocol import Evaluation + +ENTRY_POINT_GROUP = "dimos.evaluations" +LOCAL_NAME_PATTERN = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") +BUILTIN_EVALUATIONS: dict[str, str] = {} + + +class EvaluationRegistryError(ValueError): + """An Evaluation name or plugin could not be resolved.""" + + +@dataclass(frozen=True) +class ResolvedEvaluation: + name: str + provider: str + version: str + evaluation: Evaluation + + +def available_evaluations() -> list[str]: + return sorted([*BUILTIN_EVALUATIONS, *_external_entries()]) + + +def resolve_evaluation(name: str) -> ResolvedEvaluation: + if name in BUILTIN_EVALUATIONS: + target = _load_target(BUILTIN_EVALUATIONS[name], name) + return ResolvedEvaluation( + name=name, + provider="dimos", + version=_distribution_version("dimos"), + evaluation=_validate_target(name, target), + ) + + entries = _external_entries() + entry = entries.get(name) + if entry is None: + available = available_evaluations() + suffix = f" Available evaluations: {', '.join(available)}." if available else "" + raise EvaluationRegistryError(f"Unknown evaluation {name!r}.{suffix}") + try: + target = entry.load() + except Exception as exc: + raise EvaluationRegistryError( + f"Failed to load evaluation {name!r} from {entry.value!r}: {type(exc).__name__}: {exc}" + ) from exc + distribution = entry.dist + assert distribution is not None + distribution_name = distribution.metadata["Name"] + return ResolvedEvaluation( + name=name, + provider=distribution_name, + version=distribution.version, + evaluation=_validate_target(name, target), + ) + + +def _external_entries() -> dict[str, importlib_metadata.EntryPoint]: + result: dict[str, importlib_metadata.EntryPoint] = {} + for entry in importlib_metadata.entry_points(group=ENTRY_POINT_GROUP): + distribution = entry.dist + if distribution is None: + continue + distribution_name = distribution.metadata.get("Name") + if not distribution_name or LOCAL_NAME_PATTERN.fullmatch(entry.name) is None: + continue + namespace = str(canonicalize_name(distribution_name)) + qualified_name = f"{namespace}.{entry.name}" + if qualified_name in result: + raise EvaluationRegistryError( + f"Multiple installed entry points provide evaluation {qualified_name!r}" + ) + result[qualified_name] = entry + return result + + +def _load_target(path: str, name: str) -> Any: + module_name, separator, attribute = path.partition(":") + if not separator: + raise EvaluationRegistryError(f"Invalid built-in evaluation target for {name!r}: {path}") + try: + return getattr(importlib.import_module(module_name), attribute) + except Exception as exc: + raise EvaluationRegistryError( + f"Failed to load evaluation {name!r} from {path!r}: {type(exc).__name__}: {exc}" + ) from exc + + +def _validate_target(name: str, target: Any) -> Evaluation: + if not isinstance(target, Evaluation): + raise EvaluationRegistryError( + f"Evaluation {name!r} must expose name, config_model, and run()" + ) + config_model = target.config_model + if not isinstance(config_model, type) or not issubclass(config_model, BaseModel): + raise EvaluationRegistryError( + f"Evaluation {name!r} config_model must be a Pydantic BaseModel type" + ) + if target.name != name.rpartition(".")[2]: + raise EvaluationRegistryError(f"Evaluation {name!r} loaded a target named {target.name!r}") + return target + + +def _distribution_version(name: str) -> str: + try: + return importlib_metadata.version(name) + except importlib_metadata.PackageNotFoundError: + return "source" diff --git a/dimos/benchmark/evaluation/runner.py b/dimos/benchmark/evaluation/runner.py new file mode 100644 index 0000000000..0485c10e76 --- /dev/null +++ b/dimos/benchmark/evaluation/runner.py @@ -0,0 +1,141 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Resolve, execute, and atomically publish one Evaluation Run.""" + +from __future__ import annotations + +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import shutil +import tempfile +import time +from typing import Literal +from uuid import uuid4 + +from dimos.benchmark.evaluation.models import ( + EvaluationIdentity, + EvaluationRun, + EvaluationRunError, + EvaluationRunSpecification, +) +from dimos.benchmark.evaluation.progress import ProgressSink, StatusProgress, emit_progress +from dimos.benchmark.evaluation.protocol import EvaluationContext +from dimos.benchmark.evaluation.registry import resolve_evaluation +from dimos.benchmark.evaluation.runtime import CodePolicyRuntimeFactory + + +def execute_evaluation( + specification_path: Path, + *, + output: Path, + api_key_env: str = "OPENAI_API_KEY", + progress: ProgressSink | None = None, +) -> EvaluationRun: + """Run one resolved Evaluation and publish its immutable record.""" + specification_path = specification_path.expanduser().resolve() + output = output.expanduser().resolve() + _validate_output(output) + specification = EvaluationRunSpecification.model_validate_json(specification_path.read_bytes()) + resolved = resolve_evaluation(specification.evaluation.name) + config = resolved.evaluation.config_model.model_validate_json( + json.dumps(specification.evaluation.config), + strict=True, + ) + api_key = os.environ.get(api_key_env) + if not api_key: + raise ValueError(f"API key environment variable {api_key_env!r} is unset") + + output.parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix=f".{output.name}-", dir=output.parent)) + run_id = str(uuid4()) + started_at = datetime.now(timezone.utc) + started = time.monotonic() + runtime = CodePolicyRuntimeFactory( + api_key=api_key, + workspace=temporary, + progress=progress, + ) + context = EvaluationContext( + run_id=run_id, + spec_dir=specification_path.parent, + workspace=temporary, + runtime=runtime, + progress=progress, + ) + emit_progress(progress, StatusProgress(channel="eval", message="evaluation started")) + try: + status: Literal["completed", "failed", "cancelled"] + try: + report = resolved.evaluation.run(config, context) + status = "completed" + error = None + except KeyboardInterrupt: + report = None + status = "cancelled" + error = EvaluationRunError( + stage="evaluation", + error_type="KeyboardInterrupt", + message="Evaluation cancelled by user", + ) + except Exception as exc: + report = None + status = "failed" + error = EvaluationRunError( + stage="evaluation", + error_type=type(exc).__name__, + message=_redact_error(str(exc) or type(exc).__name__, api_key), + ) + finished_at = datetime.now(timezone.utc) + run = EvaluationRun( + run_id=run_id, + specification=specification, + evaluation=EvaluationIdentity( + name=resolved.name, + provider=resolved.provider, + version=resolved.version, + ), + runtime=runtime.identity, + status=status, + started_at=started_at, + finished_at=finished_at, + duration_seconds=time.monotonic() - started, + report=report, + error=error, + runtime_artifacts=runtime.runtime_artifacts, + prompt_evidence=runtime.prompt_evidence, + ) + (temporary / "run.json").write_text( + run.model_dump_json(indent=2) + "\n", + encoding="utf-8", + ) + if output.exists(): + output.rmdir() + os.replace(temporary, output) + emit_progress(progress, StatusProgress(channel="eval", message="run published")) + return run + finally: + if temporary.exists(): + shutil.rmtree(temporary) + + +def _validate_output(output: Path) -> None: + if output.exists() and (not output.is_dir() or any(output.iterdir())): + raise FileExistsError(f"Output must be absent or an empty directory: {output}") + + +def _redact_error(message: str, api_key: str) -> str: + return message.replace(api_key, "[REDACTED]") diff --git a/dimos/benchmark/evaluation/runtime.py b/dimos/benchmark/evaluation/runtime.py new file mode 100644 index 0000000000..6db53f889b --- /dev/null +++ b/dimos/benchmark/evaluation/runtime.py @@ -0,0 +1,402 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fixed Pi exploration and clean, agent-free CodePolicy execution.""" + +from __future__ import annotations + +import hashlib +import json +import multiprocessing +from pathlib import Path +import queue +import shutil +import time +from typing import Any, Literal + +from dimos.agents.code_policy_server import CodePolicyMcpServer +from dimos.benchmark.evaluation.models import ArtifactReference, RuntimeIdentity +from dimos.benchmark.evaluation.pi_process import PI_VERSION, PiCliRunner, PiRunError +from dimos.benchmark.evaluation.progress import ProgressSink +from dimos.benchmark.evaluation.protocol import ( + DebugTrialSubmitter, + ExplorationOutcome, + PolicyArtifact, + PolicyExecution, + TrialRun, +) + +CODE_POLICY_PROFILE: Literal["code-policy-v1"] = "code-policy-v1" +MODEL = "gpt-5.6-luna" +THINKING_LEVEL = "medium" +TURN_TIMEOUT_SECONDS = 600.0 +DEFAULT_MAX_SUBMISSIONS = 5 + +SYSTEM_INSTRUCTIONS = """You are a CodePolicy exploration agent. + +Use the single `python_exec` tool to solve the supplied robotics task. Python +executes in a persistent trusted environment. Define a synchronous function +with the exact signature `def policy(app: Dimos) -> None`, then call +`submit_policy(policy)` to test it. Every accepted submission starts a fresh +debug environment and a fresh policy-only DimOS blueprint. Inspect the returned +TrialRun's outcome, logs, Memory2 recording, and artifacts to diagnose failures. +You may submit at most five trials. The last accepted submission becomes the +task-level policy evaluated later without an agent. Do not connect to DimOS +directly from the exploration REPL. +""" + + +class CodePolicyRuntimeFactory: + """The one fixed CodePolicy runtime supplied to complete Evaluations.""" + + def __init__( + self, + *, + api_key: str, + workspace: Path, + progress: ProgressSink | None = None, + ) -> None: + self.api_key = api_key + self.workspace = workspace + self.progress = progress + self._exploration_count = 0 + self._prompt_evidence: list[ArtifactReference] = [] + self._runtime_artifacts: list[ArtifactReference] = [] + + @property + def identity(self) -> RuntimeIdentity: + return RuntimeIdentity( + profile=CODE_POLICY_PROFILE, + driver_version=PI_VERSION, + model=MODEL, + thinking_level=THINKING_LEVEL, + ) + + @property + def prompt_evidence(self) -> tuple[ArtifactReference, ...]: + return tuple(self._prompt_evidence) + + @property + def runtime_artifacts(self) -> tuple[ArtifactReference, ...]: + return tuple(self._runtime_artifacts) + + def explore( + self, + *, + evaluation_protocol: str, + task_input: str, + submit_debug_trial: DebugTrialSubmitter, + max_submissions: int = DEFAULT_MAX_SUBMISSIONS, + ) -> ExplorationOutcome: + if not evaluation_protocol.strip() or not task_input.strip(): + raise ValueError("evaluation_protocol and task_input must be non-empty") + if max_submissions != DEFAULT_MAX_SUBMISSIONS: + raise ValueError( + f"code-policy-v1 requires exactly {DEFAULT_MAX_SUBMISSIONS} submissions" + ) + + self._exploration_count += 1 + relative = Path("runtime") / f"exploration-{self._exploration_count:04d}" + path = self.workspace / relative + path.mkdir(parents=True) + manager = _SubmissionManager( + workspace=self.workspace, + path=path, + relative_path=relative, + submit_debug_trial=submit_debug_trial, + max_submissions=max_submissions, + record_artifact=self._runtime_artifacts.append, + ) + server = CodePolicyMcpServer(manager.submit) + server.start() + try: + user_message = _assemble_user_message(evaluation_protocol, task_input) + self._record_prompt(path, relative, evaluation_protocol, task_input, user_message) + working = path / "working" + working.mkdir() + cli, extension = _pi_paths() + runner = PiCliRunner( + cli=cli, + extension=extension, + model=MODEL, + thinking_level=THINKING_LEVEL, + timeout_s=TURN_TIMEOUT_SECONDS, + progress=self.progress, + ) + try: + result = runner.run( + prompt=user_message, + system_prompt=SYSTEM_INSTRUCTIONS, + mcp_url=server.mcp_url, + api_key=self.api_key, + run_dir=working, + ) + except PiRunError as exc: + self._record_text(path, relative, "stderr.log", exc.stderr, "Pi stderr") + raise + if result.transcript_path is not None: + target = path / "pi-transcript.jsonl" + shutil.copy2(result.transcript_path, target) + self._runtime_artifacts.append( + _artifact(relative / target.name, "Pi transcript", "application/x-ndjson") + ) + if result.stderr: + self._record_text(path, relative, "stderr.log", result.stderr, "Pi stderr") + policy = manager.last_policy + error = None if policy is not None else "Pi did not submit a valid policy" + outcome = ExplorationOutcome( + status="valid" if policy is not None else "invalid", + policy=policy, + trials=tuple(manager.trials), + final_text=result.final_text, + tool_call_count=result.tool_call_count, + duration_seconds=result.duration_seconds, + error=error, + ) + manifest = path / "exploration.json" + manifest.write_text( + json.dumps( + { + "status": outcome.status, + "policy_sha256": policy.sha256 if policy is not None else None, + "submission_count": manager.accepted_count, + "error": error, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + self._runtime_artifacts.append( + _artifact(relative / manifest.name, "Exploration manifest", "application/json") + ) + return outcome + finally: + server.stop() + shutil.rmtree(path / "working", ignore_errors=True) + + def execute(self, policy: PolicyArtifact, *, timeout_s: float) -> PolicyExecution: + if timeout_s <= 0: + raise ValueError("timeout_s must be positive") + started = time.monotonic() + context = multiprocessing.get_context("spawn") + result_queue = context.Queue(maxsize=1) + process = context.Process( + target=_execute_policy_worker, + args=(str(policy.serialized_path), result_queue), + daemon=True, + ) + try: + process.start() + process.join(timeout_s) + if process.is_alive(): + process.terminate() + process.join(5) + if process.is_alive(): + process.kill() + process.join() + return PolicyExecution( + status="timed_out", + duration_seconds=time.monotonic() - started, + error=f"policy timed out after {timeout_s:g}s", + ) + try: + status, error = result_queue.get_nowait() + except queue.Empty: + return PolicyExecution( + status="infrastructure_error", + duration_seconds=time.monotonic() - started, + error=f"policy worker exited with code {process.exitcode} without a result", + ) + return PolicyExecution( + status=status, + duration_seconds=time.monotonic() - started, + error=error, + ) + except Exception as exc: + return PolicyExecution( + status="infrastructure_error", + duration_seconds=time.monotonic() - started, + error=f"{type(exc).__name__}: {exc}", + ) + finally: + result_queue.close() + + def _record_prompt( + self, + path: Path, + relative: Path, + evaluation_protocol: str, + task_input: str, + user_message: str, + ) -> None: + components = ( + ("runtime-system.txt", "runtime", SYSTEM_INSTRUCTIONS), + ("evaluation-protocol.txt", "evaluation", evaluation_protocol), + ("task-input.txt", "evaluation", task_input), + ("assembled-user-message.txt", "runtime", user_message), + ) + manifest_components: list[dict[str, str]] = [] + for filename, owner, value in components: + target = path / filename + target.write_text(value, encoding="utf-8") + reference = _artifact(relative / filename, filename, "text/plain") + self._prompt_evidence.append(reference) + self._runtime_artifacts.append(reference) + manifest_components.append( + { + "path": reference.path, + "owner": owner, + "sha256": hashlib.sha256(value.encode()).hexdigest(), + } + ) + manifest = path / "prompt-assembly.json" + manifest.write_text( + json.dumps( + { + "schema_version": "1.0", + "runtime_profile": CODE_POLICY_PROFILE, + "components": manifest_components, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + reference = _artifact(relative / manifest.name, "Prompt assembly", "application/json") + self._prompt_evidence.append(reference) + self._runtime_artifacts.append(reference) + + def _record_text( + self, + path: Path, + relative: Path, + filename: str, + value: str, + label: str, + ) -> None: + if not value: + return + (path / filename).write_text(value, encoding="utf-8") + self._runtime_artifacts.append(_artifact(relative / filename, label, "text/plain")) + + +class _SubmissionManager: + def __init__( + self, + *, + workspace: Path, + path: Path, + relative_path: Path, + submit_debug_trial: DebugTrialSubmitter, + max_submissions: int, + record_artifact: Any, + ) -> None: + self.workspace = workspace + self.path = path + self.relative_path = relative_path + self.submit_debug_trial = submit_debug_trial + self.max_submissions = max_submissions + self.record_artifact = record_artifact + self.last_policy: PolicyArtifact | None = None + self.trials: list[TrialRun] = [] + self.accepted_count = 0 + + def submit(self, source: str, serialized: bytes) -> TrialRun: + if self.accepted_count >= self.max_submissions: + raise RuntimeError(f"submission budget exhausted ({self.max_submissions})") + import cloudpickle # type: ignore[import-untyped] + + from dimos.agents.code_policy_core import validate_policy_callable + + policy_callable = cloudpickle.loads(serialized) + validate_policy_callable(policy_callable) + self.accepted_count += 1 + submission_number = self.accepted_count + relative = self.relative_path / f"submission-{submission_number:04d}" + submission_path = self.workspace / relative + submission_path.mkdir() + source_path = submission_path / "policy.py" + serialized_path = submission_path / "policy.pkl" + source_path.write_text(source.rstrip() + "\n", encoding="utf-8") + serialized_path.write_bytes(serialized) + policy = PolicyArtifact( + source_path=source_path, + serialized_path=serialized_path, + sha256=hashlib.sha256(serialized).hexdigest(), + ) + self.last_policy = policy + self.record_artifact(_artifact(relative / "policy.py", "Policy source", "text/x-python")) + self.record_artifact( + _artifact(relative / "policy.pkl", "Serialized policy", "application/octet-stream") + ) + trial_path = submission_path / "trial" + trial_path.mkdir() + trial = self.submit_debug_trial(policy, submission_number, trial_path) + self.trials.append(trial) + return trial + + +def _execute_policy_worker(serialized_path: str, result_queue: Any) -> None: + try: + import cloudpickle + + with Path(serialized_path).open("rb") as handle: + policy = cloudpickle.load(handle) + except Exception as exc: + result_queue.put( + ("infrastructure_error", f"policy load failed: {type(exc).__name__}: {exc}") + ) + return + try: + from dimos.porcelain.dimos import Dimos + + app = Dimos.connect() + except Exception as exc: + result_queue.put( + ("infrastructure_error", f"DimOS connection failed: {type(exc).__name__}: {exc}") + ) + return + try: + result = policy(app) + if result is not None: + raise TypeError("policy(app) must return None") + except Exception as exc: + result_queue.put(("policy_error", f"{type(exc).__name__}: {exc}")) + else: + result_queue.put(("completed", None)) + finally: + app.stop() + + +def _assemble_user_message(evaluation_protocol: str, task_input: str) -> str: + return ( + "# Evaluation protocol\n\n" + f"{evaluation_protocol.strip()}\n\n" + "# Task input\n\n" + f"{task_input.strip()}\n" + ) + + +def _artifact(path: Path, label: str, media_type: str) -> ArtifactReference: + return ArtifactReference(path=path.as_posix(), label=label, media_type=media_type) + + +def _pi_paths() -> tuple[Path, Path]: + package = Path(__file__).resolve().parents[3] / "packages" / "pi-code-policy-extension" + cli = package / "node_modules" / "@earendil-works" / "pi-coding-agent" / "dist" / "cli.js" + extension = package / "dist" / "python-exec.js" + return cli, extension diff --git a/dimos/benchmark/evaluation/test_framework.py b/dimos/benchmark/evaluation/test_framework.py new file mode 100644 index 0000000000..9dabe62f1b --- /dev/null +++ b/dimos/benchmark/evaluation/test_framework.py @@ -0,0 +1,112 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from pathlib import Path + +from pydantic import BaseModel, ConfigDict +import pytest + +from dimos.benchmark.evaluation.models import ( + EvaluationReport, + InlineNativeResult, + RuntimeIdentity, + SummaryItem, +) +from dimos.benchmark.evaluation.protocol import EvaluationContext +from dimos.benchmark.evaluation.registry import ResolvedEvaluation +import dimos.benchmark.evaluation.runner as runner + + +class HarnessConfig(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + expected: int + + +class NativeHarnessEvaluation: + name = "native-harness" + config_model: type[BaseModel] = HarnessConfig + + def run(self, config: BaseModel, context: EvaluationContext) -> EvaluationReport: + assert isinstance(config, HarnessConfig) + assert context.runtime is not None + return EvaluationReport( + summary=(SummaryItem(key="score", label="Score", value=config.expected),), + native_result=InlineNativeResult(value={"native_score": config.expected}), + ) + + +class FakeRuntime: + prompt_evidence = () + runtime_artifacts = () + + def __init__(self, **_kwargs) -> None: + pass + + @property + def identity(self) -> RuntimeIdentity: + return RuntimeIdentity( + driver_version="test", + model="gpt-5.6-luna", + thinking_level="medium", + ) + + +def _write_spec(tmp_path: Path) -> Path: + path = tmp_path / "spec.json" + path.write_text( + json.dumps( + { + "schema_version": "1.0", + "evaluation": { + "name": "native-harness", + "config": {"expected": 7}, + }, + } + ) + ) + return path + + +def test_native_evaluation_owns_result_and_uses_fixed_runtime(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "secret") + monkeypatch.setattr(runner, "CodePolicyRuntimeFactory", FakeRuntime) + monkeypatch.setattr( + runner, + "resolve_evaluation", + lambda _name: ResolvedEvaluation( + name="native-harness", + provider="fixture", + version="1", + evaluation=NativeHarnessEvaluation(), + ), + ) + output = tmp_path / "output" + + result = runner.execute_evaluation(_write_spec(tmp_path), output=output) + + assert result.status == "completed" + assert result.report is not None + assert result.report.native_result.value == {"native_score": 7} + assert json.loads((output / "run.json").read_text())["runtime"]["driver"] == "pi" + + +def test_run_specification_rejects_agent_customization(tmp_path: Path) -> None: + specification = json.loads(_write_spec(tmp_path).read_text()) + specification["agent"] = {"model": "another-model"} + path = tmp_path / "custom.json" + path.write_text(json.dumps(specification)) + + with pytest.raises(Exception, match="Extra inputs are not permitted"): + runner.execute_evaluation(path, output=tmp_path / "output") diff --git a/dimos/benchmark/evaluation/test_models.py b/dimos/benchmark/evaluation/test_models.py new file mode 100644 index 0000000000..447d4ab675 --- /dev/null +++ b/dimos/benchmark/evaluation/test_models.py @@ -0,0 +1,64 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime, timezone + +from pydantic import ValidationError +import pytest + +from dimos.benchmark.evaluation.models import ( + ArtifactReference, + EvaluationIdentity, + EvaluationReference, + EvaluationReport, + EvaluationRun, + EvaluationRunSpecification, + InlineNativeResult, + RuntimeIdentity, +) + + +def test_artifact_reference_rejects_paths_outside_run() -> None: + with pytest.raises(ValidationError, match="safe relative"): + ArtifactReference(path="../private.json", label="Private") + + +def test_completed_run_requires_evaluation_report() -> None: + now = datetime.now(timezone.utc) + + with pytest.raises(ValidationError, match="require a report"): + EvaluationRun( + run_id="run", + specification=EvaluationRunSpecification( + evaluation=EvaluationReference(name="fixture") + ), + evaluation=EvaluationIdentity(name="fixture", provider="tests", version="1"), + runtime=RuntimeIdentity( + driver_version="test", + model="gpt-5.6-luna", + thinking_level="medium", + ), + status="completed", + started_at=now, + finished_at=now, + duration_seconds=0, + ) + + +def test_native_result_preserves_nested_benchmark_payload() -> None: + payload = {"metrics": {"success_rate": 0.75}, "episodes": [True, False]} + + report = EvaluationReport(native_result=InlineNativeResult(value=payload)) + + assert report.model_dump(mode="json")["native_result"]["value"] == payload diff --git a/dimos/benchmark/evaluation/test_pi_process.py b/dimos/benchmark/evaluation/test_pi_process.py new file mode 100644 index 0000000000..2eaa9bedbf --- /dev/null +++ b/dimos/benchmark/evaluation/test_pi_process.py @@ -0,0 +1,213 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from io import StringIO +import json +from pathlib import Path +import subprocess +import threading + +import pytest + +from dimos.benchmark.evaluation.pi_process import PiCliRunner, PiRunError, parse_pi_events + + +def test_parse_stock_pi_events_uses_final_message_and_counts_tools() -> None: + events = [ + {"type": "tool_execution_start", "toolName": "python_exec"}, + { + "type": "message_end", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": "Reasoning\nANSWER: 8"}], + "stopReason": "stop", + }, + }, + ] + text, count, error = parse_pi_events("\n".join(json.dumps(item) for item in events)) + assert text == "Reasoning\nANSWER: 8" + assert count == 1 + assert error is None + + +def test_stock_cli_streams_assistant_tools_and_stderr_while_running(mocker, tmp_path: Path) -> None: + cli = tmp_path / "cli.js" + extension = tmp_path / "extension.js" + cli.touch() + extension.touch() + process = mocker.Mock(returncode=0) + process.stdout = StringIO( + "\n".join( + json.dumps(event) + for event in ( + { + "type": "message_update", + "assistantMessageEvent": {"type": "text_delta", "delta": "Checking"}, + }, + { + "type": "tool_execution_start", + "toolCallId": "call-1", + "toolName": "python_exec", + "args": {"code": "memory.list_streams()"}, + }, + { + "type": "tool_execution_end", + "toolCallId": "call-1", + "toolName": "python_exec", + "result": {"content": [{"type": "text", "text": "['lidar']"}]}, + "isError": False, + }, + { + "type": "message_end", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": "ANSWER: 2"}], + "stopReason": "stop", + }, + }, + ) + ) + ) + process.stderr = StringIO("provider secret connected\n") + mocker.patch("dimos.benchmark.evaluation.pi_process.subprocess.Popen", return_value=process) + progress = [] + assistant_seen = threading.Event() + + def observe(event) -> None: + progress.append(event) + if event.kind == "assistant_text": + assistant_seen.set() + + def wait_for_process(*, timeout) -> int: + assert timeout == 10 + assert assistant_seen.wait(timeout=1) + return 0 + + process.wait.side_effect = wait_for_process + runner = PiCliRunner( + cli=cli, + extension=extension, + model="gpt-5.6-luna", + thinking_level="medium", + timeout_s=10, + progress=observe, + ) + + result = runner.run( + prompt="Count", + system_prompt="Use memory", + mcp_url="http://127.0.0.1:1234/mcp", + api_key="secret", + run_dir=tmp_path, + ) + + assert [(event.kind, getattr(event, "delta", None)) for event in progress[:1]] == [ + ("assistant_text", "Checking") + ] + structured = [event for event in progress if event.kind != "status"] + assert [event.kind for event in structured] == [ + "assistant_text", + "tool_start", + "tool_end", + "final_response", + ] + assert structured[1].code == "memory.list_streams()" + assert structured[2].result == "['lidar']" + assert [event.message for event in progress if event.kind == "status"] == [ + "provider [REDACTED] connected" + ] + assert result.stderr == "provider [REDACTED] connected\n" + assert result.final_text == "ANSWER: 2" + + +def test_stock_cli_receives_only_api_key_and_evaluator_binding(mocker, tmp_path: Path) -> None: + cli = tmp_path / "cli.js" + extension = tmp_path / "extension.js" + cli.touch() + extension.touch() + process = mocker.Mock(returncode=0) + process.stdout = StringIO( + json.dumps( + { + "type": "message_end", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": "ANSWER: 2"}], + "stopReason": "stop", + }, + } + ) + ) + process.stderr = StringIO() + process.wait.return_value = 0 + popen = mocker.patch( + "dimos.benchmark.evaluation.pi_process.subprocess.Popen", return_value=process + ) + runner = PiCliRunner( + cli=cli, + extension=extension, + model="gpt-5.6-luna", + thinking_level="medium", + timeout_s=10, + ) + result = runner.run( + prompt="Count", + system_prompt="Use memory", + mcp_url="http://127.0.0.1:1234/mcp", + api_key="secret", + run_dir=tmp_path, + ) + command = popen.call_args.args[0] + env = popen.call_args.kwargs["env"] + assert command[command.index("--mode") + 1] == "json" + assert "--no-builtin-tools" in command + assert command[command.index("--tools") + 1] == "python_exec" + assert env["OPENAI_API_KEY"] == "secret" + assert env["DIMOS_CODE_POLICY_MCP_URL"].endswith("/mcp") + assert "secret" not in command + assert result.final_text == "ANSWER: 2" + + +def test_stock_cli_timeout_terminates_the_child(mocker, tmp_path: Path) -> None: + cli = tmp_path / "cli.js" + extension = tmp_path / "extension.js" + cli.touch() + extension.touch() + process = mocker.Mock() + process.stdout = StringIO() + process.stderr = StringIO("stopped") + process.wait.side_effect = [ + subprocess.TimeoutExpired("pi", 0.01), + 0, + ] + mocker.patch("dimos.benchmark.evaluation.pi_process.subprocess.Popen", return_value=process) + runner = PiCliRunner( + cli=cli, + extension=extension, + model="gpt-5.6-luna", + thinking_level="medium", + timeout_s=0.01, + ) + + with pytest.raises(PiRunError, match="timed out"): + runner.run( + prompt="Count", + system_prompt="Use memory", + mcp_url="http://127.0.0.1:1234/mcp", + api_key="secret", + run_dir=tmp_path, + ) + + process.terminate.assert_called_once_with() + process.kill.assert_not_called() diff --git a/dimos/benchmark/evaluation/test_policy_runtime.py b/dimos/benchmark/evaluation/test_policy_runtime.py new file mode 100644 index 0000000000..75300430ae --- /dev/null +++ b/dimos/benchmark/evaluation/test_policy_runtime.py @@ -0,0 +1,168 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from pathlib import Path +import queue + +import cloudpickle +import pytest + +from dimos.benchmark.evaluation.pi_process import PiRunResult +from dimos.benchmark.evaluation.protocol import PolicyArtifact, TrialOutcome, TrialRun +import dimos.benchmark.evaluation.runtime as runtime_module +from dimos.benchmark.evaluation.runtime import ( + CodePolicyRuntimeFactory, + _execute_policy_worker, + _SubmissionManager, +) +from dimos.memory2.store.sqlite import SqliteStore +from dimos.porcelain.dimos import Dimos + + +def policy(app: Dimos) -> None: + del app + + +def _trial(path: Path, number: int) -> TrialRun: + path.mkdir(exist_ok=True) + log_path = path / "main.jsonl" + log_path.write_text(json.dumps({"module": "Planner", "number": number}) + "\n") + memory_path = path / "recording.db" + with SqliteStore(path=str(memory_path)) as memory: + memory.stream("events", int).append(number) + return TrialRun( + run_id=f"debug-{number}", + outcome=TrialOutcome( + success=number == 5, + reward=float(number), + status="completed", + error=None, + duration_seconds=1.0, + ), + artifacts=path, + log_path=log_path, + memory_path=memory_path, + ) + + +def test_submission_manager_caps_trials_and_selects_last_policy(tmp_path: Path) -> None: + artifacts = [] + manager = _SubmissionManager( + workspace=tmp_path, + path=tmp_path / "exploration", + relative_path=Path("exploration"), + submit_debug_trial=lambda _policy, number, path: _trial(path, number), + max_submissions=5, + record_artifact=artifacts.append, + ) + manager.path.mkdir() + serialized = cloudpickle.dumps(policy) + + trials = [manager.submit("def policy(app: Dimos) -> None: ...", serialized) for _ in range(5)] + + assert [trial.run_id for trial in trials] == [f"debug-{number}" for number in range(1, 6)] + assert manager.last_policy is not None + assert manager.last_policy.serialized_path.parent.name == "submission-0005" + with pytest.raises(RuntimeError, match="budget exhausted"): + manager.submit("def policy(app: Dimos) -> None: ...", serialized) + + +def test_trial_run_exposes_logs_and_read_only_memory(tmp_path: Path) -> None: + trial = _trial(tmp_path / "trial", 2) + + assert trial.read_logs(module="Planner") == ({"module": "Planner", "number": 2},) + with trial.open_memory() as memory: + assert memory.streams.events.last().data == 2 + with pytest.raises(PermissionError): + memory.streams.events.append(3) + + +def test_policy_artifact_records_serialized_callable(tmp_path: Path) -> None: + serialized_path = tmp_path / "policy.pkl" + serialized_path.write_bytes(cloudpickle.dumps(policy)) + artifact = PolicyArtifact( + source_path=tmp_path / "policy.py", + serialized_path=serialized_path, + sha256="digest", + ) + + with artifact.serialized_path.open("rb") as handle: + loaded = cloudpickle.load(handle) + + assert loaded.__name__ == "policy" + + +def test_policy_worker_connects_and_invokes_callable(mocker, tmp_path: Path) -> None: + serialized_path = tmp_path / "policy.pkl" + serialized_path.write_bytes(cloudpickle.dumps(policy)) + app = mocker.Mock(spec=Dimos) + connect = mocker.patch.object(Dimos, "connect", return_value=app) + results: queue.Queue[tuple[str, str | None]] = queue.Queue() + + _execute_policy_worker(str(serialized_path), results) + + assert results.get_nowait() == ("completed", None) + connect.assert_called_once_with() + app.stop.assert_called_once_with() + + +def test_explore_freezes_last_of_five_debug_submissions(mocker, tmp_path: Path) -> None: + class FakeServer: + current = None + mcp_url = "http://127.0.0.1:1/mcp" + + def __init__(self, submission_handler) -> None: + self.submission_handler = submission_handler + FakeServer.current = self + + def start(self) -> None: + pass + + def stop(self) -> None: + pass + + class FakePiRunner: + def __init__(self, **_kwargs) -> None: + pass + + def run(self, **_kwargs) -> PiRunResult: + assert FakeServer.current is not None + serialized = cloudpickle.dumps(policy) + for _ in range(5): + FakeServer.current.submission_handler( + "def policy(app: Dimos) -> None: ...", + serialized, + ) + return PiRunResult("done", 5, 2.0, None, "") + + marker = tmp_path / "pi" + marker.touch() + mocker.patch.object(runtime_module, "CodePolicyMcpServer", FakeServer) + mocker.patch.object(runtime_module, "PiCliRunner", FakePiRunner) + mocker.patch.object(runtime_module, "_pi_paths", return_value=(marker, marker)) + runtime = CodePolicyRuntimeFactory(api_key="secret", workspace=tmp_path) + + outcome = runtime.explore( + evaluation_protocol="Use normal DimOS information.", + task_input="Complete the task.", + submit_debug_trial=lambda _policy, number, path: _trial(path, number), + ) + + assert outcome.status == "valid" + assert len(outcome.trials) == 5 + assert outcome.policy is not None + assert outcome.policy.serialized_path.parent.name == "submission-0005" diff --git a/dimos/benchmark/evaluation/test_registry.py b/dimos/benchmark/evaluation/test_registry.py new file mode 100644 index 0000000000..5c89296d0b --- /dev/null +++ b/dimos/benchmark/evaluation/test_registry.py @@ -0,0 +1,98 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pydantic import BaseModel +import pytest + +from dimos.benchmark.evaluation.models import EvaluationReport, InlineNativeResult +from dimos.benchmark.evaluation.protocol import EvaluationContext +import dimos.benchmark.evaluation.registry as registry + + +class Config(BaseModel): + value: int + + +class PluginEvaluation: + name = "sample" + config_model: type[BaseModel] = Config + + def run(self, config: BaseModel, context: EvaluationContext) -> EvaluationReport: + return EvaluationReport(native_result=InlineNativeResult(value=None)) + + +class Distribution: + metadata = {"Name": "Acme_Evals"} + version = "2.0" + + +class EntryPoint: + name = "sample" + value = "acme.evals:sample" + dist = Distribution() + + def __init__(self, target) -> None: + self.target = target + + def load(self): + return self.target + + +def test_external_evaluation_uses_distribution_namespace(monkeypatch) -> None: + entry = EntryPoint(PluginEvaluation()) + monkeypatch.setattr( + registry.importlib_metadata, + "entry_points", + lambda **_kwargs: [entry], + ) + + resolved = registry.resolve_evaluation("acme-evals.sample") + + assert resolved.provider == "Acme_Evals" + assert resolved.version == "2.0" + assert resolved.evaluation is entry.target + + +def test_unknown_evaluation_reports_name(monkeypatch) -> None: + monkeypatch.setattr( + registry.importlib_metadata, + "entry_points", + lambda **_kwargs: [], + ) + + with pytest.raises(registry.EvaluationRegistryError, match="Unknown evaluation 'missing'"): + registry.resolve_evaluation("missing") + + +def test_external_target_must_implement_whole_evaluation(monkeypatch) -> None: + entry = EntryPoint(object()) + monkeypatch.setattr( + registry.importlib_metadata, + "entry_points", + lambda **_kwargs: [entry], + ) + + with pytest.raises(registry.EvaluationRegistryError, match="name, config_model, and run"): + registry.resolve_evaluation("acme-evals.sample") + + +def test_duplicate_external_evaluation_names_are_rejected(monkeypatch) -> None: + monkeypatch.setattr( + registry.importlib_metadata, + "entry_points", + lambda **_kwargs: [EntryPoint(PluginEvaluation()), EntryPoint(PluginEvaluation())], + ) + + with pytest.raises(registry.EvaluationRegistryError, match="Multiple installed"): + registry.available_evaluations() diff --git a/dimos/cli/dimos.py b/dimos/cli/dimos.py index 153c9effc0..fc35557734 100644 --- a/dimos/cli/dimos.py +++ b/dimos/cli/dimos.py @@ -50,6 +50,7 @@ from dimos.agents.mcp.mcp_adapter import McpAdapter, McpError from dimos.cli.cache import app as cache_app +from dimos.cli.eval import app as eval_app from dimos.cli.hardware_cli import app as hardware_app from dimos.cli.shell import shell from dimos.constants import CONFIG_DIR, LOG_DIR @@ -178,6 +179,7 @@ def callback(**kwargs) -> None: # type: ignore[no-untyped-def] main.add_typer(piper_app, name="piper") main.command()(shell) main.add_typer(cache_app, name="cache") +main.add_typer(eval_app, name="eval") def _with_relay_bridge(blueprint: Blueprint) -> Blueprint: diff --git a/dimos/cli/eval.py b/dimos/cli/eval.py new file mode 100644 index 0000000000..7a5ce1e20f --- /dev/null +++ b/dimos/cli/eval.py @@ -0,0 +1,160 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dependency-light shell for the unified Evaluation CLI.""" + +from __future__ import annotations + +from pathlib import Path +import threading +from typing import Any + +import typer + +app = typer.Typer(help="Run executable evaluations", no_args_is_help=True) + +MAX_RENDERED_TOOL_RESULT_CHARS = 2_000 + + +def execute_evaluation(*args: Any, **kwargs: Any) -> Any: + """Import and dispatch the evaluation runtime only when ``eval run`` executes.""" + try: + from dimos.benchmark.evaluation.runner import execute_evaluation as execute + except ModuleNotFoundError as exc: + raise RuntimeError( + "Evaluation dependencies are missing; run `uv sync --extra agents`" + ) from exc + + return execute(*args, **kwargs) + + +@app.command("run") +def run( + specification: Path = typer.Argument(..., exists=True, dir_okay=False, readable=True), + api_key_env: str = typer.Option("OPENAI_API_KEY", "--api-key-env"), + output: Path = typer.Option(..., "--output"), + json_output: bool = typer.Option(False, "--json", help="Print compact JSON"), + quiet: bool = typer.Option(False, "--quiet", help="Suppress live evaluation progress"), +) -> None: + """Run one Evaluation Run Specification synchronously.""" + renderer = None if quiet else ProgressRenderer() + try: + result = execute_evaluation( + specification, + output=output, + api_key_env=api_key_env, + progress=renderer, + ) + except Exception as exc: + if renderer is not None: + renderer.finish() + typer.echo(f"Evaluation preflight failed: {type(exc).__name__}: {exc}", err=True) + raise typer.Exit(2) from exc + if renderer is not None: + renderer.finish() + typer.echo(result.model_dump_json() if json_output else format_result(result, output)) + if result.status == "cancelled": + raise typer.Exit(130) + if result.status == "failed": + raise typer.Exit(1) + + +def format_result(result: Any, output: Path | None = None) -> str: + """Render infrastructure status plus Evaluation-supplied summary rows.""" + heading = { + "completed": "✓ Evaluation completed", + "failed": "! Evaluation failed", + "cancelled": "! Evaluation cancelled", + }[result.status] + rows = [ + ("Evaluation", result.evaluation.name), + ( + "Runtime", + f"Pi · {result.runtime.model} · {result.runtime.thinking_level}", + ), + ] + if result.report is not None: + rows.extend( + (item.label, "—" if item.value is None else str(item.value)) + for item in result.report.summary + ) + if result.error is not None: + rows.append(("Error", f"{result.error.error_type}: {result.error.message}")) + rows.append(("Output", str((output / "run.json") if output is not None else "run.json"))) + width = max(len(label) for label, _ in rows) + body = "\n".join(f" {label:<{width}} {value}" for label, value in rows) + return f"{heading}\n\n{body}" + + +class ProgressRenderer: + """Thread-safe concise terminal renderer for best-effort evaluation progress.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._assistant_open = False + self._saw_assistant_text = False + + def __call__(self, event: Any) -> None: + with self._lock: + if event.kind == "assistant_text": + if not self._assistant_open and not event.delta.strip(): + return + if not self._assistant_open: + typer.echo("[pi] ", err=True, nl=False) + self._assistant_open = True + typer.echo(event.delta, err=True, nl=False) + self._saw_assistant_text = True + return + self._end_assistant_line() + if event.kind == "case_header": + source = event.source + if event.progress is not None: + source += f" @ {event.progress * 100:g}%" + typer.echo("[eval] Session", err=True) + typer.echo(f" {'Case':<10} {event.case_id}", err=True) + typer.echo(f" {'Source':<10} {source}", err=True) + typer.echo(f" {'Question':<10} {event.question}", err=True) + typer.echo(f" {'Answer':<10} pending", err=True) + elif event.kind == "status": + typer.echo(f"[{event.channel}] {event.message}", err=True) + elif event.kind == "tool_start": + typer.echo("[python_exec] call", err=True) + typer.echo(_indent(event.code), err=True) + elif event.kind == "tool_end": + status = "ok" if event.ok else "error" + typer.echo(f"[python_exec] {status} ({event.duration_seconds:.1f}s)", err=True) + if event.result: + typer.echo(_indent(_truncate_tool_result(event.result)), err=True) + elif event.kind == "final_response" and not self._saw_assistant_text: + typer.echo(f"[pi] {event.text}", err=True) + + def finish(self) -> None: + with self._lock: + self._end_assistant_line() + + def _end_assistant_line(self) -> None: + if self._assistant_open: + typer.echo("", err=True) + self._assistant_open = False + + +def _indent(value: str) -> str: + return "\n".join(f" {line}" for line in value.splitlines()) + + +def _truncate_tool_result(value: str) -> str: + if len(value) <= MAX_RENDERED_TOOL_RESULT_CHARS: + return value + visible = value[:MAX_RENDERED_TOOL_RESULT_CHARS].rstrip() + return f"{visible}\n... [terminal output truncated; full result retained]" diff --git a/dimos/memory2/blobstore/sqlite.py b/dimos/memory2/blobstore/sqlite.py index 06021dcd9a..ddfa70923f 100644 --- a/dimos/memory2/blobstore/sqlite.py +++ b/dimos/memory2/blobstore/sqlite.py @@ -27,6 +27,7 @@ class SqliteBlobStoreConfig(BlobStoreConfig): conn: sqlite3.Connection | None = Field(default=None, exclude=True) path: str | None = None + read_only: bool = False @model_validator(mode="after") def _conn_xor_path(self) -> SqliteBlobStoreConfig: @@ -76,7 +77,9 @@ def _ensure_table(self, stream_name: str) -> None: def start(self) -> None: if self._conn is None: assert self._path is not None - disposable, self._conn = open_disposable_sqlite_connection(self._path) + disposable, self._conn = open_disposable_sqlite_connection( + self._path, read_only=self.config.read_only + ) self.register_disposable(disposable) def put(self, stream_name: str, key: int, data: bytes) -> None: diff --git a/dimos/memory2/observationstore/sqlite.py b/dimos/memory2/observationstore/sqlite.py index 31c6a25ea0..0dedf0ad9f 100644 --- a/dimos/memory2/observationstore/sqlite.py +++ b/dimos/memory2/observationstore/sqlite.py @@ -209,6 +209,7 @@ class SqliteObservationStoreConfig(ObservationStoreConfig): blob_store_conn_match: bool = Field(default=False, exclude=True) page_size: int = 256 path: str | None = None + read_only: bool = False @model_validator(mode="after") def _conn_xor_path(self) -> SqliteObservationStoreConfig: @@ -249,9 +250,18 @@ def __init__(self, **kwargs: Any) -> None: def start(self) -> None: if self._conn is None: assert self._path is not None - disposable, self._conn = open_disposable_sqlite_connection(self._path) + disposable, self._conn = open_disposable_sqlite_connection( + self._path, read_only=self.config.read_only + ) self.register_disposable(disposable) - self._ensure_tables() + if self.config.read_only: + found = self._conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (self._name,) + ).fetchone() + if found is None: + raise KeyError(f"Stream table {self._name!r} does not exist") + else: + self._ensure_tables() def _ensure_tables(self) -> None: """Create the metadata table and R*Tree index if they don't exist.""" @@ -331,6 +341,8 @@ def _ensure_tag_indexes(self, tags: dict[str, Any]) -> None: self._tag_indexes.add(key) def insert(self, obs: Observation[T]) -> int: + if self.config.read_only: + raise PermissionError("Cannot append to a read-only SQLite store") pose = obs.pose_tuple tags_json = json.dumps(obs.tags) if obs.tags else "{}" value = obs._data if isinstance(obs._data, (int, float)) else None diff --git a/dimos/memory2/registry.py b/dimos/memory2/registry.py index a5707a8cea..ba464ebb7f 100644 --- a/dimos/memory2/registry.py +++ b/dimos/memory2/registry.py @@ -41,6 +41,7 @@ def deserialize_component(data: dict[str, Any]) -> Any: class RegistryStoreConfig(BaseConfig): conn: sqlite3.Connection = Field(exclude=True) + read_only: bool = False class RegistryStore(Configurable): @@ -51,13 +52,20 @@ class RegistryStore(Configurable): def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) self._conn: sqlite3.Connection = self.config.conn - self._conn.execute( - "CREATE TABLE IF NOT EXISTS _streams (" - " name TEXT PRIMARY KEY," - " config TEXT NOT NULL" - ")" - ) - self._conn.commit() + if self.config.read_only: + found = self._conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='_streams'" + ).fetchone() + if found is None: + raise ValueError("SQLite recording has no Memory2 stream registry") + else: + self._conn.execute( + "CREATE TABLE IF NOT EXISTS _streams (" + " name TEXT PRIMARY KEY," + " config TEXT NOT NULL" + ")" + ) + self._conn.commit() def get(self, name: str) -> dict[str, Any] | None: row = self._conn.execute("SELECT config FROM _streams WHERE name = ?", (name,)).fetchone() diff --git a/dimos/memory2/store/sqlite.py b/dimos/memory2/store/sqlite.py index 8afab3a714..edaa0ef0d9 100644 --- a/dimos/memory2/store/sqlite.py +++ b/dimos/memory2/store/sqlite.py @@ -41,6 +41,7 @@ class SqliteStoreConfig(StoreConfig): ] = "memory.db" page_size: int = 256 must_exist: bool = False + read_only: bool = False class SqliteStore(Store): @@ -54,16 +55,22 @@ def __init__(self, **kwargs: Any) -> None: raise FileNotFoundError( f"SQLite database not found: {os.path.abspath(self.config.path)}" ) - if not self.config.must_exist: + if self.config.read_only and not os.path.exists(self.config.path): + raise FileNotFoundError( + f"SQLite database not found: {os.path.abspath(self.config.path)}" + ) + if not self.config.must_exist and not self.config.read_only: parent = os.path.dirname(self.config.path) if parent: os.makedirs(parent, exist_ok=True) self._registry_conn = self._open_connection() - self._registry = RegistryStore(conn=self._registry_conn) + self._registry = RegistryStore(conn=self._registry_conn, read_only=self.config.read_only) def _open_connection(self) -> sqlite3.Connection: """Open a new WAL-mode connection with sqlite-vec loaded.""" - disposable, connection = open_disposable_sqlite_connection(self.config.path) + disposable, connection = open_disposable_sqlite_connection( + self.config.path, read_only=self.config.read_only + ) self.register_disposable(disposable) return connection @@ -82,23 +89,31 @@ def _assemble_backend(self, name: str, stored: dict[str, Any]) -> Backend[Any]: # Reconstruct components from serialized config bs_data = stored.get("blob_store") if bs_data is not None: - bs_cfg = bs_data.get("config", {}) - if bs_cfg.get("path") is None and bs_data["class"] == qual(SqliteBlobStore): - bs: Any = SqliteBlobStore(conn=backend_conn) + bs_cfg = dict(bs_data.get("config", {})) + if bs_data["class"] == qual(SqliteBlobStore): + if self.config.read_only: + bs_cfg["read_only"] = True + if bs_cfg.get("path") is None: + bs_cfg["conn"] = backend_conn + bs: Any = SqliteBlobStore(**bs_cfg) else: bs = deserialize_component(bs_data) else: - bs = SqliteBlobStore(conn=backend_conn) + bs = SqliteBlobStore(conn=backend_conn, read_only=self.config.read_only) vs_data = stored.get("vector_store") if vs_data is not None: - vs_cfg = vs_data.get("config", {}) - if vs_cfg.get("path") is None and vs_data["class"] == qual(SqliteVectorStore): - vs: Any = SqliteVectorStore(conn=backend_conn) + vs_cfg = dict(vs_data.get("config", {})) + if vs_data["class"] == qual(SqliteVectorStore): + if self.config.read_only: + vs_cfg["read_only"] = True + if vs_cfg.get("path") is None: + vs_cfg["conn"] = backend_conn + vs: Any = SqliteVectorStore(**vs_cfg) else: vs = deserialize_component(vs_data) else: - vs = SqliteVectorStore(conn=backend_conn) + vs = SqliteVectorStore(conn=backend_conn, read_only=self.config.read_only) notifier_data = stored.get("notifier") if notifier_data is not None: @@ -116,6 +131,7 @@ def _assemble_backend(self, name: str, stored: dict[str, Any]) -> Backend[Any]: codec=codec, blob_store_conn_match=blob_store_conn_match and eager_blobs, page_size=page_size, + read_only=self.config.read_only, ) backend: Backend[Any] = Backend( metadata_store=metadata_store, @@ -164,6 +180,9 @@ def _create_backend( ) return self._assemble_backend(name, stored) + if self.config.read_only: + raise KeyError(f"Stream {name!r} does not exist in read-only store") + # Create path: inject conn-shared defaults, then delegate to base if payload_type is None: raise TypeError(f"Stream {name!r} does not exist yet — payload_type is required") @@ -211,6 +230,8 @@ def list_streams(self) -> list[str]: return sorted(db_names | set(self._streams.keys())) def delete_stream(self, name: str) -> None: + if self.config.read_only: + raise PermissionError("Cannot delete streams from a read-only store") super().delete_stream(name) self._registry_conn.execute(f'DROP TABLE IF EXISTS "{name}"') self._registry_conn.execute(f'DROP TABLE IF EXISTS "{name}_blob"') diff --git a/dimos/memory2/utils/sqlite.py b/dimos/memory2/utils/sqlite.py index 02a48f22b7..27475d4dfb 100644 --- a/dimos/memory2/utils/sqlite.py +++ b/dimos/memory2/utils/sqlite.py @@ -20,13 +20,18 @@ from reactivex.disposable import Disposable -def open_sqlite_connection(path: str | Path) -> sqlite3.Connection: +def open_sqlite_connection(path: str | Path, *, read_only: bool = False) -> sqlite3.Connection: """Open a WAL-mode SQLite connection with sqlite-vec loaded.""" import sqlite_vec - conn = sqlite3.connect(path, check_same_thread=False) - conn.execute("PRAGMA journal_mode=WAL") - conn.execute("PRAGMA synchronous=NORMAL") + if read_only: + uri = Path(path).resolve().as_uri() + "?mode=ro" + conn = sqlite3.connect(uri, uri=True, check_same_thread=False) + conn.execute("PRAGMA query_only=ON") + else: + conn = sqlite3.connect(path, check_same_thread=False) + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") conn.enable_load_extension(True) sqlite_vec.load(conn) conn.enable_load_extension(False) @@ -35,10 +40,12 @@ def open_sqlite_connection(path: str | Path) -> sqlite3.Connection: def open_disposable_sqlite_connection( path: str | Path, + *, + read_only: bool = False, ) -> tuple[Disposable, sqlite3.Connection]: """Open a WAL-mode SQLite connection and return (disposable, connection). The disposable closes the connection when disposed. """ - conn = open_sqlite_connection(path) + conn = open_sqlite_connection(path, read_only=read_only) return Disposable(lambda: conn.close()), conn diff --git a/dimos/memory2/vectorstore/sqlite.py b/dimos/memory2/vectorstore/sqlite.py index bb5e9d200e..9956548024 100644 --- a/dimos/memory2/vectorstore/sqlite.py +++ b/dimos/memory2/vectorstore/sqlite.py @@ -31,6 +31,7 @@ class SqliteVectorStoreConfig(VectorStoreConfig): conn: sqlite3.Connection | None = Field(default=None, exclude=True) path: str | None = None + read_only: bool = False @model_validator(mode="after") def _conn_xor_path(self) -> SqliteVectorStoreConfig: @@ -74,7 +75,9 @@ def _ensure_table(self, stream_name: str, dim: int) -> None: def start(self) -> None: if self._conn is None: assert self._path is not None - disposable, self._conn = open_disposable_sqlite_connection(self._path) + disposable, self._conn = open_disposable_sqlite_connection( + self._path, read_only=self.config.read_only + ) self.register_disposable(disposable) def put(self, stream_name: str, key: int, embedding: Embedding) -> None: diff --git a/docs/capabilities/agents/evaluation.md b/docs/capabilities/agents/evaluation.md new file mode 100644 index 0000000000..f11d332ea8 --- /dev/null +++ b/docs/capabilities/agents/evaluation.md @@ -0,0 +1,107 @@ +--- +title: "CodePolicy evaluation" +--- + +# CodePolicy evaluation + +CodePolicy evaluations separate exploration from measured execution. + +```text +Pi + Python REPL native Evaluation + │ │ + ├── submit_policy(policy) ── fresh debug trial + │ ◀── outcome, logs, Memory2, artifacts + │ + └── last submitted policy ── held-out trials without Pi + │ + privileged native scorer +``` + +The existing `dimos evals` command and its passive and interactive cases remain +unchanged. Complete third-party benchmarks use the additive `Evaluation` plugin +interface and the `dimos eval` command. + +## Evaluation ownership + +An `Evaluation` owns its native environment, cases, seeds, blueprint lifecycle, +step or real-time horizon, privileged scorer, aggregation, and native result. +External packages register an Evaluation through the `dimos.evaluations` entry +point group. The shared runner resolves the plugin, provides a fixed CodePolicy +runtime, and publishes one immutable result directory. + +The policy sees every capability exposed by its running blueprint through +`Dimos`: modules, RPCs, skills, and streams. The scorer may use simulator-only +state, such as true poses and task predicates. Scorer state never enters the +policy process, trial logs, or Memory2 recording. + +## Exploration + +The fixed `code-policy-v1` profile runs Pi with `gpt-5.6-luna`, medium thinking, +and one tool: `python_exec`. The persistent Python REPL contains `Dimos` and +`submit_policy`. + +```python +def policy(app: Dimos) -> None: + target = app.peek_stream("detections") + app.skills.move_to(target.position) + + +trial = submit_policy(policy) +trial.outcome +trial.read_logs(module="MotionPlanner", tail=100) +memory = trial.open_memory() +list(memory.streams) +list(trial.artifacts.iterdir()) +``` + +The callable must have the exact synchronous signature shown above. Each +accepted submission starts a fresh debug environment and a fresh policy-only +blueprint. The Evaluation chooses the debug-case sequence. Pi may submit five +trials; its last accepted callable becomes the task-level policy artifact. + +`TrialRun` describes a stopped run. It exposes debug success and reward, policy +errors, DimOS logs, a read-only Memory2 store, primitive traces, and recorded +artifacts. It does not expose simulator state or scorer internals. + +## Held-out execution + +The Evaluation reuses one task-level policy across all held-out cases. For each +case it starts a fresh environment and policy-only blueprint, waits for DimOS to +be ready, and invokes the serialized callable in a clean process: + +```python +execution = context.runtime.execute(policy, timeout_s=case.timeout_s) +``` + +Pi and the production `McpClient` are absent from measured execution. The task +horizon starts immediately before `policy(app)` runs, so blueprint startup and +model generation do not consume real-time or simulator-step budgets. + +## CLI + +Install the agent dependencies and build the Pi extension: + +```bash +uv sync --extra agents +npm --prefix packages/pi-code-policy-extension install +npm --prefix packages/pi-code-policy-extension run build +``` + +Run an installed Evaluation from a strict JSON specification: + +```json +{ + "schema_version": "1.0", + "evaluation": { + "name": "vendor-evals.benchmark-name", + "config": {} + } +} +``` + +```bash +dimos eval run specification.json --output evaluation-run +``` + +The run specification has no agent or policy-mode fields. The runtime profile +is fixed and recorded in `evaluation-run/run.json`. diff --git a/docs/capabilities/agents/index.md b/docs/capabilities/agents/index.md index 45bbad2fc1..25871ccc15 100644 --- a/docs/capabilities/agents/index.md +++ b/docs/capabilities/agents/index.md @@ -4,6 +4,9 @@ sidebarTitle: "Overview" --- LLM agents run as native dimOS modules. They subscribe to camera, LiDAR, odometry, and spatial memory streams and they control the robot through skills. +See [CodePolicy evaluation](/docs/capabilities/agents/evaluation.md) for the separate exploration and +agent-free policy execution workflow. + ## Architecture ``` diff --git a/packages/pi-code-policy-extension/.gitignore b/packages/pi-code-policy-extension/.gitignore new file mode 100644 index 0000000000..e9488b3e35 --- /dev/null +++ b/packages/pi-code-policy-extension/.gitignore @@ -0,0 +1,6 @@ +# The repository-wide ignore rules treat package manifests as local tooling files. +!package.json +!package-lock.json +dist/ +dist-test/ +node_modules/ diff --git a/packages/pi-code-policy-extension/README.md b/packages/pi-code-policy-extension/README.md new file mode 100644 index 0000000000..975615d79b --- /dev/null +++ b/packages/pi-code-policy-extension/README.md @@ -0,0 +1,8 @@ +# Pi CodePolicy extension + +This package adds one `python_exec` tool to the stock Pi CLI. The tool connects +to the evaluator-owned MCP server named by `DIMOS_CODE_POLICY_MCP_URL`. + +The Python evaluator launches Pi with all built-in tools and extension discovery +disabled, then loads only `dist/python-exec.js`. The Python REPL provides +`submit_policy(policy)`; it is not a second Pi tool. diff --git a/packages/pi-code-policy-extension/package-lock.json b/packages/pi-code-policy-extension/package-lock.json new file mode 100644 index 0000000000..e8e8841d00 --- /dev/null +++ b/packages/pi-code-policy-extension/package-lock.json @@ -0,0 +1,1894 @@ +{ + "name": "@dimos/pi-code-policy-extension", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@dimos/pi-code-policy-extension", + "version": "0.1.0", + "dependencies": { + "@earendil-works/pi-coding-agent": "0.80.10", + "@modelcontextprotocol/client": "2.0.0", + "typebox": "1.3.6" + }, + "devDependencies": { + "@types/node": "22.15.0", + "typescript": "5.8.3" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.91.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.91.1.tgz", + "integrity": "sha512-LAmu761tSN9r66ixvmciswUj/ZC+1Q4iAfpedTfSVLeswRwnY3n2Nb6Tsk+cLPP28aLOPWeMgIuTuCcMC6W/iw==", + "license": "MIT", + "dependencies": { + "json-schema-to-ts": "^3.1.1" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-bedrock-runtime": { + "version": "3.1048.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-runtime/-/client-bedrock-runtime-3.1048.0.tgz", + "integrity": "sha512-u+NT61JZEkRFtpL0CAw1N1dwxnaLgwVXQl/zjJxTGgLyS/jTIdg2SdoEoCTHxgDyCnqa1HEi9QOoE9/pYRNpOQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/credential-provider-node": "^3.972.42", + "@aws-sdk/eventstream-handler-node": "^3.972.16", + "@aws-sdk/middleware-eventstream": "^3.972.12", + "@aws-sdk/middleware-websocket": "^3.972.19", + "@aws-sdk/token-providers": "3.1048.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/fetch-http-handler": "^5.4.2", + "@smithy/node-http-handler": "^4.7.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.977.6", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.977.6.tgz", + "integrity": "sha512-QiaJV4/zDrB4ZY2mfeSXSzSTc36W16sZXcGz+SPFk0CJ26gziO0cS+4LjJUMAbdeeBOvS0k0Aq1cZpfGdUXxSw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@aws-sdk/xml-builder": "^3.972.37", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.31.1", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.16.1", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.67", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.67.tgz", + "integrity": "sha512-rcIpk5kxUqDaaNa6Xk23pQ6ViY7jlqzmfFWCahQcBT97ddXaXYYwzCen9Tz1Jvo6aJft6wDl5bN44/Jw5B4oLA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.6", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.69", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.69.tgz", + "integrity": "sha512-nggwJtZ4eeNsUw5IeWBMXsi1ryct5idi0K+/SCRF3kybLubOMaNTb3XCihXpWMiVpyzyPeIrl0zTkzhBH9porA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.6", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/fetch-http-handler": "^5.6.13", + "@smithy/node-http-handler": "^4.9.13", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http/node_modules/@smithy/node-http-handler": { + "version": "4.9.13", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.13.tgz", + "integrity": "sha512-Nmd/Nl35zfYrd+a6OO2cDJb3GPh9bgTjIUhcM+JFfjpp8/osCgboDV5nCT1I01Pv6R13eSKDKLSoVa5ZB6Zsfw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.973.12", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.12.tgz", + "integrity": "sha512-pNEf/OeyN5X3VmLKlgSO6TqaWmW10CvI3TfwL1XhsuhYjSLT2VDaxFnCPHnOeQXSaFisMX4jNhpETriqN8DOmg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.6", + "@aws-sdk/credential-provider-env": "^3.972.67", + "@aws-sdk/credential-provider-http": "^3.972.69", + "@aws-sdk/credential-provider-login": "^3.972.74", + "@aws-sdk/credential-provider-process": "^3.972.67", + "@aws-sdk/credential-provider-sso": "^3.973.11", + "@aws-sdk/credential-provider-web-identity": "^3.972.73", + "@aws-sdk/nested-clients": "^3.997.41", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.74", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.74.tgz", + "integrity": "sha512-0AQfDcf99TNmqVKv0owHrw/TQs6i4ZE5t9qmz6NvO53bE/sA/tpXhXL9AAcEP1qHc6Zzjd1UMb69+/9zdhvY3g==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.6", + "@aws-sdk/nested-clients": "^3.997.41", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.78", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.78.tgz", + "integrity": "sha512-OgPAnfvbGAMWac6yvxJ1ihslrvDpPVwR68D2csospdNCCyPvHk9JLzYKwz48SNiS1T2znDwHauywRKRFfpyYng==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.67", + "@aws-sdk/credential-provider-http": "^3.972.69", + "@aws-sdk/credential-provider-ini": "^3.973.12", + "@aws-sdk/credential-provider-process": "^3.972.67", + "@aws-sdk/credential-provider-sso": "^3.973.11", + "@aws-sdk/credential-provider-web-identity": "^3.972.73", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.67", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.67.tgz", + "integrity": "sha512-IlUEejorGTWKb4/Dm7K5Yw4QxUmXLThLhrvBmzVBqZFTbW72cv9LTcITmo1dsnYriALE4h68mOq4LB99x6sQ7Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.6", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.973.11", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.11.tgz", + "integrity": "sha512-gAQBkBZxUB84d71+pPcI9L+jh2ujhuAVxc/4FgGiWFDjkPBlMKxzd5XDtkSXTFX8Ro7ansnT88+XadasxMeCRw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.6", + "@aws-sdk/nested-clients": "^3.997.41", + "@aws-sdk/token-providers": "3.1103.0", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso/node_modules/@aws-sdk/token-providers": { + "version": "3.1103.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1103.0.tgz", + "integrity": "sha512-N4wy26MNn31ItGVHYHPrEuCIFY4MBBjC+C5v1lJKqIUSA7OZBdhleCY53zCCrXn27hsk7YNOaTuhQu807S4AfQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.6", + "@aws-sdk/nested-clients": "^3.997.41", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.73", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.73.tgz", + "integrity": "sha512-SnlEmQa6SjOgs6iOPLUQl1Eyq4AKiAdPQlkOhFhqNfDtDCwibMGvL6QlkSmf3o6vAUSImzdPCxowT5dfQUZP1A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.6", + "@aws-sdk/nested-clients": "^3.997.41", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/eventstream-handler-node": { + "version": "3.972.31", + "resolved": "https://registry.npmjs.org/@aws-sdk/eventstream-handler-node/-/eventstream-handler-node-3.972.31.tgz", + "integrity": "sha512-/BRzvkp46mF6eXBL/l9WKPQQfifLlUPaWli6n9/T/WDLUg8he7TCyuNFnk6RvHP5j5W/kMj5Gxw7W778LJaXDA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-eventstream": { + "version": "3.972.26", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-eventstream/-/middleware-eventstream-3.972.26.tgz", + "integrity": "sha512-2eIvouTZoxPu5ClHY6ij13De1yhY8Rmllt0dlGeBNXX3wmR7fU1pvMCGb50fKm1GxcuntWK0t1T0cMjTyDoUQA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-websocket": { + "version": "3.972.49", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-websocket/-/middleware-websocket-3.972.49.tgz", + "integrity": "sha512-wGYJvu1/stdWuzGcNyh44u8aY7ArU8XFrMesBlzIYn70HWVCRBNEngQexDuPIMu0NEgsKGKmTc0uvnS/bF7KWw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.6", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/fetch-http-handler": "^5.6.13", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.41.tgz", + "integrity": "sha512-RDHqPGQWlF6tatA/Tp3rg6oIwtgN9IVderxE+9av2Y93Dfyu+mO1hZ5Bu2jpfZg2rwdNbsssnwM+sLafIczMlQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.6", + "@aws-sdk/signature-v4-multi-region": "^3.996.43", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/fetch-http-handler": "^5.6.13", + "@smithy/node-http-handler": "^4.9.13", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients/node_modules/@smithy/node-http-handler": { + "version": "4.9.13", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.13.tgz", + "integrity": "sha512-Nmd/Nl35zfYrd+a6OO2cDJb3GPh9bgTjIUhcM+JFfjpp8/osCgboDV5nCT1I01Pv6R13eSKDKLSoVa5ZB6Zsfw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.43", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.43.tgz", + "integrity": "sha512-lKekx8bLBXSv4O+cslk9Zfnw2XKSkWBs3uWL5QGhH2ZAQfNS7FE0vcSSN2vD/AhxX54ZTywWxR4STThoeOXlBA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1048.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1048.0.tgz", + "integrity": "sha512-k0y/GcuesuSfWyUM0WamrGyeZmltRYaPbHO82UDA6mZ/doB+FOHKutikPAtSXMn/hDz970cF+iRuuiYO9VEbAA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.2", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.2.tgz", + "integrity": "sha512-3W6IUtSxFbH6X7Wb7DzGCV5QiFQsd0g8bOfntpmDxQlzBoKWUMBu/JPQR0DwkE+Hpnxd6db1tXbOwdeHddG6cA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/util-locate-window": { + "version": "3.965.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.965.8.tgz", + "integrity": "sha512-uUbMs1cBZPafD0ohUj6EwNf0fPZ534NvBxHox4hjX+0Rxq5paSYUem7+hi833pYrzrcnBATKIYpR02MDXT5M9g==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.37", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.37.tgz", + "integrity": "sha512-zKq4HQum8JwDyEuyfuI4bbiAcU0KxP6qy+9PR/IsR92IyE/DaBAikzAS50tjxip4bqIIANpCcG+Yyj6CVhXupg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@earendil-works/pi-agent-core": { + "version": "0.80.10", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.80.10.tgz", + "integrity": "sha512-nwnOR3SuLYGRFfyQm8ri4Nj5VGVAvAM9GuqQd3u7BUQj0d6hmD2F8w7OHAAjThE3CuySIdM+v8E22QJG6/RfCg==", + "license": "MIT", + "dependencies": { + "@earendil-works/pi-ai": "^0.80.10", + "ignore": "7.0.5", + "typebox": "1.1.38", + "yaml": "2.9.0" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-agent-core/node_modules/typebox": { + "version": "1.1.38", + "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.1.38.tgz", + "integrity": "sha512-pZ0aQPmMmXoUvSbeuWf/Hzsc+avNw/Zd6VeE8CFgkVGWyuHPJvqeJJDeJqLve+K70LvjYIoleGcoJHPT17cWoA==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-ai": { + "version": "0.80.10", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.80.10.tgz", + "integrity": "sha512-Moe/H8c87yacDGK9dPbWphZNjVsrb3nTrIHycOQJAkFEnY9PYxOOd74+ny44kATfPU9Dm7aTHefar3pZF+UKUA==", + "license": "MIT", + "dependencies": { + "@anthropic-ai/sdk": "0.91.1", + "@aws-sdk/client-bedrock-runtime": "3.1048.0", + "@google/genai": "1.52.0", + "@mistralai/mistralai": "2.2.6", + "@opentelemetry/api": "1.9.0", + "@smithy/node-http-handler": "4.7.3", + "http-proxy-agent": "7.0.2", + "https-proxy-agent": "7.0.6", + "openai": "6.26.0", + "partial-json": "0.1.7", + "typebox": "1.1.38" + }, + "bin": { + "pi-ai": "dist/cli.js" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-ai/node_modules/typebox": { + "version": "1.1.38", + "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.1.38.tgz", + "integrity": "sha512-pZ0aQPmMmXoUvSbeuWf/Hzsc+avNw/Zd6VeE8CFgkVGWyuHPJvqeJJDeJqLve+K70LvjYIoleGcoJHPT17cWoA==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent": { + "version": "0.80.10", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-coding-agent/-/pi-coding-agent-0.80.10.tgz", + "integrity": "sha512-aL4apbupCHiVLSXASXvRzH4Q2vmtfrDa+0s909CJuVu/GgGylbDzr7oyF1mPmip5E+VxYYxKWmph4hV04wUcQg==", + "license": "MIT", + "dependencies": { + "@earendil-works/pi-agent-core": "^0.80.10", + "@earendil-works/pi-ai": "^0.80.10", + "@earendil-works/pi-tui": "^0.80.10", + "@silvia-odwyer/photon-node": "0.3.4", + "chalk": "5.6.2", + "cross-spawn": "7.0.6", + "diff": "8.0.4", + "glob": "13.0.6", + "highlight.js": "10.7.3", + "hosted-git-info": "9.0.3", + "ignore": "7.0.5", + "jiti": "2.7.0", + "minimatch": "10.2.5", + "proper-lockfile": "4.1.2", + "semver": "7.8.0", + "typebox": "1.1.38", + "undici": "8.5.0", + "yaml": "2.9.0" + }, + "bin": { + "pi": "dist/cli.js" + }, + "engines": { + "node": ">=22.19.0" + }, + "optionalDependencies": { + "@mariozechner/clipboard": "0.3.9" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/typebox": { + "version": "1.1.38", + "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.1.38.tgz", + "integrity": "sha512-pZ0aQPmMmXoUvSbeuWf/Hzsc+avNw/Zd6VeE8CFgkVGWyuHPJvqeJJDeJqLve+K70LvjYIoleGcoJHPT17cWoA==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-tui": { + "version": "0.80.10", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.80.10.tgz", + "integrity": "sha512-c2JO29PbhKPEQ6fgHQKAl0WhwuFqzWfzspMmP+8B5tpDuP+0mvarRbKKg8gq4b+pQx/QX+6aVS4ko7deoyjQjg==", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "1.6.0", + "marked": "18.0.5" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@google/genai": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.52.0.tgz", + "integrity": "sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "google-auth-library": "^10.3.0", + "p-retry": "^4.6.2", + "protobufjs": "^7.5.4", + "ws": "^8.18.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@modelcontextprotocol/sdk": "^1.25.2" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/sdk": { + "optional": true + } + } + }, + "node_modules/@mariozechner/clipboard": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard/-/clipboard-0.3.9.tgz", + "integrity": "sha512-ABnA53mdfkGZwOFUdZNv2S0CWGO/EIuPj8Vv9xmBFmSYg/qFc7ihO6q5FcQjvoE67kZpWkEc4AhD6B/os04yuA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@mariozechner/clipboard-darwin-arm64": "0.3.9", + "@mariozechner/clipboard-darwin-universal": "0.3.9", + "@mariozechner/clipboard-darwin-x64": "0.3.9", + "@mariozechner/clipboard-linux-arm64-gnu": "0.3.9", + "@mariozechner/clipboard-linux-arm64-musl": "0.3.9", + "@mariozechner/clipboard-linux-riscv64-gnu": "0.3.9", + "@mariozechner/clipboard-linux-x64-gnu": "0.3.9", + "@mariozechner/clipboard-linux-x64-musl": "0.3.9", + "@mariozechner/clipboard-win32-arm64-msvc": "0.3.9", + "@mariozechner/clipboard-win32-x64-msvc": "0.3.9" + } + }, + "node_modules/@mariozechner/clipboard-darwin-arm64": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-arm64/-/clipboard-darwin-arm64-0.3.9.tgz", + "integrity": "sha512-BfgV7vCEWZwJwZJw03r6bP5+tf0iI/ANuQYCxi9RNn7FrWB3yzGuMKCrNLRl6V761vXRdL8+OqZ0wd4TqlsNOQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-darwin-universal": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-universal/-/clipboard-darwin-universal-0.3.9.tgz", + "integrity": "sha512-BGGR4iA9Z2shAjI65eI5xtyb3LYNlDW9X3gxKxDbqtbnREohsrqznov6zpKoIrsRWpzlYVEdKphS7ksJ0/ndSQ==", + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-darwin-x64": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-x64/-/clipboard-darwin-x64-0.3.9.tgz", + "integrity": "sha512-4kURmCbS6nt8uYhtmWpUcJWyPHfmAr5dTpXD1nO3pIfa+TSQ9DbrGOYCKH+aEFW47XhQ4Vp8ZTszie+wfFvDKg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-linux-arm64-gnu": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-arm64-gnu/-/clipboard-linux-arm64-gnu-0.3.9.tgz", + "integrity": "sha512-g59OkUGP2DDfCOIKypHeYgv2M55u/cKvXa5dSxFbEJ34XvIQMdcVmpKCkGUro3ZgefXiGVdwguvTMQGpHWzIXw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-linux-arm64-musl": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-arm64-musl/-/clipboard-linux-arm64-musl-0.3.9.tgz", + "integrity": "sha512-AGuJdgKsmJdm4Pych7kv3sqe591ERRaAHW3xjLooiFzn8J+PxUyof++7YZrB5Y5tpnTO+K18Og3taj2NpluCRQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-linux-riscv64-gnu": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-riscv64-gnu/-/clipboard-linux-riscv64-gnu-0.3.9.tgz", + "integrity": "sha512-DXBEAiuMpk7dhS1a9NzNxVAFi1vaKoPu7rQNgY8LIDLGrK3lnIp3nT10DUum+PKVJoJppIP+NAA8IZe4DMNDPw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-linux-x64-gnu": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-x64-gnu/-/clipboard-linux-x64-gnu-0.3.9.tgz", + "integrity": "sha512-WORrMLd6EpElEME7JRKfSaY34nW1P5LbdgK5YNCS1ncG2LqmITsSMEJ8nh2mpvxb3TxqbOOKgY7k9eMJYlW9Mw==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-linux-x64-musl": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-x64-musl/-/clipboard-linux-x64-musl-0.3.9.tgz", + "integrity": "sha512-/DHn+1DrfL6oRaPPWXaOKvonFFrni666fxd+zFqiQEfvBH0tsHVWjq9iqBk0oDp0qaPA72lIMy5BptxISBEhZQ==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-win32-arm64-msvc": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-win32-arm64-msvc/-/clipboard-win32-arm64-msvc-0.3.9.tgz", + "integrity": "sha512-O5FHD3ErkMwMhNzAfu3ggy0ug4z7btZuoQgwwxlzPrwV2bxlD6WDpqBY4NCgICAgZdDKdp+loUEKVAVt8aYnhQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mariozechner/clipboard-win32-x64-msvc": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-win32-x64-msvc/-/clipboard-win32-x64-msvc-0.3.9.tgz", + "integrity": "sha512-ihQC3EufqEY81vhXBgVBtK4prL+wc62zJsSvxrgz7K1hsdt6OObz6v9p3Rn1OG3GJksTTKMJF0u/guMISHPhSA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mistralai/mistralai": { + "version": "2.2.6", + "resolved": "https://registry.npmjs.org/@mistralai/mistralai/-/mistralai-2.2.6.tgz", + "integrity": "sha512-W8pX7zHxjJvMIpw8JMxeJEleapXX0Q9NPszdNzqkM3MIEoIGPObdodujj+WHteXEvGfaP/AMwlNyRfEzSY6dQQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.40.0", + "ws": "^8.18.0", + "zod": "^3.25.0 || ^4.0.0", + "zod-to-json-schema": "^3.25.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + } + } + }, + "node_modules/@modelcontextprotocol/client": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/client/-/client-2.0.0.tgz", + "integrity": "sha512-8f1OghQ2rjzIOfqgUCP+8GiUWqRs89njoWLNqAe8kWmDePv3s1fZXseej+QXemssEuuOvLLmLO/kqM3IQHtISw==", + "license": "MIT", + "dependencies": { + "@modelcontextprotocol/core": "2.0.0", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "jose": "^6.1.3", + "pkce-challenge": "^5.0.0", + "zod": "^4.2.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@modelcontextprotocol/core": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/core/-/core-2.0.0.tgz", + "integrity": "sha512-pJCEwGG7Lfr/+PQp9ZTwKXNeO5wzbfKL7H3MYpCorM4oFBoQrdjnBgEoqG+RjhsvS1FKrDbKux+M1HhlnGWqcA==", + "license": "MIT", + "dependencies": { + "zod": "^4.2.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.43.0.tgz", + "integrity": "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, + "node_modules/@silvia-odwyer/photon-node": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@silvia-odwyer/photon-node/-/photon-node-0.3.4.tgz", + "integrity": "sha512-bnly4BKB3KDTFxrUIcgCLbaeVVS8lrAkri1pEzskpmxu9MdfGQTy8b8EgcD83ywD3RPMsIulY8xJH5Awa+t9fA==", + "license": "Apache-2.0" + }, + "node_modules/@smithy/core": { + "version": "3.31.1", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.31.1.tgz", + "integrity": "sha512-CyogUINxvi7C7LDsh8Syo6hVJOT9ckz4rG8dRZfTJ8r91HkMY59PnNooaj7WcHyxEkxPfBAmbgztZU+xTo76lg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.4.16", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.16.tgz", + "integrity": "sha512-QfuLWAkLzptffFW980AFeHZFdqds2B64rpEd3uJ6lgs3xVn9QegGMUgUcj+4d7dRrAsya3r58ZKpku97WcFb4w==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.6.13", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.13.tgz", + "integrity": "sha512-4fW86pEUOMbrD5nkbyl/tTvPHHWJFbuB2odl6ps9lWfHoXf9HWh3Q/Smh59qH1g7+c/BSZghX6bbUk4gsiMs8A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.7.3", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.7.3.tgz", + "integrity": "sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.24.3", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.6.12", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.6.12.tgz", + "integrity": "sha512-I6KLtq3H0qqSuV9vLglfi8puHqzygzWHOnI4z/Rdoo+q50vvo18vBRdPAvvEtcaKROz7Zn6qnPa14kRfPH6PcQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.16.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.16.1.tgz", + "integrity": "sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@types/node": { + "version": "22.15.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.0.tgz", + "integrity": "sha512-99S8dWD2DkeE6PBaEDw+In3aar7hdoBvjyJMR6vaKBTzpvR0P00ClzJMOoVrj9D2+Sy/YCwACYHnBTpMhg1UCA==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "license": "MIT" + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, + "node_modules/chalk": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/diff": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.4.tgz", + "integrity": "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/gaxios": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.3.0.tgz", + "integrity": "sha512-RB5vLV+vvQeoFPCX4QMK6/hjVkbIamPp1QSUD0CiZcnj12qbpiL+pLbYtgD+oZkWl0tl9z+o2Utp+MpM3QRhBA==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gcp-metadata": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^7.0.0", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz", + "integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/google-auth-library": { + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.9.1.tgz", + "integrity": "sha512-i1ydyHrqcIxXkWh/uBmVkzCvIuq5yiK2ATndIe5XxKholrG/MTYP9xGYka4sQhrbIAgGjL2B6NOE7rFaiF3fXw==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "8.1.2", + "google-logging-utils": "1.1.3", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/highlight.js": { + "version": "10.7.3", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", + "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==", + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, + "node_modules/hosted-git-info": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-9.0.3.tgz", + "integrity": "sha512-Hc+ghLoSt6QaYZUv0WBiIvmMDZuZZ7oaDvdH8MbfOO4lOsxdXLEvuC6ePoGs9H1X9oCLyq6+NVN0MKqD+ydxyg==", + "license": "ISC", + "dependencies": { + "lru-cache": "^11.1.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jiti": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.7.0.tgz", + "integrity": "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==", + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/jose": { + "version": "6.2.8", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.8.tgz", + "integrity": "sha512-Bsdjwm3Qsd/P0jR+BHDe3LytDfY7WBq2HmCCLIwuVRHMuEC9ae7/R474GIUdF1NgCyZjzVo/A9DOiOBtXq8ZoQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/lru-cache": { + "version": "11.5.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", + "integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/marked": { + "version": "18.0.5", + "resolved": "https://registry.npmjs.org/marked/-/marked-18.0.5.tgz", + "integrity": "sha512-S6GcvALHg6K4ohtu4E7x0a1AqhAjp6cV8KhLSyN9qVapnzJkusVBxZRcIU9AeYsbe6P1hKDusSbEOzGyyuce6w==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/openai": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.26.0.tgz", + "integrity": "sha512-zd23dbWTjiJ6sSAX6s0HrCZi41JwTA1bQVs0wLQPZ2/5o2gxOJA5wh7yOAUgwYybfhDXyhwlpeQf7Mlgx8EOCA==", + "license": "Apache-2.0", + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/partial-json": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/partial-json/-/partial-json-0.1.7.tgz", + "integrity": "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA==", + "license": "MIT" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/proper-lockfile/node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/protobufjs": { + "version": "7.6.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", + "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.0.tgz", + "integrity": "sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "license": "MIT" + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/typebox": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.3.6.tgz", + "integrity": "sha512-Sc8RA0NCMEFmApHNU9ZMzqcpQj46She44J8ffpLM/bdhLNUZKq7DJumcLcsFx1gRmDfQPgCgOmFFJ7rcnfWNyA==", + "license": "MIT" + }, + "node_modules/typescript": { + "version": "5.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici": { + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.5.0.tgz", + "integrity": "sha512-xamtWoB1EshgjpmlXd7GGm2VfdDtw1+rD8uhry8pSNW3If6S8E0m2T2+orSKeZXEn/aPJMviCpDBA65WJt8zhg==", + "license": "MIT", + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/ws": { + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", + "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + } + } +} diff --git a/packages/pi-code-policy-extension/package.json b/packages/pi-code-policy-extension/package.json new file mode 100644 index 0000000000..018375f2e4 --- /dev/null +++ b/packages/pi-code-policy-extension/package.json @@ -0,0 +1,24 @@ +{ + "name": "@dimos/pi-code-policy-extension", + "version": "0.1.0", + "private": true, + "type": "module", + "engines": { + "node": ">=22.19.0" + }, + "scripts": { + "clean": "rm -rf dist dist-test", + "build": "npm run clean && tsc -p tsconfig.build.json", + "typecheck": "tsc -p tsconfig.json", + "test": "npm run clean && tsc -p tsconfig.build.json && tsc -p tsconfig.test.json && node --test dist-test/test/*.test.js" + }, + "dependencies": { + "@earendil-works/pi-coding-agent": "0.80.10", + "@modelcontextprotocol/client": "2.0.0", + "typebox": "1.3.6" + }, + "devDependencies": { + "@types/node": "22.15.0", + "typescript": "5.8.3" + } +} diff --git a/packages/pi-code-policy-extension/src/python-exec.ts b/packages/pi-code-policy-extension/src/python-exec.ts new file mode 100644 index 0000000000..ebc2540897 --- /dev/null +++ b/packages/pi-code-policy-extension/src/python-exec.ts @@ -0,0 +1,84 @@ +import { + Client, + StreamableHTTPClientTransport, + type CallToolResult, +} from "@modelcontextprotocol/client"; +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { Type } from "typebox"; + +const TOOL_NAME = "python_exec"; +const DEFAULT_TIMEOUT_SECONDS = 600; + +interface McpClient { + listTools(): Promise<{ tools: Array<{ name: string; description?: string }> }>; + callTool( + params: { name: string; arguments: Record }, + options?: { timeout?: number }, + ): Promise; + close(): Promise; +} + +export function textFromResult(result: CallToolResult): string { + return result.content + .filter((item): item is Extract => item.type === "text") + .map((item) => item.text) + .join("\n"); +} + +async function connect(url: string): Promise { + const client = new Client({ name: "dimos-pi-code-policy", version: "1.0.0" }); + await client.connect(new StreamableHTTPClientTransport(new URL(url))); + return client; +} + +export async function installPythonExec( + pi: ExtensionAPI, + mcpUrl: string, + connectClient: (url: string) => Promise = connect, +): Promise { + const client = await connectClient(mcpUrl); + const inventory = await client.listTools(); + if (inventory.tools.length !== 1 || inventory.tools[0]?.name !== TOOL_NAME) { + await client.close(); + throw new Error("CodePolicy MCP server must expose exactly python_exec"); + } + + pi.registerTool({ + name: TOOL_NAME, + label: "Execute Python", + description: + inventory.tools[0].description ?? + "Execute Python in a persistent trusted, unsandboxed session.", + parameters: Type.Object( + { + code: Type.String({ minLength: 1 }), + timeout_s: Type.Optional( + Type.Number({ exclusiveMinimum: 0, maximum: DEFAULT_TIMEOUT_SECONDS }), + ), + }, + { additionalProperties: false }, + ), + executionMode: "sequential", + execute: async (_id, params) => { + const timeoutSeconds = params.timeout_s ?? DEFAULT_TIMEOUT_SECONDS; + const result = await client.callTool( + { + name: TOOL_NAME, + arguments: { code: params.code, timeout_s: timeoutSeconds }, + }, + { timeout: (timeoutSeconds + 10) * 1000 }, + ); + return { content: [{ type: "text", text: textFromResult(result) }], details: {} }; + }, + }); + + pi.on("session_shutdown", async () => { + await client.close(); + }); +} + +export default async function pythonExecExtension(pi: ExtensionAPI): Promise { + const mcpUrl = process.env.DIMOS_CODE_POLICY_MCP_URL; + if (!mcpUrl) throw new Error("DIMOS_CODE_POLICY_MCP_URL is required"); + await installPythonExec(pi, mcpUrl); +} diff --git a/packages/pi-code-policy-extension/test/python-exec.test.ts b/packages/pi-code-policy-extension/test/python-exec.test.ts new file mode 100644 index 0000000000..2fd25148f7 --- /dev/null +++ b/packages/pi-code-policy-extension/test/python-exec.test.ts @@ -0,0 +1,51 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import type { CallToolResult } from "@modelcontextprotocol/client"; +import type { ExtensionAPI, ToolDefinition } from "@earendil-works/pi-coding-agent"; + +import { installPythonExec } from "../src/python-exec.js"; + +test("registers one tool that calls MCP directly", async () => { + let tool: ToolDefinition | undefined; + let shutdown: (() => Promise) | undefined; + let closed = false; + const pi = { + registerTool(value: ToolDefinition) { + tool = value; + }, + on(event: string, handler: () => Promise) { + if (event === "session_shutdown") shutdown = handler; + }, + } as ExtensionAPI; + const client = { + async listTools() { + return { + tools: [ + { + name: "python_exec", + description: "Canonical CodePolicy description", + }, + ], + }; + }, + async callTool(params: { name: string; arguments: Record }) { + assert.deepEqual(params, { + name: "python_exec", + arguments: { code: "1 + 1", timeout_s: 3 }, + }); + return { content: [{ type: "text", text: "2" }] } as CallToolResult; + }, + async close() { + closed = true; + }, + }; + + await installPythonExec(pi, "http://127.0.0.1:1/mcp", async () => client); + assert.equal(tool?.name, "python_exec"); + assert.equal(tool?.description, "Canonical CodePolicy description"); + const result = await tool!.execute("call-1", { code: "1 + 1", timeout_s: 3 }, undefined, undefined, {} as never); + assert.deepEqual(result.content, [{ type: "text", text: "2" }]); + await shutdown!(); + assert.equal(closed, true); +}); diff --git a/packages/pi-code-policy-extension/tsconfig.build.json b/packages/pi-code-policy-extension/tsconfig.build.json new file mode 100644 index 0000000000..7b4b891a32 --- /dev/null +++ b/packages/pi-code-policy-extension/tsconfig.build.json @@ -0,0 +1,13 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "sourceMap": true + }, + "include": [ + "src" + ] +} diff --git a/packages/pi-code-policy-extension/tsconfig.json b/packages/pi-code-policy-extension/tsconfig.json new file mode 100644 index 0000000000..fd64273095 --- /dev/null +++ b/packages/pi-code-policy-extension/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "verbatimModuleSyntax": true, + "types": [ + "node" + ] + }, + "include": [ + "src", + "test" + ] +} diff --git a/packages/pi-code-policy-extension/tsconfig.test.json b/packages/pi-code-policy-extension/tsconfig.test.json new file mode 100644 index 0000000000..b2c0cfcf0c --- /dev/null +++ b/packages/pi-code-policy-extension/tsconfig.test.json @@ -0,0 +1,14 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "outDir": "dist-test", + "rootDir": ".", + "declaration": false, + "sourceMap": false + }, + "include": [ + "src", + "test" + ] +} diff --git a/pyproject.toml b/pyproject.toml index 4e50ad93f8..4a480aa22d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -225,6 +225,14 @@ learning = [ ] agents = [ + # Persistent trusted Python runtime used by standalone CodePolicy evaluation. + "cloudpickle>=3.1,<4", + "ipykernel>=7.2.0", + "jupyter-client>=8.8.0", + # Loopback-only MCP host for standalone CodePolicy evaluation. + "mcp==2.0.0", + "requests>=2.32,<3", + "uvicorn>=0.34.0", "langchain>=1.2.3,<2", "langchain-core>=1.2.22,<2", "langchain-openai>=1,<2", @@ -409,6 +417,12 @@ project-deps = [ "lap>=0.5.12", "langchain-openai>=1,<2", "ollama>=0.6.0", + "cloudpickle>=3.1,<4", + "ipykernel>=7.2.0", + "jupyter-client>=8.8.0", + "mcp==2.0.0", + "requests>=2.32,<3", + "uvicorn>=0.34.0", ] tests = [ diff --git a/uv.lock b/uv.lock index 04a30f4013..3de78d55da 100644 --- a/uv.lock +++ b/uv.lock @@ -26,7 +26,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-03T19:08:40.466827Z" +exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. exclude-newer-span = "P7D" [options.exclude-newer-package] @@ -1686,16 +1686,22 @@ dependencies = [ [package.optional-dependencies] agents = [ + { name = "cloudpickle" }, { name = "faster-whisper" }, + { name = "ipykernel" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-huggingface" }, { name = "langchain-ollama" }, { name = "langchain-openai" }, + { name = "mcp" }, { name = "ollama" }, { name = "openai" }, { name = "openevals" }, + { name = "requests" }, { name = "sounddevice" }, + { name = "uvicorn" }, ] all = [ { name = "a750-control", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, @@ -1703,6 +1709,7 @@ all = [ { name = "aioquic" }, { name = "aiortc" }, { name = "chromadb" }, + { name = "cloudpickle" }, { name = "coacd" }, { name = "cupy-cuda12x", marker = "platform_machine == 'x86_64'" }, { name = "dimos-viewer" }, @@ -1719,6 +1726,7 @@ all = [ { name = "hydra-core" }, { name = "ipykernel" }, { name = "jinja2" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-huggingface" }, @@ -1727,6 +1735,7 @@ all = [ { name = "lap" }, { name = "manifold3d" }, { name = "matplotlib" }, + { name = "mcp" }, { name = "moondream" }, { name = "mujoco" }, { name = "ollama" }, @@ -1747,6 +1756,7 @@ all = [ { name = "python-multipart" }, { name = "pyyaml" }, { name = "reportlab" }, + { name = "requests" }, { name = "rerun-sdk" }, { name = "roboplan" }, { name = "sounddevice" }, @@ -1774,25 +1784,30 @@ apriltag = [ base = [ { name = "aioquic" }, { name = "chromadb" }, + { name = "cloudpickle" }, { name = "dimos-viewer" }, { name = "einops" }, { name = "fastapi" }, { name = "faster-whisper" }, { name = "ffmpeg-python" }, { name = "hydra-core" }, + { name = "ipykernel" }, { name = "jinja2" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-huggingface" }, { name = "langchain-ollama" }, { name = "langchain-openai" }, { name = "lap" }, + { name = "mcp" }, { name = "moondream" }, { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, { name = "openevals" }, { name = "pillow" }, + { name = "requests" }, { name = "rerun-sdk" }, { name = "sounddevice" }, { name = "soundfile" }, @@ -1889,6 +1904,7 @@ spot = [ unitree = [ { name = "aioquic" }, { name = "chromadb" }, + { name = "cloudpickle" }, { name = "dimos-viewer" }, { name = "einops" }, { name = "fastapi" }, @@ -1896,19 +1912,23 @@ unitree = [ { name = "ffmpeg-python" }, { name = "gtsam-extended" }, { name = "hydra-core" }, + { name = "ipykernel" }, { name = "jinja2" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-huggingface" }, { name = "langchain-ollama" }, { name = "langchain-openai" }, { name = "lap" }, + { name = "mcp" }, { name = "moondream" }, { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, { name = "openevals" }, { name = "pillow" }, + { name = "requests" }, { name = "rerun-sdk" }, { name = "sounddevice" }, { name = "soundfile" }, @@ -1922,6 +1942,7 @@ unitree = [ unitree-dds = [ { name = "aioquic" }, { name = "chromadb" }, + { name = "cloudpickle" }, { name = "cyclonedds" }, { name = "dimos-viewer" }, { name = "einops" }, @@ -1930,7 +1951,9 @@ unitree-dds = [ { name = "ffmpeg-python" }, { name = "gtsam-extended" }, { name = "hydra-core" }, + { name = "ipykernel" }, { name = "jinja2" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-huggingface" }, @@ -1938,12 +1961,14 @@ unitree-dds = [ { name = "langchain-openai" }, { name = "lap" }, { name = "mcap" }, + { name = "mcp" }, { name = "moondream" }, { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, { name = "openevals" }, { name = "pillow" }, + { name = "requests" }, { name = "rerun-sdk" }, { name = "sounddevice" }, { name = "soundfile" }, @@ -1984,17 +2009,21 @@ browser-tests = [ lint = [ { name = "aiortc" }, { name = "chromadb" }, + { name = "cloudpickle" }, { name = "dimos", extra = ["visualization", "web", "webrtc"] }, { name = "einops" }, { name = "gdown" }, { name = "googlemaps" }, { name = "hydra-core" }, + { name = "ipykernel" }, { name = "ipython", version = "8.38.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "ipython", version = "9.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-openai" }, { name = "lap" }, + { name = "mcp" }, { name = "moondream" }, { name = "mypy" }, { name = "ollama" }, @@ -2005,6 +2034,7 @@ lint = [ { name = "pytest" }, { name = "python-can" }, { name = "python-socketio" }, + { name = "requests" }, { name = "roboplan" }, { name = "ruff" }, { name = "sounddevice" }, @@ -2018,33 +2048,41 @@ lint = [ { name = "types-reportlab" }, { name = "types-requests" }, { name = "ultralytics" }, + { name = "uvicorn" }, { name = "watchdog" }, { name = "xacro" }, ] project-deps = [ { name = "chromadb" }, + { name = "cloudpickle" }, { name = "dimos", extra = ["visualization", "web", "webrtc"] }, { name = "einops" }, { name = "gdown" }, { name = "googlemaps" }, { name = "hydra-core" }, + { name = "ipykernel" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-openai" }, { name = "lap" }, + { name = "mcp" }, { name = "moondream" }, { name = "ollama" }, { name = "open-clip-torch" }, { name = "openai" }, + { name = "requests" }, { name = "tensorboard" }, { name = "torch" }, { name = "torchreid" }, { name = "transformers", extra = ["torch"] }, { name = "ultralytics" }, + { name = "uvicorn" }, { name = "xacro" }, ] tests = [ { name = "chromadb" }, + { name = "cloudpickle" }, { name = "coacd" }, { name = "coverage" }, { name = "dimos", extra = ["apriltag", "cpu", "drone", "learning", "mapping", "visualization", "web", "webrtc"] }, @@ -2052,11 +2090,14 @@ tests = [ { name = "gdown" }, { name = "googlemaps" }, { name = "hydra-core" }, + { name = "ipykernel" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-openai" }, { name = "lap" }, { name = "maturin" }, + { name = "mcp" }, { name = "md-babel-py" }, { name = "moondream" }, { name = "mujoco" }, @@ -2078,6 +2119,7 @@ tests = [ { name = "python-can" }, { name = "python-lsp-ruff" }, { name = "python-lsp-server", extra = ["all"] }, + { name = "requests" }, { name = "requests-mock" }, { name = "tensorboard" }, { name = "torch" }, @@ -2086,12 +2128,14 @@ tests = [ { name = "trimesh" }, { name = "ultralytics" }, { name = "unitree-webrtc-connect" }, + { name = "uvicorn" }, { name = "viser", extra = ["urdf"], marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, { name = "watchdog" }, { name = "xacro" }, ] tests-self-hosted = [ { name = "chromadb" }, + { name = "cloudpickle" }, { name = "coacd" }, { name = "coverage" }, { name = "dimos", extra = ["agents", "apriltag", "cpu", "drone", "learning", "manipulation", "mapping", "misc", "perception", "sim", "unitree", "visualization", "web", "webrtc"] }, @@ -2099,12 +2143,15 @@ tests-self-hosted = [ { name = "gdown" }, { name = "googlemaps" }, { name = "hydra-core" }, + { name = "ipykernel" }, + { name = "jupyter-client" }, { name = "langchain" }, { name = "langchain-core" }, { name = "langchain-openai" }, { name = "lap" }, { name = "maturin" }, { name = "mcap" }, + { name = "mcp" }, { name = "md-babel-py" }, { name = "moondream" }, { name = "mujoco" }, @@ -2127,6 +2174,7 @@ tests-self-hosted = [ { name = "python-can" }, { name = "python-lsp-ruff" }, { name = "python-lsp-server", extra = ["all"] }, + { name = "requests" }, { name = "requests-mock" }, { name = "tensorboard" }, { name = "torch" }, @@ -2135,6 +2183,7 @@ tests-self-hosted = [ { name = "trimesh" }, { name = "ultralytics" }, { name = "unitree-webrtc-connect" }, + { name = "uvicorn" }, { name = "viser", extra = ["urdf"], marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, { name = "watchdog" }, { name = "xacro" }, @@ -2152,6 +2201,7 @@ requires-dist = [ { name = "bosdyn-client", marker = "extra == 'spot'", specifier = ">=4.0.0" }, { name = "bosdyn-core", marker = "extra == 'spot'", specifier = ">=4.0.0" }, { name = "chromadb", marker = "extra == 'perception'", specifier = ">=1.0.0" }, + { name = "cloudpickle", marker = "extra == 'agents'", specifier = ">=3.1,<4" }, { name = "cmeel-tinyxml2", specifier = ">=11,<12" }, { name = "coacd", marker = "extra == 'scene'", specifier = ">=1.0.0" }, { name = "cryptography", specifier = ">=46.0.5" }, @@ -2182,9 +2232,11 @@ requires-dist = [ { name = "huggingface-hub", marker = "extra == 'graspgenx'", specifier = ">=0.30,<1" }, { name = "hydra-core", marker = "extra == 'perception'", specifier = ">=1.3.0" }, { name = "imagecodecs", specifier = ">=2024.6.1" }, + { name = "ipykernel", marker = "extra == 'agents'", specifier = ">=7.2.0" }, { name = "ipykernel", marker = "extra == 'misc'" }, { name = "ipython" }, { name = "jinja2", marker = "extra == 'web'", specifier = ">=3.1.6" }, + { name = "jupyter-client", marker = "extra == 'agents'", specifier = ">=8.8.0" }, { name = "langchain", marker = "extra == 'agents'", specifier = ">=1.2.3,<2" }, { name = "langchain-core", marker = "extra == 'agents'", specifier = ">=1.2.22,<2" }, { name = "langchain-huggingface", marker = "extra == 'agents'", specifier = ">=1,<2" }, @@ -2198,6 +2250,7 @@ requires-dist = [ { name = "matplotlib", marker = "extra == 'graspgenx'", specifier = ">=3.7.1" }, { name = "matplotlib", marker = "extra == 'manipulation'", specifier = ">=3.7.1" }, { name = "mcap", marker = "extra == 'unitree-dds'", specifier = ">=1.2.0" }, + { name = "mcp", marker = "extra == 'agents'", specifier = "==2.0.0" }, { name = "moondream", marker = "extra == 'perception'" }, { name = "mujoco", marker = "extra == 'sim'", specifier = ">=3.3.4" }, { name = "numba", specifier = ">=0.60.0" }, @@ -2238,6 +2291,7 @@ requires-dist = [ { name = "qpsolvers", extras = ["proxqp"], specifier = ">=4.12.0" }, { name = "reactivex" }, { name = "reportlab", marker = "extra == 'apriltag'", specifier = ">=4.5.0" }, + { name = "requests", marker = "extra == 'agents'", specifier = ">=2.32,<3" }, { name = "rerun-sdk", specifier = "==0.32.0" }, { name = "rerun-sdk", marker = "extra == 'visualization'", specifier = "==0.32.0" }, { name = "roboplan", marker = "extra == 'manipulation'", specifier = ">=0.6.0,<0.7.0" }, @@ -2267,6 +2321,7 @@ requires-dist = [ { name = "unitree-sdk2py-dimos", marker = "extra == 'unitree-dds'", specifier = ">=1.0.2" }, { name = "unitree-webrtc-connect", marker = "extra == 'unitree'", specifier = ">=2.1.2" }, { name = "usd-core", marker = "extra == 'scene'", specifier = ">=23.11" }, + { name = "uvicorn", marker = "extra == 'agents'", specifier = ">=0.34.0" }, { name = "uvicorn", marker = "extra == 'web'", specifier = ">=0.34.0" }, { name = "viser", extras = ["urdf"], marker = "extra == 'manipulation'", specifier = ">=1.0.29" }, { name = "websocket-client", specifier = ">=1.8" }, @@ -2284,16 +2339,20 @@ browser-tests = [{ name = "playwright", specifier = ">=1.55" }] lint = [ { name = "aiortc", specifier = ">=1.14.0" }, { name = "chromadb", specifier = ">=1.0.0" }, + { name = "cloudpickle", specifier = ">=3.1,<4" }, { name = "dimos", extras = ["web", "visualization", "webrtc"] }, { name = "einops", specifier = ">=0.8.1" }, { name = "gdown", specifier = "==6.0.0" }, { name = "googlemaps", specifier = ">=4.10.0" }, { name = "hydra-core", specifier = ">=1.3.0" }, + { name = "ipykernel", specifier = ">=7.2.0" }, { name = "ipython" }, + { name = "jupyter-client", specifier = ">=8.8.0" }, { name = "langchain", specifier = "==1.2.3" }, { name = "langchain-core", specifier = "==1.3.3" }, { name = "langchain-openai", specifier = ">=1,<2" }, { name = "lap", specifier = ">=0.5.12" }, + { name = "mcp", specifier = "==2.0.0" }, { name = "moondream" }, { name = "mypy", specifier = "==1.19.0" }, { name = "ollama", specifier = ">=0.6.0" }, @@ -2304,6 +2363,7 @@ lint = [ { name = "pytest", specifier = "==8.3.5" }, { name = "python-can", specifier = ">=4" }, { name = "python-socketio", specifier = ">=5.16.1" }, + { name = "requests", specifier = ">=2.32,<3" }, { name = "roboplan", specifier = ">=0.6.0,<0.7.0" }, { name = "ruff", specifier = "==0.14.3" }, { name = "sounddevice", specifier = ">=0.5.5" }, @@ -2317,33 +2377,41 @@ lint = [ { name = "types-reportlab", specifier = ">=4.5.0" }, { name = "types-requests", specifier = ">=2.32.4.20260107,<3" }, { name = "ultralytics", specifier = ">=8.3.70" }, + { name = "uvicorn", specifier = ">=0.34.0" }, { name = "watchdog", specifier = ">=3.0.0" }, { name = "xacro" }, ] project-deps = [ { name = "chromadb", specifier = ">=1.0.0" }, + { name = "cloudpickle", specifier = ">=3.1,<4" }, { name = "dimos", extras = ["web", "visualization", "webrtc"] }, { name = "einops", specifier = ">=0.8.1" }, { name = "gdown", specifier = "==6.0.0" }, { name = "googlemaps", specifier = ">=4.10.0" }, { name = "hydra-core", specifier = ">=1.3.0" }, + { name = "ipykernel", specifier = ">=7.2.0" }, + { name = "jupyter-client", specifier = ">=8.8.0" }, { name = "langchain", specifier = "==1.2.3" }, { name = "langchain-core", specifier = "==1.3.3" }, { name = "langchain-openai", specifier = ">=1,<2" }, { name = "lap", specifier = ">=0.5.12" }, + { name = "mcp", specifier = "==2.0.0" }, { name = "moondream" }, { name = "ollama", specifier = ">=0.6.0" }, { name = "open-clip-torch", specifier = "==3.2.0" }, { name = "openai" }, + { name = "requests", specifier = ">=2.32,<3" }, { name = "tensorboard", specifier = "==2.20.0" }, { name = "torch" }, { name = "torchreid", specifier = "==0.2.5" }, { name = "transformers", extras = ["torch"], specifier = "==4.53.3" }, { name = "ultralytics", specifier = ">=8.3.70" }, + { name = "uvicorn", specifier = ">=0.34.0" }, { name = "xacro" }, ] tests = [ { name = "chromadb", specifier = ">=1.0.0" }, + { name = "cloudpickle", specifier = ">=3.1,<4" }, { name = "coacd", specifier = ">=1.0.0" }, { name = "coverage", specifier = ">=7.0" }, { name = "dimos", extras = ["apriltag", "mapping", "drone", "cpu", "learning"] }, @@ -2352,11 +2420,14 @@ tests = [ { name = "gdown", specifier = "==6.0.0" }, { name = "googlemaps", specifier = ">=4.10.0" }, { name = "hydra-core", specifier = ">=1.3.0" }, + { name = "ipykernel", specifier = ">=7.2.0" }, + { name = "jupyter-client", specifier = ">=8.8.0" }, { name = "langchain", specifier = "==1.2.3" }, { name = "langchain-core", specifier = "==1.3.3" }, { name = "langchain-openai", specifier = ">=1,<2" }, { name = "lap", specifier = ">=0.5.12" }, { name = "maturin", specifier = ">=1.7" }, + { name = "mcp", specifier = "==2.0.0" }, { name = "md-babel-py", specifier = ">=1.4.0" }, { name = "moondream" }, { name = "mujoco", specifier = ">=3.3.4" }, @@ -2378,6 +2449,7 @@ tests = [ { name = "python-can", specifier = ">=4" }, { name = "python-lsp-ruff", specifier = "==2.3.0" }, { name = "python-lsp-server", extras = ["all"], specifier = "==1.14.0" }, + { name = "requests", specifier = ">=2.32,<3" }, { name = "requests-mock", specifier = "==1.12.1" }, { name = "tensorboard", specifier = "==2.20.0" }, { name = "torch" }, @@ -2386,12 +2458,14 @@ tests = [ { name = "trimesh", specifier = ">=4.0.0" }, { name = "ultralytics", specifier = ">=8.3.70" }, { name = "unitree-webrtc-connect", specifier = ">=2.1.2" }, + { name = "uvicorn", specifier = ">=0.34.0" }, { name = "viser", extras = ["urdf"], marker = "platform_machine != 'aarch64' or sys_platform != 'linux'", specifier = ">=1.0.29" }, { name = "watchdog", specifier = ">=3.0.0" }, { name = "xacro" }, ] tests-self-hosted = [ { name = "chromadb", specifier = ">=1.0.0" }, + { name = "cloudpickle", specifier = ">=3.1,<4" }, { name = "coacd", specifier = ">=1.0.0" }, { name = "coverage", specifier = ">=7.0" }, { name = "dimos", extras = ["agents", "perception", "manipulation", "sim", "unitree", "misc"] }, @@ -2401,12 +2475,15 @@ tests-self-hosted = [ { name = "gdown", specifier = "==6.0.0" }, { name = "googlemaps", specifier = ">=4.10.0" }, { name = "hydra-core", specifier = ">=1.3.0" }, + { name = "ipykernel", specifier = ">=7.2.0" }, + { name = "jupyter-client", specifier = ">=8.8.0" }, { name = "langchain", specifier = "==1.2.3" }, { name = "langchain-core", specifier = "==1.3.3" }, { name = "langchain-openai", specifier = ">=1,<2" }, { name = "lap", specifier = ">=0.5.12" }, { name = "maturin", specifier = ">=1.7" }, { name = "mcap", specifier = ">=1.2.0" }, + { name = "mcp", specifier = "==2.0.0" }, { name = "md-babel-py", specifier = ">=1.4.0" }, { name = "moondream" }, { name = "mujoco", specifier = ">=3.3.4" }, @@ -2429,6 +2506,7 @@ tests-self-hosted = [ { name = "python-can", specifier = ">=4" }, { name = "python-lsp-ruff", specifier = "==2.3.0" }, { name = "python-lsp-server", extras = ["all"], specifier = "==1.14.0" }, + { name = "requests", specifier = ">=2.32,<3" }, { name = "requests-mock", specifier = "==1.12.1" }, { name = "tensorboard", specifier = "==2.20.0" }, { name = "torch" }, @@ -2437,6 +2515,7 @@ tests-self-hosted = [ { name = "trimesh", specifier = ">=4.0.0" }, { name = "ultralytics", specifier = ">=8.3.70" }, { name = "unitree-webrtc-connect", specifier = ">=2.1.2" }, + { name = "uvicorn", specifier = ">=0.34.0" }, { name = "viser", extras = ["urdf"], marker = "platform_machine != 'aarch64' or sys_platform != 'linux'", specifier = ">=1.0.29" }, { name = "watchdog", specifier = ">=3.0.0" }, { name = "xacro" }, @@ -3495,6 +3574,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/a8/20ed1ed79cbc2ecdf5301c0968ab7c85547212e2a7bd126ddd2d986e206e/httpcore2-2.9.1.tar.gz", hash = "sha256:4d8acbf8b306f48c9d6046591fd5ba4037d1b1b1000d140fc2c3eab1e9a0c0e2", size = 67089, upload-time = "2026-07-24T09:21:03.867Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/fb/46c52b781975c335a2bcf1072c7bbc007cbdc8d674217f5ee1daba2c848b/httpcore2-2.9.1-py3-none-any.whl", hash = "sha256:6182472379e855fe4221246a2bb7ecede403bc61c6798062ae1787d051ccde26", size = 82809, upload-time = "2026-07-24T09:21:01.178Z" }, +] + [[package]] name = "httptools" version = "0.8.0" @@ -3539,6 +3631,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx2" +version = "2.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpcore2" }, + { name = "idna" }, + { name = "truststore" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/14/38128fbafd7e0ed41d874df6c9a653d47c2d111cfe59e2b4ac95161b4abd/httpx2-2.9.1.tar.gz", hash = "sha256:1932a768737e3666291582833da748cc4e563c337cf96706fccc04fa6e58764a", size = 95458, upload-time = "2026-07-24T09:21:04.972Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/b8/cfd91c4ab9134d386d48f0b6ac662ff3d4be6efdee59ee1c67ebc3c0487c/httpx2-2.9.1-py3-none-any.whl", hash = "sha256:1820fe14a9ab1107bfeff39259987429450b070ec0ff38cc87eb0d8c97fdc71a", size = 91191, upload-time = "2026-07-24T09:21:02.6Z" }, +] + [[package]] name = "huggingface-hub" version = "0.36.2" @@ -3592,11 +3700,11 @@ wheels = [ [[package]] name = "idna" -version = "3.11" +version = "3.18" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] [[package]] @@ -4991,6 +5099,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, ] +[[package]] +name = "mcp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx2" }, + { name = "jsonschema" }, + { name = "mcp-types" }, + { name = "opentelemetry-api" }, + { name = "pydantic" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "python-multipart" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "sse-starlette" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, + { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/33/32d4dff2c95bb5d897c3ef4c83649a08996b17b58f0a326d2495d4c81179/mcp-2.0.0.tar.gz", hash = "sha256:0f440e735c13ece8bb19bc62cf0b86f4313448432fbb77d35e14034f4e050728", size = 1662284, upload-time = "2026-07-28T13:45:32.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/72/7d7897418912c1d12e87556630dfb7bf0eac71160e9bef8b447960804ee3/mcp-2.0.0-py3-none-any.whl", hash = "sha256:1cb4c75d2d2c7b8c1d756355e5d82a39f2822cc7f13e22a2051d7ca3592349d6", size = 349980, upload-time = "2026-07-28T13:45:28.853Z" }, +] + +[[package]] +name = "mcp-types" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/56/9b8e1c152f61f6c6b07c4b5896c88c7d0ae90bac6ee6306f852fcc5c1eb0/mcp_types-2.0.0.tar.gz", hash = "sha256:d7d939b9285c9961ae8866ba75ef85da34d12bafe276efbf4eb6a131786d8379", size = 66632, upload-time = "2026-07-28T13:45:33.804Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/4c/c78d78c3d52b0ac594ad7cc8ef5972adfe070e3597a8a4c6ce0cd39196ea/mcp_types-2.0.0-py3-none-any.whl", hash = "sha256:6b2de797ca2797f568b79529e1b25948e34de511bcc0bd82fef1039a6d1b8eb0", size = 69649, upload-time = "2026-07-28T13:45:30.713Z" }, +] + [[package]] name = "md-babel-py" version = "1.4.0" @@ -7260,6 +7406,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, ] +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + [[package]] name = "pylibsrtp" version = "1.0.0" @@ -9476,6 +9627,15 @@ dependencies = [ { name = "importlib-metadata", marker = "platform_machine == 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "typer" version = "0.23.1"