diff --git a/dimos/agents/code_policy_server.py b/dimos/agents/code_policy_server.py index 70b89de7dd..a16765382d 100644 --- a/dimos/agents/code_policy_server.py +++ b/dimos/agents/code_policy_server.py @@ -33,9 +33,8 @@ PYTHON_EXEC_DESCRIPTION = """Execute Python in a persistent trusted, unsandboxed session. -The frozen evaluation session exposes read-only `memory`. Imports, functions, and -variables persist between calls. Use this tool to inspect the recording and compute -the answer; do not guess from the prompt. +Imports, functions, and variables persist between calls. The runtime environment +determines which globals are available. """ _NOISY_MCP_TRANSPORT_LOGGERS = ( diff --git a/dimos/benchmark/agent_eval/models.py b/dimos/benchmark/agent_eval/models.py deleted file mode 100644 index 7991d89b59..0000000000 --- a/dimos/benchmark/agent_eval/models.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Small tagged contracts for one frozen agent-evaluation case.""" - -from __future__ import annotations - -import math -from pathlib import PurePosixPath -from typing import Annotated, Literal - -from pydantic import BaseModel, ConfigDict, Field, model_validator - - -class BaseEvalModel(BaseModel): - """Strict immutable base for the compact evaluation contracts.""" - - model_config = ConfigDict(extra="forbid", frozen=True, strict=True) - schema_version: Literal["1.0"] = "1.0" - - -NonEmpty = Annotated[str, Field(min_length=1)] - - -class FrozenRecordingSource(BaseEvalModel): - kind: Literal["frozen_memory"] = "frozen_memory" - recording: NonEmpty - progress: float = Field(ge=0, le=1, allow_inf_nan=False) - - @model_validator(mode="after") - def finite_progress(self) -> FrozenRecordingSource: - if not math.isfinite(self.progress): - raise ValueError("recording progress must be finite") - return self - - -class IntegerQuestionTask(BaseEvalModel): - kind: Literal["integer_question"] = "integer_question" - prompt: NonEmpty - answer_marker: Literal["ANSWER:"] = "ANSWER:" - - -class ExactIntegerValidatorRef(BaseEvalModel): - kind: Literal["exact_integer"] = "exact_integer" - revision: NonEmpty - private_path: NonEmpty - - @model_validator(mode="after") - def safe_relative_path(self) -> ExactIntegerValidatorRef: - path = PurePosixPath(self.private_path) - if path.is_absolute() or not path.parts or ".." in path.parts: - raise ValueError("validator private_path must be a safe relative path") - return self - - -SourceSpec = Annotated[FrozenRecordingSource, Field(discriminator="kind")] -TaskSpec = Annotated[IntegerQuestionTask, Field(discriminator="kind")] -ValidatorRef = Annotated[ExactIntegerValidatorRef, Field(discriminator="kind")] - - -class EvalCase(BaseEvalModel): - case_id: NonEmpty - source: SourceSpec - task: TaskSpec - validator: ValidatorRef - - -class PiAgentConfig(BaseEvalModel): - backend: Literal["pi"] = "pi" - model: Literal["gpt-5.6-luna"] = "gpt-5.6-luna" - thinking_level: Literal["medium"] = "medium" - api_key_env: str = Field(default="OPENAI_API_KEY", min_length=1) - - -class EvalRunConfig(BaseEvalModel): - agent: PiAgentConfig = Field(default_factory=PiAgentConfig) - - -class CompactEvalResult(BaseEvalModel): - case_id: str - recording: str - progress: float - model: str - thinking_level: str - final_response: str = "" - prediction_status: Literal["parsed", "invalid", "not_evaluated"] - integer_answer: int | None = None - passed: bool | None = None - validator_revision: str - tool_call_count: int = Field(ge=0) - duration_seconds: float = Field(ge=0) - infra_error: str | None = None - - @property - def attempt_status(self) -> Literal["completed", "failed"]: - return "failed" if self.infra_error is not None else "completed" - - @property - def task_result(self) -> Literal["passed", "failed", "not_evaluated"]: - if self.passed is None: - return "not_evaluated" - return "passed" if self.passed else "failed" diff --git a/dimos/benchmark/agent_eval/single_case.py b/dimos/benchmark/agent_eval/single_case.py deleted file mode 100644 index 1c951d7825..0000000000 --- a/dimos/benchmark/agent_eval/single_case.py +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Direct runner for one frozen-memory Pi evaluation case.""" - -from __future__ import annotations - -import errno -import json -import os -from pathlib import Path -import re -import shutil -import tempfile -import time - -from dimos.agents.code_policy_core import CodePolicySessionConfig, FrozenMemoryEnvironment -from dimos.agents.code_policy_server import CodePolicyMcpServer -from dimos.benchmark.agent_eval.models import CompactEvalResult, EvalCase, EvalRunConfig -from dimos.benchmark.agent_eval.pi_process import PiCliRunner, PiRunError -from dimos.benchmark.agent_eval.progress import ProgressSink, StatusProgress, emit_progress -from dimos.benchmark.short_horizon_qa.eval import ( - load_exact_integer_oracle, - parse_integer_prediction, -) -from dimos.benchmark.short_horizon_qa.models import MapperSettings -from dimos.benchmark.short_horizon_qa.prepare import prepare_bundle -from dimos.benchmark.short_horizon_qa.service import load_bundle -from dimos.constants import CACHE_DIR -from dimos.memory2.cli.dataset import resolve_dataset - -TURN_TIMEOUT_SECONDS = 600.0 - -SYSTEM_PROMPT = """You are answering a question about a frozen robot recording. - -You have exactly one tool, `python_exec`. It runs trusted, unsandboxed Python in a -persistent Jupyter kernel with a read-only `memory` object. Inspect Memory2 streams -and compute the answer from the recording. Do not guess. End with exactly one line: -ANSWER: -""" - - -def execute_single_case( - case_path: Path, - *, - config: EvalRunConfig, - output: Path, - progress: ProgressSink | None = None, -) -> CompactEvalResult: - """Preflight, run, and atomically publish exactly one result directory.""" - path = case_path.expanduser().resolve() - output = output.expanduser().resolve() - _validate_output(output) - emit_progress(progress, StatusProgress(channel="eval", message="loading case")) - case = EvalCase.model_validate_json(path.read_bytes()) - oracle = load_exact_integer_oracle(case, path.parent) - api_key = os.environ.get(config.agent.api_key_env) - if not api_key: - raise ValueError(f"API key environment variable {config.agent.api_key_env!r} is unset") - bundle = _materialize_frozen_memory(case, progress) - _, cutoff, source_path, derived_path = load_bundle(bundle, progress=case.source.progress) - emit_progress(progress, StatusProgress(channel="eval", message="memory ready")) - cli, extension = _pi_paths() - runner = PiCliRunner( - cli=cli, - extension=extension, - model=config.agent.model, - thinking_level=config.agent.thinking_level, - timeout_s=TURN_TIMEOUT_SECONDS, - progress=progress, - ) - - output.parent.mkdir(parents=True, exist_ok=True) - temporary = Path(tempfile.mkdtemp(prefix=f".{output.name}-", dir=output.parent)) - runtime_dir = temporary / "runtime" - runtime_dir.mkdir() - started = time.monotonic() - stderr = "" - server: CodePolicyMcpServer | None = None - try: - emit_progress(progress, StatusProgress(channel="eval", message="starting agent")) - server = CodePolicyMcpServer( - CodePolicySessionConfig( - environment=FrozenMemoryEnvironment( - recording_path=str(source_path), - derived_recording_path=str(derived_path), - memory_cutoff_timestamp=cutoff.cutoff_timestamp, - ) - ) - ) - try: - server.start() - pi_result = runner.run( - prompt=_agent_prompt(case), - system_prompt=SYSTEM_PROMPT, - mcp_url=server.mcp_url, - api_key=api_key, - run_dir=runtime_dir, - ) - stderr = pi_result.stderr - if pi_result.transcript_path is not None: - shutil.copy2(pi_result.transcript_path, temporary / "pi-transcript.jsonl") - prediction = parse_integer_prediction(pi_result.final_text) - passed = ( - prediction.status == "parsed" and prediction.integer_answer == oracle.expected_count - ) - result = CompactEvalResult( - case_id=case.case_id, - recording=case.source.recording, - progress=case.source.progress, - model=config.agent.model, - thinking_level=config.agent.thinking_level, - final_response=pi_result.final_text, - prediction_status=prediction.status, - integer_answer=prediction.integer_answer, - passed=passed, - validator_revision=case.validator.revision, - tool_call_count=pi_result.tool_call_count, - duration_seconds=time.monotonic() - started, - ) - finally: - server.stop() - except Exception as exc: - if isinstance(exc, PiRunError): - stderr = exc.stderr - result = CompactEvalResult( - case_id=case.case_id, - recording=case.source.recording, - progress=case.source.progress, - model=config.agent.model, - thinking_level=config.agent.thinking_level, - prediction_status="not_evaluated", - passed=None, - validator_revision=case.validator.revision, - tool_call_count=server.session.execution_count if server is not None else 0, - duration_seconds=time.monotonic() - started, - infra_error=f"{type(exc).__name__}: {exc}", - ) - finally: - shutil.rmtree(runtime_dir, ignore_errors=True) - - if stderr: - (temporary / "stderr.log").write_text(stderr, encoding="utf-8") - (temporary / "result.json").write_text( - json.dumps(result.model_dump(mode="json"), indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - if output.exists(): - output.rmdir() - os.replace(temporary, output) - emit_progress(progress, StatusProgress(channel="eval", message="result published")) - return result - - -def _validate_output(output: Path) -> None: - if output.exists() and (not output.is_dir() or any(output.iterdir())): - raise FileExistsError(f"Output must be absent or an empty directory: {output}") - - -def _materialize_frozen_memory(case: EvalCase, progress: ProgressSink | None) -> Path: - source_path = resolve_dataset(case.source.recording).resolve() - stat = source_path.stat() - stem = re.sub(r"[^A-Za-z0-9_.-]+", "-", source_path.stem)[:64] - mapper = MapperSettings() - raw_key = ( - f"{stem}-{stat.st_size}-{stat.st_mtime_ns}-p{case.source.progress:.9f}-" - f"v{mapper.voxel_size_m}-b{mapper.block_count}-d{mapper.device}-" - f"c{int(mapper.carve_columns)}-f{mapper.frame_id}-e{mapper.emit_every}" - ) - key = re.sub(r"[^A-Za-z0-9_.-]+", "-", raw_key) - bundle = CACHE_DIR / "agent_eval" / "frozen_memory" / key - manifest = bundle / "manifest.v1.json" - if not manifest.is_file(): - emit_progress(progress, StatusProgress(channel="eval", message="preparing memory")) - bundle.parent.mkdir(parents=True, exist_ok=True) - try: - prepare_bundle( - case.source.recording, - [], - bundle, - progress=[case.source.progress], - mapper=mapper, - map_progress=lambda current, total: emit_progress( - progress, - StatusProgress(channel="eval", message=f"mapping {current}/{total} frames"), - ), - ) - except OSError as exc: - concurrent_publish = isinstance(exc, FileExistsError) or exc.errno in { - errno.EEXIST, - errno.ENOTEMPTY, - } - if not concurrent_publish or not manifest.is_file(): - raise - return bundle - - -def _pi_paths() -> tuple[Path, Path]: - package = Path(__file__).resolve().parents[3] / "packages" / "pi-code-policy-extension" - cli = package / "node_modules" / "@earendil-works" / "pi-coding-agent" / "dist" / "cli.js" - extension = package / "dist" / "python-exec.js" - return cli, extension - - -def _agent_prompt(case: EvalCase) -> str: - return ( - f"{case.task.prompt}\n\n" - "Use python_exec to inspect the read-only recording. " - f"End with `{case.task.answer_marker} `." - ) diff --git a/dimos/benchmark/agent_eval/test_single_case.py b/dimos/benchmark/agent_eval/test_single_case.py deleted file mode 100644 index 6b8deb758b..0000000000 --- a/dimos/benchmark/agent_eval/test_single_case.py +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pathlib import Path -from types import SimpleNamespace - -import pytest - -from dimos.benchmark.agent_eval.models import ( - EvalCase, - EvalRunConfig, - ExactIntegerValidatorRef, - FrozenRecordingSource, - IntegerQuestionTask, -) -from dimos.benchmark.agent_eval.pi_process import PiRunResult -import dimos.benchmark.agent_eval.single_case as single_case -from dimos.benchmark.agent_eval.single_case import execute_single_case - - -def _case(tmp_path: Path) -> Path: - private = tmp_path / "private" - private.mkdir() - (private / "oracle.json").write_text( - '{"schema_version":"1.0","expected_count":2,' - '"counting_policy":"count rooms","rooms":[],' - '"reviewed_by":["reviewer"]}' - ) - case = EvalCase( - case_id="demo", - source=FrozenRecordingSource(recording="recording", progress=1.0), - task=IntegerQuestionTask(prompt="How many rooms?"), - validator=ExactIntegerValidatorRef(revision="v1", private_path="private/oracle.json"), - ) - path = tmp_path / "case.json" - path.write_text(case.model_dump_json()) - return path - - -def test_direct_run_publishes_only_compact_result_and_native_transcript( - monkeypatch, tmp_path: Path -) -> None: - case_path = _case(tmp_path) - bundle = tmp_path / "bundle" - bundle.mkdir() - monkeypatch.setenv("OPENAI_API_KEY", "secret") - monkeypatch.setattr(single_case, "_materialize_frozen_memory", lambda *_args: bundle) - monkeypatch.setattr( - single_case, - "load_bundle", - lambda *_args, **_kwargs: ( - object(), - SimpleNamespace(cutoff_timestamp=10.0), - tmp_path / "source.db", - tmp_path / "derived.db", - ), - ) - monkeypatch.setattr(single_case, "_pi_paths", lambda: (case_path, case_path)) - - class Server: - mcp_url = "http://127.0.0.1:1234/mcp" - session = SimpleNamespace(execution_count=3) - - def __init__(self, _config): - pass - - def start(self): - pass - - def stop(self): - pass - - class Runner: - def __init__(self, **_kwargs): - pass - - def run(self, *, run_dir, **_kwargs): - transcript = run_dir / "native.jsonl" - transcript.write_text('{"type":"session"}\n') - return PiRunResult("Checked\nANSWER: 2", 3, 1.0, transcript, "") - - monkeypatch.setattr(single_case, "CodePolicyMcpServer", Server) - monkeypatch.setattr(single_case, "PiCliRunner", Runner) - output = tmp_path / "output" - result = execute_single_case(case_path, config=EvalRunConfig(), output=output) - assert result.passed is True - assert {path.name for path in output.iterdir()} == { - "result.json", - "pi-transcript.jsonl", - } - - -def test_nonempty_output_is_rejected_before_execution(tmp_path: Path) -> None: - output = tmp_path / "output" - output.mkdir() - (output / "keep").write_text("user data") - with pytest.raises(FileExistsError, match="absent or an empty"): - execute_single_case(_case(tmp_path), config=EvalRunConfig(), output=output) - assert (output / "keep").read_text() == "user data" - - -def test_materialize_accepts_bundle_published_by_concurrent_runner( - monkeypatch, tmp_path: Path -) -> None: - recording = tmp_path / "recording.db" - recording.touch() - case = EvalCase( - case_id="concurrent", - source=FrozenRecordingSource(recording=str(recording), progress=1.0), - task=IntegerQuestionTask(prompt="How many rooms?"), - validator=ExactIntegerValidatorRef(revision="v1", private_path="private/oracle.json"), - ) - monkeypatch.setattr(single_case, "CACHE_DIR", tmp_path / "cache") - monkeypatch.setattr(single_case, "resolve_dataset", lambda _recording: recording) - - def publish_first(_recording, _cutoffs, output: Path, **_kwargs) -> None: - output.mkdir() - (output / "manifest.v1.json").write_text("{}") - raise FileExistsError(output) - - monkeypatch.setattr(single_case, "prepare_bundle", publish_first) - - bundle = single_case._materialize_frozen_memory(case, None) - - assert bundle.joinpath("manifest.v1.json").is_file() diff --git a/dimos/benchmark/evaluation/models.py b/dimos/benchmark/evaluation/models.py new file mode 100644 index 0000000000..bd4839041b --- /dev/null +++ b/dimos/benchmark/evaluation/models.py @@ -0,0 +1,138 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Universal contracts for requesting and recording evaluation runs.""" + +from __future__ import annotations + +from datetime import datetime +from pathlib import PurePosixPath +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class EvaluationModel(BaseModel): + """Strict immutable base for persisted evaluation contracts.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class EvaluationReference(EvaluationModel): + name: str = Field(min_length=1) + config: dict[str, Any] = Field(default_factory=dict) + + +class CodePolicyAgentConfig(EvaluationModel): + profile: Literal["code-policy-v1"] = "code-policy-v1" + model: Literal["gpt-5.6-luna"] = "gpt-5.6-luna" + thinking_level: Literal["medium"] = "medium" + + +class EvaluationRunSpecification(EvaluationModel): + schema_version: Literal["1.0"] = "1.0" + evaluation: EvaluationReference + agent: CodePolicyAgentConfig = Field(default_factory=CodePolicyAgentConfig) + + +class ArtifactReference(EvaluationModel): + path: str = Field(min_length=1) + label: str = Field(min_length=1) + media_type: str | None = Field(default=None, min_length=1) + + @model_validator(mode="after") + def path_is_safe_and_relative(self) -> ArtifactReference: + path = PurePosixPath(self.path) + if path.is_absolute() or not path.parts or ".." in path.parts: + raise ValueError("artifact path must be a safe relative POSIX path") + return self + + +SummaryValue = str | int | float | bool | None + + +class SummaryItem(EvaluationModel): + key: str = Field(min_length=1, pattern=r"^[a-z][a-z0-9_]*$") + label: str = Field(min_length=1) + value: SummaryValue + + +class InlineNativeResult(EvaluationModel): + kind: Literal["inline"] = "inline" + value: Any + + +class ArtifactNativeResult(EvaluationModel): + kind: Literal["artifact"] = "artifact" + artifact: ArtifactReference + + +NativeResult = Annotated[ + InlineNativeResult | ArtifactNativeResult, + Field(discriminator="kind"), +] + + +class EvaluationReport(EvaluationModel): + summary: tuple[SummaryItem, ...] = () + native_result: NativeResult + artifacts: tuple[ArtifactReference, ...] = () + + +class EvaluationIdentity(EvaluationModel): + name: str = Field(min_length=1) + provider: str = Field(min_length=1) + version: str = Field(min_length=1) + + +class RuntimeIdentity(EvaluationModel): + profile: Literal["code-policy-v1"] = "code-policy-v1" + driver: Literal["pi"] = "pi" + driver_version: str = Field(min_length=1) + model: str = Field(min_length=1) + thinking_level: str = Field(min_length=1) + + +class EvaluationRunError(EvaluationModel): + stage: Literal["evaluation", "publication"] + error_type: str = Field(min_length=1) + message: str = Field(min_length=1) + + +class EvaluationRun(EvaluationModel): + schema_version: Literal["1.0"] = "1.0" + run_id: str = Field(min_length=1) + specification: EvaluationRunSpecification + evaluation: EvaluationIdentity + runtime: RuntimeIdentity + status: Literal["completed", "failed", "cancelled"] + started_at: datetime + finished_at: datetime + duration_seconds: float = Field(ge=0) + report: EvaluationReport | None = None + error: EvaluationRunError | None = None + runtime_artifacts: tuple[ArtifactReference, ...] = () + prompt_evidence: tuple[ArtifactReference, ...] = () + + @model_validator(mode="after") + def status_matches_payload(self) -> EvaluationRun: + if self.status == "completed" and self.report is None: + raise ValueError("completed evaluation runs require a report") + if self.status != "completed" and self.error is None: + raise ValueError("non-completed evaluation runs require an error") + if self.status == "completed" and self.error is not None: + raise ValueError("completed evaluation runs cannot contain an error") + if self.status != "completed" and self.report is not None: + raise ValueError("non-completed evaluation runs cannot contain a report") + return self diff --git a/dimos/benchmark/agent_eval/pi_process.py b/dimos/benchmark/evaluation/pi_process.py similarity index 99% rename from dimos/benchmark/agent_eval/pi_process.py rename to dimos/benchmark/evaluation/pi_process.py index ecd04336e0..f02fe926c1 100644 --- a/dimos/benchmark/agent_eval/pi_process.py +++ b/dimos/benchmark/evaluation/pi_process.py @@ -25,7 +25,7 @@ import time from typing import Any -from dimos.benchmark.agent_eval.progress import ( +from dimos.benchmark.evaluation.progress import ( AssistantTextProgress, FinalResponseProgress, ProgressSink, @@ -106,7 +106,7 @@ def run( "--session-dir", str(session_dir), "--name", - "dimos-frozen-eval", + "dimos-evaluation", "--no-builtin-tools", "--tools", "python_exec", diff --git a/dimos/benchmark/agent_eval/progress.py b/dimos/benchmark/evaluation/progress.py similarity index 100% rename from dimos/benchmark/agent_eval/progress.py rename to dimos/benchmark/evaluation/progress.py diff --git a/dimos/benchmark/evaluation/protocol.py b/dimos/benchmark/evaluation/protocol.py new file mode 100644 index 0000000000..1f6c61a224 --- /dev/null +++ b/dimos/benchmark/evaluation/protocol.py @@ -0,0 +1,68 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The complete Evaluation extension point.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Protocol, runtime_checkable + +from pydantic import BaseModel + +from dimos.benchmark.evaluation.models import EvaluationReport +from dimos.benchmark.evaluation.progress import ProgressSink + + +@runtime_checkable +class CodePolicyRuntime(Protocol): + """Factory supplied to evaluations for evaluation-owned agent sessions.""" + + def open_session(self, environment: BaseModel) -> CodePolicySessionHandle: ... + + +@runtime_checkable +class CodePolicySessionHandle(Protocol): + def __enter__(self) -> CodePolicySessionHandle: ... + + def __exit__(self, *args: object) -> None: ... + + def run(self, *, evaluation_protocol: str, task_input: str) -> AgentOutcome: ... + + +@dataclass(frozen=True) +class AgentOutcome: + final_text: str + tool_call_count: int + duration_seconds: float + + +@dataclass(frozen=True) +class EvaluationContext: + run_id: str + spec_dir: Path + workspace: Path + agent: CodePolicyRuntime + progress: ProgressSink | None + + +@runtime_checkable +class Evaluation(Protocol): + """A complete executable evaluation with native result semantics.""" + + name: str + config_model: type[BaseModel] + + def run(self, config: BaseModel, context: EvaluationContext) -> EvaluationReport: ... diff --git a/dimos/benchmark/evaluation/registry.py b/dimos/benchmark/evaluation/registry.py new file mode 100644 index 0000000000..a6608daf67 --- /dev/null +++ b/dimos/benchmark/evaluation/registry.py @@ -0,0 +1,136 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Lazy built-in and installed-package Evaluation discovery.""" + +from __future__ import annotations + +from dataclasses import dataclass +import importlib +import importlib.metadata as importlib_metadata +import re +from typing import Any + +from packaging.utils import canonicalize_name +from pydantic import BaseModel + +from dimos.benchmark.evaluation.protocol import Evaluation + +ENTRY_POINT_GROUP = "dimos.evaluations" +LOCAL_NAME_PATTERN = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") +BUILTIN_EVALUATIONS = { + "frozen-integer-qa": ("dimos.benchmark.short_horizon_qa.evaluation:frozen_integer_qa"), +} + + +class EvaluationRegistryError(ValueError): + """An Evaluation name or plugin could not be resolved.""" + + +@dataclass(frozen=True) +class ResolvedEvaluation: + name: str + provider: str + version: str + evaluation: Evaluation + + +def available_evaluations() -> list[str]: + return sorted([*BUILTIN_EVALUATIONS, *_external_entries()]) + + +def resolve_evaluation(name: str) -> ResolvedEvaluation: + if name in BUILTIN_EVALUATIONS: + target = _load_target(BUILTIN_EVALUATIONS[name], name) + return ResolvedEvaluation( + name=name, + provider="dimos", + version=_distribution_version("dimos"), + evaluation=_validate_target(name, target), + ) + + entries = _external_entries() + entry = entries.get(name) + if entry is None: + available = available_evaluations() + suffix = f" Available evaluations: {', '.join(available)}." if available else "" + raise EvaluationRegistryError(f"Unknown evaluation {name!r}.{suffix}") + try: + target = entry.load() + except Exception as exc: + raise EvaluationRegistryError( + f"Failed to load evaluation {name!r} from {entry.value!r}: {type(exc).__name__}: {exc}" + ) from exc + distribution = entry.dist + assert distribution is not None + distribution_name = distribution.metadata["Name"] + return ResolvedEvaluation( + name=name, + provider=distribution_name, + version=distribution.version, + evaluation=_validate_target(name, target), + ) + + +def _external_entries() -> dict[str, importlib_metadata.EntryPoint]: + result: dict[str, importlib_metadata.EntryPoint] = {} + for entry in importlib_metadata.entry_points(group=ENTRY_POINT_GROUP): + distribution = entry.dist + if distribution is None: + continue + distribution_name = distribution.metadata.get("Name") + if not distribution_name or LOCAL_NAME_PATTERN.fullmatch(entry.name) is None: + continue + namespace = str(canonicalize_name(distribution_name)) + qualified_name = f"{namespace}.{entry.name}" + if qualified_name in result: + raise EvaluationRegistryError( + f"Multiple installed entry points provide evaluation {qualified_name!r}" + ) + result[qualified_name] = entry + return result + + +def _load_target(path: str, name: str) -> Any: + module_name, separator, attribute = path.partition(":") + if not separator: + raise EvaluationRegistryError(f"Invalid built-in evaluation target for {name!r}: {path}") + try: + return getattr(importlib.import_module(module_name), attribute) + except Exception as exc: + raise EvaluationRegistryError( + f"Failed to load evaluation {name!r} from {path!r}: {type(exc).__name__}: {exc}" + ) from exc + + +def _validate_target(name: str, target: Any) -> Evaluation: + if not isinstance(target, Evaluation): + raise EvaluationRegistryError( + f"Evaluation {name!r} must expose name, config_model, and run()" + ) + config_model = target.config_model + if not isinstance(config_model, type) or not issubclass(config_model, BaseModel): + raise EvaluationRegistryError( + f"Evaluation {name!r} config_model must be a Pydantic BaseModel type" + ) + if target.name != name.rpartition(".")[2]: + raise EvaluationRegistryError(f"Evaluation {name!r} loaded a target named {target.name!r}") + return target + + +def _distribution_version(name: str) -> str: + try: + return importlib_metadata.version(name) + except importlib_metadata.PackageNotFoundError: + return "source" diff --git a/dimos/benchmark/evaluation/runner.py b/dimos/benchmark/evaluation/runner.py new file mode 100644 index 0000000000..3bef855ae2 --- /dev/null +++ b/dimos/benchmark/evaluation/runner.py @@ -0,0 +1,142 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Resolve, execute, and atomically publish one Evaluation Run.""" + +from __future__ import annotations + +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import shutil +import tempfile +import time +from typing import Literal +from uuid import uuid4 + +from dimos.benchmark.evaluation.models import ( + EvaluationIdentity, + EvaluationRun, + EvaluationRunError, + EvaluationRunSpecification, +) +from dimos.benchmark.evaluation.progress import ProgressSink, StatusProgress, emit_progress +from dimos.benchmark.evaluation.protocol import EvaluationContext +from dimos.benchmark.evaluation.registry import resolve_evaluation +from dimos.benchmark.evaluation.runtime import CodePolicyRuntimeFactory + + +def execute_evaluation( + specification_path: Path, + *, + output: Path, + api_key_env: str = "OPENAI_API_KEY", + progress: ProgressSink | None = None, +) -> EvaluationRun: + """Run one resolved Evaluation and publish its immutable record.""" + specification_path = specification_path.expanduser().resolve() + output = output.expanduser().resolve() + _validate_output(output) + specification = EvaluationRunSpecification.model_validate_json(specification_path.read_bytes()) + resolved = resolve_evaluation(specification.evaluation.name) + config = resolved.evaluation.config_model.model_validate_json( + json.dumps(specification.evaluation.config), + strict=True, + ) + api_key = os.environ.get(api_key_env) + if not api_key: + raise ValueError(f"API key environment variable {api_key_env!r} is unset") + + output.parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix=f".{output.name}-", dir=output.parent)) + run_id = str(uuid4()) + started_at = datetime.now(timezone.utc) + started = time.monotonic() + runtime = CodePolicyRuntimeFactory( + config=specification.agent, + api_key=api_key, + workspace=temporary, + progress=progress, + ) + context = EvaluationContext( + run_id=run_id, + spec_dir=specification_path.parent, + workspace=temporary, + agent=runtime, + progress=progress, + ) + emit_progress(progress, StatusProgress(channel="eval", message="evaluation started")) + try: + status: Literal["completed", "failed", "cancelled"] + try: + report = resolved.evaluation.run(config, context) + status = "completed" + error = None + except KeyboardInterrupt: + report = None + status = "cancelled" + error = EvaluationRunError( + stage="evaluation", + error_type="KeyboardInterrupt", + message="Evaluation cancelled by user", + ) + except Exception as exc: + report = None + status = "failed" + error = EvaluationRunError( + stage="evaluation", + error_type=type(exc).__name__, + message=_redact_error(str(exc) or type(exc).__name__, api_key), + ) + finished_at = datetime.now(timezone.utc) + run = EvaluationRun( + run_id=run_id, + specification=specification, + evaluation=EvaluationIdentity( + name=resolved.name, + provider=resolved.provider, + version=resolved.version, + ), + runtime=runtime.identity, + status=status, + started_at=started_at, + finished_at=finished_at, + duration_seconds=time.monotonic() - started, + report=report, + error=error, + runtime_artifacts=runtime.runtime_artifacts, + prompt_evidence=runtime.prompt_evidence, + ) + (temporary / "run.json").write_text( + run.model_dump_json(indent=2) + "\n", + encoding="utf-8", + ) + if output.exists(): + output.rmdir() + os.replace(temporary, output) + emit_progress(progress, StatusProgress(channel="eval", message="run published")) + return run + finally: + if temporary.exists(): + shutil.rmtree(temporary) + + +def _validate_output(output: Path) -> None: + if output.exists() and (not output.is_dir() or any(output.iterdir())): + raise FileExistsError(f"Output must be absent or an empty directory: {output}") + + +def _redact_error(message: str, api_key: str) -> str: + return message.replace(api_key, "[REDACTED]") diff --git a/dimos/benchmark/evaluation/runtime.py b/dimos/benchmark/evaluation/runtime.py new file mode 100644 index 0000000000..fc29befaff --- /dev/null +++ b/dimos/benchmark/evaluation/runtime.py @@ -0,0 +1,254 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The versioned Pi implementation of the CodePolicy agent runtime.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import shutil + +from pydantic import BaseModel + +from dimos.agents.code_policy_core import ( + CodePolicyEnvironment, + CodePolicySessionConfig, + FrozenMemoryEnvironment, + LiveDimosEnvironment, +) +from dimos.agents.code_policy_server import CodePolicyMcpServer +from dimos.benchmark.evaluation.models import ( + ArtifactReference, + CodePolicyAgentConfig, + RuntimeIdentity, +) +from dimos.benchmark.evaluation.pi_process import PI_VERSION, PiCliRunner, PiRunError +from dimos.benchmark.evaluation.progress import ProgressSink +from dimos.benchmark.evaluation.protocol import AgentOutcome + +CODE_POLICY_PROFILE = "code-policy-v1" +TURN_TIMEOUT_SECONDS = 600.0 +SYSTEM_INSTRUCTIONS = """You are a CodePolicy agent. + +Use the single `python_exec` tool to solve the supplied task. Python executes in a +persistent trusted, unsandboxed environment, so imports, variables, and functions +persist between tool calls. Follow the evaluation protocol exactly. +""" + + +class CodePolicyRuntimeFactory: + """Create evaluation-owned sessions with one fixed CodePolicy profile.""" + + def __init__( + self, + *, + config: CodePolicyAgentConfig, + api_key: str, + workspace: Path, + progress: ProgressSink | None = None, + ) -> None: + self.config = config + self.api_key = api_key + self.workspace = workspace + self.progress = progress + self._session_count = 0 + self._prompt_evidence: list[ArtifactReference] = [] + self._runtime_artifacts: list[ArtifactReference] = [] + + @property + def identity(self) -> RuntimeIdentity: + return RuntimeIdentity( + profile=self.config.profile, + driver_version=PI_VERSION, + model=self.config.model, + thinking_level=self.config.thinking_level, + ) + + @property + def prompt_evidence(self) -> tuple[ArtifactReference, ...]: + return tuple(self._prompt_evidence) + + @property + def runtime_artifacts(self) -> tuple[ArtifactReference, ...]: + return tuple(self._runtime_artifacts) + + def open_session(self, environment: BaseModel) -> CodePolicyRuntimeSession: + if not isinstance(environment, (FrozenMemoryEnvironment, LiveDimosEnvironment)): + raise TypeError(f"Unsupported CodePolicy environment: {type(environment).__name__}") + self._session_count += 1 + session_path = Path("runtime") / f"session-{self._session_count:04d}" + return CodePolicyRuntimeSession( + factory=self, + environment=environment, + relative_path=session_path, + ) + + def _record_prompt_evidence(self, references: list[ArtifactReference]) -> None: + self._prompt_evidence.extend(references) + self._runtime_artifacts.extend(references) + + def _record_runtime_artifact(self, reference: ArtifactReference) -> None: + self._runtime_artifacts.append(reference) + + +class CodePolicyRuntimeSession: + """One lifecycle-bounded, single-turn CodePolicy interaction.""" + + def __init__( + self, + *, + factory: CodePolicyRuntimeFactory, + environment: CodePolicyEnvironment, + relative_path: Path, + ) -> None: + self.factory = factory + self.environment = environment + self.relative_path = relative_path + self.path = factory.workspace / relative_path + self.server: CodePolicyMcpServer | None = None + self._ran = False + + def __enter__(self) -> CodePolicyRuntimeSession: + self.path.mkdir(parents=True) + self.server = CodePolicyMcpServer(CodePolicySessionConfig(environment=self.environment)) + self.server.start() + return self + + def __exit__(self, *_args: object) -> None: + if self.server is not None: + self.server.stop() + self.server = None + shutil.rmtree(self.path / "working", ignore_errors=True) + + def run(self, *, evaluation_protocol: str, task_input: str) -> AgentOutcome: + if self.server is None: + raise RuntimeError("CodePolicy session must be entered before run()") + if self._ran: + raise RuntimeError("code-policy-v1 sessions accept exactly one initial turn") + if not evaluation_protocol.strip() or not task_input.strip(): + raise ValueError("evaluation protocol and task input must be non-empty") + self._ran = True + + user_message = _assemble_user_message(evaluation_protocol, task_input) + evidence = self._write_prompt_evidence(evaluation_protocol, task_input, user_message) + self.factory._record_prompt_evidence(evidence) + working = self.path / "working" + working.mkdir() + cli, extension = _pi_paths() + runner = PiCliRunner( + cli=cli, + extension=extension, + model=self.factory.config.model, + thinking_level=self.factory.config.thinking_level, + timeout_s=TURN_TIMEOUT_SECONDS, + progress=self.factory.progress, + ) + try: + result = runner.run( + prompt=user_message, + system_prompt=SYSTEM_INSTRUCTIONS, + mcp_url=self.server.mcp_url, + api_key=self.factory.api_key, + run_dir=working, + ) + except PiRunError as exc: + self._record_stderr(exc.stderr) + raise + if result.transcript_path is not None: + target = self.path / "pi-transcript.jsonl" + shutil.copy2(result.transcript_path, target) + self.factory._record_runtime_artifact( + self._artifact(target, "Pi transcript", "application/x-ndjson") + ) + if result.stderr: + self._record_stderr(result.stderr) + return AgentOutcome( + final_text=result.final_text, + tool_call_count=result.tool_call_count, + duration_seconds=result.duration_seconds, + ) + + def _record_stderr(self, stderr: str) -> None: + if not stderr: + return + target = self.path / "stderr.log" + target.write_text(stderr, encoding="utf-8") + self.factory._record_runtime_artifact(self._artifact(target, "Pi stderr", "text/plain")) + + def _write_prompt_evidence( + self, + evaluation_protocol: str, + task_input: str, + user_message: str, + ) -> list[ArtifactReference]: + components = ( + ("runtime-system.txt", "runtime", SYSTEM_INSTRUCTIONS), + ("evaluation-protocol.txt", "evaluation", evaluation_protocol), + ("task-input.txt", "evaluation", task_input), + ("assembled-user-message.txt", "runtime", user_message), + ) + manifest_components: list[dict[str, str]] = [] + references: list[ArtifactReference] = [] + for filename, owner, text in components: + path = self.path / filename + path.write_text(text, encoding="utf-8") + manifest_components.append( + { + "path": path.relative_to(self.factory.workspace).as_posix(), + "owner": owner, + "sha256": hashlib.sha256(text.encode()).hexdigest(), + } + ) + references.append(self._artifact(path, filename, "text/plain")) + manifest = self.path / "prompt-assembly.json" + manifest.write_text( + json.dumps( + { + "schema_version": "1.0", + "runtime_profile": CODE_POLICY_PROFILE, + "components": manifest_components, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + references.append(self._artifact(manifest, "Prompt assembly", "application/json")) + return references + + def _artifact(self, path: Path, label: str, media_type: str) -> ArtifactReference: + return ArtifactReference( + path=path.relative_to(self.factory.workspace).as_posix(), + label=label, + media_type=media_type, + ) + + +def _assemble_user_message(evaluation_protocol: str, task_input: str) -> str: + return ( + "# Evaluation protocol\n\n" + f"{evaluation_protocol.strip()}\n\n" + "# Task input\n\n" + f"{task_input.strip()}\n" + ) + + +def _pi_paths() -> tuple[Path, Path]: + package = Path(__file__).resolve().parents[3] / "packages" / "pi-code-policy-extension" + cli = package / "node_modules" / "@earendil-works" / "pi-coding-agent" / "dist" / "cli.js" + extension = package / "dist" / "python-exec.js" + return cli, extension diff --git a/dimos/benchmark/evaluation/test_models.py b/dimos/benchmark/evaluation/test_models.py new file mode 100644 index 0000000000..447d4ab675 --- /dev/null +++ b/dimos/benchmark/evaluation/test_models.py @@ -0,0 +1,64 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime, timezone + +from pydantic import ValidationError +import pytest + +from dimos.benchmark.evaluation.models import ( + ArtifactReference, + EvaluationIdentity, + EvaluationReference, + EvaluationReport, + EvaluationRun, + EvaluationRunSpecification, + InlineNativeResult, + RuntimeIdentity, +) + + +def test_artifact_reference_rejects_paths_outside_run() -> None: + with pytest.raises(ValidationError, match="safe relative"): + ArtifactReference(path="../private.json", label="Private") + + +def test_completed_run_requires_evaluation_report() -> None: + now = datetime.now(timezone.utc) + + with pytest.raises(ValidationError, match="require a report"): + EvaluationRun( + run_id="run", + specification=EvaluationRunSpecification( + evaluation=EvaluationReference(name="fixture") + ), + evaluation=EvaluationIdentity(name="fixture", provider="tests", version="1"), + runtime=RuntimeIdentity( + driver_version="test", + model="gpt-5.6-luna", + thinking_level="medium", + ), + status="completed", + started_at=now, + finished_at=now, + duration_seconds=0, + ) + + +def test_native_result_preserves_nested_benchmark_payload() -> None: + payload = {"metrics": {"success_rate": 0.75}, "episodes": [True, False]} + + report = EvaluationReport(native_result=InlineNativeResult(value=payload)) + + assert report.model_dump(mode="json")["native_result"]["value"] == payload diff --git a/dimos/benchmark/agent_eval/test_pi_process.py b/dimos/benchmark/evaluation/test_pi_process.py similarity index 96% rename from dimos/benchmark/agent_eval/test_pi_process.py rename to dimos/benchmark/evaluation/test_pi_process.py index b7de377df9..2eaa9bedbf 100644 --- a/dimos/benchmark/agent_eval/test_pi_process.py +++ b/dimos/benchmark/evaluation/test_pi_process.py @@ -20,7 +20,7 @@ import pytest -from dimos.benchmark.agent_eval.pi_process import PiCliRunner, PiRunError, parse_pi_events +from dimos.benchmark.evaluation.pi_process import PiCliRunner, PiRunError, parse_pi_events def test_parse_stock_pi_events_uses_final_message_and_counts_tools() -> None: @@ -80,7 +80,7 @@ def test_stock_cli_streams_assistant_tools_and_stderr_while_running(mocker, tmp_ ) ) process.stderr = StringIO("provider secret connected\n") - mocker.patch("dimos.benchmark.agent_eval.pi_process.subprocess.Popen", return_value=process) + mocker.patch("dimos.benchmark.evaluation.pi_process.subprocess.Popen", return_value=process) progress = [] assistant_seen = threading.Event() @@ -152,7 +152,7 @@ def test_stock_cli_receives_only_api_key_and_evaluator_binding(mocker, tmp_path: process.stderr = StringIO() process.wait.return_value = 0 popen = mocker.patch( - "dimos.benchmark.agent_eval.pi_process.subprocess.Popen", return_value=process + "dimos.benchmark.evaluation.pi_process.subprocess.Popen", return_value=process ) runner = PiCliRunner( cli=cli, @@ -191,7 +191,7 @@ def test_stock_cli_timeout_terminates_the_child(mocker, tmp_path: Path) -> None: subprocess.TimeoutExpired("pi", 0.01), 0, ] - mocker.patch("dimos.benchmark.agent_eval.pi_process.subprocess.Popen", return_value=process) + mocker.patch("dimos.benchmark.evaluation.pi_process.subprocess.Popen", return_value=process) runner = PiCliRunner( cli=cli, extension=extension, diff --git a/dimos/benchmark/evaluation/test_registry.py b/dimos/benchmark/evaluation/test_registry.py new file mode 100644 index 0000000000..0c7b164d87 --- /dev/null +++ b/dimos/benchmark/evaluation/test_registry.py @@ -0,0 +1,98 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pydantic import BaseModel +import pytest + +from dimos.benchmark.evaluation.models import EvaluationReport, InlineNativeResult +from dimos.benchmark.evaluation.protocol import EvaluationContext +import dimos.benchmark.evaluation.registry as registry + + +class Config(BaseModel): + value: int + + +class PluginEvaluation: + name = "sample" + config_model: type[BaseModel] = Config + + def run(self, config: BaseModel, context: EvaluationContext) -> EvaluationReport: + return EvaluationReport(native_result=InlineNativeResult(value=None)) + + +class Distribution: + metadata = {"Name": "Acme_Evals"} + version = "2.0" + + +class EntryPoint: + name = "sample" + value = "acme.evals:sample" + dist = Distribution() + + def __init__(self, target) -> None: + self.target = target + + def load(self): + return self.target + + +def test_external_evaluation_uses_distribution_namespace(monkeypatch) -> None: + entry = EntryPoint(PluginEvaluation()) + monkeypatch.setattr( + registry.importlib_metadata, + "entry_points", + lambda **_kwargs: [entry], + ) + + resolved = registry.resolve_evaluation("acme-evals.sample") + + assert resolved.provider == "Acme_Evals" + assert resolved.version == "2.0" + assert resolved.evaluation is entry.target + + +def test_unknown_evaluation_lists_available_names(monkeypatch) -> None: + monkeypatch.setattr( + registry.importlib_metadata, + "entry_points", + lambda **_kwargs: [], + ) + + with pytest.raises(registry.EvaluationRegistryError, match="frozen-integer-qa"): + registry.resolve_evaluation("missing") + + +def test_external_target_must_implement_whole_evaluation(monkeypatch) -> None: + entry = EntryPoint(object()) + monkeypatch.setattr( + registry.importlib_metadata, + "entry_points", + lambda **_kwargs: [entry], + ) + + with pytest.raises(registry.EvaluationRegistryError, match="name, config_model, and run"): + registry.resolve_evaluation("acme-evals.sample") + + +def test_duplicate_external_evaluation_names_are_rejected(monkeypatch) -> None: + monkeypatch.setattr( + registry.importlib_metadata, + "entry_points", + lambda **_kwargs: [EntryPoint(PluginEvaluation()), EntryPoint(PluginEvaluation())], + ) + + with pytest.raises(registry.EvaluationRegistryError, match="Multiple installed"): + registry.available_evaluations() diff --git a/dimos/benchmark/evaluation/test_runner.py b/dimos/benchmark/evaluation/test_runner.py new file mode 100644 index 0000000000..d89931bda4 --- /dev/null +++ b/dimos/benchmark/evaluation/test_runner.py @@ -0,0 +1,188 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from pathlib import Path + +from pydantic import BaseModel, ConfigDict +import pytest + +from dimos.benchmark.evaluation.models import ( + CodePolicyAgentConfig, + EvaluationReport, + InlineNativeResult, + RuntimeIdentity, + SummaryItem, +) +from dimos.benchmark.evaluation.protocol import AgentOutcome, EvaluationContext +from dimos.benchmark.evaluation.registry import ResolvedEvaluation +import dimos.benchmark.evaluation.runner as runner + + +class HarnessConfig(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + samples: tuple[int, ...] + + +class Environment(BaseModel): + sample: int + + +class NativeHarnessEvaluation: + name = "native-harness" + config_model: type[BaseModel] = HarnessConfig + + def run(self, config: BaseModel, context: EvaluationContext) -> EvaluationReport: + assert isinstance(config, HarnessConfig) + predictions = [] + for sample in config.samples: + with context.agent.open_session(Environment(sample=sample)) as session: + outcome = session.run( + evaluation_protocol="Return the native benchmark answer.", + task_input=str(sample), + ) + predictions.append(int(outcome.final_text)) + native = { + "benchmark": "fixture", + "predictions": predictions, + "aggregate": {"sum": sum(predictions)}, + } + return EvaluationReport( + summary=(SummaryItem(key="native_sum", label="Native sum", value=sum(predictions)),), + native_result=InlineNativeResult(value=native), + ) + + +class FailingEvaluation: + name = "native-harness" + config_model: type[BaseModel] = HarnessConfig + + def run(self, config: BaseModel, context: EvaluationContext) -> EvaluationReport: + raise RuntimeError("credential=secret") + + +class FakeSession: + def __init__(self, sample: int) -> None: + self.sample = sample + + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def run(self, *, evaluation_protocol: str, task_input: str) -> AgentOutcome: + assert evaluation_protocol == "Return the native benchmark answer." + assert task_input == str(self.sample) + return AgentOutcome(str(self.sample * 2), 1, 0.01) + + +class FakeRuntime: + def __init__(self, *, config: CodePolicyAgentConfig, **_kwargs) -> None: + self.config = config + self.prompt_evidence = () + self.runtime_artifacts = () + + @property + def identity(self) -> RuntimeIdentity: + return RuntimeIdentity( + driver_version="test", + model=self.config.model, + thinking_level=self.config.thinking_level, + ) + + def open_session(self, environment: BaseModel) -> FakeSession: + assert isinstance(environment, Environment) + return FakeSession(environment.sample) + + +def _write_spec(tmp_path: Path) -> Path: + path = tmp_path / "spec.json" + path.write_text( + json.dumps( + { + "schema_version": "1.0", + "evaluation": { + "name": "native-harness", + "config": {"samples": [2, 3]}, + }, + "agent": {"profile": "code-policy-v1"}, + } + ) + ) + return path + + +def test_native_harness_owns_loop_scoring_and_aggregation(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "secret") + monkeypatch.setattr(runner, "CodePolicyRuntimeFactory", FakeRuntime) + monkeypatch.setattr( + runner, + "resolve_evaluation", + lambda _name: ResolvedEvaluation( + name="native-harness", + provider="fixture", + version="1", + evaluation=NativeHarnessEvaluation(), + ), + ) + output = tmp_path / "output" + + result = runner.execute_evaluation(_write_spec(tmp_path), output=output) + + assert result.status == "completed" + assert result.report is not None + assert result.report.native_result.value == { + "benchmark": "fixture", + "predictions": [4, 6], + "aggregate": {"sum": 10}, + } + assert json.loads((output / "run.json").read_text())["status"] == "completed" + + +def test_nonempty_output_is_rejected_before_execution(tmp_path: Path) -> None: + output = tmp_path / "output" + output.mkdir() + (output / "keep").write_text("user data") + + with pytest.raises(FileExistsError, match="absent or an empty"): + runner.execute_evaluation(_write_spec(tmp_path), output=output) + + assert (output / "keep").read_text() == "user data" + + +def test_started_failure_is_published_with_credentials_redacted( + monkeypatch, + tmp_path: Path, +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "secret") + monkeypatch.setattr(runner, "CodePolicyRuntimeFactory", FakeRuntime) + monkeypatch.setattr( + runner, + "resolve_evaluation", + lambda _name: ResolvedEvaluation( + name="native-harness", + provider="fixture", + version="1", + evaluation=FailingEvaluation(), + ), + ) + output = tmp_path / "output" + + result = runner.execute_evaluation(_write_spec(tmp_path), output=output) + + assert result.status == "failed" + assert result.error is not None + assert result.error.message == "credential=[REDACTED]" + assert "secret" not in (output / "run.json").read_text() diff --git a/dimos/benchmark/evaluation/test_runtime.py b/dimos/benchmark/evaluation/test_runtime.py new file mode 100644 index 0000000000..77c6493b8e --- /dev/null +++ b/dimos/benchmark/evaluation/test_runtime.py @@ -0,0 +1,117 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import json +from pathlib import Path + +import pytest + +from dimos.agents.code_policy_core import FrozenMemoryEnvironment +from dimos.benchmark.evaluation.models import CodePolicyAgentConfig +from dimos.benchmark.evaluation.pi_process import PiRunError, PiRunResult +import dimos.benchmark.evaluation.runtime as runtime + + +class FakeServer: + mcp_url = "http://127.0.0.1:1234/mcp" + + def __init__(self, config) -> None: + self.config = config + self.started = False + + def start(self) -> None: + self.started = True + + def stop(self) -> None: + self.started = False + + +class FakeRunner: + def __init__(self, **kwargs) -> None: + self.kwargs = kwargs + + def run(self, *, run_dir: Path, prompt: str, system_prompt: str, **_kwargs): + transcript = run_dir / "native.jsonl" + transcript.write_text('{"type":"session"}\n') + return PiRunResult("ANSWER: 2", 3, 1.0, transcript, "") + + +class FailingRunner(FakeRunner): + def run(self, **_kwargs): + raise PiRunError("Pi failed", stderr="diagnostic") + + +def test_runtime_records_separate_prompt_components(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setattr(runtime, "CodePolicyMcpServer", FakeServer) + monkeypatch.setattr(runtime, "PiCliRunner", FakeRunner) + marker = tmp_path / "exists" + marker.touch() + monkeypatch.setattr(runtime, "_pi_paths", lambda: (marker, marker)) + factory = runtime.CodePolicyRuntimeFactory( + config=CodePolicyAgentConfig(), + api_key="secret", + workspace=tmp_path, + ) + environment = FrozenMemoryEnvironment( + recording_path="source.db", + derived_recording_path="derived.db", + memory_cutoff_timestamp=1.0, + ) + + with factory.open_session(environment) as session: + outcome = session.run( + evaluation_protocol="End with ANSWER: .", + task_input="How many rooms?", + ) + + assert outcome.final_text == "ANSWER: 2" + session_path = tmp_path / "runtime" / "session-0001" + assert (session_path / "evaluation-protocol.txt").read_text() == ("End with ANSWER: .") + assert (session_path / "task-input.txt").read_text() == "How many rooms?" + assembly = json.loads((session_path / "prompt-assembly.json").read_text()) + task = next(item for item in assembly["components"] if item["path"].endswith("task-input.txt")) + assert task["owner"] == "evaluation" + assert task["sha256"] == hashlib.sha256(b"How many rooms?").hexdigest() + assert not (session_path / "working").exists() + assert {item.path for item in factory.prompt_evidence} >= { + "runtime/session-0001/runtime-system.txt", + "runtime/session-0001/evaluation-protocol.txt", + "runtime/session-0001/task-input.txt", + } + + +def test_runtime_retains_pi_stderr_on_failure(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setattr(runtime, "CodePolicyMcpServer", FakeServer) + monkeypatch.setattr(runtime, "PiCliRunner", FailingRunner) + marker = tmp_path / "exists" + marker.touch() + monkeypatch.setattr(runtime, "_pi_paths", lambda: (marker, marker)) + factory = runtime.CodePolicyRuntimeFactory( + config=CodePolicyAgentConfig(), + api_key="secret", + workspace=tmp_path, + ) + environment = FrozenMemoryEnvironment( + recording_path="source.db", + derived_recording_path="derived.db", + memory_cutoff_timestamp=1.0, + ) + + with pytest.raises(PiRunError, match="Pi failed"): + with factory.open_session(environment) as session: + session.run(evaluation_protocol="Use memory.", task_input="Question") + + assert (tmp_path / "runtime/session-0001/stderr.log").read_text() == "diagnostic" + assert factory.runtime_artifacts[-1].path == "runtime/session-0001/stderr.log" diff --git a/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/README.md b/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/README.md index 823d39aa3c..52ac5c7a0a 100644 --- a/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/README.md +++ b/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/README.md @@ -10,5 +10,6 @@ failed task score as an agent or mapping regression. The authoritative case remains incomplete until a human-authored room inventory, counting policy, and independent review establish the expected count. -Use this case to exercise the direct stock-Pi CLI path. Any observed answer is -experimental until the oracle is replaced with a reviewed room inventory. +Use `run.json` to exercise the direct stock-Pi CLI path; it references the +Evaluation-owned `case.json`. Any observed answer is experimental until the +oracle is replaced with a reviewed room inventory. diff --git a/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/run.json b/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/run.json new file mode 100644 index 0000000000..20d64fef70 --- /dev/null +++ b/dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/run.json @@ -0,0 +1,14 @@ +{ + "schema_version": "1.0", + "evaluation": { + "name": "frozen-integer-qa", + "config": { + "case": "case.json" + } + }, + "agent": { + "profile": "code-policy-v1", + "model": "gpt-5.6-luna", + "thinking_level": "medium" + } +} diff --git a/dimos/benchmark/short_horizon_qa/evaluation.py b/dimos/benchmark/short_horizon_qa/evaluation.py new file mode 100644 index 0000000000..90f6369307 --- /dev/null +++ b/dimos/benchmark/short_horizon_qa/evaluation.py @@ -0,0 +1,195 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Frozen integer QA as one complete built-in Evaluation.""" + +from __future__ import annotations + +import errno +from pathlib import Path +import re +import time + +from openevals.exact import exact_match +from pydantic import BaseModel + +from dimos.agents.code_policy_core import FrozenMemoryEnvironment +from dimos.benchmark.evaluation.models import ( + EvaluationReport, + InlineNativeResult, + SummaryItem, +) +from dimos.benchmark.evaluation.progress import ( + CaseHeaderProgress, + StatusProgress, + emit_progress, +) +from dimos.benchmark.evaluation.protocol import EvaluationContext +from dimos.benchmark.short_horizon_qa.integer_answer import ( + load_exact_integer_oracle, + parse_integer_prediction, +) +from dimos.benchmark.short_horizon_qa.models import ( + FrozenIntegerQaCase, + FrozenIntegerQaConfig, + MapperSettings, +) +from dimos.benchmark.short_horizon_qa.prepare import prepare_bundle +from dimos.benchmark.short_horizon_qa.service import load_bundle +from dimos.constants import CACHE_DIR +from dimos.memory2.cli.dataset import resolve_dataset + +EVALUATION_PROTOCOL = """Use `python_exec` to inspect the read-only `memory` object +for the frozen robot recording. Compute the requested integer from the recording; +do not guess. End with exactly one line in this form: + +ANSWER: +""" + + +class FrozenIntegerQaEvaluation: + name = "frozen-integer-qa" + config_model: type[BaseModel] = FrozenIntegerQaConfig + + def run(self, config: BaseModel, context: EvaluationContext) -> EvaluationReport: + if not isinstance(config, FrozenIntegerQaConfig): + raise TypeError("frozen-integer-qa received the wrong configuration type") + case_path = Path(config.case).expanduser() + if not case_path.is_absolute(): + case_path = context.spec_dir / case_path + case_path = case_path.resolve() + case = FrozenIntegerQaCase.model_validate_json(case_path.read_bytes()) + oracle = load_exact_integer_oracle(case, case_path.parent) + emit_progress( + context.progress, + CaseHeaderProgress( + case_id=case.case_id, + source=case.source.recording, + progress=case.source.progress, + question=case.task.prompt, + ), + ) + bundle = _materialize_frozen_memory(case, context) + _, cutoff, source_path, derived_path = load_bundle( + bundle, + progress=case.source.progress, + ) + emit_progress( + context.progress, + StatusProgress(channel="eval", message="memory ready"), + ) + started = time.monotonic() + with context.agent.open_session( + FrozenMemoryEnvironment( + recording_path=str(source_path), + derived_recording_path=str(derived_path), + memory_cutoff_timestamp=cutoff.cutoff_timestamp, + ) + ) as session: + outcome = session.run( + evaluation_protocol=EVALUATION_PROTOCOL, + task_input=case.task.prompt, + ) + prediction = parse_integer_prediction(outcome.final_text) + native_result = exact_match( + outputs={ + "status": prediction.status, + "integer_answer": prediction.integer_answer, + }, + reference_outputs={ + "status": "parsed", + "integer_answer": oracle.expected_count, + }, + ) + return EvaluationReport( + summary=( + SummaryItem(key="case", label="Case", value=case.case_id), + SummaryItem( + key="recording", + label="Recording", + value=f"{case.source.recording} @ {case.source.progress * 100:g}%", + ), + SummaryItem( + key="answer", + label="Answer", + value=prediction.integer_answer, + ), + SummaryItem( + key="exact_match", + label="Exact match", + value=bool(native_result["score"]), + ), + SummaryItem( + key="tool_calls", + label="Tool calls", + value=outcome.tool_call_count, + ), + SummaryItem( + key="duration", + label="Duration", + value=f"{time.monotonic() - started:.1f}s", + ), + ), + native_result=InlineNativeResult(value=native_result), + ) + + +def _materialize_frozen_memory( + case: FrozenIntegerQaCase, + context: EvaluationContext, +) -> Path: + source_path = resolve_dataset(case.source.recording).resolve() + stat = source_path.stat() + stem = re.sub(r"[^A-Za-z0-9_.-]+", "-", source_path.stem)[:64] + mapper = MapperSettings() + raw_key = ( + f"{stem}-{stat.st_size}-{stat.st_mtime_ns}-p{case.source.progress:.9f}-" + f"v{mapper.voxel_size_m}-b{mapper.block_count}-d{mapper.device}-" + f"c{int(mapper.carve_columns)}-f{mapper.frame_id}-e{mapper.emit_every}" + ) + key = re.sub(r"[^A-Za-z0-9_.-]+", "-", raw_key) + bundle = CACHE_DIR / "evaluation" / "frozen_memory" / key + manifest = bundle / "manifest.v1.json" + if not manifest.is_file(): + emit_progress( + context.progress, + StatusProgress(channel="eval", message="preparing memory"), + ) + bundle.parent.mkdir(parents=True, exist_ok=True) + try: + prepare_bundle( + case.source.recording, + [], + bundle, + progress=[case.source.progress], + mapper=mapper, + map_progress=lambda current, total: emit_progress( + context.progress, + StatusProgress( + channel="eval", + message=f"mapping {current}/{total} frames", + ), + ), + ) + except OSError as exc: + concurrent_publish = isinstance(exc, FileExistsError) or exc.errno in { + errno.EEXIST, + errno.ENOTEMPTY, + } + if not concurrent_publish or not manifest.is_file(): + raise + return bundle + + +frozen_integer_qa = FrozenIntegerQaEvaluation() diff --git a/dimos/benchmark/short_horizon_qa/eval.py b/dimos/benchmark/short_horizon_qa/integer_answer.py similarity index 63% rename from dimos/benchmark/short_horizon_qa/eval.py rename to dimos/benchmark/short_horizon_qa/integer_answer.py index 72916e9615..174b56de07 100644 --- a/dimos/benchmark/short_horizon_qa/eval.py +++ b/dimos/benchmark/short_horizon_qa/integer_answer.py @@ -12,42 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Exact-integer validation for short-horizon frozen-memory questions.""" +"""Integer-answer decoding and private oracle loading for frozen-memory questions.""" from __future__ import annotations from pathlib import Path import re -from typing import Any, Literal -from pydantic import Field - -from dimos.benchmark.agent_eval.models import ( - BaseEvalModel, - EvalCase, - ExactIntegerValidatorRef, +from dimos.benchmark.short_horizon_qa.models import ( + ExactIntegerOracle, + FrozenIntegerQaCase, + IntegerPrediction, ) _ANSWER_LINE = re.compile(r"(?m)^ANSWER:\s*") _TERMINAL_INTEGER = re.compile(r"(?:^|\n)ANSWER:\s*(-?\d+)\s*\Z") -class ExactIntegerOracle(BaseEvalModel): - expected_count: int = Field(ge=0) - counting_policy: str = Field(min_length=1) - rooms: tuple[dict[str, Any], ...] = () - reviewed_by: tuple[str, ...] = Field(min_length=1) - - -class IntegerPrediction(BaseEvalModel): - status: Literal["parsed", "invalid"] - integer_answer: int | None = None - - -def load_exact_integer_oracle(case: EvalCase, case_root: Path) -> ExactIntegerOracle: +def load_exact_integer_oracle(case: FrozenIntegerQaCase, case_root: Path) -> ExactIntegerOracle: reference = case.validator - if not isinstance(reference, ExactIntegerValidatorRef): - raise TypeError("case does not use exact-integer validation") root = case_root.resolve() path = (root / reference.private_path).resolve() if root not in path.parents: diff --git a/dimos/benchmark/short_horizon_qa/models.py b/dimos/benchmark/short_horizon_qa/models.py index 85b6d042a1..eb26237509 100644 --- a/dimos/benchmark/short_horizon_qa/models.py +++ b/dimos/benchmark/short_horizon_qa/models.py @@ -17,7 +17,8 @@ from __future__ import annotations import math -from typing import Literal +from pathlib import PurePosixPath +from typing import Annotated, Any, Literal from pydantic import BaseModel, ConfigDict, Field, model_validator @@ -70,3 +71,66 @@ class FrozenMemoryManifest(FrozenQaModel): derived_path: Literal["derived.db"] = "derived.db" mapper: MapperSettings cutoffs: tuple[CutoffRecord, ...] = Field(min_length=1) + + +NonEmpty = Annotated[str, Field(min_length=1)] + + +class FrozenRecordingSource(FrozenQaModel): + schema_version: Literal["1.0"] = "1.0" + kind: Literal["frozen_memory"] = "frozen_memory" + recording: NonEmpty + progress: float = Field(ge=0, le=1, allow_inf_nan=False) + + @model_validator(mode="after") + def finite_progress(self) -> FrozenRecordingSource: + if not math.isfinite(self.progress): + raise ValueError("recording progress must be finite") + return self + + +class IntegerQuestionTask(FrozenQaModel): + schema_version: Literal["1.0"] = "1.0" + kind: Literal["integer_question"] = "integer_question" + prompt: NonEmpty + answer_marker: Literal["ANSWER:"] = "ANSWER:" + + +class ExactIntegerValidatorRef(FrozenQaModel): + schema_version: Literal["1.0"] = "1.0" + kind: Literal["exact_integer"] = "exact_integer" + revision: NonEmpty + private_path: NonEmpty + + @model_validator(mode="after") + def safe_relative_path(self) -> ExactIntegerValidatorRef: + path = PurePosixPath(self.private_path) + if path.is_absolute() or not path.parts or ".." in path.parts: + raise ValueError("validator private_path must be a safe relative path") + return self + + +class FrozenIntegerQaCase(FrozenQaModel): + schema_version: Literal["1.0"] = "1.0" + case_id: NonEmpty + source: FrozenRecordingSource + task: IntegerQuestionTask + validator: ExactIntegerValidatorRef + + +class FrozenIntegerQaConfig(FrozenQaModel): + case: NonEmpty + + +class ExactIntegerOracle(FrozenQaModel): + schema_version: Literal["1.0"] = "1.0" + expected_count: int = Field(ge=0) + counting_policy: str = Field(min_length=1) + rooms: tuple[dict[str, Any], ...] = () + reviewed_by: tuple[str, ...] = Field(min_length=1) + + +class IntegerPrediction(FrozenQaModel): + schema_version: Literal["1.0"] = "1.0" + status: Literal["parsed", "invalid"] + integer_answer: int | None = None diff --git a/dimos/benchmark/short_horizon_qa/test_evaluation.py b/dimos/benchmark/short_horizon_qa/test_evaluation.py new file mode 100644 index 0000000000..31a3172a88 --- /dev/null +++ b/dimos/benchmark/short_horizon_qa/test_evaluation.py @@ -0,0 +1,169 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +from types import SimpleNamespace + +from dimos.benchmark.evaluation.protocol import AgentOutcome, EvaluationContext +import dimos.benchmark.short_horizon_qa.evaluation as evaluation +from dimos.benchmark.short_horizon_qa.models import ( + ExactIntegerValidatorRef, + FrozenIntegerQaCase, + FrozenIntegerQaConfig, + FrozenRecordingSource, + IntegerQuestionTask, +) + + +class FakeSession: + def __init__(self, captured: dict) -> None: + self.captured = captured + + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def run(self, *, evaluation_protocol: str, task_input: str) -> AgentOutcome: + self.captured.update(protocol=evaluation_protocol, task=task_input) + return AgentOutcome("Checked\nANSWER: 2", 3, 1.0) + + +class FakeAgent: + def __init__(self, captured: dict) -> None: + self.captured = captured + + def open_session(self, environment): + self.captured["environment"] = environment + return FakeSession(self.captured) + + +def _case(tmp_path: Path) -> Path: + private = tmp_path / "private" + private.mkdir() + (private / "oracle.json").write_text( + '{"schema_version":"1.0","expected_count":2,' + '"counting_policy":"count rooms","rooms":[],' + '"reviewed_by":["reviewer"]}' + ) + case = FrozenIntegerQaCase( + case_id="demo", + source=FrozenRecordingSource(recording="recording", progress=1.0), + task=IntegerQuestionTask(prompt="How many rooms?"), + validator=ExactIntegerValidatorRef( + revision="v1", + private_path="private/oracle.json", + ), + ) + path = tmp_path / "case.json" + path.write_text(case.model_dump_json()) + return path + + +def test_frozen_evaluation_owns_protocol_decoder_and_openevals( + monkeypatch, + tmp_path: Path, +) -> None: + case_path = _case(tmp_path) + bundle = tmp_path / "bundle" + bundle.mkdir() + captured = {} + monkeypatch.setattr(evaluation, "_materialize_frozen_memory", lambda *_args: bundle) + monkeypatch.setattr( + evaluation, + "load_bundle", + lambda *_args, **_kwargs: ( + object(), + SimpleNamespace(cutoff_timestamp=10.0), + tmp_path / "source.db", + tmp_path / "derived.db", + ), + ) + + def exact_match(*, outputs, reference_outputs): + captured.update(outputs=outputs, reference_outputs=reference_outputs) + return {"key": "exact_match", "score": True, "comment": None} + + monkeypatch.setattr(evaluation, "exact_match", exact_match) + context = EvaluationContext( + run_id="run", + spec_dir=tmp_path, + workspace=tmp_path / "workspace", + agent=FakeAgent(captured), + progress=None, + ) + + report = evaluation.frozen_integer_qa.run( + FrozenIntegerQaConfig(case=case_path.name), + context, + ) + + assert captured["task"] == "How many rooms?" + assert "ANSWER: " in captured["protocol"] + assert captured["outputs"] == {"status": "parsed", "integer_answer": 2} + assert captured["reference_outputs"] == { + "status": "parsed", + "integer_answer": 2, + } + assert report.native_result.value == { + "key": "exact_match", + "score": True, + "comment": None, + } + assert [item.key for item in report.summary] == [ + "case", + "recording", + "answer", + "exact_match", + "tool_calls", + "duration", + ] + + +def test_materialize_accepts_bundle_published_by_concurrent_runner( + monkeypatch, + tmp_path: Path, +) -> None: + recording = tmp_path / "recording.db" + recording.touch() + case = FrozenIntegerQaCase( + case_id="concurrent", + source=FrozenRecordingSource(recording=str(recording), progress=1.0), + task=IntegerQuestionTask(prompt="How many rooms?"), + validator=ExactIntegerValidatorRef( + revision="v1", + private_path="private/oracle.json", + ), + ) + monkeypatch.setattr(evaluation, "CACHE_DIR", tmp_path / "cache") + monkeypatch.setattr(evaluation, "resolve_dataset", lambda _recording: recording) + + def publish_first(_recording, _cutoffs, output: Path, **_kwargs) -> None: + output.mkdir() + (output / "manifest.v1.json").write_text("{}") + raise FileExistsError(output) + + monkeypatch.setattr(evaluation, "prepare_bundle", publish_first) + context = EvaluationContext( + run_id="run", + spec_dir=tmp_path, + workspace=tmp_path, + agent=FakeAgent({}), + progress=None, + ) + + bundle = evaluation._materialize_frozen_memory(case, context) + + assert bundle.joinpath("manifest.v1.json").is_file() diff --git a/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py b/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py index 0c7def1bfa..7406146321 100644 --- a/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py +++ b/dimos/benchmark/short_horizon_qa/test_hongkong_eval.py @@ -18,8 +18,7 @@ import pytest -from dimos.benchmark.agent_eval.models import EvalCase -from dimos.benchmark.short_horizon_qa.models import MapperSettings +from dimos.benchmark.short_horizon_qa.models import FrozenIntegerQaCase, MapperSettings from dimos.benchmark.short_horizon_qa.prepare import prepare_bundle from dimos.utils.data import get_data @@ -29,7 +28,7 @@ def test_real_hongkong_recording_prepares_direct_demo_case(tmp_path: Path) -> No case_path = ( Path(__file__).parent / "cases" / "demo_go2_hongkong_office-room-count-smoke" / "case.json" ) - case = EvalCase.model_validate_json(case_path.read_bytes()) + case = FrozenIntegerQaCase.model_validate_json(case_path.read_bytes()) map_progress: list[tuple[int, int]] = [] manifest = prepare_bundle( get_data("go2_hongkong_office.db"), diff --git a/dimos/benchmark/short_horizon_qa/test_eval.py b/dimos/benchmark/short_horizon_qa/test_integer_answer.py similarity index 93% rename from dimos/benchmark/short_horizon_qa/test_eval.py rename to dimos/benchmark/short_horizon_qa/test_integer_answer.py index 87d4adba4f..56475a4490 100644 --- a/dimos/benchmark/short_horizon_qa/test_eval.py +++ b/dimos/benchmark/short_horizon_qa/test_integer_answer.py @@ -14,7 +14,7 @@ import pytest -from dimos.benchmark.short_horizon_qa.eval import parse_integer_prediction +from dimos.benchmark.short_horizon_qa.integer_answer import parse_integer_prediction @pytest.mark.parametrize( diff --git a/dimos/cli/eval.py b/dimos/cli/eval.py index d3517a5c2a..781c8692fc 100644 --- a/dimos/cli/eval.py +++ b/dimos/cli/eval.py @@ -12,25 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Dependency-light shell for the immutable single-case evaluation CLI.""" +"""Dependency-light shell for the unified Evaluation CLI.""" from __future__ import annotations from pathlib import Path import threading -from typing import Any, Literal +from typing import Any import typer -app = typer.Typer(help="Run immutable agent evaluation cases", no_args_is_help=True) +app = typer.Typer(help="Run executable evaluations", no_args_is_help=True) MAX_RENDERED_TOOL_RESULT_CHARS = 2_000 -def execute_single_case(*args: Any, **kwargs: Any) -> Any: +def execute_evaluation(*args: Any, **kwargs: Any) -> Any: """Import and dispatch the evaluation runtime only when ``eval run`` executes.""" try: - from dimos.benchmark.agent_eval.single_case import execute_single_case as execute + from dimos.benchmark.evaluation.runner import execute_evaluation as execute except ModuleNotFoundError as exc: raise RuntimeError( "Evaluation dependencies are missing; run `uv sync --extra agents`" @@ -41,35 +41,19 @@ def execute_single_case(*args: Any, **kwargs: Any) -> Any: @app.command("run") def run( - case: Path = typer.Argument(..., exists=True, dir_okay=False, readable=True), - agent_backend: Literal["pi"] = typer.Option("pi", "--agent.backend"), - agent_model: Literal["gpt-5.6-luna"] = typer.Option("gpt-5.6-luna", "--agent.model"), - thinking_level: Literal["medium"] = typer.Option("medium", "--agent.thinking-level"), - api_key_env: str = typer.Option("OPENAI_API_KEY", "--agent.api-key-env"), + specification: Path = typer.Argument(..., exists=True, dir_okay=False, readable=True), + api_key_env: str = typer.Option("OPENAI_API_KEY", "--api-key-env"), output: Path = typer.Option(..., "--output"), json_output: bool = typer.Option(False, "--json", help="Print compact JSON"), quiet: bool = typer.Option(False, "--quiet", help="Suppress live evaluation progress"), ) -> None: - """Run one static evaluation case synchronously.""" - from dimos.benchmark.agent_eval.models import ( - EvalRunConfig, - PiAgentConfig, - ) - - config = EvalRunConfig( - agent=PiAgentConfig( - backend=agent_backend, - model=agent_model, - thinking_level=thinking_level, - api_key_env=api_key_env, - ) - ) + """Run one Evaluation Run Specification synchronously.""" renderer = None if quiet else ProgressRenderer() try: - result = execute_single_case( - case, - config=config, + result = execute_evaluation( + specification, output=output, + api_key_env=api_key_env, progress=renderer, ) except Exception as exc: @@ -80,31 +64,36 @@ def run( if renderer is not None: renderer.finish() typer.echo(result.model_dump_json() if json_output else format_result(result, output)) - if result.attempt_status == "failed": + if result.status == "cancelled": + raise typer.Exit(130) + if result.status == "failed": raise typer.Exit(1) def format_result(result: Any, output: Path | None = None) -> str: - """Render the compact typed result without exposing private oracle material.""" - if result.attempt_status == "failed": - heading = "! Evaluation not evaluated" - elif result.task_result == "passed": - heading = "✓ Evaluation passed" - else: - heading = "✗ Evaluation failed" - source = f"{result.recording} @ {result.progress * 100:g}%" - answer = str(result.integer_answer) if result.integer_answer is not None else "—" - rows = ( - ("Case", result.case_id), - ("Source", source), - ("Answer", answer), - ("Result", result.task_result), - ("Agent", f"Pi · {result.model} · {result.thinking_level}"), - ("Tool calls", str(result.tool_call_count)), - ("Duration", f"{result.duration_seconds:.1f}s"), - ("Output", str((output / "result.json") if output is not None else "result.json")), - ) - body = "\n".join(f" {label:<10} {value}" for label, value in rows) + """Render infrastructure status plus Evaluation-supplied summary rows.""" + heading = { + "completed": "✓ Evaluation completed", + "failed": "! Evaluation failed", + "cancelled": "! Evaluation cancelled", + }[result.status] + rows = [ + ("Evaluation", result.evaluation.name), + ( + "Agent", + f"Pi · {result.runtime.model} · {result.runtime.thinking_level}", + ), + ] + if result.report is not None: + rows.extend( + (item.label, "—" if item.value is None else str(item.value)) + for item in result.report.summary + ) + if result.error is not None: + rows.append(("Error", f"{result.error.error_type}: {result.error.message}")) + rows.append(("Output", str((output / "run.json") if output is not None else "run.json"))) + width = max(len(label) for label, _ in rows) + body = "\n".join(f" {label:<{width}} {value}" for label, value in rows) return f"{heading}\n\n{body}" diff --git a/dimos/cli/test_eval.py b/dimos/cli/test_eval.py index c94ded418a..47f1516a58 100644 --- a/dimos/cli/test_eval.py +++ b/dimos/cli/test_eval.py @@ -13,6 +13,7 @@ # limitations under the License. import builtins +from datetime import datetime, timezone import json from pathlib import Path import subprocess @@ -23,8 +24,19 @@ import pytest from typer.testing import CliRunner -from dimos.benchmark.agent_eval.models import CompactEvalResult -from dimos.benchmark.agent_eval.progress import ( +from dimos.benchmark.evaluation.models import ( + CodePolicyAgentConfig, + EvaluationIdentity, + EvaluationReference, + EvaluationReport, + EvaluationRun, + EvaluationRunError, + EvaluationRunSpecification, + InlineNativeResult, + RuntimeIdentity, + SummaryItem, +) +from dimos.benchmark.evaluation.progress import ( AssistantTextProgress, StatusProgress, ToolEndProgress, @@ -33,46 +45,81 @@ import dimos.cli.eval as eval_cli -def _result(*, passed: bool | None = True) -> CompactEvalResult: - return CompactEvalResult( - case_id="demo-room-count", - recording="go2_hongkong_office", - progress=1.0, - model="gpt-5.6-luna", - thinking_level="medium", - final_response="ANSWER: 4" if passed is not None else "", - prediction_status="parsed" if passed is not None else "not_evaluated", - integer_answer=4 if passed is not None else None, - passed=passed, - validator_revision="v1", - tool_call_count=7, - duration_seconds=42.75, - infra_error="Pi failed" if passed is None else None, +def _result(status: str = "completed") -> EvaluationRun: + now = datetime.now(timezone.utc) + completed = status == "completed" + return EvaluationRun( + run_id="run-1", + specification=EvaluationRunSpecification( + evaluation=EvaluationReference(name="fixture", config={}), + agent=CodePolicyAgentConfig(), + ), + evaluation=EvaluationIdentity(name="fixture", provider="tests", version="1"), + runtime=RuntimeIdentity( + driver_version="test", + model="gpt-5.6-luna", + thinking_level="medium", + ), + status=status, + started_at=now, + finished_at=now, + duration_seconds=1.0, + report=( + EvaluationReport( + summary=(SummaryItem(key="native_score", label="Native score", value=0.5),), + native_result=InlineNativeResult(value={"score": 0.5}), + ) + if completed + else None + ), + error=( + None + if completed + else EvaluationRunError( + stage="evaluation", + error_type="RuntimeError", + message="agent failed", + ) + ), ) -def _case(tmp_path: Path) -> Path: - path = tmp_path / "case.json" +def _spec(tmp_path: Path) -> Path: + path = tmp_path / "spec.json" path.write_text("{}") return path -def test_eval_run_uses_api_key_default_and_separates_progress(tmp_path, monkeypatch) -> None: +def test_eval_run_uses_operational_api_key_and_renders_native_summary( + tmp_path, + monkeypatch, +) -> None: captured = {} - def execute(path, *, config, progress, output): - captured.update(path=path, config=config, progress=progress, output=output) - progress(StatusProgress(channel="eval", message="loading case")) + def execute(path, *, api_key_env, progress, output): + captured.update( + path=path, + api_key_env=api_key_env, + progress=progress, + output=output, + ) + progress(StatusProgress(channel="eval", message="loading specification")) return _result() - monkeypatch.setattr(eval_cli, "execute_single_case", execute) + monkeypatch.setattr(eval_cli, "execute_evaluation", execute) output = tmp_path / "run" - result = CliRunner().invoke(main, ["eval", "run", str(_case(tmp_path)), f"--output={output}"]) + + result = CliRunner().invoke( + main, + ["eval", "run", str(_spec(tmp_path)), f"--output={output}"], + ) + assert result.exit_code == 0, result.output - assert captured["config"].agent.api_key_env == "OPENAI_API_KEY" + assert captured["api_key_env"] == "OPENAI_API_KEY" assert captured["output"] == output - assert "✓ Evaluation passed" in result.stdout - assert "[eval] loading case" in result.stderr + assert "✓ Evaluation completed" in result.stdout + assert "Native score" in result.stdout + assert "[eval] loading specification" in result.stderr def test_eval_run_accepts_named_api_key_env_and_json(tmp_path, monkeypatch) -> None: @@ -82,54 +129,66 @@ def execute(*args, **kwargs): captured.update(kwargs) return _result() - monkeypatch.setattr(eval_cli, "execute_single_case", execute) + monkeypatch.setattr(eval_cli, "execute_evaluation", execute) output = tmp_path / "run" result = CliRunner().invoke( main, [ "eval", "run", - str(_case(tmp_path)), - "--agent.api-key-env=MY_OPENAI_KEY", + str(_spec(tmp_path)), + "--api-key-env=MY_OPENAI_KEY", f"--output={output}", "--json", ], ) + assert result.exit_code == 0, result.output - assert captured["config"].agent.api_key_env == "MY_OPENAI_KEY" - assert json.loads(result.stdout)["passed"] is True + assert captured["api_key_env"] == "MY_OPENAI_KEY" + payload = json.loads(result.stdout) + assert payload["status"] == "completed" + assert payload["report"]["native_result"]["value"] == {"score": 0.5} -def test_eval_exit_codes_distinguish_infra_semantic_and_preflight(tmp_path, monkeypatch) -> None: - output = tmp_path / "run" - monkeypatch.setattr(eval_cli, "execute_single_case", lambda *a, **k: _result(passed=None)) - infra = CliRunner().invoke( - main, ["eval", "run", str(_case(tmp_path)), f"--output={output}", "--quiet"] - ) - monkeypatch.setattr(eval_cli, "execute_single_case", lambda *a, **k: _result(passed=False)) - semantic = CliRunner().invoke( - main, ["eval", "run", str(_case(tmp_path)), f"--output={output}", "--quiet"] +@pytest.mark.parametrize(("status", "exit_code"), [("failed", 1), ("cancelled", 130)]) +def test_eval_exit_codes_for_noncompleted_runs( + status, + exit_code, + tmp_path, + monkeypatch, +) -> None: + monkeypatch.setattr(eval_cli, "execute_evaluation", lambda *a, **k: _result(status)) + + result = CliRunner().invoke( + main, + ["eval", "run", str(_spec(tmp_path)), f"--output={tmp_path / 'run'}", "--quiet"], ) + assert result.exit_code == exit_code + + +def test_eval_preflight_failure_uses_exit_two(tmp_path, monkeypatch) -> None: def preflight(*_args, **_kwargs): - raise FileNotFoundError("extension build missing") + raise FileNotFoundError("evaluation plugin missing") - monkeypatch.setattr(eval_cli, "execute_single_case", preflight) - preflight_result = CliRunner().invoke( - main, ["eval", "run", str(_case(tmp_path)), f"--output={output}"] + monkeypatch.setattr(eval_cli, "execute_evaluation", preflight) + result = CliRunner().invoke( + main, + ["eval", "run", str(_spec(tmp_path)), f"--output={tmp_path / 'run'}"], ) - assert infra.exit_code == 1 - assert semantic.exit_code == 0 - assert preflight_result.exit_code == 2 + + assert result.exit_code == 2 -def test_eval_help_is_typed_and_output_is_required(tmp_path) -> None: +def test_eval_help_exposes_thin_operational_settings(tmp_path) -> None: runner = CliRunner() help_result = runner.invoke(main, ["eval", "run", "--help"], color=True) - missing_output = runner.invoke(main, ["eval", "run", str(_case(tmp_path))]) + missing_output = runner.invoke(main, ["eval", "run", str(_spec(tmp_path))]) + assert help_result.exit_code == 0 help_text = unstyle(help_result.stdout) - assert "--agent.api-key-env" in help_text + assert "--api-key-env" in help_text + assert "--agent.model" not in help_text assert "--output" in help_text assert missing_output.exit_code == 2 @@ -137,14 +196,15 @@ def test_eval_help_is_typed_and_output_is_required(tmp_path) -> None: def test_lazy_runtime_import_has_actionable_error(monkeypatch) -> None: original_import = builtins.__import__ - def fail_single_case(name, *args, **kwargs): - if name == "dimos.benchmark.agent_eval.single_case": + def fail_runtime(name, *args, **kwargs): + if name == "dimos.benchmark.evaluation.runner": raise ModuleNotFoundError("No module named 'mcp'") return original_import(name, *args, **kwargs) - monkeypatch.setattr(builtins, "__import__", fail_single_case) + monkeypatch.setattr(builtins, "__import__", fail_runtime) + with pytest.raises(RuntimeError, match="uv sync --extra agents"): - eval_cli.execute_single_case(Path("case.json"), config=None) + eval_cli.execute_evaluation(Path("spec.json"), output=Path("output")) def test_base_cli_help_imports_without_eval_runtime() -> None: @@ -167,8 +227,12 @@ def find_spec(self, fullname, path=None, target=None): """ ) completed = subprocess.run( - [sys.executable, "-c", script], capture_output=True, text=True, check=False + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, ) + assert completed.returncode == 0, completed.stdout + completed.stderr diff --git a/docs/capabilities/agents/evaluation.md b/docs/capabilities/agents/evaluation.md index 211c26375a..82f314ea27 100644 --- a/docs/capabilities/agents/evaluation.md +++ b/docs/capabilities/agents/evaluation.md @@ -1,79 +1,160 @@ --- -title: "Frozen recording evaluation" +title: "Agent evaluations" --- -`dimos eval run` asks Pi one integer question about a frozen Memory2 recording. -The evaluator prepares the runtime map, exposes read-only `memory` through one -`python_exec` MCP tool, and checks the final `ANSWER: ` line against a -private oracle. It does not start a robot, simulation, replay blueprint, or live -DimOS module. +DimOS runs complete **Evaluations**. An Evaluation owns its inputs, protocol, +scoring, aggregation, and native result semantics. The shared framework resolves +the Evaluation, supplies CodePolicy agent sessions, and records an immutable +Evaluation Run. + +```text +Evaluation Run Specification + | + v + Evaluation ------ dataset / cases / native harness + | + v + CodePolicy Runtime ----- Pi + one persistent python_exec tool + | + v + Evaluation Run ------- status / native result / artifacts +``` + +This is deliberately not a universal scorer. The built-in frozen integer QA +Evaluation uses OpenEvals internally. A third-party benchmark should instead call +its own harness and return that harness's native result without rescoring it. ## Setup -From a source checkout, install the lightweight Python runtime and build the Pi -extension: +From a source checkout, install the Python runtime and build the Pi extension: ```bash uv sync --extra agents npm ci --prefix packages/pi-code-policy-extension npm run build --prefix packages/pi-code-policy-extension +export OPENAI_API_KEY=... ``` -The package pins Pi `0.80.10` and requires Node 22.19.0 or newer. Set the API key -before running a case: - -```bash -export OPENAI_API_KEY=... +The initial runtime profile is `code-policy-v1`: Pi 0.80.10, model +`gpt-5.6-luna`, medium thinking, and exactly one `python_exec` MCP tool. Pi is the +profile's driver, not a user-selectable evaluation runtime. + +## Run specification and CLI + +An Evaluation Run Specification binds an Evaluation configuration to the agent +configuration: + +```json +{ + "schema_version": "1.0", + "evaluation": { + "name": "frozen-integer-qa", + "config": {"case": "case.json"} + }, + "agent": { + "profile": "code-policy-v1", + "model": "gpt-5.6-luna", + "thinking_level": "medium" + } +} ``` -## Run the direct demo case +Evaluation-owned relative paths are resolved from the specification directory. +Run the included smoke specification with: ```bash uv run dimos eval run \ - dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/case.json \ + dimos/benchmark/short_horizon_qa/cases/demo_go2_hongkong_office-room-count-smoke/run.json \ --output=/tmp/dimos-eval-smoke ``` -The demo fixture uses the synthetic sentinel `0`, not a reviewed Hong Kong office -room count. A semantic failure can therefore mean the agent and runtime worked but -the response did not match that plumbing sentinel. - -The supported options are deliberately small: +The operational CLI settings stay thin: | Option | Default | Purpose | | --- | --- | --- | -| `--agent.backend` | `pi` | Use the pinned Pi backend. | -| `--agent.model` | `gpt-5.6-luna` | Use the pinned model. | -| `--agent.thinking-level` | `medium` | Use the pinned thinking level. | -| `--agent.api-key-env` | `OPENAI_API_KEY` | Select the environment variable containing the API key. | -| `--output` | required | Publish this run to the exact directory. | -| `--json` | off | Print the compact result as JSON. | -| `--quiet` | off | Suppress status messages on stderr. | +| `--api-key-env` | `OPENAI_API_KEY` | Name of the environment variable containing the API key. | +| `--output` | required | Atomically publish the run to this directory. | +| `--json` | off | Print the complete Evaluation Run as JSON. | +| `--quiet` | off | Suppress live progress on stderr. | + +The API key is passed only to Pi. It is not written to the specification, run +record, prompt evidence, subprocess arguments, or Python kernel environment. + +## Results, artifacts, and exit status + +`--output` must be absent or empty. DimOS builds the run in a temporary sibling +directory and publishes it atomically as: + +```text +run.json +runtime/session-0001/ + runtime-system.txt + evaluation-protocol.txt + task-input.txt + assembled-user-message.txt + prompt-assembly.json + pi-transcript.jsonl # when Pi emits one + stderr.log # when nonempty +``` -The API key is passed only to the Pi subprocess. It is not placed in arguments, -results, or the Jupyter kernel environment. +`run.json` contains only universal infrastructure status—`completed`, `failed`, +or `cancelled`—plus the Evaluation's summary, opaque native result or artifact +reference, and artifact metadata. A native score of `false` can still be a +successfully completed run. + +| Exit | Meaning | +| --- | --- | +| `0` | The Evaluation completed, regardless of native semantic score. | +| `1` | Evaluation or agent infrastructure failed after execution started. | +| `2` | Specification, discovery, configuration, credential, or output preflight failed. | +| `130` | The user cancelled the Evaluation. | + +## Implement an Evaluation + +An Evaluation is the only public semantic extension point: + +```python +class MyEvaluation: + name = "my-evaluation" + config_model = MyEvaluationConfig + + def run(self, config, context): + with context.agent.open_session(environment) as session: + outcome = session.run( + evaluation_protocol="Return one answer per benchmark rules.", + task_input=sample.question, + ) + native_result = my_existing_harness.score(outcome.final_text) + return EvaluationReport( + summary=(...), + native_result=InlineNativeResult(value=native_result), + ) +``` -## Output and exit status +Built-ins are registered lazily inside DimOS. External distributions expose an +Evaluation object through the `dimos.evaluations` entry-point group: -`--output` must name an absent or empty directory. The evaluator builds the run in -a temporary sibling and atomically publishes it on completion. It never merges -with or overwrites a nonempty directory. +```toml +[project.entry-points."dimos.evaluations"] +my-evaluation = "my_package.evaluation:my_evaluation" +``` -The directory contains only: +An installed external Evaluation is addressed as +`.my-evaluation`. Keep benchmark datasets, sample +loops, success checks, and aggregation in the Evaluation or native harness. Do +not translate them into a universal DimOS case or scorer. -- `result.json`; -- `pi-transcript.jsonl`, when Pi wrote a native transcript; -- `stderr.log`, only when nonempty diagnostics are available. +## Prompt ownership -Exit code `0` means evaluation completed, whether the semantic score passed or -failed. Exit code `1` means a caught runtime or agent infrastructure failure; the -published `result.json` includes `infra_error`. Exit code `2` means preflight -failed before a run started. +The versioned runtime profile owns system instructions, the `python_exec` tool +surface, Pi flags, and deterministic assembly. Every Evaluation supplies two +immutable strings: its Evaluation Protocol and its Task Input. Their owners and +SHA-256 hashes are recorded separately even though Pi receives them together in +one user message. There are no prompt-template settings. ## Trust boundary CodePolicy executes agent-authored Python in a persistent Jupyter kernel. It is -trusted and **unsandboxed**. The `memory` object is cutoff-limited and its SQLite -connections are truly read-only, but Python can still access other host files and -processes. Run only trusted evaluation agents, or place the whole command in an OS -sandbox or container. +trusted and **unsandboxed**. Frozen Memory2 SQLite connections are read-only, but +Python can still access other host files and processes. Run only trusted agents, +or place the entire evaluation command in an OS sandbox or container. diff --git a/docs/development/testing.md b/docs/development/testing.md index 8932ad1481..ac15ffa1bd 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -89,7 +89,7 @@ uv run --extra agents pytest \ dimos/memory2/store/test_frozen.py \ dimos/agents/test_code_policy_core.py \ dimos/agents/test_code_policy_server.py \ - dimos/benchmark/agent_eval \ + dimos/benchmark/evaluation \ dimos/benchmark/short_horizon_qa \ dimos/cli/test_eval.py ``` diff --git a/packages/pi-code-policy-extension/src/python-exec.ts b/packages/pi-code-policy-extension/src/python-exec.ts index 5a74266436..5d0c05bdd6 100644 --- a/packages/pi-code-policy-extension/src/python-exec.ts +++ b/packages/pi-code-policy-extension/src/python-exec.ts @@ -10,7 +10,7 @@ const TOOL_NAME = "python_exec"; const DEFAULT_TIMEOUT_SECONDS = 110; interface McpClient { - listTools(): Promise<{ tools: Array<{ name: string }> }>; + listTools(): Promise<{ tools: Array<{ name: string; description?: string }> }>; callTool( params: { name: string; arguments: Record }, options?: { timeout?: number }, @@ -47,7 +47,8 @@ export async function installPythonExec( name: TOOL_NAME, label: "Execute Python", description: - "Execute Python in a persistent trusted, unsandboxed session with read-only memory.", + inventory.tools[0].description ?? + "Execute Python in a persistent trusted, unsandboxed session.", parameters: Type.Object( { code: Type.String({ minLength: 1 }), diff --git a/packages/pi-code-policy-extension/test/python-exec.test.ts b/packages/pi-code-policy-extension/test/python-exec.test.ts index 9f351564c7..2fd25148f7 100644 --- a/packages/pi-code-policy-extension/test/python-exec.test.ts +++ b/packages/pi-code-policy-extension/test/python-exec.test.ts @@ -20,7 +20,14 @@ test("registers one tool that calls MCP directly", async () => { } as ExtensionAPI; const client = { async listTools() { - return { tools: [{ name: "python_exec" }] }; + return { + tools: [ + { + name: "python_exec", + description: "Canonical CodePolicy description", + }, + ], + }; }, async callTool(params: { name: string; arguments: Record }) { assert.deepEqual(params, { @@ -36,6 +43,7 @@ test("registers one tool that calls MCP directly", async () => { await installPythonExec(pi, "http://127.0.0.1:1/mcp", async () => client); assert.equal(tool?.name, "python_exec"); + assert.equal(tool?.description, "Canonical CodePolicy description"); const result = await tool!.execute("call-1", { code: "1 + 1", timeout_s: 3 }, undefined, undefined, {} as never); assert.deepEqual(result.content, [{ type: "text", text: "2" }]); await shutdown!(); diff --git a/pyproject.toml b/pyproject.toml index fe37de9571..8a34b320b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -234,6 +234,7 @@ agents = [ "langchain-huggingface>=1,<2", "langchain-ollama>=1,<2", "ollama>=0.6.0", + "openevals>=0.2,<0.3", # Audio "openai", @@ -416,6 +417,7 @@ project-deps = [ "jupyter-client>=8.8.0", "mcp==2.0.0", "uvicorn>=0.34.0", + "openevals>=0.2,<0.3", ] tests = [ diff --git a/uv.lock b/uv.lock index 76c8ef989a..c309c0c078 100644 --- a/uv.lock +++ b/uv.lock @@ -1644,6 +1644,7 @@ agents = [ { name = "nbformat" }, { name = "ollama" }, { name = "openai" }, + { name = "openevals" }, { name = "pyzmq" }, { name = "sounddevice" }, { name = "uvicorn" }, @@ -1689,6 +1690,7 @@ all = [ { name = "onnxruntime-gpu", marker = "platform_machine == 'x86_64'" }, { name = "open-clip-torch" }, { name = "openai" }, + { name = "openevals" }, { name = "pillow" }, { name = "piper-sdk" }, { name = "playground" }, @@ -1749,6 +1751,7 @@ base = [ { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, + { name = "openevals" }, { name = "pillow" }, { name = "pyzmq" }, { name = "rerun-sdk" }, @@ -1864,6 +1867,7 @@ unitree = [ { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, + { name = "openevals" }, { name = "pillow" }, { name = "pyzmq" }, { name = "rerun-sdk" }, @@ -1903,6 +1907,7 @@ unitree-dds = [ { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, + { name = "openevals" }, { name = "pillow" }, { name = "pyzmq" }, { name = "rerun-sdk" }, @@ -1965,6 +1970,7 @@ lint = [ { name = "open-clip-torch" }, { name = "openai" }, { name = "openai-whisper" }, + { name = "openevals" }, { name = "pandas-stubs" }, { name = "pytest" }, { name = "python-can" }, @@ -2004,6 +2010,7 @@ project-deps = [ { name = "ollama" }, { name = "open-clip-torch" }, { name = "openai" }, + { name = "openevals" }, { name = "tensorboard" }, { name = "torch" }, { name = "torchreid" }, @@ -2035,6 +2042,7 @@ tests = [ { name = "ollama" }, { name = "open-clip-torch" }, { name = "openai" }, + { name = "openevals" }, { name = "pre-commit" }, { name = "py-spy" }, { name = "pygame" }, @@ -2087,6 +2095,7 @@ tests-self-hosted = [ { name = "ollama" }, { name = "open-clip-torch" }, { name = "openai" }, + { name = "openevals" }, { name = "pre-commit" }, { name = "py-spy" }, { name = "pybind11" }, @@ -2190,6 +2199,7 @@ requires-dist = [ { name = "open3d-unofficial-arm", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'", specifier = ">=0.19.0.post9" }, { name = "openai", marker = "extra == 'agents'" }, { name = "opencv-contrib-python", specifier = ">=4.8,<5" }, + { name = "openevals", marker = "extra == 'agents'", specifier = ">=0.2,<0.3" }, { name = "packaging", specifier = ">=24.0" }, { name = "pandas", marker = "extra == 'learning'" }, { name = "pillow", marker = "extra == 'perception'" }, @@ -2283,6 +2293,7 @@ lint = [ { name = "open-clip-torch", specifier = "==3.2.0" }, { name = "openai" }, { name = "openai-whisper" }, + { name = "openevals", specifier = ">=0.2,<0.3" }, { name = "pandas-stubs", specifier = ">=2.3.2.250926,<3" }, { name = "pytest", specifier = "==8.3.5" }, { name = "python-can", specifier = ">=4" }, @@ -2322,6 +2333,7 @@ project-deps = [ { name = "ollama", specifier = ">=0.6.0" }, { name = "open-clip-torch", specifier = "==3.2.0" }, { name = "openai" }, + { name = "openevals", specifier = ">=0.2,<0.3" }, { name = "tensorboard", specifier = "==2.20.0" }, { name = "torch" }, { name = "torchreid", specifier = "==0.2.5" }, @@ -2354,6 +2366,7 @@ tests = [ { name = "ollama", specifier = ">=0.6.0" }, { name = "open-clip-torch", specifier = "==3.2.0" }, { name = "openai" }, + { name = "openevals", specifier = ">=0.2,<0.3" }, { name = "pre-commit", specifier = "==4.2.0" }, { name = "py-spy" }, { name = "pygame", specifier = ">=2.6.1" }, @@ -2408,6 +2421,7 @@ tests-self-hosted = [ { name = "ollama", specifier = ">=0.6.0" }, { name = "open-clip-torch", specifier = "==3.2.0" }, { name = "openai" }, + { name = "openevals", specifier = ">=0.2,<0.3" }, { name = "pre-commit", specifier = "==4.2.0" }, { name = "py-spy" }, { name = "pybind11", specifier = ">=2.12" }, @@ -2489,7 +2503,7 @@ wheels = [ [[package]] name = "dm-control" -version = "1.0.43" +version = "1.0.44" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "absl-py" }, @@ -2510,9 +2524,9 @@ dependencies = [ { name = "setuptools" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f0/eb/dcb0528623f58196522d74e7b2ec4a0cb5e890b9c5d28c7730de9e346d1a/dm_control-1.0.43.tar.gz", hash = "sha256:8f0e27246939cbafb3ca37d5a620056b77a95b5b8ba061b33a7aaca3d2185338", size = 56276100, upload-time = "2026-06-22T18:55:42.307Z" } +sdist = { url = "https://files.pythonhosted.org/packages/96/52/082ddf62082d13753caca4773f16ce2fb2034be0ce7a9ae199d6490b7a28/dm_control-1.0.44.tar.gz", hash = "sha256:6a5daa60130a8ca1ada40d8a2f89ed96dab2e2f3a6b32e408cf03ff86ec87d5d", size = 56276738, upload-time = "2026-07-28T02:06:26.507Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/50/4d9a4ea0ceb5c9399412834877fba79acb4438bac9c520a9cab57513a93b/dm_control-1.0.43-py3-none-any.whl", hash = "sha256:4d79532c44fb7660825b1c201d0172b14f0c8585a1f39f6cbc0181ee16e5f8e2", size = 56446782, upload-time = "2026-06-22T18:55:36.927Z" }, + { url = "https://files.pythonhosted.org/packages/5e/12/49c1795d760dcee49f8fc2e619dc9fd2fcb5b7020822b1ae71fb5b5ef760/dm_control-1.0.44-py3-none-any.whl", hash = "sha256:4353cdf0eca4964ee776ddca680d3cba1056f0cbf57fc4fd6c31c3594a2087c5", size = 56447560, upload-time = "2026-07-28T02:06:21.521Z" }, ] [[package]] @@ -5330,7 +5344,7 @@ wheels = [ [[package]] name = "mujoco" -version = "3.10.0" +version = "3.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "absl-py" }, @@ -5340,23 +5354,20 @@ dependencies = [ { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "pyopengl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a2/3b/c76837b7fdb007f7605ff783689a0bd23a5a49b065928bec2f1fa7ea3d67/mujoco-3.10.0.tar.gz", hash = "sha256:c9e8d5d87d82204ed5bccc87d843c0a53e75aaf381de2938ec46d04f1ac6e24e", size = 1094987, upload-time = "2026-06-22T17:40:59.904Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/2d/290f3f4062eec1c0d5246de0a75f1b18b59a1961081f49ddc97333fc8263/mujoco-3.11.0.tar.gz", hash = "sha256:390856102af9547dfd87cb2791eb105a3d2e00a37323fe9a12ed17e512aff1a6", size = 1139880, upload-time = "2026-07-28T01:23:32.252Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/57/ed/9b8df6c801fa25542d4c2dc417063611474a070c4bef52e896fa57726d94/mujoco-3.10.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:c1c9dfb4ba3f1ef14b70968e9cd41b14fa1877f9697369953a471aa17324f443", size = 7745281, upload-time = "2026-06-22T17:39:47.198Z" }, - { url = "https://files.pythonhosted.org/packages/09/be/3d9a1ecfe3501a84ece0d5824ff67a0933337650fb48b26c8db0b43de0cd/mujoco-3.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c8e4688d87b85be27dfcfdf957f05f1450a99131f9f8b1818613a2e4fd8d7321", size = 19324219, upload-time = "2026-06-22T17:39:49.666Z" }, - { url = "https://files.pythonhosted.org/packages/03/37/5580b126403510a80a059a66d3ea21f3e55d47960e480e66ff2a7723b514/mujoco-3.10.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b4e1e2c0c72200340d413da4211b56a8885a7ed78e893f36d97e5696e8f4af3e", size = 19628590, upload-time = "2026-06-22T17:39:53.624Z" }, - { url = "https://files.pythonhosted.org/packages/d6/3c/e3768418794c4450c6bef971eaa2314e1fac7d46abc6968b6722b0b654a0/mujoco-3.10.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47a89c371cc171a38eb6c03172aff2f905e54c1261d87b3575a9d77fe3a29a55", size = 20763444, upload-time = "2026-06-22T17:39:56.559Z" }, - { url = "https://files.pythonhosted.org/packages/af/3c/a3c5121ca9356e78dd1f09ad48e7fa034b91e02124d533e64252c314b646/mujoco-3.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:2f71cb614559cd06a8ce57bd255146e15ccb49ee7dea316aeb895838ba82e1e0", size = 17719988, upload-time = "2026-06-22T17:39:59.614Z" }, - { url = "https://files.pythonhosted.org/packages/45/bd/c3a4ad6884e60bbbff3d77df75358570bcf9b97ff8d81a0ea3b311b0276e/mujoco-3.10.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:62b7e9faf714f1582e1dd923ba3ea769a939cc572f6d77909752cbb31db5409f", size = 7758349, upload-time = "2026-06-22T17:40:02.329Z" }, - { url = "https://files.pythonhosted.org/packages/41/69/4c55c05fe602d72be5526d911e6802de1e67ad70c9301f556eef1d78a4bb/mujoco-3.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0b812e0e36e8b2dde8ce4ad9b25e189b658b9c94f5e798aa64783261abb88321", size = 19348907, upload-time = "2026-06-22T17:40:04.634Z" }, - { url = "https://files.pythonhosted.org/packages/05/19/a8a560f29f7f0137da6d41d633bc892a9e8ebc36af64c31a3db5882d26d8/mujoco-3.10.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0eff9fb64c39c21f5e29f39996c152ed9a2a5c783818b524c77abf41f6e5751", size = 19654107, upload-time = "2026-06-22T17:40:07.594Z" }, - { url = "https://files.pythonhosted.org/packages/b1/07/a37fc7fa55d38e9225884b80c3d241e669357e83f7e23f80b8860a7e14cd/mujoco-3.10.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5489e18a8dd09da2dd71d28563e17889a2e5a0ad07943ebcaa1446d6c4d4e1dd", size = 20789312, upload-time = "2026-06-22T17:40:10.656Z" }, - { url = "https://files.pythonhosted.org/packages/82/84/6548d32afc49fb79015a0b98d1119628ce94dd8befb4526c2cd10429e13f/mujoco-3.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:45a27a8982e6f89f09e87e4578a5d3e7c9b5667f3db07052e518993b78c9b3ea", size = 17757305, upload-time = "2026-06-22T17:40:13.448Z" }, - { url = "https://files.pythonhosted.org/packages/03/a2/4dd9f4cec6ce92f836a8b2de1cc799c4458af1467d7a044ef8014217bdb4/mujoco-3.10.0-cp312-cp312-macosx_10_16_x86_64.whl", hash = "sha256:47d4a22b7667c60e24e7ef6acb027c13abe9abba9acf17cc8db6fb250ba275ea", size = 7772567, upload-time = "2026-06-22T17:40:16.539Z" }, - { url = "https://files.pythonhosted.org/packages/5d/7d/ebe5342c136de27e0c430ba781f829df2cd66c00ed22627c1964fbd5d7fe/mujoco-3.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a4d35e9d0b13ff9ad3196294a7dac363f1d0cdaa988832d0b687d42d98f4ee29", size = 19380823, upload-time = "2026-06-22T17:40:19.211Z" }, - { url = "https://files.pythonhosted.org/packages/d8/5d/43d1b2b9fe97676e5af03020e132ac497b45a0333a4c61de657d0d52170a/mujoco-3.10.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb7f0d7c148a588f3633020807fc0ec3f3a9aff1f647406e3e0ffe96b05dfd57", size = 19705628, upload-time = "2026-06-22T17:40:22.549Z" }, - { url = "https://files.pythonhosted.org/packages/c3/11/c69199e4123935f98068ab6ab6b35955b4de0f6a91d3f9883805a5789394/mujoco-3.10.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:966d12f88e77e2b188e7530667b519d6963b9b906cff83bab534e5e4279325a0", size = 20904309, upload-time = "2026-06-22T17:40:25.861Z" }, - { url = "https://files.pythonhosted.org/packages/47/13/07bf2550c7dcd69ee8c7fd1f5c400a4ba2e4ede0a29a463ad3ac4cc9da90/mujoco-3.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:708edb5aceee96f2767b1072641523060043b2c67000e39e6e9797addf073696", size = 17865123, upload-time = "2026-06-22T17:40:28.996Z" }, + { url = "https://files.pythonhosted.org/packages/ab/4f/cac3ffdff6fbd63860d1be28a7f45fa1206ed638d4c774fe4505b9563c7c/mujoco-3.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d830dab9f43781980d1e673f06938d80261f9960d8086d8a221ba0118b68c79", size = 17455595, upload-time = "2026-07-28T01:22:20.595Z" }, + { url = "https://files.pythonhosted.org/packages/be/f5/f8dfdbc9964b24bb1c2fb60467474d7a85f160c609ce29099d6eda9806ec/mujoco-3.11.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aac0102660762fad316cea63b81765593c97bdda0e22251eb64ff561f78e6853", size = 17632593, upload-time = "2026-07-28T01:22:23.616Z" }, + { url = "https://files.pythonhosted.org/packages/64/f5/9bb09c44bbc7b3d844aed8517f130cbc99e565b5f5e8a55c8f06f40b8129/mujoco-3.11.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:39b813755f4b09b75a0ccaec0e1e37d313bc7c9ca1be94203a9aae3b7829cae0", size = 18727136, upload-time = "2026-07-28T01:22:26.625Z" }, + { url = "https://files.pythonhosted.org/packages/43/40/e5123f0502dbb61158fa8f73b6d5576d3a818e3b3b07794732760759acf8/mujoco-3.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:c82557572b279dd61540f2da6d61afed64ef765c210e24abeee4f2b2a57660bb", size = 15651634, upload-time = "2026-07-28T01:22:29.765Z" }, + { url = "https://files.pythonhosted.org/packages/82/cb/eaa909bfdb093d82b80518182db89936fd0cc5486aeac31e71d9940e4800/mujoco-3.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:87a70b79d461b3afe03d1cd3dfb44b9ba1955a80b011a87c9536a6ef9c7936c8", size = 17474359, upload-time = "2026-07-28T01:22:32.578Z" }, + { url = "https://files.pythonhosted.org/packages/68/d6/74fcb2a95b21de5217f19d9eb87db923d337e204da4dedaffeacababd28a/mujoco-3.11.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d09bcec0d9338fd79095314c4bd3a3072d7063e94a27d47f78e6159ca9a2baa9", size = 17655486, upload-time = "2026-07-28T01:22:35.33Z" }, + { url = "https://files.pythonhosted.org/packages/43/3d/3c933a7a8e7f00e12260ad175195b8bbc36fd3aa62068f00db36351a2147/mujoco-3.11.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:16f94f05745225aaafb68016cbd2a1617f4af6b4bfca6c97d22c7b14e598d1f1", size = 18751041, upload-time = "2026-07-28T01:22:38.149Z" }, + { url = "https://files.pythonhosted.org/packages/a7/eb/e09d6ac38f3e7af2fec0b3501515973f8a400d9c3d1e22e727a21762f598/mujoco-3.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:5a286601c6a73a21f576ac6405e842a3221eb7db53013084a5c2e341a35112ee", size = 15687351, upload-time = "2026-07-28T01:22:40.864Z" }, + { url = "https://files.pythonhosted.org/packages/87/b3/b2b449b5978bcb13277c9e50d764e6d8cd9534f58f071fb1647724123e2c/mujoco-3.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3609c198de090c8218b9cc62ca6133f0feede8edd103b743b8002f3f735fcd9b", size = 17511145, upload-time = "2026-07-28T01:22:43.668Z" }, + { url = "https://files.pythonhosted.org/packages/1a/65/d8130bb8a673f9cdc8a8fb2ef02f9b6931708567de57f17395106e83b4e3/mujoco-3.11.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:517bff03623c3329a563f575afd7d731b0be5c4f92a54dcec934b99120b4a9ec", size = 17704436, upload-time = "2026-07-28T01:22:47.05Z" }, + { url = "https://files.pythonhosted.org/packages/41/d6/7b79f5d8fbd019658a3b5feb6ffd09c1e727eaf93f518685d3cc105a28f5/mujoco-3.11.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f40214fefc8c2fe0002a3c8abadf30de7a3330634f3c45cc29b407b40a0173fc", size = 18868030, upload-time = "2026-07-28T01:22:50.494Z" }, + { url = "https://files.pythonhosted.org/packages/19/46/f994cd7d973c4db4f6b5af19ba6eb62703b9aafc8f894c5bc5ceadd31b0d/mujoco-3.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:98c08a5eec9411ecd6f763f1e635fbc8f31e3ea3701584d6b7edcc24d8d6c575", size = 15791637, upload-time = "2026-07-28T01:22:53.922Z" }, ] [[package]] @@ -6038,6 +6049,21 @@ dependencies = [ { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')" }, ] +[[package]] +name = "openevals" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain" }, + { name = "langchain-openai" }, + { name = "langsmith" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/b1/028a05846136805b29b7a3afb58a940c2d213fa2c3d0a7d7003c7fbaa115/openevals-0.2.0.tar.gz", hash = "sha256:7e95fa64625be53eaa8c657d7f69b842a52bda10bdf3bb91781c7d09a385b069", size = 140711, upload-time = "2026-04-07T19:45:22.749Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/8b/00f402b7f3475e235c339a9bc82d2eaf46cc08b53ecd85d4a117850170d3/openevals-0.2.0-py3-none-any.whl", hash = "sha256:2bce5964be9d162e3d38c2dfd026739156e1ac521536ade6b8e2f0a89b632f2c", size = 106958, upload-time = "2026-04-07T19:45:21.575Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.42.1" @@ -7277,11 +7303,11 @@ wheels = [ [[package]] name = "pyglet" -version = "2.1.15" +version = "2.1.16" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/ee/5caebffaf0345e5ccf342c30605898bd07654cee626154f7af2e465cc81c/pyglet-2.1.15.tar.gz", hash = "sha256:0ef34fe730808a97e48c24dd6ed4ff614024b980955ed8dbc2dd01ededaa08cb", size = 6597005, upload-time = "2026-06-28T11:39:09.98Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8d/af/72a0c4fc95339ec590f441a591143e04acb27b2c94845735809c1630ade6/pyglet-2.1.16.tar.gz", hash = "sha256:ca90ee05532ced8330b8a8087ac2a7b69589b7fa7fb67f85aabe24aaf4ae42c0", size = 6598246, upload-time = "2026-08-01T01:32:35.895Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/a8/9a203001fa496934708baf6cadc54b62e453f1256d5d88589c71019dd751/pyglet-2.1.15-py3-none-any.whl", hash = "sha256:8709131baf5e96c496d38197ff9f9207ad06f273c9694570008b145c58bf51ae", size = 1036800, upload-time = "2026-06-28T11:39:04.717Z" }, + { url = "https://files.pythonhosted.org/packages/23/93/b81e304f8954876fed84c5afffe83d2c1c0ceed064d5003b87fbfe9f9076/pyglet-2.1.16-py3-none-any.whl", hash = "sha256:27f9cabf97a64b15d335cf70d9e8979bbab5f84a9900a058206bcd3a2de341ae", size = 1037857, upload-time = "2026-08-01T01:32:30.566Z" }, ] [[package]]