feat(evals): add code-as-policy evaluation runtime

CONTEXT.md

@@ -0,0 +1,49 @@
+# Evaluation
+
+The language used to describe how DimOS measures task-performing behavior across recorded, simulated, and live environments.
+
+## Language
+
+**Evaluation**:
+A complete benchmark integration that owns its environment lifecycle, cases, scoring, aggregation, and native result semantics. Third-party evaluations retain their original harness rather than translating it into DimOS scoring primitives.
+_Avoid_: Runtime, universal scorer
+
+**Evaluation Case**:
+A fixed task, environment, budget, and scoring definition. It does not select the behavior being evaluated.
+_Avoid_: Policy configuration, run
+
+**Evaluation Run**:
+One attempt of an evaluation case by a selected policy. Different policies can attempt the same case without changing its definition.
+_Avoid_: Case, suite
+
+**Runtime**:
+The injected DimOS facility that attaches to a system launched by an evaluation and executes the selected policy. It does not own the benchmark environment or scoring.
+_Avoid_: Evaluation, benchmark harness
+
+**Policy**:
+The callable produced during exploration and replayed without an agent during evaluation. Its canonical signature is `policy(app: Dimos) -> None`.
+_Avoid_: Agent, exploration transcript, execution mode
+
+**Policy Artifact**:
+The serialized callable and human-readable source captured when `submit_policy(policy)` is called. One artifact is produced per benchmark task and reused across its held-out evaluation cases or seeds. REPL outputs and the agent transcript are separate exploration evidence.
+_Avoid_: Exploration transcript, policy source
+
+**Exploration Stage**:
+The unscored stage in which an agent uses a persistent Python REPL and calls `submit_policy(policy)` to run complete debug trials in fresh environments and blueprints. Model latency does not consume the evaluation horizon.
+_Avoid_: Evaluation rollout, scoring
+
+**Evaluation Stage**:
+The measured stage in which the Policy Artifact executes without an agent against a reset or held-out environment. The native benchmark owns its real-time or step horizon and privileged scoring.
+_Avoid_: Agent session, policy generation
+
+**Policy Environment**:
+The capabilities exposed by the fresh policy-only DimOS blueprint while a policy runs. Simulated, live, and replay-backed evaluations all pass the policy a connected `Dimos` application; completed trials additionally expose their Memory2 recording read-only through `TrialRun`.
+_Avoid_: Agent tools, scorer context
+
+**Evaluation Oracle**:
+The evaluator-only source of truth used to score a policy attempt. In simulation it contains privileged state, such as true poses and object identities, that the Policy Environment cannot access.
+_Avoid_: Runtime memory, perception output
+
+**Agent**:
+The model-backed participant in the Exploration Stage that issues code through the runtime's Python REPL and produces a Policy Artifact. It is absent from the Evaluation Stage and does not receive the evaluation oracle.
+_Avoid_: Evaluator, runner

dimos/agents/code_policy_core.py

@@ -0,0 +1,325 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Persistent, module-independent Python session for trusted CodePolicy agents."""
+
+from __future__ import annotations
+
+import base64
+import inspect
+import os
+import queue
+import re
+import threading
+import time
+from typing import TYPE_CHECKING, Any, get_type_hints
+
+from pydantic import BaseModel, ConfigDict, Field
+
+if TYPE_CHECKING:
+    from dimos.benchmark.evaluation.protocol import TrialRun
+
+MAX_EXECUTION_TIMEOUT_S = 600.0
+DEFAULT_OUTPUT_LIMIT = 32_000
+_SUBMISSION_URL_ENV = "DIMOS_CODE_POLICY_SUBMISSION_URL"
+_SUBMISSION_TOKEN_ENV = "DIMOS_CODE_POLICY_SUBMISSION_TOKEN"
+_ANSI_ESCAPE_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]")
+_TRUNCATION_MARKER = "\n... [output truncated]"
+_CREDENTIAL_NAME_RE = re.compile(
+    r"(?:API_?KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|AUTH|OPENAI|ANTHROPIC|AWS_|AZURE_)",
+    re.IGNORECASE,
+)
+
+
+class CodePolicySessionConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+    submission_url: str = Field(min_length=1)
+    submission_token: str = Field(min_length=1)
+    output_limit: int = Field(default=DEFAULT_OUTPUT_LIMIT, ge=0)
+    startup_timeout_s: float = Field(default=10.0, gt=0)
+    interrupt_grace_s: float = Field(default=2.0, gt=0)
+
+
+class _BoundedOutput:
+    def __init__(self, limit: int) -> None:
+        self.limit = limit
+        self.parts: list[str] = []
+        self.length = 0
+        self.truncated = False
+
+    def __call__(self, message: dict[str, Any]) -> None:
+        message_type = message.get("header", {}).get("msg_type")
+        content = message.get("content", {})
+        value = ""
+        if message_type == "stream":
+            value = str(content.get("text", ""))
+        elif message_type in {"execute_result", "display_data"}:
+            value = str(content.get("data", {}).get("text/plain", ""))
+        elif message_type == "error":
+            traceback = content.get("traceback", [])
+            value = "\n".join(str(line) for line in traceback)
+        self._append(_ANSI_ESCAPE_RE.sub("", value))
+
+    def _append(self, value: str) -> None:
+        if not value or self.truncated:
+            return
+        remaining = self.limit - self.length
+        if len(value) <= remaining:
+            self.parts.append(value)
+            self.length += len(value)
+            return
+        marker = _TRUNCATION_MARKER[:remaining]
+        content_limit = max(0, remaining - len(marker))
+        self.parts.append(value[:content_limit] + marker)
+        self.length = self.limit
+        self.truncated = True
+
+    def text(self) -> str:
+        return "".join(self.parts)
+
+
+def _load_kernel_manager() -> type[Any]:
+    try:
+        from jupyter_client.manager import KernelManager
+    except ImportError as exc:
+        raise RuntimeError(
+            "CodePolicy requires ipykernel and jupyter-client; install the agents extra"
+        ) from exc
+    return KernelManager
+
+
+def _bootstrap_source() -> str:
+    return """
+from dimos.agents.code_policy_core import submit_policy
+from dimos.porcelain.dimos import Dimos
+"""
+
+
+def _kernel_environment(config: CodePolicySessionConfig) -> dict[str, str]:
+    """Build an exploration environment without forwarding host credentials."""
+    result = {
+        name: value for name, value in os.environ.items() if not _CREDENTIAL_NAME_RE.search(name)
+    }
+    result[_SUBMISSION_URL_ENV] = config.submission_url
+    result[_SUBMISSION_TOKEN_ENV] = config.submission_token
+    return result
+
+
+def submit_policy(policy: Any) -> TrialRun:
+    """Submit a typed callable from the exploration kernel for one fresh trial."""
+    validate_policy_callable(policy)
+    try:
+        source = inspect.getsource(policy)
+    except (OSError, TypeError) as exc:
+        raise TypeError("policy source is unavailable; define it in the exploration REPL") from exc
+    try:
+        import cloudpickle  # type: ignore[import-untyped]
+
+        serialized = cloudpickle.dumps(policy)
+    except Exception as exc:
+        raise TypeError(f"policy is not serializable: {type(exc).__name__}: {exc}") from exc
+
+    import requests
+
+    response = requests.post(
+        os.environ[_SUBMISSION_URL_ENV],
+        headers={"Authorization": f"Bearer {os.environ[_SUBMISSION_TOKEN_ENV]}"},
+        json={"source": source, "serialized": base64.b64encode(serialized).decode("ascii")},
+        timeout=None,
+    )
+    if response.status_code != 200:
+        detail = response.json().get("error", response.text)
+        raise RuntimeError(f"policy submission failed: {detail}")
+    from pathlib import Path
+
+    from dimos.benchmark.evaluation.protocol import TrialOutcome, TrialRun
+
+    payload = response.json()
+    return TrialRun(
+        run_id=payload["run_id"],
+        outcome=TrialOutcome(**payload["outcome"]),
+        artifacts=Path(payload["artifacts"]),
+        log_path=Path(payload["log_path"]),
+        memory_path=Path(payload["memory_path"]),
+    )
+
+
+def validate_policy_callable(policy: Any) -> None:
+    """Enforce the one canonical callable contract before a trial is launched."""
+    from dimos.porcelain.dimos import Dimos
+
+    if not inspect.isfunction(policy) or inspect.iscoroutinefunction(policy):
+        raise TypeError("policy must be a synchronous Python function")
+    if policy.__name__ != "policy":
+        raise TypeError("submitted function must be named 'policy'")
+    signature = inspect.signature(policy)
+    parameters = list(signature.parameters.values())
+    if len(parameters) != 1 or parameters[0].kind not in {
+        inspect.Parameter.POSITIONAL_ONLY,
+        inspect.Parameter.POSITIONAL_OR_KEYWORD,
+    }:
+        raise TypeError("policy must accept exactly one positional app parameter")
+    try:
+        hints = get_type_hints(policy)
+    except Exception as exc:
+        raise TypeError(f"policy annotations could not be resolved: {exc}") from exc
+    if hints.get(parameters[0].name) is not Dimos or hints.get("return") not in {None, type(None)}:
+        raise TypeError("policy signature must be policy(app: Dimos) -> None")
+
+
+class CodePolicySession:
+    """Execute trusted Python serially in one persistent Jupyter kernel."""
+
+    def __init__(self, config: CodePolicySessionConfig) -> None:
+        self.config = config
+        self.execution_count = 0
+        self.execution_duration_s = 0.0
+        self._execution_lock = threading.Lock()
+        self._kernel_lock = threading.RLock()
+        self._manager: Any = None
+        self._client: Any = None
+        self._stopped = True
+
+    def start(self) -> None:
+        self._stopped = False
+
+    def python_exec(self, code: str, timeout_s: float = MAX_EXECUTION_TIMEOUT_S) -> str:
+        if self._stopped:
+            return "CodePolicy session is stopped"
+        if not code:
+            return "python_exec code must be non-empty"
+        if not 0 < timeout_s <= MAX_EXECUTION_TIMEOUT_S:
+            return f"timeout_s must be in (0, {MAX_EXECUTION_TIMEOUT_S:g}]"
+        if not self._execution_lock.acquire(blocking=False):
+            return "CodePolicy session is busy"
+        started = time.monotonic()
+        self.execution_count += 1
+        try:
+            try:
+                client = self._ensure_kernel()
+            except Exception as exc:
+                return f"CodePolicy kernel failed to start: {type(exc).__name__}: {exc}"
+            output = _BoundedOutput(self.config.output_limit)
+            try:
+                reply = client.execute_interactive(
+                    code,
+                    allow_stdin=False,
+                    output_hook=output,
+                    store_history=True,
+                    timeout=timeout_s,
+                )
+            except (TimeoutError, queue.Empty):
+                if self._interrupt_and_recover():
+                    return f"Execution timed out after {timeout_s:.1f}s and was interrupted"
+                return (
+                    f"Execution timed out after {timeout_s:.1f}s; "
+                    "the kernel was restarted and its namespace was reset"
+                )
+            except Exception as exc:
+                self._shutdown_kernel()
+                return f"CodePolicy execution failed: {type(exc).__name__}: {exc}"
+            content = reply.get("content", {})
+            body = output.text().rstrip()
+            if not body and content.get("status") != "ok":
+                body = f"{content.get('ename', 'Error')}: {content.get('evalue', '')}"
+            if not body:
+                body = "(completed)"
+            state = "completed" if content.get("status") == "ok" else "failed"
+            return f"In [{content.get('execution_count', '?')}] {state}\n\n{body}"
+        finally:
+            self.execution_duration_s += time.monotonic() - started
+            self._execution_lock.release()
+
+    def stop(self) -> None:
+        self._stopped = True
+        self._shutdown_kernel()
+
+    def _ensure_kernel(self) -> Any:
+        with self._kernel_lock:
+            if self._manager is not None and self._client is not None and self._manager.is_alive():
+                return self._client
+            self._shutdown_kernel()
+            manager = _load_kernel_manager()(kernel_name="python3")
+            client = None
+            try:
+                manager.start_kernel(env=_kernel_environment(self.config))
+                client = manager.client()
+                client.start_channels()
+                client.wait_for_ready(timeout=self.config.startup_timeout_s)
+                reply = client.execute_interactive(
+                    _bootstrap_source(),
+                    allow_stdin=False,
+                    output_hook=lambda _message: None,
+                    silent=True,
+                    store_history=False,
+                    timeout=self.config.startup_timeout_s,
+                )
+                if reply.get("content", {}).get("status") != "ok":
+                    content = reply.get("content", {})
+                    raise RuntimeError(
+                        f"{content.get('ename', 'KernelBootstrapError')}: "
+                        f"{content.get('evalue', 'bootstrap failed')}"
+                    )
+            except Exception:
+                if client is not None:
+                    client.stop_channels()
+                try:
+                    manager.shutdown_kernel(now=True)
+                    manager.cleanup_resources()
+                except Exception:
+                    pass
+                raise
+            self._manager = manager
+            self._client = client
+            return client
+
+    def _interrupt_and_recover(self) -> bool:
+        manager, client = self._manager, self._client
+        if manager is None or client is None:
+            return False
+        try:
+            manager.interrupt_kernel()
+            client.wait_for_ready(timeout=self.config.interrupt_grace_s)
+            return True
+        except Exception:
+            try:
+                manager.restart_kernel(now=True)
+                client.wait_for_ready(timeout=self.config.startup_timeout_s)
+                reply = client.execute_interactive(
+                    _bootstrap_source(),
+                    allow_stdin=False,
+                    output_hook=lambda _message: None,
+                    silent=True,
+                    store_history=False,
+                    timeout=self.config.startup_timeout_s,
+                )
+                if reply.get("content", {}).get("status") != "ok":
+                    raise RuntimeError("bootstrap failed after kernel restart")
+            except Exception:
+                self._shutdown_kernel()
+            return False
+
+    def _shutdown_kernel(self) -> None:
+        with self._kernel_lock:
+            manager, client = self._manager, self._client
+            self._manager = None
+            self._client = None
+        if client is not None:
+            client.stop_channels()
+        if manager is not None:
+            try:
+                manager.shutdown_kernel(now=True)
+                manager.cleanup_resources()
+            except Exception:
+                pass