feat(evals): semantic object VQA + ImageDetections3DPC.agent_encode (evo autoresearch, 0.355 -> 0.9997)

dimos/cli/dimos.py

@@ -798,6 +798,10 @@ def dataprep_inspect(
 
 main.add_typer(mem_app, name="mem")
 
+from dimos.evals.cli import app as evals_app
+
+main.add_typer(evals_app, name="evals")
+
 
 @main.command()
 def cameracalibrate(

dimos/evals/cli.py

@@ -0,0 +1,68 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""``dimos evals`` — run and list eval suites. Heavy imports stay inside
+command bodies (test_cli_startup budget)."""
+
+from __future__ import annotations
+
+import importlib
+
+import typer
+
+app = typer.Typer(help="Run agent evals on recordings, sim, or a live robot.")
+
+
+@app.command("run")
+def run(
+    suite: str = typer.Argument(
+        help="Dotted suite module exporting SUITE, e.g. dimos.evals.suites.go2_smoke"
+    ),
+    tags: str = typer.Option("", help="Comma-separated tag filter"),
+    model: str = typer.Option("", help="Override chat model"),
+    blind: bool = typer.Option(False, help="Withhold observations (guessing ablation)"),
+    attach: bool = typer.Option(False, help="Drive an already-running dimos (interactive cases)"),
+    limit: int = typer.Option(0, help="Run at most N cases"),
+    live_db: str = typer.Option("recording.db", help="Live Recorder db (interactive cases)"),
+) -> None:
+    from dimos.evals.runner import EvalRunner, summarize
+
+    cases = importlib.import_module(suite).SUITE
+    overrides: dict[str, object] = {"blind": blind, "attach": attach, "live_db": live_db}
+    if model:
+        overrides["model"] = model
+    runner = EvalRunner(**overrides)
+    results = runner.run(
+        cases,
+        tags=frozenset(t for t in tags.split(",") if t) if tags else frozenset(),
+        limit=limit,
+    )
+
+    for r in results:
+        status = "ERROR" if r.error else ("PASS" if r.passed else "fail")
+        detail = r.error or f"score={r.score:.2f} answer={r.outputs[:60]!r}"
+        typer.echo(f"{status:5} {r.case_id:30} {detail} ({r.duration_s:.1f}s)")
+    s = summarize(results)
+    typer.echo(
+        f"\n{s.n} cases | mean {s.mean_score:.2f} | pass {s.pass_rate:.0%} "
+        f"| errors {s.errors} | {s.duration_s:.0f}s | {runner.run_dir}"
+    )
+
+
+@app.command("list")
+def list_() -> None:
+    from dimos.evals.module import list_suites
+
+    for name in list_suites():
+        typer.echo(name)

dimos/evals/generate.py

@@ -0,0 +1,88 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Eval-row generators — deferred feature (PRD: low priority), kept minimal.
+
+Ground truth is computed analytically from a *privileged* modality; the emitted
+case quizzes a different (or lossily-encoded) surface. Rows are pure data —
+a suite module maps them onto typed :class:`PassiveEval` cases.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from dimos.memory2.cli.dataset import open_dataset
+
+Row = dict[str, object]
+
+
+def displacement_rows(dataset: str, windows: Sequence[tuple[float, float]]) -> list[Row]:
+    """Straight-line displacement over each window (odom is the privileged truth;
+    the case quizzes the encoded odom summary). Sampling-invariant ground truth."""
+    store = open_dataset(dataset)
+    try:
+        rows: list[Row] = []
+        for t1, t2 in windows:
+            obs = store.streams.odom.range_time(t1, t2).to_list()
+            if len(obs) < 2:
+                continue
+            d = (obs[-1].data.position - obs[0].data.position).length()
+            rows.append(
+                {
+                    "id": f"{dataset}_disp_{t1:g}_{t2:g}",
+                    "q": "How far in a straight line is your final position from your "
+                    "position at the first shown observation, in meters?",
+                    "a": round(d, 1),
+                    "band": max(1.0, d * 0.4),
+                    "stream": "odom",
+                    "window": [t1, t2],
+                    "dataset": dataset,
+                }
+            )
+        return rows
+    finally:
+        store.stop()
+
+
+def path_length_rows(dataset: str, windows: Sequence[tuple[float, float]]) -> list[Row]:
+    """Integrated path length per window. Deliberately hard on a downsampled
+    encoding — expect partial credit; that gap is the finding."""
+    store = open_dataset(dataset)
+    try:
+        rows: list[Row] = []
+        for t1, t2 in windows:
+            path, prev = 0.0, None
+            for obs in store.streams.odom.range_time(t1, t2):
+                p = obs.data.position
+                if prev is not None:
+                    path += (p - prev).length()
+                prev = p
+            if prev is None:
+                continue
+            rows.append(
+                {
+                    "id": f"{dataset}_path_{t1:g}_{t2:g}",
+                    "q": "Roughly how many meters did you travel in total over these "
+                    "observations (path length, not displacement)?",
+                    "a": round(path, 1),
+                    "band": max(2.0, path * 0.5),
+                    "stream": "odom",
+                    "window": [t1, t2],
+                    "dataset": dataset,
+                }
+            )
+        return rows
+    finally:
+        store.stop()

dimos/evals/intro.md

@@ -0,0 +1,202 @@
+# Evals Intro
+
+Evals measure what an agent (or a bare model, or a single skill) can do with
+the robot's memory. Two kinds:
+
+- **Passive** — the world is a frozen memory2 recording. Deterministic, cheap,
+  repeatable. Run these constantly.
+- **Interactive** — a live robot or sim; actions change the world; scoring
+  samples the live memory2 store while the agent works.
+
+memory2 is the source of truth for everything an eval sees: context selectors
+return real `Stream`s, and interactive scoring reads a real `Store`.
+
+## Quick start (CLI)
+
+```bash
+# two documentation cases against the go2_short recording (needs OPENAI_API_KEY)
+dimos evals run dimos.evals.suites.examples
+
+# same questions with observations withheld — the guessing ablation
+dimos evals run dimos.evals.suites.examples --blind
+
+# list available suites
+dimos evals list
+```
+
+Each run prints a per-case table and writes `results.jsonl`, `summary.json`,
+and per-case transcripts to `~/.local/state/dimos/evals/run-*/`.
+
+## Your first eval, end to end
+
+Build a tiny recording (any memory2 store works — this is the same API the
+robot's Recorder uses; see `dimos/memory2/intro.md` for the full Stream API):
+
+```python session=evals ansi=false no-result
+import os
+from pathlib import Path
+
+os.environ["DIMOS_LOG_LEVEL"] = "WARNING"  # keep doc output stable
+
+from dimos.memory2.store.sqlite import SqliteStore
+from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped
+from dimos.msgs.geometry_msgs.Quaternion import Quaternion
+from dimos.msgs.geometry_msgs.Vector3 import make_vector3
+
+Path("/tmp/evals_intro.db").unlink(missing_ok=True)
+store = SqliteStore(path="/tmp/evals_intro.db")
+odom = store.stream("odom", PoseStamped)
+for i in range(20):
+    odom.append(
+        PoseStamped(position=make_vector3(float(i), 2.5, 0.0),
+                    orientation=Quaternion(0, 0, 0, 1), frame_id="world"),
+        ts=1000.0 + i,
+    )
+```
+
+```python session=evals ansi=false
+print(odom.summary())
+```
+
+```results
+Stream("odom"): 20 items, 1970-01-01 00:16:40 — 1970-01-01 00:16:59 (19.0s, 1.00 Hz, 1.68 KiB)
+```
+
+A passive eval is one Python literal. `context` is a tuple of callables that
+receive the opened `Store` and return the mem2 `Stream`s the model may see —
+anything the Stream API expresses (windows, filters, single frames) works, and
+the runner evenly downsamples each selected stream to `context_budget`
+observations before encoding:
+
+```python session=evals ansi=false no-result
+from dimos.evals.scorers import first_number, within
+from dimos.evals.types import PassiveEval
+
+case = PassiveEval(
+    id="how_far",
+    inputs="How far along x did you travel, in meters?",
+    expected=19.0,
+    parse=first_number,          # model text -> float
+    score=within(1.0),           # graded: 1.0 exact, linear to 0 at ±1m
+    context=(lambda s: s.streams.odom,),
+    dataset="/tmp/evals_intro.db",   # a mem2 name ("go2_short") or a path
+)
+```
+
+Run it. `chat_model=` injects any LangChain chat model — here a canned fake so
+this document runs offline; drop the argument to use the production model
+config (`gpt-5.6-luna`, same construction as the deployed `McpClient`):
+
+```python session=evals ansi=false
+from langchain_core.language_models.fake_chat_models import FakeListChatModel
+from dimos.evals.runner import EvalRunner, summarize
+
+runner = EvalRunner(chat_model=FakeListChatModel(responses=["about 19 meters"]))
+result = runner.run([case])[0]
+print(f"score={result.score} passed={result.passed} outputs={result.outputs!r}")
+s = summarize([result])
+print(f"n={s.n} mean={s.mean_score} pass_rate={s.pass_rate} errors={s.errors}")
+```
+
+```results
+score=1.0 passed=True outputs='about 19 meters'
+n=1 mean=1.0 pass_rate=1.0 errors=0
+```
+
+That's the whole loop: dataset -> context streams -> encoded prompt -> model
+-> parse -> score -> artifacts.
+
+## Scoring
+
+Scores are floats in `[0, 1]`; `passed = score >= threshold`. Scorers are
+plain functions `(expected, got) -> float` — a custom heuristic is a lambda,
+not a class:
+
+```python session=evals ansi=false
+from dimos.evals.scorers import choice, exact, first_number, ramp, within, yes_no
+
+print(exact("yes", "yes"), within(2.0)(10.0, 11.0), ramp(1.0, band=2.0))
+print(first_number("around 12.5 m"), yes_no("Yes, clearly."), choice(" Chairs. "))
+```
+
+```results
+1.0 0.5 0.5
+12.5 yes chairs
+```
+
+- `exact` — equality (the default). Pair with a parser (`yes_no`, `choice`,
+  `int`) so formatting noise doesn't fail a correct answer.
+- `within(band)` — graded numeric credit: 1.0 exact, 0.5 halfway, 0 outside.
+- `ramp(distance, band)` — same ramp over meters; msg types support
+  arithmetic, so physical scorers stay one-liners:
+  `lambda s: ramp((GOAL - s.streams.odom.last().data.position).length(), band=0.5)`
+- `judge(rubric)` — LLM-as-judge with partial credit, wrapping the
+  langchain/openevals standard (`inputs`/`reference_outputs` convention, so
+  external VQA benchmarks map on natively).
+
+Interactive evals score a *series* (one sample per `interval_s`); `aggregate`
+reduces it:
+
+```python session=evals ansi=false
+from dimos.evals.scorers import final, floor, mean
+
+print(final([0.2, 0.9]), floor([0.4, 0.2, 0.8]), mean([0.0, 1.0]))
+```
+
+```results
+0.9 0.2 0.5
+```
+
+`final` = "where did it end up", `floor` = "never left the zone",
+`mean` = "how good was it throughout".
+
+## Interactive evals
+
+The case names its environment (reproducibility); `score` reads the **live**
+store the robot's Recorder writes, sampled every `interval_s`:
+
+```python session=evals ansi=false no-result
+from dimos.evals.scorers import final, ramp
+from dimos.evals.types import InteractiveEval
+from dimos.msgs.geometry_msgs.Vector3 import Vector3
+
+BED = Vector3(-3.567, -1.332, 0.0)
+
+go_to_bed = InteractiveEval(
+    id="go_to_bed",
+    inputs="go to the bed",
+    score=lambda s: ramp((BED - s.streams.odom.last().data.position).length(), band=2.0),
+    aggregate=final,
+    interval_s=2.0,
+    timeout_s=180.0,
+    blueprint="unitree-go2-agentic go2-memory",
+    simulator="dimsim",
+    scene="apartment",
+)
+```
+
+```bash
+dimos evals run dimos.evals.suites.dimsim_house --live-db recording_go2.db
+```
+
+The result carries the full `(t, score)` series — "reached the bed at t=50s
+and stayed" and "grazed it at the deadline" score differently under `floor`
+vs `final`.
+
+## Running
+
+- **CLI**: `dimos evals run <dotted.suite> [--tags nav --blind --limit 5 --model gpt-4o]`
+- **Python**: `EvalRunner(...).run(SUITE, tags=frozenset({"encoding"}))`
+- **pytest**: suites are importable lists —
+  `@pytest.mark.parametrize("case", SUITE)` and assert on `passed`
+  (gate live-model tests with `skipif_no_openai`).
+- **MCP**: the `EvalModule` skills `run_evals` / `list_eval_suites` return the
+  summary + run dir, so a coding agent can run evals, grep transcripts, edit
+  prompts/encodings, and run again.
+- **Blind ablation**: `EvalRunner(blind=True)` withholds all observations. A
+  case that still passes blind is guessable — fix its distractors. Run every
+  new suite sighted and blind once before trusting it.
+- **Preflight**: before anything runs, every case is checked against the rig —
+  a missing stream fails with `"No stream 'lidar'. Available: [...]"`, a case
+  needing MCP/sim fails with what's missing. Errors are per-case; one broken
+  case never kills a run.