From c8db558b63348ee70e8d41e98cbdedbe7217c83f Mon Sep 17 00:00:00 2001 From: stash Date: Sat, 8 Aug 2026 19:39:50 -0700 Subject: [PATCH 01/12] feat(evals): passive/interactive eval framework over memory2 EvalCase/PassiveEval/InteractiveEval with EvalRig protocol dispatch, EvalRunner implementing the rig (model call / mcp skill / agent loop / live-store sampling), scorers as plain functions wrapping openevals, generated + hand VQA suites over go2 replays, dimsim go-to-bed interactive suite, dimos evals CLI + EvalModule MCP skills. extracts _init_model to dimos/agents/model.py for shared use. --- dimos/agents/mcp/mcp_client.py | 19 +- dimos/agents/mcp/test_mcp_client_unit.py | 2 +- dimos/agents/model.py | 44 +++ dimos/cli/dimos.py | 4 + dimos/evals/__init__.py | 48 +++ dimos/evals/cli.py | 68 ++++ dimos/evals/generate.py | 88 +++++ dimos/evals/module.py | 64 ++++ dimos/evals/runner.py | 396 +++++++++++++++++++++++ dimos/evals/scorers.py | 106 ++++++ dimos/evals/suites/__init__.py | 15 + dimos/evals/suites/dimsim_house.py | 101 ++++++ dimos/evals/suites/examples.py | 62 ++++ dimos/evals/suites/go2_smoke.py | 81 +++++ dimos/evals/suites/go2_vqa.json | 86 +++++ dimos/evals/suites/go2_vqa.py | 73 +++++ dimos/evals/test_evals.py | 312 ++++++++++++++++++ dimos/evals/test_smoke.py | 38 +++ dimos/evals/types.py | 198 ++++++++++++ pyproject.toml | 1 + uv.lock | 21 ++ 21 files changed, 1809 insertions(+), 18 deletions(-) create mode 100644 dimos/agents/model.py create mode 100644 dimos/evals/__init__.py create mode 100644 dimos/evals/cli.py create mode 100644 dimos/evals/generate.py create mode 100644 dimos/evals/module.py create mode 100644 dimos/evals/runner.py create mode 100644 dimos/evals/scorers.py create mode 100644 dimos/evals/suites/__init__.py create mode 100644 dimos/evals/suites/dimsim_house.py create mode 100644 dimos/evals/suites/examples.py create mode 100644 dimos/evals/suites/go2_smoke.py create mode 100644 dimos/evals/suites/go2_vqa.json create mode 100644 dimos/evals/suites/go2_vqa.py create mode 100644 dimos/evals/test_evals.py create mode 100644 dimos/evals/test_smoke.py create mode 100644 dimos/evals/types.py diff --git a/dimos/agents/mcp/mcp_client.py b/dimos/agents/mcp/mcp_client.py index 859b15451b..6beddb6b07 100644 --- a/dimos/agents/mcp/mcp_client.py +++ b/dimos/agents/mcp/mcp_client.py @@ -20,16 +20,15 @@ import uuid from langchain.agents import create_agent -from langchain.chat_models import init_chat_model from langchain_core.messages import HumanMessage from langchain_core.messages.base import BaseMessage from langchain_core.tools import StructuredTool -from langchain_openai import ChatOpenAI from langgraph.graph.state import CompiledStateGraph from reactivex.disposable import Disposable import requests from dimos.agents.mcp import tool_stream +from dimos.agents.model import init_model from dimos.agents.system_prompt import SYSTEM_PROMPT from dimos.agents.utils import pretty_print_langchain_message from dimos.constants import DEFAULT_THREAD_JOIN_TIMEOUT @@ -42,20 +41,6 @@ logger = setup_logger() -_RESPONSES_REASONING_MODEL_PREFIXES = ("gpt-5", "o1", "o3", "o4") - - -def _init_model(model_name: str) -> Any: - """Initialize a model while preserving LangChain provider resolution.""" - if ":" in model_name or not model_name.startswith(_RESPONSES_REASONING_MODEL_PREFIXES): - return init_chat_model(model=model_name) - - return ChatOpenAI( - model=model_name, - use_responses_api=True, - reasoning={"effort": "medium", "summary": "auto"}, - ) - class McpClientConfig(ModuleConfig): system_prompt: str | None = SYSTEM_PROMPT @@ -233,7 +218,7 @@ def on_system_modules(self, _modules: list[RPCClient]) -> None: model = MockModel(json_path=self.config.model_fixture) else: - model = _init_model(self.config.model) + model = init_model(self.config.model) with self._lock: self._state_graph = create_agent( diff --git a/dimos/agents/mcp/test_mcp_client_unit.py b/dimos/agents/mcp/test_mcp_client_unit.py index a49df130ff..dc1af78dbf 100644 --- a/dimos/agents/mcp/test_mcp_client_unit.py +++ b/dimos/agents/mcp/test_mcp_client_unit.py @@ -260,7 +260,7 @@ def test_on_system_modules_resolves_non_reasoning_models( with ( patch("dimos.agents.mcp.mcp_client.create_agent"), - patch("dimos.agents.mcp.mcp_client.init_chat_model", return_value=resolved_model) as init, + patch("dimos.agents.model.init_chat_model", return_value=resolved_model) as init, ): configured_mcp_client.on_system_modules([]) diff --git a/dimos/agents/model.py b/dimos/agents/model.py new file mode 100644 index 0000000000..b71471c01f --- /dev/null +++ b/dimos/agents/model.py @@ -0,0 +1,44 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared chat-model construction for agents and evals.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from langchain.chat_models import init_chat_model +from langchain_openai import ChatOpenAI + +if TYPE_CHECKING: + from langchain_core.language_models.chat_models import BaseChatModel + +_RESPONSES_REASONING_MODEL_PREFIXES = ("gpt-5", "o1", "o3", "o4") + + +def init_model(model_name: str) -> BaseChatModel: + """Initialize a model while preserving LangChain provider resolution. + + OpenAI reasoning models (gpt-5*/o*) without an explicit ``provider:`` prefix + go through the Responses API with reasoning enabled — the same configuration + the production ``McpClient`` runs, so evals measure what deploys. + """ + if ":" in model_name or not model_name.startswith(_RESPONSES_REASONING_MODEL_PREFIXES): + return init_chat_model(model=model_name) + + return ChatOpenAI( + model=model_name, + use_responses_api=True, + reasoning={"effort": "medium", "summary": "auto"}, + ) diff --git a/dimos/cli/dimos.py b/dimos/cli/dimos.py index ee6a92f457..f4c83f3315 100644 --- a/dimos/cli/dimos.py +++ b/dimos/cli/dimos.py @@ -798,6 +798,10 @@ def dataprep_inspect( main.add_typer(mem_app, name="mem") +from dimos.evals.cli import app as evals_app + +main.add_typer(evals_app, name="evals") + @main.command() def cameracalibrate( diff --git a/dimos/evals/__init__.py b/dimos/evals/__init__.py new file mode 100644 index 0000000000..f43b4f11cc --- /dev/null +++ b/dimos/evals/__init__.py @@ -0,0 +1,48 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DimOS evals: passive (frozen mem2 recordings) and interactive (live robot/sim).""" + +from dimos.evals.runner import EvalRunner, EvalRunnerConfig, RunSummary, summarize +from dimos.evals.scorers import exact, final, floor, judge, mean, ramp, within +from dimos.evals.types import ( + EvalCase, + EvalResult, + EvalRig, + InteractiveEval, + PassiveEval, + Select, + Suite, +) + +__all__ = [ + "EvalCase", + "EvalResult", + "EvalRig", + "EvalRunner", + "EvalRunnerConfig", + "InteractiveEval", + "PassiveEval", + "RunSummary", + "Select", + "Suite", + "exact", + "final", + "floor", + "judge", + "mean", + "ramp", + "summarize", + "within", +] diff --git a/dimos/evals/cli.py b/dimos/evals/cli.py new file mode 100644 index 0000000000..7df7b7618e --- /dev/null +++ b/dimos/evals/cli.py @@ -0,0 +1,68 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``dimos evals`` — run and list eval suites. Heavy imports stay inside +command bodies (test_cli_startup budget).""" + +from __future__ import annotations + +import importlib + +import typer + +app = typer.Typer(help="Run agent evals on recordings, sim, or a live robot.") + + +@app.command("run") +def run( + suite: str = typer.Argument( + help="Dotted suite module exporting SUITE, e.g. dimos.evals.suites.go2_smoke" + ), + tags: str = typer.Option("", help="Comma-separated tag filter"), + model: str = typer.Option("", help="Override chat model"), + blind: bool = typer.Option(False, help="Withhold observations (guessing ablation)"), + attach: bool = typer.Option(False, help="Drive an already-running dimos (interactive cases)"), + limit: int = typer.Option(0, help="Run at most N cases"), + live_db: str = typer.Option("recording.db", help="Live Recorder db (interactive cases)"), +) -> None: + from dimos.evals.runner import EvalRunner, summarize + + cases = importlib.import_module(suite).SUITE + overrides: dict[str, object] = {"blind": blind, "attach": attach, "live_db": live_db} + if model: + overrides["model"] = model + runner = EvalRunner(**overrides) + results = runner.run( + cases, + tags=frozenset(t for t in tags.split(",") if t) if tags else frozenset(), + limit=limit, + ) + + for r in results: + status = "ERROR" if r.error else ("PASS" if r.passed else "fail") + detail = r.error or f"score={r.score:.2f} answer={r.outputs[:60]!r}" + typer.echo(f"{status:5} {r.case_id:30} {detail} ({r.duration_s:.1f}s)") + s = summarize(results) + typer.echo( + f"\n{s.n} cases | mean {s.mean_score:.2f} | pass {s.pass_rate:.0%} " + f"| errors {s.errors} | {s.duration_s:.0f}s | {runner.run_dir}" + ) + + +@app.command("list") +def list_() -> None: + from dimos.evals.module import list_suites + + for name in list_suites(): + typer.echo(name) diff --git a/dimos/evals/generate.py b/dimos/evals/generate.py new file mode 100644 index 0000000000..45bf0c4bee --- /dev/null +++ b/dimos/evals/generate.py @@ -0,0 +1,88 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Eval-row generators — deferred feature (PRD: low priority), kept minimal. + +Ground truth is computed analytically from a *privileged* modality; the emitted +case quizzes a different (or lossily-encoded) surface. Rows are pure data — +a suite module maps them onto typed :class:`PassiveEval` cases. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from dimos.memory2.cli.dataset import open_dataset + +Row = dict[str, object] + + +def displacement_rows(dataset: str, windows: Sequence[tuple[float, float]]) -> list[Row]: + """Straight-line displacement over each window (odom is the privileged truth; + the case quizzes the encoded odom summary). Sampling-invariant ground truth.""" + store = open_dataset(dataset) + try: + rows: list[Row] = [] + for t1, t2 in windows: + obs = store.streams.odom.range_time(t1, t2).to_list() + if len(obs) < 2: + continue + d = (obs[-1].data.position - obs[0].data.position).length() + rows.append( + { + "id": f"{dataset}_disp_{t1:g}_{t2:g}", + "q": "How far in a straight line is your final position from your " + "position at the first shown observation, in meters?", + "a": round(d, 1), + "band": max(1.0, d * 0.4), + "stream": "odom", + "window": [t1, t2], + "dataset": dataset, + } + ) + return rows + finally: + store.stop() + + +def path_length_rows(dataset: str, windows: Sequence[tuple[float, float]]) -> list[Row]: + """Integrated path length per window. Deliberately hard on a downsampled + encoding — expect partial credit; that gap is the finding.""" + store = open_dataset(dataset) + try: + rows: list[Row] = [] + for t1, t2 in windows: + path, prev = 0.0, None + for obs in store.streams.odom.range_time(t1, t2): + p = obs.data.position + if prev is not None: + path += (p - prev).length() + prev = p + if prev is None: + continue + rows.append( + { + "id": f"{dataset}_path_{t1:g}_{t2:g}", + "q": "Roughly how many meters did you travel in total over these " + "observations (path length, not displacement)?", + "a": round(path, 1), + "band": max(2.0, path * 0.5), + "stream": "odom", + "window": [t1, t2], + "dataset": dataset, + } + ) + return rows + finally: + store.stop() diff --git a/dimos/evals/module.py b/dimos/evals/module.py new file mode 100644 index 0000000000..8849d9bc4a --- /dev/null +++ b/dimos/evals/module.py @@ -0,0 +1,64 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MCP surface for evals — lets a coding agent spin runs and grep the run dir.""" + +from __future__ import annotations + +import importlib +import pkgutil + +from dimos.agents.annotation import skill +from dimos.agents.skill_result import SkillResult +from dimos.core.module import Module + + +def list_suites() -> list[str]: + """Dotted module paths under dimos.evals.suites exporting ``SUITE``.""" + from dimos.evals import suites + + return [ + name for _, name, _ in pkgutil.iter_modules(suites.__path__, prefix=f"{suites.__name__}.") + ] + + +class EvalModule(Module): + """Expose eval runs as skills so agents can iterate: run, read the summary, + grep transcripts in the returned run_dir, edit code/prompts, run again.""" + + @skill + def run_evals(self, suite: str, tags: str = "") -> SkillResult: + """Run an eval suite by dotted module path (see list_eval_suites). + + Args: + suite: e.g. "dimos.evals.suites.go2_smoke" (must export SUITE). + tags: optional comma-separated tag filter. + """ + from dimos.evals.runner import EvalRunner, summarize + + cases = importlib.import_module(suite).SUITE + runner = EvalRunner(attach=True) + results = runner.run( + cases, tags=frozenset(t for t in tags.split(",") if t) if tags else frozenset() + ) + s = summarize(results) + return SkillResult.ok( + f"{s.n} cases: mean={s.mean_score:.2f} pass={s.pass_rate:.0%} errors={s.errors}", + run_dir=str(runner.run_dir), + ) + + @skill + def list_eval_suites(self) -> SkillResult: + """List available eval suite module paths.""" + return SkillResult.ok(", ".join(list_suites())) diff --git a/dimos/evals/runner.py b/dimos/evals/runner.py new file mode 100644 index 0000000000..9fdfec96ff --- /dev/null +++ b/dimos/evals/runner.py @@ -0,0 +1,396 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""EvalRunner — the one engine behind CLI, MCP skill, and pytest. + +Implements the :class:`~dimos.evals.types.EvalRig` protocol structurally. +Cases own their evaluation flow (``case.evaluate(rig)``); the runner owns +resources (model client, MCP adapter, sim process, live store) plus run +lifecycle: preflight, timing, error isolation, artifacts. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from dataclasses import asdict, dataclass, replace +import json +from pathlib import Path +import subprocess +import time +from typing import TYPE_CHECKING, Any + +from dimos.core.resource import CompositeResource +from dimos.evals.types import EvalCase, EvalResult, InteractiveEval, Suite +from dimos.protocol.service.spec import BaseConfig, Configurable +from dimos.utils.logging_config import setup_logger + +if TYPE_CHECKING: + from langchain_core.language_models.chat_models import BaseChatModel + + from dimos.e2e_tests.dimos_cli_call import DimosCliCall + from dimos.memory2.store.base import Store + from dimos.memory2.stream import Stream + +logger = setup_logger() + +EVAL_SYSTEM_PROMPT = ( + "You are evaluating a robot's perception and memory. Answer the question " + "using only the provided observations. Reply with the answer value only — " + "a bare number or a short phrase. No explanation, no units unless asked." +) + +BLIND_BLOCK: dict[str, str] = { + "type": "text", + "text": "[observations withheld — answer anyway]", +} + + +class EvalRunnerConfig(BaseConfig): + model: str = "gpt-5.6-luna" # mirrors McpClientConfig.model + # House convention (StoreConfig): pass an instance to inject, e.g. a fake + # chat model in tests. None -> built from `model` via init_model(). + chat_model: Any | None = None + mcp_url: str = "http://localhost:9990/mcp" + live_db: str = "recording.db" # store the Recorder writes (interactive) + blind: bool = False # ablation: context withheld (SPACE guessing check) + threshold: float = 1.0 # passed = score >= threshold + strict: bool = False # preflight failure aborts the whole run + context_budget: int = 8 # max observations encoded per context Select + attach: bool = False # True: drive an already-running dimos + launch_timeout_s: float = 1200.0 # blueprint + MCP readiness (e2e parity) + out_dir: Path = Path("~/.local/state/dimos/evals").expanduser() + + +@dataclass(frozen=True, kw_only=True) +class RunSummary: + n: int + mean_score: float + pass_rate: float + errors: int + duration_s: float + + +def summarize(results: Sequence[EvalResult]) -> RunSummary: + scored = [r for r in results if not r.error] + return RunSummary( + n=len(results), + mean_score=sum(r.score for r in scored) / len(scored) if scored else 0.0, + pass_rate=sum(r.passed for r in scored) / len(scored) if scored else 0.0, + errors=sum(1 for r in results if r.error), + duration_s=sum(r.duration_s for r in results), + ) + + +class EvalRunner(Configurable, CompositeResource): + config: EvalRunnerConfig + + def __init__(self, **kwargs: Any) -> None: + Configurable.__init__(self, **kwargs) + CompositeResource.__init__(self) + self._model: BaseChatModel | None = None + self._proc: DimosCliCall | None = None + self._run_dir: Path | None = None + + # -- run lifecycle ----------------------------------------------------------- + + def run( + self, + cases: Suite, + *, + tags: frozenset[str] = frozenset(), + limit: int = 0, + ) -> list[EvalResult]: + selected = [c for c in cases if not tags or tags & c.tags] + if limit: + selected = selected[:limit] + self._run_dir = self._new_run_dir() + + results: list[EvalResult] = [] + runnable: list[EvalCase] = [] + for case in selected: + try: + case.preflight(self) + runnable.append(case) + except Exception as e: + if self.config.strict: + raise + logger.warning("preflight failed", case=case.id, error=str(e)) + results.append(EvalResult(case_id=case.id, error=f"preflight: {e}")) + + for case in runnable: + result = self._guarded(case) + logger.info( + "eval case done", + case=case.id, + score=round(result.score, 3), + error=result.error or None, + ) + results.append(result) + + self._write_artifacts(results) + self.stop() + return results + + def _guarded(self, case: EvalCase) -> EvalResult: + t0 = time.monotonic() + try: + result = case.evaluate(self) + transcript = self.run_dir / f"{case.id}.jsonl" + return replace( + result, + duration_s=time.monotonic() - t0, + passed=result.score >= self.config.threshold and not result.error, + transcript=str(transcript) if transcript.exists() else result.transcript, + ) + except Exception as e: + return EvalResult(case_id=case.id, error=repr(e), duration_s=time.monotonic() - t0) + + @property + def run_dir(self) -> Path: + assert self._run_dir is not None, "run_dir is available only during run()" + return self._run_dir + + def _new_run_dir(self) -> Path: + run_dir = self.config.out_dir / time.strftime("run-%Y%m%d-%H%M%S") + run_dir.mkdir(parents=True, exist_ok=True) + return run_dir + + def _write_artifacts(self, results: list[EvalResult]) -> None: + lines = [json.dumps(asdict(r)) for r in results] + (self.run_dir / "results.jsonl").write_text("\n".join(lines) + "\n") + summary: dict[str, Any] = asdict(summarize(results)) + summary |= {"model": self.config.model, "blind": self.config.blind, "git": _git_sha()} + (self.run_dir / "summary.json").write_text(json.dumps(summary, indent=2)) + + def stop(self) -> None: + if self._proc is not None: + self._proc.stop() + self._proc = None + super().stop() + + # -- EvalRig: shared ------------------------------------------------------------ + + @property + def blind(self) -> bool: + return self.config.blind + + @property + def mcp_url(self) -> str: + return self.config.mcp_url + + def open_dataset(self, name: str) -> Store: + from dimos.memory2.cli.dataset import open_dataset + + return open_dataset(name) + + def live_store(self) -> Store: + from dimos.memory2.store.sqlite import SqliteStore + + return SqliteStore(path=self.config.live_db, must_exist=True) + + def encode(self, stream: Stream[Any, Any]) -> list[dict[str, Any]]: + """mem2 Stream -> model-legible content blocks (the surface under test). + + Metadata iterates lazily; blobs load only for the <= context_budget + observations actually encoded. ``agent_encode()`` is used where a type + provides it; ``str(data)`` otherwise (an encoding gap the eval will + surface, by design). + """ + observations = list(stream) + if not observations: + return [{"type": "text", "text": f"stream {stream.name!r}: no observations"}] + + budget = self.config.context_budget + if len(observations) > budget: + step = (len(observations) - 1) / (budget - 1) + observations = [observations[round(i * step)] for i in range(budget)] + + t0 = observations[0].ts + blocks: list[dict[str, Any]] = [ + { + "type": "text", + "text": f"observations from stream {stream.name!r} " + f"(t is seconds from the first shown):", + } + ] + for obs in observations: + data = obs.data + encoded = data.agent_encode() if hasattr(data, "agent_encode") else None + stamp = f"[t={obs.ts - t0:.1f}s]" + if isinstance(encoded, list): # e.g. Image -> image_url blocks + blocks.append({"type": "text", "text": stamp}) + blocks.extend(encoded) + elif encoded is not None: + blocks.append( + {"type": "text", "text": f"{stamp} {json.dumps(encoded, default=str)}"} + ) + else: + blocks.append({"type": "text", "text": f"{stamp} {data}"}) + return blocks + + def ask(self, context: Sequence[dict[str, Any]], question: str) -> str: + from langchain_core.messages import HumanMessage, SystemMessage + + blocks = list(context) if context else [BLIND_BLOCK] + message = HumanMessage(content=[*blocks, {"type": "text", "text": question}]) + response = self.model.invoke([SystemMessage(EVAL_SYSTEM_PROMPT), message]) + return str(response.text) + + @property + def model(self) -> BaseChatModel: + if self.config.chat_model is not None: + return self.config.chat_model # type: ignore[no-any-return] + if self._model is None: + from dimos.agents.model import init_model + + self._model = init_model(self.config.model) + return self._model + + def call_skill(self, name: str, args: Mapping[str, object]) -> str: + from dimos.agents.mcp.mcp_adapter import McpAdapter + + return McpAdapter(self.config.mcp_url).call_tool_text(name, dict(args)) + + def mcp_ready(self) -> bool: + from dimos.agents.mcp.mcp_adapter import McpAdapter + + return McpAdapter(self.config.mcp_url).wait_for_ready(timeout=2.0) + + def agent_loop(self, case: EvalCase) -> str: + """Fresh create_agent per case over the MCP toolset — the McpClient loop + minus its queue/thread shell. Transcript -> /.jsonl.""" + from langchain.agents import create_agent + from langchain_core.messages import HumanMessage, SystemMessage + from langchain_core.tools import StructuredTool + + from dimos.agents.mcp.mcp_adapter import McpAdapter + + adapter = McpAdapter(self.config.mcp_url) + tools = [ + StructuredTool( + name=t["name"], + description=t.get("description", ""), + args_schema=t.get("inputSchema", {}), + func=lambda _name=t["name"], **kwargs: adapter.call_tool_text(_name, kwargs), + ) + for t in adapter.list_tools() + ] + graph: Any = create_agent(self.model, tools) + messages: list[Any] = [SystemMessage(EVAL_SYSTEM_PROMPT), HumanMessage(case.inputs)] + transcript = self.run_dir / f"{case.id}.jsonl" + final_text = "" + with transcript.open("w") as fh: + for update in graph.stream({"messages": messages}, stream_mode="updates"): + for _node, payload in update.items(): + for msg in payload.get("messages", []): + fh.write( + json.dumps({"type": type(msg).__name__, "content": str(msg.content)}) + + "\n" + ) + final_text = str(msg.content) + return final_text + + # -- EvalRig: interactive ---------------------------------------------------------- + + def setup_env(self, case: InteractiveEval) -> None: + from dimos.evals.types import _no_setup + + if case.simulator and not self.config.attach: + from dimos.e2e_tests.dimos_cli_call import DimosCliCall + + proc = DimosCliCall() + proc.simulator = case.simulator + proc.global_args = ["--dimsim-scene", case.scene] + proc.demo_args = ["run", *case.blueprint.split()] + proc.start() + self._proc = proc + if not self._wait_mcp(self.config.launch_timeout_s): + raise RuntimeError(f"MCP at {self.config.mcp_url} not ready — is dimos up?") + if case.setup is not _no_setup: + from dimos.e2e_tests.dim_sim_client import DimSimClient + + sim = DimSimClient() + sim.start() + case.setup(sim) + + def check_env(self, case: InteractiveEval) -> None: + if self.config.attach or not case.simulator: + if not self.mcp_ready(): + raise RuntimeError( + f"{case.id}: attach mode needs a running dimos at {self.config.mcp_url}" + ) + return + import shutil + + if case.simulator == "dimsim" and shutil.which("deno") is None: + raise RuntimeError(f"{case.id}: dimsim requires deno on PATH") + + def _wait_mcp(self, timeout: float) -> bool: + from dimos.agents.mcp.mcp_adapter import McpAdapter + + return McpAdapter(self.config.mcp_url).wait_for_ready(timeout=timeout, interval=2.0) + + def instruct(self, text: str) -> None: + from dimos.core.transport import pLCMTransport + + transport: pLCMTransport[str] = pLCMTransport("/human_input") + transport.lcm.start() + try: + transport.publish(text) + time.sleep(0.5) # let LCM flush before teardown + finally: + transport.lcm.stop() + + def sample( + self, score: Callable[[Store], float], interval_s: float, timeout_s: float + ) -> list[tuple[float, float]]: + """Score the live Recorder store on an interval — the mem2 analogue of + lcm_spy.wait_until_odom_position, but it returns a graded series.""" + deadline = time.monotonic() + timeout_s + t0 = time.monotonic() + series: list[tuple[float, float]] = [] + store = self._wait_live_store(deadline) + try: + while time.monotonic() < deadline: + try: + value = score(store) + except LookupError: + value = None # stream not written yet — keep waiting + if value is not None: + series.append((time.monotonic() - t0, value)) + if value >= 0.999: # ponytail: early exit on success; drop if + break # aggregates ever need the full window + time.sleep(interval_s) + finally: + store.stop() + return series + + def _wait_live_store(self, deadline: float) -> Store: + path = Path(self.config.live_db) + while not path.exists() and time.monotonic() < deadline: + time.sleep(1.0) + return self.live_store() + + +def _git_sha() -> str: + try: + return subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + timeout=5, + check=False, + ).stdout.strip() + except OSError: + return "" diff --git a/dimos/evals/scorers.py b/dimos/evals/scorers.py new file mode 100644 index 0000000000..49d6c0d369 --- /dev/null +++ b/dimos/evals/scorers.py @@ -0,0 +1,106 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Scoring helpers: plain functions over typed values, graded credit in one line. + +Scores are floats in ``[0, 1]``. Msg types support arithmetic, so physical +scorers stay one-liners:: + + lambda s: ramp((GOAL - s.streams.odom.last().data.position).length(), band=0.5) + +LLM-based scoring wraps ``openevals`` — a function library (nothing to +subclass): factories return evaluators called with +``inputs/outputs/reference_outputs`` returning ``{"key", "score", "comment"}``. +""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from typing import TypeVar + +T = TypeVar("T") + + +def exact(expected: T, got: T) -> float: + return float(expected == got) + + +# -- parsers (model text -> typed answer) ----------------------------------------- + + +def first_number(text: str) -> float: + """Pull the first number out of a model reply ("about 12.5 meters" -> 12.5).""" + import re + + match = re.search(r"-?\d+(?:\.\d+)?", text) + if match is None: + raise ValueError(f"no number in reply: {text[:80]!r}") + return float(match.group()) + + +def yes_no(text: str) -> str: + """Normalize a reply to "yes"/"no".""" + t = text.strip().lower() + if t.startswith(("yes", "no")): + return "yes" if t.startswith("yes") else "no" + raise ValueError(f"not a yes/no reply: {text[:80]!r}") + + +def choice(text: str) -> str: + """Normalize a multiple-choice reply for exact comparison.""" + return text.strip().lower().rstrip(".") + + +def within(band: float) -> Callable[[float, float], float]: + """1.0 at exact, linear to 0.0 at ``band`` away.""" + return lambda expected, got: max(0.0, 1.0 - abs(got - expected) / band) + + +def ramp(distance: float, band: float) -> float: + """Distance (meters) -> [0, 1] credit inside ``band``.""" + return max(0.0, 1.0 - distance / band) + + +def judge(rubric: str, *, model: str = "openai:gpt-5.6-luna") -> Callable[[str, str], float]: + """LLM-as-judge with partial credit via openevals ``continuous=True``. + + ``rubric`` may reference ``{inputs}``, ``{outputs}``, ``{reference_outputs}``. + """ + from openevals.llm import create_llm_as_judge + + evaluator = create_llm_as_judge(prompt=rubric, model=model, continuous=True) + + def _score(expected: str, got: str) -> float: + result = evaluator(inputs="", outputs=got, reference_outputs=expected) + if isinstance(result, list): + result = result[0] + return float(result["score"]) + + return _score + + +# -- aggregates for interactive score series ------------------------------------- + + +def final(scores: Sequence[float]) -> float: + return scores[-1] + + +def floor(scores: Sequence[float]) -> float: + """Worst moment wins — "never left the zone".""" + return min(scores) + + +def mean(scores: Sequence[float]) -> float: + return sum(scores) / len(scores) diff --git a/dimos/evals/suites/__init__.py b/dimos/evals/suites/__init__.py new file mode 100644 index 0000000000..f02f56de83 --- /dev/null +++ b/dimos/evals/suites/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Eval suites. Each module exports ``SUITE: Suite``.""" diff --git a/dimos/evals/suites/dimsim_house.py b/dimos/evals/suites/dimsim_house.py new file mode 100644 index 0000000000..3e685f7c89 --- /dev/null +++ b/dimos/evals/suites/dimsim_house.py @@ -0,0 +1,101 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Interactive suite — dimsim apartment, parity with test_dimsim_spatial_memory. + +The e2e test asserts ``wait_until_odom_position(-3.567, -1.332, threshold=2)`` +after "go to the bed"; here the same success condition is a graded ramp scored +against the live mem2 store written by the ``go2-memory`` Recorder. + +Exploration (the e2e ``explore_house`` fixture) is the case's ``setup`` — the +agent needs spatial memory of the apartment before it can navigate it. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING + +from dimos.evals.scorers import final, ramp +from dimos.evals.types import InteractiveEval, Suite +from dimos.msgs.geometry_msgs.Vector3 import Vector3 + +if TYPE_CHECKING: + from dimos.e2e_tests.dim_sim_client import DimSimClient + from dimos.memory2.store.base import Store + +BED = Vector3(-3.567, -1.332, 0.0) + +_HOUSE_TOUR = [ + (3.881, 4.803), + (4.160, 1.615), + (1.596, 1.505), + (1.649, 0.137), + (-3.644, -0.064), + (-3.759, -2.661), + (-4.186, -4.830), + (-3.759, -2.661), + (-1.070, -3.285), + (-2.504, -2.452), + (-2.647, 5.243), + (-3.663, 3.591), + (-1.178, 1.974), + (-2.416, 2.629), + (-2.581, 0.164), + (1.834, 0.072), + (3.010, -3.883), + (1.756, -3.742), + (6.336, -4.077), + (8.264, -5.119), + (6.258, -0.964), + (6.453, 5.327), +] + + +def _explore_house(sim: DimSimClient) -> None: + from dimos.simulation.mujoco.direct_cmd_vel_explorer import DirectCmdVelExplorer + + explorer = DirectCmdVelExplorer() + explorer.linear_speed = 0.5 + explorer.start() + try: + explorer.follow_points(_HOUSE_TOUR) + finally: + explorer.stop() + + +def _xy_distance_to(target: Vector3) -> Callable[[Store], float]: + def distance(store: Store) -> float: + p = store.streams.odom.last().data.position + d = Vector3(p.x - target.x, p.y - target.y, 0.0).length() + return ramp(d, band=2.0) # e2e parity: threshold=2 -> full credit inside 2m + + return distance + + +go_to_bed = InteractiveEval( + id="dimsim_go_to_bed", + inputs="go to the bed", + score=_xy_distance_to(BED), + aggregate=final, + interval_s=2.0, + timeout_s=180.0, # e2e parity + blueprint="unitree-go2-agentic go2-memory", + simulator="dimsim", + scene="apartment", + setup=_explore_house, + tags=frozenset({"nav", "system"}), +) + +SUITE: Suite = [go_to_bed] diff --git a/dimos/evals/suites/examples.py b/dimos/evals/suites/examples.py new file mode 100644 index 0000000000..4dda943a9e --- /dev/null +++ b/dimos/evals/suites/examples.py @@ -0,0 +1,62 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Documentation examples — the smallest useful evals, one concept each. + +Run them:: + + dimos evals run dimos.evals.suites.examples + +Or from pytest / a notebook:: + + from dimos.evals import EvalRunner + from dimos.evals.suites.examples import SUITE + + results = EvalRunner().run(SUITE) +""" + +from __future__ import annotations + +from dimos.evals.scorers import exact, first_number, within, yes_no +from dimos.evals.types import PassiveEval, Suite + +# One lidar frame from the unitree go2 replay. The context selector returns the +# real mem2 Stream — `.limit(1)` keeps exactly the first PointCloud2. The str() +# fallback encoding exposes `num_points`, so the answer is verifiable. +single_lidar_frame = PassiveEval( + id="example_single_lidar_frame", + inputs="How many points does the shown pointcloud contain?", + expected=20834.0, + parse=first_number, + score=within(5000.0), + context=(lambda s: s.streams.lidar.limit(1),), + dataset="go2_short", + tags=frozenset({"example", "encoding", "pointcloud"}), +) + +# A range of 10 image frames from the same replay: seconds 45..61 of the +# recording, capped to 10 observations. Image.agent_encode() turns each into an +# image content block; the person at the table appears near the end. +ten_image_range = PassiveEval( + id="example_ten_image_range", + inputs="Is a person visible in any of these images?", + expected="yes", + parse=yes_no, + score=exact, + context=(lambda s: s.streams.color_image.range_time(45, 61).limit(10),), + dataset="go2_short", + tags=frozenset({"example", "encoding", "image"}), +) + +SUITE: Suite = [single_lidar_frame, ten_image_range] diff --git a/dimos/evals/suites/go2_smoke.py b/dimos/evals/suites/go2_smoke.py new file mode 100644 index 0000000000..6c1a5168b4 --- /dev/null +++ b/dimos/evals/suites/go2_smoke.py @@ -0,0 +1,81 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hand-written passive smoke suite over the go2 replays. + +Ground truth verified against the recordings (contact sheets + odom math): +go2_short is 60s — chairs room, store shelves, robot kiosk, glass booths, a +person at a table at the end; path 37.9m, displacement 1.7m. +""" + +from __future__ import annotations + +from dimos.evals.scorers import choice, exact, first_number, within, yes_no +from dimos.evals.types import PassiveEval, Suite + +SUITE: Suite = [ + PassiveEval( + id="short_person_visible", + inputs="Is a person visible in any of these images?", + expected="yes", + parse=yes_no, + score=exact, + context=(lambda s: s.streams.color_image.range_time(40, 61),), + dataset="go2_short", + tags=frozenset({"image", "presence"}), + ), + PassiveEval( + id="short_start_furniture", + inputs="At the start of these observations, which furniture is most numerous? " + "Answer with one of: chairs, sofas, beds, desks.", + expected="chairs", + parse=choice, + score=exact, + context=(lambda s: s.streams.color_image.range_time(0, 8),), + dataset="go2_short", + tags=frozenset({"image", "mcq"}), + ), + PassiveEval( + id="short_displacement", + inputs="How far in a straight line is your final position from your first " + "shown position, in meters?", + expected=1.7, + parse=first_number, + score=within(1.5), + context=(lambda s: s.streams.odom,), + dataset="go2_short", + tags=frozenset({"odom", "numeric"}), + ), + PassiveEval( + id="short_lidar_points", + inputs="How many points does the shown pointcloud contain?", + expected=20834.0, + parse=first_number, + score=within(5000.0), + context=(lambda s: s.streams.lidar.limit(1),), + dataset="go2_short", + tags=frozenset({"pointcloud", "numeric"}), + ), + PassiveEval( + id="hk_not_seen", + inputs="Which of these did you NOT see anywhere in the observations? " + "Answer with one of: a couch, store shelves, a swimming pool, an office chair.", + expected="a swimming pool", + parse=choice, + score=exact, + context=(lambda s: s.streams.color_image,), + dataset="go2_hongkong_office", + tags=frozenset({"image", "mcq"}), + ), +] diff --git a/dimos/evals/suites/go2_vqa.json b/dimos/evals/suites/go2_vqa.json new file mode 100644 index 0000000000..aa2262f25d --- /dev/null +++ b/dimos/evals/suites/go2_vqa.json @@ -0,0 +1,86 @@ +[ + { + "id": "go2_short_disp_0_60", + "q": "How far in a straight line is your final position from your position at the first shown observation, in meters?", + "a": 1.7, + "band": 1.0, + "stream": "odom", + "window": [ + 0, + 60 + ], + "dataset": "go2_short" + }, + { + "id": "go2_short_disp_0_30", + "q": "How far in a straight line is your final position from your position at the first shown observation, in meters?", + "a": 11.4, + "band": 4.543036677935427, + "stream": "odom", + "window": [ + 0, + 30 + ], + "dataset": "go2_short" + }, + { + "id": "go2_short_disp_30_60", + "q": "How far in a straight line is your final position from your position at the first shown observation, in meters?", + "a": 11.8, + "band": 4.738980277235515, + "stream": "odom", + "window": [ + 30, + 60 + ], + "dataset": "go2_short" + }, + { + "id": "go2_short_path_0_60", + "q": "Roughly how many meters did you travel in total over these observations (path length, not displacement)?", + "a": 37.9, + "band": 18.963056711366217, + "stream": "odom", + "window": [ + 0, + 60 + ], + "dataset": "go2_short" + }, + { + "id": "go2_hongkong_office_disp_0_558", + "q": "How far in a straight line is your final position from your position at the first shown observation, in meters?", + "a": 9.6, + "band": 3.8486174236058126, + "stream": "odom", + "window": [ + 0, + 558 + ], + "dataset": "go2_hongkong_office" + }, + { + "id": "go2_hongkong_office_disp_100_300", + "q": "How far in a straight line is your final position from your position at the first shown observation, in meters?", + "a": 27.5, + "band": 11.016777380908813, + "stream": "odom", + "window": [ + 100, + 300 + ], + "dataset": "go2_hongkong_office" + }, + { + "id": "go2_hongkong_office_path_0_558", + "q": "Roughly how many meters did you travel in total over these observations (path length, not displacement)?", + "a": 192.5, + "band": 96.26788393075581, + "stream": "odom", + "window": [ + 0, + 558 + ], + "dataset": "go2_hongkong_office" + } +] diff --git a/dimos/evals/suites/go2_vqa.py b/dimos/evals/suites/go2_vqa.py new file mode 100644 index 0000000000..8377434e78 --- /dev/null +++ b/dimos/evals/suites/go2_vqa.py @@ -0,0 +1,73 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generated VQA suite over the go2 replays. + +Rows (``go2_vqa.json``) are pure data emitted by :mod:`dimos.evals.generate` — +ground truth computed analytically from odom, quizzing the encoded odom +summary. Typing and scoring live here; the JSON stays behavior-free. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from dimos.evals.scorers import exact, first_number, within, yes_no +from dimos.evals.types import PassiveEval, Suite + +_ROWS = json.loads((Path(__file__).parent / "go2_vqa.json").read_text()) + +_generated: list[PassiveEval[float]] = [ + PassiveEval( + id=str(row["id"]), + inputs=str(row["q"]), + expected=float(row["a"]), # type: ignore[arg-type] + parse=first_number, + score=within(float(row["band"])), # type: ignore[arg-type] + context=( + lambda s, name=str(row["stream"]), w=tuple(row["window"]): # type: ignore[misc] + s.streams[name].range_time(*w), + ), + dataset=str(row["dataset"]), + tags=frozenset({"generated", "odom", "numeric"}), + ) + for row in _ROWS +] + +# Hand-labeled presence questions, verified against the recording imagery. +_hand: list[PassiveEval[str]] = [ + PassiveEval( + id="hk_couch_seen", + inputs="Did you see a couch or sofa at any point?", + expected="yes", + parse=yes_no, + score=exact, + context=(lambda s: s.streams.color_image.range_time(150, 250),), + dataset="go2_hongkong_office", + tags=frozenset({"image", "presence"}), + ), + PassiveEval( + id="hk_plants_seen", + inputs="Did you see any potted plants?", + expected="yes", + parse=yes_no, + score=exact, + context=(lambda s: s.streams.color_image.range_time(0, 60),), + dataset="go2_hongkong_office", + tags=frozenset({"image", "presence"}), + ), +] + +SUITE: Suite = [*_generated, *_hand] diff --git a/dimos/evals/test_evals.py b/dimos/evals/test_evals.py new file mode 100644 index 0000000000..3dd97c8289 --- /dev/null +++ b/dimos/evals/test_evals.py @@ -0,0 +1,312 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline unit tests: scorers, case dispatch, preflight, runner artifacts. + +No network, no robot, no LLM — the chat model is a fake and the rig in case +tests is a plain object satisfying the EvalRig protocol structurally. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +import json +from pathlib import Path +from typing import Any + +import pytest + +from dimos.evals.scorers import ( + choice, + exact, + final, + first_number, + floor, + mean, + ramp, + within, + yes_no, +) +from dimos.evals.types import EvalCase, InteractiveEval, PassiveEval +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Vector3 import make_vector3 + + +def _pose(x: float, y: float) -> PoseStamped: + return PoseStamped( + position=make_vector3(x, y, 0.0), + orientation=Quaternion(0.0, 0.0, 0.0, 1.0), + frame_id="world", + ) + + +@pytest.fixture +def dataset(tmp_path: Path) -> str: + """A tiny on-disk mem2 dataset: 5 odom poses walking 4m in +x over 4s.""" + from dimos.memory2.store.sqlite import SqliteStore + + path = tmp_path / "tiny.db" + try: + store = SqliteStore(path=str(path)) + except Exception as e: # pragma: no cover — sqlite-vec unavailable platforms + pytest.skip(f"SqliteStore unavailable: {e}") + stream = store.stream("odom", PoseStamped) + for i in range(5): + stream.append(_pose(float(i), 0.0), ts=1000.0 + i) + store.stop() + return str(path) + + +class FakeRig: + """Structural EvalRig for case-level tests.""" + + blind = False + mcp_url = "http://localhost:9990/mcp" + + def __init__(self, answer: str = "", series: list[tuple[float, float]] | None = None): + self.answer = answer + self.series = series or [] + self.calls: list[str] = [] + + def open_dataset(self, name: str) -> Any: + from dimos.memory2.cli.dataset import open_dataset + + return open_dataset(name) + + def live_store(self) -> Any: + raise NotImplementedError + + def encode(self, stream: Any) -> list[dict[str, Any]]: + return [{"type": "text", "text": f"{len(list(stream))} observations"}] + + def ask(self, context: Sequence[dict[str, Any]], question: str) -> str: + self.calls.append("ask") + return self.answer + + def call_skill(self, name: str, args: Mapping[str, object]) -> str: + self.calls.append(f"skill:{name}") + return self.answer + + def agent_loop(self, case: EvalCase) -> str: + self.calls.append("agent_loop") + return self.answer + + def mcp_ready(self) -> bool: + return False + + def setup_env(self, case: InteractiveEval) -> None: + self.calls.append("setup_env") + + def check_env(self, case: InteractiveEval) -> None: + pass + + def instruct(self, text: str) -> None: + self.calls.append(f"instruct:{text}") + + def sample( + self, score: Callable[[Any], float], interval_s: float, timeout_s: float + ) -> list[tuple[float, float]]: + return self.series + + +# -- scorers ------------------------------------------------------------------------ + + +def test_scorer_math() -> None: + assert exact(6, 6) == 1.0 + assert exact("yes", "no") == 0.0 + assert within(2.0)(10.0, 10.0) == 1.0 + assert within(2.0)(10.0, 11.0) == 0.5 + assert within(2.0)(10.0, 13.0) == 0.0 + assert ramp(0.0, band=2.0) == 1.0 + assert ramp(1.0, band=2.0) == 0.5 + assert ramp(5.0, band=2.0) == 0.0 + assert final([0.1, 0.9]) == 0.9 + assert floor([0.4, 0.2, 0.8]) == 0.2 + assert mean([0.0, 1.0]) == 0.5 + + +def test_parsers() -> None: + assert first_number("about 12.5 meters") == 12.5 + assert first_number("-3") == -3.0 + with pytest.raises(ValueError): + first_number("none") + assert yes_no("Yes, there is.") == "yes" + assert yes_no("no") == "no" + with pytest.raises(ValueError): + yes_no("maybe") + assert choice(" Chairs. ") == "chairs" + + +# -- case dispatch --------------------------------------------------------------------- + + +def test_passive_model_path(dataset: str) -> None: + case = PassiveEval( + id="disp", + inputs="how far?", + expected=4.0, + parse=first_number, + score=within(1.0), + context=(lambda s: s.streams.odom,), + dataset=dataset, + ) + rig = FakeRig(answer="about 4 meters") + result = case.evaluate(rig) + assert result.score == 1.0 + assert rig.calls == ["ask"] + + +def test_passive_skill_path(dataset: str) -> None: + case = PassiveEval( + id="sk", + inputs="", + skill="detect", + skill_args={"query": "person"}, + expected="yes", + parse=yes_no, + dataset=dataset, + ) + rig = FakeRig(answer="yes") + assert case.evaluate(rig).score == 1.0 + assert rig.calls == ["skill:detect"] + + +def test_interactive_dispatch() -> None: + case = InteractiveEval( + id="nav", + inputs="go to the bed", + score=lambda store: 1.0, + aggregate=floor, + simulator="", + ) + rig = FakeRig(series=[(0.0, 0.2), (1.0, 0.6), (2.0, 0.9)]) + result = case.evaluate(rig) + assert result.score == 0.2 # floor aggregate + assert result.series == ((0.0, 0.2), (1.0, 0.6), (2.0, 0.9)) + assert rig.calls == ["setup_env", "instruct:go to the bed"] + + +def test_interactive_no_samples_is_error() -> None: + case = InteractiveEval(id="n", inputs="x", score=lambda s: 1.0, simulator="") + assert "no samples" in case.evaluate(FakeRig()).error + + +# -- preflight ---------------------------------------------------------------------- + + +def test_preflight_missing_stream(dataset: str) -> None: + case = PassiveEval( + id="bad", + inputs="?", + expected=1.0, + parse=first_number, + context=(lambda s: s.streams.lidar.limit(1),), + dataset=dataset, + ) + with pytest.raises(AttributeError, match="No stream 'lidar'"): + case.preflight(FakeRig()) + + +def test_preflight_needs_mcp(dataset: str) -> None: + case = PassiveEval( + id="needs_mcp", + inputs="?", + expected="yes", + parse=yes_no, + tools=True, + dataset=dataset, + ) + with pytest.raises(RuntimeError, match="needs MCP"): + case.preflight(FakeRig()) + + +# -- runner ------------------------------------------------------------------------ + + +def test_runner_end_to_end_offline(dataset: str, tmp_path: Path) -> None: + from langchain_core.language_models.fake_chat_models import FakeListChatModel + + from dimos.evals.runner import EvalRunner, summarize + + cases = [ + PassiveEval( + id="disp", + inputs="straight-line distance in meters?", + expected=4.0, + parse=first_number, + score=within(1.0), + context=(lambda s: s.streams.odom,), + dataset=dataset, + ), + PassiveEval( # parse failure -> error result, run survives + id="unparseable", + inputs="?", + expected=1.0, + parse=first_number, + context=(lambda s: s.streams.odom.limit(1),), + dataset=dataset, + ), + PassiveEval( # preflight failure -> error result, run survives + id="missing_stream", + inputs="?", + expected=1.0, + parse=first_number, + context=(lambda s: s.streams.lidar,), + dataset=dataset, + ), + ] + runner = EvalRunner( + chat_model=FakeListChatModel(responses=["4.0", "no numbers here"]), + out_dir=tmp_path / "evals", + ) + results = runner.run(cases) + + by_id = {r.case_id: r for r in results} + assert by_id["disp"].passed and by_id["disp"].score == 1.0 + assert "ValueError" in by_id["unparseable"].error + assert by_id["missing_stream"].error.startswith("preflight:") + + s = summarize(results) + assert s.n == 3 and s.errors == 2 + + run_dir = runner.run_dir + lines = (run_dir / "results.jsonl").read_text().strip().splitlines() + assert len(lines) == 3 + summary = json.loads((run_dir / "summary.json").read_text()) + assert summary["n"] == 3 and "model" in summary + + +def test_runner_encode_budget(dataset: str, tmp_path: Path) -> None: + from dimos.evals.runner import EvalRunner + + runner = EvalRunner(context_budget=3, out_dir=tmp_path) + store = runner.open_dataset(dataset) + try: + blocks = runner.encode(store.streams.odom) + finally: + store.stop() + # 1 header + 3 sampled observations, all text (PoseStamped str fallback) + assert len(blocks) == 4 + assert all(b["type"] == "text" for b in blocks) + assert "pos=" in blocks[1]["text"] + + +def test_suites_importable() -> None: + """Suite modules construct without data or network (lambdas stay lazy).""" + from dimos.evals.suites import dimsim_house, examples, go2_smoke, go2_vqa + + for module in (examples, go2_smoke, go2_vqa, dimsim_house): + assert module.SUITE, module.__name__ diff --git a/dimos/evals/test_smoke.py b/dimos/evals/test_smoke.py new file mode 100644 index 0000000000..891d3db11d --- /dev/null +++ b/dimos/evals/test_smoke.py @@ -0,0 +1,38 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Live passive smoke: real model, real LFS recordings. Self-hosted + API key.""" + +from __future__ import annotations + +import pytest + +pytestmark = [pytest.mark.self_hosted, pytest.mark.skipif_no_openai] + + +def test_passive_smoke(tmp_path) -> None: # type: ignore[no-untyped-def] + from dimos.evals.runner import EvalRunner, summarize + from dimos.evals.suites.examples import SUITE + from dimos.utils.data import get_data + + get_data("go2_short.db") + + runner = EvalRunner(model="gpt-4o-mini", out_dir=tmp_path / "evals") + results = runner.run(SUITE) + + assert not any(r.error for r in results), [r.error for r in results] + s = summarize(results) + # The lidar-points case reads a number embedded in the str() encoding and + # the image case is unambiguous — a competent VLM should clear both. + assert s.mean_score >= 0.5 diff --git a/dimos/evals/types.py b/dimos/evals/types.py new file mode 100644 index 0000000000..be54495621 --- /dev/null +++ b/dimos/evals/types.py @@ -0,0 +1,198 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Eval case primitives. + +Two taxonomies, orthogonal: + +- **Passive** evals: world state is immutable (a frozen frame or replay, any + time window). The model's output never feeds back into its input. Cheap, + deterministic, repeatable. +- **Interactive** evals: actions feed back into observations; state is + mutable. Needs sim or a real robot, scored by sampling the live memory2 + store the robot's Recorder writes. + +Suites are Python modules exporting ``SUITE: Suite`` (behavior is typed code; +JSON holds only data rows). memory2 is the source of truth for all input and +perception: context selectors return real :class:`~dimos.memory2.stream.Stream` +objects and scoring reads :class:`~dimos.memory2.store.base.Store`. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Generic, Protocol, TypeVar + +from dimos.evals.scorers import exact, final + +if TYPE_CHECKING: + from dimos.e2e_tests.dim_sim_client import DimSimClient + from dimos.memory2.store.base import Store + from dimos.memory2.stream import Stream + +T = TypeVar("T") + +Select = Callable[["Store"], "Stream[Any, Any]"] +"""Context selector — hands the model real mem2 streams, whole or windowed:: + + lambda s: s.streams.lidar.limit(1) + lambda s: s.streams.odom.range_time(0, 600) +""" + + +@dataclass(frozen=True, kw_only=True) +class EvalResult: + case_id: str + outputs: str = "" + score: float = 0.0 + passed: bool = False + duration_s: float = 0.0 + error: str = "" + series: tuple[tuple[float, float], ...] = () # (t, score) — interactive only + transcript: str = "" # path within the run dir, when an agent loop ran + + +class EvalRig(Protocol): + """What a case may ask of the runner. :class:`EvalRunner` implements this + structurally — no import cycle, mypy-checked at call sites, and a fake rig + in tests is any object with these methods.""" + + @property + def blind(self) -> bool: ... + @property + def mcp_url(self) -> str: ... + + def open_dataset(self, name: str) -> Store: ... + def live_store(self) -> Store: ... + def encode(self, stream: Stream[Any, Any]) -> list[dict[str, Any]]: ... + def ask(self, context: Sequence[dict[str, Any]], question: str) -> str: ... + def call_skill(self, name: str, args: Mapping[str, object]) -> str: ... + def agent_loop(self, case: EvalCase) -> str: ... + def mcp_ready(self) -> bool: ... + def setup_env(self, case: InteractiveEval) -> None: ... + def check_env(self, case: InteractiveEval) -> None: ... + def instruct(self, text: str) -> None: ... + def sample( + self, score: Callable[[Store], float], interval_s: float, timeout_s: float + ) -> list[tuple[float, float]]: ... + + +@dataclass(frozen=True, kw_only=True) +class EvalCase(ABC): + """Common surface the runner, report, and filters operate on. + + ``skill`` set -> score one tool call (no agent loop): ``detect()`` on a + replay, ``grasp()`` in sim. ``skill`` empty -> subclass decides. + """ + + id: str + inputs: str + skill: str = "" + skill_args: Mapping[str, object] = field(default_factory=dict) + tags: frozenset[str] = frozenset() + timeout_s: float = 60.0 + + @abstractmethod + def evaluate(self, rig: EvalRig) -> EvalResult: + """Produce this case's result using the rig's resources.""" + + def preflight(self, rig: EvalRig) -> None: + """Raise with a precise message if this case cannot run on this rig. + + Cheap: resolves resources, reads no data, starts no processes. + """ + if self.skill and not rig.mcp_ready(): + raise RuntimeError(f"{self.id}: needs MCP at {rig.mcp_url}, nothing listening") + + +@dataclass(frozen=True, kw_only=True) +class PassiveEval(EvalCase, Generic[T]): + """World state immutable; ``T`` ties ``expected``/``parse``/``score`` + together so mypy checks the triple agrees per case.""" + + expected: T + parse: Callable[[str], T] + score: Callable[[T, T], float] = exact + context: tuple[Select, ...] = () + dataset: str = "go2_short" + tools: bool = False # True: full agent loop over the frozen store + + def evaluate(self, rig: EvalRig) -> EvalResult: + store = rig.open_dataset(self.dataset) + try: + if self.skill: + outputs = rig.call_skill(self.skill, self.skill_args) + elif self.tools: + outputs = rig.agent_loop(self) + else: + blocks = ( + [] if rig.blind else [b for sel in self.context for b in rig.encode(sel(store))] + ) + outputs = rig.ask(blocks, self.inputs) + finally: + store.stop() + got = self.parse(outputs) + return EvalResult(case_id=self.id, outputs=outputs, score=self.score(self.expected, got)) + + def preflight(self, rig: EvalRig) -> None: + store = rig.open_dataset(self.dataset) # raises: dataset unresolvable + try: + for sel in self.context: + sel(store) # raises: "No stream 'x'. Available: [...]" — no data read + finally: + store.stop() + if (self.skill or self.tools) and not rig.mcp_ready(): + raise RuntimeError(f"{self.id}: needs MCP at {rig.mcp_url}, nothing listening") + + +def _no_setup(sim: DimSimClient) -> None: + return None + + +@dataclass(frozen=True, kw_only=True) +class InteractiveEval(EvalCase): + """Actions feed back into observations. The case names its environment so + the eval is reproducible; the runner only decides attach-vs-launch.""" + + score: Callable[[Store], float] # sampled every interval_s against live mem2 + aggregate: Callable[[Sequence[float]], float] = final + interval_s: float = 1.0 + timeout_s: float = 300.0 + blueprint: str = "unitree-go2-agentic" + simulator: str = "dimsim" # "" = attach to a running dimos / real robot + scene: str = "apartment" # --dimsim-scene name (ScenePackage name later) + setup: Callable[[DimSimClient], None] = _no_setup + + def evaluate(self, rig: EvalRig) -> EvalResult: + rig.setup_env(self) + if self.skill: + rig.call_skill(self.skill, self.skill_args) + else: + rig.instruct(self.inputs) + series = rig.sample(self.score, self.interval_s, self.timeout_s) + if not series: + return EvalResult(case_id=self.id, error=f"{self.id}: no samples collected") + return EvalResult( + case_id=self.id, + score=self.aggregate([s for _, s in series]), + series=tuple(series), + ) + + def preflight(self, rig: EvalRig) -> None: + rig.check_env(self) + + +Suite = Sequence[EvalCase] diff --git a/pyproject.toml b/pyproject.toml index 1fc0193bab..a2bc303efd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -226,6 +226,7 @@ agents = [ "langchain-huggingface>=1,<2", "langchain-ollama>=1,<2", "ollama>=0.6.0", + "openevals>=0.1", # eval scorers (LLM-as-judge etc.) — dimos/evals # Audio "openai", diff --git a/uv.lock b/uv.lock index e2313b7058..0eac32ef6b 100644 --- a/uv.lock +++ b/uv.lock @@ -1640,6 +1640,7 @@ agents = [ { name = "langchain-openai" }, { name = "ollama" }, { name = "openai" }, + { name = "openevals" }, { name = "sounddevice" }, ] all = [ @@ -1680,6 +1681,7 @@ all = [ { name = "onnxruntime-gpu", marker = "platform_machine == 'x86_64'" }, { name = "open-clip-torch" }, { name = "openai" }, + { name = "openevals" }, { name = "pillow" }, { name = "piper-sdk" }, { name = "playground" }, @@ -1735,6 +1737,7 @@ base = [ { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, + { name = "openevals" }, { name = "pillow" }, { name = "rerun-sdk" }, { name = "sounddevice" }, @@ -1845,6 +1848,7 @@ unitree = [ { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, + { name = "openevals" }, { name = "pillow" }, { name = "rerun-sdk" }, { name = "sounddevice" }, @@ -1879,6 +1883,7 @@ unitree-dds = [ { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, + { name = "openevals" }, { name = "pillow" }, { name = "rerun-sdk" }, { name = "sounddevice" }, @@ -2145,6 +2150,7 @@ requires-dist = [ { name = "open3d-unofficial-arm", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'", specifier = ">=0.19.0.post9" }, { name = "openai", marker = "extra == 'agents'" }, { name = "opencv-contrib-python", specifier = ">=4.8,<5" }, + { name = "openevals", marker = "extra == 'agents'", specifier = ">=0.1" }, { name = "packaging", specifier = ">=24.0" }, { name = "pandas", marker = "extra == 'learning'" }, { name = "pillow", marker = "extra == 'perception'" }, @@ -5908,6 +5914,21 @@ dependencies = [ { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')" }, ] +[[package]] +name = "openevals" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain" }, + { name = "langchain-openai" }, + { name = "langsmith" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/b1/028a05846136805b29b7a3afb58a940c2d213fa2c3d0a7d7003c7fbaa115/openevals-0.2.0.tar.gz", hash = "sha256:7e95fa64625be53eaa8c657d7f69b842a52bda10bdf3bb91781c7d09a385b069", size = 140711, upload-time = "2026-04-07T19:45:22.749Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/8b/00f402b7f3475e235c339a9bc82d2eaf46cc08b53ecd85d4a117850170d3/openevals-0.2.0-py3-none-any.whl", hash = "sha256:2bce5964be9d162e3d38c2dfd026739156e1ac521536ade6b8e2f0a89b632f2c", size = 106958, upload-time = "2026-04-07T19:45:21.575Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.42.1" From 4ea607ab548096c7c29f327ecdb092387a83bf3b Mon Sep 17 00:00:00 2001 From: stash Date: Sat, 8 Aug 2026 19:56:41 -0700 Subject: [PATCH 02/12] feat(evals): keyless local eval model (moondream2 chat adapter) --- dimos/evals/local.py | 105 +++++++++++++++++++++++++++++++++ dimos/evals/suites/examples.py | 9 ++- 2 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 dimos/evals/local.py diff --git a/dimos/evals/local.py b/dimos/evals/local.py new file mode 100644 index 0000000000..2281b11ae1 --- /dev/null +++ b/dimos/evals/local.py @@ -0,0 +1,105 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Keyless local eval model: MoondreamVlModel behind the chat-model interface. + +Lets ``EvalRunner(chat_model=MoondreamChat())`` run passive evals with zero API +keys on any GPU box. Moondream is single-image, so multi-image contexts are +tiled into one contact sheet; text blocks concatenate into the question. +""" + +from __future__ import annotations + +import base64 +from functools import cached_property +from typing import Any + +from langchain_core.callbacks import CallbackManagerForLLMRun +from langchain_core.language_models.chat_models import BaseChatModel +from langchain_core.messages import AIMessage, BaseMessage +from langchain_core.outputs import ChatGeneration, ChatResult +import numpy as np + +from dimos.msgs.sensor_msgs.Image import Image + + +class MoondreamChat(BaseChatModel): + """Chat-model adapter over the local moondream2 VLM (dimos MoondreamVlModel).""" + + tile_columns: int = 3 + + @property + def _llm_type(self) -> str: + return "moondream-local" + + @cached_property + def _vl(self) -> Any: + from dimos.models.vl.moondream import MoondreamVlModel + + model = MoondreamVlModel() + model.start() + return model + + def _generate( + self, + messages: list[BaseMessage], + stop: list[str] | None = None, + run_manager: CallbackManagerForLLMRun | None = None, + **kwargs: Any, + ) -> ChatResult: + texts: list[str] = [] + frames: list[np.ndarray[Any, Any]] = [] + for message in messages: + content = message.content + if isinstance(content, str): + texts.append(content) + continue + for block in content: + if not isinstance(block, dict): + texts.append(str(block)) + elif block.get("type") == "text": + texts.append(str(block["text"])) + elif block.get("type") == "image_url": + frames.append(_decode_data_uri(str(block["image_url"]["url"]))) + + image = Image.from_numpy(_tile(frames) if frames else _BLANK) + answer = self._vl.query(image, "\n".join(texts)) + return ChatResult(generations=[ChatGeneration(message=AIMessage(content=str(answer)))]) + + +_BLANK = np.full((64, 64, 3), 128, dtype=np.uint8) + + +def _decode_data_uri(uri: str) -> np.ndarray[Any, Any]: + import cv2 + + payload = uri.split(",", 1)[1] + buffer = np.frombuffer(base64.b64decode(payload), dtype=np.uint8) + frame: np.ndarray[Any, Any] = cv2.imdecode(buffer, cv2.IMREAD_COLOR) + return cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + + +def _tile(frames: list[np.ndarray[Any, Any]], columns: int = 3) -> np.ndarray[Any, Any]: + """Grid-tile frames into one contact sheet (moondream is single-image).""" + if len(frames) == 1: + return frames[0] + height = min(f.shape[0] for f in frames) + width = min(f.shape[1] for f in frames) + import cv2 + + resized = [cv2.resize(f, (width, height)) for f in frames] + rows = [np.hstack(resized[i : i + columns]) for i in range(0, len(resized), columns)] + max_w = max(r.shape[1] for r in rows) + rows = [np.pad(r, ((0, 0), (0, max_w - r.shape[1]), (0, 0))) for r in rows] + return np.vstack(rows) diff --git a/dimos/evals/suites/examples.py b/dimos/evals/suites/examples.py index 4dda943a9e..c4f5f700d3 100644 --- a/dimos/evals/suites/examples.py +++ b/dimos/evals/suites/examples.py @@ -45,16 +45,19 @@ tags=frozenset({"example", "encoding", "pointcloud"}), ) -# A range of 10 image frames from the same replay: seconds 45..61 of the +# A range of 10 image frames from the same replay: seconds 58..61 of the # recording, capped to 10 observations. Image.agent_encode() turns each into an -# image content block; the person at the table appears near the end. +# image content block; a person sits at a table in this stretch. +# Note: `.limit(n)` keeps the *first* n observations of the window — for a +# spread across a long window, give the runner the whole range and let its +# context budget downsample evenly instead. ten_image_range = PassiveEval( id="example_ten_image_range", inputs="Is a person visible in any of these images?", expected="yes", parse=yes_no, score=exact, - context=(lambda s: s.streams.color_image.range_time(45, 61).limit(10),), + context=(lambda s: s.streams.color_image.range_time(58, 61).limit(10),), dataset="go2_short", tags=frozenset({"example", "encoding", "image"}), ) From f99a4931d4d540da2c2bdd29f9d3ddcf655e2c15 Mon Sep 17 00:00:00 2001 From: stash Date: Sat, 8 Aug 2026 20:57:56 -0700 Subject: [PATCH 03/12] test(evals): mem2 wiring integration tests (passive prompt path + live-store sampling) --- dimos/evals/local.py | 4 +- dimos/evals/test_mem2_wiring.py | 208 ++++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 dimos/evals/test_mem2_wiring.py diff --git a/dimos/evals/local.py b/dimos/evals/local.py index 2281b11ae1..d0537db1f6 100644 --- a/dimos/evals/local.py +++ b/dimos/evals/local.py @@ -86,7 +86,9 @@ def _decode_data_uri(uri: str) -> np.ndarray[Any, Any]: payload = uri.split(",", 1)[1] buffer = np.frombuffer(base64.b64decode(payload), dtype=np.uint8) - frame: np.ndarray[Any, Any] = cv2.imdecode(buffer, cv2.IMREAD_COLOR) + frame = cv2.imdecode(buffer, cv2.IMREAD_COLOR) + if frame is None: + raise ValueError("undecodable image data URI") return cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) diff --git a/dimos/evals/test_mem2_wiring.py b/dimos/evals/test_mem2_wiring.py new file mode 100644 index 0000000000..47b9d7b91e --- /dev/null +++ b/dimos/evals/test_mem2_wiring.py @@ -0,0 +1,208 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Integration tests for the memory2 <-> EvalCase connection. + +Passive: a case's context Selects pull real Streams from a recording, the +runner encodes them, and the *actual observation data* (image blocks, pose +text) reaches the model prompt. + +Interactive: a case's score callable reads the *live* store while a writer is +appending — the mem2 analogue of a robot's Recorder running mid-task. +""" + +from __future__ import annotations + +from pathlib import Path +import threading +import time +from typing import Any + +from langchain_core.callbacks import CallbackManagerForLLMRun +from langchain_core.language_models.chat_models import BaseChatModel +from langchain_core.messages import AIMessage, BaseMessage +from langchain_core.outputs import ChatGeneration, ChatResult +import numpy as np +import pytest + +from dimos.evals.runner import EvalRunner +from dimos.evals.scorers import final, first_number, ramp, within +from dimos.evals.types import InteractiveEval, PassiveEval +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Vector3 import make_vector3 +from dimos.msgs.sensor_msgs.Image import Image + + +def _pose(x: float, y: float) -> PoseStamped: + return PoseStamped( + position=make_vector3(x, y, 0.0), + orientation=Quaternion(0.0, 0.0, 0.0, 1.0), + frame_id="world", + ) + + +def _open_store(path: Path) -> Any: + from dimos.memory2.store.sqlite import SqliteStore + + try: + return SqliteStore(path=str(path)) + except Exception as e: # pragma: no cover — sqlite-vec unavailable platforms + pytest.skip(f"SqliteStore unavailable: {e}") + + +class SpyChat(BaseChatModel): + """Captures the exact messages the runner sends; replies with a constant.""" + + reply: str = "42" + seen: list[list[BaseMessage]] = [] + + @property + def _llm_type(self) -> str: + return "spy" + + def _generate( + self, + messages: list[BaseMessage], + stop: list[str] | None = None, + run_manager: CallbackManagerForLLMRun | None = None, + **kwargs: Any, + ) -> ChatResult: + self.seen.append(messages) + return ChatResult(generations=[ChatGeneration(message=AIMessage(content=self.reply))]) + + +# -- passive: recording -> Select -> encode -> prompt -------------------------------- + + +def test_passive_streams_reach_the_prompt(tmp_path: Path) -> None: + store = _open_store(tmp_path / "rec.db") + odom = store.stream("odom", PoseStamped) + for i in range(20): + odom.append(_pose(float(i), 2.5), ts=1000.0 + i) + frame = np.full((16, 16, 3), 200, dtype=np.uint8) + images = store.stream("color_image", Image) + for i in range(3): + images.append(Image.from_numpy(frame, frame_id="cam", ts=1000.0 + i), ts=1000.0 + i) + store.stop() + + case = PassiveEval( + id="wiring", + inputs="how far along x did you travel?", + expected=19.0, + parse=first_number, + score=within(1.0), + context=( + lambda s: s.streams.odom.range_time(0, 100), + lambda s: s.streams.color_image.limit(2), + ), + dataset=str(tmp_path / "rec.db"), + ) + + spy = SpyChat(reply="19") + spy.seen.clear() + runner = EvalRunner(chat_model=spy, out_dir=tmp_path / "evals") + results = runner.run([case]) + + assert results[0].passed, results[0] + blocks = [b for m in spy.seen[0] for b in (m.content if isinstance(m.content, list) else [])] + image_blocks = [b for b in blocks if b.get("type") == "image_url"] + text = " ".join(b["text"] for b in blocks if b.get("type") == "text") + # the actual observation data crossed from mem2 into the prompt: + assert len(image_blocks) == 2, "both selected image observations should be encoded" + assert image_blocks[0]["image_url"]["url"].startswith("data:image/jpeg;base64,") + assert "pos=[0.000, 2.500" in text.replace(" ", " ") or "0.000" in text + assert "19.000" in text or "19.0" in text, "last odom pose must reach the prompt" + assert case.inputs in text + + +def test_passive_context_budget_downsamples_not_truncates(tmp_path: Path) -> None: + store = _open_store(tmp_path / "rec.db") + odom = store.stream("odom", PoseStamped) + for i in range(100): + odom.append(_pose(float(i), 0.0), ts=1000.0 + i) + store.stop() + + runner = EvalRunner(context_budget=5, out_dir=tmp_path / "evals") + reopened = runner.open_dataset(str(tmp_path / "rec.db")) + try: + blocks = runner.encode(reopened.streams.odom) + finally: + reopened.stop() + texts = [b["text"] for b in blocks[1:]] # skip header + assert len(texts) == 5 + assert "0.000" in texts[0] and "99.000" in texts[-1], "spread must cover the whole window" + + +# -- interactive: live store -> score sampling ---------------------------------------- + + +def test_interactive_scores_live_store_while_writing(tmp_path: Path) -> None: + """A writer thread plays the Recorder role: the case's score callable must + see fresh observations appear in the live store as they are appended.""" + db = tmp_path / "live.db" + store = _open_store(db) + odom = store.stream("odom", PoseStamped) + odom.append(_pose(5.0, 0.0), ts=time.time()) # robot starts 5m from goal + + stop = threading.Event() + + def writer() -> None: + for i in range(1, 26): + if stop.is_set(): + return + odom.append(_pose(max(0.0, 5.0 - i * 0.2), 0.0), ts=time.time()) + time.sleep(0.05) + + thread = threading.Thread(target=writer) + + class NoEnvRunner(EvalRunner): + """Rig with the sim/MCP environment stubbed out — mem2 path stays real.""" + + def check_env(self, case: InteractiveEval) -> None: + pass + + def setup_env(self, case: InteractiveEval) -> None: + thread.start() + + def instruct(self, text: str) -> None: + pass + + case = InteractiveEval( + id="live_wiring", + inputs="go to the goal", + score=lambda s: ramp(abs(s.streams.odom.last().data.position.x), band=2.0), + aggregate=final, + interval_s=0.1, + timeout_s=10.0, + simulator="", + ) + + runner = NoEnvRunner(live_db=str(db), out_dir=tmp_path / "evals") + try: + results = runner.run([case]) + finally: + stop.set() + if thread.ident is not None: + thread.join(timeout=5.0) + store.stop() + + r = results[0] + assert not r.error, r.error + assert len(r.series) >= 3, "sampler must observe multiple live states" + scores = [s for _, s in r.series] + assert scores[0] < 0.9, "first sample sees the robot far from the goal" + assert scores[-1] >= 0.99, "last sample sees the robot arrive (live data flowed)" + assert r.score >= 0.99 # aggregate=final + assert scores == sorted(scores), "monotonic approach must be visible in the series" From 09d0f5db801577b67f759554200f8abfc15ed562 Mon Sep 17 00:00:00 2001 From: stash Date: Sat, 8 Aug 2026 21:15:56 -0700 Subject: [PATCH 04/12] fix(evals): wait for odom before dimsim house exploration --- dimos/evals/suites/dimsim_house.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/dimos/evals/suites/dimsim_house.py b/dimos/evals/suites/dimsim_house.py index 3e685f7c89..975264a564 100644 --- a/dimos/evals/suites/dimsim_house.py +++ b/dimos/evals/suites/dimsim_house.py @@ -64,8 +64,22 @@ def _explore_house(sim: DimSimClient) -> None: + import time + + from dimos.core.transport import LCMTransport + from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped from dimos.simulation.mujoco.direct_cmd_vel_explorer import DirectCmdVelExplorer + # dimsim spawns the robot well after MCP is up — wait for odom before driving + seen: list[PoseStamped] = [] + probe: LCMTransport[PoseStamped] = LCMTransport("/odom", PoseStamped) + probe.subscribe(lambda msg, *args: seen.append(msg)) + deadline = time.time() + 180.0 + while not seen and time.time() < deadline: + time.sleep(1.0) + if not seen: + raise TimeoutError("no /odom within 180s — robot never spawned") + explorer = DirectCmdVelExplorer() explorer.linear_speed = 0.5 explorer.start() From a6e809a409382b37025dfa19fdda8378d5a50c95 Mon Sep 17 00:00:00 2001 From: stash Date: Sat, 8 Aug 2026 21:55:26 -0700 Subject: [PATCH 05/12] chore(evals): drop __init__.py files per repo convention (namespace packages, no __all__) --- dimos/evals/__init__.py | 48 ---------------------------------- dimos/evals/suites/__init__.py | 15 ----------- dimos/evals/suites/examples.py | 2 +- 3 files changed, 1 insertion(+), 64 deletions(-) delete mode 100644 dimos/evals/__init__.py delete mode 100644 dimos/evals/suites/__init__.py diff --git a/dimos/evals/__init__.py b/dimos/evals/__init__.py deleted file mode 100644 index f43b4f11cc..0000000000 --- a/dimos/evals/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""DimOS evals: passive (frozen mem2 recordings) and interactive (live robot/sim).""" - -from dimos.evals.runner import EvalRunner, EvalRunnerConfig, RunSummary, summarize -from dimos.evals.scorers import exact, final, floor, judge, mean, ramp, within -from dimos.evals.types import ( - EvalCase, - EvalResult, - EvalRig, - InteractiveEval, - PassiveEval, - Select, - Suite, -) - -__all__ = [ - "EvalCase", - "EvalResult", - "EvalRig", - "EvalRunner", - "EvalRunnerConfig", - "InteractiveEval", - "PassiveEval", - "RunSummary", - "Select", - "Suite", - "exact", - "final", - "floor", - "judge", - "mean", - "ramp", - "summarize", - "within", -] diff --git a/dimos/evals/suites/__init__.py b/dimos/evals/suites/__init__.py deleted file mode 100644 index f02f56de83..0000000000 --- a/dimos/evals/suites/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Eval suites. Each module exports ``SUITE: Suite``.""" diff --git a/dimos/evals/suites/examples.py b/dimos/evals/suites/examples.py index c4f5f700d3..288bf2665e 100644 --- a/dimos/evals/suites/examples.py +++ b/dimos/evals/suites/examples.py @@ -20,7 +20,7 @@ Or from pytest / a notebook:: - from dimos.evals import EvalRunner + from dimos.evals.runner import EvalRunner from dimos.evals.suites.examples import SUITE results = EvalRunner().run(SUITE) From 48c36de60acb9cbe1dbf317e0d5c1280ea1acea3 Mon Sep 17 00:00:00 2001 From: stash Date: Sat, 8 Aug 2026 22:42:15 -0700 Subject: [PATCH 06/12] chore(evals): keep model construction private to evals, no agents/ changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit reuse mcp_client._init_model lazily instead of extracting it — keeps this PR scoped to dimos/evals (+ cli registration). extraction can be its own PR if we want it shared properly. --- dimos/agents/mcp/mcp_client.py | 19 ++++++++-- dimos/agents/mcp/test_mcp_client_unit.py | 2 +- dimos/agents/model.py | 44 ------------------------ dimos/evals/runner.py | 8 +++-- 4 files changed, 23 insertions(+), 50 deletions(-) delete mode 100644 dimos/agents/model.py diff --git a/dimos/agents/mcp/mcp_client.py b/dimos/agents/mcp/mcp_client.py index 6beddb6b07..859b15451b 100644 --- a/dimos/agents/mcp/mcp_client.py +++ b/dimos/agents/mcp/mcp_client.py @@ -20,15 +20,16 @@ import uuid from langchain.agents import create_agent +from langchain.chat_models import init_chat_model from langchain_core.messages import HumanMessage from langchain_core.messages.base import BaseMessage from langchain_core.tools import StructuredTool +from langchain_openai import ChatOpenAI from langgraph.graph.state import CompiledStateGraph from reactivex.disposable import Disposable import requests from dimos.agents.mcp import tool_stream -from dimos.agents.model import init_model from dimos.agents.system_prompt import SYSTEM_PROMPT from dimos.agents.utils import pretty_print_langchain_message from dimos.constants import DEFAULT_THREAD_JOIN_TIMEOUT @@ -41,6 +42,20 @@ logger = setup_logger() +_RESPONSES_REASONING_MODEL_PREFIXES = ("gpt-5", "o1", "o3", "o4") + + +def _init_model(model_name: str) -> Any: + """Initialize a model while preserving LangChain provider resolution.""" + if ":" in model_name or not model_name.startswith(_RESPONSES_REASONING_MODEL_PREFIXES): + return init_chat_model(model=model_name) + + return ChatOpenAI( + model=model_name, + use_responses_api=True, + reasoning={"effort": "medium", "summary": "auto"}, + ) + class McpClientConfig(ModuleConfig): system_prompt: str | None = SYSTEM_PROMPT @@ -218,7 +233,7 @@ def on_system_modules(self, _modules: list[RPCClient]) -> None: model = MockModel(json_path=self.config.model_fixture) else: - model = init_model(self.config.model) + model = _init_model(self.config.model) with self._lock: self._state_graph = create_agent( diff --git a/dimos/agents/mcp/test_mcp_client_unit.py b/dimos/agents/mcp/test_mcp_client_unit.py index dc1af78dbf..a49df130ff 100644 --- a/dimos/agents/mcp/test_mcp_client_unit.py +++ b/dimos/agents/mcp/test_mcp_client_unit.py @@ -260,7 +260,7 @@ def test_on_system_modules_resolves_non_reasoning_models( with ( patch("dimos.agents.mcp.mcp_client.create_agent"), - patch("dimos.agents.model.init_chat_model", return_value=resolved_model) as init, + patch("dimos.agents.mcp.mcp_client.init_chat_model", return_value=resolved_model) as init, ): configured_mcp_client.on_system_modules([]) diff --git a/dimos/agents/model.py b/dimos/agents/model.py deleted file mode 100644 index b71471c01f..0000000000 --- a/dimos/agents/model.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Shared chat-model construction for agents and evals.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from langchain.chat_models import init_chat_model -from langchain_openai import ChatOpenAI - -if TYPE_CHECKING: - from langchain_core.language_models.chat_models import BaseChatModel - -_RESPONSES_REASONING_MODEL_PREFIXES = ("gpt-5", "o1", "o3", "o4") - - -def init_model(model_name: str) -> BaseChatModel: - """Initialize a model while preserving LangChain provider resolution. - - OpenAI reasoning models (gpt-5*/o*) without an explicit ``provider:`` prefix - go through the Responses API with reasoning enabled — the same configuration - the production ``McpClient`` runs, so evals measure what deploys. - """ - if ":" in model_name or not model_name.startswith(_RESPONSES_REASONING_MODEL_PREFIXES): - return init_chat_model(model=model_name) - - return ChatOpenAI( - model=model_name, - use_responses_api=True, - reasoning={"effort": "medium", "summary": "auto"}, - ) diff --git a/dimos/evals/runner.py b/dimos/evals/runner.py index 9fdfec96ff..489ce11b25 100644 --- a/dimos/evals/runner.py +++ b/dimos/evals/runner.py @@ -59,7 +59,7 @@ class EvalRunnerConfig(BaseConfig): model: str = "gpt-5.6-luna" # mirrors McpClientConfig.model # House convention (StoreConfig): pass an instance to inject, e.g. a fake - # chat model in tests. None -> built from `model` via init_model(). + # chat model in tests. None -> built from `model` like McpClient does. chat_model: Any | None = None mcp_url: str = "http://localhost:9990/mcp" live_db: str = "recording.db" # store the Recorder writes (interactive) @@ -252,9 +252,11 @@ def model(self) -> BaseChatModel: if self.config.chat_model is not None: return self.config.chat_model # type: ignore[no-any-return] if self._model is None: - from dimos.agents.model import init_model + # Same construction as the production agent (Responses-API branch + # for gpt-5.x) so evals measure the deployed model config. + from dimos.agents.mcp.mcp_client import _init_model - self._model = init_model(self.config.model) + self._model = _init_model(self.config.model) return self._model def call_skill(self, name: str, args: Mapping[str, object]) -> str: From c67a2da2809617b60ea2e47556172de112c75636 Mon Sep 17 00:00:00 2001 From: stash Date: Sun, 9 Aug 2026 00:34:11 -0700 Subject: [PATCH 07/12] fix blueprint test --- dimos/robot/all_blueprints.py | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/dimos/robot/all_blueprints.py b/dimos/robot/all_blueprints.py index 58b941ae79..f6f7738ae8 100644 --- a/dimos/robot/all_blueprints.py +++ b/dimos/robot/all_blueprints.py @@ -192,6 +192,7 @@ "drone-tracking-module": "dimos.robot.drone.drone_tracking_module.DroneTrackingModule", "emitter-module": "dimos.utils.demo_image_encoding.EmitterModule", "episode-monitor-module": "dimos.imitation.collection.episode_monitor.EpisodeMonitorModule", + "eval-module": "dimos.evals.module.EvalModule", "evaluator": "dimos.navigation.nav_3d.evaluator.evaluator.Evaluator", "far-planner": "dimos.navigation.cmu_nav.modules.far_planner.far_planner.FarPlanner", "fast-lio2": "dimos.hardware.sensors.lidar.fastlio2.module.FastLio2", diff --git a/pyproject.toml b/pyproject.toml index a2bc303efd..31d62180e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -613,6 +613,7 @@ module = [ "mujoco_playground.*", "nav_msgs.*", "open_clip", + "openevals.*", "pinocchio", "pink", "pink.*", From 0ba6f9264b2e7e4513c1e0dc94a1b7323aa559d3 Mon Sep 17 00:00:00 2001 From: stash Date: Sun, 9 Aug 2026 00:36:15 -0700 Subject: [PATCH 08/12] docs(evals): runnable intro (memory2 doc conventions) --- dimos/evals/intro.md | 199 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 dimos/evals/intro.md diff --git a/dimos/evals/intro.md b/dimos/evals/intro.md new file mode 100644 index 0000000000..a8ed863180 --- /dev/null +++ b/dimos/evals/intro.md @@ -0,0 +1,199 @@ +# Evals Intro + +Evals measure what an agent (or a bare model, or a single skill) can do with +the robot's memory. Two kinds: + +- **Passive** — the world is a frozen memory2 recording. Deterministic, cheap, + repeatable. Run these constantly. +- **Interactive** — a live robot or sim; actions change the world; scoring + samples the live memory2 store while the agent works. + +memory2 is the source of truth for everything an eval sees: context selectors +return real `Stream`s, and interactive scoring reads a real `Store`. + +## Quick start (CLI) + +```bash +# two documentation cases against the go2_short recording (needs OPENAI_API_KEY) +dimos evals run dimos.evals.suites.examples + +# same questions with observations withheld — the guessing ablation +dimos evals run dimos.evals.suites.examples --blind + +# list available suites +dimos evals list +``` + +Each run prints a per-case table and writes `results.jsonl`, `summary.json`, +and per-case transcripts to `~/.local/state/dimos/evals/run-*/`. + +## Your first eval, end to end + +Build a tiny recording (any memory2 store works — this is the same API the +robot's Recorder uses; see `dimos/memory2/intro.md` for the full Stream API): + +```python session=evals ansi=false no-result +from dimos.memory2.store.sqlite import SqliteStore +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Vector3 import make_vector3 + +store = SqliteStore(path="/tmp/evals_intro.db") +odom = store.stream("odom", PoseStamped) +for i in range(20): + odom.append( + PoseStamped(position=make_vector3(float(i), 2.5, 0.0), + orientation=Quaternion(0, 0, 0, 1), frame_id="world"), + ts=1000.0 + i, + ) +``` + +```python session=evals ansi=false +print(odom.summary()) +``` + + +``` +Stream("odom"): 20 items, 1970-01-01 00:16:40 — 1970-01-01 00:16:59 (19.0s, 1.00 Hz, 1.68 KiB) +``` + +A passive eval is one Python literal. `context` is a tuple of callables that +receive the opened `Store` and return the mem2 `Stream`s the model may see — +anything the Stream API expresses (windows, filters, single frames) works, and +the runner evenly downsamples each selected stream to `context_budget` +observations before encoding: + +```python session=evals ansi=false no-result +from dimos.evals.scorers import first_number, within +from dimos.evals.types import PassiveEval + +case = PassiveEval( + id="how_far", + inputs="How far along x did you travel, in meters?", + expected=19.0, + parse=first_number, # model text -> float + score=within(1.0), # graded: 1.0 exact, linear to 0 at ±1m + context=(lambda s: s.streams.odom,), + dataset="/tmp/evals_intro.db", # a mem2 name ("go2_short") or a path +) +``` + +Run it. `chat_model=` injects any LangChain chat model — here a canned fake so +this document runs offline; drop the argument to use the production model +config (`gpt-5.6-luna`, same construction as the deployed `McpClient`): + +```python session=evals ansi=false +from langchain_core.language_models.fake_chat_models import FakeListChatModel +from dimos.evals.runner import EvalRunner, summarize + +runner = EvalRunner(chat_model=FakeListChatModel(responses=["about 19 meters"])) +result = runner.run([case])[0] +print(f"score={result.score} passed={result.passed} outputs={result.outputs!r}") +print(summarize([result])) +``` + + +``` +score=1.0 passed=True outputs='about 19 meters' +RunSummary(n=1, mean_score=1.0, pass_rate=1.0, errors=0, duration_s=0.31) +``` + +That's the whole loop: dataset -> context streams -> encoded prompt -> model +-> parse -> score -> artifacts. + +## Scoring + +Scores are floats in `[0, 1]`; `passed = score >= threshold`. Scorers are +plain functions `(expected, got) -> float` — a custom heuristic is a lambda, +not a class: + +```python session=evals ansi=false +from dimos.evals.scorers import choice, exact, first_number, ramp, within, yes_no + +print(exact("yes", "yes"), within(2.0)(10.0, 11.0), ramp(1.0, band=2.0)) +print(first_number("around 12.5 m"), yes_no("Yes, clearly."), choice(" Chairs. ")) +``` + + +``` +1.0 0.5 0.5 +12.5 yes chairs +``` + +- `exact` — equality (the default). Pair with a parser (`yes_no`, `choice`, + `int`) so formatting noise doesn't fail a correct answer. +- `within(band)` — graded numeric credit: 1.0 exact, 0.5 halfway, 0 outside. +- `ramp(distance, band)` — same ramp over meters; msg types support + arithmetic, so physical scorers stay one-liners: + `lambda s: ramp((GOAL - s.streams.odom.last().data.position).length(), band=0.5)` +- `judge(rubric)` — LLM-as-judge with partial credit, wrapping the + langchain/openevals standard (`inputs`/`reference_outputs` convention, so + external VQA benchmarks map on natively). + +Interactive evals score a *series* (one sample per `interval_s`); `aggregate` +reduces it: + +```python session=evals ansi=false +from dimos.evals.scorers import final, floor, mean + +print(final([0.2, 0.9]), floor([0.4, 0.2, 0.8]), mean([0.0, 1.0])) +``` + + +``` +0.9 0.2 0.5 +``` + +`final` = "where did it end up", `floor` = "never left the zone", +`mean` = "how good was it throughout". + +## Interactive evals + +The case names its environment (reproducibility); `score` reads the **live** +store the robot's Recorder writes, sampled every `interval_s`: + +```python session=evals ansi=false no-result +from dimos.evals.scorers import final, ramp +from dimos.evals.types import InteractiveEval +from dimos.msgs.geometry_msgs.Vector3 import Vector3 + +BED = Vector3(-3.567, -1.332, 0.0) + +go_to_bed = InteractiveEval( + id="go_to_bed", + inputs="go to the bed", + score=lambda s: ramp((BED - s.streams.odom.last().data.position).length(), band=2.0), + aggregate=final, + interval_s=2.0, + timeout_s=180.0, + blueprint="unitree-go2-agentic go2-memory", + simulator="dimsim", + scene="apartment", +) +``` + +```bash +dimos evals run dimos.evals.suites.dimsim_house --live-db recording_go2.db +``` + +The result carries the full `(t, score)` series — "reached the bed at t=50s +and stayed" and "grazed it at the deadline" score differently under `floor` +vs `final`. + +## Running + +- **CLI**: `dimos evals run [--tags nav --blind --limit 5 --model gpt-4o]` +- **Python**: `EvalRunner(...).run(SUITE, tags=frozenset({"encoding"}))` +- **pytest**: suites are importable lists — + `@pytest.mark.parametrize("case", SUITE)` and assert on `passed` + (gate live-model tests with `skipif_no_openai`). +- **MCP**: the `EvalModule` skills `run_evals` / `list_eval_suites` return the + summary + run dir, so a coding agent can run evals, grep transcripts, edit + prompts/encodings, and run again. +- **Blind ablation**: `EvalRunner(blind=True)` withholds all observations. A + case that still passes blind is guessable — fix its distractors. Run every + new suite sighted and blind once before trusting it. +- **Preflight**: before anything runs, every case is checked against the rig — + a missing stream fails with `"No stream 'lidar'. Available: [...]"`, a case + needing MCP/sim fails with what's missing. Errors are per-case; one broken + case never kills a run. From c250cd1a1edbbc691d2175122514ececd12ae802 Mon Sep 17 00:00:00 2001 From: stash Date: Sun, 9 Aug 2026 00:43:35 -0700 Subject: [PATCH 09/12] docs(evals): md-babel executable intro, stable results --- dimos/evals/intro.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/dimos/evals/intro.md b/dimos/evals/intro.md index a8ed863180..771b57f366 100644 --- a/dimos/evals/intro.md +++ b/dimos/evals/intro.md @@ -33,11 +33,17 @@ Build a tiny recording (any memory2 store works — this is the same API the robot's Recorder uses; see `dimos/memory2/intro.md` for the full Stream API): ```python session=evals ansi=false no-result +import os +from pathlib import Path + +os.environ["DIMOS_LOG_LEVEL"] = "WARNING" # keep doc output stable + from dimos.memory2.store.sqlite import SqliteStore from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped from dimos.msgs.geometry_msgs.Quaternion import Quaternion from dimos.msgs.geometry_msgs.Vector3 import make_vector3 +Path("/tmp/evals_intro.db").unlink(missing_ok=True) store = SqliteStore(path="/tmp/evals_intro.db") odom = store.stream("odom", PoseStamped) for i in range(20): @@ -52,8 +58,7 @@ for i in range(20): print(odom.summary()) ``` - -``` +```results Stream("odom"): 20 items, 1970-01-01 00:16:40 — 1970-01-01 00:16:59 (19.0s, 1.00 Hz, 1.68 KiB) ``` @@ -89,13 +94,13 @@ from dimos.evals.runner import EvalRunner, summarize runner = EvalRunner(chat_model=FakeListChatModel(responses=["about 19 meters"])) result = runner.run([case])[0] print(f"score={result.score} passed={result.passed} outputs={result.outputs!r}") -print(summarize([result])) +s = summarize([result]) +print(f"n={s.n} mean={s.mean_score} pass_rate={s.pass_rate} errors={s.errors}") ``` - -``` +```results score=1.0 passed=True outputs='about 19 meters' -RunSummary(n=1, mean_score=1.0, pass_rate=1.0, errors=0, duration_s=0.31) +n=1 mean=1.0 pass_rate=1.0 errors=0 ``` That's the whole loop: dataset -> context streams -> encoded prompt -> model @@ -114,8 +119,7 @@ print(exact("yes", "yes"), within(2.0)(10.0, 11.0), ramp(1.0, band=2.0)) print(first_number("around 12.5 m"), yes_no("Yes, clearly."), choice(" Chairs. ")) ``` - -``` +```results 1.0 0.5 0.5 12.5 yes chairs ``` @@ -139,8 +143,7 @@ from dimos.evals.scorers import final, floor, mean print(final([0.2, 0.9]), floor([0.4, 0.2, 0.8]), mean([0.0, 1.0])) ``` - -``` +```results 0.9 0.2 0.5 ``` From 0ddf8b493ac0fc50b9e5d1c263e6725fdeaecbe9 Mon Sep 17 00:00:00 2001 From: stash Date: Sun, 9 Aug 2026 01:23:59 -0700 Subject: [PATCH 10/12] =?UTF-8?q?fix(evals):=20smoke=20test=20skips=20jpeg?= =?UTF-8?q?=20cases=20=E2=80=94=20ros-dev=20container=20lacks=20libturbojp?= =?UTF-8?q?eg?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dimos/evals/test_smoke.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/dimos/evals/test_smoke.py b/dimos/evals/test_smoke.py index 891d3db11d..c9ffa3ee69 100644 --- a/dimos/evals/test_smoke.py +++ b/dimos/evals/test_smoke.py @@ -12,27 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Live passive smoke: real model, real LFS recordings. Self-hosted + API key.""" +"""Live passive smoke: real model, real LFS recordings. Self-hosted + API key. + +Runs only the ``numeric`` cases (odom + pointcloud str encodings) — the +self-hosted ros-dev container has no libturbojpeg, so image-encoding cases +are exercised locally via ``dimos evals run dimos.evals.suites.examples``. +""" from __future__ import annotations +from pathlib import Path + import pytest pytestmark = [pytest.mark.self_hosted, pytest.mark.skipif_no_openai] -def test_passive_smoke(tmp_path) -> None: # type: ignore[no-untyped-def] +def test_passive_smoke(tmp_path: Path) -> None: from dimos.evals.runner import EvalRunner, summarize - from dimos.evals.suites.examples import SUITE + from dimos.evals.suites.go2_smoke import SUITE from dimos.utils.data import get_data get_data("go2_short.db") runner = EvalRunner(model="gpt-4o-mini", out_dir=tmp_path / "evals") - results = runner.run(SUITE) + results = runner.run(SUITE, tags=frozenset({"numeric"})) assert not any(r.error for r in results), [r.error for r in results] s = summarize(results) - # The lidar-points case reads a number embedded in the str() encoding and - # the image case is unambiguous — a competent VLM should clear both. + # The lidar-points case reads a number embedded in the str() encoding — + # a competent model clears it outright; displacement earns graded credit. assert s.mean_score >= 0.5 From 7977fd974efbca4f654392d7fe24f42207afdd79 Mon Sep 17 00:00:00 2001 From: stash Date: Sun, 9 Aug 2026 20:26:33 -0700 Subject: [PATCH 11/12] add: semantic object VQA benchmark (47 cases over go2_bigoffice YOLO+odom object memory) Frozen detections + independent truth clusterer, 12 families incl. VQASynth-style distances, set-F1 recall, egocentric and betweenness. Gates: blind-ablation ceiling, answer-leak grep, abstraction budget (max one agent_encode + one memory2 search skill), target-only diff. --- dimos/memory2/objects.py | 90 + dimos/memory2/test_objects.py | 43 + evals_bench/semantic/abstraction_check.py | 144 + evals_bench/semantic/benchmark.py | 274 ++ evals_bench/semantic/cheat_check.py | 64 + evals_bench/semantic/detections.json | 2639 +++++++++++++++++++ evals_bench/semantic/generate_detections.py | 94 + evals_bench/semantic/generate_rows.py | 632 +++++ evals_bench/semantic/rows.json | 869 ++++++ 9 files changed, 4849 insertions(+) create mode 100644 dimos/memory2/objects.py create mode 100644 dimos/memory2/test_objects.py create mode 100644 evals_bench/semantic/abstraction_check.py create mode 100644 evals_bench/semantic/benchmark.py create mode 100644 evals_bench/semantic/cheat_check.py create mode 100644 evals_bench/semantic/detections.json create mode 100644 evals_bench/semantic/generate_detections.py create mode 100644 evals_bench/semantic/generate_rows.py create mode 100644 evals_bench/semantic/rows.json diff --git a/dimos/memory2/objects.py b/dimos/memory2/objects.py new file mode 100644 index 0000000000..9da1e34b6b --- /dev/null +++ b/dimos/memory2/objects.py @@ -0,0 +1,90 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Materialize frozen raw detections as the canonical perception detection stream. + +``detections3d_stream`` takes raw per-frame detection dicts (``class_name``, +``confidence``, ``ts``, odom-grounded world ``x``/``y``/``z``) and materializes +the memory2 convention established by ``dimos/perception/memory/tool_localize.py``: +a stream named ``"detections3d"`` whose payload is +:class:`~dimos.perception.detection.type.detection3d.imageDetections3DPC.ImageDetections3DPC` +— one observation per camera frame, holding one +:class:`~dimos.perception.detection.type.detection3d.pointcloud.Detection3DPC` +per detection. + +The source recording has no depth_image/camera_info, so tool_localize's depth +projection cannot run. Positions come from the YOLO+odom grounding instead +(object world position = robot odom position at detection time) and are stored +as a single-point world-frame pointcloud per detection; the frame image is a +stub ``Image`` carrying only the timestamp. +""" + +from __future__ import annotations + +from collections import defaultdict +from typing import Any + +import numpy as np + +from dimos.memory2.store.memory import MemoryStore +from dimos.memory2.stream import Stream +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.perception.detection.type.detection3d.imageDetections3DPC import ImageDetections3DPC +from dimos.perception.detection.type.detection3d.pointcloud import Detection3DPC + +DETECTIONS_STREAM = "detections3d" # stream name convention from tool_localize.py + + +def detections3d_stream(detections: list[dict[str, Any]]) -> Stream[ImageDetections3DPC]: + """Materialize raw detection dicts as the full-span ``detections3d`` stream. + + No curation: every detection of every frame is stored, matching what the + perception pipeline itself would have written. Summarizing this stream for + an agent is the encoder's job, not the store's. + """ + frames: dict[float, list[dict[str, Any]]] = defaultdict(list) + for det in sorted(detections, key=lambda d: d["ts"]): + frames[det["ts"]].append(det) + + stream: Stream[ImageDetections3DPC] = MemoryStore().stream( + DETECTIONS_STREAM, ImageDetections3DPC + ) + for ts, dets in frames.items(): + image = Image(ts=ts) + stream.append( + ImageDetections3DPC( + image=image, + detections=[ + Detection3DPC( + bbox=(0.0, 0.0, 0.0, 0.0), + track_id=-1, + class_id=-1, + confidence=d["confidence"], + name=d["class_name"], + ts=ts, + image=image, + frame_id="world", + pointcloud=PointCloud2.from_numpy( + np.array([[d["x"], d["y"], d["z"]]]), + frame_id="world", + timestamp=ts, + ), + ) + for d in dets + ], + ), + ts=ts, + ) + return stream diff --git a/dimos/memory2/test_objects.py b/dimos/memory2/test_objects.py new file mode 100644 index 0000000000..170f3dfebb --- /dev/null +++ b/dimos/memory2/test_objects.py @@ -0,0 +1,43 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dimos.memory2.objects import DETECTIONS_STREAM, detections3d_stream +from dimos.perception.detection.type.detection3d.imageDetections3DPC import ImageDetections3DPC + + +def test_detections3d_stream_materializes_canonical_frames() -> None: + detections = [ + {"class_name": "chair", "confidence": 0.8, "ts": 1.0, "x": 0.0, "y": 0.0, "z": 0.3}, + {"class_name": "tv", "confidence": 0.9, "ts": 1.0, "x": 5.0, "y": 5.0, "z": 1.0}, + {"class_name": "chair", "confidence": 0.6, "ts": 2.0, "x": 1.0, "y": 0.0, "z": 0.3}, + ] + stream = detections3d_stream(detections) + assert stream.name == DETECTIONS_STREAM == "detections3d" + + observations = stream.to_list() + # one observation per frame ts, every detection kept (no curation) + assert [o.ts for o in observations] == [1.0, 2.0] + frame = observations[0].data + assert isinstance(frame, ImageDetections3DPC) + assert sorted(det.name for det in frame) == ["chair", "tv"] + + tv = next(det for det in frame if det.name == "tv") + assert tv.frame_id == "world" + assert (tv.center.x, tv.center.y, tv.center.z) == (5.0, 5.0, 1.0) + assert tv.confidence == 0.9 + assert len(tv.pointcloud) == 1 + + later = observations[1].data + assert [det.name for det in later] == ["chair"] + assert later[0].ts == 2.0 diff --git a/evals_bench/semantic/abstraction_check.py b/evals_bench/semantic/abstraction_check.py new file mode 100644 index 0000000000..c00ea2f0f1 --- /dev/null +++ b/evals_bench/semantic/abstraction_check.py @@ -0,0 +1,144 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Abstraction-budget gate: deterministic AST diff against the fork point. + +An experiment may add AT MOST one @skill function that touches memory2 APIs, +at most ONE ``agent_encode`` per class, and NO new store/stream wrapper +classes over memory2. More than that is abstraction laundering — moving the +benchmark's work into unpenalized scaffolding — and fails the gate (exit 1). + +Usage: abstraction_check.py --worktree [--fork 0ddf8b493] +""" + +from __future__ import annotations + +import argparse +import ast +from pathlib import Path +import re +import subprocess +import sys + +FORK_SHA = "0ddf8b493" +MEMORY2_API = re.compile(r"memory2|MemoryStore|SqliteStore|\.streams\b|\.stream\(") +WRAPPER_BASES = ("Store", "Stream") + + +def _git(worktree: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(worktree), *args], capture_output=True, text=True, check=False + ) + return result.stdout + + +def changed_py_files(worktree: Path, fork: str) -> list[str]: + tracked = _git(worktree, "diff", "--name-only", fork, "--", "*.py").splitlines() + untracked = _git( + worktree, "ls-files", "--others", "--exclude-standard", "--", "*.py" + ).splitlines() + return sorted({p for p in tracked + untracked if p}) + + +def _decorator_is_skill(dec: ast.expr) -> bool: + if isinstance(dec, ast.Call): + dec = dec.func + if isinstance(dec, ast.Attribute): + return dec.attr == "skill" + return isinstance(dec, ast.Name) and dec.id == "skill" + + +def _analyze(src: str) -> dict[str, object] | None: + """AST facts for one file: skill fns, agent_encode counts, wrapper classes.""" + try: + tree = ast.parse(src) + except SyntaxError: + return None + imports_memory2 = "memory2" in src + + skills: dict[str, bool] = {} # fn name -> touches memory2 + encodes: dict[str, int] = {} # class name (or "") -> agent_encode defs + wrappers: set[str] = set() + + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + if any(_decorator_is_skill(d) for d in node.decorator_list): + segment = ast.get_source_segment(src, node) or "" + skills[node.name] = imports_memory2 or bool(MEMORY2_API.search(segment)) + if isinstance(node, ast.ClassDef): + count = sum( + isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)) + and item.name == "agent_encode" + for item in node.body + ) + if count: + encodes[node.name] = count + base_names = [ + b.attr if isinstance(b, ast.Attribute) else getattr(b, "id", "") for b in node.bases + ] + if imports_memory2 and any(n.endswith(WRAPPER_BASES) for n in base_names if n): + wrappers.add(node.name) + for node in tree.body: # module-level agent_encode defs + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and ( + node.name == "agent_encode" + ): + encodes[""] = encodes.get("", 0) + 1 + return {"skills": skills, "encodes": encodes, "wrappers": wrappers} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--worktree", required=True, type=Path) + parser.add_argument("--fork", default=FORK_SHA) + args = parser.parse_args() + + new_memory2_skills: list[str] = [] + violations: list[str] = [] + + for rel in changed_py_files(args.worktree, args.fork): + path = args.worktree / rel + if not path.exists(): # deleted file + continue + new = _analyze(path.read_text()) + old = _analyze(_git(args.worktree, "show", f"{args.fork}:{rel}")) + if new is None: + continue + old = old or {"skills": {}, "encodes": {}, "wrappers": set()} + + for name, touches in new["skills"].items(): # type: ignore[union-attr] + if name not in old["skills"] and touches: # type: ignore[operator] + new_memory2_skills.append(f"{rel}:{name}") + for cls, count in new["encodes"].items(): # type: ignore[union-attr] + added = count - old["encodes"].get(cls, 0) # type: ignore[union-attr] + if added > 1: + violations.append(f"{rel}: {added} agent_encode defs added on {cls!r} (max 1)") + for cls in new["wrappers"] - old["wrappers"]: # type: ignore[operator] + violations.append(f"{rel}: new class {cls!r} wraps a memory2 Store/Stream") + + if len(new_memory2_skills) > 1: + violations.append( + f"{len(new_memory2_skills)} @skill functions touching memory2 added (max 1): " + + ", ".join(new_memory2_skills) + ) + + if violations: + print("abstraction check FAILED:", file=sys.stderr) + for v in violations: + print(f" - {v}", file=sys.stderr) + sys.exit(1) + print(f"abstraction check ok ({len(new_memory2_skills)} memory2 skill added)") + + +if __name__ == "__main__": + main() diff --git a/evals_bench/semantic/benchmark.py b/evals_bench/semantic/benchmark.py new file mode 100644 index 0000000000..38e86cc8fc --- /dev/null +++ b/evals_bench/semantic/benchmark.py @@ -0,0 +1,274 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Semantic-map VQA benchmark over go2_bigoffice via the dimos evals framework. + +Score = mean [0,1] credit across generated cases (rows.json). The surface +under test is the encoding the agent receives for the raw semantic detection +stream: ``detections3d_stream()`` (dimos/memory2/objects.py) materializes the +frozen detections (detections.json) as the canonical perception payload — +``ImageDetections3DPC`` observations on a memory2 stream named +``"detections3d"``, the tool_localize.py convention — and the eval runner +encodes each observation (``agent_encode()`` when the type provides it, +``str(data)`` fallback). The evo target is the file where that +``agent_encode`` lives: +``dimos/perception/detection/type/detection3d/imageDetections3DPC.py``. + +CONTEXT PARITY CONTRACT (load-bearing — generate_rows.py and this harness +must agree; do not change one side without the other): + + 1. Ground truth in generate_rows.py is computed from the FULL recording's + detections (all of detections.json, minus the documented class filters). + The student's objects context is therefore the FULL-span detections3d + stream: every frame, every detection, no curation and no truncation. + ``build_cases`` asserts frame count <= CONTEXT_BUDGET so the runner's + evenly-spaced subsampler never drops a frame. + 2. For time-conditioned families (nearest / egoside), truth is the robot + odom pose at the question timestamp. The odom context window therefore + ENDS EXACTLY at the question timestamp (row ``odom_window[1] == t``): + "your current pose is the last odom observation shown". The objects + context stays full-span for these rows too — truth ranks the full map. + 3. Odom may be subsampled by the runner (evenly spaced, ~19 Hz source); + the subsampler always keeps the last observation, so the pose the + question is conditioned on survives subsampling. + +Flags: + --blind withhold context (guessing ablation) + --max-mean X exit 1 if mean score exceeds X (blind-gate mode) + --min-mean X exit 1 if mean score falls below X (floor-gate mode) + --limit N run only the first N cases +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import re +import sys + +# Anti-gaming + cost ceiling per case context. The uncurated full-span +# detections3d stream str()-encodes to ~156k chars (rich table + ANSI), so the +# ceiling sits above the raw baseline while still refusing point-dump blowups. +MAX_CONTEXT_CHARS = 200_000 +CONTEXT_BUDGET = 256 # >= frame count (161): full detection stream, no subsampling +TRANSIENT = re.compile(r"429|rate.?limit|timeout|connection|temporar", re.IGNORECASE) + + +def pick(choices: list[str]): + ordered = sorted(choices, key=len, reverse=True) + pattern = re.compile(r"\b(" + "|".join(ordered) + r")\b", re.IGNORECASE) + + def parse(text: str) -> str: + matches = pattern.findall(text) + return matches[-1].lower() if matches else "" + + return parse + + +def class_set(vocab: list[str]): + """Parse every class name mentioned in a reply into a set. + + ``vocab`` = mapped classes + verified-absent probes, so hallucinated + extras are caught and penalized via F1 precision. + """ + ordered = sorted(vocab, key=len, reverse=True) + pattern = re.compile(r"\b(" + "|".join(re.escape(v) for v in ordered) + r")\b", re.IGNORECASE) + + def parse(text: str) -> frozenset[str]: + return frozenset(m.lower() for m in pattern.findall(text)) + + return parse + + +def set_f1(expected: frozenset[str], got: frozenset[str]) -> float: + """F1 between predicted and truth class sets ([0,1]; listing every class + loses precision, listing nothing scores 0).""" + tp = len(expected & got) + return 2 * tp / (len(expected) + len(got)) if tp else 0.0 + + +def build_cases(rows: list[dict]): + from dimos.evals.scorers import exact, first_number, within + from dimos.evals.types import PassiveEval + from dimos.memory2.objects import detections3d_stream + + detections = json.loads((Path(__file__).parent / "detections.json").read_text()) + # parity contract #1: full stream must fit the runner's context budget + n_frames = len({d["ts"] for d in detections}) + assert n_frames <= CONTEXT_BUDGET, ( + f"{n_frames} detection frames > CONTEXT_BUDGET {CONTEXT_BUDGET}: " + "the runner would subsample the objects context — parity broken" + ) + + def objects_select(store): + return detections3d_stream(detections) + + def odom_select(window): + return lambda s, w=tuple(window): s.streams.odom.range_time(*w) + + cases = [] + for row in rows: + context = [objects_select] + if row["ctx"] == "objects+odom": + context.append(odom_select(row["odom_window"])) + if row["type"] == "numeric": + case = PassiveEval( + id=row["id"], + inputs=row["q"], + expected=float(row["a"]), + parse=first_number, + score=within(float(row["band"])), + context=tuple(context), + dataset=row["dataset"], + tags=frozenset({row["family"], "numeric"}), + ) + elif row["type"] == "set": + case = PassiveEval( + id=row["id"], + inputs=row["q"], + expected=frozenset(row["a"]), + parse=class_set(list(row["vocab"])), + score=set_f1, + context=tuple(context), + dataset=row["dataset"], + tags=frozenset({row["family"], "set"}), + ) + else: + case = PassiveEval( + id=row["id"], + inputs=row["q"], + expected=str(row["a"]), + parse=pick(list(row["choices"])), + score=exact, + context=tuple(context), + dataset=row["dataset"], + tags=frozenset({row["family"], "mcq"}), + ) + cases.append(case) + return cases + + +def check_context_budget(runner, cases) -> None: + """Fail fast if any case's encoded context exceeds the ceiling. + + Text blocks are measured as the raw text the model receives (json.dumps + would count every ANSI/box-drawing char as a 6-char \\uXXXX escape and + overstate rich-table encodings ~2.6x); other blocks by their JSON size. + """ + for case in cases: + store = runner.open_dataset(case.dataset) + try: + total = sum( + len(block["text"]) + if block.get("type") == "text" + else len(json.dumps(block, default=str)) + for select in case.context + for block in runner.encode(select(store)) + ) + finally: + store.stop() + if total > MAX_CONTEXT_CHARS: + print( + f"context for {case.id} is {total} chars > {MAX_CONTEXT_CHARS};" + " encoding too verbose — refusing to run", + file=sys.stderr, + ) + sys.exit(2) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--blind", action="store_true") + parser.add_argument("--max-mean", type=float, default=None) + parser.add_argument("--min-mean", type=float, default=None) + parser.add_argument("--limit", type=int, default=0) + args = parser.parse_args() + + from evo_agent import Run + + from dimos.evals.runner import EvalRunner + + rows = json.loads((Path(__file__).parent / "rows.json").read_text()) + if args.limit: + rows = rows[: args.limit] + by_id = {row["id"]: row for row in rows} + cases = build_cases(rows) + + runner = EvalRunner(blind=args.blind, context_budget=CONTEXT_BUDGET) + if not args.blind: + check_context_budget(runner, cases) + results = list(runner.run(cases)) + + # one retry pass for transport-flavored failures; API blips are not + # encoding regressions + retry = [ + c for c, r in zip(cases, results, strict=False) if r.error and TRANSIENT.search(r.error) + ] + if retry: + print(f"retrying {len(retry)} transient failures", file=sys.stderr) + retried = { + r.case_id: r + for r in EvalRunner(blind=args.blind, context_budget=CONTEXT_BUDGET).run(retry) + } + results = [retried.get(r.case_id, r) for r in results] + + infra = re.compile( + r"Authentication|RateLimit|quota|APIConnection|APIError|NotFound|Timeout" + r"|api.?key|OpenAIError", + re.IGNORECASE, + ) + outage = [r for r in results if r.error and infra.search(r.error)] + if len(outage) / len(results) >= 0.5: + print( + f"FAIL: {len(outage)}/{len(results)} cases hit API/infra errors — " + "refusing to report a fake 0.0", + file=sys.stderr, + ) + sys.exit(3) + + run = Run() + try: + for result in results: + row = by_id[result.case_id] + run.report( + result.case_id, + score=result.score, + summary=f"{row['family']} expected={row['a']} score={result.score:.2f}", + failure_reason=(result.error or None) if result.score < 1.0 else None, + extras={ + "question": row["q"], + "expected": row["a"], + "family": row["family"], + "kind": row["type"], + "blind": args.blind, + "output": result.outputs[:800], + "error": result.error, + }, + ) + finally: + run.finish() + + mean = sum(r.score for r in results) / len(results) + print(f"mean={mean:.4f} n={len(results)} blind={args.blind}") + if args.max_mean is not None and mean > args.max_mean: + print(f"FAIL: mean {mean:.3f} > ceiling {args.max_mean}", file=sys.stderr) + sys.exit(1) + if args.min_mean is not None and mean < args.min_mean: + print(f"FAIL: mean {mean:.3f} < floor {args.min_mean}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/evals_bench/semantic/cheat_check.py b/evals_bench/semantic/cheat_check.py new file mode 100644 index 0000000000..b3fc0d6915 --- /dev/null +++ b/evals_bench/semantic/cheat_check.py @@ -0,0 +1,64 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Deterministic answer-leak gate: the evo target (--target, the file holding +``ImageDetections3DPC.agent_encode``) must not reference the benchmark, its +data files, or hardcode row answers. Exits non-zero on a hit.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys + +FORBIDDEN_SUBSTRINGS = [ + "evals_bench", + "rows.json", + "detections.json", + "go2_bigoffice", + "go2_short", +] + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--target", required=True, type=Path) + args = parser.parse_args() + + source = args.target.read_text() + hits: list[str] = [] + + for token in FORBIDDEN_SUBSTRINGS: + if token in source: + hits.append(f"forbidden reference {token!r}") + + rows = json.loads((Path(__file__).parent / "rows.json").read_text()) + answers = { + str(row["a"]) for row in rows if row["type"] == "numeric" and len(str(row["a"])) >= 3 + } + for answer in sorted(answers): + if answer in source: + hits.append(f"row answer literal {answer!r} appears in target") + + if hits: + print("cheat check FAILED:", file=sys.stderr) + for hit in hits: + print(f" - {hit}", file=sys.stderr) + sys.exit(1) + print("cheat check ok") + + +if __name__ == "__main__": + main() diff --git a/evals_bench/semantic/detections.json b/evals_bench/semantic/detections.json new file mode 100644 index 0000000000..8475a5534e --- /dev/null +++ b/evals_bench/semantic/detections.json @@ -0,0 +1,2639 @@ +[ + { + "class_name": "surfboard", + "confidence": 0.507, + "ts": 6.67, + "x": -11.659, + "y": 12.972, + "z": 0.312, + "yaw": -0.43 + }, + { + "class_name": "bed", + "confidence": 0.499, + "ts": 8.08, + "x": -10.973, + "y": 12.633, + "z": 0.312, + "yaw": -0.472 + }, + { + "class_name": "bed", + "confidence": 0.825, + "ts": 8.78, + "x": -10.592, + "y": 12.444, + "z": 0.307, + "yaw": -0.45 + }, + { + "class_name": "bed", + "confidence": 0.625, + "ts": 9.13, + "x": -10.404, + "y": 12.35, + "z": 0.307, + "yaw": -0.514 + }, + { + "class_name": "car", + "confidence": 0.595, + "ts": 10.18, + "x": -10.008, + "y": 12.16, + "z": 0.307, + "yaw": -0.183 + }, + { + "class_name": "bed", + "confidence": 0.459, + "ts": 11.24, + "x": -9.483, + "y": 12.027, + "z": 0.311, + "yaw": -0.277 + }, + { + "class_name": "bed", + "confidence": 0.643, + "ts": 11.59, + "x": -9.142, + "y": 11.935, + "z": 0.316, + "yaw": -0.266 + }, + { + "class_name": "bench", + "confidence": 0.426, + "ts": 18.27, + "x": -4.488, + "y": 10.47, + "z": 0.31, + "yaw": -0.248 + }, + { + "class_name": "chair", + "confidence": 0.571, + "ts": 22.13, + "x": -3.003, + "y": 9.609, + "z": 0.315, + "yaw": -1.72 + }, + { + "class_name": "chair", + "confidence": 0.476, + "ts": 22.13, + "x": -3.003, + "y": 9.609, + "z": 0.315, + "yaw": -1.72 + }, + { + "class_name": "chair", + "confidence": 0.469, + "ts": 22.13, + "x": -3.003, + "y": 9.609, + "z": 0.315, + "yaw": -1.72 + }, + { + "class_name": "chair", + "confidence": 0.527, + "ts": 23.89, + "x": -3.245, + "y": 8.332, + "z": 0.31, + "yaw": -1.902 + }, + { + "class_name": "chair", + "confidence": 0.487, + "ts": 23.89, + "x": -3.245, + "y": 8.332, + "z": 0.31, + "yaw": -1.902 + }, + { + "class_name": "chair", + "confidence": 0.437, + "ts": 24.59, + "x": -3.366, + "y": 8.016, + "z": 0.309, + "yaw": -1.97 + }, + { + "class_name": "chair", + "confidence": 0.509, + "ts": 24.94, + "x": -3.463, + "y": 7.769, + "z": 0.314, + "yaw": -1.957 + }, + { + "class_name": "chair", + "confidence": 0.479, + "ts": 24.94, + "x": -3.463, + "y": 7.769, + "z": 0.314, + "yaw": -1.957 + }, + { + "class_name": "chair", + "confidence": 0.468, + "ts": 24.94, + "x": -3.463, + "y": 7.769, + "z": 0.314, + "yaw": -1.957 + }, + { + "class_name": "chair", + "confidence": 0.47, + "ts": 25.29, + "x": -3.578, + "y": 7.507, + "z": 0.314, + "yaw": -1.961 + }, + { + "class_name": "chair", + "confidence": 0.522, + "ts": 25.64, + "x": -3.727, + "y": 7.157, + "z": 0.314, + "yaw": -1.956 + }, + { + "class_name": "chair", + "confidence": 0.558, + "ts": 29.16, + "x": -4.171, + "y": 4.982, + "z": 0.309, + "yaw": -0.625 + }, + { + "class_name": "chair", + "confidence": 0.48, + "ts": 29.16, + "x": -4.171, + "y": 4.982, + "z": 0.309, + "yaw": -0.625 + }, + { + "class_name": "tv", + "confidence": 0.455, + "ts": 29.16, + "x": -4.171, + "y": 4.982, + "z": 0.309, + "yaw": -0.625 + }, + { + "class_name": "chair", + "confidence": 0.409, + "ts": 29.16, + "x": -4.171, + "y": 4.982, + "z": 0.309, + "yaw": -0.625 + }, + { + "class_name": "chair", + "confidence": 0.467, + "ts": 29.5, + "x": -4.029, + "y": 4.886, + "z": 0.309, + "yaw": -0.565 + }, + { + "class_name": "chair", + "confidence": 0.678, + "ts": 29.86, + "x": -3.867, + "y": 4.79, + "z": 0.316, + "yaw": -0.558 + }, + { + "class_name": "chair", + "confidence": 0.545, + "ts": 29.86, + "x": -3.867, + "y": 4.79, + "z": 0.316, + "yaw": -0.558 + }, + { + "class_name": "chair", + "confidence": 0.424, + "ts": 29.86, + "x": -3.867, + "y": 4.79, + "z": 0.316, + "yaw": -0.558 + }, + { + "class_name": "tv", + "confidence": 0.423, + "ts": 29.86, + "x": -3.867, + "y": 4.79, + "z": 0.316, + "yaw": -0.558 + }, + { + "class_name": "chair", + "confidence": 0.532, + "ts": 30.92, + "x": -3.392, + "y": 4.535, + "z": 0.312, + "yaw": -0.381 + }, + { + "class_name": "chair", + "confidence": 0.428, + "ts": 30.92, + "x": -3.392, + "y": 4.535, + "z": 0.312, + "yaw": -0.381 + }, + { + "class_name": "chair", + "confidence": 0.464, + "ts": 31.62, + "x": -3.062, + "y": 4.456, + "z": 0.31, + "yaw": -0.026 + }, + { + "class_name": "chair", + "confidence": 0.783, + "ts": 33.38, + "x": -2.377, + "y": 4.796, + "z": 0.308, + "yaw": 0.981 + }, + { + "class_name": "chair", + "confidence": 0.462, + "ts": 33.38, + "x": -2.377, + "y": 4.796, + "z": 0.308, + "yaw": 0.981 + }, + { + "class_name": "person", + "confidence": 0.437, + "ts": 33.72, + "x": -2.264, + "y": 4.985, + "z": 0.314, + "yaw": 1.001 + }, + { + "class_name": "chair", + "confidence": 0.506, + "ts": 34.42, + "x": -2.112, + "y": 5.334, + "z": 0.31, + "yaw": 1.31 + }, + { + "class_name": "chair", + "confidence": 0.467, + "ts": 35.48, + "x": -2.121, + "y": 5.497, + "z": 0.313, + "yaw": 0.386 + }, + { + "class_name": "chair", + "confidence": 0.413, + "ts": 35.48, + "x": -2.121, + "y": 5.497, + "z": 0.313, + "yaw": 0.386 + }, + { + "class_name": "tv", + "confidence": 0.579, + "ts": 37.95, + "x": -1.862, + "y": 4.724, + "z": 0.323, + "yaw": -0.872 + }, + { + "class_name": "tv", + "confidence": 0.474, + "ts": 38.3, + "x": -1.699, + "y": 4.517, + "z": 0.312, + "yaw": -0.864 + }, + { + "class_name": "toilet", + "confidence": 0.448, + "ts": 38.3, + "x": -1.699, + "y": 4.517, + "z": 0.312, + "yaw": -0.864 + }, + { + "class_name": "tv", + "confidence": 0.575, + "ts": 38.65, + "x": -1.565, + "y": 4.369, + "z": 0.316, + "yaw": -0.859 + }, + { + "class_name": "tv", + "confidence": 0.495, + "ts": 39.0, + "x": -1.41, + "y": 4.187, + "z": 0.311, + "yaw": -0.849 + }, + { + "class_name": "microwave", + "confidence": 0.436, + "ts": 39.0, + "x": -1.41, + "y": 4.187, + "z": 0.311, + "yaw": -0.849 + }, + { + "class_name": "microwave", + "confidence": 0.698, + "ts": 39.36, + "x": -1.249, + "y": 4.013, + "z": 0.307, + "yaw": -0.831 + }, + { + "class_name": "person", + "confidence": 0.506, + "ts": 39.7, + "x": -1.146, + "y": 3.886, + "z": 0.318, + "yaw": -1.099 + }, + { + "class_name": "surfboard", + "confidence": 0.569, + "ts": 42.86, + "x": -0.679, + "y": 2.291, + "z": 0.31, + "yaw": -1.492 + }, + { + "class_name": "refrigerator", + "confidence": 0.418, + "ts": 43.92, + "x": -0.625, + "y": 1.634, + "z": 0.309, + "yaw": -1.328 + }, + { + "class_name": "person", + "confidence": 0.861, + "ts": 50.58, + "x": 0.864, + "y": 0.548, + "z": 0.31, + "yaw": 2.55 + }, + { + "class_name": "person", + "confidence": 0.768, + "ts": 51.3, + "x": 0.589, + "y": 0.609, + "z": 0.308, + "yaw": 2.99 + }, + { + "class_name": "person", + "confidence": 0.533, + "ts": 51.64, + "x": 0.428, + "y": 0.635, + "z": 0.309, + "yaw": 3.004 + }, + { + "class_name": "person", + "confidence": 0.885, + "ts": 51.99, + "x": 0.214, + "y": 0.653, + "z": 0.315, + "yaw": 3.011 + }, + { + "class_name": "refrigerator", + "confidence": 0.593, + "ts": 51.99, + "x": 0.214, + "y": 0.653, + "z": 0.315, + "yaw": 3.011 + }, + { + "class_name": "person", + "confidence": 0.562, + "ts": 52.35, + "x": 0.026, + "y": 0.684, + "z": 0.313, + "yaw": 2.883 + }, + { + "class_name": "refrigerator", + "confidence": 0.529, + "ts": 52.35, + "x": 0.026, + "y": 0.684, + "z": 0.313, + "yaw": 2.883 + }, + { + "class_name": "person", + "confidence": 0.6, + "ts": 52.7, + "x": -0.162, + "y": 0.724, + "z": 0.31, + "yaw": 2.898 + }, + { + "class_name": "tv", + "confidence": 0.492, + "ts": 52.7, + "x": -0.162, + "y": 0.724, + "z": 0.31, + "yaw": 2.898 + }, + { + "class_name": "person", + "confidence": 0.425, + "ts": 52.7, + "x": -0.162, + "y": 0.724, + "z": 0.31, + "yaw": 2.898 + }, + { + "class_name": "train", + "confidence": 0.469, + "ts": 53.05, + "x": -0.365, + "y": 0.773, + "z": 0.31, + "yaw": 2.901 + }, + { + "class_name": "toothbrush", + "confidence": 0.476, + "ts": 54.1, + "x": -0.853, + "y": 0.925, + "z": 0.311, + "yaw": 3.055 + }, + { + "class_name": "person", + "confidence": 0.448, + "ts": 55.51, + "x": -1.34, + "y": 0.603, + "z": 0.315, + "yaw": -1.968 + }, + { + "class_name": "person", + "confidence": 0.452, + "ts": 57.27, + "x": -1.367, + "y": -0.034, + "z": 0.309, + "yaw": -1.653 + }, + { + "class_name": "bottle", + "confidence": 0.483, + "ts": 57.62, + "x": -1.385, + "y": -0.241, + "z": 0.314, + "yaw": -1.637 + }, + { + "class_name": "cake", + "confidence": 0.57, + "ts": 58.68, + "x": -1.555, + "y": -1.061, + "z": 0.312, + "yaw": -1.793 + }, + { + "class_name": "bottle", + "confidence": 0.407, + "ts": 58.68, + "x": -1.555, + "y": -1.061, + "z": 0.312, + "yaw": -1.793 + }, + { + "class_name": "bottle", + "confidence": 0.486, + "ts": 59.37, + "x": -1.694, + "y": -1.596, + "z": 0.314, + "yaw": -1.884 + }, + { + "class_name": "bottle", + "confidence": 0.464, + "ts": 59.37, + "x": -1.694, + "y": -1.596, + "z": 0.314, + "yaw": -1.884 + }, + { + "class_name": "bottle", + "confidence": 0.413, + "ts": 59.37, + "x": -1.694, + "y": -1.596, + "z": 0.314, + "yaw": -1.884 + }, + { + "class_name": "bottle", + "confidence": 0.445, + "ts": 60.78, + "x": -1.981, + "y": -2.564, + "z": 0.307, + "yaw": -1.839 + }, + { + "class_name": "bottle", + "confidence": 0.43, + "ts": 60.78, + "x": -1.981, + "y": -2.564, + "z": 0.307, + "yaw": -1.839 + }, + { + "class_name": "bottle", + "confidence": 0.413, + "ts": 60.78, + "x": -1.981, + "y": -2.564, + "z": 0.307, + "yaw": -1.839 + }, + { + "class_name": "bottle", + "confidence": 0.508, + "ts": 61.13, + "x": -2.038, + "y": -2.751, + "z": 0.314, + "yaw": -1.837 + }, + { + "class_name": "bottle", + "confidence": 0.464, + "ts": 61.13, + "x": -2.038, + "y": -2.751, + "z": 0.314, + "yaw": -1.837 + }, + { + "class_name": "bottle", + "confidence": 0.439, + "ts": 61.48, + "x": -2.1, + "y": -2.981, + "z": 0.313, + "yaw": -1.817 + }, + { + "class_name": "bottle", + "confidence": 0.417, + "ts": 61.48, + "x": -2.1, + "y": -2.981, + "z": 0.313, + "yaw": -1.817 + }, + { + "class_name": "bottle", + "confidence": 0.413, + "ts": 61.48, + "x": -2.1, + "y": -2.981, + "z": 0.313, + "yaw": -1.817 + }, + { + "class_name": "bottle", + "confidence": 0.409, + "ts": 61.48, + "x": -2.1, + "y": -2.981, + "z": 0.313, + "yaw": -1.817 + }, + { + "class_name": "cup", + "confidence": 0.81, + "ts": 62.87, + "x": -2.29, + "y": -3.841, + "z": 0.315, + "yaw": -1.742 + }, + { + "class_name": "pizza", + "confidence": 0.62, + "ts": 62.87, + "x": -2.29, + "y": -3.841, + "z": 0.315, + "yaw": -1.742 + }, + { + "class_name": "cup", + "confidence": 0.791, + "ts": 63.22, + "x": -2.326, + "y": -4.033, + "z": 0.309, + "yaw": -1.727 + }, + { + "class_name": "pizza", + "confidence": 0.66, + "ts": 63.22, + "x": -2.326, + "y": -4.033, + "z": 0.309, + "yaw": -1.727 + }, + { + "class_name": "cup", + "confidence": 0.885, + "ts": 63.57, + "x": -2.357, + "y": -4.257, + "z": 0.306, + "yaw": -1.665 + }, + { + "class_name": "refrigerator", + "confidence": 0.624, + "ts": 63.57, + "x": -2.357, + "y": -4.257, + "z": 0.306, + "yaw": -1.665 + }, + { + "class_name": "refrigerator", + "confidence": 0.461, + "ts": 64.62, + "x": -2.23, + "y": -4.666, + "z": 0.303, + "yaw": -1.071 + }, + { + "class_name": "tv", + "confidence": 0.411, + "ts": 64.97, + "x": -2.146, + "y": -4.775, + "z": 0.309, + "yaw": -0.773 + }, + { + "class_name": "chair", + "confidence": 0.44, + "ts": 66.38, + "x": -1.823, + "y": -4.947, + "z": 0.303, + "yaw": -0.258 + }, + { + "class_name": "chair", + "confidence": 0.458, + "ts": 66.73, + "x": -1.697, + "y": -4.97, + "z": 0.302, + "yaw": -0.154 + }, + { + "class_name": "chair", + "confidence": 0.518, + "ts": 74.11, + "x": -2.653, + "y": -4.675, + "z": 0.309, + "yaw": -0.548 + }, + { + "class_name": "sandwich", + "confidence": 0.459, + "ts": 76.22, + "x": -2.647, + "y": -4.261, + "z": 0.305, + "yaw": 3.09 + }, + { + "class_name": "bottle", + "confidence": 0.647, + "ts": 79.39, + "x": -4.021, + "y": -3.786, + "z": 0.317, + "yaw": 1.668 + }, + { + "class_name": "bottle", + "confidence": 0.623, + "ts": 79.39, + "x": -4.021, + "y": -3.786, + "z": 0.317, + "yaw": 1.668 + }, + { + "class_name": "bottle", + "confidence": 0.607, + "ts": 79.39, + "x": -4.021, + "y": -3.786, + "z": 0.317, + "yaw": 1.668 + }, + { + "class_name": "bottle", + "confidence": 0.602, + "ts": 79.39, + "x": -4.021, + "y": -3.786, + "z": 0.317, + "yaw": 1.668 + }, + { + "class_name": "bottle", + "confidence": 0.602, + "ts": 79.39, + "x": -4.021, + "y": -3.786, + "z": 0.317, + "yaw": 1.668 + }, + { + "class_name": "bottle", + "confidence": 0.572, + "ts": 79.39, + "x": -4.021, + "y": -3.786, + "z": 0.317, + "yaw": 1.668 + }, + { + "class_name": "bottle", + "confidence": 0.563, + "ts": 79.39, + "x": -4.021, + "y": -3.786, + "z": 0.317, + "yaw": 1.668 + }, + { + "class_name": "bottle", + "confidence": 0.541, + "ts": 79.39, + "x": -4.021, + "y": -3.786, + "z": 0.317, + "yaw": 1.668 + }, + { + "class_name": "bottle", + "confidence": 0.489, + "ts": 79.39, + "x": -4.021, + "y": -3.786, + "z": 0.317, + "yaw": 1.668 + }, + { + "class_name": "bottle", + "confidence": 0.418, + "ts": 79.39, + "x": -4.021, + "y": -3.786, + "z": 0.317, + "yaw": 1.668 + }, + { + "class_name": "bottle", + "confidence": 0.406, + "ts": 79.39, + "x": -4.021, + "y": -3.786, + "z": 0.317, + "yaw": 1.668 + }, + { + "class_name": "bottle", + "confidence": 0.754, + "ts": 80.08, + "x": -4.06, + "y": -3.432, + "z": 0.311, + "yaw": 1.653 + }, + { + "class_name": "bottle", + "confidence": 0.704, + "ts": 80.08, + "x": -4.06, + "y": -3.432, + "z": 0.311, + "yaw": 1.653 + }, + { + "class_name": "bottle", + "confidence": 0.679, + "ts": 80.08, + "x": -4.06, + "y": -3.432, + "z": 0.311, + "yaw": 1.653 + }, + { + "class_name": "bottle", + "confidence": 0.678, + "ts": 80.08, + "x": -4.06, + "y": -3.432, + "z": 0.311, + "yaw": 1.653 + }, + { + "class_name": "bottle", + "confidence": 0.658, + "ts": 80.08, + "x": -4.06, + "y": -3.432, + "z": 0.311, + "yaw": 1.653 + }, + { + "class_name": "bottle", + "confidence": 0.656, + "ts": 80.08, + "x": -4.06, + "y": -3.432, + "z": 0.311, + "yaw": 1.653 + }, + { + "class_name": "bottle", + "confidence": 0.588, + "ts": 80.08, + "x": -4.06, + "y": -3.432, + "z": 0.311, + "yaw": 1.653 + }, + { + "class_name": "bottle", + "confidence": 0.549, + "ts": 80.08, + "x": -4.06, + "y": -3.432, + "z": 0.311, + "yaw": 1.653 + }, + { + "class_name": "pizza", + "confidence": 0.849, + "ts": 80.44, + "x": -4.077, + "y": -3.24, + "z": 0.312, + "yaw": 1.574 + }, + { + "class_name": "bottle", + "confidence": 0.728, + "ts": 80.78, + "x": -4.076, + "y": -3.076, + "z": 0.31, + "yaw": 1.563 + }, + { + "class_name": "bottle", + "confidence": 0.695, + "ts": 80.78, + "x": -4.076, + "y": -3.076, + "z": 0.31, + "yaw": 1.563 + }, + { + "class_name": "bottle", + "confidence": 0.687, + "ts": 80.78, + "x": -4.076, + "y": -3.076, + "z": 0.31, + "yaw": 1.563 + }, + { + "class_name": "bottle", + "confidence": 0.684, + "ts": 80.78, + "x": -4.076, + "y": -3.076, + "z": 0.31, + "yaw": 1.563 + }, + { + "class_name": "bottle", + "confidence": 0.681, + "ts": 80.78, + "x": -4.076, + "y": -3.076, + "z": 0.31, + "yaw": 1.563 + }, + { + "class_name": "bottle", + "confidence": 0.523, + "ts": 80.78, + "x": -4.076, + "y": -3.076, + "z": 0.31, + "yaw": 1.563 + }, + { + "class_name": "bottle", + "confidence": 0.575, + "ts": 81.48, + "x": -3.987, + "y": -2.767, + "z": 0.308, + "yaw": 1.199 + }, + { + "class_name": "bottle", + "confidence": 0.573, + "ts": 81.48, + "x": -3.987, + "y": -2.767, + "z": 0.308, + "yaw": 1.199 + }, + { + "class_name": "bottle", + "confidence": 0.537, + "ts": 81.48, + "x": -3.987, + "y": -2.767, + "z": 0.308, + "yaw": 1.199 + }, + { + "class_name": "bottle", + "confidence": 0.463, + "ts": 81.48, + "x": -3.987, + "y": -2.767, + "z": 0.308, + "yaw": 1.199 + }, + { + "class_name": "bottle", + "confidence": 0.609, + "ts": 82.18, + "x": -3.844, + "y": -2.371, + "z": 0.315, + "yaw": 1.219 + }, + { + "class_name": "bottle", + "confidence": 0.542, + "ts": 82.18, + "x": -3.844, + "y": -2.371, + "z": 0.315, + "yaw": 1.219 + }, + { + "class_name": "refrigerator", + "confidence": 0.527, + "ts": 82.18, + "x": -3.844, + "y": -2.371, + "z": 0.315, + "yaw": 1.219 + }, + { + "class_name": "bottle", + "confidence": 0.499, + "ts": 82.18, + "x": -3.844, + "y": -2.371, + "z": 0.315, + "yaw": 1.219 + }, + { + "class_name": "bottle", + "confidence": 0.462, + "ts": 82.18, + "x": -3.844, + "y": -2.371, + "z": 0.315, + "yaw": 1.219 + }, + { + "class_name": "bottle", + "confidence": 0.435, + "ts": 82.18, + "x": -3.844, + "y": -2.371, + "z": 0.315, + "yaw": 1.219 + }, + { + "class_name": "bottle", + "confidence": 0.418, + "ts": 82.18, + "x": -3.844, + "y": -2.371, + "z": 0.315, + "yaw": 1.219 + }, + { + "class_name": "bottle", + "confidence": 0.407, + "ts": 82.18, + "x": -3.844, + "y": -2.371, + "z": 0.315, + "yaw": 1.219 + }, + { + "class_name": "bottle", + "confidence": 0.679, + "ts": 83.23, + "x": -3.633, + "y": -1.78, + "z": 0.31, + "yaw": 1.225 + }, + { + "class_name": "bottle", + "confidence": 0.667, + "ts": 83.23, + "x": -3.633, + "y": -1.78, + "z": 0.31, + "yaw": 1.225 + }, + { + "class_name": "bottle", + "confidence": 0.666, + "ts": 83.23, + "x": -3.633, + "y": -1.78, + "z": 0.31, + "yaw": 1.225 + }, + { + "class_name": "bottle", + "confidence": 0.634, + "ts": 83.23, + "x": -3.633, + "y": -1.78, + "z": 0.31, + "yaw": 1.225 + }, + { + "class_name": "bottle", + "confidence": 0.581, + "ts": 83.23, + "x": -3.633, + "y": -1.78, + "z": 0.31, + "yaw": 1.225 + }, + { + "class_name": "bottle", + "confidence": 0.556, + "ts": 83.23, + "x": -3.633, + "y": -1.78, + "z": 0.31, + "yaw": 1.225 + }, + { + "class_name": "bottle", + "confidence": 0.466, + "ts": 83.23, + "x": -3.633, + "y": -1.78, + "z": 0.31, + "yaw": 1.225 + }, + { + "class_name": "tv", + "confidence": 0.44, + "ts": 83.23, + "x": -3.633, + "y": -1.78, + "z": 0.31, + "yaw": 1.225 + }, + { + "class_name": "bottle", + "confidence": 0.421, + "ts": 83.94, + "x": -3.501, + "y": -1.401, + "z": 0.316, + "yaw": 1.24 + }, + { + "class_name": "suitcase", + "confidence": 0.632, + "ts": 85.0, + "x": -3.279, + "y": -0.878, + "z": 0.306, + "yaw": 1.171 + }, + { + "class_name": "tv", + "confidence": 0.484, + "ts": 85.0, + "x": -3.279, + "y": -0.878, + "z": 0.306, + "yaw": 1.171 + }, + { + "class_name": "bottle", + "confidence": 0.517, + "ts": 88.51, + "x": -3.387, + "y": -1.109, + "z": 0.309, + "yaw": -1.89 + }, + { + "class_name": "bowl", + "confidence": 0.46, + "ts": 88.51, + "x": -3.387, + "y": -1.109, + "z": 0.309, + "yaw": -1.89 + }, + { + "class_name": "bottle", + "confidence": 0.432, + "ts": 88.51, + "x": -3.387, + "y": -1.109, + "z": 0.309, + "yaw": -1.89 + }, + { + "class_name": "bottle", + "confidence": 0.558, + "ts": 88.86, + "x": -3.45, + "y": -1.318, + "z": 0.304, + "yaw": -1.872 + }, + { + "class_name": "bottle", + "confidence": 0.52, + "ts": 88.86, + "x": -3.45, + "y": -1.318, + "z": 0.304, + "yaw": -1.872 + }, + { + "class_name": "book", + "confidence": 0.516, + "ts": 88.86, + "x": -3.45, + "y": -1.318, + "z": 0.304, + "yaw": -1.872 + }, + { + "class_name": "bottle", + "confidence": 0.43, + "ts": 88.86, + "x": -3.45, + "y": -1.318, + "z": 0.304, + "yaw": -1.872 + }, + { + "class_name": "book", + "confidence": 0.571, + "ts": 89.21, + "x": -3.508, + "y": -1.496, + "z": 0.311, + "yaw": -1.874 + }, + { + "class_name": "book", + "confidence": 0.465, + "ts": 89.21, + "x": -3.508, + "y": -1.496, + "z": 0.311, + "yaw": -1.874 + }, + { + "class_name": "book", + "confidence": 0.44, + "ts": 89.21, + "x": -3.508, + "y": -1.496, + "z": 0.311, + "yaw": -1.874 + }, + { + "class_name": "book", + "confidence": 0.44, + "ts": 89.21, + "x": -3.508, + "y": -1.496, + "z": 0.311, + "yaw": -1.874 + }, + { + "class_name": "bottle", + "confidence": 0.402, + "ts": 89.21, + "x": -3.508, + "y": -1.496, + "z": 0.311, + "yaw": -1.874 + }, + { + "class_name": "book", + "confidence": 0.636, + "ts": 89.91, + "x": -3.62, + "y": -1.919, + "z": 0.313, + "yaw": -1.849 + }, + { + "class_name": "book", + "confidence": 0.567, + "ts": 89.91, + "x": -3.62, + "y": -1.919, + "z": 0.313, + "yaw": -1.849 + }, + { + "class_name": "book", + "confidence": 0.524, + "ts": 89.91, + "x": -3.62, + "y": -1.919, + "z": 0.313, + "yaw": -1.849 + }, + { + "class_name": "book", + "confidence": 0.498, + "ts": 89.91, + "x": -3.62, + "y": -1.919, + "z": 0.313, + "yaw": -1.849 + }, + { + "class_name": "book", + "confidence": 0.497, + "ts": 89.91, + "x": -3.62, + "y": -1.919, + "z": 0.313, + "yaw": -1.849 + }, + { + "class_name": "book", + "confidence": 0.49, + "ts": 89.91, + "x": -3.62, + "y": -1.919, + "z": 0.313, + "yaw": -1.849 + }, + { + "class_name": "bottle", + "confidence": 0.484, + "ts": 89.91, + "x": -3.62, + "y": -1.919, + "z": 0.313, + "yaw": -1.849 + }, + { + "class_name": "book", + "confidence": 0.455, + "ts": 89.91, + "x": -3.62, + "y": -1.919, + "z": 0.313, + "yaw": -1.849 + }, + { + "class_name": "bottle", + "confidence": 0.45, + "ts": 89.91, + "x": -3.62, + "y": -1.919, + "z": 0.313, + "yaw": -1.849 + }, + { + "class_name": "book", + "confidence": 0.446, + "ts": 89.91, + "x": -3.62, + "y": -1.919, + "z": 0.313, + "yaw": -1.849 + }, + { + "class_name": "book", + "confidence": 0.566, + "ts": 90.27, + "x": -3.685, + "y": -2.15, + "z": 0.31, + "yaw": -1.84 + }, + { + "class_name": "book", + "confidence": 0.526, + "ts": 90.27, + "x": -3.685, + "y": -2.15, + "z": 0.31, + "yaw": -1.84 + }, + { + "class_name": "book", + "confidence": 0.467, + "ts": 90.27, + "x": -3.685, + "y": -2.15, + "z": 0.31, + "yaw": -1.84 + }, + { + "class_name": "book", + "confidence": 0.459, + "ts": 90.27, + "x": -3.685, + "y": -2.15, + "z": 0.31, + "yaw": -1.84 + }, + { + "class_name": "book", + "confidence": 0.426, + "ts": 90.27, + "x": -3.685, + "y": -2.15, + "z": 0.31, + "yaw": -1.84 + }, + { + "class_name": "book", + "confidence": 0.658, + "ts": 90.62, + "x": -3.744, + "y": -2.377, + "z": 0.312, + "yaw": -1.822 + }, + { + "class_name": "book", + "confidence": 0.653, + "ts": 90.62, + "x": -3.744, + "y": -2.377, + "z": 0.312, + "yaw": -1.822 + }, + { + "class_name": "person", + "confidence": 0.442, + "ts": 90.62, + "x": -3.744, + "y": -2.377, + "z": 0.312, + "yaw": -1.822 + }, + { + "class_name": "book", + "confidence": 0.435, + "ts": 90.62, + "x": -3.744, + "y": -2.377, + "z": 0.312, + "yaw": -1.822 + }, + { + "class_name": "book", + "confidence": 0.429, + "ts": 90.62, + "x": -3.744, + "y": -2.377, + "z": 0.312, + "yaw": -1.822 + }, + { + "class_name": "book", + "confidence": 0.417, + "ts": 90.62, + "x": -3.744, + "y": -2.377, + "z": 0.312, + "yaw": -1.822 + }, + { + "class_name": "person", + "confidence": 0.878, + "ts": 90.97, + "x": -3.789, + "y": -2.577, + "z": 0.314, + "yaw": -1.819 + }, + { + "class_name": "book", + "confidence": 0.553, + "ts": 91.34, + "x": -3.848, + "y": -2.817, + "z": 0.309, + "yaw": -1.803 + }, + { + "class_name": "person", + "confidence": 0.534, + "ts": 91.34, + "x": -3.848, + "y": -2.817, + "z": 0.309, + "yaw": -1.803 + }, + { + "class_name": "person", + "confidence": 0.889, + "ts": 91.68, + "x": -3.893, + "y": -3.019, + "z": 0.305, + "yaw": -1.793 + }, + { + "class_name": "book", + "confidence": 0.425, + "ts": 91.68, + "x": -3.893, + "y": -3.019, + "z": 0.305, + "yaw": -1.793 + }, + { + "class_name": "person", + "confidence": 0.618, + "ts": 92.02, + "x": -3.946, + "y": -3.236, + "z": 0.316, + "yaw": -1.786 + }, + { + "class_name": "book", + "confidence": 0.582, + "ts": 92.37, + "x": -3.992, + "y": -3.437, + "z": 0.312, + "yaw": -1.767 + }, + { + "class_name": "book", + "confidence": 0.418, + "ts": 92.37, + "x": -3.992, + "y": -3.437, + "z": 0.312, + "yaw": -1.767 + }, + { + "class_name": "tv", + "confidence": 0.429, + "ts": 103.25, + "x": -7.594, + "y": -3.512, + "z": 0.302, + "yaw": 1.641 + }, + { + "class_name": "tv", + "confidence": 0.533, + "ts": 103.61, + "x": -7.608, + "y": -3.406, + "z": 0.307, + "yaw": 1.598 + }, + { + "class_name": "tv", + "confidence": 0.593, + "ts": 104.31, + "x": -7.621, + "y": -3.266, + "z": 0.299, + "yaw": 1.508 + }, + { + "class_name": "tv", + "confidence": 0.442, + "ts": 105.01, + "x": -7.613, + "y": -3.035, + "z": 0.31, + "yaw": 1.419 + }, + { + "class_name": "tv", + "confidence": 0.42, + "ts": 108.17, + "x": -6.769, + "y": -0.446, + "z": 0.309, + "yaw": 1.269 + }, + { + "class_name": "bottle", + "confidence": 0.438, + "ts": 128.2, + "x": -6.03, + "y": -7.746, + "z": 0.314, + "yaw": 0.096 + }, + { + "class_name": "toothbrush", + "confidence": 0.494, + "ts": 133.82, + "x": -5.222, + "y": -7.444, + "z": 0.313, + "yaw": 0.889 + }, + { + "class_name": "kite", + "confidence": 0.536, + "ts": 144.01, + "x": -5.842, + "y": -10.475, + "z": 0.318, + "yaw": -0.344 + }, + { + "class_name": "frisbee", + "confidence": 0.433, + "ts": 159.13, + "x": -6.603, + "y": -7.972, + "z": 0.311, + "yaw": 0.687 + }, + { + "class_name": "chair", + "confidence": 0.462, + "ts": 163.34, + "x": -5.295, + "y": -6.458, + "z": 0.311, + "yaw": 1.513 + }, + { + "class_name": "car", + "confidence": 0.431, + "ts": 164.39, + "x": -5.175, + "y": -5.686, + "z": 0.314, + "yaw": 1.319 + }, + { + "class_name": "bus", + "confidence": 0.617, + "ts": 167.2, + "x": -4.539, + "y": -4.108, + "z": 0.314, + "yaw": 0.401 + }, + { + "class_name": "cup", + "confidence": 0.618, + "ts": 170.01, + "x": -3.125, + "y": -4.119, + "z": 0.309, + "yaw": -0.129 + }, + { + "class_name": "train", + "confidence": 0.506, + "ts": 171.42, + "x": -2.581, + "y": -3.82, + "z": 0.319, + "yaw": 0.958 + }, + { + "class_name": "bottle", + "confidence": 0.654, + "ts": 173.18, + "x": -2.241, + "y": -2.896, + "z": 0.315, + "yaw": 1.278 + }, + { + "class_name": "bottle", + "confidence": 0.449, + "ts": 173.18, + "x": -2.241, + "y": -2.896, + "z": 0.315, + "yaw": 1.278 + }, + { + "class_name": "bottle", + "confidence": 0.444, + "ts": 173.18, + "x": -2.241, + "y": -2.896, + "z": 0.315, + "yaw": 1.278 + }, + { + "class_name": "bottle", + "confidence": 0.433, + "ts": 173.18, + "x": -2.241, + "y": -2.896, + "z": 0.315, + "yaw": 1.278 + }, + { + "class_name": "cup", + "confidence": 0.487, + "ts": 176.69, + "x": -1.665, + "y": -0.548, + "z": 0.3, + "yaw": 0.518 + }, + { + "class_name": "cup", + "confidence": 0.496, + "ts": 177.03, + "x": -1.524, + "y": -0.485, + "z": 0.311, + "yaw": 0.513 + }, + { + "class_name": "vase", + "confidence": 0.48, + "ts": 177.03, + "x": -1.524, + "y": -0.485, + "z": 0.311, + "yaw": 0.513 + }, + { + "class_name": "cup", + "confidence": 0.47, + "ts": 177.38, + "x": -1.394, + "y": -0.415, + "z": 0.306, + "yaw": 0.335 + }, + { + "class_name": "person", + "confidence": 0.906, + "ts": 186.17, + "x": 0.919, + "y": -1.444, + "z": 0.307, + "yaw": 2.788 + }, + { + "class_name": "person", + "confidence": 0.405, + "ts": 186.52, + "x": 0.789, + "y": -1.4, + "z": 0.304, + "yaw": 2.67 + }, + { + "class_name": "person", + "confidence": 0.768, + "ts": 186.88, + "x": 0.637, + "y": -1.315, + "z": 0.303, + "yaw": 2.666 + }, + { + "class_name": "person", + "confidence": 0.778, + "ts": 187.22, + "x": 0.454, + "y": -1.222, + "z": 0.307, + "yaw": 2.632 + }, + { + "class_name": "person", + "confidence": 0.401, + "ts": 187.93, + "x": 0.167, + "y": -1.046, + "z": 0.307, + "yaw": 2.551 + }, + { + "class_name": "person", + "confidence": 0.67, + "ts": 188.28, + "x": 0.021, + "y": -0.947, + "z": 0.314, + "yaw": 2.549 + }, + { + "class_name": "chair", + "confidence": 0.465, + "ts": 189.34, + "x": -0.733, + "y": -0.464, + "z": 0.311, + "yaw": 2.577 + }, + { + "class_name": "chair", + "confidence": 0.444, + "ts": 189.34, + "x": -0.733, + "y": -0.464, + "z": 0.311, + "yaw": 2.577 + }, + { + "class_name": "chair", + "confidence": 0.4, + "ts": 189.34, + "x": -0.733, + "y": -0.464, + "z": 0.311, + "yaw": 2.577 + }, + { + "class_name": "refrigerator", + "confidence": 0.498, + "ts": 190.04, + "x": -1.076, + "y": -0.228, + "z": 0.312, + "yaw": 2.395 + }, + { + "class_name": "refrigerator", + "confidence": 0.488, + "ts": 190.74, + "x": -1.281, + "y": 0.001, + "z": 0.306, + "yaw": 2.217 + }, + { + "class_name": "chair", + "confidence": 0.401, + "ts": 194.25, + "x": -1.164, + "y": 1.987, + "z": 0.31, + "yaw": 1.207 + }, + { + "class_name": "chair", + "confidence": 0.452, + "ts": 194.95, + "x": -0.911, + "y": 2.617, + "z": 0.305, + "yaw": 1.239 + }, + { + "class_name": "chair", + "confidence": 0.66, + "ts": 195.31, + "x": -0.851, + "y": 2.723, + "z": 0.311, + "yaw": 1.257 + }, + { + "class_name": "chair", + "confidence": 0.747, + "ts": 195.66, + "x": -0.853, + "y": 2.751, + "z": 0.297, + "yaw": 1.25 + }, + { + "class_name": "person", + "confidence": 0.451, + "ts": 196.36, + "x": -0.883, + "y": 2.772, + "z": 0.309, + "yaw": 1.717 + }, + { + "class_name": "chair", + "confidence": 0.406, + "ts": 196.36, + "x": -0.883, + "y": 2.772, + "z": 0.309, + "yaw": 1.717 + }, + { + "class_name": "chair", + "confidence": 0.762, + "ts": 197.07, + "x": -0.968, + "y": 3.043, + "z": 0.303, + "yaw": 2.067 + }, + { + "class_name": "chair", + "confidence": 0.523, + "ts": 197.07, + "x": -0.968, + "y": 3.043, + "z": 0.303, + "yaw": 2.067 + }, + { + "class_name": "chair", + "confidence": 0.705, + "ts": 197.42, + "x": -1.083, + "y": 3.251, + "z": 0.315, + "yaw": 2.106 + }, + { + "class_name": "chair", + "confidence": 0.595, + "ts": 197.42, + "x": -1.083, + "y": 3.251, + "z": 0.315, + "yaw": 2.106 + }, + { + "class_name": "chair", + "confidence": 0.499, + "ts": 197.42, + "x": -1.083, + "y": 3.251, + "z": 0.315, + "yaw": 2.106 + }, + { + "class_name": "chair", + "confidence": 0.823, + "ts": 197.77, + "x": -1.246, + "y": 3.524, + "z": 0.31, + "yaw": 2.123 + }, + { + "class_name": "chair", + "confidence": 0.676, + "ts": 197.77, + "x": -1.246, + "y": 3.524, + "z": 0.31, + "yaw": 2.123 + }, + { + "class_name": "chair", + "confidence": 0.674, + "ts": 197.77, + "x": -1.246, + "y": 3.524, + "z": 0.31, + "yaw": 2.123 + }, + { + "class_name": "train", + "confidence": 0.468, + "ts": 198.11, + "x": -1.393, + "y": 3.739, + "z": 0.314, + "yaw": 2.499 + }, + { + "class_name": "chair", + "confidence": 0.77, + "ts": 198.82, + "x": -1.835, + "y": 4.024, + "z": 0.316, + "yaw": 2.571 + }, + { + "class_name": "chair", + "confidence": 0.698, + "ts": 198.82, + "x": -1.835, + "y": 4.024, + "z": 0.316, + "yaw": 2.571 + }, + { + "class_name": "chair", + "confidence": 0.454, + "ts": 198.82, + "x": -1.835, + "y": 4.024, + "z": 0.316, + "yaw": 2.571 + }, + { + "class_name": "chair", + "confidence": 0.598, + "ts": 199.87, + "x": -2.359, + "y": 4.578, + "z": 0.31, + "yaw": 2.074 + }, + { + "class_name": "chair", + "confidence": 0.598, + "ts": 199.87, + "x": -2.359, + "y": 4.578, + "z": 0.31, + "yaw": 2.074 + }, + { + "class_name": "chair", + "confidence": 0.403, + "ts": 199.87, + "x": -2.359, + "y": 4.578, + "z": 0.31, + "yaw": 2.074 + }, + { + "class_name": "chair", + "confidence": 0.532, + "ts": 200.57, + "x": -2.478, + "y": 4.967, + "z": 0.311, + "yaw": 1.736 + }, + { + "class_name": "chair", + "confidence": 0.779, + "ts": 200.93, + "x": -2.531, + "y": 5.233, + "z": 0.314, + "yaw": 1.768 + }, + { + "class_name": "chair", + "confidence": 0.774, + "ts": 200.93, + "x": -2.531, + "y": 5.233, + "z": 0.314, + "yaw": 1.768 + }, + { + "class_name": "person", + "confidence": 0.55, + "ts": 200.93, + "x": -2.531, + "y": 5.233, + "z": 0.314, + "yaw": 1.768 + }, + { + "class_name": "chair", + "confidence": 0.784, + "ts": 201.63, + "x": -2.552, + "y": 5.73, + "z": 0.321, + "yaw": 1.34 + }, + { + "class_name": "teddy bear", + "confidence": 0.466, + "ts": 201.63, + "x": -2.552, + "y": 5.73, + "z": 0.321, + "yaw": 1.34 + }, + { + "class_name": "chair", + "confidence": 0.916, + "ts": 201.99, + "x": -2.474, + "y": 5.967, + "z": 0.314, + "yaw": 1.291 + }, + { + "class_name": "chair", + "confidence": 0.855, + "ts": 201.99, + "x": -2.474, + "y": 5.967, + "z": 0.314, + "yaw": 1.291 + }, + { + "class_name": "person", + "confidence": 0.462, + "ts": 201.99, + "x": -2.474, + "y": 5.967, + "z": 0.314, + "yaw": 1.291 + }, + { + "class_name": "person", + "confidence": 0.438, + "ts": 201.99, + "x": -2.474, + "y": 5.967, + "z": 0.314, + "yaw": 1.291 + }, + { + "class_name": "chair", + "confidence": 0.404, + "ts": 202.33, + "x": -2.401, + "y": 6.299, + "z": 0.315, + "yaw": 1.391 + }, + { + "class_name": "teddy bear", + "confidence": 0.795, + "ts": 202.68, + "x": -2.383, + "y": 6.519, + "z": 0.315, + "yaw": 1.539 + }, + { + "class_name": "teddy bear", + "confidence": 0.547, + "ts": 202.68, + "x": -2.383, + "y": 6.519, + "z": 0.315, + "yaw": 1.539 + }, + { + "class_name": "potted plant", + "confidence": 0.406, + "ts": 202.68, + "x": -2.383, + "y": 6.519, + "z": 0.315, + "yaw": 1.539 + }, + { + "class_name": "teddy bear", + "confidence": 0.89, + "ts": 203.38, + "x": -2.275, + "y": 6.905, + "z": 0.315, + "yaw": 1.006 + }, + { + "class_name": "teddy bear", + "confidence": 0.576, + "ts": 203.38, + "x": -2.275, + "y": 6.905, + "z": 0.315, + "yaw": 1.006 + }, + { + "class_name": "teddy bear", + "confidence": 0.873, + "ts": 203.74, + "x": -2.191, + "y": 7.039, + "z": 0.309, + "yaw": 0.883 + }, + { + "class_name": "teddy bear", + "confidence": 0.479, + "ts": 203.74, + "x": -2.191, + "y": 7.039, + "z": 0.309, + "yaw": 0.883 + }, + { + "class_name": "teddy bear", + "confidence": 0.849, + "ts": 204.44, + "x": -1.967, + "y": 7.231, + "z": 0.303, + "yaw": 0.633 + }, + { + "class_name": "teddy bear", + "confidence": 0.609, + "ts": 204.44, + "x": -1.967, + "y": 7.231, + "z": 0.303, + "yaw": 0.633 + }, + { + "class_name": "person", + "confidence": 0.476, + "ts": 205.14, + "x": -1.682, + "y": 7.358, + "z": 0.308, + "yaw": 0.276 + }, + { + "class_name": "person", + "confidence": 0.718, + "ts": 220.6, + "x": 3.058, + "y": 16.821, + "z": 0.308, + "yaw": -2.029 + }, + { + "class_name": "person", + "confidence": 0.695, + "ts": 220.95, + "x": 2.961, + "y": 16.623, + "z": 0.315, + "yaw": -2.03 + }, + { + "class_name": "person", + "confidence": 0.57, + "ts": 221.65, + "x": 2.819, + "y": 16.158, + "z": 0.324, + "yaw": -1.776 + }, + { + "class_name": "person", + "confidence": 0.739, + "ts": 222.01, + "x": 2.734, + "y": 15.9, + "z": 0.316, + "yaw": -1.893 + }, + { + "class_name": "person", + "confidence": 0.731, + "ts": 222.36, + "x": 2.644, + "y": 15.61, + "z": 0.313, + "yaw": -1.884 + }, + { + "class_name": "person", + "confidence": 0.612, + "ts": 222.7, + "x": 2.532, + "y": 15.263, + "z": 0.298, + "yaw": -1.879 + }, + { + "class_name": "person", + "confidence": 0.728, + "ts": 223.06, + "x": 2.444, + "y": 14.993, + "z": 0.315, + "yaw": -1.872 + }, + { + "class_name": "person", + "confidence": 0.744, + "ts": 223.41, + "x": 2.332, + "y": 14.597, + "z": 0.317, + "yaw": -1.853 + }, + { + "class_name": "person", + "confidence": 0.763, + "ts": 223.76, + "x": 2.251, + "y": 14.315, + "z": 0.31, + "yaw": -1.848 + }, + { + "class_name": "person", + "confidence": 0.703, + "ts": 224.11, + "x": 2.154, + "y": 14.007, + "z": 0.304, + "yaw": -1.88 + }, + { + "class_name": "person", + "confidence": 0.758, + "ts": 224.47, + "x": 2.071, + "y": 13.76, + "z": 0.311, + "yaw": -1.89 + }, + { + "class_name": "person", + "confidence": 0.755, + "ts": 224.82, + "x": 1.956, + "y": 13.403, + "z": 0.307, + "yaw": -1.868 + }, + { + "class_name": "person", + "confidence": 0.732, + "ts": 225.17, + "x": 1.866, + "y": 13.187, + "z": 0.308, + "yaw": -1.954 + }, + { + "class_name": "person", + "confidence": 0.724, + "ts": 225.52, + "x": 1.758, + "y": 12.897, + "z": 0.308, + "yaw": -1.934 + }, + { + "class_name": "person", + "confidence": 0.773, + "ts": 225.87, + "x": 1.628, + "y": 12.553, + "z": 0.31, + "yaw": -1.93 + }, + { + "class_name": "person", + "confidence": 0.82, + "ts": 226.22, + "x": 1.506, + "y": 12.214, + "z": 0.315, + "yaw": -1.932 + }, + { + "class_name": "person", + "confidence": 0.498, + "ts": 226.57, + "x": 1.372, + "y": 11.93, + "z": 0.308, + "yaw": -1.665 + }, + { + "class_name": "airplane", + "confidence": 0.802, + "ts": 232.54, + "x": 2.942, + "y": 11.322, + "z": 0.312, + "yaw": 0.372 + }, + { + "class_name": "person", + "confidence": 0.545, + "ts": 238.17, + "x": 2.597, + "y": 11.49, + "z": 0.313, + "yaw": -3.093 + }, + { + "class_name": "train", + "confidence": 0.442, + "ts": 238.17, + "x": 2.597, + "y": 11.49, + "z": 0.313, + "yaw": -3.093 + }, + { + "class_name": "person", + "confidence": 0.627, + "ts": 238.88, + "x": 2.177, + "y": 11.38, + "z": 0.313, + "yaw": -2.647 + }, + { + "class_name": "potted plant", + "confidence": 0.593, + "ts": 241.33, + "x": 0.297, + "y": 10.697, + "z": 0.305, + "yaw": 3.088 + }, + { + "class_name": "potted plant", + "confidence": 0.523, + "ts": 241.68, + "x": 0.091, + "y": 10.715, + "z": 0.311, + "yaw": 3.085 + }, + { + "class_name": "potted plant", + "confidence": 0.594, + "ts": 243.43, + "x": -1.425, + "y": 10.974, + "z": 0.314, + "yaw": 2.991 + }, + { + "class_name": "potted plant", + "confidence": 0.532, + "ts": 243.79, + "x": -1.678, + "y": 11.017, + "z": 0.312, + "yaw": 2.99 + }, + { + "class_name": "potted plant", + "confidence": 0.618, + "ts": 244.14, + "x": -2.041, + "y": 11.074, + "z": 0.312, + "yaw": 3.009 + }, + { + "class_name": "bench", + "confidence": 0.445, + "ts": 245.19, + "x": -2.712, + "y": 11.037, + "z": 0.32, + "yaw": -2.477 + }, + { + "class_name": "train", + "confidence": 0.603, + "ts": 245.9, + "x": -3.21, + "y": 10.668, + "z": 0.314, + "yaw": -2.462 + }, + { + "class_name": "bed", + "confidence": 0.44, + "ts": 250.12, + "x": -6.248, + "y": 10.52, + "z": 0.317, + "yaw": 3.024 + }, + { + "class_name": "bed", + "confidence": 0.609, + "ts": 250.47, + "x": -6.541, + "y": 10.584, + "z": 0.315, + "yaw": 2.912 + }, + { + "class_name": "bed", + "confidence": 0.475, + "ts": 253.28, + "x": -8.562, + "y": 11.484, + "z": 0.309, + "yaw": 2.634 + }, + { + "class_name": "laptop", + "confidence": 0.434, + "ts": 253.28, + "x": -8.562, + "y": 11.484, + "z": 0.309, + "yaw": 2.634 + }, + { + "class_name": "laptop", + "confidence": 0.445, + "ts": 253.62, + "x": -8.868, + "y": 11.647, + "z": 0.307, + "yaw": 2.645 + }, + { + "class_name": "laptop", + "confidence": 0.783, + "ts": 253.97, + "x": -9.159, + "y": 11.818, + "z": 0.302, + "yaw": 2.662 + }, + { + "class_name": "keyboard", + "confidence": 0.499, + "ts": 253.97, + "x": -9.159, + "y": 11.818, + "z": 0.302, + "yaw": 2.662 + }, + { + "class_name": "laptop", + "confidence": 0.453, + "ts": 254.33, + "x": -9.524, + "y": 12.009, + "z": 0.303, + "yaw": 2.677 + }, + { + "class_name": "laptop", + "confidence": 0.778, + "ts": 254.68, + "x": -9.76, + "y": 12.129, + "z": 0.306, + "yaw": 2.692 + }, + { + "class_name": "laptop", + "confidence": 0.484, + "ts": 255.03, + "x": -10.067, + "y": 12.281, + "z": 0.3, + "yaw": 2.71 + }, + { + "class_name": "refrigerator", + "confidence": 0.548, + "ts": 279.98, + "x": -15.282, + "y": 1.695, + "z": 0.305, + "yaw": -1.361 + }, + { + "class_name": "refrigerator", + "confidence": 0.485, + "ts": 285.25, + "x": -13.843, + "y": -0.249, + "z": 0.311, + "yaw": -1.861 + } +] diff --git a/evals_bench/semantic/generate_detections.py b/evals_bench/semantic/generate_detections.py new file mode 100644 index 0000000000..8ea311af68 --- /dev/null +++ b/evals_bench/semantic/generate_detections.py @@ -0,0 +1,94 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate raw semantic detections for go2_bigoffice (run once; frozen data). + +Runs a YOLO detector over every FRAME_STRIDE-th color frame and records, per +detection: class_name, confidence, ts (seconds from first frame), and the +robot odom pose at that ts — world x/y/z plus yaw (radians, from the odom +quaternion). Object world position = robot position at detection time +(odom-grounding convention: ~1-2 m error, benchmark bands account for it). + +Output: detections.json next to this script, plus a class histogram on stdout. +""" + +from __future__ import annotations + +from collections import Counter +import json +from pathlib import Path + +import numpy as np + +from dimos.memory2.cli.dataset import open_dataset + +DATASET = "go2_bigoffice" +MODEL = "yolov8m.pt" +CONF = 0.4 +FRAME_STRIDE = 5 +WEIGHTS_DIR = Path.home() / ".cache" / "ultralytics" # never the repo + + +def load_model(): + from ultralytics import YOLO + from ultralytics.utils.downloads import attempt_download_asset + + WEIGHTS_DIR.mkdir(parents=True, exist_ok=True) + weights = WEIGHTS_DIR / MODEL + if not weights.exists(): + attempt_download_asset(str(weights)) + return YOLO(str(weights)) + + +def main() -> None: + model = load_model() + store = open_dataset(DATASET) + detections: list[dict[str, object]] = [] + try: + odom = store.streams.odom.to_list() + odom_ts = np.array([o.ts for o in odom]) + frames = store.streams.color_image.to_list()[::FRAME_STRIDE] + t0 = frames[0].ts + for k, obs in enumerate(frames): + result = model.predict(obs.data.to_opencv(), conf=CONF, verbose=False)[0] + if len(result.boxes) == 0: + continue + od = odom[int(np.abs(odom_ts - obs.ts).argmin())].data + yaw = float(od.orientation.euler[2]) + for box in result.boxes: + detections.append( + { + "class_name": result.names[int(box.cls)], + "confidence": round(float(box.conf), 3), + "ts": round(obs.ts - t0, 2), + "x": round(float(od.position.x), 3), + "y": round(float(od.position.y), 3), + "z": round(float(od.position.z), 3), + "yaw": round(yaw, 3), + } + ) + if k % 100 == 0: + print(f"frame {k}/{len(frames)}: {len(detections)} detections so far") + finally: + store.stop() + + out = Path(__file__).parent / "detections.json" + out.write_text(json.dumps(detections, indent=1) + "\n") + print(f"\nwrote {len(detections)} detections from {len(frames)} frames -> {out}") + for name, count in Counter(d["class_name"] for d in detections).most_common(): + print(f" {name:20s} {count}") + + +if __name__ == "__main__": + main() diff --git a/evals_bench/semantic/generate_rows.py b/evals_bench/semantic/generate_rows.py new file mode 100644 index 0000000000..da33c867a1 --- /dev/null +++ b/evals_bench/semantic/generate_rows.py @@ -0,0 +1,632 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate semantic-map VQA rows from detections.json (run once; frozen data). + +Ground truth is derived here, independently of the evo target: detections are +clustered per class within MERGE_RADIUS_M, weak classes (=EGO_MARGIN_DEG inside a band) + compass : 8-way compass relation between two objects >=3 m apart (MCQ) + objdist : horizontal distance between two objects >=3 m apart (numeric, + band max(1.0, 25%); VQASynth-inspired; round 2 diversifies + anchors beyond the alphabetically-first class) + robotdist : robot-to-object horizontal distance at time t (numeric, same + band rule; timestamps >=15 s apart; VQASynth-inspired) + recall : list every mapped class, full map + zone-scoped (set-F1) + within : list classes with an instance within R m of an anchor; R + chosen so no cluster sits within 1.5 m of the boundary (set-F1) + nextto : nearest class to an object, runner-up margin >=1.5 m (MCQ) + between : the one class in the corridor between two anchors (MCQ; + exactly one qualifying cluster, corridor otherwise clear) + +Set-answer rows carry a ``vocab`` (mapped + verified-absent classes) for +reply parsing; scored by set-F1 in benchmark.py. No bare yes/no proximity +binaries (blind-gate variance — nextto MCQ covers the relation). + +Context parity (see benchmark.py header): truth always uses the FULL +detection span; time-conditioned rows (nearest/egoside/robotdist) carry an +odom_window that ends exactly at the question timestamp. +""" + +from __future__ import annotations + +from collections import defaultdict +import json +import math +from pathlib import Path + +DATASET = "go2_bigoffice" +MERGE_RADIUS_M = 1.5 +MIN_SIGHTINGS = 3 +COMPASS_MIN_PAIR_DIST = 3.0 # grounding error ~1-2 m; closer pairs are noise +NEAREST_MARGIN_M = 1.0 # nearest class must beat runner-up by this margin +EGO_AHEAD_DEG = 40.0 # |rel bearing| <= this: ahead; >= 180-this: behind +EGO_MARGIN_DEG = 15.0 # picked bearings sit this far inside a band boundary +COMPASS_NAMES = [ + "east", + "northeast", + "north", + "northwest", + "west", + "southwest", + "south", + "southeast", +] +# COCO classes implausible indoors — detector noise on an office run. +NON_INDOOR = { + "car", + "truck", + "bus", + "train", + "airplane", + "boat", + "motorcycle", + "bicycle", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "frisbee", + "skis", + "snowboard", + "kite", + "surfboard", + "skateboard", + "sports ball", + "baseball bat", + "baseball glove", + "tennis racket", +} +# Moving objects break the static-map odom-grounding convention. +NON_STATIC = {"person"} +# Presence probes are chosen so an office prior anti-correlates with truth: +# "yes" cases are classes a blind guesser would not expect mapped in an office, +# "no" cases are office-plausible classes verified absent from raw detections. +PRESENT_SURPRISES = ["bed", "refrigerator", "teddy bear"] +ABSENT_CANDIDATES = ["couch", "sink"] # hardest "no" probes: blind said yes to these +# 4-way forced choice (chance 0.25): the one mapped class is the option an +# office prior ranks LAST; the three distractors are verified-absent classes. +RECORDED_MCQS = [ + ("teddy bear", ["mouse", "teddy bear", "dining table", "cell phone"]), + ("bed", ["clock", "oven", "bed", "scissors"]), +] + + +def cluster(detections: list[dict]) -> list[dict]: + """Greedy same-class clustering within MERGE_RADIUS_M of the running mean.""" + clusters: list[dict] = [] + for det in sorted(detections, key=lambda d: d["ts"]): + home = None + for c in clusters: + if c["class_name"] != det["class_name"]: + continue + cx, cy = c["sx"] / c["n"], c["sy"] / c["n"] + if math.hypot(det["x"] - cx, det["y"] - cy) <= MERGE_RADIUS_M: + home = c + break + if home is None: + clusters.append( + { + "class_name": det["class_name"], + "sx": det["x"], + "sy": det["y"], + "n": 1, + } + ) + else: + home["sx"] += det["x"] + home["sy"] += det["y"] + home["n"] += 1 + for c in clusters: + c["x"], c["y"] = c["sx"] / c["n"], c["sy"] / c["n"] + return clusters + + +def compass_of(dx: float, dy: float) -> str: + return COMPASS_NAMES[round(math.atan2(dy, dx) / (math.pi / 4)) % 8] + + +def robot_pose_at(detections: list[dict], t: float) -> dict: + return min(detections, key=lambda d: abs(d["ts"] - t)) + + +def main() -> None: + here = Path(__file__).parent + detections = json.loads((here / "detections.json").read_text()) + # "no" presence probes must be classes the detector NEVER saw — a faithful + # map of the raw detections must agree the class is absent. + ever_detected = {d["class_name"] for d in detections} + detections = [d for d in detections if d["class_name"] not in NON_INDOOR | NON_STATIC] + + clusters = [c for c in cluster(detections) if c["n"] >= MIN_SIGHTINGS] + by_class: dict[str, list[dict]] = defaultdict(list) + for c in clusters: + by_class[c["class_name"]].append(c) + singles = {name: cs[0] for name, cs in by_class.items() if len(cs) == 1} + print(f"{len(clusters)} clusters across {len(by_class)} classes") + for name, cs in sorted(by_class.items(), key=lambda kv: -len(kv[1])): + print(f" {name:15s} {len(cs)} objects, sightings {[c['n'] for c in cs]}") + + # Zones: quadrants of the odom-track bounding box, +x east / +y north. + xs = [d["x"] for d in detections] + ys = [d["y"] for d in detections] + mx, my = round((min(xs) + max(xs)) / 2, 1), round((min(ys) + max(ys)) / 2, 1) + zones = { + "northwest area": lambda x, y: x < mx and y >= my, + "northeast area": lambda x, y: x >= mx and y >= my, + "southwest area": lambda x, y: x < mx and y < my, + "southeast area": lambda x, y: x >= mx and y < my, + } + zone_clause = ( + ( + f"The map is divided into four zones at the point (x={mx}, y={my}), " + "with +x east and +y north: the northwest area (x<{mx}, y>={my}), northeast " + "area (x>={mx}, y>={my}), southwest area (x<{mx}, y<{my}), and southeast area " + "(x>={mx}, y<{my})." + ) + .replace("{mx}", str(mx)) + .replace("{my}", str(my)) + ) + + rows: list[dict] = [] + + def add(row_id: str, family: str, q: str, a, *, ctx: str = "objects", **extra) -> None: + if "vocab" in extra: + kind = "set" + elif isinstance(a, (int, float)) and "choices" not in extra: + kind = "numeric" + else: + kind = "mcq" + rows.append( + {"id": row_id, "family": family, "type": kind, "q": q, "a": a, "ctx": ctx} + | extra + | {"dataset": DATASET} + ) + + # -- presence: priors anti-correlated with truth so blind guessing loses + present = [n for n in PRESENT_SURPRISES if n in by_class] + absent = [n for n in ABSENT_CANDIDATES if n not in ever_detected] + for name in present + absent: + truth = "yes" if name in by_class else "no" + add( + f"sm_presence_{name.replace(' ', '_')}", + "presence", + f"Did the robot's semantic object map record at least one " + f"{name} anywhere in the mapped area? Answer with exactly one word: yes or no.", + truth, + choices=["yes", "no"], + ) + for truth, options in RECORDED_MCQS: + assert truth in by_class + assert all(o not in ever_detected for o in options if o != truth) + add( + f"sm_presence_which_{truth.replace(' ', '_')}", + "presence", + f"Exactly one of these object classes was actually recorded in the " + f"robot's semantic object map: {', '.join(options)}. Which one? " + f"Answer with exactly one of: {', '.join(options)}.", + truth, + choices=list(options), + ) + + popular = sorted(by_class, key=lambda n: -sum(c["n"] for c in by_class[n])) + + # -- global counting per class (skip rows a generic small-count guess can hit) + for name in popular: + count = len(by_class[name]) + band = max(1, round(0.2 * count)) + if count < 5 and any(abs(g - count) <= band for g in (2, 3)): + continue # a blind "2 or 3" would land in-band + add( + f"sm_count_{name.replace(' ', '_')}", + "count", + f"Based on the semantic object map shown, how many distinct " + f"{name} objects are in the mapped area? Answer with a single number.", + count, + band=band, + ) + + # -- zone counting: per zone, quiz the class most represented there + for zone_name, inside in zones.items(): + name = max(popular[:6], key=lambda n: sum(1 for c in by_class[n] if inside(c["x"], c["y"]))) + count = sum(1 for c in by_class[name] if inside(c["x"], c["y"])) + if count == 0: + continue # empty zone — a truth of 0 is prior-guessable + add( + f"sm_zonecount_{zone_name.split()[0]}_{name.replace(' ', '_')}", + "zonecount", + f"{zone_clause} How many distinct {name} objects are in the " + f"{zone_name}? Answer with a single number.", + count, + band=1.0, + ) + + # -- nearest-class MCQ at sampled timestamps + choices = sorted(by_class) + duration = max(d["ts"] for d in detections) + nearest_added = 0 + for t in [duration * f / 20 for f in range(1, 20)]: + if nearest_added >= 5: + break + pose = robot_pose_at(detections, t) + dists = { + name: min(math.hypot(c["x"] - pose["x"], c["y"] - pose["y"]) for c in cs) + for name, cs in by_class.items() + } + ranked = sorted(dists.items(), key=lambda kv: kv[1]) + if ranked[1][1] - ranked[0][1] < NEAREST_MARGIN_M: + continue # ambiguous — skip timestamp + if any( + r["family"] == "nearest" and abs(r["odom_window"][1] - pose["ts"]) < 15.0 for r in rows + ): + continue # keep quizzed timestamps well separated + nearest_added += 1 + add( + f"sm_nearest_t{pose['ts']:g}", + "nearest", + f"You are the robot; your current pose is the last odom observation " + f"shown. Based on the semantic object map, which object class is " + f"horizontally nearest to you? Answer with exactly one of: " + f"{', '.join(choices)}.", + ranked[0][0], + choices=choices, + ctx="objects+odom", + odom_window=[round(max(0.0, pose["ts"] - 0.5), 2), pose["ts"]], + ) + + # -- egocentric ahead/behind/left/right at sampled timestamps + # (single-instance classes, rotating so one object doesn't dominate) + ego_added = 0 + single_names = sorted(singles) + for i, t in enumerate([duration * f for f in (0.15, 0.3, 0.45, 0.6, 0.75, 0.9)]): + if ego_added >= 6: + break + pose = robot_pose_at(detections, t) + rotation = single_names[i % len(single_names) :] + single_names[: i % len(single_names)] + for name in rotation: + c = singles[name] + bearing = math.atan2(c["y"] - pose["y"], c["x"] - pose["x"]) + rel = math.atan2(math.sin(bearing - pose["yaw"]), math.cos(bearing - pose["yaw"])) + # 4-way bands on |rel|: ahead <=40, behind >=140, else side by sign; + # only pick bearings >=EGO_MARGIN_DEG inside a band boundary. + rel_deg = abs(math.degrees(rel)) + if rel_deg <= EGO_AHEAD_DEG - EGO_MARGIN_DEG: + truth = "ahead" + elif rel_deg >= 180.0 - EGO_AHEAD_DEG + EGO_MARGIN_DEG: + truth = "behind" + elif ( + EGO_AHEAD_DEG + EGO_MARGIN_DEG <= rel_deg <= 180.0 - EGO_AHEAD_DEG - EGO_MARGIN_DEG + ): + # positive relative bearing -> object on the left + truth = "left" if rel > 0 else "right" + else: + continue # too close to a band boundary — ambiguous + add( + f"sm_egoside_t{pose['ts']:g}_{name.replace(' ', '_')}", + "egoside", + f"You are the robot; your current pose is the last odom " + f"observation shown, and you face your direction of heading " + f"(the odom yaw). Based on the semantic object map, is the " + f"{name} ahead of you, behind you, on your left, or on your " + f"right? Answer with exactly one word: ahead, behind, left, " + f"or right.", + truth, + choices=["ahead", "behind", "left", "right"], + ctx="objects+odom", + odom_window=[round(max(0.0, pose["ts"] - 0.5), 2), pose["ts"]], + ) + ego_added += 1 + break # one object per timestamp + + # -- object<->object compass relations (single-instance classes, >=3 m apart) + names = sorted(singles) + compass_added = 0 + for i, a in enumerate(names): + for b in names[i + 1 :]: + if compass_added >= 5: + break + ca, cb = singles[a], singles[b] + if math.hypot(ca["x"] - cb["x"], ca["y"] - cb["y"]) < COMPASS_MIN_PAIR_DIST: + continue + add( + f"sm_compass_{a.replace(' ', '_')}_{b.replace(' ', '_')}", + "compass", + f"Based on the semantic object map (world frame, +x east, +y " + f"north), in which compass direction is the {a} from the {b}? " + f"Answer with exactly one word: " + f"{', '.join(COMPASS_NAMES)}.", + compass_of(ca["x"] - cb["x"], ca["y"] - cb["y"]), + choices=list(COMPASS_NAMES), + ) + compass_added += 1 + + # -- object<->object distances (VQASynth-inspired; single-instance, >=3 m + # apart so the odom-grounding error stays small relative to truth) + objdist_added = 0 + objdist_pairs: set[tuple[str, str]] = set() + for i, a in enumerate(names): + for b in names[i + 1 :]: + if objdist_added >= 4: + break + ca, cb = singles[a], singles[b] + dist = math.hypot(ca["x"] - cb["x"], ca["y"] - cb["y"]) + if dist < COMPASS_MIN_PAIR_DIST: + continue + add( + f"sm_objdist_{a.replace(' ', '_')}_{b.replace(' ', '_')}", + "objdist", + f"Based on the semantic object map, how far apart horizontally " + f"are the {a} and the {b}, in meters? Answer with a single number.", + round(dist, 2), + band=max(1.0, round(0.25 * dist, 2)), + ) + objdist_pairs.add((a, b)) + objdist_added += 1 + + # -- robot-to-object distances at sampled timestamps (VQASynth-inspired; + # single-instance classes, quizzed timestamps >=15 s apart) + robotdist_added = 0 + for t in [duration * f for f in (0.2, 0.5, 0.8, 0.35, 0.65)]: + if robotdist_added >= 3: + break + pose = robot_pose_at(detections, t) + if any( + r["family"] == "robotdist" and abs(r["odom_window"][1] - pose["ts"]) < 15.0 + for r in rows + ): + continue + name = single_names[robotdist_added % len(single_names)] + c = singles[name] + dist = math.hypot(c["x"] - pose["x"], c["y"] - pose["y"]) + add( + f"sm_robotdist_t{pose['ts']:g}_{name.replace(' ', '_')}", + "robotdist", + f"You are the robot; your current pose is the last odom observation " + f"shown. Based on the semantic object map, how far are you " + f"horizontally from the {name}, in meters? Answer with a single " + f"number.", + round(dist, 2), + band=max(1.0, round(0.25 * dist, 2)), + ctx="objects+odom", + odom_window=[round(max(0.0, pose["ts"] - 0.5), 2), pose["ts"]], + ) + robotdist_added += 1 + + # -- objdist round 2 (appended so pre-existing row positions are stable): + # diversify anchors — round 1 concentrates on the alphabetically-first + # class; one new pair per other anchor + for a in names: + if objdist_added >= 8: + break + if a == names[0]: + continue + for b in names: + pair = tuple(sorted((a, b))) + if b == a or pair in objdist_pairs: + continue + ca, cb = singles[a], singles[b] + dist = math.hypot(ca["x"] - cb["x"], ca["y"] - cb["y"]) + if dist < COMPASS_MIN_PAIR_DIST: + continue + add( + f"sm_objdist_{a.replace(' ', '_')}_{b.replace(' ', '_')}", + "objdist", + f"Based on the semantic object map, how far apart horizontally " + f"are the {a} and the {b}, in meters? Answer with a single number.", + round(dist, 2), + band=max(1.0, round(0.25 * dist, 2)), + ) + objdist_pairs.add(pair) + objdist_added += 1 + break # one new pair per anchor + + # vocabulary for set-answer parsing: mapped classes + verified-absent + # probes so hallucinated extras cost F1 precision. The probe list is + # office-plausible COCO classes (things a blind prior WOULD list) filtered + # to those never detected — the larger it is, the lower the F1 floor for + # exhaustive or prior-based listing. + office_probes = [ + "backpack", + "bowl", + "cell phone", + "clock", + "couch", + "dining table", + "handbag", + "keyboard", + "microwave", + "mouse", + "oven", + "remote", + "scissors", + "sink", + "suitcase", + "toaster", + "umbrella", + "vase", + "wine glass", + ] + set_vocab = choices + sorted( + {p for p in office_probes if p not in ever_detected} + | set(absent) + | {o for t, opts in RECORDED_MCQS for o in opts if o != t} + ) + list_clause = "Answer with a comma-separated list of class names." + + # -- recall: list every mapped class (full map + two zone-scoped variants) + add( + "sm_recall_all", + "recall", + f"List every distinct object class recorded in the robot's semantic " + f"object map. {list_clause}", + sorted(by_class), + vocab=set_vocab, + ) + recall_zones = 0 + for zone_name, inside in zones.items(): + if recall_zones >= 2: + break + zone_classes = sorted( + n for n, cs in by_class.items() if any(inside(c["x"], c["y"]) for c in cs) + ) + if not (2 <= len(zone_classes) <= len(by_class) - 2): + continue # empty or near-total zones are prior-guessable + add( + f"sm_recall_{zone_name.split()[0]}", + "recall", + f"{zone_clause} List every distinct object class recorded in the " + f"{zone_name}. {list_clause}", + zone_classes, + vocab=set_vocab, + ) + recall_zones += 1 + + # -- within-radius set listing; anchor+R chosen so NO cluster sits within + # 1.5 m of the boundary (odom-grounding hysteresis, asserted by the skip) + within_added = 0 + for x in single_names: + if within_added >= 3: + break + ax, ay = singles[x]["x"], singles[x]["y"] + others = [c for c in clusters if c is not singles[x]] + dists = [math.hypot(c["x"] - ax, c["y"] - ay) for c in others] + best: tuple[int, float, list[str]] | None = None # (len(truth), -R, truth) + for radius in (2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0): + if any(abs(d - radius) < 1.5 for d in dists): + continue # a cluster rides the boundary — ambiguous under grounding error + truth = sorted( + {c["class_name"] for c, d in zip(others, dists, strict=True) if d < radius} + ) + if not truth: + continue # empty truth is prior-guessable ("none") + if best is None or (len(truth), -radius) > (best[0], best[1]): + best = (len(truth), -radius, truth) + if best is not None: + radius = -best[1] + add( + f"sm_within_{x.replace(' ', '_')}_{radius:g}m", + "within", + f"Based on the semantic object map, list every object class other " + f"than {x} with at least one instance within {radius:g} meters " + f"horizontally of the {x}. {list_clause}", + best[2], + vocab=set_vocab, + ) + within_added += 1 + + # -- nextto: nearest class to an object (MCQ; runner-up margin >= 1.5 m) + nextto_added = 0 + for x in single_names: + if nextto_added >= 3: + break + ax, ay = singles[x]["x"], singles[x]["y"] + ranked = sorted( + (min(math.hypot(c["x"] - ax, c["y"] - ay) for c in cs), n) + for n, cs in by_class.items() + if n != x + ) + if ranked[1][0] - ranked[0][0] < 1.5: + continue # ambiguous under grounding error + opts = [n for n in choices if n != x] + add( + f"sm_nextto_{x.replace(' ', '_')}", + "nextto", + f"Based on the semantic object map, which object class is " + f"horizontally nearest to the {x} (other than {x} itself)? " + f"Answer with exactly one of: {', '.join(opts)}.", + ranked[0][1], + choices=opts, + ) + nextto_added += 1 + + # -- between: exactly one cluster in the corridor between two anchors + # (perp < 1.5 m, projection in the middle 60%; no other cluster within + # 2.5 m of the corridor, else the pair is skipped) + between_added = 0 + for i, a in enumerate(single_names): + for b in single_names[i + 1 :]: + if between_added >= 2: + break + ca, cb = singles[a], singles[b] + vx, vy = cb["x"] - ca["x"], cb["y"] - ca["y"] + seg2 = vx * vx + vy * vy + if seg2 < COMPASS_MIN_PAIR_DIST**2: + continue + inside_corridor = [] + near_corridor = 0 + for c in clusters: + if c is ca or c is cb: + continue + s = ((c["x"] - ca["x"]) * vx + (c["y"] - ca["y"]) * vy) / seg2 + perp = abs((c["x"] - ca["x"]) * vy - (c["y"] - ca["y"]) * vx) / math.sqrt(seg2) + if not 0.2 <= s <= 0.8: + continue + if perp < 1.5: + inside_corridor.append(c) + elif perp < 2.5: + near_corridor += 1 + if len(inside_corridor) != 1 or near_corridor: + continue + mid = inside_corridor[0]["class_name"] + if mid in (a, b): + continue + opts = [n for n in choices if n not in (a, b)] + add( + f"sm_between_{a.replace(' ', '_')}_{b.replace(' ', '_')}", + "between", + f"Based on the semantic object map, which single object class " + f"lies between the {a} and the {b}? Answer with exactly one of: " + f"{', '.join(opts)}.", + mid, + choices=opts, + ) + between_added += 1 + + out = here / "rows.json" + out.write_text(json.dumps(rows, indent=2) + "\n") # matches pretty-format-json hook + families = defaultdict(int) + for r in rows: + families[r["family"]] += 1 + print(f"\nwrote {len(rows)} rows -> {out}") + for fam, n in families.items(): + print(f" {fam:10s} {n}") + for r in rows: + print(json.dumps(r)) + + +if __name__ == "__main__": + main() diff --git a/evals_bench/semantic/rows.json b/evals_bench/semantic/rows.json new file mode 100644 index 0000000000..724b502413 --- /dev/null +++ b/evals_bench/semantic/rows.json @@ -0,0 +1,869 @@ +[ + { + "id": "sm_presence_bed", + "family": "presence", + "type": "mcq", + "q": "Did the robot's semantic object map record at least one bed anywhere in the mapped area? Answer with exactly one word: yes or no.", + "a": "yes", + "ctx": "objects", + "choices": [ + "yes", + "no" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_presence_refrigerator", + "family": "presence", + "type": "mcq", + "q": "Did the robot's semantic object map record at least one refrigerator anywhere in the mapped area? Answer with exactly one word: yes or no.", + "a": "yes", + "ctx": "objects", + "choices": [ + "yes", + "no" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_presence_teddy_bear", + "family": "presence", + "type": "mcq", + "q": "Did the robot's semantic object map record at least one teddy bear anywhere in the mapped area? Answer with exactly one word: yes or no.", + "a": "yes", + "ctx": "objects", + "choices": [ + "yes", + "no" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_presence_couch", + "family": "presence", + "type": "mcq", + "q": "Did the robot's semantic object map record at least one couch anywhere in the mapped area? Answer with exactly one word: yes or no.", + "a": "no", + "ctx": "objects", + "choices": [ + "yes", + "no" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_presence_sink", + "family": "presence", + "type": "mcq", + "q": "Did the robot's semantic object map record at least one sink anywhere in the mapped area? Answer with exactly one word: yes or no.", + "a": "no", + "ctx": "objects", + "choices": [ + "yes", + "no" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_presence_which_teddy_bear", + "family": "presence", + "type": "mcq", + "q": "Exactly one of these object classes was actually recorded in the robot's semantic object map: mouse, teddy bear, dining table, cell phone. Which one? Answer with exactly one of: mouse, teddy bear, dining table, cell phone.", + "a": "teddy bear", + "ctx": "objects", + "choices": [ + "mouse", + "teddy bear", + "dining table", + "cell phone" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_presence_which_bed", + "family": "presence", + "type": "mcq", + "q": "Exactly one of these object classes was actually recorded in the robot's semantic object map: clock, oven, bed, scissors. Which one? Answer with exactly one of: clock, oven, bed, scissors.", + "a": "bed", + "ctx": "objects", + "choices": [ + "clock", + "oven", + "bed", + "scissors" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_count_chair", + "family": "count", + "type": "numeric", + "q": "Based on the semantic object map shown, how many distinct chair objects are in the mapped area? Answer with a single number.", + "a": 6, + "ctx": "objects", + "band": 1, + "dataset": "go2_bigoffice" + }, + { + "id": "sm_zonecount_northeast_chair", + "family": "zonecount", + "type": "numeric", + "q": "The map is divided into four zones at the point (x=-7.5, y=2.4), with +x east and +y north: the northwest area (x<-7.5, y>=2.4), northeast area (x>=-7.5, y>=2.4), southwest area (x<-7.5, y<2.4), and southeast area (x>=-7.5, y<2.4). How many distinct chair objects are in the northeast area? Answer with a single number.", + "a": 4, + "ctx": "objects", + "band": 1.0, + "dataset": "go2_bigoffice" + }, + { + "id": "sm_zonecount_southwest_tv", + "family": "zonecount", + "type": "numeric", + "q": "The map is divided into four zones at the point (x=-7.5, y=2.4), with +x east and +y north: the northwest area (x<-7.5, y>=2.4), northeast area (x>=-7.5, y>=2.4), southwest area (x<-7.5, y<2.4), and southeast area (x>=-7.5, y<2.4). How many distinct tv objects are in the southwest area? Answer with a single number.", + "a": 1, + "ctx": "objects", + "band": 1.0, + "dataset": "go2_bigoffice" + }, + { + "id": "sm_zonecount_southeast_bottle", + "family": "zonecount", + "type": "numeric", + "q": "The map is divided into four zones at the point (x=-7.5, y=2.4), with +x east and +y north: the northwest area (x<-7.5, y>=2.4), northeast area (x>=-7.5, y>=2.4), southwest area (x<-7.5, y<2.4), and southeast area (x>=-7.5, y<2.4). How many distinct bottle objects are in the southeast area? Answer with a single number.", + "a": 3, + "ctx": "objects", + "band": 1.0, + "dataset": "go2_bigoffice" + }, + { + "id": "sm_nearest_t29.16", + "family": "nearest", + "type": "mcq", + "q": "You are the robot; your current pose is the last odom observation shown. Based on the semantic object map, which object class is horizontally nearest to you? Answer with exactly one of: bed, book, bottle, chair, cup, laptop, potted plant, refrigerator, teddy bear, tv.", + "a": "chair", + "ctx": "objects+odom", + "choices": [ + "bed", + "book", + "bottle", + "chair", + "cup", + "laptop", + "potted plant", + "refrigerator", + "teddy bear", + "tv" + ], + "odom_window": [ + 28.66, + 29.16 + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_nearest_t103.25", + "family": "nearest", + "type": "mcq", + "q": "You are the robot; your current pose is the last odom observation shown. Based on the semantic object map, which object class is horizontally nearest to you? Answer with exactly one of: bed, book, bottle, chair, cup, laptop, potted plant, refrigerator, teddy bear, tv.", + "a": "tv", + "ctx": "objects+odom", + "choices": [ + "bed", + "book", + "bottle", + "chair", + "cup", + "laptop", + "potted plant", + "refrigerator", + "teddy bear", + "tv" + ], + "odom_window": [ + 102.75, + 103.25 + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_nearest_t204.44", + "family": "nearest", + "type": "mcq", + "q": "You are the robot; your current pose is the last odom observation shown. Based on the semantic object map, which object class is horizontally nearest to you? Answer with exactly one of: bed, book, bottle, chair, cup, laptop, potted plant, refrigerator, teddy bear, tv.", + "a": "teddy bear", + "ctx": "objects+odom", + "choices": [ + "bed", + "book", + "bottle", + "chair", + "cup", + "laptop", + "potted plant", + "refrigerator", + "teddy bear", + "tv" + ], + "odom_window": [ + 203.94, + 204.44 + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_nearest_t241.33", + "family": "nearest", + "type": "mcq", + "q": "You are the robot; your current pose is the last odom observation shown. Based on the semantic object map, which object class is horizontally nearest to you? Answer with exactly one of: bed, book, bottle, chair, cup, laptop, potted plant, refrigerator, teddy bear, tv.", + "a": "potted plant", + "ctx": "objects+odom", + "choices": [ + "bed", + "book", + "bottle", + "chair", + "cup", + "laptop", + "potted plant", + "refrigerator", + "teddy bear", + "tv" + ], + "odom_window": [ + 240.83, + 241.33 + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_nearest_t279.98", + "family": "nearest", + "type": "mcq", + "q": "You are the robot; your current pose is the last odom observation shown. Based on the semantic object map, which object class is horizontally nearest to you? Answer with exactly one of: bed, book, bottle, chair, cup, laptop, potted plant, refrigerator, teddy bear, tv.", + "a": "tv", + "ctx": "objects+odom", + "choices": [ + "bed", + "book", + "bottle", + "chair", + "cup", + "laptop", + "potted plant", + "refrigerator", + "teddy bear", + "tv" + ], + "odom_window": [ + 279.48, + 279.98 + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_egoside_t43.92_potted_plant", + "family": "egoside", + "type": "mcq", + "q": "You are the robot; your current pose is the last odom observation shown, and you face your direction of heading (the odom yaw). Based on the semantic object map, is the potted plant ahead of you, behind you, on your left, or on your right? Answer with exactly one word: ahead, behind, left, or right.", + "a": "behind", + "ctx": "objects+odom", + "choices": [ + "ahead", + "behind", + "left", + "right" + ], + "odom_window": [ + 43.42, + 43.92 + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_egoside_t85_book", + "family": "egoside", + "type": "mcq", + "q": "You are the robot; your current pose is the last odom observation shown, and you face your direction of heading (the odom yaw). Based on the semantic object map, is the book ahead of you, behind you, on your left, or on your right? Answer with exactly one word: ahead, behind, left, or right.", + "a": "behind", + "ctx": "objects+odom", + "choices": [ + "ahead", + "behind", + "left", + "right" + ], + "odom_window": [ + 84.5, + 85.0 + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_egoside_t128.2_laptop", + "family": "egoside", + "type": "mcq", + "q": "You are the robot; your current pose is the last odom observation shown, and you face your direction of heading (the odom yaw). Based on the semantic object map, is the laptop ahead of you, behind you, on your left, or on your right? Answer with exactly one word: ahead, behind, left, or right.", + "a": "left", + "ctx": "objects+odom", + "choices": [ + "ahead", + "behind", + "left", + "right" + ], + "odom_window": [ + 127.7, + 128.2 + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_egoside_t170.01_potted_plant", + "family": "egoside", + "type": "mcq", + "q": "You are the robot; your current pose is the last odom observation shown, and you face your direction of heading (the odom yaw). Based on the semantic object map, is the potted plant ahead of you, behind you, on your left, or on your right? Answer with exactly one word: ahead, behind, left, or right.", + "a": "left", + "ctx": "objects+odom", + "choices": [ + "ahead", + "behind", + "left", + "right" + ], + "odom_window": [ + 169.51, + 170.01 + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_egoside_t204.44_refrigerator", + "family": "egoside", + "type": "mcq", + "q": "You are the robot; your current pose is the last odom observation shown, and you face your direction of heading (the odom yaw). Based on the semantic object map, is the refrigerator ahead of you, behind you, on your left, or on your right? Answer with exactly one word: ahead, behind, left, or right.", + "a": "right", + "ctx": "objects+odom", + "choices": [ + "ahead", + "behind", + "left", + "right" + ], + "odom_window": [ + 203.94, + 204.44 + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_egoside_t255.03_teddy_bear", + "family": "egoside", + "type": "mcq", + "q": "You are the robot; your current pose is the last odom observation shown, and you face your direction of heading (the odom yaw). Based on the semantic object map, is the teddy bear ahead of you, behind you, on your left, or on your right? Answer with exactly one word: ahead, behind, left, or right.", + "a": "behind", + "ctx": "objects+odom", + "choices": [ + "ahead", + "behind", + "left", + "right" + ], + "odom_window": [ + 254.53, + 255.03 + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_compass_bed_book", + "family": "compass", + "type": "mcq", + "q": "Based on the semantic object map (world frame, +x east, +y north), in which compass direction is the bed from the book? Answer with exactly one word: east, northeast, north, northwest, west, southwest, south, southeast.", + "a": "northwest", + "ctx": "objects", + "choices": [ + "east", + "northeast", + "north", + "northwest", + "west", + "southwest", + "south", + "southeast" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_compass_bed_potted_plant", + "family": "compass", + "type": "mcq", + "q": "Based on the semantic object map (world frame, +x east, +y north), in which compass direction is the bed from the potted plant? Answer with exactly one word: east, northeast, north, northwest, west, southwest, south, southeast.", + "a": "west", + "ctx": "objects", + "choices": [ + "east", + "northeast", + "north", + "northwest", + "west", + "southwest", + "south", + "southeast" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_compass_bed_refrigerator", + "family": "compass", + "type": "mcq", + "q": "Based on the semantic object map (world frame, +x east, +y north), in which compass direction is the bed from the refrigerator? Answer with exactly one word: east, northeast, north, northwest, west, southwest, south, southeast.", + "a": "northwest", + "ctx": "objects", + "choices": [ + "east", + "northeast", + "north", + "northwest", + "west", + "southwest", + "south", + "southeast" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_compass_bed_teddy_bear", + "family": "compass", + "type": "mcq", + "q": "Based on the semantic object map (world frame, +x east, +y north), in which compass direction is the bed from the teddy bear? Answer with exactly one word: east, northeast, north, northwest, west, southwest, south, southeast.", + "a": "northwest", + "ctx": "objects", + "choices": [ + "east", + "northeast", + "north", + "northwest", + "west", + "southwest", + "south", + "southeast" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_compass_book_laptop", + "family": "compass", + "type": "mcq", + "q": "Based on the semantic object map (world frame, +x east, +y north), in which compass direction is the book from the laptop? Answer with exactly one word: east, northeast, north, northwest, west, southwest, south, southeast.", + "a": "south", + "ctx": "objects", + "choices": [ + "east", + "northeast", + "north", + "northwest", + "west", + "southwest", + "south", + "southeast" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_objdist_bed_book", + "family": "objdist", + "type": "numeric", + "q": "Based on the semantic object map, how far apart horizontally are the bed and the book, in meters? Answer with a single number.", + "a": 15.8, + "ctx": "objects", + "band": 3.95, + "dataset": "go2_bigoffice" + }, + { + "id": "sm_objdist_bed_potted_plant", + "family": "objdist", + "type": "numeric", + "q": "Based on the semantic object map, how far apart horizontally are the bed and the potted plant, in meters? Answer with a single number.", + "a": 8.5, + "ctx": "objects", + "band": 2.12, + "dataset": "go2_bigoffice" + }, + { + "id": "sm_objdist_bed_refrigerator", + "family": "objdist", + "type": "numeric", + "q": "Based on the semantic object map, how far apart horizontally are the bed and the refrigerator, in meters? Answer with a single number.", + "a": 15.07, + "ctx": "objects", + "band": 3.77, + "dataset": "go2_bigoffice" + }, + { + "id": "sm_objdist_bed_teddy_bear", + "family": "objdist", + "type": "numeric", + "q": "Based on the semantic object map, how far apart horizontally are the bed and the teddy bear, in meters? Answer with a single number.", + "a": 9.6, + "ctx": "objects", + "band": 2.4, + "dataset": "go2_bigoffice" + }, + { + "id": "sm_robotdist_t57.62_bed", + "family": "robotdist", + "type": "numeric", + "q": "You are the robot; your current pose is the last odom observation shown. Based on the semantic object map, how far are you horizontally from the bed, in meters? Answer with a single number.", + "a": 15.26, + "ctx": "objects+odom", + "band": 3.82, + "odom_window": [ + 57.12, + 57.62 + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_robotdist_t133.82_book", + "family": "robotdist", + "type": "numeric", + "q": "You are the robot; your current pose is the last odom observation shown. Based on the semantic object map, how far are you horizontally from the book, in meters? Answer with a single number.", + "a": 5.52, + "ctx": "objects+odom", + "band": 1.38, + "odom_window": [ + 133.32, + 133.82 + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_robotdist_t241.33_laptop", + "family": "robotdist", + "type": "numeric", + "q": "You are the robot; your current pose is the last odom observation shown. Based on the semantic object map, how far are you horizontally from the laptop, in meters? Answer with a single number.", + "a": 9.69, + "ctx": "objects+odom", + "band": 2.42, + "odom_window": [ + 240.83, + 241.33 + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_objdist_book_laptop", + "family": "objdist", + "type": "numeric", + "q": "Based on the semantic object map, how far apart horizontally are the book and the laptop, in meters? Answer with a single number.", + "a": 15.13, + "ctx": "objects", + "band": 3.78, + "dataset": "go2_bigoffice" + }, + { + "id": "sm_objdist_laptop_potted_plant", + "family": "objdist", + "type": "numeric", + "q": "Based on the semantic object map, how far apart horizontally are the laptop and the potted plant, in meters? Answer with a single number.", + "a": 7.66, + "ctx": "objects", + "band": 1.91, + "dataset": "go2_bigoffice" + }, + { + "id": "sm_objdist_potted_plant_book", + "family": "objdist", + "type": "numeric", + "q": "Based on the semantic object map, how far apart horizontally are the potted plant and the book, in meters? Answer with a single number.", + "a": 13.32, + "ctx": "objects", + "band": 3.33, + "dataset": "go2_bigoffice" + }, + { + "id": "sm_objdist_refrigerator_book", + "family": "objdist", + "type": "numeric", + "q": "Based on the semantic object map, how far apart horizontally are the refrigerator and the book, in meters? Answer with a single number.", + "a": 4.74, + "ctx": "objects", + "band": 1.18, + "dataset": "go2_bigoffice" + }, + { + "id": "sm_recall_all", + "family": "recall", + "type": "set", + "q": "List every distinct object class recorded in the robot's semantic object map. Answer with a comma-separated list of class names.", + "a": [ + "bed", + "book", + "bottle", + "chair", + "cup", + "laptop", + "potted plant", + "refrigerator", + "teddy bear", + "tv" + ], + "ctx": "objects", + "vocab": [ + "bed", + "book", + "bottle", + "chair", + "cup", + "laptop", + "potted plant", + "refrigerator", + "teddy bear", + "tv", + "backpack", + "cell phone", + "clock", + "couch", + "dining table", + "handbag", + "mouse", + "oven", + "remote", + "scissors", + "sink", + "toaster", + "umbrella", + "wine glass" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_recall_northwest", + "family": "recall", + "type": "set", + "q": "The map is divided into four zones at the point (x=-7.5, y=2.4), with +x east and +y north: the northwest area (x<-7.5, y>=2.4), northeast area (x>=-7.5, y>=2.4), southwest area (x<-7.5, y<2.4), and southeast area (x>=-7.5, y<2.4). List every distinct object class recorded in the northwest area. Answer with a comma-separated list of class names.", + "a": [ + "bed", + "laptop" + ], + "ctx": "objects", + "vocab": [ + "bed", + "book", + "bottle", + "chair", + "cup", + "laptop", + "potted plant", + "refrigerator", + "teddy bear", + "tv", + "backpack", + "cell phone", + "clock", + "couch", + "dining table", + "handbag", + "mouse", + "oven", + "remote", + "scissors", + "sink", + "toaster", + "umbrella", + "wine glass" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_recall_northeast", + "family": "recall", + "type": "set", + "q": "The map is divided into four zones at the point (x=-7.5, y=2.4), with +x east and +y north: the northwest area (x<-7.5, y>=2.4), northeast area (x>=-7.5, y>=2.4), southwest area (x<-7.5, y<2.4), and southeast area (x>=-7.5, y<2.4). List every distinct object class recorded in the northeast area. Answer with a comma-separated list of class names.", + "a": [ + "chair", + "potted plant", + "teddy bear", + "tv" + ], + "ctx": "objects", + "vocab": [ + "bed", + "book", + "bottle", + "chair", + "cup", + "laptop", + "potted plant", + "refrigerator", + "teddy bear", + "tv", + "backpack", + "cell phone", + "clock", + "couch", + "dining table", + "handbag", + "mouse", + "oven", + "remote", + "scissors", + "sink", + "toaster", + "umbrella", + "wine glass" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_within_bed_3m", + "family": "within", + "type": "set", + "q": "Based on the semantic object map, list every object class other than bed with at least one instance within 3 meters horizontally of the bed. Answer with a comma-separated list of class names.", + "a": [ + "laptop" + ], + "ctx": "objects", + "vocab": [ + "bed", + "book", + "bottle", + "chair", + "cup", + "laptop", + "potted plant", + "refrigerator", + "teddy bear", + "tv", + "backpack", + "cell phone", + "clock", + "couch", + "dining table", + "handbag", + "mouse", + "oven", + "remote", + "scissors", + "sink", + "toaster", + "umbrella", + "wine glass" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_within_laptop_3m", + "family": "within", + "type": "set", + "q": "Based on the semantic object map, list every object class other than laptop with at least one instance within 3 meters horizontally of the laptop. Answer with a comma-separated list of class names.", + "a": [ + "bed" + ], + "ctx": "objects", + "vocab": [ + "bed", + "book", + "bottle", + "chair", + "cup", + "laptop", + "potted plant", + "refrigerator", + "teddy bear", + "tv", + "backpack", + "cell phone", + "clock", + "couch", + "dining table", + "handbag", + "mouse", + "oven", + "remote", + "scissors", + "sink", + "toaster", + "umbrella", + "wine glass" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_nextto_bed", + "family": "nextto", + "type": "mcq", + "q": "Based on the semantic object map, which object class is horizontally nearest to the bed (other than bed itself)? Answer with exactly one of: book, bottle, chair, cup, laptop, potted plant, refrigerator, teddy bear, tv.", + "a": "laptop", + "ctx": "objects", + "choices": [ + "book", + "bottle", + "chair", + "cup", + "laptop", + "potted plant", + "refrigerator", + "teddy bear", + "tv" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_nextto_laptop", + "family": "nextto", + "type": "mcq", + "q": "Based on the semantic object map, which object class is horizontally nearest to the laptop (other than laptop itself)? Answer with exactly one of: bed, book, bottle, chair, cup, potted plant, refrigerator, teddy bear, tv.", + "a": "bed", + "ctx": "objects", + "choices": [ + "bed", + "book", + "bottle", + "chair", + "cup", + "potted plant", + "refrigerator", + "teddy bear", + "tv" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_between_laptop_teddy_bear", + "family": "between", + "type": "mcq", + "q": "Based on the semantic object map, which single object class lies between the laptop and the teddy bear? Answer with exactly one of: bed, book, bottle, chair, cup, potted plant, refrigerator, tv.", + "a": "chair", + "ctx": "objects", + "choices": [ + "bed", + "book", + "bottle", + "chair", + "cup", + "potted plant", + "refrigerator", + "tv" + ], + "dataset": "go2_bigoffice" + }, + { + "id": "sm_between_potted_plant_teddy_bear", + "family": "between", + "type": "mcq", + "q": "Based on the semantic object map, which single object class lies between the potted plant and the teddy bear? Answer with exactly one of: bed, book, bottle, chair, cup, laptop, refrigerator, tv.", + "a": "chair", + "ctx": "objects", + "choices": [ + "bed", + "book", + "bottle", + "chair", + "cup", + "laptop", + "refrigerator", + "tv" + ], + "dataset": "go2_bigoffice" + } +] From 45ebc7b00810c604c81b144f1033545d04ce6370 Mon Sep 17 00:00:00 2001 From: stash Date: Sun, 9 Aug 2026 20:26:37 -0700 Subject: [PATCH 12/12] =?UTF-8?q?feat(perception):=20ImageDetections3DPC.a?= =?UTF-8?q?gent=5Fencode()=20=E2=80=94=20LLM-legible=20semantic=20object?= =?UTF-8?q?=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Distilled from evo autoresearch winner exp_0013 (semantic VQA: baseline 0.3546 -> 0.9997; replicates 0.953/0.9998; blind control 0.06). Compact per-frame 'name#k conf pos' entries with encoder-side greedy instance clustering plus a once-per-pass P1-P9 procedure legend (closed-form clearance/compass/ego-frame rules, noise deletion, distance-table discipline). --- .../type/detection3d/imageDetections3DPC.py | 124 +++++++++++++++++- 1 file changed, 123 insertions(+), 1 deletion(-) diff --git a/dimos/perception/detection/type/detection3d/imageDetections3DPC.py b/dimos/perception/detection/type/detection3d/imageDetections3DPC.py index e63d415b8a..7cf85b36d5 100644 --- a/dimos/perception/detection/type/detection3d/imageDetections3DPC.py +++ b/dimos/perception/detection/type/detection3d/imageDetections3DPC.py @@ -14,7 +14,8 @@ from __future__ import annotations -from typing import TYPE_CHECKING +import math +from typing import TYPE_CHECKING, ClassVar from dimos.perception.detection.type.detection3d.pointcloud import Detection3DPC from dimos.perception.detection.type.imageDetections import ImageDetections @@ -28,10 +29,131 @@ from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D from dimos.perception.detection.type.detection3d.pointcloud_filters import PointCloudFilter +# Legend prepended once per encoding pass: how to read the per-frame detection +# lines, plus closed-form decision procedures for the spatial reasoning an +# agent is asked to do over a semantic object map. Procedures are deliberately +# compare-and-sign recipes (no free-form trigonometry) so a model can follow +# them step by step. +_AGENT_LEGEND = """\ +SEMANTIC OBJECT MAP -- reading guide and reasoning procedures. +Each observation below is one camera frame; each entry reads +"name#k conf=C pos=(x, y, z)": one detection of class name at world-frame +position (x, y, z) in meters (+x east, +y north, z up). The same physical +object is re-detected on many frames, so it appears as many entries with +slightly different positions; #k is its instance id — entries sharing a +tag (class AND id) are sightings of the SAME physical object (ids come +from an exact greedy scan: a detection joins the first same-class instance +whose running-mean position lies within 1.5 m, else it founds the next id). + +Use these procedures exactly; do every computation numerically, step by step. + +P1 BUILD THE OBJECT LIST FIRST (before answering anything): + a. Group entries by their full tag (class AND id). Each tag is ONE + physical object: position = the mean (x, y) of its entries, sighting + count = the number of entries carrying the tag. + b. Different ids of the same class are DISTINCT objects even when their + means sit only one-to-two meters apart. Never merge tags, and never + measure a distance between entries of the same tag. + c. Then DELETE noise, before anything else uses the table: tags with + fewer than 3 entries, classes implausible indoors (vehicles, street + objects, wild animals — a plush toy is a household object, not an + animal — and outdoor sports gear), and inherently mobile classes + (people). A deleted tag must never appear in any later step or answer + — if a class keeps only one tag, that survivor is THE object of its + class. + d. Write out the final table (tag, mean x, mean y, count) and answer + every question from that table only. +P2 DISTANCES are horizontal: dist = sqrt((x1-x2)^2 + (y1-y2)^2) between + cluster positions (robot-relative distances use the robot pose of P4). + Answer numeric questions with the computed number only; never refuse. +P3 COMPASS direction of A from B: dx = xA-xB, dy = yA-yB. Write down both + products 2.41*|dx| and 2.41*|dy| before comparing anything. A bigger |dx| + alone does NOT mean east/west: the pure directions need a 2.41x dominance, + otherwise the answer is diagonal. + If |dx| > 2.41*|dy| -> "east" if dx>0 else "west". + Else if |dy| > 2.41*|dx| -> "north" if dy>0 else "south". + Else diagonal -> "northeast" (dx>0, dy>0), "northwest" (dx<0, dy>0), + "southeast" (dx>0, dy<0), "southwest" (dx<0, dy<0). +P4 ROBOT POSE: the LAST odom observation shown. Its pos gives your (xr, yr); + its euler [roll, pitch, yaw] is in DEGREES -- your heading is th = yaw + (use degree-mode trig, or convert to radians first). +P5 EGO FRAME (is an object ahead/behind/left/right of YOU): never judge from + raw world coordinates; transform first. With dx = xo-xr, dy = yo-yr to the + object cluster: fwd = dx*cos(th) + dy*sin(th); left = -dx*sin(th) + + dy*cos(th). Write both numbers down. Then: + if |fwd| >= |left| -> "ahead" if fwd > 0 else "behind"; + otherwise -> "left" if left > 0 else "right". +P6 COUNTS AND ZONES: count DISTINCT clusters (after P1), not entries. For a + zone question, test each cluster's mean (x, y) against the zone's + inequalities and count/list only the clusters that satisfy them. +P7 NEAREST / NEXT-TO: write a full table — one line per candidate class with + its distance from the reference point (to the class's nearest cluster) — + then answer the class with the smallest distance. Never answer before the + table is complete. Recency and salience are irrelevant: a prominent or + recently-seen object is often NOT the nearest — before answering, verify + no other class in the table has a smaller distance. +P8 LISTS (which classes exist / lie within a radius / lie in a zone): apply + P1 first, then list each qualifying class exactly once, comma-separated. +P9 BETWEEN A and B: first admit only candidates strictly between the two + anchors: a cluster qualifies only if BOTH its distance to A and its + distance to B are smaller than the A-to-B distance (deleted noise tags + never qualify; a cluster on the far side of an anchor fails this test + even if it lies on the A-B line). Among qualifying clusters, answer the + class of the one nearest the midpoint ((xA+xB)/2, (yA+yB)/2). +""" + class ImageDetections3DPC(ImageDetections[Detection3DPC]): """Specialized class for 3D detections in an image.""" + # Max frame timestamp seen by agent_encode; a non-increasing ts means a + # new encoding pass started (frames are re-encoded from the top). + _agent_last_ts: float | None = None + # Per-pass online instance clustering: class name -> per-instance + # [sum x, sum y, count], in founding order. A detection joins the first + # instance of its class whose running mean lies within 1.5 m, else it + # founds a new one. Reset at each new encoding pass. + _agent_instances: ClassVar[dict[str, list[list[float]]]] = {} + + def agent_encode(self) -> list[dict[str, str]]: + """Model-legible encoding: compact per-frame detection lines with + greedily-assigned instance-id tags (``name#k``). + + Prepends the reasoning-procedure legend once per encoding pass. + Frame timestamps are monotonic within a pass, so a non-increasing + timestamp signals a fresh pass (ponytail: monotonic-ts heuristic; + move legend + clustering state into the encoder loop if streams + ever interleave). + """ + cls = ImageDetections3DPC + ts = self.image.ts + new_pass = cls._agent_last_ts is None or ts <= cls._agent_last_ts + cls._agent_last_ts = ts + if new_pass: + cls._agent_instances = {} + + parts: list[str] = [] + for det in self.detections: + x, y = det.center.x, det.center.y + instances = cls._agent_instances.setdefault(det.name, []) + for k, inst in enumerate(instances): # noqa: B007 - k used after break + if math.hypot(x - inst[0] / inst[2], y - inst[1] / inst[2]) <= 1.5: + inst[0] += x + inst[1] += y + inst[2] += 1 + break + else: + instances.append([x, y, 1.0]) + k = len(instances) - 1 + parts.append( + f"{det.name}#{k + 1} conf={det.confidence:.2f} " + f"pos=({x:.2f}, {y:.2f}, {det.center.z:.2f})" + ) + blocks = [{"type": "text", "text": "; ".join(parts) or "no detections"}] + if new_pass: + blocks.insert(0, {"type": "text", "text": _AGENT_LEGEND}) + return blocks + @classmethod def from_2d( cls,