diff --git a/dimos/benchmark/evaluation/registry.py b/dimos/benchmark/evaluation/registry.py index a6608daf67..14e929d911 100644 --- a/dimos/benchmark/evaluation/registry.py +++ b/dimos/benchmark/evaluation/registry.py @@ -31,6 +31,7 @@ LOCAL_NAME_PATTERN = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") BUILTIN_EVALUATIONS = { "frozen-integer-qa": ("dimos.benchmark.short_horizon_qa.evaluation:frozen_integer_qa"), + "point-cloud-vqa": ("dimos.benchmark.vqa.evaluation:point_cloud_vqa"), } diff --git a/dimos/benchmark/vqa/evaluation.py b/dimos/benchmark/vqa/evaluation.py new file mode 100644 index 0000000000..c7ae259156 --- /dev/null +++ b/dimos/benchmark/vqa/evaluation.py @@ -0,0 +1,144 @@ +# Copyright 2026 Dimensional Inc. +"""Multiple-choice VQA evaluation plugin using public image artifacts only.""" + +from __future__ import annotations + +from collections.abc import Callable +import json +from pathlib import Path +import re + +import cv2 +from pydantic import BaseModel, ConfigDict, Field + +from dimos.benchmark.evaluation.models import ( + ArtifactNativeResult, + ArtifactReference, + EvaluationReport, + SummaryItem, +) +from dimos.benchmark.evaluation.protocol import EvaluationContext +from dimos.models.vl.openai import OpenAIVlModel +from dimos.msgs.sensor_msgs.Image import Image + + +class VqaEvaluationConfig(BaseModel): + """Location and vision model for a generated VQA evaluation dataset.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + dataset: str = Field(min_length=1) + model: str = Field(default="gpt-4o-mini", min_length=1) + + +class MultipleChoiceVqaEvaluation: + """Score a vision model against generated image-question-choice VQA cases.""" + + name = "point-cloud-vqa" + config_model: type[BaseModel] = VqaEvaluationConfig + + def __init__(self, vision_factory: Callable[[str], OpenAIVlModel] | None = None) -> None: + self._vision_factory = vision_factory or (lambda model: OpenAIVlModel(model_name=model)) + + def run(self, config: BaseModel, context: EvaluationContext) -> EvaluationReport: + if not isinstance(config, VqaEvaluationConfig): + raise TypeError("point-cloud-vqa received the wrong configuration type") + dataset = Path(config.dataset).expanduser() + if not dataset.is_absolute(): + dataset = context.spec_dir / dataset + dataset = dataset.resolve() + cases = _load_jsonl(dataset / "cases.jsonl") + labels = {item["id"]: item["answer"] for item in _load_jsonl(dataset / "labels.jsonl")} + model = self._vision_factory(config.model) + results = [_evaluate_case(dataset, model, case, labels) for case in cases] + artifact = context.workspace / "vqa-results.json" + artifact.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8") + passed = sum(item["passed"] for item in results) + total = len(results) + return EvaluationReport( + summary=( + SummaryItem(key="cases", label="Cases", value=total), + SummaryItem(key="passed", label="Passed", value=passed), + SummaryItem( + key="accuracy", label="Accuracy", value=passed / total if total else 0.0 + ), + ), + native_result=ArtifactNativeResult( + artifact=ArtifactReference( + path=artifact.relative_to(context.workspace).as_posix(), + label="VQA case results", + media_type="application/json", + ) + ), + artifacts=( + ArtifactReference( + path=artifact.relative_to(context.workspace).as_posix(), + label="VQA case results", + media_type="application/json", + ), + ), + ) + + +def _evaluate_case( + dataset: Path, model: OpenAIVlModel, case: dict[str, object], labels: dict[str, str] +) -> dict[str, object]: + case_id = _required_string(case, "id") + choices = _choices(case) + expected = labels.get(case_id) + if expected is None: + raise ValueError(f"missing private label for case {case_id}") + if expected not in choices: + raise ValueError(f"private label for {case_id} is not an allowed choice") + image = cv2.imread(str(dataset / _required_string(case, "image"))) + if image is None: + raise ValueError(f"unable to load public image for case {case_id}") + prompt = ( + f"{_required_string(case, 'question')}\n\n" + f"Choices: {', '.join(choices)}.\n" + "Use only the supplied image. End with exactly `ANSWER: `." + ) + response = model.query(Image.from_numpy(image), prompt) + answer = _parse_choice(response, choices) + return { + "id": case_id, + "expected": expected, + "answer": answer, + "passed": answer == expected, + "raw_response": response, + } + + +def _load_jsonl(path: Path) -> list[dict[str, object]]: + if not path.is_file(): + raise ValueError(f"missing VQA dataset file: {path}") + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line] + + +def _required_string(item: dict[str, object], key: str) -> str: + value = item.get(key) + if not isinstance(value, str) or not value: + raise ValueError(f"VQA case requires non-empty {key}") + return value + + +def _choices(case: dict[str, object]) -> tuple[str, ...]: + value = case.get("choices") + if ( + not isinstance(value, list) + or len(value) < 2 + or not all(isinstance(item, str) for item in value) + ): + raise ValueError("VQA case requires at least two string choices") + return tuple(value) + + +def _parse_choice(response: str, choices: tuple[str, ...]) -> str | None: + match = re.search(r"^ANSWER:\s*(.+?)\s*$", response, re.MULTILINE) + if match is None: + return None + answer = match.group(1) + return answer if answer in choices else None + + +point_cloud_vqa = MultipleChoiceVqaEvaluation() diff --git a/dimos/benchmark/vqa/generation/adapters.py b/dimos/benchmark/vqa/generation/adapters.py new file mode 100644 index 0000000000..ebe4323599 --- /dev/null +++ b/dimos/benchmark/vqa/generation/adapters.py @@ -0,0 +1,51 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MoonDream and EdgeTAM adapters for the single-frame VQA pipeline.""" + +from __future__ import annotations + +from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter +from dimos.models.vl.moondream import MoondreamVlModel +from dimos.msgs.sensor_msgs.Image import Image +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D + + +class MoondreamObjectDetector: + """Adapt MoonDream's query detection API to the grounding interface.""" + + def __init__(self, model: MoondreamVlModel) -> None: + self._model = model + + def detect(self, image: Image, query: str) -> ImageDetections2D: + return self._model.query_detections(image, query) + + def locate(self, image: Image, query: str) -> ImageDetections2D: + points = self._model.query_points(image, f"center of the {query}") + for point in points: + point.name = query + return points + + +class EdgeTamObjectSegmenter: + """Adapt EdgeTAM single-image segmentation to the grounding interface.""" + + def __init__(self, segmenter: EdgeTAMImageSegmenter) -> None: + self._segmenter = segmenter + + def segment(self, detections: ImageDetections2D) -> ImageDetections2D: + return self._segmenter.segment(detections) + + def segment_points(self, points: ImageDetections2D) -> ImageDetections2D: + return self._segmenter.segment_points(points) diff --git a/dimos/benchmark/vqa/generation/dataset.py b/dimos/benchmark/vqa/generation/dataset.py new file mode 100644 index 0000000000..987e1ced7d --- /dev/null +++ b/dimos/benchmark/vqa/generation/dataset.py @@ -0,0 +1,149 @@ +# Copyright 2026 Dimensional Inc. +"""Persist VQA generation evidence and a simple multiple-choice evaluation export.""" + +from __future__ import annotations + +from dataclasses import asdict +import json +from pathlib import Path +from typing import Any + +import cv2 + +from dimos.benchmark.vqa.models import ( + AcceptedOracleResult, + BooleanAnswerContract, + CalibratedFrame, + GroundTruthResult, + QuestionIntent, + QuestionProposal, + RejectedOracleResult, +) + + +def write_frame_record( + output: Path, + frame: CalibratedFrame, + recording: str, + frame_index: int, + intents: list[QuestionIntent | QuestionProposal], + results: list[GroundTruthResult | AcceptedOracleResult | RejectedOracleResult], + metadata: dict[str, Any], +) -> None: + """Write one frame's public cases alongside its private generation audit record.""" + output.mkdir(parents=True, exist_ok=False) + image_path = output / "image.jpg" + if not cv2.imwrite(str(image_path), frame.image.data): + raise RuntimeError(f"failed to write {image_path}") + accepted = [result for result in results if _is_accepted(result)] + cases, labels = _evaluation_rows(frame.id, accepted) + _write_json( + output / "frame.json", + { + "schema_version": "1.0", + "frame_id": frame.id, + "recording": recording, + "frame_index": frame_index, + "image": image_path.name, + "question_count": len(intents), + "accepted_question_count": len(accepted), + "rejected_question_count": len(results) - len(accepted), + **metadata, + }, + ) + _write_json(output / "ground_truth.json", [_private_result(item) for item in results]) + _write_json(output / "cases.json", cases) + _write_json(output / "labels.json", labels) + + +def write_dataset_manifest(output: Path) -> dict[str, int]: + """Build aggregate public cases and private labels from completed frame records.""" + frames = sorted(path for path in output.glob("frame-*") if (path / "frame.json").is_file()) + case_rows: list[dict[str, Any]] = [] + label_rows: list[dict[str, Any]] = [] + accepted = 0 + rejected = 0 + for path in frames: + frame = json.loads((path / "frame.json").read_text()) + case_rows.extend( + {**case, "image": f"{path.name}/{case['image']}"} + for case in json.loads((path / "cases.json").read_text()) + ) + label_rows.extend(json.loads((path / "labels.json").read_text())) + accepted += frame["accepted_question_count"] + rejected += frame["rejected_question_count"] + _write_jsonl(output / "cases.jsonl", case_rows) + _write_jsonl(output / "labels.jsonl", label_rows) + return { + "frame_count": len(frames), + "accepted_question_count": accepted, + "rejected_question_count": rejected, + } + + +def _is_accepted(result: GroundTruthResult | AcceptedOracleResult | RejectedOracleResult) -> bool: + return isinstance(result, AcceptedOracleResult) or ( + isinstance(result, GroundTruthResult) and result.status == "answered" + ) + + +def _evaluation_rows( + frame_id: str, results: list[GroundTruthResult | AcceptedOracleResult | RejectedOracleResult] +) -> tuple[list[dict[str, Any]], list[dict[str, str]]]: + cases: list[dict[str, Any]] = [] + labels: list[dict[str, str]] = [] + for result in results: + if isinstance(result, RejectedOracleResult): + continue + if isinstance(result, AcceptedOracleResult): + contract = result.answer_contract + choices = ( + ("yes", "no") if isinstance(contract, BooleanAnswerContract) else contract.choices + ) + case_id = f"{frame_id}-{result.proposal.id}" + question = result.proposal.question + answer = result.answer + else: + case_id = result.question.id + question = result.question.question + choices = result.question.allowed_answers + answer = result.answer + if answer is None or answer not in choices: + raise ValueError(f"accepted VQA case {case_id} must have a choice answer") + cases.append( + {"id": case_id, "image": "image.jpg", "question": question, "choices": choices} + ) + labels.append({"id": case_id, "answer": answer}) + return cases, labels + + +def _private_result( + result: GroundTruthResult | AcceptedOracleResult | RejectedOracleResult, +) -> dict[str, Any]: + if isinstance(result, AcceptedOracleResult): + return { + "status": "answered", + "answer": result.answer, + "proposal": asdict(result.proposal), + "answer_contract": asdict(result.answer_contract), + "evidence_ids": result.evidence_ids, + "tool_results": [asdict(item) for item in result.tool_results], + "trace": [asdict(item) for item in result.trace], + } + if isinstance(result, RejectedOracleResult): + return { + "status": "rejected", + "reason": result.reason, + "proposal": asdict(result.proposal), + "tool_results": [asdict(item) for item in result.tool_results], + "trace": [asdict(item) for item in result.trace], + } + return asdict(result) + + +def _write_json(path: Path, payload: Any) -> None: + path.write_text(json.dumps(payload, indent=2) + "\n") + + +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.write_text("".join(f"{json.dumps(row, sort_keys=True)}\n" for row in rows)) diff --git a/dimos/benchmark/vqa/generation/geometry.py b/dimos/benchmark/vqa/generation/geometry.py new file mode 100644 index 0000000000..db094027b6 --- /dev/null +++ b/dimos/benchmark/vqa/generation/geometry.py @@ -0,0 +1,85 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Static calibrated point-cloud projection for single-frame VQA.""" + +from __future__ import annotations + +import numpy as np + +from dimos.benchmark.vqa.models import CalibratedFrame, ProjectedPoints, ProjectionConfig + + +def project_visible_points( + frame: CalibratedFrame, config: ProjectionConfig = ProjectionConfig() +) -> ProjectedPoints: + """Project the nearest point at each image pixel using static calibration. + + The supplied transform maps point-cloud coordinates to the camera optical + frame. This function intentionally has no time or TF dependency. + """ + if not frame.image_is_rectified: + raise ValueError("VQA projection requires a rectified pinhole image") + if config.min_depth_m <= 0: + raise ValueError("min_depth_m must be positive") + + camera_info = frame.camera_info + if camera_info.width != frame.image.width or camera_info.height != frame.image.height: + raise ValueError("camera intrinsics dimensions must match the image") + if len(camera_info.K) != 9: + raise ValueError("camera intrinsics must contain a 3x3 matrix") + + points, _ = frame.pointcloud.as_numpy() + if len(points) == 0: + return ProjectedPoints([], [], []) + + homogeneous = np.column_stack((points, np.ones(len(points), dtype=points.dtype))) + camera_points = (frame.pointcloud_to_camera.to_matrix() @ homogeneous.T).T[:, :3] + source_indices = np.arange(len(points)) + + depth_mask = camera_points[:, 2] >= config.min_depth_m + camera_points = camera_points[depth_mask] + source_indices = source_indices[depth_mask] + if len(camera_points) == 0: + return ProjectedPoints([], [], []) + + fx, fy = camera_info.K[0], camera_info.K[4] + cx, cy = camera_info.K[2], camera_info.K[5] + if fx <= 0 or fy <= 0: + raise ValueError("camera focal lengths must be positive") + + u = np.floor(fx * camera_points[:, 0] / camera_points[:, 2] + cx).astype(np.int64) + v = np.floor(fy * camera_points[:, 1] / camera_points[:, 2] + cy).astype(np.int64) + in_image = (u >= 0) & (u < frame.image.width) & (v >= 0) & (v < frame.image.height) + camera_points = camera_points[in_image] + source_indices = source_indices[in_image] + u = u[in_image] + v = v[in_image] + if len(camera_points) == 0: + return ProjectedPoints([], [], []) + + pixel_ids = v * frame.image.width + u + nearest_first = np.lexsort((camera_points[:, 2], pixel_ids)) + first_per_pixel = np.concatenate( + ([True], pixel_ids[nearest_first][1:] != pixel_ids[nearest_first][:-1]) + ) + visible = nearest_first[first_per_pixel] + + return ProjectedPoints( + camera_points=[ + (float(point[0]), float(point[1]), float(point[2])) for point in camera_points[visible] + ], + pixels=[(int(x), int(y)) for x, y in zip(u[visible], v[visible], strict=True)], + source_indices=[int(index) for index in source_indices[visible]], + ) diff --git a/dimos/benchmark/vqa/generation/ground_truth_generator.py b/dimos/benchmark/vqa/generation/ground_truth_generator.py new file mode 100644 index 0000000000..3987d5ee2d --- /dev/null +++ b/dimos/benchmark/vqa/generation/ground_truth_generator.py @@ -0,0 +1,438 @@ +# Copyright 2026 Dimensional Inc. +"""Tool-driven private answer generation for single-frame VQA.""" + +from __future__ import annotations + +from dimos.benchmark.vqa.generation.primitives.choices import ( + CAMERA_RANGE_CHOICES, + COUNT_CHOICES, + camera_range_choice, + count_choice, +) +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives +from dimos.benchmark.vqa.generation.primitives.selection import select_nearest_object +from dimos.benchmark.vqa.generation.questions import generate_questions +from dimos.benchmark.vqa.models import ( + CalibratedFrame, + GroundedObject, + GroundTruthResult, + QuestionIntent, + ToolTrace, + VqaExample, +) + + +class VqaGroundTruthGenerator: + """Answer constrained questions by calling detection, segmentation, and geometry tools.""" + + def __init__(self, primitives: FramePerceptionPrimitives) -> None: + self.primitives = primitives + + def answer(self, frame: CalibratedFrame, intent: QuestionIntent) -> GroundTruthResult: + if intent.kind == "forward_path": + return _classify_forward_path(frame, intent, self.primitives) + objects, trace = self.ground(frame, intent.object_query) + return self._answer_from_objects(frame, intent, objects, trace) + + def ground( + self, frame: CalibratedFrame, object_query: str + ) -> tuple[list[GroundedObject], tuple[ToolTrace, ...]]: + """Run the fixed constrained grounding recipe over shared primitives.""" + if self.primitives.has_grounding(object_query): + return self.primitives.ground_masks(object_query), ( + ToolTrace("reuse_grounding", object_query), + ) + trace: list[ToolTrace] = [ToolTrace("detect_objects", object_query)] + detections = self.primitives.detect_objects(object_query) + if len(detections): + trace.append(ToolTrace("segment_objects", f"count={len(detections)}")) + elif self.primitives.can_localize_points: + trace.append(ToolTrace("locate_object_point", object_query)) + masks = self.primitives.segment_detections(object_query) + if not len(detections) and self.primitives.used_point_localization(object_query): + trace.append(ToolTrace("segment_object_point", object_query)) + trace.append(ToolTrace("get_foreground_geometry", f"masks={len(masks)}")) + objects = self.primitives.ground_masks(object_query) + return objects, tuple(trace) + + def _answer_from_objects( + self, + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + ) -> GroundTruthResult: + if not objects: + rejected = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.kind}", + _render_question(intent), + "", + "", + (), + ) + return GroundTruthResult( + intent, rejected, "rejected", None, "no_grounded_object", (), trace + ) + if intent.kind == "compare_nearest_by_side": + return _compare_nearest_by_side(frame, intent, objects, trace) + if intent.kind == "visible_count": + return _count_visible_objects(frame, intent, objects, trace) + if intent.kind == "camera_range": + return _bucket_camera_range(frame, intent, objects, trace) + if intent.kind == "compare_left_right": + return _compare_left_right(frame, intent, objects, trace, self) + if intent.kind == "compare_height": + return _compare_heights(frame, intent, objects, trace, self) + if intent.kind == "door_state": + return _classify_door_state(frame, intent, objects, trace, self.primitives) + if intent.kind == "closest_object": + return _select_closest_object(frame, intent, objects, trace, self) + examples = generate_questions( + frame.id, objects, [intent.object_query], distance_m=intent.threshold_m or 3.0 + ) + suffix = { + "presence": "presence", + "horizontal_direction": "direction", + "within_distance": "range", + }[intent.kind] + example = next((item for item in examples if item.id.endswith(f"-{suffix}")), None) + if example is not None: + return GroundTruthResult( + intent, + example, + "answered", + example.expected_answer, + None, + tuple(objects), + trace, + ) + rejected = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.kind}", _render_question(intent), "", "", () + ) + return GroundTruthResult( + intent, rejected, "rejected", None, "no_grounded_object", tuple(objects), trace + ) + + +def _render_question(intent: QuestionIntent) -> str: + if intent.kind == "presence": + return f"Is there a {intent.object_query} in the image? Answer yes or no." + if intent.kind == "horizontal_direction": + return f"Where is the nearest {intent.object_query}: left, center, or right?" + if intent.kind == "visible_count": + return f"How many {intent.object_query}s are visible?" + if intent.kind == "camera_range": + return f"How far is the nearest {intent.object_query} from the camera?" + if intent.kind == "compare_nearest_by_side": + return f"Which {intent.object_query} is closer: the left one or the right one?" + if intent.kind == "compare_left_right": + return ( + f"Is the {intent.object_query} to the left or right of the {intent.comparison_query}?" + ) + if intent.kind == "compare_height": + return f"Which is taller: the {intent.object_query} or the {intent.comparison_query}?" + if intent.kind == "door_state": + return f"Is the {intent.object_query} open or closed?" + if intent.kind == "closest_object": + return f"Which object is closest to the {intent.object_query}: {', '.join(intent.candidate_queries)}?" + if intent.kind == "forward_path": + return "Is the path directly ahead clear or blocked?" + return f"Is the nearest {intent.object_query} within {intent.threshold_m or 3:g} meters? Answer yes or no." + + +def _count_visible_objects( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], +) -> GroundTruthResult: + answer = count_choice(len(objects)) + example = VqaExample( + f"{frame.id}-{intent.object_query}-visible-count", + _render_question(intent), + answer, + "choice", + tuple(item.id for item in objects), + COUNT_CHOICES, + ) + return GroundTruthResult(intent, example, "answered", answer, None, tuple(objects), trace) + + +def _bucket_camera_range( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], +) -> GroundTruthResult: + selected = select_nearest_object(objects) + if selected is None: + return _rejected_result(frame, intent, objects, trace, "no_grounded_object") + answer = camera_range_choice(selected.range_m) + example = VqaExample( + f"{frame.id}-{intent.object_query}-camera-range", + _render_question(intent), + answer, + "choice", + (selected.id,), + CAMERA_RANGE_CHOICES, + ) + return GroundTruthResult(intent, example, "answered", answer, None, (selected,), trace) + + +def _compare_heights( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + generator: VqaGroundTruthGenerator, +) -> GroundTruthResult: + if len(objects) != 1 or intent.comparison_query is None: + return _rejected_result(frame, intent, objects, trace, "ambiguous_first_height_object") + other_objects, other_trace = generator.ground(frame, intent.comparison_query) + trace = (*trace, *other_trace) + if len(other_objects) != 1: + return _rejected_result( + frame, intent, [*objects, *other_objects], trace, "ambiguous_second_height_object" + ) + plane_fit = generator.primitives.fit_ground_plane() + trace = (*trace, ToolTrace("fit_ground_plane", plane_fit.rejection_reason or "accepted")) + if plane_fit.estimate is None: + return _rejected_result( + frame, + intent, + [*objects, *other_objects], + trace, + plane_fit.rejection_reason or "ground_plane_rejected", + ) + first = generator.primitives.measure_height(objects[0], plane_fit.estimate) + second = generator.primitives.measure_height(other_objects[0], plane_fit.estimate) + trace = ( + *trace, + ToolTrace("measure_height", first.rejection_reason or objects[0].id), + ToolTrace("measure_height", second.rejection_reason or other_objects[0].id), + ) + if first.measurement is None or second.measurement is None: + return _rejected_result( + frame, + intent, + [*objects, *other_objects], + trace, + first.rejection_reason or second.rejection_reason or "height_measurement_rejected", + ) + first_lower = first.measurement.value - first.measurement.tolerance + second_lower = second.measurement.value - second.measurement.tolerance + first_upper = first.measurement.value + first.measurement.tolerance + second_upper = second.measurement.value + second.measurement.tolerance + if first_lower <= second_upper and second_lower <= first_upper: + return _rejected_result( + frame, intent, [*objects, *other_objects], trace, "ambiguous_height_comparison" + ) + answer = intent.object_query if first_lower > second_upper else intent.comparison_query + example = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.comparison_query}-height-comparison", + _render_question(intent), + answer, + "choice", + (objects[0].id, other_objects[0].id), + (intent.object_query, intent.comparison_query), + ) + return GroundTruthResult( + intent, example, "answered", answer, None, (objects[0], other_objects[0]), trace + ) + + +def _compare_left_right( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + generator: VqaGroundTruthGenerator, +) -> GroundTruthResult: + if len(objects) != 1 or intent.comparison_query is None: + return _rejected_result(frame, intent, objects, trace, "ambiguous_first_relation_object") + other_objects, other_trace = generator.ground(frame, intent.comparison_query) + trace = (*trace, *other_trace) + if len(other_objects) != 1: + return _rejected_result( + frame, intent, [*objects, *other_objects], trace, "ambiguous_second_relation_object" + ) + relation = generator.primitives.classify_horizontal_relation(objects[0], other_objects[0]) + trace = ( + *trace, + ToolTrace( + "classify_horizontal_relation", + relation.relation or relation.rejection_reason or "rejected", + ), + ) + if relation.relation is None: + return _rejected_result( + frame, + intent, + [*objects, *other_objects], + trace, + relation.rejection_reason or "horizontal_relation_rejected", + ) + example = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.comparison_query}-left-right", + _render_question(intent), + relation.relation, + "choice", + (objects[0].id, other_objects[0].id), + ("left", "right"), + ) + return GroundTruthResult( + intent, + example, + "answered", + relation.relation, + None, + (objects[0], other_objects[0]), + trace, + ) + + +def _compare_nearest_by_side( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], +) -> GroundTruthResult: + left = select_nearest_object(objects, "left") + right = select_nearest_object(objects, "right") + if left is None or right is None: + return _rejected_result(frame, intent, objects, trace, "missing_grounded_side") + if left.range_m == right.range_m: + return _rejected_result(frame, intent, objects, trace, "ambiguous_nearest_by_side") + answer = "left" if left.range_m < right.range_m else "right" + example = VqaExample( + f"{frame.id}-{intent.object_query}-nearest-by-side", + _render_question(intent), + answer, + "choice", + (left.id, right.id), + ("left", "right"), + ) + return GroundTruthResult(intent, example, "answered", answer, None, (left, right), trace) + + +def _classify_door_state( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + primitives: FramePerceptionPrimitives, +) -> GroundTruthResult: + if "door" not in intent.object_query.lower(): + return _rejected_result(frame, intent, objects, trace, "door_state_requires_door_query") + if len(objects) != 1: + return _rejected_result(frame, intent, objects, trace, "ambiguous_door_instances") + result = primitives.classify_door_state(objects[0]) + trace = ( + *trace, + ToolTrace("classify_door_state", result.state or result.rejection_reason or "rejected"), + ) + if result.state is None: + return _rejected_result( + frame, intent, objects, trace, result.rejection_reason or "door_state_rejected" + ) + example = VqaExample( + f"{frame.id}-{intent.object_query}-state", + _render_question(intent), + result.state, + "choice", + (objects[0].id,), + ("open", "closed"), + ) + return GroundTruthResult(intent, example, "answered", result.state, None, tuple(objects), trace) + + +def _select_closest_object( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + generator: VqaGroundTruthGenerator, +) -> GroundTruthResult: + if len(objects) != 1: + return _rejected_result(frame, intent, objects, trace, "ambiguous_target_object") + candidates: list[GroundedObject] = [] + for query in intent.candidate_queries: + matches, candidate_trace = generator.ground(frame, query) + trace = (*trace, *candidate_trace) + if len(matches) != 1: + return _rejected_result( + frame, + intent, + [*objects, *candidates, *matches], + trace, + "ambiguous_candidate_object", + ) + candidates.append(matches[0]) + selected = generator.primitives.select_closest_object(objects[0], candidates) + trace = ( + *trace, + ToolTrace( + "select_closest_object", + selected.object.id if selected.object else selected.rejection_reason or "rejected", + ), + ) + if selected.object is None: + return _rejected_result( + frame, + intent, + [*objects, *candidates], + trace, + selected.rejection_reason or "closest_object_rejected", + ) + example = VqaExample( + f"{frame.id}-{intent.object_query}-closest-object", + _render_question(intent), + selected.object.label, + "choice", + (objects[0].id, *(item.id for item in candidates)), + intent.candidate_queries, + ) + return GroundTruthResult( + intent, + example, + "answered", + selected.object.label, + None, + tuple([*objects, *candidates]), + trace, + ) + + +def _classify_forward_path( + frame: CalibratedFrame, intent: QuestionIntent, primitives: FramePerceptionPrimitives +) -> GroundTruthResult: + result = primitives.classify_forward_path() + trace = ( + ToolTrace("classify_forward_path", result.state or result.rejection_reason or "rejected"), + ) + if result.state is None: + return _rejected_result( + frame, intent, [], trace, result.rejection_reason or "forward_path_rejected" + ) + example = VqaExample( + f"{frame.id}-forward-path", + _render_question(intent), + result.state, + "choice", + (), + ("clear", "blocked"), + ) + return GroundTruthResult(intent, example, "answered", result.state, None, (), trace) + + +def _rejected_result( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + reason: str, +) -> GroundTruthResult: + rejected = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.kind}", _render_question(intent), "", "", () + ) + return GroundTruthResult(intent, rejected, "rejected", None, reason, tuple(objects), trace) diff --git a/dimos/benchmark/vqa/generation/grounding.py b/dimos/benchmark/vqa/generation/grounding.py new file mode 100644 index 0000000000..b6b6aa6c61 --- /dev/null +++ b/dimos/benchmark/vqa/generation/grounding.py @@ -0,0 +1,72 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Foreground-mask point-cloud grounding for one calibrated frame.""" + +from __future__ import annotations + +import numpy as np + +from dimos.benchmark.vqa.generation.geometry import project_visible_points +from dimos.benchmark.vqa.models import CalibratedFrame, GroundedObject, ProjectionConfig +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg + + +def ground_segmented_objects( + frame: CalibratedFrame, + detections: list[Detection2DSeg], + *, + min_foreground_points: int = 3, + projection: ProjectionConfig = ProjectionConfig(), +) -> list[GroundedObject]: + """Create object geometry from visible points covered by each foreground mask.""" + if min_foreground_points < 1: + raise ValueError("min_foreground_points must be positive") + + projected = project_visible_points(frame, projection) + grounded: list[GroundedObject] = [] + for index, detection in enumerate(detections): + mask = detection.mask + if mask.shape != (frame.image.height, frame.image.width): + raise ValueError("segmentation mask dimensions must match the image") + selected = [ + point + for point, (x, y) in zip(projected.camera_points, projected.pixels, strict=True) + if mask[y, x] > 0 + ] + if len(selected) < min_foreground_points: + continue + + ranges = np.linalg.norm(np.asarray(selected), axis=1) + image_x = [x for x, y in projected.pixels if mask[y, x] > 0] + median_x = float(np.median(image_x)) + direction = _horizontal_direction(median_x, frame.image.width) + grounded.append( + GroundedObject( + id=f"{frame.id}-{detection.name}-{index}", + label=detection.name, + point_count=len(selected), + range_m=float(np.median(ranges)), + horizontal_direction=direction, + ) + ) + return grounded + + +def _horizontal_direction(x: float, width: int) -> str: + if x < width / 3: + return "left" + if x >= 2 * width / 3: + return "right" + return "center" diff --git a/dimos/benchmark/vqa/generation/oracle.py b/dimos/benchmark/vqa/generation/oracle.py new file mode 100644 index 0000000000..2007082afc --- /dev/null +++ b/dimos/benchmark/vqa/generation/oracle.py @@ -0,0 +1,328 @@ +# Copyright 2026 Dimensional Inc. +"""Private bounded LangChain oracle for generic VQA question proposals.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, replace +import json +import re +from typing import TYPE_CHECKING, Any, Protocol + +from dimos.benchmark.vqa.generation.oracle_tools import LocalOracleToolRegistry +from dimos.benchmark.vqa.models import ( + AcceptedOracleResult, + AnswerContract, + BooleanAnswerContract, + ChoiceAnswerContract, + DeferredHeightChoiceContract, + OracleToolResult, + OracleTrace, + QuestionProposal, + RejectedOracleResult, + ResolvedAnswerContract, +) + +if TYPE_CHECKING: + from langchain_core.language_models.chat_models import BaseChatModel + + +@dataclass(frozen=True) +class SemanticEvidenceValidation: + """Private verdict on whether cited tool evidence supports an oracle answer.""" + + accepted: bool + reason: str + + +class SemanticEvidenceValidator(Protocol): + """Validate an answer only against the frozen question and cited local evidence.""" + + def validate( + self, + proposal: QuestionProposal, + answer: str, + cited_results: tuple[OracleToolResult, ...], + ) -> SemanticEvidenceValidation: ... + + +class OpenAISemanticEvidenceValidator: + """Private no-tools model judge for semantic grounding of a proposed answer.""" + + def __init__(self, model: BaseChatModel) -> None: + self._model = model + + def validate( + self, + proposal: QuestionProposal, + answer: str, + cited_results: tuple[OracleToolResult, ...], + ) -> SemanticEvidenceValidation: + from langchain_core.messages import HumanMessage, SystemMessage + + response = self._model.invoke( + [ + SystemMessage( + "You validate a private VQA oracle answer. Decide only whether the cited " + "structured local-tool evidence supports the answer to the frozen question. " + "Reject claims requiring measurements not present in the evidence; for example, " + "height is not supported by range or side alone. A cited measurement bucket must " + "exactly match the selected choice. Return strict JSON only: " + '{"accepted": true|false, "reason": "concise reason"}. Do not call tools.' + ), + HumanMessage( + json.dumps( + { + "question": proposal.question, + "answer": answer, + "answer_contract": asdict(proposal.answer_contract), + "cited_evidence": [asdict(result) for result in cited_results], + } + ) + ), + ] + ) + try: + payload = _parse_strict_json_object(_response_text(response.content)) + accepted, reason = payload.get("accepted"), payload.get("reason") + if not isinstance(accepted, bool) or not isinstance(reason, str) or not reason: + raise ValueError( + "validator response requires boolean accepted and non-empty reason" + ) + except (ValueError, json.JSONDecodeError, AttributeError) as exc: + return SemanticEvidenceValidation(False, f"invalid_validator_response:{exc}") + return SemanticEvidenceValidation(accepted, reason) + + +class PrivateToolCallingOracle: + """Use direct local tools only, then validate a model's final JSON response.""" + + def __init__( + self, + model: BaseChatModel, + max_tool_calls: int = 8, + semantic_validator: SemanticEvidenceValidator | None = None, + ) -> None: + if max_tool_calls < 1: + raise ValueError("max_tool_calls must be positive") + self._model = model + self._max_tool_calls = max_tool_calls + self._semantic_validator = semantic_validator + + def answer( + self, proposal: QuestionProposal, registry: LocalOracleToolRegistry + ) -> AcceptedOracleResult | RejectedOracleResult: + from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage + + tools = registry.tools() + model = self._model.bind_tools(tools) + messages: list[Any] = [ + SystemMessage( + "You are a private VQA oracle. Use only supplied local tools. Do not invent " + 'evidence. Finish with JSON only: {"answer": value, "evidence_ids": [..]}.' + ), + HumanMessage(_proposal_prompt(proposal)), + ] + trace: list[OracleTrace] = [] + calls = 0 + while calls < self._max_tool_calls: + response = model.invoke(messages) + messages.append(response) + tool_calls = getattr(response, "tool_calls", []) + if not tool_calls: + return _validated_result( + proposal, + _response_text(response.content), + registry.results, + trace, + self._semantic_validator, + ) + for call in tool_calls: + if calls >= self._max_tool_calls: + break + name = call.get("name") + tool = next((item for item in tools if item.name == name), None) + if tool is None: + return _rejected(proposal, "unsupported_tool", registry.results, trace) + try: + output = tool.invoke(call.get("args", {})) + except (TypeError, ValueError) as exc: + return _rejected(proposal, f"tool_error:{exc}", registry.results, trace) + calls += 1 + trace.append(OracleTrace("tool", str(name))) + messages.append(ToolMessage(content=str(output), tool_call_id=call["id"])) + return _rejected(proposal, "tool_call_limit", registry.results, trace) + + +def create_openai_oracle(model: str, max_tool_calls: int = 8) -> PrivateToolCallingOracle: + """Construct the private-only OpenAI tool-calling oracle.""" + from langchain_openai import ChatOpenAI + + return PrivateToolCallingOracle( + ChatOpenAI(model=model), + max_tool_calls=max_tool_calls, + semantic_validator=OpenAISemanticEvidenceValidator(ChatOpenAI(model=model)), + ) + + +def validate_oracle_answer( + proposal: QuestionProposal, + answer: Any, + evidence_ids: Any, + results: tuple[OracleToolResult, ...], +) -> str: + """Deterministically validate an answer format and citations.""" + return _resolve_oracle_answer(proposal, answer, evidence_ids, results)[0] + + +def _resolve_oracle_answer( + proposal: QuestionProposal, + answer: Any, + evidence_ids: Any, + results: tuple[OracleToolResult, ...], +) -> tuple[str, ResolvedAnswerContract]: + """Return a validated answer and the public contract resolved from private evidence.""" + if ( + not isinstance(evidence_ids, list) + or not evidence_ids + or not all(isinstance(item, str) for item in evidence_ids) + ): + raise ValueError("answer requires non-empty evidence_ids") + known_ids = {item.id for result in results for item in result.evidence} + if not set(evidence_ids).issubset(known_ids): + raise ValueError("answer cites unknown evidence") + contract = proposal.answer_contract + if isinstance(contract, BooleanAnswerContract): + if answer not in ("yes", "no"): + raise ValueError("boolean answer must be yes or no") + return str(answer), contract + if isinstance(contract, ChoiceAnswerContract): + if not isinstance(answer, str) or answer not in contract.choices: + raise ValueError("choice answer is not allowed") + cited = set(evidence_ids) + measured_choices = { + result.choice + for result in results + if result.choice is not None and any(item.id in cited for item in result.evidence) + } + if measured_choices and answer not in measured_choices: + raise ValueError("choice answer does not match cited measurement bucket") + return answer, contract + if isinstance(contract, DeferredHeightChoiceContract): + bucket_results = [ + result + for result in results + if result.tool == "bucket_measurement" and result.choice is not None and result.choices + ] + if len(bucket_results) != 1: + raise ValueError("deferred height answer requires exactly one measurement bucket") + bucket = bucket_results[0] + if not any(item.id in evidence_ids for item in bucket.evidence): + raise ValueError("deferred height answer must cite its measurement") + if answer != bucket.choice: + raise ValueError("deferred height answer does not match measurement bucket") + if bucket.choice not in bucket.choices: + raise ValueError("measurement bucket choice is not public") + return bucket.choice, ChoiceAnswerContract(bucket.choices) + raise ValueError("unsupported answer contract") + + +def _validated_result( + proposal: QuestionProposal, + response: str, + results: tuple[OracleToolResult, ...], + trace: list[OracleTrace], + semantic_validator: SemanticEvidenceValidator | None, +) -> AcceptedOracleResult | RejectedOracleResult: + try: + payload = _parse_json_object(response) + answer, answer_contract = _resolve_oracle_answer( + proposal, payload.get("answer"), payload.get("evidence_ids"), results + ) + evidence_ids = tuple(payload["evidence_ids"]) + except (ValueError, json.JSONDecodeError, AttributeError) as exc: + return _rejected(proposal, f"invalid_final_answer:{exc}", results, trace) + cited_results = _cited_results(evidence_ids, results) + if semantic_validator is None: + return _rejected(proposal, "semantic_validator_not_configured", results, trace) + resolved_proposal = replace(proposal, answer_contract=answer_contract) + verdict = semantic_validator.validate(resolved_proposal, answer, cited_results) + trace.append( + OracleTrace( + "semantic_validation", + f"{'accepted' if verdict.accepted else 'rejected'}:{verdict.reason}", + ) + ) + if not verdict.accepted: + return _rejected(proposal, f"unsupported_evidence:{verdict.reason}", results, trace) + return AcceptedOracleResult( + proposal, answer, answer_contract, evidence_ids, results, tuple(trace) + ) + + +def _cited_results( + evidence_ids: tuple[str, ...], results: tuple[OracleToolResult, ...] +) -> tuple[OracleToolResult, ...]: + cited = set(evidence_ids) + return tuple( + replace( + result, + evidence=tuple(evidence for evidence in result.evidence if evidence.id in cited), + ) + for result in results + if any(evidence.id in cited for evidence in result.evidence) + ) + + +def _rejected( + proposal: QuestionProposal, + reason: str, + results: tuple[OracleToolResult, ...], + trace: list[OracleTrace], +) -> RejectedOracleResult: + return RejectedOracleResult(proposal, reason, results, tuple(trace)) + + +def _proposal_prompt(proposal: QuestionProposal) -> str: + return ( + f"Question: {proposal.question}\nAnswer contract: {_contract_prompt(proposal.answer_contract)}\n" + f"Suggested object queries: {', '.join(proposal.object_queries) or 'none'}" + ) + + +def _contract_prompt(contract: AnswerContract) -> str: + if isinstance(contract, BooleanAnswerContract): + return "boolean: yes or no" + if isinstance(contract, ChoiceAnswerContract): + return f"choice: {', '.join(contract.choices)}" + if isinstance(contract, DeferredHeightChoiceContract): + return ( + "deferred height choice: call measure_height, then bucket_measurement, and return " + "the exact choice from that result" + ) + raise ValueError("unsupported answer contract") + + +def _parse_json_object(response: str) -> dict[str, Any]: + stripped = re.sub(r"^```(?:json)?\s*|\s*```$", "", response.strip(), flags=re.IGNORECASE) + start, end = stripped.find("{"), stripped.rfind("}") + if start < 0 or end < start: + raise json.JSONDecodeError("expected JSON object", stripped, 0) + payload: Any = json.loads(stripped[start : end + 1]) + if not isinstance(payload, dict): + raise ValueError("final response must be an object") + return payload + + +def _parse_strict_json_object(response: str) -> dict[str, Any]: + payload: Any = json.loads(response) + if not isinstance(payload, dict): + raise ValueError("validator response must be an object") + return payload + + +def _response_text(content: Any) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + return "".join(str(item.get("text", "")) for item in content if isinstance(item, dict)) + return str(content) diff --git a/dimos/benchmark/vqa/generation/oracle_tools.py b/dimos/benchmark/vqa/generation/oracle_tools.py new file mode 100644 index 0000000000..73cca5958d --- /dev/null +++ b/dimos/benchmark/vqa/generation/oracle_tools.py @@ -0,0 +1,632 @@ +# Copyright 2026 Dimensional Inc. +"""Typed private perception primitives over one frozen VQA frame.""" + +from __future__ import annotations + +import json +from typing import Any + +from langchain_core.tools import StructuredTool + +from dimos.benchmark.vqa.generation.primitives.choices import ( + CAMERA_RANGE_CHOICES, + COUNT_CHOICES, + camera_range_choice, + count_choice, + height_choice_window, +) +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives +from dimos.benchmark.vqa.generation.primitives.selection import select_nearest_object +from dimos.benchmark.vqa.models import ( + GroundedObject, + GroundPlaneEstimate, + OracleEvidence, + OracleMeasurement, + OracleToolResult, +) + + +class LocalOracleToolRegistry: + """Expose the same private perception primitives used by constrained recipes.""" + + def __init__(self, primitives: FramePerceptionPrimitives) -> None: + self._primitives = primitives + self._results: list[OracleToolResult] = [] + self._detections: dict[str, str] = {} + self._masks: dict[str, str] = {} + self._objects: dict[str, GroundedObject] = {} + self._planes: dict[str, GroundPlaneEstimate] = {} + self._measurements: dict[str, OracleToolResult] = {} + self._next_id = 0 + + @property + def results(self) -> tuple[OracleToolResult, ...]: + return tuple(self._results) + + def tools(self) -> list[StructuredTool]: + return [ + StructuredTool.from_function( + self.detect_objects, + name="detect_objects", + description="Run private MoonDream detection for one visible semantic query.", + ), + StructuredTool.from_function( + self.segment_detections, + name="segment_detections", + description="Run private EdgeTAM segmentation for one opaque detection ID.", + ), + StructuredTool.from_function( + self.ground_masks, + name="ground_masks", + description=( + "Project visible calibrated point-cloud support through one opaque mask ID. " + "Returns grounded object IDs and citable evidence." + ), + ), + StructuredTool.from_function( + self.select_nearest_object, + name="select_nearest_object", + description=( + "Select the nearest opaque grounded object ID, optionally restricted to left, center, " + "or right." + ), + ), + StructuredTool.from_function( + self.count_grounded_objects, + name="count_grounded_objects", + description="Count opaque grounded object IDs into fixed public count buckets.", + ), + StructuredTool.from_function( + self.bucket_camera_range, + name="bucket_camera_range", + description="Bucket one opaque object's private camera-origin range into fixed public choices.", + ), + StructuredTool.from_function( + self.compare_nearest_by_side, + name="compare_nearest_by_side", + description="Choose whether the nearest opaque left or right object is closer to the camera.", + ), + StructuredTool.from_function( + self.compare_left_right, + name="compare_left_right", + description=( + "Choose whether one opaque object is left or right of another from private " + "camera-frame support centroids. Rejects ambiguous separation." + ), + ), + StructuredTool.from_function( + self.select_closest_object, + name="select_closest_object", + description=( + "Select which opaque candidate object is closest to one opaque target object " + "by private point-cloud support. Rejects ambiguous proximity." + ), + ), + StructuredTool.from_function( + self.fit_ground_plane, + name="fit_ground_plane", + description="Fit a quality-gated Open3D ground plane to the frozen visible point cloud.", + ), + StructuredTool.from_function( + self.measure_height, + name="measure_height", + description=( + "Measure one opaque grounded object above one opaque accepted ground-plane ID." + ), + ), + StructuredTool.from_function( + self.compare_heights, + name="compare_heights", + description=( + "Compare two opaque grounded objects against one opaque accepted ground-plane ID. " + "Rejects overlapping physical-height uncertainty." + ), + ), + StructuredTool.from_function( + self.classify_door_state, + name="classify_door_state", + description=( + "Classify one opaque grounded door as open or closed from point-cloud planes. " + "Rejects insufficient or ambiguous geometry." + ), + ), + StructuredTool.from_function( + self.classify_forward_path, + name="classify_forward_path", + description=( + "Classify the visible camera-forward corridor as clear or blocked from point-cloud " + "ground and obstacle support. Rejects incomplete or ambiguous visibility." + ), + ), + StructuredTool.from_function( + self.bucket_measurement, + name="bucket_measurement", + description=( + "Map one opaque height measurement ID to four public, answer-conditioned " + "height choices and the one matching choice." + ), + ), + ] + + def detect_objects(self, query: str) -> str: + """Detect objects and return an opaque ID for a later segmentation call.""" + detections = self._primitives.detect_objects(query) + detection_id = self._id("detection") + self._detections[detection_id] = query + result = OracleToolResult("detect_objects", query, ()) + self._results.append(result) + boxes = [list(item.bbox) for item in detections] + return json.dumps(_tool_payload(result, detection_id=detection_id, boxes=boxes)) + + def segment_detections(self, detection_id: str) -> str: + """Segment one earlier opaque detection result and return an opaque mask ID.""" + query = self._detections.get(detection_id) + if query is None: + return self._record_rejection("segment_detections", "", [], "unknown_detection_id") + masks = self._primitives.segment_detections(query) + mask_id = self._id("mask") + self._masks[mask_id] = query + result = OracleToolResult("segment_detections", query, ()) + self._results.append(result) + return json.dumps(_tool_payload(result, mask_id=mask_id, mask_count=len(masks))) + + def ground_masks(self, mask_id: str) -> str: + """Ground one earlier opaque mask result against the visible point cloud.""" + query = self._masks.get(mask_id) + if query is None: + return self._record_rejection("ground_masks", "", [], "unknown_mask_id") + objects = self._primitives.ground_masks(query) + evidence = tuple(_grounding_evidence(item) for item in objects) + for item in objects: + self._objects[item.id] = item + result = OracleToolResult("ground_masks", query, evidence) + self._results.append(result) + return json.dumps(_tool_payload(result, object_ids=[item.id for item in objects])) + + def fit_ground_plane(self) -> str: + """Fit the private ground plane and return an opaque ID for measurements.""" + fit = self._primitives.fit_ground_plane() + if fit.estimate is None: + return self._record_rejection( + "fit_ground_plane", "", list(fit.quality_flags), fit.rejection_reason + ) + plane_id = self._id("plane") + self._planes[plane_id] = fit.estimate + measurement = OracleMeasurement( + fit.estimate.offset_m, + "m", + max(fit.estimate.residual_m, 0.01), + fit.quality_flags, + (f"frame:{self._primitives.frame.id}",), + ) + evidence = OracleEvidence( + f"ground-plane:v1:{self._primitives.frame.id}", + "v1", + "ground-plane", + "ground", + 0.0, + "n/a", + fit.estimate.inlier_count, + measurement, + ) + result = OracleToolResult( + "fit_ground_plane", + "", + (evidence,), + measurement=measurement, + plane=fit.estimate, + quality_flags=fit.quality_flags, + ) + self._results.append(result) + return json.dumps(_tool_payload(result, plane_id=plane_id)) + + def select_nearest_object(self, object_ids: list[str], side: str | None = None) -> str: + """Select one opaque grounded object by private point-cloud range.""" + selected: list[GroundedObject] = [] + for object_id in object_ids: + item = self._objects.get(object_id) + if item is None: + return self._record_rejection( + "select_nearest_object", object_id, [], "unknown_object_id" + ) + selected.append(item) + nearest = select_nearest_object(selected, side) + if nearest is None: + return self._record_rejection("select_nearest_object", "", [], "no_object_matches_side") + result = OracleToolResult( + "select_nearest_object", nearest.label, (_grounding_evidence(nearest),) + ) + self._results.append(result) + return json.dumps(_tool_payload(result, object_id=nearest.id)) + + def count_grounded_objects(self, object_ids: list[str]) -> str: + """Count unique grounded object IDs into fixed public count choices.""" + objects = self._lookup_objects("count_grounded_objects", object_ids) + if objects is None: + return self._record_rejection( + "count_grounded_objects", "", [], "unknown_or_duplicate_object_id" + ) + if not objects: + return self._record_rejection("count_grounded_objects", "", [], "no_grounded_object") + result = OracleToolResult( + "count_grounded_objects", + objects[0].label, + tuple(_grounding_evidence(item) for item in objects), + choice=count_choice(len(objects)), + choices=COUNT_CHOICES, + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def bucket_camera_range(self, object_id: str) -> str: + """Map one grounded object's camera-origin range into fixed public choices.""" + object = self._objects.get(object_id) + if object is None: + return self._record_rejection("bucket_camera_range", object_id, [], "unknown_object_id") + measurement = OracleMeasurement( + object.range_m, + "m", + 0.0, + ("camera_origin_euclidean_range",), + (f"grounding:v1:{object.id}",), + ) + result = OracleToolResult( + "bucket_camera_range", + object.label, + (_grounding_evidence(object),), + measurement=measurement, + choice=camera_range_choice(object.range_m), + choices=CAMERA_RANGE_CHOICES, + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def compare_nearest_by_side(self, object_ids: list[str]) -> str: + """Compare the nearest left and right grounded objects by camera range.""" + objects = self._lookup_objects("compare_nearest_by_side", object_ids) + if objects is None: + return self._record_rejection( + "compare_nearest_by_side", "", [], "unknown_or_duplicate_object_id" + ) + left = select_nearest_object(objects, "left") + right = select_nearest_object(objects, "right") + if left is None or right is None: + return self._record_rejection( + "compare_nearest_by_side", "", [], "missing_grounded_side" + ) + if left.range_m == right.range_m: + return self._record_rejection( + "compare_nearest_by_side", "", [], "ambiguous_nearest_by_side" + ) + choice = "left" if left.range_m < right.range_m else "right" + result = OracleToolResult( + "compare_nearest_by_side", + left.label, + (_grounding_evidence(left), _grounding_evidence(right)), + choice=choice, + choices=("left", "right"), + ) + self._results.append(result) + return json.dumps(_tool_payload(result, left_object_id=left.id, right_object_id=right.id)) + + def compare_left_right(self, first_object_id: str, second_object_id: str) -> str: + """Classify one grounded object's left/right relation to another grounded object.""" + first = self._objects.get(first_object_id) + second = self._objects.get(second_object_id) + if first is None or second is None: + return self._record_rejection("compare_left_right", "", [], "unknown_object_id") + relation = self._primitives.classify_horizontal_relation(first, second) + if relation.relation is None: + return self._record_rejection( + "compare_left_right", + f"{first.label},{second.label}", + list(relation.quality_flags), + relation.rejection_reason, + ) + result = OracleToolResult( + "compare_left_right", + f"{first.label},{second.label}", + (_grounding_evidence(first), _grounding_evidence(second)), + choice=relation.relation, + choices=("left", "right"), + quality_flags=relation.quality_flags, + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def select_closest_object(self, target_id: str, candidate_ids: list[str]) -> str: + """Select one candidate closest to a target by private 3D support-point proximity.""" + target = self._objects.get(target_id) + if target is None: + return self._record_rejection( + "select_closest_object", target_id, [], "unknown_target_id" + ) + candidates: list[GroundedObject] = [] + for candidate_id in candidate_ids: + candidate = self._objects.get(candidate_id) + if candidate is None: + return self._record_rejection( + "select_closest_object", candidate_id, [], "unknown_candidate_id" + ) + candidates.append(candidate) + selected = self._primitives.select_closest_object(target, candidates) + if selected.object is None: + return self._record_rejection( + "select_closest_object", + target.label, + list(selected.quality_flags), + selected.rejection_reason, + ) + result = OracleToolResult( + "select_closest_object", + target.label, + (_grounding_evidence(target), _grounding_evidence(selected.object)), + choice=selected.object.label, + quality_flags=selected.quality_flags, + ) + self._results.append(result) + return json.dumps(_tool_payload(result, object_id=selected.object.id)) + + def measure_height(self, object_id: str, plane_id: str) -> str: + """Measure one grounded object against one previously accepted plane.""" + object = self._objects.get(object_id) + plane = self._planes.get(plane_id) + if object is None: + return self._record_rejection("measure_height", object_id, [], "unknown_object_id") + if plane is None: + return self._record_rejection("measure_height", object_id, [], "unknown_plane_id") + measured = self._primitives.measure_height(object, plane) + if measured.measurement is None: + return self._record_rejection( + "measure_height", + object.label, + list(measured.quality_flags), + measured.rejection_reason, + ) + measurement = measured.measurement + evidence = _height_evidence(object, measurement) + result = OracleToolResult( + "measure_height", + object.label, + (evidence,), + measurement=measurement, + plane=plane, + quality_flags=measured.quality_flags, + ) + measurement_id = self._id("measurement") + self._measurements[measurement_id] = result + self._results.append(result) + return json.dumps(_tool_payload(result, measurement_id=measurement_id)) + + def compare_heights(self, first_object_id: str, second_object_id: str, plane_id: str) -> str: + """Choose the taller object only when one shared-plane measurement is unambiguous.""" + first = self._objects.get(first_object_id) + second = self._objects.get(second_object_id) + plane = self._planes.get(plane_id) + if first is None or second is None: + return self._record_rejection("compare_heights", "", [], "unknown_object_id") + if first.id == second.id: + return self._record_rejection("compare_heights", first.label, [], "duplicate_object_id") + if plane is None: + return self._record_rejection("compare_heights", "", [], "unknown_plane_id") + first_height = self._primitives.measure_height(first, plane) + second_height = self._primitives.measure_height(second, plane) + if first_height.measurement is None or second_height.measurement is None: + return self._record_rejection( + "compare_heights", + "", + [*first_height.quality_flags, *second_height.quality_flags], + first_height.rejection_reason + or second_height.rejection_reason + or "height_measurement_rejected", + ) + first_measurement = first_height.measurement + second_measurement = second_height.measurement + first_lower = first_measurement.value - first_measurement.tolerance + second_lower = second_measurement.value - second_measurement.tolerance + first_upper = first_measurement.value + first_measurement.tolerance + second_upper = second_measurement.value + second_measurement.tolerance + if first_lower <= second_upper and second_lower <= first_upper: + return self._record_rejection("compare_heights", "", [], "ambiguous_height_comparison") + choice = first.label if first_lower > second_upper else second.label + result = OracleToolResult( + "compare_heights", + f"{first.label},{second.label}", + ( + _height_evidence(first, first_measurement), + _height_evidence(second, second_measurement), + ), + choice=choice, + choices=(first.label, second.label), + plane=plane, + quality_flags=(*first_height.quality_flags, *second_height.quality_flags), + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def classify_door_state(self, object_id: str) -> str: + """Classify one grounded door against the surrounding point-cloud plane.""" + object = self._objects.get(object_id) + if object is None: + return self._record_rejection("classify_door_state", object_id, [], "unknown_object_id") + if "door" not in object.label.lower(): + return self._record_rejection( + "classify_door_state", object.label, [], "door_state_requires_door_query" + ) + result = self._primitives.classify_door_state(object) + if result.state is None: + return self._record_rejection( + "classify_door_state", + object.label, + list(result.quality_flags), + result.rejection_reason, + ) + evidence = OracleEvidence( + f"door-state:v1:{object.id}", + "v1", + object.id, + object.label, + object.range_m, + object.horizontal_direction, + object.point_count, + ) + tool_result = OracleToolResult( + "classify_door_state", + object.label, + (evidence,), + choice=result.state, + quality_flags=result.quality_flags, + ) + self._results.append(tool_result) + return json.dumps(_tool_payload(tool_result)) + + def classify_forward_path(self) -> str: + """Classify the observed local corridor directly ahead of the camera.""" + result = self._primitives.classify_forward_path() + if result.state is None: + return self._record_rejection( + "classify_forward_path", + "", + list(result.quality_flags), + result.rejection_reason, + ) + evidence = OracleEvidence( + f"forward-path:v1:{self._primitives.frame.id}", + "v1", + "forward-path", + "forward path", + 0.0, + "center", + result.point_count, + ) + tool_result = OracleToolResult( + "classify_forward_path", + "forward path", + (evidence,), + choice=result.state, + quality_flags=result.quality_flags, + ) + self._results.append(tool_result) + return json.dumps(_tool_payload(tool_result)) + + def bucket_measurement(self, measurement_id: str) -> str: + """Map one accepted private height measurement to its public choice.""" + source = self._measurements.get(measurement_id) + if source is None or source.measurement is None: + return self._record_rejection("bucket_measurement", "", [], "unknown_measurement_id") + choices, choice = height_choice_window(source.measurement.value) + result = OracleToolResult( + "bucket_measurement", + source.query, + source.evidence, + measurement=source.measurement, + choice=choice, + choices=choices, + plane=source.plane, + quality_flags=source.quality_flags, + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def _record_rejection(self, tool: str, query: str, flags: list[str], reason: str | None) -> str: + result = OracleToolResult( + tool, query, (), quality_flags=tuple(flags), rejection_reason=reason + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def _lookup_objects(self, tool: str, object_ids: list[str]) -> list[GroundedObject] | None: + if len(set(object_ids)) != len(object_ids): + return None + objects: list[GroundedObject] = [] + for object_id in object_ids: + object = self._objects.get(object_id) + if object is None: + return None + objects.append(object) + return objects + + def _id(self, kind: str) -> str: + self._next_id += 1 + return f"{kind}:v1:{self._next_id:04d}" + + +def _grounding_evidence(item: GroundedObject) -> OracleEvidence: + return OracleEvidence( + f"grounding:v1:{item.id}", + "v1", + item.id, + item.label, + item.range_m, + item.horizontal_direction, + item.point_count, + ) + + +def _height_evidence(item: GroundedObject, measurement: OracleMeasurement) -> OracleEvidence: + return OracleEvidence( + f"height:v1:{item.id}", + "v1", + item.id, + item.label, + item.range_m, + item.horizontal_direction, + item.point_count, + measurement, + ) + + +def _tool_payload(result: OracleToolResult, **identifiers: Any) -> dict[str, Any]: + return { + "tool": result.tool, + "query": result.query, + "version": result.version, + "measurement": ( + { + "value": result.measurement.value, + "unit": result.measurement.unit, + "tolerance": result.measurement.tolerance, + "quality_flags": result.measurement.quality_flags, + "provenance_ids": result.measurement.provenance_ids, + } + if result.measurement is not None + else None + ), + "choice": result.choice, + "choices": result.choices, + "quality_flags": result.quality_flags, + "rejection_reason": result.rejection_reason, + "plane": ( + { + "normal": result.plane.normal, + "offset_m": result.plane.offset_m, + "sample_count": result.plane.sample_count, + "inlier_count": result.plane.inlier_count, + "residual_m": result.plane.residual_m, + } + if result.plane is not None + else None + ), + "objects": [_evidence_payload(item) for item in result.evidence], + **identifiers, + } + + +def _evidence_payload(item: OracleEvidence) -> dict[str, Any]: + payload: dict[str, Any] = { + "evidence_id": item.id, + "id": item.object_id, + "label": item.label, + "range_m": item.range_m, + "side": item.side, + "point_count": item.point_count, + } + if item.measurement is not None: + payload["measurement"] = { + "value": item.measurement.value, + "unit": item.measurement.unit, + "tolerance": item.measurement.tolerance, + "quality_flags": item.measurement.quality_flags, + "provenance_ids": item.measurement.provenance_ids, + } + return payload diff --git a/dimos/benchmark/vqa/generation/pipeline.py b/dimos/benchmark/vqa/generation/pipeline.py new file mode 100644 index 0000000000..19476557f6 --- /dev/null +++ b/dimos/benchmark/vqa/generation/pipeline.py @@ -0,0 +1,43 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Single-frame VQA private ground-truth generation.""" + +from __future__ import annotations + +from dimos.benchmark.vqa.generation.grounding import ground_segmented_objects +from dimos.benchmark.vqa.generation.questions import generate_questions +from dimos.benchmark.vqa.models import ( + CalibratedFrame, + ObjectDetector, + ObjectSegmenter, + VqaExample, +) +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg + + +def generate_ground_truth( + frame: CalibratedFrame, + queries: list[str], + detector: ObjectDetector, + segmenter: ObjectSegmenter, +) -> list[VqaExample]: + """Generate VQA examples from MoonDream/EdgeTAM-style image models and LiDAR.""" + segmented: list[Detection2DSeg] = [] + for query in queries: + detections = detector.detect(frame.image, query) + result = segmenter.segment(detections) + segmented.extend(detection for detection in result if isinstance(detection, Detection2DSeg)) + objects = ground_segmented_objects(frame, segmented) + return generate_questions(frame.id, objects, queries) diff --git a/dimos/benchmark/vqa/generation/primitives/choices.py b/dimos/benchmark/vqa/generation/primitives/choices.py new file mode 100644 index 0000000000..af45ca20c8 --- /dev/null +++ b/dimos/benchmark/vqa/generation/primitives/choices.py @@ -0,0 +1,52 @@ +"""Deterministic public-choice resolution from private measurements.""" + +from __future__ import annotations + +from bisect import bisect_right + +COUNT_CHOICES = ("1-2", "3-4", "5-7", "8+") +CAMERA_RANGE_CHOICES = ("under 1 m", "1 to under 2 m", "2 to under 4 m", "4 m or more") + + +def count_choice(count: int) -> str: + """Return the public count bucket for one or more grounded instances.""" + if count < 1: + raise ValueError("count must be positive") + if count <= 2: + return COUNT_CHOICES[0] + if count <= 4: + return COUNT_CHOICES[1] + if count <= 7: + return COUNT_CHOICES[2] + return COUNT_CHOICES[3] + + +def camera_range_choice(range_m: float) -> str: + """Return the public camera-origin range bucket for a grounded object.""" + if range_m < 0: + raise ValueError("range must be non-negative") + if range_m < 1.0: + return CAMERA_RANGE_CHOICES[0] + if range_m < 2.0: + return CAMERA_RANGE_CHOICES[1] + if range_m < 4.0: + return CAMERA_RANGE_CHOICES[2] + return CAMERA_RANGE_CHOICES[3] + + +def height_choice_window(height_m: float) -> tuple[tuple[str, ...], str]: + """Generate a local, exhaustive four-choice window around a private height.""" + breakpoints = (0.1, 0.2, 0.6, 1.0, 2.0) + start = min(max(bisect_right(breakpoints, height_m) - 1, 0), len(breakpoints) - 3) + lower, middle, upper = breakpoints[start : start + 3] + choices = ( + f"under {_format_height(lower)} m", + f"{_format_height(lower)}-{_format_height(middle)} m", + f"{_format_height(middle)}-{_format_height(upper)} m", + f"over {_format_height(upper)} m", + ) + return choices, choices[bisect_right((lower, middle, upper), height_m)] + + +def _format_height(value: float) -> str: + return f"{value:.1f}" diff --git a/dimos/benchmark/vqa/generation/primitives/contracts.py b/dimos/benchmark/vqa/generation/primitives/contracts.py new file mode 100644 index 0000000000..e27e6b027f --- /dev/null +++ b/dimos/benchmark/vqa/generation/primitives/contracts.py @@ -0,0 +1,59 @@ +"""Typed results returned by frame-scoped private perception primitives.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +from dimos.benchmark.vqa.models import GroundedObject, GroundPlaneEstimate, OracleMeasurement + + +@dataclass(frozen=True) +class HeightMeasurementResult: + """A private object-height measurement or its explicit rejection.""" + + object: GroundedObject + plane: GroundPlaneEstimate + measurement: OracleMeasurement | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + + +@dataclass(frozen=True) +class DoorStateResult: + """A conservative point-cloud classification of one door's state.""" + + object: GroundedObject + state: Literal["open", "closed"] | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + angle_deg: float | None = None + + +@dataclass(frozen=True) +class ClosestObjectResult: + """A point-cloud selected candidate nearest to one grounded target object.""" + + object: GroundedObject | None + distance_m: float | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + + +@dataclass(frozen=True) +class HorizontalRelationResult: + """A pairwise camera-frame horizontal relation or its explicit rejection.""" + + relation: Literal["left", "right"] | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + + +@dataclass(frozen=True) +class ForwardPathResult: + """A conservative visible-corridor classification from point-cloud evidence.""" + + state: Literal["clear", "blocked"] | None + point_count: int + quality_flags: tuple[str, ...] + rejection_reason: str | None = None diff --git a/dimos/benchmark/vqa/generation/primitives/frame.py b/dimos/benchmark/vqa/generation/primitives/frame.py new file mode 100644 index 0000000000..80009f7281 --- /dev/null +++ b/dimos/benchmark/vqa/generation/primitives/frame.py @@ -0,0 +1,276 @@ +"""Frame-scoped private perception primitives shared by VQA generation modes.""" + +from __future__ import annotations + +import numpy as np + +from dimos.benchmark.vqa.generation.geometry import project_visible_points +from dimos.benchmark.vqa.generation.grounding import ground_segmented_objects +from dimos.benchmark.vqa.generation.primitives.contracts import ( + ClosestObjectResult, + DoorStateResult, + ForwardPathResult, + HeightMeasurementResult, + HorizontalRelationResult, +) +from dimos.benchmark.vqa.generation.primitives.geometry import ( + PlaneFitResult, + classify_door_plane_angle, + classify_forward_corridor, + estimate_ground_plane, + fit_surface_plane, + points_around_mask, + points_in_mask, +) +from dimos.benchmark.vqa.models import ( + CalibratedFrame, + GroundedObject, + GroundingConfig, + GroundPlaneEstimate, + ObjectDetector, + ObjectPointLocalizer, + ObjectSegmenter, + OracleMeasurement, + PointObjectSegmenter, +) +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D +from dimos.perception.detection.type.detection2d.point import Detection2DPoint +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg + + +class FramePerceptionPrimitives: + """Cached private perception and geometry operations over one frozen frame.""" + + def __init__( + self, + frame: CalibratedFrame, + detector: ObjectDetector, + segmenter: ObjectSegmenter, + localizer: ObjectPointLocalizer | None = None, + point_segmenter: PointObjectSegmenter | None = None, + config: GroundingConfig = GroundingConfig(), + ) -> None: + if config.min_mask_area_px < 1 or config.min_foreground_points < 1: + raise ValueError("grounding thresholds must be positive") + self.frame = frame + self._detector = detector + self._segmenter = segmenter + self._localizer = localizer + self._point_segmenter = point_segmenter + self._config = config + self._detected: dict[str, ImageDetections2D] = {} + self._segmented_queries: set[str] = set() + self._masks: dict[str, list[Detection2DSeg]] = {} + self._points: dict[str, list[Detection2DPoint]] = {} + self._groundings: dict[str, list[GroundedObject]] = {} + self._object_masks: dict[str, Detection2DSeg] = {} + self._plane_fit: PlaneFitResult | None = None + + def detect_objects(self, query: str) -> ImageDetections2D: + """Run private object detection once for a semantic query.""" + cached = self._detected.get(query) + if cached is not None: + return cached + detections = self._detector.detect(self.frame.image, query) + self._detected[query] = detections + return detections + + def segment_detections(self, query: str) -> list[Detection2DSeg]: + """Segment accepted detections, falling back to point localization when available.""" + if query in self._segmented_queries: + return self._masks.get(query, []) + detections = self.detect_objects(query) + if len(detections): + segmented = self._segmenter.segment(detections) + elif self._localizer is not None and self._point_segmenter is not None: + points = self._localizer.locate(self.frame.image, query) + self._points[query] = [item for item in points if isinstance(item, Detection2DPoint)] + segmented = self._point_segmenter.segment_points(points) + else: + segmented = detections + masks = [ + item + for item in segmented + if isinstance(item, Detection2DSeg) + and int((item.mask > 0).sum()) >= self._config.min_mask_area_px + ] + self._masks[query] = masks + self._segmented_queries.add(query) + return masks + + def ground_masks(self, query: str) -> list[GroundedObject]: + """Project visible point-cloud support through accepted masks.""" + cached = self._groundings.get(query) + if cached is not None: + return cached + masks = self.segment_detections(query) + objects = ground_segmented_objects( + self.frame, masks, min_foreground_points=self._config.min_foreground_points + ) + for item in objects: + index = _object_mask_index(item) + if index < len(masks): + self._object_masks[item.id] = masks[index] + self._groundings[query] = objects + return objects + + def used_point_localization(self, query: str) -> bool: + """Return whether segmentation for a query used positive-point localization.""" + return query in self._points + + def has_grounding(self, query: str) -> bool: + """Return whether a query has already been grounded for this frame.""" + return query in self._groundings + + @property + def can_localize_points(self) -> bool: + """Return whether point localization can supplement empty detections.""" + return self._localizer is not None and self._point_segmenter is not None + + def fit_ground_plane(self) -> PlaneFitResult: + """Fit and cache one quality-gated ground plane for the frozen frame.""" + if self._plane_fit is None: + self._plane_fit = estimate_ground_plane(self.frame) + return self._plane_fit + + def measure_height( + self, object: GroundedObject, plane: GroundPlaneEstimate + ) -> HeightMeasurementResult: + """Measure one grounded object's visible height above an accepted plane.""" + mask = self._object_masks.get(object.id) + if mask is None: + raise ValueError(f"unknown grounded object: {object.id}") + selected = points_in_mask(self.frame, mask.mask) + flags = ["visible_point_cloud_height"] + if len(selected) < 6: + flags.append("sparse_object_point_support") + return HeightMeasurementResult( + object, plane, None, tuple(flags), "insufficient_object_support" + ) + normal = np.asarray(plane.normal) + distances = selected @ normal + plane.offset_m + positive = distances[distances > 0.02] + if len(positive) < 4 or len(positive) / len(selected) < 0.6: + flags.append("partial_or_non_elevated_object_support") + return HeightMeasurementResult( + object, plane, None, tuple(flags), "ambiguous_object_extent" + ) + flags.append("conservative_upper_percentile") + measurement = OracleMeasurement( + float(np.percentile(positive, 85)), + "m", + float(max(0.05, plane.residual_m + np.std(positive) * 0.25)), + tuple(flags), + ( + f"frame:{self.frame.id}", + f"ground-plane:v1:{self.frame.id}", + f"grounding:v1:{object.id}", + ), + ) + return HeightMeasurementResult(object, plane, measurement, tuple(flags)) + + def classify_door_state(self, object: GroundedObject) -> DoorStateResult: + """Classify a door as open or closed from its plane relative to nearby structure.""" + mask = self._object_masks.get(object.id) + if mask is None: + raise ValueError(f"unknown grounded object: {object.id}") + door_fit = fit_surface_plane(points_in_mask(self.frame, mask.mask)) + if door_fit.estimate is None: + return DoorStateResult( + object, + None, + ("door_plane_rejected", *door_fit.quality_flags), + door_fit.rejection_reason, + ) + surrounding_fit = fit_surface_plane(points_around_mask(self.frame, mask.mask)) + if surrounding_fit.estimate is None: + return DoorStateResult( + object, + None, + ("surrounding_plane_rejected", *surrounding_fit.quality_flags), + surrounding_fit.rejection_reason, + ) + state, reason, angle_deg = classify_door_plane_angle( + door_fit.estimate, surrounding_fit.estimate + ) + return DoorStateResult( + object, + state, + ( + "door_and_surrounding_planes_accepted", + *door_fit.quality_flags, + *surrounding_fit.quality_flags, + ), + reason, + angle_deg, + ) + + def select_closest_object( + self, target: GroundedObject, candidates: list[GroundedObject] + ) -> ClosestObjectResult: + """Select the unambiguously closest candidate by private support-point centroids.""" + if not candidates: + return ClosestObjectResult(None, None, (), "no_candidate_objects") + if any(item.id == target.id for item in candidates): + return ClosestObjectResult(None, None, (), "target_cannot_be_candidate") + target_points = self._object_points(target) + if target_points is None: + return ClosestObjectResult(None, None, (), "insufficient_target_support") + target_center = np.median(target_points, axis=0) + distances: list[tuple[float, GroundedObject]] = [] + for candidate in candidates: + candidate_points = self._object_points(candidate) + if candidate_points is None: + return ClosestObjectResult(None, None, (), "insufficient_candidate_support") + distance = float(np.linalg.norm(np.median(candidate_points, axis=0) - target_center)) + distances.append((distance, candidate)) + distances.sort(key=lambda item: item[0]) + if len(distances) > 1 and distances[1][0] - distances[0][0] < 0.15: + return ClosestObjectResult(None, None, (), "ambiguous_object_proximity") + return ClosestObjectResult(distances[0][1], distances[0][0], ("object_centroid_proximity",)) + + def classify_horizontal_relation( + self, first: GroundedObject, second: GroundedObject + ) -> HorizontalRelationResult: + """Classify whether the first object's support centroid is left or right of the second's.""" + if first.id == second.id: + return HorizontalRelationResult(None, (), "duplicate_object_id") + first_points = self._object_points(first) + second_points = self._object_points(second) + if first_points is None or second_points is None: + return HorizontalRelationResult(None, (), "insufficient_object_support") + horizontal_offset_m = float(np.median(first_points[:, 0]) - np.median(second_points[:, 0])) + if abs(horizontal_offset_m) < 0.1: + return HorizontalRelationResult(None, (), "ambiguous_horizontal_relation") + return HorizontalRelationResult( + "left" if horizontal_offset_m < 0 else "right", ("camera_frame_support_centroids",) + ) + + def classify_forward_path(self) -> ForwardPathResult: + """Classify the observed camera-forward corridor as clear or blocked.""" + ground_fit = self.fit_ground_plane() + if ground_fit.estimate is None: + return ForwardPathResult( + None, + 0, + ("ground_plane_rejected", *ground_fit.quality_flags), + ground_fit.rejection_reason, + ) + projected = project_visible_points(self.frame) + points = np.asarray(projected.camera_points, dtype=np.float64) + state, flags, reason = classify_forward_corridor(points, ground_fit.estimate) + return ForwardPathResult(state, len(points), flags, reason) + + def _object_points(self, object: GroundedObject) -> np.ndarray | None: + mask = self._object_masks.get(object.id) + if mask is None: + raise ValueError(f"unknown grounded object: {object.id}") + points = points_in_mask(self.frame, mask.mask) + return points if len(points) >= 6 else None + + +def _object_mask_index(item: GroundedObject) -> int: + try: + return int(item.id.rsplit("-", 1)[1]) + except (IndexError, ValueError) as exc: + raise ValueError(f"grounded object ID lacks mask index: {item.id}") from exc diff --git a/dimos/benchmark/vqa/generation/primitives/geometry.py b/dimos/benchmark/vqa/generation/primitives/geometry.py new file mode 100644 index 0000000000..a1314506a9 --- /dev/null +++ b/dimos/benchmark/vqa/generation/primitives/geometry.py @@ -0,0 +1,187 @@ +"""Deterministic point-cloud geometry helpers for private VQA primitives.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import cv2 +import numpy as np + +from dimos.benchmark.vqa.generation.geometry import project_visible_points +from dimos.benchmark.vqa.models import CalibratedFrame, GroundPlaneEstimate + + +@dataclass(frozen=True) +class PlaneFitResult: + """Accepted plane or explicit quality-gated rejection.""" + + estimate: GroundPlaneEstimate | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + + +def estimate_ground_plane(frame: CalibratedFrame) -> PlaneFitResult: + """Fit a robust plane to visible points in the lower image ground band.""" + projected = project_visible_points(frame) + candidates = np.asarray( + [ + point + for point, (_, y) in zip(projected.camera_points, projected.pixels, strict=True) + if y >= int(frame.image.height * 0.6) + ], + dtype=np.float64, + ) + min_points, min_inliers, threshold = 12, 10, 0.06 + if len(candidates) < min_points: + return PlaneFitResult(None, ("insufficient_ground_band_points",), "insufficient_support") + + import open3d as o3d + + o3d.utility.random.seed(0) + point_cloud = o3d.geometry.PointCloud(o3d.utility.Vector3dVector(candidates)) + _, inlier_indices = point_cloud.segment_plane( + distance_threshold=threshold, + ransac_n=3, + num_iterations=816, + ) + if len(inlier_indices) < min_inliers: + return PlaneFitResult(None, ("insufficient_plane_inliers",), "insufficient_inliers") + + inlier_points = candidates[np.asarray(inlier_indices, dtype=int)] + center = np.mean(inlier_points, axis=0) + _, _, right = np.linalg.svd(inlier_points - center, full_matrices=False) + normal = right[-1] + offset = -float(normal @ center) + if offset < 0: + normal, offset = -normal, -offset + residuals = np.abs(candidates @ normal + offset) + inliers = residuals <= threshold + residual_m = ( + float(np.sqrt(np.mean(np.square(residuals[inliers])))) if inliers.any() else float("inf") + ) + if int(inliers.sum()) < min_inliers: + return PlaneFitResult(None, ("insufficient_refined_inliers",), "insufficient_inliers") + if residual_m > 0.035: + return PlaneFitResult(None, ("high_plane_residual",), "residual_too_high") + return PlaneFitResult( + GroundPlaneEstimate( + tuple(float(value) for value in normal), + float(offset), + len(candidates), + int(inliers.sum()), + residual_m, + ), + ("ground_band_visible", "ransac_inliers_accepted"), + ) + + +def points_in_mask(frame: CalibratedFrame, mask: np.ndarray) -> np.ndarray: + """Return nearest visible camera points covered by one foreground mask.""" + if mask.shape != (frame.image.height, frame.image.width): + raise ValueError("segmentation mask dimensions must match the image") + projected = project_visible_points(frame) + return np.asarray( + [ + point + for point, (x, y) in zip(projected.camera_points, projected.pixels, strict=True) + if mask[y, x] > 0 + ], + dtype=np.float64, + ) + + +def points_around_mask(frame: CalibratedFrame, mask: np.ndarray, radius_px: int = 12) -> np.ndarray: + """Return visible points in an annulus surrounding one foreground mask.""" + if radius_px < 1: + raise ValueError("mask ring radius must be positive") + if mask.shape != (frame.image.height, frame.image.width): + raise ValueError("segmentation mask dimensions must match the image") + foreground = mask > 0 + expanded = cv2.dilate( + foreground.astype(np.uint8), + np.ones((radius_px * 2 + 1, radius_px * 2 + 1), dtype=np.uint8), + ).astype(bool) + return points_in_mask(frame, expanded & ~foreground) + + +def fit_surface_plane(points: np.ndarray) -> PlaneFitResult: + """Fit a robust plane to private surface points without assuming ground orientation.""" + min_points, min_inliers, threshold = 12, 10, 0.06 + if len(points) < min_points: + return PlaneFitResult(None, ("insufficient_surface_points",), "insufficient_support") + + import open3d as o3d + + o3d.utility.random.seed(0) + point_cloud = o3d.geometry.PointCloud(o3d.utility.Vector3dVector(points)) + _, inlier_indices = point_cloud.segment_plane( + distance_threshold=threshold, + ransac_n=3, + num_iterations=816, + ) + if len(inlier_indices) < min_inliers: + return PlaneFitResult(None, ("insufficient_surface_inliers",), "insufficient_inliers") + + inlier_points = points[np.asarray(inlier_indices, dtype=int)] + center = np.mean(inlier_points, axis=0) + _, _, right = np.linalg.svd(inlier_points - center, full_matrices=False) + normal = right[-1] + offset = -float(normal @ center) + residuals = np.abs(points @ normal + offset) + inliers = residuals <= threshold + residual_m = ( + float(np.sqrt(np.mean(np.square(residuals[inliers])))) if inliers.any() else float("inf") + ) + if int(inliers.sum()) < min_inliers: + return PlaneFitResult( + None, ("insufficient_refined_surface_inliers",), "insufficient_inliers" + ) + if residual_m > 0.035: + return PlaneFitResult(None, ("high_surface_residual",), "residual_too_high") + return PlaneFitResult( + GroundPlaneEstimate( + tuple(float(value) for value in normal), + float(offset), + len(points), + int(inliers.sum()), + residual_m, + ), + ("surface_plane_accepted",), + ) + + +def classify_door_plane_angle( + door: GroundPlaneEstimate, surrounding: GroundPlaneEstimate +) -> tuple[str | None, str | None, float]: + """Classify only clearly coplanar or rotated door and surrounding planes.""" + alignment = abs(float(np.dot(door.normal, surrounding.normal))) + angle_deg = float(np.degrees(np.arccos(np.clip(alignment, -1.0, 1.0)))) + if angle_deg <= 12.0: + return "closed", None, angle_deg + if angle_deg >= 25.0: + return "open", None, angle_deg + return None, "ambiguous_door_angle", angle_deg + + +def classify_forward_corridor( + points: np.ndarray, ground: GroundPlaneEstimate +) -> tuple[str | None, tuple[str, ...], str | None]: + """Classify a visible camera-forward corridor as clear or blocked.""" + if len(points) == 0: + return None, (), "insufficient_forward_support" + depth = points[:, 2] + lateral_limit = depth * np.tan(np.radians(20.0)) + corridor = points[(depth >= 0.5) & (depth <= 3.0) & (np.abs(points[:, 0]) <= lateral_limit)] + if len(corridor) < 12: + return None, (), "insufficient_forward_support" + elevation = corridor @ np.asarray(ground.normal) + ground.offset_m + ground_points = corridor[np.abs(elevation) <= 0.08] + for start, stop in ((0.5, 1.33), (1.33, 2.16), (2.16, 3.0)): + if int(((ground_points[:, 2] >= start) & (ground_points[:, 2] < stop)).sum()) < 3: + return None, (), "incomplete_forward_ground_support" + obstacle_count = int((elevation > 0.15).sum()) + if obstacle_count >= 4: + return "blocked", ("visible_forward_obstacle", "forward_ground_supported"), None + if obstacle_count: + return None, (), "ambiguous_forward_obstacle" + return "clear", ("forward_ground_supported", "no_supported_forward_obstacle"), None diff --git a/dimos/benchmark/vqa/generation/primitives/selection.py b/dimos/benchmark/vqa/generation/primitives/selection.py new file mode 100644 index 0000000000..a9263ab1fc --- /dev/null +++ b/dimos/benchmark/vqa/generation/primitives/selection.py @@ -0,0 +1,13 @@ +"""Deterministic selection over grounded object evidence.""" + +from __future__ import annotations + +from dimos.benchmark.vqa.models import GroundedObject + + +def select_nearest_object( + objects: list[GroundedObject], side: str | None = None +) -> GroundedObject | None: + """Return the nearest grounded object, optionally restricted to one image side.""" + candidates = [item for item in objects if side is None or item.horizontal_direction == side] + return min(candidates, key=lambda item: item.range_m) if candidates else None diff --git a/dimos/benchmark/vqa/generation/question_agent.py b/dimos/benchmark/vqa/generation/question_agent.py new file mode 100644 index 0000000000..1f2cdb5fba --- /dev/null +++ b/dimos/benchmark/vqa/generation/question_agent.py @@ -0,0 +1,222 @@ +# Copyright 2026 Dimensional Inc. +"""Image-only constrained VQA question proposal.""" + +from __future__ import annotations + +import json +import re +from typing import Any + +from dimos.benchmark.vqa.models import ( + AnswerContract, + BooleanAnswerContract, + ChoiceAnswerContract, + DeferredHeightChoiceContract, + QuestionIntent, + QuestionProposal, +) +from dimos.models.vl.openai import OpenAIVlModel +from dimos.msgs.sensor_msgs.Image import Image + +QUESTION_PROMPT = """Select up to 5 challenging but visually well-supported single-frame VQA intents. +Inspect only this image. Do not assume depth, point clouds, calibration, metadata, or temporal context. +Return JSON only: an array of objects with kind, object_query, and threshold_m only for +within_distance, candidate_queries only for closest_object, and comparison_query only for +compare_height. kind must be one of presence, horizontal_direction, within_distance, visible_count, +camera_range, compare_nearest_by_side, compare_left_right, compare_height, door_state, closest_object, +or forward_path. +Use threshold_m: 3.0 for within_distance. +Use image context to select only intents likely to produce a useful geometric case: emit +compare_nearest_by_side only when at least two visible instances of the same object appear on +opposite image sides; emit horizontal_direction only for a visible object; and prefer relational +or directional intents over presence. Emit door_state only for a clearly visible door with nearby +visible structure. Diversify object classes and question families when the scene supports them. +Emit visible_count only when at least one repeated visible object is present. Emit camera_range only +for a visible object. Emit compare_height only for two distinct visible upright object types resting +on the visible ground; comparison_query must name the other object type. Emit compare_left_right only +for two distinct visible object types; comparison_query must name the other object type. Emit closest_object only when +one target and at least two distinct candidate object types are visible; candidate_queries must name +the visible candidate types. +Emit forward_path only when the center foreground and visible floor provide enough context to judge +the local path directly ahead. Use object_query: "forward path" for forward_path. +Do not return bare object names, floors, walls, ceilings, background surfaces, +questions, answers, explanations, Markdown, or information not visible in the image.""" + +AGENTIC_QUESTION_PROMPT = """Author up to 5 challenging, visually answerable single-frame VQA questions. +Inspect only this image. Do not use or infer depth, point clouds, calibration, metadata, or answers. +Return JSON only: an array of objects with question, answer_contract, optional object_queries, +and optional tool_hints. answer_contract is {"kind":"boolean"}, {"kind":"choice","choices":[...]}, +or {"kind":"deferred_height_choice","strategy":"height-window-v1"}. +Prioritize questions that a private point-cloud oracle can validate. For the height of one upright +object resting on visible ground, use deferred_height_choice. Its choices are generated privately +from a successful height measurement. Aim for a diverse set of questions that use the visible scene +composition: relative left/center/right position, which listed object is closest to a named target, +and height for an upright grounded object. For a visibly repeated object, count questions must use exactly +["1-2", "3-4", "5-7", "8+"]; camera-distance questions must use exactly ["under 1 m", +"1 to under 2 m", "2 to under 4 m", "4 m or more"]. For a pairwise left/right range question, +use exactly ["left", "right"]. For an A/B left-right relation question, use exactly ["left", "right"] +and name both objects. For a pairwise height question, use the two distinct object types as the choices. +For closest-object questions, use distinct visible candidate object types as the fixed choices. Use concise, +mutually exclusive fixed choices with two to four options. For a clearly visible door with nearby visible +structure, you may ask whether it is open or closed with exactly ["open", "closed"]. Use the fixed choices +["clear", "blocked"] only for a visibly supported local path directly ahead. +Use object_queries for every referenced object and tool_hints from "detect_objects", "segment_detections", +"ground_masks", "count_grounded_objects", "bucket_camera_range", "compare_nearest_by_side", +"compare_left_right", "fit_ground_plane", "measure_height", "compare_heights", "classify_door_state", "select_closest_object", +"classify_forward_path", or "bucket_measurement" when applicable. Use visibility/presence questions only +when no stronger geometric question is available. +Do not ask about color, material, text, intent, full physical size, hidden parts, or exact +metric distances without a supplied choice contract. Use only visible objects. Do not include +answers, explanations, Markdown, or background surfaces.""" + +_UNSUPPORTED_QUERIES = {"background", "ceiling", "floor", "ground", "room", "wall"} + + +class OpenAIQuestionAgent: + """Propose constrained VQA intents from an image without geometry access.""" + + def __init__(self, model: OpenAIVlModel) -> None: + self._model = model + + def propose(self, image: Image) -> list[QuestionIntent]: + try: + payload: Any = _parse_json_array(self._model.query(image, QUESTION_PROMPT)) + except json.JSONDecodeError as exc: + raise ValueError("question agent did not return JSON") from exc + if not isinstance(payload, list) or len(payload) > 5: + raise ValueError("question agent must return an array of at most five intents") + intents: list[QuestionIntent] = [] + if all(isinstance(item, str) for item in payload): + return [ + intent + for query in dict.fromkeys(item.strip() for item in payload if item.strip()) + if query.lower() not in _UNSUPPORTED_QUERIES + for intent in _intents_for_query(query) + ] + for item in payload: + if not isinstance(item, dict): + raise ValueError("question intent must be an object") + kind, query, threshold = ( + item.get("kind"), + item.get("object_query"), + item.get("threshold_m"), + ) + if kind not in ( + "presence", + "horizontal_direction", + "within_distance", + "visible_count", + "camera_range", + "compare_nearest_by_side", + "compare_left_right", + "compare_height", + "door_state", + "closest_object", + "forward_path", + ): + raise ValueError(f"unsupported question kind: {kind!r}") + if not isinstance(query, str) or not query: + raise ValueError("question intent requires object_query") + if kind == "within_distance" and ( + not isinstance(threshold, (int, float)) or threshold <= 0 + ): + raise ValueError("within_distance requires a positive threshold_m") + if kind != "within_distance": + threshold = None + candidates = _string_tuple(item.get("candidate_queries"), "candidate_queries") + comparison_query = item.get("comparison_query") + if kind == "closest_object" and (len(candidates) < 2 or query in candidates): + raise ValueError("closest_object requires two distinct candidate_queries") + if kind == "forward_path" and query != "forward path": + raise ValueError('forward_path requires object_query "forward path"') + if kind in ("compare_left_right", "compare_height") and ( + not isinstance(comparison_query, str) + or not comparison_query + or comparison_query == query + ): + raise ValueError(f"{kind} requires a distinct comparison_query") + if kind not in ("compare_left_right", "compare_height"): + comparison_query = None + intents.append(QuestionIntent(kind, query, threshold, candidates, comparison_query)) + return intents + + +class OpenAIFreeformQuestionAuthor: + """Image-only author for generic public questions and answer contracts.""" + + def __init__(self, model: OpenAIVlModel, max_questions: int = 5) -> None: + self._model = model + self._max_questions = max_questions + + def propose(self, image: Image) -> list[QuestionProposal]: + try: + payload: Any = _parse_json_array(self._model.query(image, AGENTIC_QUESTION_PROMPT)) + except json.JSONDecodeError as exc: + raise ValueError("question author did not return JSON") from exc + if not isinstance(payload, list) or len(payload) > self._max_questions: + raise ValueError("question author returned too many questions") + proposals = [ + _proposal_from_json(item, index) for index, item in enumerate(payload, start=1) + ] + if len({item.id for item in proposals}) != len(proposals): + raise ValueError("question ids must be unique") + return proposals + + +def _proposal_from_json(item: Any, index: int) -> QuestionProposal: + if not isinstance(item, dict): + raise ValueError("question proposal must be an object") + identifier, question = item.get("id"), item.get("question") + if not isinstance(question, str) or not question: + raise ValueError("question proposal requires question") + if not isinstance(identifier, str) or not identifier: + identifier = f"proposal-{index:02d}" + queries = _string_tuple(item.get("object_queries", []), "object_queries") + hints = _string_tuple(item.get("tool_hints", []), "tool_hints") + contract = item.get("answer_contract") + if not isinstance(contract, dict): + raise ValueError("question proposal requires answer_contract") + kind = contract.get("kind") + if kind == "boolean": + answer_contract: AnswerContract = BooleanAnswerContract() + elif kind == "choice": + choices = _string_tuple(contract.get("choices"), "choices") + if len(choices) < 2: + raise ValueError("choice contract requires at least two choices") + answer_contract = ChoiceAnswerContract(choices) + elif kind == "deferred_height_choice" and contract.get("strategy") == "height-window-v1": + answer_contract = DeferredHeightChoiceContract() + else: + raise ValueError("unsupported answer contract") + return QuestionProposal(identifier, question, answer_contract, queries, hints) + + +def _string_tuple(value: Any, name: str) -> tuple[str, ...]: + if value is None: + return () + if isinstance(value, str): + value = [value] + if not isinstance(value, list): + return () + return tuple(item.strip() for item in value if isinstance(item, str) and item.strip()) + + +def _intents_for_query(query: str) -> list[QuestionIntent]: + return [ + QuestionIntent(kind="presence", object_query=query), + QuestionIntent(kind="horizontal_direction", object_query=query), + QuestionIntent(kind="within_distance", object_query=query, threshold_m=3.0), + QuestionIntent(kind="visible_count", object_query=query), + QuestionIntent(kind="camera_range", object_query=query), + QuestionIntent(kind="compare_nearest_by_side", object_query=query), + ] + + +def _parse_json_array(response: str) -> Any: + stripped = response.strip() + if stripped.startswith("```"): + stripped = re.sub(r"^```(?:json)?\s*|\s*```$", "", stripped, flags=re.IGNORECASE) + start, end = stripped.find("["), stripped.rfind("]") + if start < 0 or end < start: + raise json.JSONDecodeError("expected JSON array", stripped, 0) + return json.loads(stripped[start : end + 1]) diff --git a/dimos/benchmark/vqa/generation/questions.py b/dimos/benchmark/vqa/generation/questions.py new file mode 100644 index 0000000000..d442232972 --- /dev/null +++ b/dimos/benchmark/vqa/generation/questions.py @@ -0,0 +1,65 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Deterministic closed-answer questions from grounded objects.""" + +from __future__ import annotations + +from dimos.benchmark.vqa.generation.primitives.selection import select_nearest_object +from dimos.benchmark.vqa.models import GroundedObject, VqaExample + + +def generate_questions( + frame_id: str, objects: list[GroundedObject], queries: list[str], *, distance_m: float = 3.0 +) -> list[VqaExample]: + """Generate presence, range, and direction questions for one frame.""" + if distance_m <= 0: + raise ValueError("distance_m must be positive") + + examples: list[VqaExample] = [] + for query in queries: + nearest = select_nearest_object([item for item in objects if item.label == query]) + examples.append( + VqaExample( + id=f"{frame_id}-{query}-presence", + question=f"Is there a {query} in the image? Answer yes or no.", + expected_answer="yes" if nearest is not None else "no", + answer_type="boolean", + object_ids=(nearest.id,) if nearest is not None else (), + allowed_answers=("yes", "no"), + ) + ) + if nearest is None: + continue + examples.extend( + [ + VqaExample( + id=f"{frame_id}-{query}-direction", + question=f"Where is the nearest {query}: left, center, or right?", + expected_answer=nearest.horizontal_direction, + answer_type="choice", + object_ids=(nearest.id,), + allowed_answers=("left", "center", "right"), + ), + VqaExample( + id=f"{frame_id}-{query}-range", + question=f"Is the nearest {query} within {distance_m:g} meters? Answer yes or no.", + expected_answer="yes" if nearest.range_m <= distance_m else "no", + answer_type="boolean", + object_ids=(nearest.id,), + allowed_answers=("yes", "no"), + ), + ] + ) + return examples diff --git a/dimos/benchmark/vqa/generation/recording.py b/dimos/benchmark/vqa/generation/recording.py new file mode 100644 index 0000000000..ff3cd6d49e --- /dev/null +++ b/dimos/benchmark/vqa/generation/recording.py @@ -0,0 +1,64 @@ +# Copyright 2026 Dimensional Inc. +"""Build self-contained VQA frames from Go2 Memory2 recordings.""" + +from __future__ import annotations + +import numpy as np + +from dimos.benchmark.vqa.models import CalibratedFrame +from dimos.memory2.store.sqlite import SqliteStore +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.robot.unitree.go2.connection import BASE_TO_OPTICAL, GO2Connection + + +def load_go2_frame(recording: str, frame_index: int, tolerance_s: float = 0.25) -> CalibratedFrame: + """Load one image with its nearest LiDAR and odometry observations.""" + if frame_index < 0 or tolerance_s <= 0: + raise ValueError("frame_index must be non-negative and tolerance_s must be positive") + store = SqliteStore(path=recording, must_exist=True) + store.start() + try: + image_obs = store.streams.color_image.offset(frame_index).first() + lidar_obs = store.streams.lidar.at(image_obs.ts, tolerance_s).first() + odom_obs = store.streams.odom.at(image_obs.ts, tolerance_s).first() + image_data = image_obs.data + lidar_data = lidar_obs.data + odom_data = odom_obs.data + finally: + store.stop() + image, camera_info = _rectify_go2_image(image_data) + world_to_camera = -(Transform.from_pose("base_link", odom_data) + BASE_TO_OPTICAL) + return CalibratedFrame( + id=f"go2-{frame_index}", + image=image, + pointcloud=lidar_data, + camera_info=camera_info, + pointcloud_to_camera=world_to_camera, + image_is_rectified=True, + original_image=image_data, + ) + + +def _rectify_go2_image(image: Image) -> tuple[Image, CameraInfo]: + import cv2 + + source = GO2Connection.camera_info_static + matrix = np.asarray(source.K, dtype=np.float64).reshape(3, 3) + distortion = np.asarray(source.D, dtype=np.float64) + size = (image.width, image.height) + map_x, map_y = cv2.fisheye.initUndistortRectifyMap( + matrix, distortion, np.eye(3), matrix, size, cv2.CV_32FC1 + ) + data = cv2.remap(image.data, map_x, map_y, interpolation=cv2.INTER_LINEAR) + camera_info = CameraInfo.from_intrinsics( + matrix[0, 0], + matrix[1, 1], + matrix[0, 2], + matrix[1, 2], + image.width, + image.height, + frame_id="camera_optical", + ) + return Image(data=data, format=image.format, frame_id=image.frame_id, ts=image.ts), camera_info diff --git a/dimos/benchmark/vqa/generation/specification.py b/dimos/benchmark/vqa/generation/specification.py new file mode 100644 index 0000000000..f0c9ce20af --- /dev/null +++ b/dimos/benchmark/vqa/generation/specification.py @@ -0,0 +1,30 @@ +"""Validated input configuration for resumable VQA dataset generation.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class VqaGroundingSpecification(BaseModel): + """Private grounding quality thresholds for generated VQA labels.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + min_mask_area_px: int = Field(default=128, ge=1) + min_foreground_points: int = Field(default=3, ge=1) + + +class VqaGenerationSpecification(BaseModel): + """One reproducible multi-frame VQA generation request.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + recording: str = Field(min_length=1) + start_index: int = Field(default=0, ge=0) + stop_index: int = Field(gt=0) + stride: int = Field(default=1, ge=1) + question_mode: Literal["constrained", "agentic"] = "constrained" + grounding: VqaGroundingSpecification = VqaGroundingSpecification() + output: str | None = None diff --git a/dimos/benchmark/vqa/generation/test_agents.py b/dimos/benchmark/vqa/generation/test_agents.py new file mode 100644 index 0000000000..6b001341e9 --- /dev/null +++ b/dimos/benchmark/vqa/generation/test_agents.py @@ -0,0 +1,444 @@ +# Copyright 2026 Dimensional Inc. + +from __future__ import annotations + +from typing import cast + +import numpy as np + +from dimos.benchmark.vqa.generation.ground_truth_generator import VqaGroundTruthGenerator +from dimos.benchmark.vqa.generation.primitives.contracts import ( + HeightMeasurementResult, + HorizontalRelationResult, +) +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives +from dimos.benchmark.vqa.generation.question_agent import OpenAIQuestionAgent +from dimos.benchmark.vqa.models import ( + CalibratedFrame, + GroundedObject, + GroundingConfig, + GroundPlaneEstimate, + OracleMeasurement, + QuestionIntent, +) +from dimos.models.vl.openai import OpenAIVlModel +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D +from dimos.perception.detection.type.detection2d.point import Detection2DPoint +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg + + +class _QuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert image.width == 6 + assert "point clouds" in prompt + return '```json\n["chair"]\n```' + + +class _Detector: + def __init__(self, image: Image, detection: Detection2DSeg) -> None: + self._image = image + self._detection = detection + + def detect(self, image: Image, query: str) -> ImageDetections2D: + return ImageDetections2D(image, [self._detection] if query == "chair" else []) + + +class _MultiDetector: + def __init__(self, image: Image, detections: list[Detection2DSeg]) -> None: + self._image = image + self._detections = detections + + def detect(self, image: Image, query: str) -> ImageDetections2D: + return ImageDetections2D(self._image, self._detections if query == "chair" else []) + + +def _agent( + frame: CalibratedFrame, + detector: _Detector | _MultiDetector, + segmenter: _Segmenter, + *, + localizer: _PointLocalizer | None = None, + point_segmenter: _PointSegmenter | None = None, + config: GroundingConfig = GroundingConfig(), +) -> VqaGroundTruthGenerator: + return VqaGroundTruthGenerator( + FramePerceptionPrimitives(frame, detector, segmenter, localizer, point_segmenter, config) + ) + + +class _Segmenter: + def segment(self, detections: ImageDetections2D) -> ImageDetections2D: + return detections + + +class _PointLocalizer: + def locate(self, image: Image, query: str) -> ImageDetections2D: + return ImageDetections2D(image, [Detection2DPoint(3.0, 3.0, query, 0.0, image)]) + + +class _PointSegmenter: + def __init__(self, detection: Detection2DSeg) -> None: + self._detection = detection + + def segment_points(self, points: ImageDetections2D) -> ImageDetections2D: + return ImageDetections2D(points.image, [self._detection]) + + +def _frame_and_detection() -> tuple[CalibratedFrame, Detection2DSeg]: + image = Image.from_numpy(np.zeros((6, 6, 3), dtype=np.uint8)) + frame = CalibratedFrame( + id="frame-1", + image=image, + pointcloud=PointCloud2.from_numpy( + np.array([[0.0, 0.0, 1.0], [0.4, 0.0, 1.0], [-0.4, 0.0, 1.0]], dtype=np.float32) + ), + camera_info=CameraInfo.from_intrinsics(3.0, 3.0, 3.0, 3.0, 6, 6), + pointcloud_to_camera=Transform.identity(), + image_is_rectified=True, + ) + detection = Detection2DSeg( + (0.0, 0.0, 5.0, 5.0), 0, -1, 1.0, "chair", 0.0, image, np.full((6, 6), 255, dtype=np.uint8) + ) + return frame, detection + + +def test_question_agent_returns_constrained_intents() -> None: + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _QuestionModel())).propose(frame.image) + + assert intents == [ + QuestionIntent(kind="presence", object_query="chair"), + QuestionIntent(kind="horizontal_direction", object_query="chair"), + QuestionIntent(kind="within_distance", object_query="chair", threshold_m=3.0), + QuestionIntent(kind="visible_count", object_query="chair"), + QuestionIntent(kind="camera_range", object_query="chair"), + QuestionIntent(kind="compare_nearest_by_side", object_query="chair"), + ] + + +def test_question_agent_uses_image_selected_structured_intents() -> None: + class _StructuredQuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "Do not return bare object names" in prompt + assert "opposite image sides" in prompt + return '[{"kind":"compare_nearest_by_side","object_query":"chair"}]' + + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _StructuredQuestionModel())).propose( + frame.image + ) + + assert intents == [QuestionIntent(kind="compare_nearest_by_side", object_query="chair")] + + +def test_question_agent_accepts_door_state_intent() -> None: + class _DoorQuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "door_state" in prompt + return '[{"kind":"door_state","object_query":"door"}]' + + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _DoorQuestionModel())).propose(frame.image) + + assert intents == [QuestionIntent(kind="door_state", object_query="door")] + + +def test_question_agent_accepts_closest_object_intent() -> None: + class _ClosestQuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "candidate_queries" in prompt + return ( + '[{"kind":"closest_object","object_query":"chair",' + '"candidate_queries":["table","lamp"]}]' + ) + + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _ClosestQuestionModel())).propose( + frame.image + ) + + assert intents == [QuestionIntent("closest_object", "chair", None, ("table", "lamp"))] + + +def test_question_agent_accepts_forward_path_intent() -> None: + class _ForwardPathQuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "forward_path" in prompt + return '[{"kind":"forward_path","object_query":"forward path"}]' + + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _ForwardPathQuestionModel())).propose( + frame.image + ) + + assert intents == [QuestionIntent(kind="forward_path", object_query="forward path")] + + +def test_question_agent_accepts_count_range_and_height_comparison_intents() -> None: + class _GeometryQuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "visible_count" in prompt + assert "comparison_query" in prompt + return """[ + {"kind":"visible_count","object_query":"chair"}, + {"kind":"camera_range","object_query":"lamp"}, + {"kind":"compare_left_right","object_query":"chair","comparison_query":"table"}, + {"kind":"compare_height","object_query":"chair","comparison_query":"table"} + ]""" + + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _GeometryQuestionModel())).propose( + frame.image + ) + + assert intents == [ + QuestionIntent(kind="visible_count", object_query="chair"), + QuestionIntent(kind="camera_range", object_query="lamp"), + QuestionIntent(kind="compare_left_right", object_query="chair", comparison_query="table"), + QuestionIntent(kind="compare_height", object_query="chair", comparison_query="table"), + ] + + +def test_ground_truth_agent_records_tools_and_rejects_unsupported_question() -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + + answered = agent.answer( + frame, QuestionIntent(kind="within_distance", object_query="chair", threshold_m=3.0) + ) + rejected = agent.answer( + frame, QuestionIntent(kind="horizontal_direction", object_query="table") + ) + absent = agent.answer(frame, QuestionIntent(kind="presence", object_query="table")) + + assert answered.status == "answered" + assert answered.answer == "yes" + assert answered.evidence[0].point_count == 3 + assert [item.tool for item in answered.trace] == [ + "detect_objects", + "segment_objects", + "get_foreground_geometry", + ] + assert rejected.status == "rejected" + assert rejected.reason == "no_grounded_object" + assert absent.status == "rejected" + + +def test_ground_truth_agent_rejects_small_masks() -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=37), + ) + + result = agent.answer(frame, QuestionIntent(kind="presence", object_query="chair")) + + assert result.status == "rejected" + + +def test_ground_truth_agent_falls_back_to_point_prompt() -> None: + frame, detection = _frame_and_detection() + point_detection = Detection2DSeg( + detection.bbox, + detection.track_id, + detection.class_id, + detection.confidence, + "plant", + detection.ts, + detection.image, + detection.mask, + ) + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + localizer=_PointLocalizer(), + point_segmenter=_PointSegmenter(point_detection), + config=GroundingConfig(min_mask_area_px=1), + ) + + result = agent.answer(frame, QuestionIntent(kind="presence", object_query="plant")) + + assert result.answer == "yes" + assert [item.tool for item in result.trace] == [ + "detect_objects", + "locate_object_point", + "segment_object_point", + "get_foreground_geometry", + ] + + +def test_ground_truth_agent_compares_nearest_objects_by_side() -> None: + image = Image.from_numpy(np.zeros((6, 6, 3), dtype=np.uint8)) + frame = CalibratedFrame( + id="frame-1", + image=image, + pointcloud=PointCloud2.from_numpy( + np.array( + [ + [-0.5, -0.6, 1.0], + [-0.5, 0.0, 1.0], + [-0.5, 0.6, 1.0], + [1.0, -1.0, 2.0], + [1.0, 0.0, 2.0], + [1.0, 1.0, 2.0], + ], + dtype=np.float32, + ) + ), + camera_info=CameraInfo.from_intrinsics(3.0, 3.0, 3.0, 3.0, 6, 6), + pointcloud_to_camera=Transform.identity(), + image_is_rectified=True, + ) + left_mask = np.zeros((6, 6), dtype=np.uint8) + left_mask[:, :3] = 255 + right_mask = np.zeros((6, 6), dtype=np.uint8) + right_mask[:, 3:] = 255 + detections = [ + Detection2DSeg((0.0, 0.0, 2.0, 5.0), 0, -1, 1.0, "chair", 0.0, image, left_mask), + Detection2DSeg((3.0, 0.0, 5.0, 5.0), 1, -1, 1.0, "chair", 0.0, image, right_mask), + ] + agent = _agent( + frame, + _MultiDetector(image, detections), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + + result = agent.answer( + frame, QuestionIntent(kind="compare_nearest_by_side", object_query="chair") + ) + + assert result.status == "answered" + assert result.answer == "left" + assert result.question.object_ids == ("frame-1-chair-0", "frame-1-chair-1") + + +def test_ground_truth_agent_rejects_side_comparison_without_both_sides() -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + + result = agent.answer( + frame, QuestionIntent(kind="compare_nearest_by_side", object_query="chair") + ) + + assert result.status == "rejected" + assert result.reason == "missing_grounded_side" + + +def test_ground_truth_agent_buckets_visible_count_and_camera_range() -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + + count = agent.answer(frame, QuestionIntent(kind="visible_count", object_query="chair")) + camera_range = agent.answer(frame, QuestionIntent(kind="camera_range", object_query="chair")) + + assert count.answer == "1-2" + assert count.question.allowed_answers == ("1-2", "3-4", "5-7", "8+") + assert camera_range.answer == "1 to under 2 m" + assert camera_range.question.allowed_answers == ( + "under 1 m", + "1 to under 2 m", + "2 to under 4 m", + "4 m or more", + ) + + +def test_ground_truth_agent_compares_ground_plane_relative_heights(monkeypatch: object) -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + chair = GroundedObject("chair-0", "chair", 8, 1.0, "left") + table = GroundedObject("table-0", "table", 8, 2.0, "right") + plane = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + monkeypatch.setattr( + agent, + "ground", + lambda _frame, query: ([chair] if query == "chair" else [table], ()), + ) + monkeypatch.setattr( + agent.primitives, + "fit_ground_plane", + lambda: type("Fit", (), {"estimate": plane, "rejection_reason": None})(), + ) + monkeypatch.setattr( + agent.primitives, + "measure_height", + lambda item, accepted_plane: HeightMeasurementResult( + item, + accepted_plane, + OracleMeasurement(0.8 if item == chair else 0.5, "m", 0.05, (), ()), + (), + ), + ) + + result = agent.answer( + frame, + QuestionIntent(kind="compare_height", object_query="chair", comparison_query="table"), + ) + + assert result.status == "answered" + assert result.answer == "chair" + assert result.question.allowed_answers == ("chair", "table") + + +def test_ground_truth_agent_compares_pairwise_left_right(monkeypatch: object) -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + chair = GroundedObject("chair-0", "chair", 8, 1.0, "left") + table = GroundedObject("table-0", "table", 8, 2.0, "right") + monkeypatch.setattr( + agent, + "ground", + lambda _frame, query: ([chair] if query == "chair" else [table], ()), + ) + monkeypatch.setattr( + agent.primitives, + "classify_horizontal_relation", + lambda first, second: HorizontalRelationResult("left", ("camera_frame_support_centroids",)), + ) + + result = agent.answer( + frame, + QuestionIntent(kind="compare_left_right", object_query="chair", comparison_query="table"), + ) + + assert result.status == "answered" + assert result.answer == "left" + assert result.question.allowed_answers == ("left", "right") diff --git a/dimos/benchmark/vqa/generation/test_dataset.py b/dimos/benchmark/vqa/generation/test_dataset.py new file mode 100644 index 0000000000..6790044cd4 --- /dev/null +++ b/dimos/benchmark/vqa/generation/test_dataset.py @@ -0,0 +1,104 @@ +# Copyright 2026 Dimensional Inc. + +import json +from pathlib import Path + +from dimos.benchmark.vqa.generation.dataset import _evaluation_rows, write_dataset_manifest +from dimos.benchmark.vqa.models import ( + AcceptedOracleResult, + ChoiceAnswerContract, + DeferredHeightChoiceContract, + GroundTruthResult, + QuestionIntent, + QuestionProposal, + ToolTrace, + VqaExample, +) + + +def test_constrained_results_export_simple_multiple_choice_rows() -> None: + result = GroundTruthResult( + intent=QuestionIntent("presence", "chair"), + question=VqaExample( + "frame-chair-presence", + "Is there a chair?", + "yes", + "boolean", + (), + ("yes", "no"), + ), + status="answered", + answer="yes", + reason=None, + evidence=(), + trace=(ToolTrace("ground", "chair"),), + ) + + cases, labels = _evaluation_rows("frame", [result]) + + assert cases == [ + { + "id": "frame-chair-presence", + "image": "image.jpg", + "question": "Is there a chair?", + "choices": ("yes", "no"), + } + ] + assert labels == [{"id": "frame-chair-presence", "answer": "yes"}] + + +def test_deferred_height_result_exports_resolved_public_choices() -> None: + choices = ("under 0.2 m", "0.2-0.6 m", "0.6-1.0 m", "over 1.0 m") + result = AcceptedOracleResult( + QuestionProposal("chair-height", "How tall is the chair?", DeferredHeightChoiceContract()), + "0.2-0.6 m", + ChoiceAnswerContract(choices), + ("height-1",), + (), + (), + ) + + cases, labels = _evaluation_rows("frame", [result]) + + assert cases[0]["choices"] == choices + assert labels == [{"id": "frame-chair-height", "answer": "0.2-0.6 m"}] + + +def test_dataset_manifest_exports_public_cases_and_private_labels(tmp_path: Path) -> None: + frame = tmp_path / "frame-000040" + frame.mkdir() + (frame / "frame.json").write_text( + json.dumps( + { + "frame_id": "frame-40", + "accepted_question_count": 1, + "rejected_question_count": 2, + } + ) + ) + (frame / "cases.json").write_text( + json.dumps( + [ + { + "id": "case-1", + "image": "image.jpg", + "question": "Is it visible?", + "choices": ["yes", "no"], + } + ] + ) + ) + (frame / "labels.json").write_text(json.dumps([{"id": "case-1", "answer": "yes"}])) + + summary = write_dataset_manifest(tmp_path) + + assert summary == {"frame_count": 1, "accepted_question_count": 1, "rejected_question_count": 2} + assert json.loads((tmp_path / "cases.jsonl").read_text()) == { + "id": "case-1", + "image": "frame-000040/image.jpg", + "question": "Is it visible?", + "choices": ["yes", "no"], + } + assert json.loads((tmp_path / "labels.jsonl").read_text()) == {"id": "case-1", "answer": "yes"} + assert not (tmp_path / "frames.jsonl").exists() + assert not (tmp_path / "manifest.json").exists() diff --git a/dimos/benchmark/vqa/generation/test_geometry.py b/dimos/benchmark/vqa/generation/test_geometry.py new file mode 100644 index 0000000000..d4977c0adb --- /dev/null +++ b/dimos/benchmark/vqa/generation/test_geometry.py @@ -0,0 +1,61 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import numpy as np +import pytest + +from dimos.benchmark.vqa.generation.geometry import project_visible_points +from dimos.benchmark.vqa.models import CalibratedFrame +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 + + +def _frame(points: np.ndarray, *, rectified: bool = True) -> CalibratedFrame: + return CalibratedFrame( + id="frame-1", + image=Image.from_numpy(np.zeros((4, 4, 3), dtype=np.uint8)), + pointcloud=PointCloud2.from_numpy(points), + camera_info=CameraInfo.from_intrinsics(2.0, 2.0, 2.0, 2.0, 4, 4), + pointcloud_to_camera=Transform.identity(), + image_is_rectified=rectified, + ) + + +def test_project_visible_points_keeps_nearest_point_per_pixel() -> None: + frame = _frame( + np.array( + [ + [0.0, 0.0, 2.0], + [0.0, 0.0, 1.0], + [0.5, 0.0, 1.0], + [0.0, 0.0, -1.0], + ], + dtype=np.float32, + ) + ) + + projected = project_visible_points(frame) + + assert projected.pixels == [(2, 2), (3, 2)] + assert projected.source_indices == [1, 2] + assert projected.camera_points == [(0.0, 0.0, 1.0), (0.5, 0.0, 1.0)] + + +def test_project_visible_points_rejects_unrectified_images() -> None: + with pytest.raises(ValueError, match="rectified"): + project_visible_points(_frame(np.zeros((1, 3), dtype=np.float32), rectified=False)) diff --git a/dimos/benchmark/vqa/generation/test_oracle.py b/dimos/benchmark/vqa/generation/test_oracle.py new file mode 100644 index 0000000000..53fcec067e --- /dev/null +++ b/dimos/benchmark/vqa/generation/test_oracle.py @@ -0,0 +1,637 @@ +# Copyright 2026 Dimensional Inc. + +from __future__ import annotations + +import json +from typing import Any, cast + +from langchain_core.messages import AIMessage +import numpy as np + +from dimos.benchmark.vqa.generation.geometry import project_visible_points +from dimos.benchmark.vqa.generation.oracle import ( + PrivateToolCallingOracle, + SemanticEvidenceValidation, + validate_oracle_answer, +) +from dimos.benchmark.vqa.generation.oracle_tools import LocalOracleToolRegistry +from dimos.benchmark.vqa.generation.primitives.choices import ( + camera_range_choice, + count_choice, + height_choice_window, +) +from dimos.benchmark.vqa.generation.primitives.contracts import ( + HeightMeasurementResult, + HorizontalRelationResult, +) +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives +from dimos.benchmark.vqa.generation.primitives.geometry import ( + classify_door_plane_angle, + classify_forward_corridor, + estimate_ground_plane, +) +from dimos.benchmark.vqa.generation.question_agent import ( + AGENTIC_QUESTION_PROMPT, + OpenAIFreeformQuestionAuthor, +) +from dimos.benchmark.vqa.models import ( + BooleanAnswerContract, + CalibratedFrame, + ChoiceAnswerContract, + DeferredHeightChoiceContract, + GroundedObject, + GroundingConfig, + GroundPlaneEstimate, + OracleEvidence, + OracleMeasurement, + OracleToolResult, + QuestionProposal, +) +from dimos.models.vl.openai import OpenAIVlModel +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg + + +class _QuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "answer_contract" in prompt + return '[{"id":"chair-presence","question":"Is there a chair?","answer_contract":{"kind":"boolean"},"object_queries":["chair"]}]' + + +class _Grounding: + frame = cast("Any", object()) + + def detect_objects(self, query: str) -> list[Any]: + return [] + + def segment_detections(self, query: str) -> list[Any]: + return [] + + def ground_masks(self, query: str) -> list[Any]: + return [ + type( + "Object", + (), + { + "id": "synthetic-chair-0", + "label": query, + "range_m": 1.0, + "horizontal_direction": "left", + "point_count": 4, + }, + )() + ] + + +def _measurement_frame(points: np.ndarray | None = None) -> CalibratedFrame: + ground = [[x, 1.0, z] for z in (3.0, 4.0, 5.0) for x in (-1.0, -0.5, 0.0, 0.5, 1.0)] + object_points = [[2.0, 1.0 - height, 4.0] for height in np.linspace(0.2, 1.0, 9)] + image = Image.from_numpy(np.zeros((100, 100, 3), dtype=np.uint8)) + return CalibratedFrame( + id="synthetic", + image=image, + pointcloud=PointCloud2.from_numpy( + points + if points is not None + else np.asarray([*ground, *object_points], dtype=np.float32) + ), + camera_info=CameraInfo.from_intrinsics(50.0, 50.0, 50.0, 50.0, 100, 100), + pointcloud_to_camera=Transform.identity(), + image_is_rectified=True, + ) + + +class _MaskDetector: + def __init__(self, mask: np.ndarray) -> None: + self._mask = mask + + def detect(self, image: Image, query: str) -> ImageDetections2D: + detection = Detection2DSeg( + (0.0, 0.0, float(image.width - 1), float(image.height - 1)), + 0, + -1, + 1.0, + query, + 0.0, + image, + self._mask, + ) + return ImageDetections2D(image, [detection]) + + +class _IdentitySegmenter: + def segment(self, detections: Any) -> Any: + return detections + + +def _frame_primitives( + frame: CalibratedFrame, mask: np.ndarray | None = None +) -> FramePerceptionPrimitives: + if mask is None: + mask = np.full((frame.image.height, frame.image.width), 255, dtype=np.uint8) + return FramePerceptionPrimitives( + frame, _MaskDetector(mask), _IdentitySegmenter(), config=GroundingConfig(min_mask_area_px=1) + ) + + +class _BoundModel: + def __init__(self) -> None: + self._calls = 0 + + def bind_tools(self, tools: Any) -> _BoundModel: + return self + + def invoke(self, messages: Any) -> AIMessage: + self._calls += 1 + if self._calls == 1: + return AIMessage( + content="", + tool_calls=[{"name": "detect_objects", "args": {"query": "chair"}, "id": "call-1"}], + ) + if self._calls == 2: + return AIMessage( + content="", + tool_calls=[ + { + "name": "segment_detections", + "args": {"detection_id": "detection:v1:0001"}, + "id": "call-2", + } + ], + ) + if self._calls == 3: + return AIMessage( + content="", + tool_calls=[ + {"name": "ground_masks", "args": {"mask_id": "mask:v1:0002"}, "id": "call-3"} + ], + ) + return AIMessage( + content='{"answer":"yes","evidence_ids":["grounding:v1:synthetic-chair-0"]}' + ) + + +class _SemanticValidator: + def __init__(self, verdict: SemanticEvidenceValidation) -> None: + self.verdict = verdict + self.calls: list[tuple[QuestionProposal, str | float, tuple[OracleToolResult, ...]]] = [] + + def validate( + self, + proposal: QuestionProposal, + answer: str | float, + cited_results: tuple[OracleToolResult, ...], + ) -> SemanticEvidenceValidation: + self.calls.append((proposal, answer, cited_results)) + return self.verdict + + +def test_freeform_question_author_parses_public_contract() -> None: + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + proposals = OpenAIFreeformQuestionAuthor(cast("OpenAIVlModel", _QuestionModel())).propose(image) + + assert proposals == [ + QuestionProposal( + "chair-presence", "Is there a chair?", BooleanAnswerContract(), ("chair",), () + ) + ] + + +def test_freeform_author_prompt_prioritizes_geometric_questions() -> None: + assert "measure_height" in AGENTIC_QUESTION_PROMPT + assert "closest to a named target" in AGENTIC_QUESTION_PROMPT + assert "visibly repeated object" in AGENTIC_QUESTION_PROMPT + assert "diverse set of questions" in AGENTIC_QUESTION_PROMPT + assert "Use visibility/presence questions only" in AGENTIC_QUESTION_PROMPT + + +def test_freeform_question_author_assigns_missing_ids() -> None: + class _ModelWithoutId: + def query(self, image: Image, prompt: str) -> str: + return '[{"question":"Is there a chair?","answer_contract":{"kind":"boolean"}}]' + + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + proposals = OpenAIFreeformQuestionAuthor(cast("OpenAIVlModel", _ModelWithoutId())).propose( + image + ) + + assert proposals[0].id == "proposal-01" + + +def test_freeform_question_author_rejects_numeric_contracts() -> None: + class _NumericContractModel: + def query(self, image: Image, prompt: str) -> str: + return ( + '[{"question":"How tall is the chair?","answer_contract":' + '{"kind":"numeric","unit":"m","tolerance":0.1}}]' + ) + + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + try: + OpenAIFreeformQuestionAuthor(cast("OpenAIVlModel", _NumericContractModel())).propose(image) + except ValueError as exc: + assert "unsupported answer contract" in str(exc) + else: + raise AssertionError("numeric answer contract was accepted") + + +def test_freeform_question_author_parses_deferred_height_contract() -> None: + class _HeightContractModel: + def query(self, image: Image, prompt: str) -> str: + return ( + '[{"question":"How tall is the chair?","answer_contract":' + '{"kind":"deferred_height_choice","strategy":"height-window-v1"},' + '"object_queries":["chair"]}]' + ) + + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + proposal = OpenAIFreeformQuestionAuthor(cast("OpenAIVlModel", _HeightContractModel())).propose( + image + )[0] + + assert proposal.answer_contract == DeferredHeightChoiceContract() + assert proposal.object_queries == ("chair",) + + +def test_freeform_question_author_normalizes_optional_query_hints() -> None: + class _ModelWithStringHints: + def query(self, image: Image, prompt: str) -> str: + return ( + '[{"question":"How tall is the chair?","answer_contract":' + '{"kind":"choice","choices":["under 0.5 m","0.5-1.0 m"]},' + '"object_queries":"chair","tool_hints":null}]' + ) + + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + proposal = OpenAIFreeformQuestionAuthor(cast("OpenAIVlModel", _ModelWithStringHints())).propose( + image + )[0] + + assert proposal.object_queries == ("chair",) + assert proposal.tool_hints == () + + +def test_freeform_question_author_ignores_malformed_optional_query_hints() -> None: + class _ModelWithMalformedHints: + def query(self, image: Image, prompt: str) -> str: + return ( + '[{"question":"How tall is the chair?","answer_contract":' + '{"kind":"choice","choices":["under 0.5 m","0.5-1.0 m"]},' + '"object_queries":{"query":"chair"}}]' + ) + + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + proposal = OpenAIFreeformQuestionAuthor( + cast("OpenAIVlModel", _ModelWithMalformedHints()) + ).propose(image)[0] + + assert proposal.object_queries == () + + +def test_local_tool_returns_geometry_and_evidence_ids() -> None: + registry = LocalOracleToolRegistry(cast("Any", _Grounding())) + + detection = json.loads(registry.detect_objects("chair")) + masks = json.loads(registry.segment_detections(detection["detection_id"])) + payload = json.loads(registry.ground_masks(masks["mask_id"])) + + assert payload["objects"][0] == { + "evidence_id": "grounding:v1:synthetic-chair-0", + "id": "synthetic-chair-0", + "label": "chair", + "range_m": 1.0, + "side": "left", + "point_count": 4, + } + assert registry.results[-1].version == "v1" + + +def test_ground_plane_estimator_fits_visible_lower_band() -> None: + fit = estimate_ground_plane(_measurement_frame()) + + assert fit.rejection_reason is None + assert fit.estimate is not None + assert fit.estimate.inlier_count >= 12 + assert fit.estimate.residual_m < 0.001 + assert np.allclose(fit.estimate.normal, (0.0, -1.0, 0.0), atol=0.001) + assert fit.estimate.offset_m == 1.0 + + +def test_ground_plane_tool_returns_quality_gated_rejection() -> None: + frame = _measurement_frame(np.asarray([[0.0, 1.0, 3.0]], dtype=np.float32)) + registry = LocalOracleToolRegistry(_frame_primitives(frame)) + + payload = json.loads(registry.fit_ground_plane()) + + assert payload["measurement"] is None + assert payload["rejection_reason"] == "insufficient_support" + assert "insufficient_ground_band_points" in payload["quality_flags"] + + +def test_height_tool_measures_visible_object_points_above_plane() -> None: + frame = _measurement_frame() + mask = np.zeros((100, 100), dtype=np.uint8) + projected = project_visible_points(frame) + for (x, y), point in zip(projected.pixels, projected.camera_points, strict=True): + if point[0] > 1.5: + mask[y, x] = 255 + registry = LocalOracleToolRegistry(_frame_primitives(frame, mask)) + + detection = json.loads(registry.detect_objects("chair")) + masks = json.loads(registry.segment_detections(detection["detection_id"])) + grounded = json.loads(registry.ground_masks(masks["mask_id"])) + plane = json.loads(registry.fit_ground_plane()) + payload = json.loads(registry.measure_height(grounded["object_ids"][0], plane["plane_id"])) + + assert payload["measurement"]["unit"] == "m" + assert 0.8 < payload["measurement"]["value"] < 1.0 + assert payload["measurement"]["tolerance"] >= 0.05 + assert payload["objects"][0]["evidence_id"] == "height:v1:synthetic-chair-0" + assert "visible_point_cloud_height" in payload["quality_flags"] + + +def test_local_registry_exposes_geometry_tools() -> None: + registry = LocalOracleToolRegistry(cast("Any", _Grounding())) + + assert {tool.name for tool in registry.tools()} == { + "detect_objects", + "segment_detections", + "ground_masks", + "select_nearest_object", + "count_grounded_objects", + "bucket_camera_range", + "compare_nearest_by_side", + "compare_left_right", + "select_closest_object", + "fit_ground_plane", + "measure_height", + "compare_heights", + "classify_door_state", + "classify_forward_path", + "bucket_measurement", + } + + +def test_height_choice_window_is_local_and_deterministic() -> None: + choices, answer = height_choice_window(0.42) + + assert choices == ( + "under 0.2 m", + "0.2-0.6 m", + "0.6-1.0 m", + "over 1.0 m", + ) + assert answer == "0.2-0.6 m" + assert height_choice_window(3.0)[1] == "over 2.0 m" + + +def test_count_and_camera_range_choices_are_fixed_and_non_overlapping() -> None: + assert [count_choice(value) for value in (1, 2, 3, 4, 5, 7, 8)] == [ + "1-2", + "1-2", + "3-4", + "3-4", + "5-7", + "5-7", + "8+", + ] + assert [camera_range_choice(value) for value in (0.99, 1.0, 2.0, 4.0)] == [ + "under 1 m", + "1 to under 2 m", + "2 to under 4 m", + "4 m or more", + ] + + +def test_local_registry_buckets_count_range_and_pairwise_side() -> None: + registry = LocalOracleToolRegistry(cast("Any", _Grounding())) + left = GroundedObject("left", "chair", 8, 1.0, "left") + right = GroundedObject("right", "chair", 8, 2.0, "right") + registry._objects = {left.id: left, right.id: right} + + count = json.loads(registry.count_grounded_objects([left.id, right.id])) + camera_range = json.loads(registry.bucket_camera_range(right.id)) + side = json.loads(registry.compare_nearest_by_side([left.id, right.id])) + + assert count["choice"] == "1-2" + assert camera_range["choice"] == "2 to under 4 m" + assert side["choice"] == "left" + + +def test_local_registry_compares_pairwise_relation_and_height(monkeypatch: Any) -> None: + primitives = _frame_primitives(_measurement_frame()) + registry = LocalOracleToolRegistry(primitives) + chair = GroundedObject("chair", "chair", 8, 1.0, "left") + table = GroundedObject("table", "table", 8, 2.0, "right") + plane = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + registry._objects = {chair.id: chair, table.id: table} + registry._planes = {"plane": plane} + monkeypatch.setattr( + primitives, + "classify_horizontal_relation", + lambda first, second: HorizontalRelationResult("left", ("camera_frame_support_centroids",)), + ) + monkeypatch.setattr( + primitives, + "measure_height", + lambda item, accepted_plane: HeightMeasurementResult( + item, + accepted_plane, + OracleMeasurement(0.8 if item == chair else 0.5, "m", 0.05, (), ()), + (), + ), + ) + + relation = json.loads(registry.compare_left_right(chair.id, table.id)) + height = json.loads(registry.compare_heights(chair.id, table.id, "plane")) + + assert relation["choice"] == "left" + assert height["choice"] == "chair" + + +def test_door_state_accepts_clear_plane_angles_and_rejects_ajar() -> None: + door = GroundPlaneEstimate((1.0, 0.0, 0.0), 0.0, 20, 20, 0.01) + closed = GroundPlaneEstimate((1.0, 0.0, 0.0), 0.0, 20, 20, 0.01) + open_door = GroundPlaneEstimate((0.0, 0.0, 1.0), 0.0, 20, 20, 0.01) + ajar_door = GroundPlaneEstimate((0.95, 0.0, 0.31), 0.0, 20, 20, 0.01) + + assert classify_door_plane_angle(door, closed)[0] == "closed" + assert classify_door_plane_angle(door, open_door)[0] == "open" + assert classify_door_plane_angle(door, ajar_door)[1] == "ambiguous_door_angle" + + +def test_closest_object_uses_point_cloud_centroids_and_rejects_ties(monkeypatch: Any) -> None: + target = GroundedObject("target", "chair", 8, 1.0, "left") + close = GroundedObject("close", "table", 8, 2.0, "center") + far = GroundedObject("far", "lamp", 8, 3.0, "right") + primitives = _frame_primitives(_measurement_frame()) + centers = { + "target": np.zeros((6, 3)), + "close": np.tile((1.0, 0.0, 0.0), (6, 1)), + "far": np.tile((2.0, 0.0, 0.0), (6, 1)), + } + monkeypatch.setattr(primitives, "_object_points", lambda item: centers[item.id]) + + selected = primitives.select_closest_object(target, [close, far]) + + assert selected.object == close + assert selected.distance_m == 1.0 + monkeypatch.setattr( + primitives, + "_object_points", + lambda item: np.tile((1.0, 0.0, 0.0), (6, 1)) if item.id != "target" else centers["target"], + ) + assert ( + primitives.select_closest_object(target, [close, far]).rejection_reason + == "ambiguous_object_proximity" + ) + + +def test_horizontal_relation_uses_camera_frame_support_centroids(monkeypatch: Any) -> None: + left = GroundedObject("left", "chair", 8, 1.0, "left") + right = GroundedObject("right", "table", 8, 2.0, "right") + primitives = _frame_primitives(_measurement_frame()) + centers = { + "left": np.tile((-0.3, 0.0, 1.0), (6, 1)), + "right": np.tile((0.3, 0.0, 1.0), (6, 1)), + } + monkeypatch.setattr(primitives, "_object_points", lambda item: centers[item.id]) + + assert primitives.classify_horizontal_relation(left, right).relation == "left" + monkeypatch.setattr(primitives, "_object_points", lambda item: np.zeros((6, 3))) + assert ( + primitives.classify_horizontal_relation(left, right).rejection_reason + == "ambiguous_horizontal_relation" + ) + + +def test_forward_corridor_requires_ground_support_and_detects_obstacles() -> None: + ground = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + floor = np.asarray( + [ + [0.0, 1.0, depth] + for depth in (0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8) + ] + ) + + assert classify_forward_corridor(floor, ground)[0] == "clear" + obstacle = np.repeat(np.asarray([[0.0, 0.5, 1.0]]), 4, axis=0) + assert classify_forward_corridor(np.vstack((floor, obstacle)), ground)[0] == "blocked" + + +def test_oracle_validates_evidence_and_answer_contract() -> None: + proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract()) + result = OracleToolResult( + "ground", "chair", (OracleEvidence("e1", "v1", "o1", "chair", 1.0, "left", 3),) + ) + + assert validate_oracle_answer(proposal, "yes", ["e1"], (result,)) == "yes" + try: + validate_oracle_answer(proposal, "maybe", ["e1"], (result,)) + except ValueError as exc: + assert "boolean" in str(exc) + else: + raise AssertionError("invalid boolean answer was accepted") + try: + validate_oracle_answer(proposal, "yes", ["unknown"], (result,)) + except ValueError as exc: + assert "unknown evidence" in str(exc) + else: + raise AssertionError("unknown evidence was accepted") + + +def test_oracle_derives_deferred_height_answer_from_measurement_bucket() -> None: + proposal = QuestionProposal( + "q", "How tall is the chair?", DeferredHeightChoiceContract(), ("chair",) + ) + evidence = OracleEvidence("height-1", "v1", "chair-1", "chair", 1.0, "left", 8) + result = OracleToolResult( + "bucket_measurement", + "chair", + (evidence,), + choice="0.2-0.6 m", + choices=("under 0.2 m", "0.2-0.6 m", "0.6-1.0 m", "over 1.0 m"), + ) + + assert validate_oracle_answer(proposal, "0.2-0.6 m", ["height-1"], (result,)) == "0.2-0.6 m" + try: + validate_oracle_answer(proposal, "under 0.2 m", ["height-1"], (result,)) + except ValueError as exc: + assert "does not match measurement bucket" in str(exc) + else: + raise AssertionError("non-derived deferred height answer was accepted") + + +def test_private_oracle_runs_direct_structured_tool() -> None: + proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract(), ("chair",)) + registry = LocalOracleToolRegistry(cast("Any", _Grounding())) + validator = _SemanticValidator(SemanticEvidenceValidation(True, "chair grounding supports yes")) + + result = PrivateToolCallingOracle( + cast("Any", _BoundModel()), semantic_validator=validator + ).answer(proposal, registry) + + assert result.answer == "yes" + assert result.evidence_ids == ("grounding:v1:synthetic-chair-0",) + assert validator.calls[0][2][-1].evidence[0].id == "grounding:v1:synthetic-chair-0" + assert result.trace[-1].detail == "accepted:chair grounding supports yes" + + +def test_private_oracle_rejects_unsupported_measurement_claim() -> None: + class _ChoiceModel(_BoundModel): + def invoke(self, messages: Any) -> AIMessage: + self._calls += 1 + if self._calls == 1: + return AIMessage( + content="", + tool_calls=[ + { + "name": "detect_objects", + "args": {"query": "chair"}, + "id": "call-1", + } + ], + ) + return AIMessage( + content='{"answer":"0.5-1.0 m","evidence_ids":["grounding:v1:synthetic-chair-0"]}' + ) + + proposal = QuestionProposal( + "q", "How tall is the chair?", ChoiceAnswerContract(("under 0.5 m", "0.5-1.0 m")) + ) + registry = LocalOracleToolRegistry(cast("Any", _Grounding())) + validator = _SemanticValidator( + SemanticEvidenceValidation(False, "range and side do not measure height") + ) + + result = PrivateToolCallingOracle( + cast("Any", _ChoiceModel()), semantic_validator=validator + ).answer(proposal, registry) + + assert result.reason == "invalid_final_answer:answer cites unknown evidence" + + +def test_agentic_oracle_never_uses_legacy_answer_program() -> None: + class _GroundingWithoutLegacyAnswer(_Grounding): + def answer(self, frame: Any, intent: Any) -> None: + raise AssertionError("agentic oracle must not call legacy answer") + + proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract(), ("chair",)) + registry = LocalOracleToolRegistry(cast("Any", _GroundingWithoutLegacyAnswer())) + validator = _SemanticValidator(SemanticEvidenceValidation(True, "chair grounding supports yes")) + + result = PrivateToolCallingOracle( + cast("Any", _BoundModel()), semantic_validator=validator + ).answer(proposal, registry) + + assert result.answer == "yes" diff --git a/dimos/benchmark/vqa/generation/test_selection.py b/dimos/benchmark/vqa/generation/test_selection.py new file mode 100644 index 0000000000..2218a72df8 --- /dev/null +++ b/dimos/benchmark/vqa/generation/test_selection.py @@ -0,0 +1,15 @@ +# Copyright 2026 Dimensional Inc. + +from dimos.benchmark.vqa.generation.primitives.selection import select_nearest_object +from dimos.benchmark.vqa.models import GroundedObject + + +def test_select_nearest_object_optionally_restricts_to_image_side() -> None: + objects = [ + GroundedObject("left", "chair", 3, 2.0, "left"), + GroundedObject("right", "chair", 3, 1.0, "right"), + ] + + assert select_nearest_object(objects).id == "right" + assert select_nearest_object(objects, "left").id == "left" + assert select_nearest_object(objects, "center") is None diff --git a/dimos/benchmark/vqa/generation/test_single_frame.py b/dimos/benchmark/vqa/generation/test_single_frame.py new file mode 100644 index 0000000000..be6ef78c29 --- /dev/null +++ b/dimos/benchmark/vqa/generation/test_single_frame.py @@ -0,0 +1,64 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import numpy as np + +from dimos.benchmark.vqa.generation.pipeline import generate_ground_truth +from dimos.benchmark.vqa.models import CalibratedFrame +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg + + +class _Detector: + def __init__(self, image: Image, detection: Detection2DSeg) -> None: + self._image = image + self._detection = detection + + def detect(self, image: Image, query: str) -> ImageDetections2D: + assert image is self._image + return ImageDetections2D(image, [self._detection] if query == "chair" else []) + + +class _Segmenter: + def segment(self, detections: ImageDetections2D) -> ImageDetections2D: + return detections + + +def test_single_frame_ground_truth_generates_multiple_choice_cases() -> None: + image = Image.from_numpy(np.zeros((6, 6, 3), dtype=np.uint8)) + frame = CalibratedFrame( + id="frame-1", + image=image, + pointcloud=PointCloud2.from_numpy( + np.array([[0.0, 0.0, 1.0], [0.4, 0.0, 1.0], [-0.4, 0.0, 1.0]], dtype=np.float32) + ), + camera_info=CameraInfo.from_intrinsics(3.0, 3.0, 3.0, 3.0, 6, 6), + pointcloud_to_camera=Transform.identity(), + image_is_rectified=True, + ) + mask = np.full((6, 6), 255, dtype=np.uint8) + detection = Detection2DSeg((0.0, 0.0, 5.0, 5.0), 0, -1, 1.0, "chair", 0.0, image, mask) + + examples = generate_ground_truth( + frame, ["chair", "table"], _Detector(image, detection), _Segmenter() + ) + + assert {example.expected_answer for example in examples} == {"yes", "no", "center"} + assert all(example.expected_answer in example.allowed_answers for example in examples) diff --git a/dimos/benchmark/vqa/models.py b/dimos/benchmark/vqa/models.py new file mode 100644 index 0000000000..33c2f81061 --- /dev/null +++ b/dimos/benchmark/vqa/models.py @@ -0,0 +1,277 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Contracts shared by the single-frame perception VQA pipeline.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal, Protocol + +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D + + +@dataclass(frozen=True) +class CalibratedFrame: + """One self-contained image and point-cloud pair for VQA generation.""" + + id: str + image: Image + pointcloud: PointCloud2 + camera_info: CameraInfo + pointcloud_to_camera: Transform + image_is_rectified: bool + original_image: Image | None = None + + +@dataclass(frozen=True) +class ProjectionConfig: + """Controls static pinhole projection of a frame's point cloud.""" + + min_depth_m: float = 0.01 + + +@dataclass(frozen=True) +class GroundingConfig: + """Quality thresholds for accepting an image mask as a grounded object.""" + + min_mask_area_px: int = 128 + min_foreground_points: int = 3 + + +@dataclass(frozen=True) +class ProjectedPoints: + """Visible point-cloud samples represented in the camera image.""" + + camera_points: list[tuple[float, float, float]] + pixels: list[tuple[int, int]] + source_indices: list[int] + + +@dataclass(frozen=True) +class GroundedObject: + """One semantic object with foreground point-cloud support.""" + + id: str + label: str + point_count: int + range_m: float + horizontal_direction: str + + +@dataclass(frozen=True) +class VqaExample: + """A closed-answer question generated from grounded objects.""" + + id: str + question: str + expected_answer: str + answer_type: str + object_ids: tuple[str, ...] + allowed_answers: tuple[str, ...] = () + + +QuestionKind = Literal[ + "presence", + "horizontal_direction", + "within_distance", + "visible_count", + "camera_range", + "compare_nearest_by_side", + "compare_left_right", + "compare_height", + "door_state", + "closest_object", + "forward_path", +] + + +@dataclass(frozen=True) +class QuestionIntent: + """A constrained question proposed from an image.""" + + kind: QuestionKind + object_query: str + threshold_m: float | None = None + candidate_queries: tuple[str, ...] = () + comparison_query: str | None = None + + +@dataclass(frozen=True) +class ToolTrace: + """One perception operation used to establish a ground-truth answer.""" + + tool: str + detail: str + + +@dataclass(frozen=True) +class GroundTruthResult: + """An answered or rejected question with private perception evidence.""" + + intent: QuestionIntent + question: VqaExample + status: Literal["answered", "rejected"] + answer: str | None + reason: str | None + evidence: tuple[GroundedObject, ...] + trace: tuple[ToolTrace, ...] + + +@dataclass(frozen=True) +class BooleanAnswerContract: + """A yes/no answer required from a private oracle.""" + + kind: Literal["boolean"] = "boolean" + + +@dataclass(frozen=True) +class ChoiceAnswerContract: + """An answer selected exactly from the supplied choices.""" + + choices: tuple[str, ...] + kind: Literal["choice"] = "choice" + + +@dataclass(frozen=True) +class DeferredHeightChoiceContract: + """A height question whose public choices follow a private measurement.""" + + strategy: Literal["height-window-v1"] = "height-window-v1" + kind: Literal["deferred_height_choice"] = "deferred_height_choice" + + +AnswerContract = BooleanAnswerContract | ChoiceAnswerContract | DeferredHeightChoiceContract +ResolvedAnswerContract = BooleanAnswerContract | ChoiceAnswerContract + + +@dataclass(frozen=True) +class QuestionProposal: + """Public image-only question frozen before private oracle execution.""" + + id: str + question: str + answer_contract: AnswerContract + object_queries: tuple[str, ...] = () + tool_hints: tuple[str, ...] = () + + +@dataclass(frozen=True) +class OracleEvidence: + """A private evidence item emitted by a registered local tool.""" + + id: str + version: str + object_id: str + label: str + range_m: float + side: str + point_count: int + measurement: OracleMeasurement | None = None + + +@dataclass(frozen=True) +class OracleMeasurement: + """A private scalar measurement with its uncertainty and provenance.""" + + value: float + unit: str + tolerance: float + quality_flags: tuple[str, ...] + provenance_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class GroundPlaneEstimate: + """A robust ground-plane fit in frozen camera coordinates.""" + + normal: tuple[float, float, float] + offset_m: float + sample_count: int + inlier_count: int + residual_m: float + + +@dataclass(frozen=True) +class OracleToolResult: + """Structured result and evidence IDs from one local tool invocation.""" + + tool: str + query: str + evidence: tuple[OracleEvidence, ...] + version: str = "v1" + measurement: OracleMeasurement | None = None + choice: str | None = None + choices: tuple[str, ...] = () + plane: GroundPlaneEstimate | None = None + quality_flags: tuple[str, ...] = () + rejection_reason: str | None = None + + +@dataclass(frozen=True) +class OracleTrace: + """Audit record for a private oracle model or tool operation.""" + + operation: str + detail: str + + +@dataclass(frozen=True) +class AcceptedOracleResult: + """Validated private answer for a frozen public proposal.""" + + proposal: QuestionProposal + answer: str + answer_contract: ResolvedAnswerContract + evidence_ids: tuple[str, ...] + tool_results: tuple[OracleToolResult, ...] + trace: tuple[OracleTrace, ...] + + +@dataclass(frozen=True) +class RejectedOracleResult: + """Private oracle attempt that cannot be safely exported as a case.""" + + proposal: QuestionProposal + reason: str + tool_results: tuple[OracleToolResult, ...] + trace: tuple[OracleTrace, ...] + + +class ObjectDetector(Protocol): + """Produces object-query detections from one image.""" + + def detect(self, image: Image, query: str) -> ImageDetections2D: ... + + +class ObjectSegmenter(Protocol): + """Refines object detections into foreground masks in one image.""" + + def segment(self, detections: ImageDetections2D) -> ImageDetections2D: ... + + +class ObjectPointLocalizer(Protocol): + """Locates a queried object with positive image points.""" + + def locate(self, image: Image, query: str) -> ImageDetections2D: ... + + +class PointObjectSegmenter(Protocol): + """Creates foreground masks from positive image-point prompts.""" + + def segment_points(self, points: ImageDetections2D) -> ImageDetections2D: ... diff --git a/dimos/benchmark/vqa/test_evaluation.py b/dimos/benchmark/vqa/test_evaluation.py new file mode 100644 index 0000000000..2fab9b85c8 --- /dev/null +++ b/dimos/benchmark/vqa/test_evaluation.py @@ -0,0 +1,47 @@ +# Copyright 2026 Dimensional Inc. + +import json +from pathlib import Path +from typing import Any, cast + +import cv2 +import numpy as np + +from dimos.benchmark.evaluation.models import ArtifactNativeResult +from dimos.benchmark.evaluation.protocol import EvaluationContext +from dimos.benchmark.vqa.evaluation import MultipleChoiceVqaEvaluation, VqaEvaluationConfig + + +class _VisionModel: + def query(self, image: Any, prompt: str) -> str: + assert "Choices: left, right." in prompt + return "ANSWER: left" + + +def test_vqa_evaluation_uses_only_public_case_image_and_private_label(tmp_path: Path) -> None: + dataset = tmp_path / "dataset" + dataset.mkdir() + assert cv2.imwrite(str(dataset / "image.jpg"), np.zeros((1, 1, 3), dtype=np.uint8)) + (dataset / "cases.jsonl").write_text( + json.dumps( + { + "id": "case-1", + "image": "image.jpg", + "question": "Which side?", + "choices": ["left", "right"], + } + ) + + "\n" + ) + (dataset / "labels.jsonl").write_text(json.dumps({"id": "case-1", "answer": "left"}) + "\n") + workspace = tmp_path / "workspace" + workspace.mkdir() + + report = MultipleChoiceVqaEvaluation(lambda _: cast("Any", _VisionModel())).run( + VqaEvaluationConfig(dataset=str(dataset)), + EvaluationContext("run", tmp_path, workspace, cast("Any", None), None), + ) + + assert report.summary[1].value == 1 + assert isinstance(report.native_result, ArtifactNativeResult) + assert json.loads((workspace / "vqa-results.json").read_text())[0]["passed"] is True diff --git a/dimos/cli/dimos.py b/dimos/cli/dimos.py index 0f50caaaea..9137621c5f 100644 --- a/dimos/cli/dimos.py +++ b/dimos/cli/dimos.py @@ -53,6 +53,7 @@ from dimos.cli.eval import app as eval_app from dimos.cli.hardware_cli import app as hardware_app from dimos.cli.shell import shell +from dimos.cli.vqa import app as vqa_app from dimos.constants import CONFIG_DIR, LOG_DIR from dimos.core.daemon import daemonize, install_signal_handlers from dimos.core.global_config import GlobalConfig, global_config @@ -77,6 +78,7 @@ help="Dimensional CLI", no_args_is_help=True, ) +main.add_typer(vqa_app, name="vqa") load_dotenv() diff --git a/dimos/cli/test_vqa.py b/dimos/cli/test_vqa.py new file mode 100644 index 0000000000..abf7e5bf49 --- /dev/null +++ b/dimos/cli/test_vqa.py @@ -0,0 +1,70 @@ +# Copyright 2026 Dimensional Inc. + +import json +from pathlib import Path + +from typer.testing import CliRunner + +from dimos.benchmark.vqa.generation.specification import VqaGenerationSpecification +from dimos.cli import vqa + + +def test_vqa_generation_cli_has_no_explicit_query_or_model_options() -> None: + output = CliRunner().invoke(vqa.app, ["generate", "--help"]).output + + assert "--query" not in output + assert "--propose-questions" not in output + assert "--question-model" not in output + assert "--oracle-model" not in output + assert "--spec" in output + + +def test_generation_spec_resolves_the_same_options_as_the_cli(tmp_path: Path) -> None: + spec = tmp_path / "generation.json" + spec.write_text( + json.dumps( + { + "recording": "go2_bigoffice.db", + "start_index": 10, + "stop_index": 40, + "stride": 5, + "question_mode": "agentic", + "grounding": {"min_mask_area_px": 256, "min_foreground_points": 4}, + "output": "/tmp/vqa", + } + ) + ) + + generation = vqa._resolve_generation_spec(spec, None, None, None, None, None, None, None, None) + + assert generation.recording == "go2_bigoffice.db" + assert generation.question_mode == "agentic" + assert generation.grounding.min_mask_area_px == 256 + assert generation.output == "/tmp/vqa" + + +def test_generation_spec_rejects_mixed_cli_options(tmp_path: Path) -> None: + spec = tmp_path / "generation.json" + spec.write_text('{"recording":"go2.db","stop_index":10}') + + result = CliRunner().invoke( + vqa.app, + ["generate", "--spec", str(spec), "--recording", "other.db"], + ) + + assert result.exit_code != 0 + assert "cannot be combined" in result.output + + +def test_generation_run_records_resolved_request(tmp_path: Path) -> None: + vqa._write_generation_run( + tmp_path, + VqaGenerationSpecification(recording="go2.db", stop_index=10), + {"frame_count": 2, "accepted_question_count": 3, "rejected_question_count": 1}, + ) + + payload = json.loads((tmp_path / "run.json").read_text()) + + assert payload["generation"]["recording"] == "go2.db" + assert payload["generation"]["output"] == str(tmp_path) + assert payload["summary"]["accepted_question_count"] == 3 diff --git a/dimos/cli/vqa.py b/dimos/cli/vqa.py new file mode 100644 index 0000000000..3d0260dc12 --- /dev/null +++ b/dimos/cli/vqa.py @@ -0,0 +1,373 @@ +# Copyright 2026 Dimensional Inc. +"""Single-frame point-cloud-grounded VQA commands.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import cast + +import typer + +from dimos.benchmark.vqa.generation.adapters import ( + EdgeTamObjectSegmenter, + MoondreamObjectDetector, +) +from dimos.benchmark.vqa.generation.dataset import write_dataset_manifest, write_frame_record +from dimos.benchmark.vqa.generation.ground_truth_generator import VqaGroundTruthGenerator +from dimos.benchmark.vqa.generation.oracle import create_openai_oracle +from dimos.benchmark.vqa.generation.oracle_tools import LocalOracleToolRegistry +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives +from dimos.benchmark.vqa.generation.question_agent import ( + OpenAIFreeformQuestionAuthor, + OpenAIQuestionAgent, +) +from dimos.benchmark.vqa.generation.recording import load_go2_frame +from dimos.benchmark.vqa.generation.specification import VqaGenerationSpecification +from dimos.benchmark.vqa.models import ( + AcceptedOracleResult, + CalibratedFrame, + GroundingConfig, + GroundTruthResult, + QuestionIntent, + QuestionProposal, + RejectedOracleResult, +) +from dimos.constants import STATE_DIR +from dimos.models.base import default_local_model_device +from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter +from dimos.models.vl.moondream import MoondreamVlModel +from dimos.models.vl.openai import OpenAIVlModel +from dimos.utils.data import resolve_named_path + +app = typer.Typer(help="Generate point-cloud-grounded VQA benchmark examples") +QUESTION_MODEL = "gpt-4o-mini" +ORACLE_MODEL = "gpt-4o-mini" + + +@app.command("single-frame") +def single_frame( + recording: str = typer.Option(..., "--recording"), + frame_index: int = typer.Option(0, "--frame-index"), + question_mode: str = typer.Option("constrained", "--question-mode"), + min_mask_area_px: int = typer.Option(128, "--min-mask-area-px"), + min_foreground_points: int = typer.Option(3, "--min-foreground-points"), + output: Path | None = typer.Option(None, "--output"), +) -> None: + """Generate private-grounded questions for one Go2 recording frame.""" + output = output or ( + STATE_DIR / "datasets" / "vqa" / f"{Path(recording).stem}-frame-{frame_index:06d}" + ) + if output.exists(): + raise typer.BadParameter("output must not already exist") + _validate_question_mode(question_mode) + _require_openai_for_question_author() + _require_edgetam_cuda() + typer.echo(f"Loading frame {frame_index} from {recording}") + frame = load_go2_frame(str(resolve_named_path(recording, ".db")), frame_index) + model = MoondreamVlModel() + typer.echo("Loading private MoonDream model") + model.start() + question_agent = OpenAIQuestionAgent(OpenAIVlModel(model_name=QUESTION_MODEL)) + try: + typer.echo(f"Proposing questions with {QUESTION_MODEL}") + intents: list[QuestionIntent] | list[QuestionProposal] = ( + OpenAIFreeformQuestionAuthor(OpenAIVlModel(model_name=QUESTION_MODEL)).propose( + frame.image + ) + if question_mode == "agentic" + else question_agent.propose(frame.image) + ) + typer.echo(f"Grounding {len(intents)} questions for frame {frame_index}") + primitives = FramePerceptionPrimitives( + frame, + detector := MoondreamObjectDetector(model), + segmenter := EdgeTamObjectSegmenter(EdgeTAMImageSegmenter()), + localizer=detector, + point_segmenter=segmenter, + config=GroundingConfig( + min_mask_area_px=min_mask_area_px, + min_foreground_points=min_foreground_points, + ), + ) + ground_truth = VqaGroundTruthGenerator(primitives) + results: list[GroundTruthResult] | list[AcceptedOracleResult | RejectedOracleResult] = ( + _answer_agentic(ground_truth, frame, cast("list[QuestionProposal]", intents)) + if question_mode == "agentic" + else _answer_intents( + ground_truth, frame, cast("list[QuestionIntent]", intents), f"Frame {frame_index}" + ) + ) + examples = [ + result + for result in results + if isinstance(result, AcceptedOracleResult) + or (isinstance(result, GroundTruthResult) and result.status == "answered") + ] + finally: + model.stop() + write_frame_record( + output, + frame, + recording, + frame_index, + cast("list[QuestionIntent | QuestionProposal]", intents), + cast("list[GroundTruthResult | AcceptedOracleResult | RejectedOracleResult]", results), + { + "question_source": "agentic_image_author" + if question_mode == "agentic" + else "openai_image_agent", + "question_model": QUESTION_MODEL, + "oracle_model": ORACLE_MODEL if question_mode == "agentic" else None, + "grounding": { + "min_mask_area_px": min_mask_area_px, + "min_foreground_points": min_foreground_points, + }, + }, + ) + typer.echo(f"Wrote {len(examples)} examples to {output}") + + +@app.command("generate") +def generate( + recording: str | None = typer.Option(None, "--recording"), + start_index: int | None = typer.Option(None, "--start-index"), + stop_index: int | None = typer.Option(None, "--stop-index"), + stride: int | None = typer.Option(None, "--stride"), + question_mode: str | None = typer.Option(None, "--question-mode"), + min_mask_area_px: int | None = typer.Option(None, "--min-mask-area-px"), + min_foreground_points: int | None = typer.Option(None, "--min-foreground-points"), + output: Path | None = typer.Option(None, "--output"), + spec: Path | None = typer.Option(None, "--spec", exists=True, dir_okay=False, readable=True), +) -> None: + """Generate a resumable VQA dataset from sampled Go2 recording frames.""" + generation = _resolve_generation_spec( + spec, + recording, + start_index, + stop_index, + stride, + question_mode, + min_mask_area_px, + min_foreground_points, + output, + ) + if generation.stop_index <= generation.start_index: + raise typer.BadParameter("provide valid frame bounds") + recording = generation.recording + start_index = generation.start_index + stop_index = generation.stop_index + stride = generation.stride + question_mode = generation.question_mode + min_mask_area_px = generation.grounding.min_mask_area_px + min_foreground_points = generation.grounding.min_foreground_points + output = ( + Path(generation.output).expanduser() + if generation.output is not None + else STATE_DIR / "datasets" / "vqa" / f"{Path(recording).stem}-frames" + ) + _require_openai_for_question_author() + output.mkdir(parents=True, exist_ok=True) + _require_edgetam_cuda() + frame_indices = range(start_index, stop_index, stride) + typer.echo(f"Generating {len(frame_indices)} sampled frames from {recording} into {output}") + model = MoondreamVlModel() + typer.echo("Loading private MoonDream model") + model.start() + try: + detector = MoondreamObjectDetector(model) + segmenter = EdgeTamObjectSegmenter(EdgeTAMImageSegmenter()) + question_agent = OpenAIQuestionAgent(OpenAIVlModel(model_name=QUESTION_MODEL)) + for frame_number, frame_index in enumerate(frame_indices, start=1): + frame_output = output / f"frame-{frame_index:06d}" + if (frame_output / "frame.json").is_file(): + typer.echo( + f"Skipping completed frame {frame_number}/{len(frame_indices)}: {frame_index}" + ) + continue + typer.echo( + f"Frame {frame_number}/{len(frame_indices)}: loading recording index {frame_index}" + ) + frame = load_go2_frame(str(resolve_named_path(recording, ".db")), frame_index) + typer.echo(f"Frame {frame_index}: proposing questions with {QUESTION_MODEL}") + intents: list[QuestionIntent] | list[QuestionProposal] = ( + OpenAIFreeformQuestionAuthor(OpenAIVlModel(model_name=QUESTION_MODEL)).propose( + frame.image + ) + if question_mode == "agentic" + else question_agent.propose(frame.image) + ) + typer.echo(f"Frame {frame_index}: grounding {len(intents)} questions") + primitives = FramePerceptionPrimitives( + frame, + detector, + segmenter, + localizer=detector, + point_segmenter=segmenter, + config=GroundingConfig( + min_mask_area_px=min_mask_area_px, min_foreground_points=min_foreground_points + ), + ) + ground_truth = VqaGroundTruthGenerator(primitives) + results: list[GroundTruthResult] | list[AcceptedOracleResult | RejectedOracleResult] = ( + _answer_agentic(ground_truth, frame, cast("list[QuestionProposal]", intents)) + if question_mode == "agentic" + else _answer_intents( + ground_truth, + frame, + cast("list[QuestionIntent]", intents), + f"Frame {frame_index}", + ) + ) + write_frame_record( + frame_output, + frame, + recording, + frame_index, + cast("list[QuestionIntent | QuestionProposal]", intents), + cast( + "list[GroundTruthResult | AcceptedOracleResult | RejectedOracleResult]", results + ), + { + "question_source": "agentic_image_author" + if question_mode == "agentic" + else "openai_image_agent", + "question_model": QUESTION_MODEL, + "oracle_model": ORACLE_MODEL if question_mode == "agentic" else None, + "grounding": { + "min_mask_area_px": min_mask_area_px, + "min_foreground_points": min_foreground_points, + }, + }, + ) + typer.echo(f"Generated {frame_output}") + finally: + model.stop() + summary = write_dataset_manifest(output) + _write_generation_run(output, generation, summary) + typer.echo(f"Dataset manifest: {summary}") + + +def _answer_intents( + ground_truth: VqaGroundTruthGenerator, + frame: CalibratedFrame, + intents: list[QuestionIntent], + label: str, +) -> list[GroundTruthResult]: + results: list[GroundTruthResult] = [] + for number, intent in enumerate(intents, start=1): + typer.echo( + f"{label}: grounding question {number}/{len(intents)}: " + f"{intent.kind} ({intent.object_query})" + ) + result = ground_truth.answer(frame, intent) + results.append(result) + typer.echo(f"{label}: question {number}/{len(intents)} {result.status}") + return results + + +def _answer_agentic( + ground_truth: VqaGroundTruthGenerator, + frame: CalibratedFrame, + proposals: list[QuestionProposal], +) -> list[AcceptedOracleResult | RejectedOracleResult]: + oracle = create_openai_oracle(ORACLE_MODEL) + results: list[AcceptedOracleResult | RejectedOracleResult] = [] + for number, proposal in enumerate(proposals, start=1): + typer.echo(f"Agentic question {number}/{len(proposals)}: {proposal.question}") + result = oracle.answer(proposal, LocalOracleToolRegistry(ground_truth.primitives)) + results.append(result) + if isinstance(result, AcceptedOracleResult): + typer.echo(f"Agentic question {number}/{len(proposals)} accepted") + else: + typer.echo(f"Agentic question {number}/{len(proposals)} rejected: {result.reason}") + return results + + +def _validate_question_mode(question_mode: str) -> None: + if question_mode not in ("constrained", "agentic"): + raise typer.BadParameter("question mode must be constrained or agentic") + + +def _resolve_generation_spec( + spec: Path | None, + recording: str | None, + start_index: int | None, + stop_index: int | None, + stride: int | None, + question_mode: str | None, + min_mask_area_px: int | None, + min_foreground_points: int | None, + output: Path | None, +) -> VqaGenerationSpecification: + """Load a JSON generation specification or resolve the explicit CLI alternatives.""" + values = ( + recording, + start_index, + stop_index, + stride, + question_mode, + min_mask_area_px, + min_foreground_points, + output, + ) + if spec is not None: + if any(value is not None for value in values): + raise typer.BadParameter("--spec cannot be combined with generation options") + try: + return VqaGenerationSpecification.model_validate_json(spec.read_bytes()) + except ValueError as exc: + raise typer.BadParameter(f"invalid generation specification: {exc}") from exc + if recording is None or stop_index is None: + raise typer.BadParameter("--recording and --stop-index are required without --spec") + try: + return VqaGenerationSpecification( + recording=recording, + start_index=0 if start_index is None else start_index, + stop_index=stop_index, + stride=1 if stride is None else stride, + question_mode="constrained" if question_mode is None else question_mode, + grounding={ + "min_mask_area_px": 128 if min_mask_area_px is None else min_mask_area_px, + "min_foreground_points": 3 + if min_foreground_points is None + else min_foreground_points, + }, + output=str(output) if output is not None else None, + ) + except ValueError as exc: + raise typer.BadParameter(f"invalid generation options: {exc}") from exc + + +def _write_generation_run( + output: Path, + generation: VqaGenerationSpecification, + summary: dict[str, int], +) -> None: + """Record the resolved request that produced one generated dataset.""" + payload = { + "schema_version": "1.0", + "generation": { + **generation.model_dump(mode="json"), + "output": str(output), + }, + "models": { + "question_author": QUESTION_MODEL, + "oracle": ORACLE_MODEL if generation.question_mode == "agentic" else None, + }, + "summary": summary, + } + (output / "run.json").write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + +def _require_openai_for_question_author() -> None: + if not os.environ.get("OPENAI_API_KEY"): + raise typer.BadParameter("OPENAI_API_KEY must be set for image-authored question modes") + + +def _require_edgetam_cuda() -> None: + if default_local_model_device() != "cuda": + raise typer.BadParameter( + "VQA generation requires an installed PyTorch CUDA build that supports this GPU for EdgeTAM" + ) diff --git a/dimos/models/base.py b/dimos/models/base.py index 65393405c5..0fd8291366 100644 --- a/dimos/models/base.py +++ b/dimos/models/base.py @@ -28,8 +28,20 @@ DeviceType = Annotated[str, "Device identifier (e.g., 'cuda', 'cpu', 'cuda:0')"] +def default_local_model_device() -> str: + """Select CUDA only when this Torch build supports the detected GPU.""" + if not torch.cuda.is_available(): + return "cpu" + try: + major, minor = torch.cuda.get_device_capability() + capability = f"sm_{major}{minor}" + return "cuda" if capability in torch.cuda.get_arch_list() else "cpu" + except RuntimeError: + return "cpu" + + class LocalModelConfig(BaseConfig): - device: DeviceType = "cuda" if torch.cuda.is_available() else "cpu" + device: DeviceType = default_local_model_device() dtype: torch.dtype = torch.float32 warmup: bool = False autostart: bool = False diff --git a/dimos/models/segmentation/edge_tam.py b/dimos/models/segmentation/edge_tam.py index 72f1484af8..3f644bd0a0 100644 --- a/dimos/models/segmentation/edge_tam.py +++ b/dimos/models/segmentation/edge_tam.py @@ -30,6 +30,7 @@ from dimos.perception.detection.detectors.base import Detector from dimos.perception.detection.type.detection2d.bbox import Detection2DBBox from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D +from dimos.perception.detection.type.detection2d.point import Detection2DPoint from dimos.perception.detection.type.detection2d.seg import Detection2DSeg from dimos.utils.data import get_data from dimos.utils.logging_config import setup_logger @@ -132,6 +133,37 @@ def segment(self, detections: ImageDetections2D) -> ImageDetections2D: ] return ImageDetections2D(image, segmented) + def segment_points(self, points: ImageDetections2D) -> ImageDetections2D: + """Create one foreground mask for every positive point prompt.""" + import cv2 + + if not len(points): + return points + image = points.image + rgb = cv2.cvtColor(image.to_opencv(), cv2.COLOR_BGR2RGB) + segmented: list[Detection2DBBox] = [] + with torch.no_grad(), torch.autocast("cuda", dtype=torch.bfloat16): + self._predictor.set_image(rgb) + for point in points: + if not isinstance(point, Detection2DPoint): + continue + mask, _, _ = self._predictor.predict( + point_coords=np.array([[point.x, point.y]], dtype=np.float32), + point_labels=np.array([1], dtype=np.int32), + multimask_output=False, + ) + segmented.append( + Detection2DSeg.from_sam2_result( + mask.squeeze(), + point.track_id, + image, + class_id=point.class_id, + name=point.name, + confidence=point.confidence, + ) + ) + return ImageDetections2D(image, segmented) + class EdgeTAMProcessor(Detector): _predictor: "SAM2VideoPredictor" diff --git a/docs/benchmarking/vqa-generation/infrastructure.md b/docs/benchmarking/vqa-generation/infrastructure.md new file mode 100644 index 0000000000..bfd03e4a9f --- /dev/null +++ b/docs/benchmarking/vqa-generation/infrastructure.md @@ -0,0 +1,161 @@ +--- +title: "VQA Generation Infrastructure" +--- + +# VQA Generation Infrastructure + +This document describes the software stack behind the point-cloud-grounded VQA benchmark. For the +step-by-step generation procedure and dataset schema, see [Pipeline](/docs/benchmarking/vqa-generation/pipeline.md). + +## Design Boundary + +The benchmark has two intentionally separate phases: + +1. Generation privately uses calibrated point clouds and perception tools to establish labels. +2. Evaluation receives only public images, questions, choices, and private expected-choice labels. + +The evaluated vision model never receives point clouds, camera calibration, masks, measurements, +tool traces, or rejected attempts. + +```text +recording + calibration + point cloud + | + v + private generation runtime + | + +-- public: image.jpg, cases.jsonl + +-- private: labels.jsonl, ground_truth.json + | + v + image-only evaluation runtime + | + v + run.json + vqa-results.json +``` + +## Package Layout + +```text +dimos/benchmark/vqa/ + models.py shared immutable VQA contracts + generation/ + primitives/ + frame.py cached frame-scoped perception runtime + contracts.py typed private primitive results + geometry.py plane fitting and masked-point helpers + selection.py nearest-object selection + choices.py deterministic answer-choice resolution + ground_truth_generator.py deterministic constrained recipe runner + oracle_tools.py opaque-ID and LangChain adapter for primitives + oracle.py bounded tool-calling and evidence validation + question_agent.py image-only question author + specification.py validated generation-specification schema + dataset.py frame records and evaluation export + evaluation.py shared point-cloud-vqa Evaluation plugin + +dimos/cli/vqa.py generation CLI commands +``` + +## Generation Runtime + +`dimos vqa generate` and `dimos vqa single-frame` load a frozen Go2 recording frame containing: + +- A rectified RGB image. +- A calibrated visible point cloud. +- Camera intrinsics and the point-cloud-to-camera transform. + +The generator accepts either explicit CLI flags or a reproducible JSON specification through +`dimos vqa generate --spec `. It writes the resolved generation request and +aggregate counts to `run.json` at the dataset root. The generator constructs one +`FramePerceptionPrimitives` instance per frame. It owns MoonDream and +EdgeTAM calls, intermediate-result caches, grounded masks, and the accepted ground-plane fit. +Projected visible point-cloud samples establish whether a mask has enough foreground support to +become a grounded object. The generation runtime writes complete frame directories, so multi-frame +generation can skip completed frames after an interrupted run. + +### Shared Perception Primitives + +Constrained and agentic generation use the same private primitives: + +```text +detect_objects(query) +-> segment_detections(detection_id) +-> ground_masks(mask_id) +-> select_nearest_object(object_ids, side) +-> fit_ground_plane() +-> measure_height(object_id, plane_id) +-> bucket_measurement(measurement_id) +``` + +Constrained generation selects a fixed sequence for each question family. Agentic generation lets +the oracle select a bounded sequence of these tools, passing opaque IDs rather than masks or point +arrays between calls. + +## Question And Answer Contracts + +The image author sees only RGB and proposes a question plus one of these contracts: + +- Boolean: the final public choices are `yes` and `no`. +- Fixed choice: the author supplies at least two public choices. +- Deferred height choice: the author freezes the question, then private geometry deterministically + creates the public height choices and matching answer after a valid measurement. + +The deferred height contract prevents the image author from guessing a numeric range. The private +`height-window-v1` policy derives four local, exhaustive choices around a measured height. The raw +measurement, uncertainty, plane fit, and support remain private. + +## Private Validation + +Every accepted agentic answer must have valid cited evidence. The deterministic validator checks: + +- Cited IDs were emitted by private tools for the current frame. +- The answer is allowed by the resolved public contract. +- A deferred height answer exactly matches the cited measurement bucket. + +A private semantic validator then determines whether the cited structured evidence supports the +question and answer. Invalid calls, inadequate geometry, bad citations, and unsupported answers +are retained as private rejected records rather than exported as evaluation cases. + +## Dataset Assembly + +Each completed frame retains the public `image.jpg`, resumability metadata, private generation +audit, and temporary per-frame case/label rows. `write_dataset_manifest()` aggregates those rows +into root `cases.jsonl` and `labels.jsonl` files. + +The root dataset is deliberately minimal: + +```text +cases.jsonl public case ID, image path, question, and choices +labels.jsonl private case ID and expected choice +frame-*/image.jpg +``` + +`ground_truth.json` is not an evaluator dependency. It exists solely to make generated labels +auditable. + +## Evaluation Runtime + +`point-cloud-vqa` is registered in the shared evaluation framework and is invoked through: + +```bash +dimos eval run --output +``` + +For each case, the evaluator loads the referenced public image, sends the question and choices to +the configured vision model, normalizes an `ANSWER: ` response, and compares it with the +private label. It writes `vqa-results.json` with the expected answer, normalized model answer, raw +model response, and pass/fail result. The shared evaluation runtime also writes its immutable +`run.json` record. + +## Operational Dependencies + +Generation requires local CUDA support for EdgeTAM and credentials for the image author and, in +agentic mode, the private oracle and semantic validator. Evaluation requires only the selected +vision-model credentials and the exported public images/cases plus private labels; it has no +perception-model, CUDA, recording, or point-cloud dependency. + +## Future Work + +Evaluation visualization is intentionally separate from generation. The planned report should be +derived from evaluator inputs and `vqa-results.json` only, so it can show public images, questions, +choices, predictions, labels, and pass/fail results without exposing private generation evidence. diff --git a/docs/benchmarking/vqa-generation/pipeline.md b/docs/benchmarking/vqa-generation/pipeline.md new file mode 100644 index 0000000000..487c0bbfbe --- /dev/null +++ b/docs/benchmarking/vqa-generation/pipeline.md @@ -0,0 +1,296 @@ +--- +title: "VQA Generation Pipeline" +--- + +# VQA Generation Pipeline + +## 1. Run Generation + +```bash +dimos vqa generate \ + --recording \ + --start-index \ + --stop-index \ + --stride \ + --question-mode constrained|agentic \ + --output +``` + +Flags: + +- `--recording`: Memory2 Go2 recording path or named recording. +- `--start-index`, `--stop-index`, `--stride`: sampled recording frames. +- `--question-mode`: `constrained` or `agentic`; default is `constrained`. +- `--min-mask-area-px`: minimum accepted segmentation-mask area; default `128`. +- `--min-foreground-points`: minimum point-cloud support inside a mask; default `3`. +- `--output`: dataset root; completed frame directories are skipped on rerun. + +`dimos vqa single-frame` accepts the same generation settings and uses `--frame-index` instead of frame bounds. + +### Generation Specification + +`dimos vqa generate --spec ` is an alternative to the explicit generation flags. +Do not combine `--spec` with `--recording`, frame bounds, question mode, grounding thresholds, or +output flags. A specification is a reproducible generation request: + +```json +{ + "recording": "go2_bigoffice.db", + "start_index": 0, + "stop_index": 100, + "stride": 20, + "question_mode": "agentic", + "grounding": { + "min_mask_area_px": 128, + "min_foreground_points": 3 + }, + "output": "~/.local/state/dimos/datasets/vqa/go2-bigoffice" +} +``` + +Generation writes the resolved request, model IDs, and aggregate counts to the dataset root's +private `run.json`. This is an output record, not the input specification. + +## 2. Create Questions + +### Constrained + +The image author inspects the scene and returns up to five structured intents. It selects only +families likely to be useful for the visible arrangement, rather than expanding every object into +every family. Private grounding still rejects unsupported or ambiguous candidates. + +| Family | Sample question | Choices | +|---|---|---| +| Presence | `Is there a chair in the image?` | `yes`, `no` | +| Horizontal direction | `Where is the nearest chair?` | `left`, `center`, `right` | +| Distance threshold | `Is the nearest chair within 3 meters?` | `yes`, `no` | +| Visible count | `How many chairs are visible?` | `1-2`, `3-4`, `5-7`, `8+` | +| Camera range | `How far is the nearest chair from the camera?` | `under 1 m`, `1 to under 2 m`, `2 to under 4 m`, `4 m or more` | +| Nearest by side | `Which chair is closer, the left or right one?` | `left`, `right` | +| Pairwise left/right | `Is the chair to the left or right of the table?` | `left`, `right` | +| Height comparison | `Which is taller: the chair or the table?` | `chair`, `table` | +| Closest object | `Which object is closest to the chair: table or lamp?` | `table`, `lamp` | +| Door state | `Is the door open or closed?` | `open`, `closed` | +| Forward path | `Is the path directly ahead clear or blocked?` | `clear`, `blocked` | + +### Agentic + +The image-only author returns a frozen question with one contract: + +- Boolean: `{"kind":"boolean"}`; public choices are `yes`, `no`. +- Choice: `{"kind":"choice","choices":[...]}`; at least two choices. +- Height: `{"kind":"deferred_height_choice","strategy":"height-window-v1"}`; the question is + frozen before private geometry, while four public choices are generated from a successful private + height measurement. + +Numeric contracts are rejected. `height-window-v1` selects a local four-choice window from private +measurement against the fixed internal breakpoints `0.1`, `0.2`, `0.6`, `1.0`, and `2.0` meters. For +example, a private height of `0.42 m` produces: + +```text +under 0.2 m | 0.2-0.6 m | 0.6-1.0 m | over 1.0 m +``` + +## 3. Pre-Answer Grounding Checks + +For each referenced object: + +1. MoonDream detects or point-localizes the object. +2. EdgeTAM produces a mask. +3. Visible calibrated point-cloud samples are projected into the mask. +4. Mask area must meet `--min-mask-area-px`. +5. Point support must meet `--min-foreground-points`. + +Height questions also require: + +1. Accepted Open3D RANSAC ground plane. +2. Exactly one grounded object and one mask. +3. At least six points inside the object mask. +4. At least four elevated points, with at least 60% of selected points elevated more than `0.02 m` above the plane. + +Door-state questions also require one grounded door, a robust plane fit for the door mask, and a +robust plane fit in a narrow ring around that mask. The planes must be either nearly aligned +(`closed`) or clearly rotated (`open`); slightly ajar or otherwise ambiguous doors are rejected. + +Closest-object questions require exactly one grounded target and one grounded instance for every +candidate choice. They compare private support-point centroids and reject candidates whose nearest +two distances are within `0.15 m`. + +Visible-count questions count only accepted grounded instances, rather than raw detections. Camera-range +questions use the nearest grounded instance's camera-origin Euclidean range. Height comparisons require +one accepted shared ground plane and one successful physical-height measurement per distinct object; +overlapping measurement uncertainty intervals are rejected. + +Pairwise left/right questions require exactly one grounded instance for each named object. They compare +their visible support-point centroids in the camera horizontal axis and reject separation under `0.1 m`. + +Forward-path questions require a fitted visible ground plane and enough ground support in each third +of the center camera-forward corridor from `0.5-3.0 m`. Supported non-ground points block the +corridor; incomplete ground support or sparse obstacle evidence is rejected. + +## 4. Create Answers + +### Constrained + +Each deterministic family runs its own fixed sequence. + +```text +presence(A) +-> detect_objects(A) -> detection_id +-> segment_detections(detection_id) -> mask_id +-> ground_masks(mask_id) -> grounded A instances +-> no grounded A: reject +-> one or more grounded A instances: yes +``` + +```text +horizontal_direction(A) +-> detect_objects(A) -> detection_id +-> segment_detections(detection_id) -> mask_id +-> ground_masks(mask_id) -> grounded A instances +-> select_nearest_object(grounded A instances) -> nearest A +-> nearest A horizontal_direction: left/center/right +``` + +```text +within_distance(A, T) +-> detect_objects(A) -> detection_id +-> segment_detections(detection_id) -> mask_id +-> ground_masks(mask_id) -> grounded A instances +-> select_nearest_object(grounded A instances) -> nearest A +-> nearest A range_m <= T: yes/no +``` + +```text +compare_nearest_by_side(A) +-> detect_objects(A) -> detection_id +-> segment_detections(detection_id) -> mask_id +-> ground_masks(mask_id) -> grounded A instances +-> select_nearest_object(grounded A instances, left) -> nearest left A +-> select_nearest_object(grounded A instances, right) -> nearest right A +-> either side missing or tied: reject +-> compare the two range_m values: left/right +``` + +```text +visible_count(A) +-> detect_objects(A) -> segment_detections(...) -> ground_masks(...) -> grounded A instances +-> bucket accepted instance count: 1-2 / 3-4 / 5-7 / 8+ +``` + +```text +camera_range(A) +-> detect_objects(A) -> segment_detections(...) -> ground_masks(...) -> grounded A instances +-> select_nearest_object(...) -> bucket camera-origin range +``` + +```text +compare_left_right(A, B) +-> ground exactly one A and one B -> compare camera-frame support centroids +-> separation under 0.1 m: reject -> otherwise choose left/right +``` + +```text +compare_height(A, B) +-> ground exactly one A and one B -> fit_ground_plane() +-> measure_height(A, plane) and measure_height(B, plane) +-> reject overlapping uncertainty intervals -> choose taller A/B +``` + +### Agentic + +The private oracle chooses a sequence from the same read-only primitives used by constrained recipes: + +| Tool | Input | Output | +|---|---|---| +| `detect_objects` | semantic query | Detection ID and private boxes. | +| `segment_detections` | detection ID | Mask ID and accepted mask count. | +| `ground_masks` | mask ID | Grounded object IDs, range, side, point support, evidence IDs. | +| `select_nearest_object` | object IDs, optional side | Nearest grounded object ID. | +| `count_grounded_objects` | object IDs | Fixed count bucket and cited grounded instances. | +| `bucket_camera_range` | object ID | Fixed camera-origin range bucket and cited object. | +| `compare_nearest_by_side` | object IDs | Public `left` or `right` choice from nearest grounded objects. | +| `compare_left_right` | two object IDs | Public pairwise `left` or `right` relation, or an ambiguity rejection. | +| `select_closest_object` | target ID, candidate IDs | Candidate nearest to target by private support-point centroids. | +| `fit_ground_plane` | none | Plane ID, plane estimate, residual, inlier support, quality flags. | +| `measure_height` | object ID, plane ID | Measurement ID, private height, uncertainty, provenance, quality flags. | +| `compare_heights` | two object IDs, plane ID | Taller object choice from shared-plane measurements, or rejection. | +| `classify_door_state` | object ID | Public `open` or `closed` choice, or a private geometry rejection. | +| `classify_forward_path` | none | Public `clear` or `blocked` choice, or a private visibility rejection. | +| `bucket_measurement` | measurement ID | Public answer-conditioned height choices and matching choice. | + +Opaque IDs chain tool results; raw masks and point-cloud arrays are not exposed to the oracle. The +oracle returns a candidate answer and cited evidence IDs. Deferred height choices are the sole +exception to frozen public choices: the question remains frozen, but the public options and answer +are deterministically derived from the private measurement. + +## 5. Post-Answer Validation + +The candidate is rejected unless: + +1. Its answer exactly matches the fixed public choices, or the public choices deterministically + generated from a cited private height measurement. +2. It cites one or more known evidence IDs. +3. A cited height bucket matches the deterministic measurement bucket. +4. The private semantic validator confirms the cited tool output supports the question and answer. + +Tool failures, quality-gate failures, invalid citations, invalid answer format, and unsupported claims are retained as rejected generation records. + +## 6. Write Dataset Artifacts + +Each `frame-*` directory contains: + +```text +image.jpg public rectified image +frame.json frame metadata and accepted/rejected counts +ground_truth.json private tool evidence, checks, answers, and rejections +cases.json public per-frame image/question/choice rows +labels.json private per-frame correct-choice rows +``` + +The dataset root aggregates: + +```text +cases.jsonl public id, image path, question, choices +labels.jsonl private id and expected choice +run.json private resolved generation request and aggregate counts +``` + +Each line in `cases.jsonl` is one public evaluation case. Formatted for readability, one record is: + +```json +{ + "id": "go2-40-chair-height", + "image": "frame-000040/image.jpg", + "question": "How tall is the chair?", + "choices": [ + "under 0.2 m", + "0.2-0.6 m", + "0.6-1.0 m", + "over 1.0 m" + ] +} +``` + +- `id`: unique case identifier. +- `image`: dataset-relative path to the public image. +- `question`: question shown to the evaluated vision model. +- `choices`: at least two allowed answer strings. + +Each line in `labels.jsonl` is the corresponding private correct answer. Formatted for readability, +one record is: + +```json +{ + "id": "go2-40-chair-height", + "answer": "0.2-0.6 m" +} +``` + +- `id`: case identifier matching exactly one `cases.jsonl` row. +- `answer`: one of that case's `choices`. + +The files store each record as one JSON object per physical line; the examples above are expanded +only for documentation readability. + +The shared `point-cloud-vqa` evaluator reads public `cases.jsonl`, public images, and private `labels.jsonl`. It does not read generation evidence or point-cloud data. diff --git a/docs/docs.json b/docs/docs.json index 9b4246f2be..94659c93a6 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -124,7 +124,9 @@ { "group": "Perception", "pages": [ - "capabilities/perception/index" + "capabilities/perception/index", + "benchmarking/vqa-generation/pipeline", + "benchmarking/vqa-generation/infrastructure" ] }, {